Repository: hibiken/asynq Branch: master Commit: d704b68a426d Files: 101 Total size: 1.1 MB Directory structure: gitextract_6qfa6b_y/ ├── .github/ │ ├── FUNDING.yml │ ├── ISSUE_TEMPLATE/ │ │ ├── bug_report.md │ │ └── feature_request.md │ ├── dependabot.yaml │ └── workflows/ │ ├── benchstat.yml │ └── build.yml ├── .gitignore ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── README.md ├── aggregator.go ├── aggregator_test.go ├── asynq.go ├── asynq_test.go ├── benchmark_test.go ├── client.go ├── client_test.go ├── context.go ├── doc.go ├── example_test.go ├── forwarder.go ├── forwarder_test.go ├── go.mod ├── go.sum ├── healthcheck.go ├── healthcheck_test.go ├── heartbeat.go ├── heartbeat_test.go ├── inspector.go ├── inspector_test.go ├── internal/ │ ├── base/ │ │ ├── base.go │ │ └── base_test.go │ ├── context/ │ │ ├── context.go │ │ └── context_test.go │ ├── errors/ │ │ ├── errors.go │ │ └── errors_test.go │ ├── log/ │ │ ├── log.go │ │ └── log_test.go │ ├── proto/ │ │ ├── asynq.pb.go │ │ └── asynq.proto │ ├── rdb/ │ │ ├── benchmark_test.go │ │ ├── inspect.go │ │ ├── inspect_test.go │ │ ├── rdb.go │ │ └── rdb_test.go │ ├── testbroker/ │ │ └── testbroker.go │ ├── testutil/ │ │ ├── builder.go │ │ ├── builder_test.go │ │ └── testutil.go │ └── timeutil/ │ ├── timeutil.go │ └── timeutil_test.go ├── janitor.go ├── janitor_test.go ├── periodic_task_manager.go ├── periodic_task_manager_test.go ├── processor.go ├── processor_test.go ├── recoverer.go ├── recoverer_test.go ├── scheduler.go ├── scheduler_test.go ├── servemux.go ├── servemux_test.go ├── server.go ├── server_test.go ├── signals_unix.go ├── signals_windows.go ├── subscriber.go ├── subscriber_test.go ├── syncer.go ├── syncer_test.go ├── tools/ │ ├── asynq/ │ │ ├── README.md │ │ ├── cmd/ │ │ │ ├── cron.go │ │ │ ├── dash/ │ │ │ │ ├── dash.go │ │ │ │ ├── draw.go │ │ │ │ ├── draw_test.go │ │ │ │ ├── fetch.go │ │ │ │ ├── key_event.go │ │ │ │ ├── key_event_test.go │ │ │ │ ├── screen_drawer.go │ │ │ │ └── table.go │ │ │ ├── dash.go │ │ │ ├── group.go │ │ │ ├── queue.go │ │ │ ├── root.go │ │ │ ├── server.go │ │ │ ├── stats.go │ │ │ └── task.go │ │ └── main.go │ ├── go.mod │ ├── go.sum │ └── metrics_exporter/ │ └── main.go └── x/ ├── go.mod ├── go.sum ├── metrics/ │ └── metrics.go └── rate/ ├── example_test.go ├── semaphore.go └── semaphore_test.go ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/FUNDING.yml ================================================ # These are supported funding model platforms github: [hibiken] open_collective: ken-hibino ================================================ FILE: .github/ISSUE_TEMPLATE/bug_report.md ================================================ --- name: Bug report about: Create a report to help us improve title: "[BUG] Description of the bug" labels: bug assignees: - hibiken - kamikazechaser --- **Describe the bug** A clear and concise description of what the bug is. **Environment (please complete the following information):** - OS: [e.g. MacOS, Linux] - `asynq` package version [e.g. v0.25.0] - Redis/Valkey version **To Reproduce** Steps to reproduce the behavior (Code snippets if applicable): 1. Setup background processing ... 2. Enqueue tasks ... 3. See Error ... **Expected behavior** A clear and concise description of what you expected to happen. **Screenshots** If applicable, add screenshots to help explain your problem. **Additional context** Add any other context about the problem here. ================================================ FILE: .github/ISSUE_TEMPLATE/feature_request.md ================================================ --- name: Feature request about: Suggest an idea for this project title: "[FEATURE REQUEST] Description of the feature request" labels: enhancement assignees: - hibiken - kamikazechaser --- **Is your feature request related to a problem? Please describe.** A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] **Describe the solution you'd like** A clear and concise description of what you want to happen. **Describe alternatives you've considered** A clear and concise description of any alternative solutions or features you've considered. **Additional context** Add any other context or screenshots about the feature request here. ================================================ FILE: .github/dependabot.yaml ================================================ version: 2 updates: - package-ecosystem: "gomod" directory: "/" schedule: interval: "weekly" labels: - "pr-deps" - package-ecosystem: "gomod" directory: "/tools" schedule: interval: "weekly" labels: - "pr-deps" - package-ecosystem: "gomod" directory: "/x" schedule: interval: "weekly" labels: - "pr-deps" - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" ================================================ FILE: .github/workflows/benchstat.yml ================================================ # This workflow runs benchmarks against the current branch, # compares them to benchmarks against master, # and uploads the results as an artifact. name: benchstat on: [pull_request] jobs: incoming: runs-on: ubuntu-latest if: false services: redis: image: redis:7 ports: - 6379:6379 steps: - name: Checkout uses: actions/checkout@v4 - name: Set up Go uses: actions/setup-go@v5 with: go-version: 1.23.x - name: Benchmark run: go test -run=^$ -bench=. -count=5 -timeout=60m ./... | tee -a new.txt - name: Upload Benchmark uses: actions/upload-artifact@v4 with: name: bench-incoming path: new.txt current: runs-on: ubuntu-latest if: false services: redis: image: redis:7 ports: - 6379:6379 steps: - name: Checkout uses: actions/checkout@v4 with: ref: master - name: Set up Go uses: actions/setup-go@v5 with: go-version: 1.23.x - name: Benchmark run: go test -run=^$ -bench=. -count=5 -timeout=60m ./... | tee -a old.txt - name: Upload Benchmark uses: actions/upload-artifact@v4 with: name: bench-current path: old.txt benchstat: needs: [incoming, current] if: false runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - name: Set up Go uses: actions/setup-go@v5 with: go-version: 1.23.x - name: Install benchstat run: go get -u golang.org/x/perf/cmd/benchstat - name: Download Incoming uses: actions/download-artifact@v4 with: name: bench-incoming - name: Download Current uses: actions/download-artifact@v4 with: name: bench-current - name: Benchstat Results run: benchstat old.txt new.txt | tee -a benchstat.txt - name: Upload benchstat results uses: actions/upload-artifact@v4 with: name: benchstat path: benchstat.txt ================================================ FILE: .github/workflows/build.yml ================================================ name: build on: [push, pull_request] jobs: build: strategy: matrix: os: [ubuntu-latest] go-version: [1.24.x, 1.25.x] runs-on: ${{ matrix.os }} services: redis: image: redis:7 ports: - 6379:6379 steps: - uses: actions/checkout@v4 - name: Set up Go uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} cache: false - name: Build core module run: go build -v ./... - name: Build x module run: cd x && go build -v ./... && cd .. - name: Test core module run: go test -race -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Test x module run: cd x && go test -race -v ./... && cd .. - name: Benchmark Test run: go test -run=^$ -bench=. -loglevel=debug ./... - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 build-tool: strategy: matrix: os: [ubuntu-latest] go-version: [1.24.x, 1.25.x] runs-on: ${{ matrix.os }} services: redis: image: redis:7 ports: - 6379:6379 steps: - uses: actions/checkout@v4 - name: Set up Go uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} cache: false - name: Build tools module run: cd tools && go build -v ./... && cd .. - name: Test tools module run: cd tools && go test -race -v ./... && cd .. golangci: name: lint runs-on: ubuntu-latest if: false steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: stable - name: golangci-lint uses: golangci/golangci-lint-action@v6 with: version: v1.61 ================================================ FILE: .gitignore ================================================ vendor # Binaries for programs and plugins *.exe *.exe~ *.dll *.so *.dylib # Test binary, built with `go test -c` *.test # Output of the go coverage tool, specifically when used with LiteIDE *.out # Ignore examples for now /examples # Ignore tool binaries /tools/asynq/asynq /tools/metrics_exporter/metrics_exporter # Ignore asynq config file .asynq.* # Ignore editor config files .vscode .idea ================================================ FILE: CHANGELOG.md ================================================ # Changelog All notable changes to this project will be documented in this file. The format is based on ["Keep a Changelog"](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] ## [0.26.0] - 2026-02-03 ### Upgrades - Prepare CI for Go 1.24.x and 1.25.x (commit: e9037f0) ### Added - Add Headers support to tasks (PR: https://github.com/hibiken/asynq/pull/1070) - Add `--tls` option to dash command (PR: https://github.com/hibiken/asynq/pull/1073) - Add `--username` CLI flag for Redis ACL authentication (PR: https://github.com/hibiken/asynq/pull/1083) - Add `UpdateTaskPayload` method for inspector (PR: https://github.com/hibiken/asynq/pull/1042) ### Fixes - Fix: Correct error message text in ResultWriter.Write (PR: https://github.com/hibiken/asynq/pull/1054) - Fix: Wrap all fmt.Errorf errors with %w (PR: https://github.com/hibiken/asynq/pull/1047) - Fix: ServeMux.NotFoundHandler returns ErrHandlerNotFound error (PR: https://github.com/hibiken/asynq/pull/1031) ### Changed - Docs: Update server.go documentation (PR: https://github.com/hibiken/asynq/pull/1010) - Chore: Fix godoc comment (PR: https://github.com/hibiken/asynq/pull/1009) ## [0.25.1] - 2024-12-11 ### Upgrades * Some packages ### Added * Add `HeartbeatInterval` option to the scheduler (PR: https://github.com/hibiken/asynq/pull/956) * Add `RedisUniversalClient` support to periodic task manager (PR: https://github.com/hibiken/asynq/pull/958) * Add `--insecure` flag to CLI dash command (PR: https://github.com/hibiken/asynq/pull/980) * Add logging for registration errors (PR: https://github.com/hibiken/asynq/pull/657) ### Fixes - Perf: Use string concat inplace of fmt.Sprintf in hotpath (PR: https://github.com/hibiken/asynq/pull/962) - Perf: Init map with size (PR: https://github.com/hibiken/asynq/pull/673) - Fix: `Scheduler` and `PeriodicTaskManager` graceful shutdown (PR: https://github.com/hibiken/asynq/pull/977) - Fix: `Server` graceful shutdown on UNIX systems (PR: https://github.com/hibiken/asynq/pull/982) ## [0.25.0] - 2024-10-29 ### Upgrades - Minumum go version is set to 1.22 (PR: https://github.com/hibiken/asynq/pull/925) - Internal protobuf package is upgraded to address security advisories (PR: https://github.com/hibiken/asynq/pull/925) - Most packages are upgraded - CI/CD spec upgraded ### Added - `IsPanicError` function is introduced to support catching of panic errors when processing tasks (PR: https://github.com/hibiken/asynq/pull/491) - `JanitorInterval` and `JanitorBatchSize` are added as Server options (PR: https://github.com/hibiken/asynq/pull/715) - `NewClientFromRedisClient` is introduced to allow reusing an existing redis client (PR: https://github.com/hibiken/asynq/pull/742) - `TaskCheckInterval` config option is added to specify the interval between checks for new tasks to process when all queues are empty (PR: https://github.com/hibiken/asynq/pull/694) - `Ping` method is added to Client, Server and Scheduler ((PR: https://github.com/hibiken/asynq/pull/585)) - `RevokeTask` error type is introduced to prevent a task from being retried or archived (PR: https://github.com/hibiken/asynq/pull/882) - `SentinelUsername` is added as a redis config option (PR: https://github.com/hibiken/asynq/pull/924) - Some jitter is introduced to improve latency when fetching jobs in the processor (PR: https://github.com/hibiken/asynq/pull/868) - Add task enqueue command to the CLI (PR: https://github.com/hibiken/asynq/pull/918) - Add a map cache (concurrent safe) to keep track of queues that ultimately reduces redis load when enqueuing tasks (PR: https://github.com/hibiken/asynq/pull/946) ### Fixes - Archived tasks that are trimmed should now be deleted (PR: https://github.com/hibiken/asynq/pull/743) - Fix lua script when listing task messages with an expired lease (PR: https://github.com/hibiken/asynq/pull/709) - Fix potential context leaks due to cancellation not being called (PR: https://github.com/hibiken/asynq/pull/926) - Misc documentation fixes - Misc test fixes ## [0.24.1] - 2023-05-01 ### Changed - Updated package version dependency for go-redis ## [0.24.0] - 2023-01-02 ### Added - `PreEnqueueFunc`, `PostEnqueueFunc` is added in `Scheduler` and deprecated `EnqueueErrorHandler` (PR: https://github.com/hibiken/asynq/pull/476) ### Changed - Removed error log when `Scheduler` failed to enqueue a task. Use `PostEnqueueFunc` to check for errors and task actions if needed. - Changed log level from ERROR to WARNINING when `Scheduler` failed to record `SchedulerEnqueueEvent`. ## [0.23.0] - 2022-04-11 ### Added - `Group` option is introduced to enqueue task in a group. - `GroupAggregator` and related types are introduced for task aggregation feature. - `GroupGracePeriod`, `GroupMaxSize`, `GroupMaxDelay`, and `GroupAggregator` fields are added to `Config`. - `Inspector` has new methods related to "aggregating tasks". - `Group` field is added to `TaskInfo`. - (CLI): `group ls` command is added - (CLI): `task ls` supports listing aggregating tasks via `--state=aggregating --group=` flags - Enable rediss url parsing support ### Fixed - Fixed overflow issue with 32-bit systems (For details, see https://github.com/hibiken/asynq/pull/426) ## [0.22.1] - 2022-02-20 ### Fixed - Fixed Redis version compatibility: Keep support for redis v4.0+ ## [0.22.0] - 2022-02-19 ### Added - `BaseContext` is introduced in `Config` to specify callback hook to provide a base `context` from which `Handler` `context` is derived - `IsOrphaned` field is added to `TaskInfo` to describe a task left in active state with no worker processing it. ### Changed - `Server` now recovers tasks with an expired lease. Recovered tasks are retried/archived with `ErrLeaseExpired` error. ## [0.21.0] - 2022-01-22 ### Added - `PeriodicTaskManager` is added. Prefer using this over `Scheduler` as it has better support for dynamic periodic tasks. - The `asynq stats` command now supports a `--json` option, making its output a JSON object - Introduced new configuration for `DelayedTaskCheckInterval`. See [godoc](https://godoc.org/github.com/hibiken/asynq) for more details. ## [0.20.0] - 2021-12-19 ### Added - Package `x/metrics` is added. - Tool `tools/metrics_exporter` binary is added. - `ProcessedTotal` and `FailedTotal` fields were added to `QueueInfo` struct. ## [0.19.1] - 2021-12-12 ### Added - `Latency` field is added to `QueueInfo`. - `EnqueueContext` method is added to `Client`. ### Fixed - Fixed an error when user pass a duration less than 1s to `Unique` option ## [0.19.0] - 2021-11-06 ### Changed - `NewTask` takes `Option` as variadic argument - Bumped minimum supported go version to 1.14 (i.e. go1.14 or higher is required). ### Added - `Retention` option is added to allow user to specify task retention duration after completion. - `TaskID` option is added to allow user to specify task ID. - `ErrTaskIDConflict` sentinel error value is added. - `ResultWriter` type is added and provided through `Task.ResultWriter` method. - `TaskInfo` has new fields `CompletedAt`, `Result` and `Retention`. ### Removed - `Client.SetDefaultOptions` is removed. Use `NewTask` instead to pass default options for tasks. ## [0.18.6] - 2021-10-03 ### Changed - Updated `github.com/go-redis/redis` package to v8 ## [0.18.5] - 2021-09-01 ### Added - `IsFailure` config option is added to determine whether error returned from Handler counts as a failure. ## [0.18.4] - 2021-08-17 ### Fixed - Scheduler methods are now thread-safe. It's now safe to call `Register` and `Unregister` concurrently. ## [0.18.3] - 2021-08-09 ### Changed - `Client.Enqueue` no longer enqueues tasks with empty typename; Error message is returned. ## [0.18.2] - 2021-07-15 ### Changed - Changed `Queue` function to not to convert the provided queue name to lowercase. Queue names are now case-sensitive. - `QueueInfo.MemoryUsage` is now an approximate usage value. ### Fixed - Fixed latency issue around memory usage (see https://github.com/hibiken/asynq/issues/309). ## [0.18.1] - 2021-07-04 ### Changed - Changed to execute task recovering logic when server starts up; Previously it needed to wait for a minute for task recovering logic to exeucte. ### Fixed - Fixed task recovering logic to execute every minute ## [0.18.0] - 2021-06-29 ### Changed - NewTask function now takes array of bytes as payload. - Task `Type` and `Payload` should be accessed by a method call. - `Server` API has changed. Renamed `Quiet` to `Stop`. Renamed `Stop` to `Shutdown`. _Note:_ As a result of this renaming, the behavior of `Stop` has changed. Please update the exising code to call `Shutdown` where it used to call `Stop`. - `Scheduler` API has changed. Renamed `Stop` to `Shutdown`. - Requires redis v4.0+ for multiple field/value pair support - `Client.Enqueue` now returns `TaskInfo` - `Inspector.RunTaskByKey` is replaced with `Inspector.RunTask` - `Inspector.DeleteTaskByKey` is replaced with `Inspector.DeleteTask` - `Inspector.ArchiveTaskByKey` is replaced with `Inspector.ArchiveTask` - `inspeq` package is removed. All types and functions from the package is moved to `asynq` package. - `WorkerInfo` field names have changed. - `Inspector.CancelActiveTask` is renamed to `Inspector.CancelProcessing` ## [0.17.2] - 2021-06-06 ### Fixed - Free unique lock when task is deleted (https://github.com/hibiken/asynq/issues/275). ## [0.17.1] - 2021-04-04 ### Fixed - Fix bug in internal `RDB.memoryUsage` method. ## [0.17.0] - 2021-03-24 ### Added - `DialTimeout`, `ReadTimeout`, and `WriteTimeout` options are added to `RedisConnOpt`. ## [0.16.1] - 2021-03-20 ### Fixed - Replace `KEYS` command with `SCAN` as recommended by [redis doc](https://redis.io/commands/KEYS). ## [0.16.0] - 2021-03-10 ### Added - `Unregister` method is added to `Scheduler` to remove a registered entry. ## [0.15.0] - 2021-01-31 **IMPORTATNT**: All `Inspector` related code are moved to subpackage "github.com/hibiken/asynq/inspeq" ### Changed - `Inspector` related code are moved to subpackage "github.com/hibken/asynq/inspeq". - `RedisConnOpt` interface has changed slightly. If you have been passing `RedisClientOpt`, `RedisFailoverClientOpt`, or `RedisClusterClientOpt` as a pointer, update your code to pass as a value. - `ErrorMsg` field in `RetryTask` and `ArchivedTask` was renamed to `LastError`. ### Added - `MaxRetry`, `Retried`, `LastError` fields were added to all task types returned from `Inspector`. - `MemoryUsage` field was added to `QueueStats`. - `DeleteAllPendingTasks`, `ArchiveAllPendingTasks` were added to `Inspector` - `DeleteTaskByKey` and `ArchiveTaskByKey` now supports deleting/archiving `PendingTask`. - asynq CLI now supports deleting/archiving pending tasks. ## [0.14.1] - 2021-01-19 ### Fixed - `go.mod` file for CLI ## [0.14.0] - 2021-01-14 **IMPORTATNT**: Please run `asynq migrate` command to migrate from the previous versions. ### Changed - Renamed `DeadTask` to `ArchivedTask`. - Renamed the operation `Kill` to `Archive` in `Inpsector`. - Print stack trace when Handler panics. - Include a file name and a line number in the error message when recovering from a panic. ### Added - `DefaultRetryDelayFunc` is now a public API, which can be used in the custom `RetryDelayFunc`. - `SkipRetry` error is added to be used as a return value from `Handler`. - `Servers` method is added to `Inspector` - `CancelActiveTask` method is added to `Inspector`. - `ListSchedulerEnqueueEvents` method is added to `Inspector`. - `SchedulerEntries` method is added to `Inspector`. - `DeleteQueue` method is added to `Inspector`. ## [0.13.1] - 2020-11-22 ### Fixed - Fixed processor to wait for specified time duration before forcefully shutdown workers. ## [0.13.0] - 2020-10-13 ### Added - `Scheduler` type is added to enable periodic tasks. See the godoc for its APIs and [wiki](https://github.com/hibiken/asynq/wiki/Periodic-Tasks) for the getting-started guide. ### Changed - interface `Option` has changed. See the godoc for the new interface. This change would have no impact as long as you are using exported functions (e.g. `MaxRetry`, `Queue`, etc) to create `Option`s. ### Added - `Payload.String() string` method is added - `Payload.MarshalJSON() ([]byte, error)` method is added ## [0.12.0] - 2020-09-12 **IMPORTANT**: If you are upgrading from a previous version, please install the latest version of the CLI `go get -u github.com/hibiken/asynq/tools/asynq` and run `asynq migrate` command. No process should be writing to Redis while you run the migration command. ## The semantics of queue have changed Previously, we called tasks that are ready to be processed _"Enqueued tasks"_, and other tasks that are scheduled to be processed in the future _"Scheduled tasks"_, etc. We changed the semantics of _"Enqueue"_ slightly; All tasks that client pushes to Redis are _Enqueued_ to a queue. Within a queue, tasks will transition from one state to another. Possible task states are: - `Pending`: task is ready to be processed (previously called "Enqueued") - `Active`: tasks is currently being processed (previously called "InProgress") - `Scheduled`: task is scheduled to be processed in the future - `Retry`: task failed to be processed and will be retried again in the future - `Dead`: task has exhausted all of its retries and stored for manual inspection purpose **These semantics change is reflected in the new `Inspector` API and CLI commands.** --- ### Changed #### `Client` Use `ProcessIn` or `ProcessAt` option to schedule a task instead of `EnqueueIn` or `EnqueueAt`. | Previously | v0.12.0 | | --------------------------- | ------------------------------------------ | | `client.EnqueueAt(t, task)` | `client.Enqueue(task, asynq.ProcessAt(t))` | | `client.EnqueueIn(d, task)` | `client.Enqueue(task, asynq.ProcessIn(d))` | #### `Inspector` All Inspector methods are scoped to a queue, and the methods take `qname (string)` as the first argument. `EnqueuedTask` is renamed to `PendingTask` and its corresponding methods. `InProgressTask` is renamed to `ActiveTask` and its corresponding methods. Command "Enqueue" is replaced by the verb "Run" (e.g. `EnqueueAllScheduledTasks` --> `RunAllScheduledTasks`) #### `CLI` CLI commands are restructured to use subcommands. Commands are organized into a few management commands: To view details on any command, use `asynq help `. - `asynq stats` - `asynq queue [ls inspect history rm pause unpause]` - `asynq task [ls cancel delete kill run delete-all kill-all run-all]` - `asynq server [ls]` ### Added #### `RedisConnOpt` - `RedisClusterClientOpt` is added to connect to Redis Cluster. - `Username` field is added to all `RedisConnOpt` types in order to authenticate connection when Redis ACLs are used. #### `Client` - `ProcessIn(d time.Duration) Option` and `ProcessAt(t time.Time) Option` are added to replace `EnqueueIn` and `EnqueueAt` functionality. #### `Inspector` - `Queues() ([]string, error)` method is added to get all queue names. - `ClusterKeySlot(qname string) (int64, error)` method is added to get queue's hash slot in Redis cluster. - `ClusterNodes(qname string) ([]ClusterNode, error)` method is added to get a list of Redis cluster nodes for the given queue. - `Close() error` method is added to close connection with redis. ### `Handler` - `GetQueueName(ctx context.Context) (string, bool)` helper is added to extract queue name from a context. ## [0.11.0] - 2020-07-28 ### Added - `Inspector` type was added to monitor and mutate state of queues and tasks. - `HealthCheckFunc` and `HealthCheckInterval` fields were added to `Config` to allow user to specify a callback function to check for broker connection. ## [0.10.0] - 2020-07-06 ### Changed - All tasks now requires timeout or deadline. By default, timeout is set to 30 mins. - Tasks that exceed its deadline are automatically retried. - Encoding schema for task message has changed. Please install the latest CLI and run `migrate` command if you have tasks enqueued with the previous version of asynq. - API of `(*Client).Enqueue`, `(*Client).EnqueueIn`, and `(*Client).EnqueueAt` has changed to return a `*Result`. - API of `ErrorHandler` has changed. It now takes context as the first argument and removed `retried`, `maxRetry` from the argument list. Use `GetRetryCount` and/or `GetMaxRetry` to get the count values. ## [0.9.4] - 2020-06-13 ### Fixed - Fixes issue of same tasks processed by more than one worker (https://github.com/hibiken/asynq/issues/90). ## [0.9.3] - 2020-06-12 ### Fixed - Fixes the JSON number overflow issue (https://github.com/hibiken/asynq/issues/166). ## [0.9.2] - 2020-06-08 ### Added - The `pause` and `unpause` commands were added to the CLI. See README for the CLI for details. ## [0.9.1] - 2020-05-29 ### Added - `GetTaskID`, `GetRetryCount`, and `GetMaxRetry` functions were added to extract task metadata from context. ## [0.9.0] - 2020-05-16 ### Changed - `Logger` interface has changed. Please see the godoc for the new interface. ### Added - `LogLevel` type is added. Server's log level can be specified through `LogLevel` field in `Config`. ## [0.8.3] - 2020-05-08 ### Added - `Close` method is added to `Client`. ## [0.8.2] - 2020-05-03 ### Fixed - [Fixed cancelfunc leak](https://github.com/hibiken/asynq/pull/145) ## [0.8.1] - 2020-04-27 ### Added - `ParseRedisURI` helper function is added to create a `RedisConnOpt` from a URI string. - `SetDefaultOptions` method is added to `Client`. ## [0.8.0] - 2020-04-19 ### Changed - `Background` type is renamed to `Server`. - To upgrade from the previous version, Update `NewBackground` to `NewServer` and pass `Config` by value. - CLI is renamed to `asynq`. - To upgrade the CLI to the latest version run `go get -u github.com/hibiken/tools/asynq` - The `ps` command in CLI is renamed to `servers` - `Concurrency` defaults to the number of CPUs when unset or set to a negative value. ### Added - `ShutdownTimeout` field is added to `Config` to speicfy timeout duration used during graceful shutdown. - New `Server` type exposes `Start`, `Stop`, and `Quiet` as well as `Run`. ## [0.7.1] - 2020-04-05 ### Fixed - Fixed signal handling for windows. ## [0.7.0] - 2020-03-22 ### Changed - Support Go v1.13+, dropped support for go v1.12 ### Added - `Unique` option was added to allow client to enqueue a task only if it's unique within a certain time period. ## [0.6.2] - 2020-03-15 ### Added - `Use` method was added to `ServeMux` to apply middlewares to all handlers. ## [0.6.1] - 2020-03-12 ### Added - `Client` can optionally schedule task with `asynq.Deadline(time)` to specify deadline for task's context. Default is no deadline. - `Logger` option was added to config, which allows user to specify the logger used by the background instance. ## [0.6.0] - 2020-03-01 ### Added - Added `ServeMux` type to make it easy for users to implement Handler interface. - `ErrorHandler` type was added. Allow users to specify error handling function (e.g. Report error to error reporting service such as Honeybadger, Bugsnag, etc) ## [0.5.0] - 2020-02-23 ### Changed - `Client` API has changed. Use `Enqueue`, `EnqueueAt` and `EnqueueIn` to enqueue and schedule tasks. ### Added - `asynqmon workers` was added to list all running workers information ## [0.4.0] - 2020-02-13 ### Changed - `Handler` interface has changed. `ProcessTask` method takes two arguments `context.Context` and `*asynq.Task` - `Queues` field in `Config` has change from `map[string]uint` to `map[string]int` ### Added - `Client` can optionally schedule task with `asynq.Timeout(duration)` to specify timeout duration for task. Default is no timeout. - `asynqmon cancel [task id]` will send a cancelation signal to the goroutine processing the speicified task. ## [0.3.0] - 2020-02-04 ### Added - `asynqmon ps` was added to list all background worker processes ## [0.2.2] - 2020-01-26 ### Fixed - Fixed restoring unfinished tasks back to correct queues. ### Changed - `asynqmon ls` command is now paginated (default 30 tasks from first page) - `asynqmon ls enqueued:[queue name]` requires queue name to be specified ## [0.2.1] - 2020-01-22 ### Fixed - More structured log messages - Prevent spamming logs with a bunch of errors when Redis connection is lost - Fixed and updated README doc ## [0.2.0] - 2020-01-19 ### Added - NewTask constructor - `Queues` option in `Config` to specify mutiple queues with priority level - `Client` can schedule a task with `asynq.Queue(name)` to specify which queue to use - `StrictPriority` option in `Config` to specify whether the priority should be followed strictly - `RedisConnOpt` to abstract away redis client implementation - [CLI] `asynqmon rmq` command to remove queue ### Changed - `Client` and `Background` constructors take `RedisConnOpt` as their first argument. - `asynqmon stats` now shows the total of all enqueued tasks under "Enqueued" - `asynqmon stats` now shows each queue's task count - `asynqmon history` now doesn't take any arguments and shows data from the last 10 days by default (use `--days` flag to change the number of days) - Task type is now immutable (i.e., Payload is read-only) ## [0.1.0] - 2020-01-04 ### Added - Initial version of asynq package - Initial version of asynqmon CLI ================================================ FILE: CODE_OF_CONDUCT.md ================================================ # Contributor Covenant Code of Conduct ## Our Pledge We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. ## Our Standards Examples of behavior that contributes to a positive environment for our community include: * Demonstrating empathy and kindness toward other people * Being respectful of differing opinions, viewpoints, and experiences * Giving and gracefully accepting constructive feedback * Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience * Focusing on what is best not just for us as individuals, but for the overall community Examples of unacceptable behavior include: * The use of sexualized language or imagery, and sexual attention or advances of any kind * Trolling, insulting or derogatory comments, and personal or political attacks * Public or private harassment * Publishing others' private information, such as a physical or email address, without their explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting ## Enforcement Responsibilities Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate. ## Scope This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at ken.hibino7@gmail.com. All complaints will be reviewed and investigated promptly and fairly. All community leaders are obligated to respect the privacy and security of the reporter of any incident. ## Enforcement Guidelines Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: ### 1. Correction **Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. **Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested. ### 2. Warning **Community Impact**: A violation through a single incident or series of actions. **Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. ### 3. Temporary Ban **Community Impact**: A serious violation of community standards, including sustained inappropriate behavior. **Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. ### 4. Permanent Ban **Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. **Consequence**: A permanent ban from any sort of public interaction within the community. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0, available at https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity). [homepage]: https://www.contributor-covenant.org For answers to common questions about this code of conduct, see the FAQ at https://www.contributor-covenant.org/faq. Translations are available at https://www.contributor-covenant.org/translations. ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing Thanks for your interest in contributing to Asynq! We are open to, and grateful for, any contributions made by the community. ## Reporting Bugs Have a look at our [issue tracker](https://github.com/hibiken/asynq/issues). If you can't find an issue (open or closed) describing your problem (or a very similar one) there, please open a new issue with the following details: - Which versions of Go and Redis are you using? - What are you trying to accomplish? - What is the full error you are seeing? - How can we reproduce this? - Please quote as much of your code as needed to reproduce (best link to a public repository or Gist) ## Getting Help We run a [Gitter channel](https://gitter.im/go-asynq/community) where you can ask questions and get help. Feel free to ask there before opening a GitHub issue. ## Submitting Feature Requests If you can't find an issue (open or closed) describing your idea on our [issue tracker](https://github.com/hibiken/asynq/issues), open an issue. Adding answers to the following questions in your description is +1: - What do you want to do, and how do you expect Asynq to support you with that? - How might this be added to Asynq? - What are possible alternatives? - Are there any disadvantages? Thank you! We'll try to respond as quickly as possible. ## Contributing Code 1. Fork this repo 2. Download your fork `git clone git@github.com:your-username/asynq.git && cd asynq` 3. Create your branch `git checkout -b your-branch-name` 4. Make and commit your changes 5. Push the branch `git push origin your-branch-name` 6. Create a new pull request Please try to keep your pull request focused in scope and avoid including unrelated commits. Please run tests against redis cluster locally with `--redis_cluster` flag to ensure that code works for Redis cluster. TODO: Run tests using Redis cluster on CI. After you have submitted your pull request, we'll try to get back to you as soon as possible. We may suggest some changes or improvements. Thank you for contributing! ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) 2019 Kentaro Hibino Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: Makefile ================================================ ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) proto: internal/proto/asynq.proto protoc -I=$(ROOT_DIR)/internal/proto \ --go_out=$(ROOT_DIR)/internal/proto \ --go_opt=module=github.com/hibiken/asynq/internal/proto \ $(ROOT_DIR)/internal/proto/asynq.proto .PHONY: lint lint: golangci-lint run ================================================ FILE: README.md ================================================ Asynq logo # Simple, reliable & efficient distributed task queue in Go [![GoDoc](https://godoc.org/github.com/hibiken/asynq?status.svg)](https://godoc.org/github.com/hibiken/asynq) [![Go Report Card](https://goreportcard.com/badge/github.com/hibiken/asynq)](https://goreportcard.com/report/github.com/hibiken/asynq) ![Build Status](https://github.com/hibiken/asynq/workflows/build/badge.svg) [![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](https://opensource.org/licenses/MIT) [![Gitter chat](https://badges.gitter.im/go-asynq/gitter.svg)](https://gitter.im/go-asynq/community) Asynq is a Go library for queueing tasks and processing them asynchronously with workers. It's backed by [Redis](https://redis.io/) and is designed to be scalable yet easy to get started. Highlevel overview of how Asynq works: - Client puts tasks on a queue - Server pulls tasks off queues and starts a worker goroutine for each task - Tasks are processed concurrently by multiple workers Task queues are used as a mechanism to distribute work across multiple machines. A system can consist of multiple worker servers and brokers, giving way to high availability and horizontal scaling. **Example use case** ![Task Queue Diagram](https://user-images.githubusercontent.com/11155743/116358505-656f5f80-a806-11eb-9c16-94e49dab0f99.jpg) ## Features - Guaranteed [at least one execution](https://www.cloudcomputingpatterns.org/at_least_once_delivery/) of a task - Scheduling of tasks - [Retries](https://github.com/hibiken/asynq/wiki/Task-Retry) of failed tasks - Automatic recovery of tasks in the event of a worker crash - [Weighted priority queues](https://github.com/hibiken/asynq/wiki/Queue-Priority#weighted-priority) - [Strict priority queues](https://github.com/hibiken/asynq/wiki/Queue-Priority#strict-priority) - Low latency to add a task since writes are fast in Redis - De-duplication of tasks using [unique option](https://github.com/hibiken/asynq/wiki/Unique-Tasks) - Allow [timeout and deadline per task](https://github.com/hibiken/asynq/wiki/Task-Timeout-and-Cancelation) - Allow [aggregating group of tasks](https://github.com/hibiken/asynq/wiki/Task-aggregation) to batch multiple successive operations - [Flexible handler interface with support for middlewares](https://github.com/hibiken/asynq/wiki/Handler-Deep-Dive) - [Ability to pause queue](/tools/asynq/README.md#pause) to stop processing tasks from the queue - [Periodic Tasks](https://github.com/hibiken/asynq/wiki/Periodic-Tasks) - [Support Redis Sentinels](https://github.com/hibiken/asynq/wiki/Automatic-Failover) for high availability - Integration with [Prometheus](https://prometheus.io/) to collect and visualize queue metrics - [Web UI](#web-ui) to inspect and remote-control queues and tasks - [CLI](#command-line-tool) to inspect and remote-control queues and tasks ## Stability and Compatibility **Status**: The library relatively stable and is currently undergoing **moderate development** with less frequent breaking API changes. > ☝️ **Important Note**: Current major version is zero (`v0.x.x`) to accommodate rapid development and fast iteration while getting early feedback from users (_feedback on APIs are appreciated!_). The public API could change without a major version update before `v1.0.0` release. ### Redis Cluster Compatibility Some of the lua scripts in this library may not be compatible with Redis Cluster. ## Sponsoring If you are using this package in production, **please consider sponsoring the project to show your support!** ## Quickstart Make sure you have Go installed ([download](https://golang.org/dl/)). The **last two** Go versions are supported (See https://go.dev/dl). Initialize your project by creating a folder and then running `go mod init github.com/your/repo` ([learn more](https://blog.golang.org/using-go-modules)) inside the folder. Then install Asynq library with the [`go get`](https://golang.org/cmd/go/#hdr-Add_dependencies_to_current_module_and_install_them) command: ```sh go get -u github.com/hibiken/asynq ``` Make sure you're running a Redis server locally or from a [Docker](https://hub.docker.com/_/redis) container. Version `4.0` or higher is required. Next, write a package that encapsulates task creation and task handling. ```go package tasks import ( "context" "encoding/json" "fmt" "log" "time" "github.com/hibiken/asynq" ) // A list of task types. const ( TypeEmailDelivery = "email:deliver" TypeImageResize = "image:resize" ) type EmailDeliveryPayload struct { UserID int TemplateID string } type ImageResizePayload struct { SourceURL string } //---------------------------------------------- // Write a function NewXXXTask to create a task. // A task consists of a type and a payload. //---------------------------------------------- func NewEmailDeliveryTask(userID int, tmplID string) (*asynq.Task, error) { payload, err := json.Marshal(EmailDeliveryPayload{UserID: userID, TemplateID: tmplID}) if err != nil { return nil, err } return asynq.NewTask(TypeEmailDelivery, payload), nil } func NewImageResizeTask(src string) (*asynq.Task, error) { payload, err := json.Marshal(ImageResizePayload{SourceURL: src}) if err != nil { return nil, err } // task options can be passed to NewTask, which can be overridden at enqueue time. return asynq.NewTask(TypeImageResize, payload, asynq.MaxRetry(5), asynq.Timeout(20 * time.Minute)), nil } //--------------------------------------------------------------- // Write a function HandleXXXTask to handle the input task. // Note that it satisfies the asynq.HandlerFunc interface. // // Handler doesn't need to be a function. You can define a type // that satisfies asynq.Handler interface. See examples below. //--------------------------------------------------------------- func HandleEmailDeliveryTask(ctx context.Context, t *asynq.Task) error { var p EmailDeliveryPayload if err := json.Unmarshal(t.Payload(), &p); err != nil { return fmt.Errorf("json.Unmarshal failed: %v: %w", err, asynq.SkipRetry) } log.Printf("Sending Email to User: user_id=%d, template_id=%s", p.UserID, p.TemplateID) // Email delivery code ... return nil } // ImageProcessor implements asynq.Handler interface. type ImageProcessor struct { // ... fields for struct } func (processor *ImageProcessor) ProcessTask(ctx context.Context, t *asynq.Task) error { var p ImageResizePayload if err := json.Unmarshal(t.Payload(), &p); err != nil { return fmt.Errorf("json.Unmarshal failed: %v: %w", err, asynq.SkipRetry) } log.Printf("Resizing image: src=%s", p.SourceURL) // Image resizing code ... return nil } func NewImageProcessor() *ImageProcessor { return &ImageProcessor{} } ``` In your application code, import the above package and use [`Client`](https://pkg.go.dev/github.com/hibiken/asynq?tab=doc#Client) to put tasks on queues. ```go package main import ( "log" "time" "github.com/hibiken/asynq" "your/app/package/tasks" ) const redisAddr = "127.0.0.1:6379" func main() { client := asynq.NewClient(asynq.RedisClientOpt{Addr: redisAddr}) defer client.Close() // ------------------------------------------------------ // Example 1: Enqueue task to be processed immediately. // Use (*Client).Enqueue method. // ------------------------------------------------------ task, err := tasks.NewEmailDeliveryTask(42, "some:template:id") if err != nil { log.Fatalf("could not create task: %v", err) } info, err := client.Enqueue(task) if err != nil { log.Fatalf("could not enqueue task: %v", err) } log.Printf("enqueued task: id=%s queue=%s", info.ID, info.Queue) // ------------------------------------------------------------ // Example 2: Schedule task to be processed in the future. // Use ProcessIn or ProcessAt option. // ------------------------------------------------------------ info, err = client.Enqueue(task, asynq.ProcessIn(24*time.Hour)) if err != nil { log.Fatalf("could not schedule task: %v", err) } log.Printf("enqueued task: id=%s queue=%s", info.ID, info.Queue) // ---------------------------------------------------------------------------- // Example 3: Set other options to tune task processing behavior. // Options include MaxRetry, Queue, Timeout, Deadline, Unique etc. // ---------------------------------------------------------------------------- task, err = tasks.NewImageResizeTask("https://example.com/myassets/image.jpg") if err != nil { log.Fatalf("could not create task: %v", err) } info, err = client.Enqueue(task, asynq.MaxRetry(10), asynq.Timeout(3 * time.Minute)) if err != nil { log.Fatalf("could not enqueue task: %v", err) } log.Printf("enqueued task: id=%s queue=%s", info.ID, info.Queue) } ``` Next, start a worker server to process these tasks in the background. To start the background workers, use [`Server`](https://pkg.go.dev/github.com/hibiken/asynq?tab=doc#Server) and provide your [`Handler`](https://pkg.go.dev/github.com/hibiken/asynq?tab=doc#Handler) to process the tasks. You can optionally use [`ServeMux`](https://pkg.go.dev/github.com/hibiken/asynq?tab=doc#ServeMux) to create a handler, just as you would with [`net/http`](https://golang.org/pkg/net/http/) Handler. ```go package main import ( "log" "github.com/hibiken/asynq" "your/app/package/tasks" ) const redisAddr = "127.0.0.1:6379" func main() { srv := asynq.NewServer( asynq.RedisClientOpt{Addr: redisAddr}, asynq.Config{ // Specify how many concurrent workers to use Concurrency: 10, // Optionally specify multiple queues with different priority. Queues: map[string]int{ "critical": 6, "default": 3, "low": 1, }, // See the godoc for other configuration options }, ) // mux maps a type to a handler mux := asynq.NewServeMux() mux.HandleFunc(tasks.TypeEmailDelivery, tasks.HandleEmailDeliveryTask) mux.Handle(tasks.TypeImageResize, tasks.NewImageProcessor()) // ...register other handlers... if err := srv.Run(mux); err != nil { log.Fatalf("could not run server: %v", err) } } ``` For a more detailed walk-through of the library, see our [Getting Started](https://github.com/hibiken/asynq/wiki/Getting-Started) guide. To learn more about `asynq` features and APIs, see the package [godoc](https://godoc.org/github.com/hibiken/asynq). ## Web UI [Asynqmon](https://github.com/hibiken/asynqmon) is a web based tool for monitoring and administrating Asynq queues and tasks. Here's a few screenshots of the Web UI: **Queues view** ![Web UI Queues View](https://user-images.githubusercontent.com/11155743/114697016-07327f00-9d26-11eb-808c-0ac841dc888e.png) **Tasks view** ![Web UI TasksView](https://user-images.githubusercontent.com/11155743/114697070-1f0a0300-9d26-11eb-855c-d3ec263865b7.png) **Metrics view** Screen Shot 2021-12-19 at 4 37 19 PM **Settings and adaptive dark mode** ![Web UI Settings and adaptive dark mode](https://user-images.githubusercontent.com/11155743/114697149-3517c380-9d26-11eb-9f7a-ae2dd00aad5b.png) For details on how to use the tool, refer to the tool's [README](https://github.com/hibiken/asynqmon#readme). ## Command Line Tool Asynq ships with a command line tool to inspect the state of queues and tasks. To install the CLI tool, run the following command: ```sh go install github.com/hibiken/asynq/tools/asynq@latest ``` Here's an example of running the `asynq dash` command: ![Gif](/docs/assets/dash.gif) For details on how to use the tool, refer to the tool's [README](/tools/asynq/README.md). ## Contributing We are open to, and grateful for, any contributions (GitHub issues/PRs, feedback on [Gitter channel](https://gitter.im/go-asynq/community), etc) made by the community. Please see the [Contribution Guide](/CONTRIBUTING.md) before contributing. ## License Copyright (c) 2019-present [Ken Hibino](https://github.com/hibiken) and [Contributors](https://github.com/hibiken/asynq/graphs/contributors). `Asynq` is free and open-source software licensed under the [MIT License](https://github.com/hibiken/asynq/blob/master/LICENSE). Official logo was created by [Vic Shóstak](https://github.com/koddr) and distributed under [Creative Commons](https://creativecommons.org/publicdomain/zero/1.0/) license (CC0 1.0 Universal). ================================================ FILE: aggregator.go ================================================ // Copyright 2022 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "context" "sync" "time" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/log" ) // An aggregator is responsible for checking groups and aggregate into one task // if any of the grouping condition is met. type aggregator struct { logger *log.Logger broker base.Broker client *Client // channel to communicate back to the long running "aggregator" goroutine. done chan struct{} // list of queue names to check and aggregate. queues []string // Group configurations gracePeriod time.Duration maxDelay time.Duration maxSize int // User provided group aggregator. ga GroupAggregator // interval used to check for aggregation interval time.Duration // sema is a counting semaphore to ensure the number of active aggregating function // does not exceed the limit. sema chan struct{} } type aggregatorParams struct { logger *log.Logger broker base.Broker queues []string gracePeriod time.Duration maxDelay time.Duration maxSize int groupAggregator GroupAggregator } const ( // Maximum number of aggregation checks in flight concurrently. maxConcurrentAggregationChecks = 3 // Default interval used for aggregation checks. If the provided gracePeriod is less than // the default, use the gracePeriod. defaultAggregationCheckInterval = 7 * time.Second ) func newAggregator(params aggregatorParams) *aggregator { interval := defaultAggregationCheckInterval if params.gracePeriod < interval { interval = params.gracePeriod } return &aggregator{ logger: params.logger, broker: params.broker, client: &Client{broker: params.broker}, done: make(chan struct{}), queues: params.queues, gracePeriod: params.gracePeriod, maxDelay: params.maxDelay, maxSize: params.maxSize, ga: params.groupAggregator, sema: make(chan struct{}, maxConcurrentAggregationChecks), interval: interval, } } func (a *aggregator) shutdown() { if a.ga == nil { return } a.logger.Debug("Aggregator shutting down...") // Signal the aggregator goroutine to stop. a.done <- struct{}{} } func (a *aggregator) start(wg *sync.WaitGroup) { if a.ga == nil { return } wg.Add(1) go func() { defer wg.Done() ticker := time.NewTicker(a.interval) for { select { case <-a.done: a.logger.Debug("Waiting for all aggregation checks to finish...") // block until all aggregation checks released the token for i := 0; i < cap(a.sema); i++ { a.sema <- struct{}{} } a.logger.Debug("Aggregator done") ticker.Stop() return case t := <-ticker.C: a.exec(t) } } }() } func (a *aggregator) exec(t time.Time) { select { case a.sema <- struct{}{}: // acquire token go a.aggregate(t) default: // If the semaphore blocks, then we are currently running max number of // aggregation checks. Skip this round and log warning. a.logger.Warnf("Max number of aggregation checks in flight. Skipping") } } func (a *aggregator) aggregate(t time.Time) { defer func() { <-a.sema /* release token */ }() for _, qname := range a.queues { groups, err := a.broker.ListGroups(qname) if err != nil { a.logger.Errorf("Failed to list groups in queue: %q", qname) continue } for _, gname := range groups { aggregationSetID, err := a.broker.AggregationCheck( qname, gname, t, a.gracePeriod, a.maxDelay, a.maxSize) if err != nil { a.logger.Errorf("Failed to run aggregation check: queue=%q group=%q", qname, gname) continue } if aggregationSetID == "" { a.logger.Debugf("No aggregation needed at this time: queue=%q group=%q", qname, gname) continue } // Aggregate and enqueue. msgs, deadline, err := a.broker.ReadAggregationSet(qname, gname, aggregationSetID) if err != nil { a.logger.Errorf("Failed to read aggregation set: queue=%q, group=%q, setID=%q", qname, gname, aggregationSetID) continue } tasks := make([]*Task, len(msgs)) for i, m := range msgs { tasks[i] = NewTaskWithHeaders(m.Type, m.Payload, m.Headers) } aggregatedTask := a.ga.Aggregate(gname, tasks) ctx, cancel := context.WithDeadline(context.Background(), deadline) if _, err := a.client.EnqueueContext(ctx, aggregatedTask, Queue(qname)); err != nil { a.logger.Errorf("Failed to enqueue aggregated task (queue=%q, group=%q, setID=%q): %v", qname, gname, aggregationSetID, err) cancel() continue } if err := a.broker.DeleteAggregationSet(ctx, qname, gname, aggregationSetID); err != nil { a.logger.Warnf("Failed to delete aggregation set: queue=%q, group=%q, setID=%q", qname, gname, aggregationSetID) } cancel() } } } ================================================ FILE: aggregator_test.go ================================================ // Copyright 2022 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "sync" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/rdb" h "github.com/hibiken/asynq/internal/testutil" ) func TestAggregator(t *testing.T) { r := setup(t) defer r.Close() rdbClient := rdb.NewRDB(r) client := Client{broker: rdbClient} tests := []struct { desc string gracePeriod time.Duration maxDelay time.Duration maxSize int aggregateFunc func(gname string, tasks []*Task) *Task tasks []*Task // tasks to enqueue enqueueFrequency time.Duration // time between one enqueue event to another waitTime time.Duration // time to wait wantGroups map[string]map[string][]base.Z wantPending map[string][]*base.TaskMessage }{ { desc: "group older than the grace period should be aggregated", gracePeriod: 1 * time.Second, maxDelay: 0, // no maxdelay limit maxSize: 0, // no maxsize limit aggregateFunc: func(gname string, tasks []*Task) *Task { return NewTask(gname, nil, MaxRetry(len(tasks))) // use max retry to see how many tasks were aggregated }, tasks: []*Task{ NewTask("task1", nil, Group("mygroup")), NewTask("task2", nil, Group("mygroup")), NewTask("task3", nil, Group("mygroup")), }, enqueueFrequency: 300 * time.Millisecond, waitTime: 3 * time.Second, wantGroups: map[string]map[string][]base.Z{ "default": { "mygroup": {}, }, }, wantPending: map[string][]*base.TaskMessage{ "default": { h.NewTaskMessageBuilder().SetType("mygroup").SetRetry(3).Build(), }, }, }, { desc: "group older than the max-delay should be aggregated", gracePeriod: 2 * time.Second, maxDelay: 4 * time.Second, maxSize: 0, // no maxsize limit aggregateFunc: func(gname string, tasks []*Task) *Task { return NewTask(gname, nil, MaxRetry(len(tasks))) // use max retry to see how many tasks were aggregated }, tasks: []*Task{ NewTask("task1", nil, Group("mygroup")), // time 0 NewTask("task2", nil, Group("mygroup")), // time 1s NewTask("task3", nil, Group("mygroup")), // time 2s NewTask("task4", nil, Group("mygroup")), // time 3s }, enqueueFrequency: 1 * time.Second, waitTime: 4 * time.Second, wantGroups: map[string]map[string][]base.Z{ "default": { "mygroup": {}, }, }, wantPending: map[string][]*base.TaskMessage{ "default": { h.NewTaskMessageBuilder().SetType("mygroup").SetRetry(4).Build(), }, }, }, { desc: "group reached the max-size should be aggregated", gracePeriod: 1 * time.Minute, maxDelay: 0, // no maxdelay limit maxSize: 5, aggregateFunc: func(gname string, tasks []*Task) *Task { return NewTask(gname, nil, MaxRetry(len(tasks))) // use max retry to see how many tasks were aggregated }, tasks: []*Task{ NewTask("task1", nil, Group("mygroup")), NewTask("task2", nil, Group("mygroup")), NewTask("task3", nil, Group("mygroup")), NewTask("task4", nil, Group("mygroup")), NewTask("task5", nil, Group("mygroup")), }, enqueueFrequency: 300 * time.Millisecond, waitTime: defaultAggregationCheckInterval * 2, wantGroups: map[string]map[string][]base.Z{ "default": { "mygroup": {}, }, }, wantPending: map[string][]*base.TaskMessage{ "default": { h.NewTaskMessageBuilder().SetType("mygroup").SetRetry(5).Build(), }, }, }, } for _, tc := range tests { h.FlushDB(t, r) aggregator := newAggregator(aggregatorParams{ logger: testLogger, broker: rdbClient, queues: []string{"default"}, gracePeriod: tc.gracePeriod, maxDelay: tc.maxDelay, maxSize: tc.maxSize, groupAggregator: GroupAggregatorFunc(tc.aggregateFunc), }) var wg sync.WaitGroup aggregator.start(&wg) for _, task := range tc.tasks { if _, err := client.Enqueue(task); err != nil { t.Errorf("%s: Client Enqueue failed: %v", tc.desc, err) aggregator.shutdown() wg.Wait() continue } time.Sleep(tc.enqueueFrequency) } time.Sleep(tc.waitTime) for qname, groups := range tc.wantGroups { for gname, want := range groups { gotGroup := h.GetGroupEntries(t, r, qname, gname) if diff := cmp.Diff(want, gotGroup, h.SortZSetEntryOpt); diff != "" { t.Errorf("%s: mismatch found in %q; (-want,+got)\n%s", tc.desc, base.GroupKey(qname, gname), diff) } } } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt, h.IgnoreIDOpt); diff != "" { t.Errorf("%s: mismatch found in %q; (-want,+got)\n%s", tc.desc, base.PendingKey(qname), diff) } } aggregator.shutdown() wg.Wait() } } ================================================ FILE: asynq.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "context" "crypto/tls" "fmt" "maps" "net" "net/url" "strconv" "strings" "time" "github.com/hibiken/asynq/internal/base" "github.com/redis/go-redis/v9" ) // Task represents a unit of work to be performed. type Task struct { // typename indicates the type of task to be performed. typename string // payload holds data needed to perform the task. payload []byte // headers holds additional metadata for the task. headers map[string]string // opts holds options for the task. opts []Option // w is the ResultWriter for the task. w *ResultWriter } func (t *Task) Type() string { return t.typename } func (t *Task) Payload() []byte { return t.payload } func (t *Task) Headers() map[string]string { return t.headers } // ResultWriter returns a pointer to the ResultWriter associated with the task. // // Nil pointer is returned if called on a newly created task (i.e. task created by calling NewTask). // Only the tasks passed to Handler.ProcessTask have a valid ResultWriter pointer. func (t *Task) ResultWriter() *ResultWriter { return t.w } // NewTask returns a new Task given a type name and payload data. // Options can be passed to configure task processing behavior. func NewTask(typename string, payload []byte, opts ...Option) *Task { return &Task{ typename: typename, payload: payload, headers: nil, opts: opts, } } // NewTaskWithHeaders returns a new Task given a type name, payload data, and headers. // Options can be passed to configure task processing behavior. // TODO: In the next major (breaking) release, fold this functionality into NewTask // // so that headers are supported directly. After that, remove this method. func NewTaskWithHeaders(typename string, payload []byte, headers map[string]string, opts ...Option) *Task { return &Task{ typename: typename, payload: payload, headers: maps.Clone(headers), opts: opts, } } // newTask creates a task with the given typename, payload and ResultWriter. func newTask(typename string, payload []byte, w *ResultWriter) *Task { return &Task{ typename: typename, payload: payload, headers: make(map[string]string), w: w, } } // A TaskInfo describes a task and its metadata. type TaskInfo struct { // ID is the identifier of the task. ID string // Queue is the name of the queue in which the task belongs. Queue string // Type is the type name of the task. Type string // Payload is the payload data of the task. Payload []byte // Headers holds additional metadata for the task. Headers map[string]string // State indicates the task state. State TaskState // MaxRetry is the maximum number of times the task can be retried. MaxRetry int // Retried is the number of times the task has retried so far. Retried int // LastErr is the error message from the last failure. LastErr string // LastFailedAt is the time time of the last failure if any. // If the task has no failures, LastFailedAt is zero time (i.e. time.Time{}). LastFailedAt time.Time // Timeout is the duration the task can be processed by Handler before being retried, // zero if not specified Timeout time.Duration // Deadline is the deadline for the task, zero value if not specified. Deadline time.Time // Group is the name of the group in which the task belongs. // // Tasks in the same queue can be grouped together by Group name and will be aggregated into one task // by a Server processing the queue. // // Empty string (default) indicates task does not belong to any groups, and no aggregation will be applied to the task. Group string // NextProcessAt is the time the task is scheduled to be processed, // zero if not applicable. NextProcessAt time.Time // IsOrphaned describes whether the task is left in active state with no worker processing it. // An orphaned task indicates that the worker has crashed or experienced network failures and was not able to // extend its lease on the task. // // This task will be recovered by running a server against the queue the task is in. // This field is only applicable to tasks with TaskStateActive. IsOrphaned bool // Retention is duration of the retention period after the task is successfully processed. Retention time.Duration // CompletedAt is the time when the task is processed successfully. // Zero value (i.e. time.Time{}) indicates no value. CompletedAt time.Time // Result holds the result data associated with the task. // Use ResultWriter to write result data from the Handler. Result []byte } // If t is non-zero, returns time converted from t as unix time in seconds. // If t is zero, returns zero value of time.Time. func fromUnixTimeOrZero(t int64) time.Time { if t == 0 { return time.Time{} } return time.Unix(t, 0) } func newTaskInfo(msg *base.TaskMessage, state base.TaskState, nextProcessAt time.Time, result []byte) *TaskInfo { info := TaskInfo{ ID: msg.ID, Queue: msg.Queue, Type: msg.Type, Payload: msg.Payload, // Do we need to make a copy? Headers: msg.Headers, MaxRetry: msg.Retry, Retried: msg.Retried, LastErr: msg.ErrorMsg, Group: msg.GroupKey, Timeout: time.Duration(msg.Timeout) * time.Second, Deadline: fromUnixTimeOrZero(msg.Deadline), Retention: time.Duration(msg.Retention) * time.Second, NextProcessAt: nextProcessAt, LastFailedAt: fromUnixTimeOrZero(msg.LastFailedAt), CompletedAt: fromUnixTimeOrZero(msg.CompletedAt), Result: result, } switch state { case base.TaskStateActive: info.State = TaskStateActive case base.TaskStatePending: info.State = TaskStatePending case base.TaskStateScheduled: info.State = TaskStateScheduled case base.TaskStateRetry: info.State = TaskStateRetry case base.TaskStateArchived: info.State = TaskStateArchived case base.TaskStateCompleted: info.State = TaskStateCompleted case base.TaskStateAggregating: info.State = TaskStateAggregating default: panic(fmt.Sprintf("internal error: unknown state: %d", state)) } return &info } // TaskState denotes the state of a task. type TaskState int const ( // Indicates that the task is currently being processed by Handler. TaskStateActive TaskState = iota + 1 // Indicates that the task is ready to be processed by Handler. TaskStatePending // Indicates that the task is scheduled to be processed some time in the future. TaskStateScheduled // Indicates that the task has previously failed and scheduled to be processed some time in the future. TaskStateRetry // Indicates that the task is archived and stored for inspection purposes. TaskStateArchived // Indicates that the task is processed successfully and retained until the retention TTL expires. TaskStateCompleted // Indicates that the task is waiting in a group to be aggregated into one task. TaskStateAggregating ) func (s TaskState) String() string { switch s { case TaskStateActive: return "active" case TaskStatePending: return "pending" case TaskStateScheduled: return "scheduled" case TaskStateRetry: return "retry" case TaskStateArchived: return "archived" case TaskStateCompleted: return "completed" case TaskStateAggregating: return "aggregating" } panic("asynq: unknown task state") } // RedisConnOpt is a discriminated union of types that represent Redis connection configuration option. // // RedisConnOpt represents a sum of following types: // // - RedisClientOpt // - RedisFailoverClientOpt // - RedisClusterClientOpt type RedisConnOpt interface { // MakeRedisClient returns a new redis client instance. // Return value is intentionally opaque to hide the implementation detail of redis client. MakeRedisClient() interface{} } // RedisClientOpt is used to create a redis client that connects // to a redis server directly. type RedisClientOpt struct { // Network type to use, either tcp or unix. // Default is tcp. Network string // Redis server address in "host:port" format. Addr string // Username to authenticate the current connection when Redis ACLs are used. // See: https://redis.io/commands/auth. Username string // Password to authenticate the current connection. // See: https://redis.io/commands/auth. Password string // Redis DB to select after connecting to a server. // See: https://redis.io/commands/select. DB int // Dial timeout for establishing new connections. // Default is 5 seconds. DialTimeout time.Duration // Timeout for socket reads. // If timeout is reached, read commands will fail with a timeout error // instead of blocking. // // Use value -1 for no timeout and 0 for default. // Default is 3 seconds. ReadTimeout time.Duration // Timeout for socket writes. // If timeout is reached, write commands will fail with a timeout error // instead of blocking. // // Use value -1 for no timeout and 0 for default. // Default is ReadTimout. WriteTimeout time.Duration // Maximum number of socket connections. // Default is 10 connections per every CPU as reported by runtime.NumCPU. PoolSize int // TLS Config used to connect to a server. // TLS will be negotiated only if this field is set. TLSConfig *tls.Config } func (opt RedisClientOpt) MakeRedisClient() interface{} { return redis.NewClient(&redis.Options{ Network: opt.Network, Addr: opt.Addr, Username: opt.Username, Password: opt.Password, DB: opt.DB, DialTimeout: opt.DialTimeout, ReadTimeout: opt.ReadTimeout, WriteTimeout: opt.WriteTimeout, PoolSize: opt.PoolSize, TLSConfig: opt.TLSConfig, }) } // RedisFailoverClientOpt is used to creates a redis client that talks // to redis sentinels for service discovery and has an automatic failover // capability. type RedisFailoverClientOpt struct { // Redis master name that monitored by sentinels. MasterName string // Addresses of sentinels in "host:port" format. // Use at least three sentinels to avoid problems described in // https://redis.io/topics/sentinel. SentinelAddrs []string // Redis sentinel username. SentinelUsername string // Redis sentinel password. SentinelPassword string // Username to authenticate the current connection when Redis ACLs are used. // See: https://redis.io/commands/auth. Username string // Password to authenticate the current connection. // See: https://redis.io/commands/auth. Password string // Redis DB to select after connecting to a server. // See: https://redis.io/commands/select. DB int // Dial timeout for establishing new connections. // Default is 5 seconds. DialTimeout time.Duration // Timeout for socket reads. // If timeout is reached, read commands will fail with a timeout error // instead of blocking. // // Use value -1 for no timeout and 0 for default. // Default is 3 seconds. ReadTimeout time.Duration // Timeout for socket writes. // If timeout is reached, write commands will fail with a timeout error // instead of blocking. // // Use value -1 for no timeout and 0 for default. // Default is ReadTimeout WriteTimeout time.Duration // Maximum number of socket connections. // Default is 10 connections per every CPU as reported by runtime.NumCPU. PoolSize int // TLS Config used to connect to a server. // TLS will be negotiated only if this field is set. TLSConfig *tls.Config } func (opt RedisFailoverClientOpt) MakeRedisClient() interface{} { return redis.NewFailoverClient(&redis.FailoverOptions{ MasterName: opt.MasterName, SentinelAddrs: opt.SentinelAddrs, SentinelUsername: opt.SentinelUsername, SentinelPassword: opt.SentinelPassword, Username: opt.Username, Password: opt.Password, DB: opt.DB, DialTimeout: opt.DialTimeout, ReadTimeout: opt.ReadTimeout, WriteTimeout: opt.WriteTimeout, PoolSize: opt.PoolSize, TLSConfig: opt.TLSConfig, }) } // RedisClusterClientOpt is used to creates a redis client that connects to // redis cluster. type RedisClusterClientOpt struct { // A seed list of host:port addresses of cluster nodes. Addrs []string // The maximum number of retries before giving up. // Command is retried on network errors and MOVED/ASK redirects. // Default is 8 retries. MaxRedirects int // Username to authenticate the current connection when Redis ACLs are used. // See: https://redis.io/commands/auth. Username string // Password to authenticate the current connection. // See: https://redis.io/commands/auth. Password string // Dial timeout for establishing new connections. // Default is 5 seconds. DialTimeout time.Duration // Timeout for socket reads. // If timeout is reached, read commands will fail with a timeout error // instead of blocking. // // Use value -1 for no timeout and 0 for default. // Default is 3 seconds. ReadTimeout time.Duration // Timeout for socket writes. // If timeout is reached, write commands will fail with a timeout error // instead of blocking. // // Use value -1 for no timeout and 0 for default. // Default is ReadTimeout. WriteTimeout time.Duration // TLS Config used to connect to a server. // TLS will be negotiated only if this field is set. TLSConfig *tls.Config } func (opt RedisClusterClientOpt) MakeRedisClient() interface{} { return redis.NewClusterClient(&redis.ClusterOptions{ Addrs: opt.Addrs, MaxRedirects: opt.MaxRedirects, Username: opt.Username, Password: opt.Password, DialTimeout: opt.DialTimeout, ReadTimeout: opt.ReadTimeout, WriteTimeout: opt.WriteTimeout, TLSConfig: opt.TLSConfig, }) } // ParseRedisURI parses redis uri string and returns RedisConnOpt if uri is valid. // It returns a non-nil error if uri cannot be parsed. // // Three URI schemes are supported, which are redis:, rediss:, redis-socket:, and redis-sentinel:. // Supported formats are: // // redis://[:password@]host[:port][/dbnumber] // rediss://[:password@]host[:port][/dbnumber] // redis-socket://[:password@]path[?db=dbnumber] // redis-sentinel://[:password@]host1[:port][,host2:[:port]][,hostN:[:port]][?master=masterName] func ParseRedisURI(uri string) (RedisConnOpt, error) { u, err := url.Parse(uri) if err != nil { return nil, fmt.Errorf("asynq: could not parse redis uri: %w", err) } switch u.Scheme { case "redis", "rediss": return parseRedisURI(u) case "redis-socket": return parseRedisSocketURI(u) case "redis-sentinel": return parseRedisSentinelURI(u) default: return nil, fmt.Errorf("asynq: unsupported uri scheme: %q", u.Scheme) } } func parseRedisURI(u *url.URL) (RedisConnOpt, error) { var db int var err error var redisConnOpt RedisClientOpt if len(u.Path) > 0 { xs := strings.Split(strings.Trim(u.Path, "/"), "/") db, err = strconv.Atoi(xs[0]) if err != nil { return nil, fmt.Errorf("asynq: could not parse redis uri: database number should be the first segment of the path") } } var password string if v, ok := u.User.Password(); ok { password = v } if u.Scheme == "rediss" { h, _, err := net.SplitHostPort(u.Host) if err != nil { h = u.Host } redisConnOpt.TLSConfig = &tls.Config{ServerName: h} } redisConnOpt.Addr = u.Host redisConnOpt.Password = password redisConnOpt.DB = db return redisConnOpt, nil } func parseRedisSocketURI(u *url.URL) (RedisConnOpt, error) { const errPrefix = "asynq: could not parse redis socket uri" if len(u.Path) == 0 { return nil, fmt.Errorf("%s: path does not exist", errPrefix) } q := u.Query() var db int var err error if n := q.Get("db"); n != "" { db, err = strconv.Atoi(n) if err != nil { return nil, fmt.Errorf("%s: query param `db` should be a number", errPrefix) } } var password string if v, ok := u.User.Password(); ok { password = v } return RedisClientOpt{Network: "unix", Addr: u.Path, DB: db, Password: password}, nil } func parseRedisSentinelURI(u *url.URL) (RedisConnOpt, error) { addrs := strings.Split(u.Host, ",") master := u.Query().Get("master") var password string if v, ok := u.User.Password(); ok { password = v } return RedisFailoverClientOpt{MasterName: master, SentinelAddrs: addrs, SentinelPassword: password}, nil } // ResultWriter is a client interface to write result data for a task. // It writes the data to the redis instance the server is connected to. type ResultWriter struct { id string // task ID this writer is responsible for qname string // queue name the task belongs to broker base.Broker ctx context.Context // context associated with the task } // Write writes the given data as a result of the task the ResultWriter is associated with. func (w *ResultWriter) Write(data []byte) (n int, err error) { select { case <-w.ctx.Done(): return 0, fmt.Errorf("failed to write task result: %w", w.ctx.Err()) default: } return w.broker.WriteResult(w.qname, w.id, data) } // TaskID returns the ID of the task the ResultWriter is associated with. func (w *ResultWriter) TaskID() string { return w.id } ================================================ FILE: asynq_test.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "crypto/tls" "flag" "sort" "strings" "testing" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/hibiken/asynq/internal/log" h "github.com/hibiken/asynq/internal/testutil" "github.com/redis/go-redis/v9" ) //============================================================================ // This file defines helper functions and variables used in other test files. //============================================================================ // variables used for package testing. var ( redisAddr string redisDB int useRedisCluster bool redisClusterAddrs string // comma-separated list of host:port testLogLevel = FatalLevel ) var testLogger *log.Logger func init() { flag.StringVar(&redisAddr, "redis_addr", "localhost:6379", "redis address to use in testing") flag.IntVar(&redisDB, "redis_db", 14, "redis db number to use in testing") flag.BoolVar(&useRedisCluster, "redis_cluster", false, "use redis cluster as a broker in testing") flag.StringVar(&redisClusterAddrs, "redis_cluster_addrs", "localhost:7000,localhost:7001,localhost:7002", "comma separated list of redis server addresses") flag.Var(&testLogLevel, "loglevel", "log level to use in testing") testLogger = log.NewLogger(nil) testLogger.SetLevel(toInternalLogLevel(testLogLevel)) } func setup(tb testing.TB) (r redis.UniversalClient) { tb.Helper() if useRedisCluster { addrs := strings.Split(redisClusterAddrs, ",") if len(addrs) == 0 { tb.Fatal("No redis cluster addresses provided. Please set addresses using --redis_cluster_addrs flag.") } r = redis.NewClusterClient(&redis.ClusterOptions{ Addrs: addrs, }) } else { r = redis.NewClient(&redis.Options{ Addr: redisAddr, DB: redisDB, }) } // Start each test with a clean slate. h.FlushDB(tb, r) return r } func getRedisConnOpt(tb testing.TB) RedisConnOpt { tb.Helper() if useRedisCluster { addrs := strings.Split(redisClusterAddrs, ",") if len(addrs) == 0 { tb.Fatal("No redis cluster addresses provided. Please set addresses using --redis_cluster_addrs flag.") } return RedisClusterClientOpt{ Addrs: addrs, } } return RedisClientOpt{ Addr: redisAddr, DB: redisDB, } } var sortTaskOpt = cmp.Transformer("SortMsg", func(in []*Task) []*Task { out := append([]*Task(nil), in...) // Copy input to avoid mutating it sort.Slice(out, func(i, j int) bool { return out[i].Type() < out[j].Type() }) return out }) func TestParseRedisURI(t *testing.T) { tests := []struct { uri string want RedisConnOpt }{ { "redis://localhost:6379", RedisClientOpt{Addr: "localhost:6379"}, }, { "rediss://localhost:6379", RedisClientOpt{Addr: "localhost:6379", TLSConfig: &tls.Config{ServerName: "localhost"}}, }, { "redis://localhost:6379/3", RedisClientOpt{Addr: "localhost:6379", DB: 3}, }, { "redis://:mypassword@localhost:6379", RedisClientOpt{Addr: "localhost:6379", Password: "mypassword"}, }, { "redis://:mypassword@127.0.0.1:6379/11", RedisClientOpt{Addr: "127.0.0.1:6379", Password: "mypassword", DB: 11}, }, { "redis-socket:///var/run/redis/redis.sock", RedisClientOpt{Network: "unix", Addr: "/var/run/redis/redis.sock"}, }, { "redis-socket://:mypassword@/var/run/redis/redis.sock", RedisClientOpt{Network: "unix", Addr: "/var/run/redis/redis.sock", Password: "mypassword"}, }, { "redis-socket:///var/run/redis/redis.sock?db=7", RedisClientOpt{Network: "unix", Addr: "/var/run/redis/redis.sock", DB: 7}, }, { "redis-socket://:mypassword@/var/run/redis/redis.sock?db=12", RedisClientOpt{Network: "unix", Addr: "/var/run/redis/redis.sock", Password: "mypassword", DB: 12}, }, { "redis-sentinel://localhost:5000,localhost:5001,localhost:5002?master=mymaster", RedisFailoverClientOpt{ MasterName: "mymaster", SentinelAddrs: []string{"localhost:5000", "localhost:5001", "localhost:5002"}, }, }, { "redis-sentinel://:mypassword@localhost:5000,localhost:5001,localhost:5002?master=mymaster", RedisFailoverClientOpt{ MasterName: "mymaster", SentinelAddrs: []string{"localhost:5000", "localhost:5001", "localhost:5002"}, SentinelPassword: "mypassword", }, }, } for _, tc := range tests { got, err := ParseRedisURI(tc.uri) if err != nil { t.Errorf("ParseRedisURI(%q) returned an error: %v", tc.uri, err) continue } if diff := cmp.Diff(tc.want, got, cmpopts.IgnoreUnexported(tls.Config{})); diff != "" { t.Errorf("ParseRedisURI(%q) = %+v, want %+v\n(-want,+got)\n%s", tc.uri, got, tc.want, diff) } } } func TestParseRedisURIErrors(t *testing.T) { tests := []struct { desc string uri string }{ { "unsupported scheme", "rdb://localhost:6379", }, { "missing scheme", "localhost:6379", }, { "multiple db numbers", "redis://localhost:6379/1,2,3", }, { "missing path for socket connection", "redis-socket://?db=one", }, { "non integer for db numbers for socket", "redis-socket:///some/path/to/redis?db=one", }, } for _, tc := range tests { _, err := ParseRedisURI(tc.uri) if err == nil { t.Errorf("%s: ParseRedisURI(%q) succeeded for malformed input, want error", tc.desc, tc.uri) } } } ================================================ FILE: benchmark_test.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "context" "encoding/json" "fmt" "sync" "testing" "time" h "github.com/hibiken/asynq/internal/testutil" ) // Creates a new task of type "task" with payload {"data": n}. func makeTask(n int) *Task { b, err := json.Marshal(map[string]int{"data": n}) if err != nil { panic(err) } return NewTask(fmt.Sprintf("task%d", n), b) } // Simple E2E Benchmark testing with no scheduled tasks and retries. func BenchmarkEndToEndSimple(b *testing.B) { const count = 100000 for n := 0; n < b.N; n++ { b.StopTimer() // begin setup setup(b) redis := getRedisConnOpt(b) client := NewClient(redis) srv := NewServer(redis, Config{ Concurrency: 10, RetryDelayFunc: func(n int, err error, t *Task) time.Duration { return time.Second }, LogLevel: testLogLevel, }) // Create a bunch of tasks for i := 0; i < count; i++ { if _, err := client.Enqueue(makeTask(i)); err != nil { b.Fatalf("could not enqueue a task: %v", err) } } client.Close() var wg sync.WaitGroup wg.Add(count) handler := func(ctx context.Context, t *Task) error { wg.Done() return nil } b.StartTimer() // end setup _ = srv.Start(HandlerFunc(handler)) wg.Wait() b.StopTimer() // begin teardown srv.Stop() b.StartTimer() // end teardown } } // E2E benchmark with scheduled tasks and retries. func BenchmarkEndToEnd(b *testing.B) { const count = 100000 for n := 0; n < b.N; n++ { b.StopTimer() // begin setup setup(b) redis := getRedisConnOpt(b) client := NewClient(redis) srv := NewServer(redis, Config{ Concurrency: 10, RetryDelayFunc: func(n int, err error, t *Task) time.Duration { return time.Second }, LogLevel: testLogLevel, }) // Create a bunch of tasks for i := 0; i < count; i++ { if _, err := client.Enqueue(makeTask(i)); err != nil { b.Fatalf("could not enqueue a task: %v", err) } } for i := 0; i < count; i++ { if _, err := client.Enqueue(makeTask(i), ProcessIn(1*time.Second)); err != nil { b.Fatalf("could not enqueue a task: %v", err) } } client.Close() var wg sync.WaitGroup wg.Add(count * 2) handler := func(ctx context.Context, t *Task) error { var p map[string]int if err := json.Unmarshal(t.Payload(), &p); err != nil { b.Logf("internal error: %v", err) } n, ok := p["data"] if !ok { n = 1 b.Logf("internal error: could not get data from payload") } retried, ok := GetRetryCount(ctx) if !ok { b.Logf("internal error: could not get retry count from context") } // Fail 1% of tasks for the first attempt. if retried == 0 && n%100 == 0 { return fmt.Errorf(":(") } wg.Done() return nil } b.StartTimer() // end setup _ = srv.Start(HandlerFunc(handler)) wg.Wait() b.StopTimer() // begin teardown srv.Stop() b.StartTimer() // end teardown } } // Simple E2E Benchmark testing with no scheduled tasks and retries with multiple queues. func BenchmarkEndToEndMultipleQueues(b *testing.B) { // number of tasks to create for each queue const ( highCount = 20000 defaultCount = 20000 lowCount = 20000 ) for n := 0; n < b.N; n++ { b.StopTimer() // begin setup setup(b) redis := getRedisConnOpt(b) client := NewClient(redis) srv := NewServer(redis, Config{ Concurrency: 10, Queues: map[string]int{ "high": 6, "default": 3, "low": 1, }, LogLevel: testLogLevel, }) // Create a bunch of tasks for i := 0; i < highCount; i++ { if _, err := client.Enqueue(makeTask(i), Queue("high")); err != nil { b.Fatalf("could not enqueue a task: %v", err) } } for i := 0; i < defaultCount; i++ { if _, err := client.Enqueue(makeTask(i)); err != nil { b.Fatalf("could not enqueue a task: %v", err) } } for i := 0; i < lowCount; i++ { if _, err := client.Enqueue(makeTask(i), Queue("low")); err != nil { b.Fatalf("could not enqueue a task: %v", err) } } client.Close() var wg sync.WaitGroup wg.Add(highCount + defaultCount + lowCount) handler := func(ctx context.Context, t *Task) error { wg.Done() return nil } b.StartTimer() // end setup _ = srv.Start(HandlerFunc(handler)) wg.Wait() b.StopTimer() // begin teardown srv.Stop() b.StartTimer() // end teardown } } // E2E benchmark to check client enqueue operation performs correctly, // while server is busy processing tasks. func BenchmarkClientWhileServerRunning(b *testing.B) { const count = 10000 for n := 0; n < b.N; n++ { b.StopTimer() // begin setup setup(b) redis := getRedisConnOpt(b) client := NewClient(redis) srv := NewServer(redis, Config{ Concurrency: 10, RetryDelayFunc: func(n int, err error, t *Task) time.Duration { return time.Second }, LogLevel: testLogLevel, }) // Enqueue 10,000 tasks. for i := 0; i < count; i++ { if _, err := client.Enqueue(makeTask(i)); err != nil { b.Fatalf("could not enqueue a task: %v", err) } } // Schedule 10,000 tasks. for i := 0; i < count; i++ { if _, err := client.Enqueue(makeTask(i), ProcessIn(1*time.Second)); err != nil { b.Fatalf("could not enqueue a task: %v", err) } } handler := func(ctx context.Context, t *Task) error { return nil } _ = srv.Start(HandlerFunc(handler)) b.StartTimer() // end setup b.Log("Starting enqueueing") enqueued := 0 for enqueued < 100000 { t := NewTask(fmt.Sprintf("enqueued%d", enqueued), h.JSON(map[string]interface{}{"data": enqueued})) if _, err := client.Enqueue(t); err != nil { b.Logf("could not enqueue task %d: %v", enqueued, err) continue } enqueued++ } b.Logf("Finished enqueueing %d tasks", enqueued) b.StopTimer() // begin teardown srv.Stop() client.Close() b.StartTimer() // end teardown } } ================================================ FILE: client.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "context" "fmt" "strings" "time" "github.com/google/uuid" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/errors" "github.com/hibiken/asynq/internal/rdb" "github.com/redis/go-redis/v9" ) // A Client is responsible for scheduling tasks. // // A Client is used to register tasks that should be processed // immediately or some time in the future. // // Clients are safe for concurrent use by multiple goroutines. type Client struct { broker base.Broker // When a Client has been created with an existing Redis connection, we do // not want to close it. sharedConnection bool } // NewClient returns a new Client instance given a redis connection option. func NewClient(r RedisConnOpt) *Client { redisClient, ok := r.MakeRedisClient().(redis.UniversalClient) if !ok { panic(fmt.Sprintf("asynq: unsupported RedisConnOpt type %T", r)) } client := NewClientFromRedisClient(redisClient) client.sharedConnection = false return client } // NewClientFromRedisClient returns a new instance of Client given a redis.UniversalClient // Warning: The underlying redis connection pool will not be closed by Asynq, you are responsible for closing it. func NewClientFromRedisClient(c redis.UniversalClient) *Client { return &Client{broker: rdb.NewRDB(c), sharedConnection: true} } type OptionType int const ( MaxRetryOpt OptionType = iota QueueOpt TimeoutOpt DeadlineOpt UniqueOpt ProcessAtOpt ProcessInOpt TaskIDOpt RetentionOpt GroupOpt ) // Option specifies the task processing behavior. type Option interface { // String returns a string representation of the option. String() string // Type describes the type of the option. Type() OptionType // Value returns a value used to create this option. Value() interface{} } // Internal option representations. type ( retryOption int queueOption string taskIDOption string timeoutOption time.Duration deadlineOption time.Time uniqueOption time.Duration processAtOption time.Time processInOption time.Duration retentionOption time.Duration groupOption string ) // MaxRetry returns an option to specify the max number of times // the task will be retried. // // Negative retry count is treated as zero retry. func MaxRetry(n int) Option { if n < 0 { n = 0 } return retryOption(n) } func (n retryOption) String() string { return fmt.Sprintf("MaxRetry(%d)", int(n)) } func (n retryOption) Type() OptionType { return MaxRetryOpt } func (n retryOption) Value() interface{} { return int(n) } // Queue returns an option to specify the queue to enqueue the task into. func Queue(name string) Option { return queueOption(name) } func (name queueOption) String() string { return fmt.Sprintf("Queue(%q)", string(name)) } func (name queueOption) Type() OptionType { return QueueOpt } func (name queueOption) Value() interface{} { return string(name) } // TaskID returns an option to specify the task ID. func TaskID(id string) Option { return taskIDOption(id) } func (id taskIDOption) String() string { return fmt.Sprintf("TaskID(%q)", string(id)) } func (id taskIDOption) Type() OptionType { return TaskIDOpt } func (id taskIDOption) Value() interface{} { return string(id) } // Timeout returns an option to specify how long a task may run. // If the timeout elapses before the Handler returns, then the task // will be retried. // // Zero duration means no limit. // // If there's a conflicting Deadline option, whichever comes earliest // will be used. func Timeout(d time.Duration) Option { return timeoutOption(d) } func (d timeoutOption) String() string { return fmt.Sprintf("Timeout(%v)", time.Duration(d)) } func (d timeoutOption) Type() OptionType { return TimeoutOpt } func (d timeoutOption) Value() interface{} { return time.Duration(d) } // Deadline returns an option to specify the deadline for the given task. // If it reaches the deadline before the Handler returns, then the task // will be retried. // // If there's a conflicting Timeout option, whichever comes earliest // will be used. func Deadline(t time.Time) Option { return deadlineOption(t) } func (t deadlineOption) String() string { return fmt.Sprintf("Deadline(%v)", time.Time(t).Format(time.UnixDate)) } func (t deadlineOption) Type() OptionType { return DeadlineOpt } func (t deadlineOption) Value() interface{} { return time.Time(t) } // Unique returns an option to enqueue a task only if the given task is unique. // Task enqueued with this option is guaranteed to be unique within the given ttl. // Once the task gets processed successfully or once the TTL has expired, // another task with the same uniqueness may be enqueued. // ErrDuplicateTask error is returned when enqueueing a duplicate task. // TTL duration must be greater than or equal to 1 second. // // Uniqueness of a task is based on the following properties: // - Task Type // - Task Payload // - Queue Name func Unique(ttl time.Duration) Option { return uniqueOption(ttl) } func (ttl uniqueOption) String() string { return fmt.Sprintf("Unique(%v)", time.Duration(ttl)) } func (ttl uniqueOption) Type() OptionType { return UniqueOpt } func (ttl uniqueOption) Value() interface{} { return time.Duration(ttl) } // ProcessAt returns an option to specify when to process the given task. // // If there's a conflicting ProcessIn option, the last option passed to Enqueue overrides the others. func ProcessAt(t time.Time) Option { return processAtOption(t) } func (t processAtOption) String() string { return fmt.Sprintf("ProcessAt(%v)", time.Time(t).Format(time.UnixDate)) } func (t processAtOption) Type() OptionType { return ProcessAtOpt } func (t processAtOption) Value() interface{} { return time.Time(t) } // ProcessIn returns an option to specify when to process the given task relative to the current time. // // If there's a conflicting ProcessAt option, the last option passed to Enqueue overrides the others. func ProcessIn(d time.Duration) Option { return processInOption(d) } func (d processInOption) String() string { return fmt.Sprintf("ProcessIn(%v)", time.Duration(d)) } func (d processInOption) Type() OptionType { return ProcessInOpt } func (d processInOption) Value() interface{} { return time.Duration(d) } // Retention returns an option to specify the duration of retention period for the task. // If this option is provided, the task will be stored as a completed task after successful processing. // A completed task will be deleted after the specified duration elapses. func Retention(d time.Duration) Option { return retentionOption(d) } func (ttl retentionOption) String() string { return fmt.Sprintf("Retention(%v)", time.Duration(ttl)) } func (ttl retentionOption) Type() OptionType { return RetentionOpt } func (ttl retentionOption) Value() interface{} { return time.Duration(ttl) } // Group returns an option to specify the group used for the task. // Tasks in a given queue with the same group will be aggregated into one task before passed to Handler. func Group(name string) Option { return groupOption(name) } func (name groupOption) String() string { return fmt.Sprintf("Group(%q)", string(name)) } func (name groupOption) Type() OptionType { return GroupOpt } func (name groupOption) Value() interface{} { return string(name) } // ErrDuplicateTask indicates that the given task could not be enqueued since it's a duplicate of another task. // // ErrDuplicateTask error only applies to tasks enqueued with a Unique option. var ErrDuplicateTask = errors.New("task already exists") // ErrTaskIDConflict indicates that the given task could not be enqueued since its task ID already exists. // // ErrTaskIDConflict error only applies to tasks enqueued with a TaskID option. var ErrTaskIDConflict = errors.New("task ID conflicts with another task") type option struct { retry int queue string taskID string timeout time.Duration deadline time.Time uniqueTTL time.Duration processAt time.Time retention time.Duration group string } // composeOptions merges user provided options into the default options // and returns the composed option. // It also validates the user provided options and returns an error if any of // the user provided options fail the validations. func composeOptions(opts ...Option) (option, error) { res := option{ retry: defaultMaxRetry, queue: base.DefaultQueueName, taskID: uuid.NewString(), timeout: 0, // do not set to defaultTimeout here deadline: time.Time{}, processAt: time.Now(), } for _, opt := range opts { switch opt := opt.(type) { case retryOption: res.retry = int(opt) case queueOption: qname := string(opt) if err := base.ValidateQueueName(qname); err != nil { return option{}, err } res.queue = qname case taskIDOption: id := string(opt) if isBlank(id) { return option{}, errors.New("task ID cannot be empty") } res.taskID = id case timeoutOption: res.timeout = time.Duration(opt) case deadlineOption: res.deadline = time.Time(opt) case uniqueOption: ttl := time.Duration(opt) if ttl < 1*time.Second { return option{}, errors.New("Unique TTL cannot be less than 1s") } res.uniqueTTL = ttl case processAtOption: res.processAt = time.Time(opt) case processInOption: res.processAt = time.Now().Add(time.Duration(opt)) case retentionOption: res.retention = time.Duration(opt) case groupOption: key := string(opt) if isBlank(key) { return option{}, errors.New("group key cannot be empty") } res.group = key default: // ignore unexpected option } } return res, nil } // isBlank returns true if the given s is empty or consist of all whitespaces. func isBlank(s string) bool { return strings.TrimSpace(s) == "" } const ( // Default max retry count used if nothing is specified. defaultMaxRetry = 25 // Default timeout used if both timeout and deadline are not specified. defaultTimeout = 30 * time.Minute ) // Value zero indicates no timeout and no deadline. var ( noTimeout time.Duration = 0 noDeadline time.Time = time.Unix(0, 0) ) // Close closes the connection with redis. func (c *Client) Close() error { if c.sharedConnection { return fmt.Errorf("redis connection is shared so the Client can't be closed through asynq") } return c.broker.Close() } // Enqueue enqueues the given task to a queue. // // Enqueue returns TaskInfo and nil error if the task is enqueued successfully, otherwise returns a non-nil error. // // The argument opts specifies the behavior of task processing. // If there are conflicting Option values the last one overrides others. // Any options provided to NewTask can be overridden by options passed to Enqueue. // By default, max retry is set to 25 and timeout is set to 30 minutes. // // If no ProcessAt or ProcessIn options are provided, the task will be pending immediately. // // Enqueue uses context.Background internally; to specify the context, use EnqueueContext. func (c *Client) Enqueue(task *Task, opts ...Option) (*TaskInfo, error) { return c.EnqueueContext(context.Background(), task, opts...) } // EnqueueContext enqueues the given task to a queue. // // EnqueueContext returns TaskInfo and nil error if the task is enqueued successfully, otherwise returns a non-nil error. // // The argument opts specifies the behavior of task processing. // If there are conflicting Option values the last one overrides others. // Any options provided to NewTask can be overridden by options passed to Enqueue. // By default, max retry is set to 25 and timeout is set to 30 minutes. // // If no ProcessAt or ProcessIn options are provided, the task will be pending immediately. // // The first argument context applies to the enqueue operation. To specify task timeout and deadline, use Timeout and Deadline option instead. func (c *Client) EnqueueContext(ctx context.Context, task *Task, opts ...Option) (*TaskInfo, error) { if task == nil { return nil, fmt.Errorf("task cannot be nil") } if strings.TrimSpace(task.Type()) == "" { return nil, fmt.Errorf("task typename cannot be empty") } // merge task options with the options provided at enqueue time. opts = append(task.opts, opts...) opt, err := composeOptions(opts...) if err != nil { return nil, err } deadline := noDeadline if !opt.deadline.IsZero() { deadline = opt.deadline } timeout := noTimeout if opt.timeout != 0 { timeout = opt.timeout } if deadline.Equal(noDeadline) && timeout == noTimeout { // If neither deadline nor timeout are set, use default timeout. timeout = defaultTimeout } var uniqueKey string if opt.uniqueTTL > 0 { uniqueKey = base.UniqueKey(opt.queue, task.Type(), task.Payload()) } msg := &base.TaskMessage{ ID: opt.taskID, Type: task.Type(), Payload: task.Payload(), Headers: task.Headers(), Queue: opt.queue, Retry: opt.retry, Deadline: deadline.Unix(), Timeout: int64(timeout.Seconds()), UniqueKey: uniqueKey, GroupKey: opt.group, Retention: int64(opt.retention.Seconds()), } now := time.Now() var state base.TaskState if opt.processAt.After(now) { err = c.schedule(ctx, msg, opt.processAt, opt.uniqueTTL) state = base.TaskStateScheduled } else if opt.group != "" { // Use zero value for processAt since we don't know when the task will be aggregated and processed. opt.processAt = time.Time{} err = c.addToGroup(ctx, msg, opt.group, opt.uniqueTTL) state = base.TaskStateAggregating } else { opt.processAt = now err = c.enqueue(ctx, msg, opt.uniqueTTL) state = base.TaskStatePending } switch { case errors.Is(err, errors.ErrDuplicateTask): return nil, fmt.Errorf("%w", ErrDuplicateTask) case errors.Is(err, errors.ErrTaskIdConflict): return nil, fmt.Errorf("%w", ErrTaskIDConflict) case err != nil: return nil, err } return newTaskInfo(msg, state, opt.processAt, nil), nil } // Ping performs a ping against the redis connection. func (c *Client) Ping() error { return c.broker.Ping() } func (c *Client) enqueue(ctx context.Context, msg *base.TaskMessage, uniqueTTL time.Duration) error { if uniqueTTL > 0 { return c.broker.EnqueueUnique(ctx, msg, uniqueTTL) } return c.broker.Enqueue(ctx, msg) } func (c *Client) schedule(ctx context.Context, msg *base.TaskMessage, t time.Time, uniqueTTL time.Duration) error { if uniqueTTL > 0 { ttl := time.Until(t.Add(uniqueTTL)) return c.broker.ScheduleUnique(ctx, msg, t, ttl) } return c.broker.Schedule(ctx, msg, t) } func (c *Client) addToGroup(ctx context.Context, msg *base.TaskMessage, group string, uniqueTTL time.Duration) error { if uniqueTTL > 0 { return c.broker.AddToGroupUnique(ctx, msg, group, uniqueTTL) } return c.broker.AddToGroup(ctx, msg, group) } ================================================ FILE: client_test.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "context" "errors" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/hibiken/asynq/internal/base" h "github.com/hibiken/asynq/internal/testutil" "github.com/redis/go-redis/v9" ) func TestClientEnqueueWithProcessAtOption(t *testing.T) { r := setup(t) client := NewClient(getRedisConnOpt(t)) defer client.Close() task := NewTask("send_email", h.JSON(map[string]interface{}{"to": "customer@gmail.com", "from": "merchant@example.com"})) var ( now = time.Now() oneHourLater = now.Add(time.Hour) ) tests := []struct { desc string task *Task processAt time.Time // value for ProcessAt option opts []Option // other options wantInfo *TaskInfo wantPending map[string][]*base.TaskMessage wantScheduled map[string][]base.Z }{ { desc: "Process task immediately", task: task, processAt: now, opts: []Option{}, wantInfo: &TaskInfo{ Queue: "default", Type: task.Type(), Payload: task.Payload(), State: TaskStatePending, MaxRetry: defaultMaxRetry, Retried: 0, LastErr: "", LastFailedAt: time.Time{}, Timeout: defaultTimeout, Deadline: time.Time{}, NextProcessAt: now, }, wantPending: map[string][]*base.TaskMessage{ "default": { { Type: task.Type(), Payload: task.Payload(), Retry: defaultMaxRetry, Queue: "default", Timeout: int64(defaultTimeout.Seconds()), Deadline: noDeadline.Unix(), }, }, }, wantScheduled: map[string][]base.Z{ "default": {}, }, }, { desc: "Schedule task to be processed in the future", task: task, processAt: oneHourLater, opts: []Option{}, wantInfo: &TaskInfo{ Queue: "default", Type: task.Type(), Payload: task.Payload(), State: TaskStateScheduled, MaxRetry: defaultMaxRetry, Retried: 0, LastErr: "", LastFailedAt: time.Time{}, Timeout: defaultTimeout, Deadline: time.Time{}, NextProcessAt: oneHourLater, }, wantPending: map[string][]*base.TaskMessage{ "default": {}, }, wantScheduled: map[string][]base.Z{ "default": { { Message: &base.TaskMessage{ Type: task.Type(), Payload: task.Payload(), Retry: defaultMaxRetry, Queue: "default", Timeout: int64(defaultTimeout.Seconds()), Deadline: noDeadline.Unix(), }, Score: oneHourLater.Unix(), }, }, }, }, } for _, tc := range tests { h.FlushDB(t, r) // clean up db before each test case. opts := append(tc.opts, ProcessAt(tc.processAt)) gotInfo, err := client.Enqueue(tc.task, opts...) if err != nil { t.Error(err) continue } cmpOptions := []cmp.Option{ cmpopts.IgnoreFields(TaskInfo{}, "ID"), cmpopts.EquateApproxTime(500 * time.Millisecond), } if diff := cmp.Diff(tc.wantInfo, gotInfo, cmpOptions...); diff != "" { t.Errorf("%s;\nEnqueue(task, ProcessAt(%v)) returned %v, want %v; (-want,+got)\n%s", tc.desc, tc.processAt, gotInfo, tc.wantInfo, diff) } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r, qname) if diff := cmp.Diff(want, gotPending, h.IgnoreIDOpt, cmpopts.EquateEmpty()); diff != "" { t.Errorf("%s;\nmismatch found in %q; (-want,+got)\n%s", tc.desc, base.PendingKey(qname), diff) } } for qname, want := range tc.wantScheduled { gotScheduled := h.GetScheduledEntries(t, r, qname) if diff := cmp.Diff(want, gotScheduled, h.IgnoreIDOpt, cmpopts.EquateEmpty()); diff != "" { t.Errorf("%s;\nmismatch found in %q; (-want,+got)\n%s", tc.desc, base.ScheduledKey(qname), diff) } } } } func testClientEnqueue(t *testing.T, client *Client, r redis.UniversalClient) { task := NewTask("send_email", h.JSON(map[string]interface{}{"to": "customer@gmail.com", "from": "merchant@example.com"})) now := time.Now() tests := []struct { desc string task *Task opts []Option wantInfo *TaskInfo wantPending map[string][]*base.TaskMessage }{ { desc: "Process task immediately with a custom retry count", task: task, opts: []Option{ MaxRetry(3), }, wantInfo: &TaskInfo{ Queue: "default", Type: task.Type(), Payload: task.Payload(), State: TaskStatePending, MaxRetry: 3, Retried: 0, LastErr: "", LastFailedAt: time.Time{}, Timeout: defaultTimeout, Deadline: time.Time{}, NextProcessAt: now, }, wantPending: map[string][]*base.TaskMessage{ "default": { { Type: task.Type(), Payload: task.Payload(), Retry: 3, Queue: "default", Timeout: int64(defaultTimeout.Seconds()), Deadline: noDeadline.Unix(), }, }, }, }, { desc: "Negative retry count", task: task, opts: []Option{ MaxRetry(-2), }, wantInfo: &TaskInfo{ Queue: "default", Type: task.Type(), Payload: task.Payload(), State: TaskStatePending, MaxRetry: 0, // Retry count should be set to zero Retried: 0, LastErr: "", LastFailedAt: time.Time{}, Timeout: defaultTimeout, Deadline: time.Time{}, NextProcessAt: now, }, wantPending: map[string][]*base.TaskMessage{ "default": { { Type: task.Type(), Payload: task.Payload(), Retry: 0, // Retry count should be set to zero Queue: "default", Timeout: int64(defaultTimeout.Seconds()), Deadline: noDeadline.Unix(), }, }, }, }, { desc: "Conflicting options", task: task, opts: []Option{ MaxRetry(2), MaxRetry(10), }, wantInfo: &TaskInfo{ Queue: "default", Type: task.Type(), Payload: task.Payload(), State: TaskStatePending, MaxRetry: 10, // Last option takes precedence Retried: 0, LastErr: "", LastFailedAt: time.Time{}, Timeout: defaultTimeout, Deadline: time.Time{}, NextProcessAt: now, }, wantPending: map[string][]*base.TaskMessage{ "default": { { Type: task.Type(), Payload: task.Payload(), Retry: 10, // Last option takes precedence Queue: "default", Timeout: int64(defaultTimeout.Seconds()), Deadline: noDeadline.Unix(), }, }, }, }, { desc: "With queue option", task: task, opts: []Option{ Queue("custom"), }, wantInfo: &TaskInfo{ Queue: "custom", Type: task.Type(), Payload: task.Payload(), State: TaskStatePending, MaxRetry: defaultMaxRetry, Retried: 0, LastErr: "", LastFailedAt: time.Time{}, Timeout: defaultTimeout, Deadline: time.Time{}, NextProcessAt: now, }, wantPending: map[string][]*base.TaskMessage{ "custom": { { Type: task.Type(), Payload: task.Payload(), Retry: defaultMaxRetry, Queue: "custom", Timeout: int64(defaultTimeout.Seconds()), Deadline: noDeadline.Unix(), }, }, }, }, { desc: "Queue option should be case sensitive", task: task, opts: []Option{ Queue("MyQueue"), }, wantInfo: &TaskInfo{ Queue: "MyQueue", Type: task.Type(), Payload: task.Payload(), State: TaskStatePending, MaxRetry: defaultMaxRetry, Retried: 0, LastErr: "", LastFailedAt: time.Time{}, Timeout: defaultTimeout, Deadline: time.Time{}, NextProcessAt: now, }, wantPending: map[string][]*base.TaskMessage{ "MyQueue": { { Type: task.Type(), Payload: task.Payload(), Retry: defaultMaxRetry, Queue: "MyQueue", Timeout: int64(defaultTimeout.Seconds()), Deadline: noDeadline.Unix(), }, }, }, }, { desc: "With timeout option", task: task, opts: []Option{ Timeout(20 * time.Second), }, wantInfo: &TaskInfo{ Queue: "default", Type: task.Type(), Payload: task.Payload(), State: TaskStatePending, MaxRetry: defaultMaxRetry, Retried: 0, LastErr: "", LastFailedAt: time.Time{}, Timeout: 20 * time.Second, Deadline: time.Time{}, NextProcessAt: now, }, wantPending: map[string][]*base.TaskMessage{ "default": { { Type: task.Type(), Payload: task.Payload(), Retry: defaultMaxRetry, Queue: "default", Timeout: 20, Deadline: noDeadline.Unix(), }, }, }, }, { desc: "With deadline option", task: task, opts: []Option{ Deadline(time.Date(2020, time.June, 24, 0, 0, 0, 0, time.UTC)), }, wantInfo: &TaskInfo{ Queue: "default", Type: task.Type(), Payload: task.Payload(), State: TaskStatePending, MaxRetry: defaultMaxRetry, Retried: 0, LastErr: "", LastFailedAt: time.Time{}, Timeout: noTimeout, Deadline: time.Date(2020, time.June, 24, 0, 0, 0, 0, time.UTC), NextProcessAt: now, }, wantPending: map[string][]*base.TaskMessage{ "default": { { Type: task.Type(), Payload: task.Payload(), Retry: defaultMaxRetry, Queue: "default", Timeout: int64(noTimeout.Seconds()), Deadline: time.Date(2020, time.June, 24, 0, 0, 0, 0, time.UTC).Unix(), }, }, }, }, { desc: "With both deadline and timeout options", task: task, opts: []Option{ Timeout(20 * time.Second), Deadline(time.Date(2020, time.June, 24, 0, 0, 0, 0, time.UTC)), }, wantInfo: &TaskInfo{ Queue: "default", Type: task.Type(), Payload: task.Payload(), State: TaskStatePending, MaxRetry: defaultMaxRetry, Retried: 0, LastErr: "", LastFailedAt: time.Time{}, Timeout: 20 * time.Second, Deadline: time.Date(2020, time.June, 24, 0, 0, 0, 0, time.UTC), NextProcessAt: now, }, wantPending: map[string][]*base.TaskMessage{ "default": { { Type: task.Type(), Payload: task.Payload(), Retry: defaultMaxRetry, Queue: "default", Timeout: 20, Deadline: time.Date(2020, time.June, 24, 0, 0, 0, 0, time.UTC).Unix(), }, }, }, }, { desc: "With Retention option", task: task, opts: []Option{ Retention(24 * time.Hour), }, wantInfo: &TaskInfo{ Queue: "default", Type: task.Type(), Payload: task.Payload(), State: TaskStatePending, MaxRetry: defaultMaxRetry, Retried: 0, LastErr: "", LastFailedAt: time.Time{}, Timeout: defaultTimeout, Deadline: time.Time{}, NextProcessAt: now, Retention: 24 * time.Hour, }, wantPending: map[string][]*base.TaskMessage{ "default": { { Type: task.Type(), Payload: task.Payload(), Retry: defaultMaxRetry, Queue: "default", Timeout: int64(defaultTimeout.Seconds()), Deadline: noDeadline.Unix(), Retention: int64((24 * time.Hour).Seconds()), }, }, }, }, } for _, tc := range tests { h.FlushDB(t, r) // clean up db before each test case. gotInfo, err := client.Enqueue(tc.task, tc.opts...) if err != nil { t.Error(err) continue } cmpOptions := []cmp.Option{ cmpopts.IgnoreFields(TaskInfo{}, "ID"), cmpopts.EquateApproxTime(500 * time.Millisecond), } if diff := cmp.Diff(tc.wantInfo, gotInfo, cmpOptions...); diff != "" { t.Errorf("%s;\nEnqueue(task) returned %v, want %v; (-want,+got)\n%s", tc.desc, gotInfo, tc.wantInfo, diff) } for qname, want := range tc.wantPending { got := h.GetPendingMessages(t, r, qname) if diff := cmp.Diff(want, got, h.IgnoreIDOpt); diff != "" { t.Errorf("%s;\nmismatch found in %q; (-want,+got)\n%s", tc.desc, base.PendingKey(qname), diff) } } } } func TestClientEnqueue(t *testing.T) { r := setup(t) client := NewClient(getRedisConnOpt(t)) defer client.Close() testClientEnqueue(t, client, r) } func TestClientFromRedisClientEnqueue(t *testing.T) { r := setup(t) redisClient := getRedisConnOpt(t).MakeRedisClient().(redis.UniversalClient) client := NewClientFromRedisClient(redisClient) testClientEnqueue(t, client, r) err := client.Close() if err == nil { t.Error("client.Close() should have failed because of a shared client but it didn't") } } func TestClientEnqueueWithGroupOption(t *testing.T) { r := setup(t) client := NewClient(getRedisConnOpt(t)) defer client.Close() task := NewTask("mytask", []byte("foo")) now := time.Now() tests := []struct { desc string task *Task opts []Option wantInfo *TaskInfo wantPending map[string][]*base.TaskMessage wantGroups map[string]map[string][]base.Z // map queue name to a set of groups wantScheduled map[string][]base.Z }{ { desc: "With only Group option", task: task, opts: []Option{ Group("mygroup"), }, wantInfo: &TaskInfo{ Queue: "default", Group: "mygroup", Type: task.Type(), Payload: task.Payload(), State: TaskStateAggregating, MaxRetry: defaultMaxRetry, Retried: 0, LastErr: "", LastFailedAt: time.Time{}, Timeout: defaultTimeout, Deadline: time.Time{}, NextProcessAt: time.Time{}, }, wantPending: map[string][]*base.TaskMessage{ "default": {}, // should not be pending }, wantGroups: map[string]map[string][]base.Z{ "default": { "mygroup": { { Message: &base.TaskMessage{ Type: task.Type(), Payload: task.Payload(), Retry: defaultMaxRetry, Queue: "default", Timeout: int64(defaultTimeout.Seconds()), Deadline: noDeadline.Unix(), GroupKey: "mygroup", }, Score: now.Unix(), }, }, }, }, wantScheduled: map[string][]base.Z{ "default": {}, }, }, { desc: "With Group and ProcessAt options", task: task, opts: []Option{ Group("mygroup"), ProcessAt(now.Add(30 * time.Minute)), }, wantInfo: &TaskInfo{ Queue: "default", Group: "mygroup", Type: task.Type(), Payload: task.Payload(), State: TaskStateScheduled, MaxRetry: defaultMaxRetry, Retried: 0, LastErr: "", LastFailedAt: time.Time{}, Timeout: defaultTimeout, Deadline: time.Time{}, NextProcessAt: now.Add(30 * time.Minute), }, wantPending: map[string][]*base.TaskMessage{ "default": {}, // should not be pending }, wantGroups: map[string]map[string][]base.Z{ "default": { "mygroup": {}, // should not be added to the group yet }, }, wantScheduled: map[string][]base.Z{ "default": { { Message: &base.TaskMessage{ Type: task.Type(), Payload: task.Payload(), Retry: defaultMaxRetry, Queue: "default", Timeout: int64(defaultTimeout.Seconds()), Deadline: noDeadline.Unix(), GroupKey: "mygroup", }, Score: now.Add(30 * time.Minute).Unix(), }, }, }, }, } for _, tc := range tests { h.FlushDB(t, r) // clean up db before each test case. gotInfo, err := client.Enqueue(tc.task, tc.opts...) if err != nil { t.Error(err) continue } cmpOptions := []cmp.Option{ cmpopts.IgnoreFields(TaskInfo{}, "ID"), cmpopts.EquateApproxTime(500 * time.Millisecond), } if diff := cmp.Diff(tc.wantInfo, gotInfo, cmpOptions...); diff != "" { t.Errorf("%s;\nEnqueue(task) returned %v, want %v; (-want,+got)\n%s", tc.desc, gotInfo, tc.wantInfo, diff) } for qname, want := range tc.wantPending { got := h.GetPendingMessages(t, r, qname) if diff := cmp.Diff(want, got, h.IgnoreIDOpt, cmpopts.EquateEmpty()); diff != "" { t.Errorf("%s;\nmismatch found in %q; (-want,+got)\n%s", tc.desc, base.PendingKey(qname), diff) } } for qname, groups := range tc.wantGroups { for groupKey, want := range groups { got := h.GetGroupEntries(t, r, qname, groupKey) if diff := cmp.Diff(want, got, h.IgnoreIDOpt, cmpopts.EquateEmpty()); diff != "" { t.Errorf("%s;\nmismatch found in %q; (-want,+got)\n%s", tc.desc, base.GroupKey(qname, groupKey), diff) } } } for qname, want := range tc.wantScheduled { gotScheduled := h.GetScheduledEntries(t, r, qname) if diff := cmp.Diff(want, gotScheduled, h.IgnoreIDOpt, cmpopts.EquateEmpty()); diff != "" { t.Errorf("%s;\nmismatch found in %q; (-want,+got)\n%s", tc.desc, base.ScheduledKey(qname), diff) } } } } func TestClientEnqueueWithTaskIDOption(t *testing.T) { r := setup(t) client := NewClient(getRedisConnOpt(t)) defer client.Close() task := NewTask("send_email", nil) now := time.Now() tests := []struct { desc string task *Task opts []Option wantInfo *TaskInfo wantPending map[string][]*base.TaskMessage }{ { desc: "With a valid TaskID option", task: task, opts: []Option{ TaskID("custom_id"), }, wantInfo: &TaskInfo{ ID: "custom_id", Queue: "default", Type: task.Type(), Payload: task.Payload(), State: TaskStatePending, MaxRetry: defaultMaxRetry, Retried: 0, LastErr: "", LastFailedAt: time.Time{}, Timeout: defaultTimeout, Deadline: time.Time{}, NextProcessAt: now, }, wantPending: map[string][]*base.TaskMessage{ "default": { { ID: "custom_id", Type: task.Type(), Payload: task.Payload(), Retry: defaultMaxRetry, Queue: "default", Timeout: int64(defaultTimeout.Seconds()), Deadline: noDeadline.Unix(), }, }, }, }, } for _, tc := range tests { h.FlushDB(t, r) // clean up db before each test case. gotInfo, err := client.Enqueue(tc.task, tc.opts...) if err != nil { t.Errorf("got non-nil error %v, want nil", err) continue } cmpOptions := []cmp.Option{ cmpopts.EquateApproxTime(500 * time.Millisecond), } if diff := cmp.Diff(tc.wantInfo, gotInfo, cmpOptions...); diff != "" { t.Errorf("%s;\nEnqueue(task) returned %v, want %v; (-want,+got)\n%s", tc.desc, gotInfo, tc.wantInfo, diff) } for qname, want := range tc.wantPending { got := h.GetPendingMessages(t, r, qname) if diff := cmp.Diff(want, got); diff != "" { t.Errorf("%s;\nmismatch found in %q; (-want,+got)\n%s", tc.desc, base.PendingKey(qname), diff) } } } } func TestClientEnqueueWithConflictingTaskID(t *testing.T) { setup(t) client := NewClient(getRedisConnOpt(t)) defer client.Close() const taskID = "custom_id" task := NewTask("foo", nil) if _, err := client.Enqueue(task, TaskID(taskID)); err != nil { t.Fatalf("First task: Enqueue failed: %v", err) } _, err := client.Enqueue(task, TaskID(taskID)) if !errors.Is(err, ErrTaskIDConflict) { t.Errorf("Second task: Enqueue returned %v, want %v", err, ErrTaskIDConflict) } } func TestClientEnqueueWithProcessInOption(t *testing.T) { r := setup(t) client := NewClient(getRedisConnOpt(t)) defer client.Close() task := NewTask("send_email", h.JSON(map[string]interface{}{"to": "customer@gmail.com", "from": "merchant@example.com"})) now := time.Now() tests := []struct { desc string task *Task delay time.Duration // value for ProcessIn option opts []Option // other options wantInfo *TaskInfo wantPending map[string][]*base.TaskMessage wantScheduled map[string][]base.Z }{ { desc: "schedule a task to be processed in one hour", task: task, delay: 1 * time.Hour, opts: []Option{}, wantInfo: &TaskInfo{ Queue: "default", Type: task.Type(), Payload: task.Payload(), State: TaskStateScheduled, MaxRetry: defaultMaxRetry, Retried: 0, LastErr: "", LastFailedAt: time.Time{}, Timeout: defaultTimeout, Deadline: time.Time{}, NextProcessAt: time.Now().Add(1 * time.Hour), }, wantPending: map[string][]*base.TaskMessage{ "default": {}, }, wantScheduled: map[string][]base.Z{ "default": { { Message: &base.TaskMessage{ Type: task.Type(), Payload: task.Payload(), Retry: defaultMaxRetry, Queue: "default", Timeout: int64(defaultTimeout.Seconds()), Deadline: noDeadline.Unix(), }, Score: time.Now().Add(time.Hour).Unix(), }, }, }, }, { desc: "Zero delay", task: task, delay: 0, opts: []Option{}, wantInfo: &TaskInfo{ Queue: "default", Type: task.Type(), Payload: task.Payload(), State: TaskStatePending, MaxRetry: defaultMaxRetry, Retried: 0, LastErr: "", LastFailedAt: time.Time{}, Timeout: defaultTimeout, Deadline: time.Time{}, NextProcessAt: now, }, wantPending: map[string][]*base.TaskMessage{ "default": { { Type: task.Type(), Payload: task.Payload(), Retry: defaultMaxRetry, Queue: "default", Timeout: int64(defaultTimeout.Seconds()), Deadline: noDeadline.Unix(), }, }, }, wantScheduled: map[string][]base.Z{ "default": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r) // clean up db before each test case. opts := append(tc.opts, ProcessIn(tc.delay)) gotInfo, err := client.Enqueue(tc.task, opts...) if err != nil { t.Error(err) continue } cmpOptions := []cmp.Option{ cmpopts.IgnoreFields(TaskInfo{}, "ID"), cmpopts.EquateApproxTime(500 * time.Millisecond), } if diff := cmp.Diff(tc.wantInfo, gotInfo, cmpOptions...); diff != "" { t.Errorf("%s;\nEnqueue(task, ProcessIn(%v)) returned %v, want %v; (-want,+got)\n%s", tc.desc, tc.delay, gotInfo, tc.wantInfo, diff) } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r, qname) if diff := cmp.Diff(want, gotPending, h.IgnoreIDOpt, cmpopts.EquateEmpty()); diff != "" { t.Errorf("%s;\nmismatch found in %q; (-want,+got)\n%s", tc.desc, base.PendingKey(qname), diff) } } for qname, want := range tc.wantScheduled { gotScheduled := h.GetScheduledEntries(t, r, qname) if diff := cmp.Diff(want, gotScheduled, h.IgnoreIDOpt, cmpopts.EquateEmpty()); diff != "" { t.Errorf("%s;\nmismatch found in %q; (-want,+got)\n%s", tc.desc, base.ScheduledKey(qname), diff) } } } } func TestClientEnqueueError(t *testing.T) { r := setup(t) client := NewClient(getRedisConnOpt(t)) defer client.Close() task := NewTask("send_email", h.JSON(map[string]interface{}{"to": "customer@gmail.com", "from": "merchant@example.com"})) tests := []struct { desc string task *Task opts []Option }{ { desc: "With nil task", task: nil, opts: []Option{}, }, { desc: "With empty queue name", task: task, opts: []Option{ Queue(""), }, }, { desc: "With empty task typename", task: NewTask("", h.JSON(map[string]interface{}{})), opts: []Option{}, }, { desc: "With blank task typename", task: NewTask(" ", h.JSON(map[string]interface{}{})), opts: []Option{}, }, { desc: "With empty task ID", task: NewTask("foo", nil), opts: []Option{TaskID("")}, }, { desc: "With blank task ID", task: NewTask("foo", nil), opts: []Option{TaskID(" ")}, }, { desc: "With unique option less than 1s", task: NewTask("foo", nil), opts: []Option{Unique(300 * time.Millisecond)}, }, } for _, tc := range tests { h.FlushDB(t, r) _, err := client.Enqueue(tc.task, tc.opts...) if err == nil { t.Errorf("%s; client.Enqueue(task, opts...) did not return non-nil error", tc.desc) } } } func TestClientWithDefaultOptions(t *testing.T) { r := setup(t) now := time.Now() tests := []struct { desc string defaultOpts []Option // options set at task initialization time opts []Option // options used at enqueue time. tasktype string payload []byte wantInfo *TaskInfo queue string // queue that the message should go into. want *base.TaskMessage }{ { desc: "With queue routing option", defaultOpts: []Option{Queue("feed")}, opts: []Option{}, tasktype: "feed:import", payload: nil, wantInfo: &TaskInfo{ Queue: "feed", Type: "feed:import", Payload: nil, State: TaskStatePending, MaxRetry: defaultMaxRetry, Retried: 0, LastErr: "", LastFailedAt: time.Time{}, Timeout: defaultTimeout, Deadline: time.Time{}, NextProcessAt: now, }, queue: "feed", want: &base.TaskMessage{ Type: "feed:import", Payload: nil, Retry: defaultMaxRetry, Queue: "feed", Timeout: int64(defaultTimeout.Seconds()), Deadline: noDeadline.Unix(), }, }, { desc: "With multiple options", defaultOpts: []Option{Queue("feed"), MaxRetry(5)}, opts: []Option{}, tasktype: "feed:import", payload: nil, wantInfo: &TaskInfo{ Queue: "feed", Type: "feed:import", Payload: nil, State: TaskStatePending, MaxRetry: 5, Retried: 0, LastErr: "", LastFailedAt: time.Time{}, Timeout: defaultTimeout, Deadline: time.Time{}, NextProcessAt: now, }, queue: "feed", want: &base.TaskMessage{ Type: "feed:import", Payload: nil, Retry: 5, Queue: "feed", Timeout: int64(defaultTimeout.Seconds()), Deadline: noDeadline.Unix(), }, }, { desc: "With overriding options at enqueue time", defaultOpts: []Option{Queue("feed"), MaxRetry(5)}, opts: []Option{Queue("critical")}, tasktype: "feed:import", payload: nil, wantInfo: &TaskInfo{ Queue: "critical", Type: "feed:import", Payload: nil, State: TaskStatePending, MaxRetry: 5, LastErr: "", LastFailedAt: time.Time{}, Timeout: defaultTimeout, Deadline: time.Time{}, NextProcessAt: now, }, queue: "critical", want: &base.TaskMessage{ Type: "feed:import", Payload: nil, Retry: 5, Queue: "critical", Timeout: int64(defaultTimeout.Seconds()), Deadline: noDeadline.Unix(), }, }, } for _, tc := range tests { h.FlushDB(t, r) c := NewClient(getRedisConnOpt(t)) defer c.Close() task := NewTask(tc.tasktype, tc.payload, tc.defaultOpts...) gotInfo, err := c.Enqueue(task, tc.opts...) if err != nil { t.Fatal(err) } cmpOptions := []cmp.Option{ cmpopts.IgnoreFields(TaskInfo{}, "ID"), cmpopts.EquateApproxTime(500 * time.Millisecond), } if diff := cmp.Diff(tc.wantInfo, gotInfo, cmpOptions...); diff != "" { t.Errorf("%s;\nEnqueue(task, opts...) returned %v, want %v; (-want,+got)\n%s", tc.desc, gotInfo, tc.wantInfo, diff) } pending := h.GetPendingMessages(t, r, tc.queue) if len(pending) != 1 { t.Errorf("%s;\nexpected queue %q to have one message; got %d messages in the queue.", tc.desc, tc.queue, len(pending)) continue } got := pending[0] if diff := cmp.Diff(tc.want, got, h.IgnoreIDOpt); diff != "" { t.Errorf("%s;\nmismatch found in pending task message; (-want,+got)\n%s", tc.desc, diff) } } } func TestClientEnqueueUnique(t *testing.T) { r := setup(t) c := NewClient(getRedisConnOpt(t)) defer c.Close() tests := []struct { task *Task ttl time.Duration }{ { NewTask("email", h.JSON(map[string]interface{}{"user_id": 123})), time.Hour, }, } for _, tc := range tests { h.FlushDB(t, r) // clean up db before each test case. // Enqueue the task first. It should succeed. _, err := c.Enqueue(tc.task, Unique(tc.ttl)) if err != nil { t.Fatal(err) } gotTTL := r.TTL(context.Background(), base.UniqueKey(base.DefaultQueueName, tc.task.Type(), tc.task.Payload())).Val() if !cmp.Equal(tc.ttl.Seconds(), gotTTL.Seconds(), cmpopts.EquateApprox(0, 1)) { t.Errorf("TTL = %v, want %v", gotTTL, tc.ttl) continue } // Enqueue the task again. It should fail. _, err = c.Enqueue(tc.task, Unique(tc.ttl)) if err == nil { t.Errorf("Enqueueing %+v did not return an error", tc.task) continue } if !errors.Is(err, ErrDuplicateTask) { t.Errorf("Enqueueing %+v returned an error that is not ErrDuplicateTask", tc.task) continue } } } func TestClientEnqueueUniqueWithProcessInOption(t *testing.T) { r := setup(t) c := NewClient(getRedisConnOpt(t)) defer c.Close() tests := []struct { task *Task d time.Duration ttl time.Duration }{ { NewTask("reindex", nil), time.Hour, 10 * time.Minute, }, } for _, tc := range tests { h.FlushDB(t, r) // clean up db before each test case. // Enqueue the task first. It should succeed. _, err := c.Enqueue(tc.task, ProcessIn(tc.d), Unique(tc.ttl)) if err != nil { t.Fatal(err) } gotTTL := r.TTL(context.Background(), base.UniqueKey(base.DefaultQueueName, tc.task.Type(), tc.task.Payload())).Val() wantTTL := time.Duration(tc.ttl.Seconds()+tc.d.Seconds()) * time.Second if !cmp.Equal(wantTTL.Seconds(), gotTTL.Seconds(), cmpopts.EquateApprox(0, 1)) { t.Errorf("TTL = %v, want %v", gotTTL, wantTTL) continue } // Enqueue the task again. It should fail. _, err = c.Enqueue(tc.task, ProcessIn(tc.d), Unique(tc.ttl)) if err == nil { t.Errorf("Enqueueing %+v did not return an error", tc.task) continue } if !errors.Is(err, ErrDuplicateTask) { t.Errorf("Enqueueing %+v returned an error that is not ErrDuplicateTask", tc.task) continue } } } func TestClientEnqueueUniqueWithProcessAtOption(t *testing.T) { r := setup(t) c := NewClient(getRedisConnOpt(t)) defer c.Close() tests := []struct { task *Task at time.Time ttl time.Duration }{ { NewTask("reindex", nil), time.Now().Add(time.Hour), 10 * time.Minute, }, } for _, tc := range tests { h.FlushDB(t, r) // clean up db before each test case. // Enqueue the task first. It should succeed. _, err := c.Enqueue(tc.task, ProcessAt(tc.at), Unique(tc.ttl)) if err != nil { t.Fatal(err) } gotTTL := r.TTL(context.Background(), base.UniqueKey(base.DefaultQueueName, tc.task.Type(), tc.task.Payload())).Val() wantTTL := time.Until(tc.at.Add(tc.ttl)) if !cmp.Equal(wantTTL.Seconds(), gotTTL.Seconds(), cmpopts.EquateApprox(0, 1)) { t.Errorf("TTL = %v, want %v", gotTTL, wantTTL) continue } // Enqueue the task again. It should fail. _, err = c.Enqueue(tc.task, ProcessAt(tc.at), Unique(tc.ttl)) if err == nil { t.Errorf("Enqueueing %+v did not return an error", tc.task) continue } if !errors.Is(err, ErrDuplicateTask) { t.Errorf("Enqueueing %+v returned an error that is not ErrDuplicateTask", tc.task) continue } } } func TestClientEnqueueWithHeaders(t *testing.T) { r := setup(t) client := NewClient(getRedisConnOpt(t)) defer client.Close() now := time.Now() headers := map[string]string{ "user-id": "123", "request-id": "abc-def-ghi", "priority": "high", } tests := []struct { desc string task *Task opts []Option wantInfo *TaskInfo wantPending map[string][]*base.TaskMessage }{ { desc: "Task with headers", task: NewTaskWithHeaders("send_email", h.JSON(map[string]interface{}{"to": "user@example.com"}), headers), opts: []Option{}, wantInfo: &TaskInfo{ Queue: "default", Type: "send_email", Payload: h.JSON(map[string]interface{}{"to": "user@example.com"}), Headers: headers, State: TaskStatePending, MaxRetry: defaultMaxRetry, Retried: 0, LastErr: "", LastFailedAt: time.Time{}, Timeout: defaultTimeout, Deadline: time.Time{}, NextProcessAt: now, }, wantPending: map[string][]*base.TaskMessage{ "default": { { Type: "send_email", Payload: h.JSON(map[string]interface{}{"to": "user@example.com"}), Headers: headers, Retry: defaultMaxRetry, Queue: "default", Timeout: int64(defaultTimeout.Seconds()), Deadline: noDeadline.Unix(), }, }, }, }, { desc: "Task with empty headers", task: NewTaskWithHeaders("process_data", []byte("data"), map[string]string{}), opts: []Option{}, wantInfo: &TaskInfo{ Queue: "default", Type: "process_data", Payload: []byte("data"), Headers: map[string]string{}, State: TaskStatePending, MaxRetry: defaultMaxRetry, Retried: 0, LastErr: "", LastFailedAt: time.Time{}, Timeout: defaultTimeout, Deadline: time.Time{}, NextProcessAt: now, }, wantPending: map[string][]*base.TaskMessage{ "default": { { Type: "process_data", Payload: []byte("data"), Headers: nil, Retry: defaultMaxRetry, Queue: "default", Timeout: int64(defaultTimeout.Seconds()), Deadline: noDeadline.Unix(), }, }, }, }, { desc: "Task with nil headers", task: NewTaskWithHeaders("cleanup", nil, nil), opts: []Option{}, wantInfo: &TaskInfo{ Queue: "default", Type: "cleanup", Payload: nil, Headers: nil, State: TaskStatePending, MaxRetry: defaultMaxRetry, Retried: 0, LastErr: "", LastFailedAt: time.Time{}, Timeout: defaultTimeout, Deadline: time.Time{}, NextProcessAt: now, }, wantPending: map[string][]*base.TaskMessage{ "default": { { Type: "cleanup", Payload: nil, Headers: nil, Retry: defaultMaxRetry, Queue: "default", Timeout: int64(defaultTimeout.Seconds()), Deadline: noDeadline.Unix(), }, }, }, }, { desc: "Task with headers and custom options", task: NewTaskWithHeaders("notify", []byte("notification"), map[string]string{"channel": "email"}), opts: []Option{MaxRetry(5), Queue("notifications")}, wantInfo: &TaskInfo{ Queue: "notifications", Type: "notify", Payload: []byte("notification"), Headers: map[string]string{"channel": "email"}, State: TaskStatePending, MaxRetry: 5, Retried: 0, LastErr: "", LastFailedAt: time.Time{}, Timeout: defaultTimeout, Deadline: time.Time{}, NextProcessAt: now, }, wantPending: map[string][]*base.TaskMessage{ "notifications": { { Type: "notify", Payload: []byte("notification"), Headers: map[string]string{"channel": "email"}, Retry: 5, Queue: "notifications", Timeout: int64(defaultTimeout.Seconds()), Deadline: noDeadline.Unix(), }, }, }, }, } for _, tc := range tests { h.FlushDB(t, r) gotInfo, err := client.Enqueue(tc.task, tc.opts...) if err != nil { t.Errorf("%s: Enqueue failed: %v", tc.desc, err) continue } cmpOptions := []cmp.Option{ cmpopts.IgnoreFields(TaskInfo{}, "ID"), cmpopts.EquateApproxTime(500 * time.Millisecond), } if diff := cmp.Diff(tc.wantInfo, gotInfo, cmpOptions...); diff != "" { t.Errorf("%s;\nEnqueue(task) returned %v, want %v; (-want,+got)\n%s", tc.desc, gotInfo, tc.wantInfo, diff) } for qname, want := range tc.wantPending { got := h.GetPendingMessages(t, r, qname) if diff := cmp.Diff(want, got, h.IgnoreIDOpt); diff != "" { t.Errorf("%s;\nmismatch found in %q; (-want,+got)\n%s", tc.desc, base.PendingKey(qname), diff) } } } } func TestClientEnqueueWithHeadersScheduled(t *testing.T) { r := setup(t) client := NewClient(getRedisConnOpt(t)) defer client.Close() now := time.Now() oneHourLater := now.Add(time.Hour) headers := map[string]string{ "correlation-id": "xyz-123", "source": "api", } tests := []struct { desc string task *Task processAt time.Time opts []Option wantInfo *TaskInfo wantScheduled map[string][]base.Z }{ { desc: "Schedule task with headers", task: NewTaskWithHeaders("scheduled_task", []byte("payload"), headers), processAt: oneHourLater, opts: []Option{}, wantInfo: &TaskInfo{ Queue: "default", Type: "scheduled_task", Payload: []byte("payload"), Headers: headers, State: TaskStateScheduled, MaxRetry: defaultMaxRetry, Retried: 0, LastErr: "", LastFailedAt: time.Time{}, Timeout: defaultTimeout, Deadline: time.Time{}, NextProcessAt: oneHourLater, }, wantScheduled: map[string][]base.Z{ "default": { { Message: &base.TaskMessage{ Type: "scheduled_task", Payload: []byte("payload"), Headers: headers, Retry: defaultMaxRetry, Queue: "default", Timeout: int64(defaultTimeout.Seconds()), Deadline: noDeadline.Unix(), }, Score: oneHourLater.Unix(), }, }, }, }, } for _, tc := range tests { h.FlushDB(t, r) opts := append(tc.opts, ProcessAt(tc.processAt)) gotInfo, err := client.Enqueue(tc.task, opts...) if err != nil { t.Errorf("%s: Enqueue failed: %v", tc.desc, err) continue } cmpOptions := []cmp.Option{ cmpopts.IgnoreFields(TaskInfo{}, "ID"), cmpopts.EquateApproxTime(500 * time.Millisecond), } if diff := cmp.Diff(tc.wantInfo, gotInfo, cmpOptions...); diff != "" { t.Errorf("%s;\nEnqueue(task, ProcessAt(%v)) returned %v, want %v; (-want,+got)\n%s", tc.desc, tc.processAt, gotInfo, tc.wantInfo, diff) } for qname, want := range tc.wantScheduled { gotScheduled := h.GetScheduledEntries(t, r, qname) if diff := cmp.Diff(want, gotScheduled, h.IgnoreIDOpt, cmpopts.EquateEmpty()); diff != "" { t.Errorf("%s;\nmismatch found in %q; (-want,+got)\n%s", tc.desc, base.ScheduledKey(qname), diff) } } } } func TestNewTaskWithHeaders(t *testing.T) { tests := []struct { desc string typename string payload []byte headers map[string]string opts []Option want *Task }{ { desc: "Task with headers", typename: "test_task", payload: []byte("test payload"), headers: map[string]string{"key1": "value1", "key2": "value2"}, opts: []Option{MaxRetry(3)}, want: &Task{ typename: "test_task", payload: []byte("test payload"), headers: map[string]string{"key1": "value1", "key2": "value2"}, opts: []Option{MaxRetry(3)}, }, }, { desc: "Task with empty headers", typename: "empty_headers", payload: nil, headers: map[string]string{}, opts: nil, want: &Task{ typename: "empty_headers", payload: nil, headers: map[string]string{}, opts: nil, }, }, { desc: "Task with nil headers", typename: "nil_headers", payload: []byte("data"), headers: nil, opts: []Option{Queue("test")}, want: &Task{ typename: "nil_headers", payload: []byte("data"), headers: nil, opts: []Option{Queue("test")}, }, }, } for _, tc := range tests { got := NewTaskWithHeaders(tc.typename, tc.payload, tc.headers, tc.opts...) if got.Type() != tc.want.typename { t.Errorf("%s: Type() = %q, want %q", tc.desc, got.Type(), tc.want.typename) } if diff := cmp.Diff(tc.want.payload, got.Payload()); diff != "" { t.Errorf("%s: Payload() mismatch (-want,+got)\n%s", tc.desc, diff) } if diff := cmp.Diff(tc.want.headers, got.Headers()); diff != "" { t.Errorf("%s: Headers() mismatch (-want,+got)\n%s", tc.desc, diff) } if tc.headers != nil && got.Headers() != nil { tc.headers["modified"] = "test" if _, exists := got.Headers()["modified"]; exists { t.Errorf("%s: Headers should be cloned, but modification affected task headers", tc.desc) } } } } func TestTaskHeadersMethod(t *testing.T) { tests := []struct { desc string task *Task want map[string]string wantNil bool }{ { desc: "Task created with NewTask has nil headers", task: NewTask("test", []byte("data")), want: nil, wantNil: true, }, { desc: "Task created with NewTaskWithHeaders has headers", task: NewTaskWithHeaders("test", []byte("data"), map[string]string{"key": "value"}), want: map[string]string{"key": "value"}, }, { desc: "Task created with empty headers", task: NewTaskWithHeaders("test", []byte("data"), map[string]string{}), want: map[string]string{}, }, { desc: "Task created with nil headers", task: NewTaskWithHeaders("test", []byte("data"), nil), want: nil, wantNil: true, }, } for _, tc := range tests { got := tc.task.Headers() if tc.wantNil { if got != nil { t.Errorf("%s: Headers() = %v, want nil", tc.desc, got) } } else { if diff := cmp.Diff(tc.want, got); diff != "" { t.Errorf("%s: Headers() mismatch (-want,+got)\n%s", tc.desc, diff) } } } } func TestClientEnqueueWithHeadersAndGroup(t *testing.T) { r := setup(t) client := NewClient(getRedisConnOpt(t)) defer client.Close() now := time.Now() headers := map[string]string{ "batch-id": "batch-123", "priority": "high", } tests := []struct { desc string task *Task opts []Option wantInfo *TaskInfo wantGroups map[string]map[string][]base.Z }{ { desc: "Task with headers and group", task: NewTaskWithHeaders("batch_process", []byte("item1"), headers), opts: []Option{Group("batch-123")}, wantInfo: &TaskInfo{ Queue: "default", Group: "batch-123", Type: "batch_process", Payload: []byte("item1"), Headers: headers, State: TaskStateAggregating, MaxRetry: defaultMaxRetry, Retried: 0, LastErr: "", LastFailedAt: time.Time{}, Timeout: defaultTimeout, Deadline: time.Time{}, NextProcessAt: time.Time{}, }, wantGroups: map[string]map[string][]base.Z{ "default": { "batch-123": { { Message: &base.TaskMessage{ Type: "batch_process", Payload: []byte("item1"), Headers: headers, Retry: defaultMaxRetry, Queue: "default", Timeout: int64(defaultTimeout.Seconds()), Deadline: noDeadline.Unix(), GroupKey: "batch-123", }, Score: now.Unix(), }, }, }, }, }, } for _, tc := range tests { h.FlushDB(t, r) gotInfo, err := client.Enqueue(tc.task, tc.opts...) if err != nil { t.Errorf("%s: Enqueue failed: %v", tc.desc, err) continue } cmpOptions := []cmp.Option{ cmpopts.IgnoreFields(TaskInfo{}, "ID"), cmpopts.EquateApproxTime(500 * time.Millisecond), } if diff := cmp.Diff(tc.wantInfo, gotInfo, cmpOptions...); diff != "" { t.Errorf("%s;\nEnqueue(task) returned %v, want %v; (-want,+got)\n%s", tc.desc, gotInfo, tc.wantInfo, diff) } for qname, groups := range tc.wantGroups { for groupKey, want := range groups { got := h.GetGroupEntries(t, r, qname, groupKey) if diff := cmp.Diff(want, got, h.IgnoreIDOpt, cmpopts.EquateEmpty()); diff != "" { t.Errorf("%s;\nmismatch found in %q; (-want,+got)\n%s", tc.desc, base.GroupKey(qname, groupKey), diff) } } } } } ================================================ FILE: context.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "context" asynqcontext "github.com/hibiken/asynq/internal/context" ) // GetTaskID extracts a task ID from a context, if any. // // ID of a task is guaranteed to be unique. // ID of a task doesn't change if the task is being retried. func GetTaskID(ctx context.Context) (id string, ok bool) { return asynqcontext.GetTaskID(ctx) } // GetRetryCount extracts retry count from a context, if any. // // Return value n indicates the number of times associated task has been // retried so far. func GetRetryCount(ctx context.Context) (n int, ok bool) { return asynqcontext.GetRetryCount(ctx) } // GetMaxRetry extracts maximum retry from a context, if any. // // Return value n indicates the maximum number of times the associated task // can be retried if ProcessTask returns a non-nil error. func GetMaxRetry(ctx context.Context) (n int, ok bool) { return asynqcontext.GetMaxRetry(ctx) } // GetQueueName extracts queue name from a context, if any. // // Return value queue indicates which queue the task was pulled from. func GetQueueName(ctx context.Context) (queue string, ok bool) { return asynqcontext.GetQueueName(ctx) } ================================================ FILE: doc.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. /* Package asynq provides a framework for Redis based distrubted task queue. Asynq uses Redis as a message broker. To connect to redis, specify the connection using one of RedisConnOpt types. redisConnOpt = asynq.RedisClientOpt{ Addr: "127.0.0.1:6379", Password: "xxxxx", DB: 2, } The Client is used to enqueue a task. client := asynq.NewClient(redisConnOpt) // Task is created with two parameters: its type and payload. // Payload data is simply an array of bytes. It can be encoded in JSON, Protocol Buffer, Gob, etc. b, err := json.Marshal(ExamplePayload{UserID: 42}) if err != nil { log.Fatal(err) } task := asynq.NewTask("example", b) // Enqueue the task to be processed immediately. info, err := client.Enqueue(task) // Schedule the task to be processed after one minute. info, err = client.Enqueue(t, asynq.ProcessIn(1*time.Minute)) The Server is used to run the task processing workers with a given handler. srv := asynq.NewServer(redisConnOpt, asynq.Config{ Concurrency: 10, }) if err := srv.Run(handler); err != nil { log.Fatal(err) } Handler is an interface type with a method which takes a task and returns an error. Handler should return nil if the processing is successful, otherwise return a non-nil error. If handler panics or returns a non-nil error, the task will be retried in the future. Example of a type that implements the Handler interface. type TaskHandler struct { // ... } func (h *TaskHandler) ProcessTask(ctx context.Context, task *asynq.Task) error { switch task.Type { case "example": var data ExamplePayload if err := json.Unmarshal(task.Payload(), &data); err != nil { return err } // perform task with the data default: return fmt.Errorf("unexpected task type %q", task.Type) } return nil } */ package asynq ================================================ FILE: example_test.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq_test import ( "context" "fmt" "log" "os" "os/signal" "time" "github.com/hibiken/asynq" "golang.org/x/sys/unix" ) func ExampleServer_Run() { srv := asynq.NewServer( asynq.RedisClientOpt{Addr: ":6379"}, asynq.Config{Concurrency: 20}, ) h := asynq.NewServeMux() // ... Register handlers // Run blocks and waits for os signal to terminate the program. if err := srv.Run(h); err != nil { log.Fatal(err) } } func ExampleServer_Shutdown() { srv := asynq.NewServer( asynq.RedisClientOpt{Addr: ":6379"}, asynq.Config{Concurrency: 20}, ) h := asynq.NewServeMux() // ... Register handlers if err := srv.Start(h); err != nil { log.Fatal(err) } sigs := make(chan os.Signal, 1) signal.Notify(sigs, unix.SIGTERM, unix.SIGINT) <-sigs // wait for termination signal srv.Shutdown() } func ExampleServer_Stop() { srv := asynq.NewServer( asynq.RedisClientOpt{Addr: ":6379"}, asynq.Config{Concurrency: 20}, ) h := asynq.NewServeMux() // ... Register handlers if err := srv.Start(h); err != nil { log.Fatal(err) } sigs := make(chan os.Signal, 1) signal.Notify(sigs, unix.SIGTERM, unix.SIGINT, unix.SIGTSTP) // Handle SIGTERM, SIGINT to exit the program. // Handle SIGTSTP to stop processing new tasks. for { s := <-sigs if s == unix.SIGTSTP { srv.Stop() // stop processing new tasks continue } break // received SIGTERM or SIGINT signal } srv.Shutdown() } func ExampleScheduler() { scheduler := asynq.NewScheduler( asynq.RedisClientOpt{Addr: ":6379"}, &asynq.SchedulerOpts{Location: time.Local}, ) if _, err := scheduler.Register("* * * * *", asynq.NewTask("task1", nil)); err != nil { log.Fatal(err) } if _, err := scheduler.Register("@every 30s", asynq.NewTask("task2", nil)); err != nil { log.Fatal(err) } // Run blocks and waits for os signal to terminate the program. if err := scheduler.Run(); err != nil { log.Fatal(err) } } func ExampleParseRedisURI() { rconn, err := asynq.ParseRedisURI("redis://localhost:6379/10") if err != nil { log.Fatal(err) } r, ok := rconn.(asynq.RedisClientOpt) if !ok { log.Fatal("unexpected type") } fmt.Println(r.Addr) fmt.Println(r.DB) // Output: // localhost:6379 // 10 } func ExampleResultWriter() { // ResultWriter is only accessible in Handler. h := func(ctx context.Context, task *asynq.Task) error { // .. do task processing work res := []byte("task result data") n, err := task.ResultWriter().Write(res) // implements io.Writer if err != nil { return fmt.Errorf("failed to write task result: %w", err) } log.Printf(" %d bytes written", n) return nil } _ = h } ================================================ FILE: forwarder.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "sync" "time" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/log" ) // A forwarder is responsible for moving scheduled and retry tasks to pending state // so that the tasks get processed by the workers. type forwarder struct { logger *log.Logger broker base.Broker // channel to communicate back to the long running "forwarder" goroutine. done chan struct{} // list of queue names to check and enqueue. queues []string // poll interval on average avgInterval time.Duration } type forwarderParams struct { logger *log.Logger broker base.Broker queues []string interval time.Duration } func newForwarder(params forwarderParams) *forwarder { return &forwarder{ logger: params.logger, broker: params.broker, done: make(chan struct{}), queues: params.queues, avgInterval: params.interval, } } func (f *forwarder) shutdown() { f.logger.Debug("Forwarder shutting down...") // Signal the forwarder goroutine to stop polling. f.done <- struct{}{} } // start starts the "forwarder" goroutine. func (f *forwarder) start(wg *sync.WaitGroup) { wg.Add(1) go func() { defer wg.Done() timer := time.NewTimer(f.avgInterval) for { select { case <-f.done: f.logger.Debug("Forwarder done") return case <-timer.C: f.exec() timer.Reset(f.avgInterval) } } }() } func (f *forwarder) exec() { if err := f.broker.ForwardIfReady(f.queues...); err != nil { f.logger.Errorf("Failed to forward scheduled tasks: %v", err) } } ================================================ FILE: forwarder_test.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "sync" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/rdb" h "github.com/hibiken/asynq/internal/testutil" ) func TestForwarder(t *testing.T) { r := setup(t) defer r.Close() rdbClient := rdb.NewRDB(r) const pollInterval = time.Second s := newForwarder(forwarderParams{ logger: testLogger, broker: rdbClient, queues: []string{"default", "critical"}, interval: pollInterval, }) t1 := h.NewTaskMessageWithQueue("gen_thumbnail", nil, "default") t2 := h.NewTaskMessageWithQueue("send_email", nil, "critical") t3 := h.NewTaskMessageWithQueue("reindex", nil, "default") t4 := h.NewTaskMessageWithQueue("sync", nil, "critical") now := time.Now() tests := []struct { initScheduled map[string][]base.Z // scheduled queue initial state initRetry map[string][]base.Z // retry queue initial state initPending map[string][]*base.TaskMessage // default queue initial state wait time.Duration // wait duration before checking for final state wantScheduled map[string][]*base.TaskMessage // schedule queue final state wantRetry map[string][]*base.TaskMessage // retry queue final state wantPending map[string][]*base.TaskMessage // default queue final state }{ { initScheduled: map[string][]base.Z{ "default": {{Message: t1, Score: now.Add(time.Hour).Unix()}}, "critical": {{Message: t2, Score: now.Add(-2 * time.Second).Unix()}}, }, initRetry: map[string][]base.Z{ "default": {{Message: t3, Score: time.Now().Add(-500 * time.Millisecond).Unix()}}, "critical": {}, }, initPending: map[string][]*base.TaskMessage{ "default": {}, "critical": {t4}, }, wait: pollInterval * 2, wantScheduled: map[string][]*base.TaskMessage{ "default": {t1}, "critical": {}, }, wantRetry: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, }, wantPending: map[string][]*base.TaskMessage{ "default": {t3}, "critical": {t2, t4}, }, }, { initScheduled: map[string][]base.Z{ "default": { {Message: t1, Score: now.Unix()}, {Message: t3, Score: now.Add(-500 * time.Millisecond).Unix()}, }, "critical": { {Message: t2, Score: now.Add(-2 * time.Second).Unix()}, }, }, initRetry: map[string][]base.Z{ "default": {}, "critical": {}, }, initPending: map[string][]*base.TaskMessage{ "default": {}, "critical": {t4}, }, wait: pollInterval * 2, wantScheduled: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, }, wantRetry: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, }, wantPending: map[string][]*base.TaskMessage{ "default": {t1, t3}, "critical": {t2, t4}, }, }, } for _, tc := range tests { h.FlushDB(t, r) // clean up db before each test case. h.SeedAllScheduledQueues(t, r, tc.initScheduled) // initialize scheduled queue h.SeedAllRetryQueues(t, r, tc.initRetry) // initialize retry queue h.SeedAllPendingQueues(t, r, tc.initPending) // initialize default queue var wg sync.WaitGroup s.start(&wg) time.Sleep(tc.wait) s.shutdown() for qname, want := range tc.wantScheduled { gotScheduled := h.GetScheduledMessages(t, r, qname) if diff := cmp.Diff(want, gotScheduled, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q after running forwarder: (-want, +got)\n%s", base.ScheduledKey(qname), diff) } } for qname, want := range tc.wantRetry { gotRetry := h.GetRetryMessages(t, r, qname) if diff := cmp.Diff(want, gotRetry, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q after running forwarder: (-want, +got)\n%s", base.RetryKey(qname), diff) } } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q after running forwarder: (-want, +got)\n%s", base.PendingKey(qname), diff) } } } } ================================================ FILE: go.mod ================================================ module github.com/hibiken/asynq go 1.24.0 require ( github.com/google/go-cmp v0.7.0 github.com/google/uuid v1.6.0 github.com/redis/go-redis/v9 v9.14.1 github.com/robfig/cron/v3 v3.0.1 github.com/spf13/cast v1.10.0 go.uber.org/goleak v1.3.0 golang.org/x/sys v0.37.0 golang.org/x/time v0.14.0 google.golang.org/protobuf v1.36.10 ) require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect ) ================================================ FILE: go.sum ================================================ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.14.1 h1:nDCrEiJmfOWhD76xlaw+HXT0c9hfNWeXgl0vIRYSDvQ= github.com/redis/go-redis/v9 v9.14.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= ================================================ FILE: healthcheck.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "sync" "time" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/log" ) // healthchecker is responsible for pinging broker periodically // and call user provided HeathCheckFunc with the ping result. type healthchecker struct { logger *log.Logger broker base.Broker // channel to communicate back to the long running "healthchecker" goroutine. done chan struct{} // interval between healthchecks. interval time.Duration // function to call periodically. healthcheckFunc func(error) } type healthcheckerParams struct { logger *log.Logger broker base.Broker interval time.Duration healthcheckFunc func(error) } func newHealthChecker(params healthcheckerParams) *healthchecker { return &healthchecker{ logger: params.logger, broker: params.broker, done: make(chan struct{}), interval: params.interval, healthcheckFunc: params.healthcheckFunc, } } func (hc *healthchecker) shutdown() { if hc.healthcheckFunc == nil { return } hc.logger.Debug("Healthchecker shutting down...") // Signal the healthchecker goroutine to stop. hc.done <- struct{}{} } func (hc *healthchecker) start(wg *sync.WaitGroup) { if hc.healthcheckFunc == nil { return } wg.Add(1) go func() { defer wg.Done() timer := time.NewTimer(hc.interval) for { select { case <-hc.done: hc.logger.Debug("Healthchecker done") timer.Stop() return case <-timer.C: err := hc.broker.Ping() hc.healthcheckFunc(err) timer.Reset(hc.interval) } } }() } ================================================ FILE: healthcheck_test.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "sync" "testing" "time" "github.com/hibiken/asynq/internal/rdb" "github.com/hibiken/asynq/internal/testbroker" ) func TestHealthChecker(t *testing.T) { r := setup(t) defer r.Close() rdbClient := rdb.NewRDB(r) var ( // mu guards called and e variables. mu sync.Mutex called int e error ) checkFn := func(err error) { mu.Lock() defer mu.Unlock() called++ e = err } hc := newHealthChecker(healthcheckerParams{ logger: testLogger, broker: rdbClient, interval: 1 * time.Second, healthcheckFunc: checkFn, }) hc.start(&sync.WaitGroup{}) time.Sleep(2 * time.Second) mu.Lock() if called == 0 { t.Errorf("Healthchecker did not call the provided HealthCheckFunc") } if e != nil { t.Errorf("HealthCheckFunc was called with non-nil error: %v", e) } mu.Unlock() hc.shutdown() } func TestHealthCheckerWhenRedisDown(t *testing.T) { // Make sure that healthchecker goroutine doesn't panic // if it cannot connect to redis. defer func() { if r := recover(); r != nil { t.Errorf("panic occurred: %v", r) } }() r := rdb.NewRDB(setup(t)) defer r.Close() testBroker := testbroker.NewTestBroker(r) var ( // mu guards called and e variables. mu sync.Mutex called int e error ) checkFn := func(err error) { mu.Lock() defer mu.Unlock() called++ e = err } hc := newHealthChecker(healthcheckerParams{ logger: testLogger, broker: testBroker, interval: 1 * time.Second, healthcheckFunc: checkFn, }) testBroker.Sleep() hc.start(&sync.WaitGroup{}) time.Sleep(2 * time.Second) mu.Lock() if called == 0 { t.Errorf("Healthchecker did not call the provided HealthCheckFunc") } if e == nil { t.Errorf("HealthCheckFunc was called with nil; want non-nil error") } mu.Unlock() hc.shutdown() } ================================================ FILE: heartbeat.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "os" "sync" "time" "github.com/google/uuid" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/log" "github.com/hibiken/asynq/internal/timeutil" ) // heartbeater is responsible for writing process info to redis periodically to // indicate that the background worker process is up. type heartbeater struct { logger *log.Logger broker base.Broker clock timeutil.Clock // channel to communicate back to the long running "heartbeater" goroutine. done chan struct{} // interval between heartbeats. interval time.Duration // following fields are initialized at construction time and are immutable. host string pid int serverID string concurrency int queues map[string]int strictPriority bool // following fields are mutable and should be accessed only by the // heartbeater goroutine. In other words, confine these variables // to this goroutine only. started time.Time workers map[string]*workerInfo // state is shared with other goroutine but is concurrency safe. state *serverState // channels to receive updates on active workers. starting <-chan *workerInfo finished <-chan *base.TaskMessage } type heartbeaterParams struct { logger *log.Logger broker base.Broker interval time.Duration concurrency int queues map[string]int strictPriority bool state *serverState starting <-chan *workerInfo finished <-chan *base.TaskMessage } func newHeartbeater(params heartbeaterParams) *heartbeater { host, err := os.Hostname() if err != nil { host = "unknown-host" } return &heartbeater{ logger: params.logger, broker: params.broker, clock: timeutil.NewRealClock(), done: make(chan struct{}), interval: params.interval, host: host, pid: os.Getpid(), serverID: uuid.New().String(), concurrency: params.concurrency, queues: params.queues, strictPriority: params.strictPriority, state: params.state, workers: make(map[string]*workerInfo), starting: params.starting, finished: params.finished, } } func (h *heartbeater) shutdown() { h.logger.Debug("Heartbeater shutting down...") // Signal the heartbeater goroutine to stop. h.done <- struct{}{} } // A workerInfo holds an active worker information. type workerInfo struct { // the task message the worker is processing. msg *base.TaskMessage // the time the worker has started processing the message. started time.Time // deadline the worker has to finish processing the task by. deadline time.Time // lease the worker holds for the task. lease *base.Lease } func (h *heartbeater) start(wg *sync.WaitGroup) { wg.Add(1) go func() { defer wg.Done() h.started = h.clock.Now() h.beat() timer := time.NewTimer(h.interval) for { select { case <-h.done: if err := h.broker.ClearServerState(h.host, h.pid, h.serverID); err != nil { h.logger.Errorf("Failed to clear server state: %v", err) } h.logger.Debug("Heartbeater done") timer.Stop() return case <-timer.C: h.beat() timer.Reset(h.interval) case w := <-h.starting: h.workers[w.msg.ID] = w case msg := <-h.finished: delete(h.workers, msg.ID) } } }() } // beat extends lease for workers and writes server/worker info to redis. func (h *heartbeater) beat() { h.state.mu.Lock() srvStatus := h.state.value.String() h.state.mu.Unlock() info := base.ServerInfo{ Host: h.host, PID: h.pid, ServerID: h.serverID, Concurrency: h.concurrency, Queues: h.queues, StrictPriority: h.strictPriority, Status: srvStatus, Started: h.started, ActiveWorkerCount: len(h.workers), } var ws []*base.WorkerInfo idsByQueue := make(map[string][]string) for id, w := range h.workers { ws = append(ws, &base.WorkerInfo{ Host: h.host, PID: h.pid, ServerID: h.serverID, ID: id, Type: w.msg.Type, Queue: w.msg.Queue, Payload: w.msg.Payload, Started: w.started, Deadline: w.deadline, }) // Check lease before adding to the set to make sure not to extend the lease if the lease is already expired. if w.lease.IsValid() { idsByQueue[w.msg.Queue] = append(idsByQueue[w.msg.Queue], id) } else { w.lease.NotifyExpiration() // notify processor if the lease is expired } } // Note: Set TTL to be long enough so that it won't expire before we write again // and short enough to expire quickly once the process is shut down or killed. if err := h.broker.WriteServerState(&info, ws, h.interval*2); err != nil { h.logger.Errorf("Failed to write server state data: %v", err) } for qname, ids := range idsByQueue { expirationTime, err := h.broker.ExtendLease(qname, ids...) if err != nil { h.logger.Errorf("Failed to extend lease for tasks %v: %v", ids, err) continue } for _, id := range ids { if l := h.workers[id].lease; !l.Reset(expirationTime) { h.logger.Warnf("Lease reset failed for %s; lease deadline: %v", id, l.Deadline()) } } } } ================================================ FILE: heartbeat_test.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "context" "sync" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/rdb" "github.com/hibiken/asynq/internal/testbroker" h "github.com/hibiken/asynq/internal/testutil" "github.com/hibiken/asynq/internal/timeutil" ) // Test goes through a few phases. // // Phase1: Simulate Server startup; Simulate starting tasks listed in startedWorkers // Phase2: Simulate finishing tasks listed in finishedTasks // Phase3: Simulate Server shutdown; func TestHeartbeater(t *testing.T) { r := setup(t) defer r.Close() rdbClient := rdb.NewRDB(r) now := time.Now() const elapsedTime = 10 * time.Second // simulated time elapsed between phase1 and phase2 clock := timeutil.NewSimulatedClock(time.Time{}) // time will be set in each test t1 := h.NewTaskMessageWithQueue("task1", nil, "default") t2 := h.NewTaskMessageWithQueue("task2", nil, "default") t3 := h.NewTaskMessageWithQueue("task3", nil, "default") t4 := h.NewTaskMessageWithQueue("task4", nil, "custom") t5 := h.NewTaskMessageWithQueue("task5", nil, "custom") t6 := h.NewTaskMessageWithQueue("task6", nil, "default") // Note: intentionally set to time less than now.Add(rdb.LeaseDuration) to test lease extension is working. lease1 := h.NewLeaseWithClock(now.Add(10*time.Second), clock) lease2 := h.NewLeaseWithClock(now.Add(10*time.Second), clock) lease3 := h.NewLeaseWithClock(now.Add(10*time.Second), clock) lease4 := h.NewLeaseWithClock(now.Add(10*time.Second), clock) lease5 := h.NewLeaseWithClock(now.Add(10*time.Second), clock) lease6 := h.NewLeaseWithClock(now.Add(10*time.Second), clock) tests := []struct { desc string // Interval between heartbeats. interval time.Duration // Server info. host string pid int queues map[string]int concurrency int active map[string][]*base.TaskMessage // initial active set state lease map[string][]base.Z // initial lease set state wantLease1 map[string][]base.Z // expected lease set state after starting all startedWorkers wantLease2 map[string][]base.Z // expected lease set state after finishing all finishedTasks startedWorkers []*workerInfo // workerInfo to send via the started channel finishedTasks []*base.TaskMessage // tasks to send via the finished channel startTime time.Time // simulated start time elapsedTime time.Duration // simulated time elapsed between starting and finishing processing tasks }{ { desc: "With single queue", interval: 2 * time.Second, host: "localhost", pid: 45678, queues: map[string]int{"default": 1}, concurrency: 10, active: map[string][]*base.TaskMessage{ "default": {t1, t2, t3}, }, lease: map[string][]base.Z{ "default": { {Message: t1, Score: now.Add(10 * time.Second).Unix()}, {Message: t2, Score: now.Add(10 * time.Second).Unix()}, {Message: t3, Score: now.Add(10 * time.Second).Unix()}, }, }, startedWorkers: []*workerInfo{ {msg: t1, started: now, deadline: now.Add(2 * time.Minute), lease: lease1}, {msg: t2, started: now, deadline: now.Add(2 * time.Minute), lease: lease2}, {msg: t3, started: now, deadline: now.Add(2 * time.Minute), lease: lease3}, }, finishedTasks: []*base.TaskMessage{t1, t2}, wantLease1: map[string][]base.Z{ "default": { {Message: t1, Score: now.Add(rdb.LeaseDuration).Unix()}, {Message: t2, Score: now.Add(rdb.LeaseDuration).Unix()}, {Message: t3, Score: now.Add(rdb.LeaseDuration).Unix()}, }, }, wantLease2: map[string][]base.Z{ "default": { {Message: t3, Score: now.Add(elapsedTime).Add(rdb.LeaseDuration).Unix()}, }, }, startTime: now, elapsedTime: elapsedTime, }, { desc: "With multiple queue", interval: 2 * time.Second, host: "localhost", pid: 45678, queues: map[string]int{"default": 1, "custom": 2}, concurrency: 10, active: map[string][]*base.TaskMessage{ "default": {t6}, "custom": {t4, t5}, }, lease: map[string][]base.Z{ "default": { {Message: t6, Score: now.Add(10 * time.Second).Unix()}, }, "custom": { {Message: t4, Score: now.Add(10 * time.Second).Unix()}, {Message: t5, Score: now.Add(10 * time.Second).Unix()}, }, }, startedWorkers: []*workerInfo{ {msg: t6, started: now, deadline: now.Add(2 * time.Minute), lease: lease6}, {msg: t4, started: now, deadline: now.Add(2 * time.Minute), lease: lease4}, {msg: t5, started: now, deadline: now.Add(2 * time.Minute), lease: lease5}, }, finishedTasks: []*base.TaskMessage{t6, t5}, wantLease1: map[string][]base.Z{ "default": { {Message: t6, Score: now.Add(rdb.LeaseDuration).Unix()}, }, "custom": { {Message: t4, Score: now.Add(rdb.LeaseDuration).Unix()}, {Message: t5, Score: now.Add(rdb.LeaseDuration).Unix()}, }, }, wantLease2: map[string][]base.Z{ "default": {}, "custom": { {Message: t4, Score: now.Add(elapsedTime).Add(rdb.LeaseDuration).Unix()}, }, }, startTime: now, elapsedTime: elapsedTime, }, } timeCmpOpt := cmpopts.EquateApproxTime(10 * time.Millisecond) ignoreOpt := cmpopts.IgnoreUnexported(base.ServerInfo{}) ignoreFieldOpt := cmpopts.IgnoreFields(base.ServerInfo{}, "ServerID") for _, tc := range tests { h.FlushDB(t, r) h.SeedAllActiveQueues(t, r, tc.active) h.SeedAllLease(t, r, tc.lease) clock.SetTime(tc.startTime) rdbClient.SetClock(clock) srvState := &serverState{} startingCh := make(chan *workerInfo) finishedCh := make(chan *base.TaskMessage) hb := newHeartbeater(heartbeaterParams{ logger: testLogger, broker: rdbClient, interval: tc.interval, concurrency: tc.concurrency, queues: tc.queues, strictPriority: false, state: srvState, starting: startingCh, finished: finishedCh, }) hb.clock = clock // Change host and pid fields for testing purpose. hb.host = tc.host hb.pid = tc.pid //=================== // Start Phase1 //=================== srvState.mu.Lock() srvState.value = srvStateActive // simulating Server.Start srvState.mu.Unlock() var wg sync.WaitGroup hb.start(&wg) // Simulate processor starting to work on tasks. for _, w := range tc.startedWorkers { startingCh <- w } // Wait for heartbeater to write to redis time.Sleep(tc.interval * 2) ss, err := rdbClient.ListServers() if err != nil { t.Errorf("%s: could not read server info from redis: %v", tc.desc, err) hb.shutdown() continue } if len(ss) != 1 { t.Errorf("%s: (*RDB).ListServers returned %d server info, want 1", tc.desc, len(ss)) hb.shutdown() continue } wantInfo := &base.ServerInfo{ Host: tc.host, PID: tc.pid, Queues: tc.queues, Concurrency: tc.concurrency, Started: now, Status: "active", ActiveWorkerCount: len(tc.startedWorkers), } if diff := cmp.Diff(wantInfo, ss[0], timeCmpOpt, ignoreOpt, ignoreFieldOpt); diff != "" { t.Errorf("%s: redis stored server status %+v, want %+v; (-want, +got)\n%s", tc.desc, ss[0], wantInfo, diff) hb.shutdown() continue } for qname, wantLease := range tc.wantLease1 { gotLease := h.GetLeaseEntries(t, r, qname) if diff := cmp.Diff(wantLease, gotLease, h.SortZSetEntryOpt); diff != "" { t.Errorf("%s: mismatch found in %q: (-want,+got):\n%s", tc.desc, base.LeaseKey(qname), diff) } } for _, w := range tc.startedWorkers { if want := now.Add(rdb.LeaseDuration); w.lease.Deadline() != want { t.Errorf("%s: lease deadline for %v is set to %v, want %v", tc.desc, w.msg, w.lease.Deadline(), want) } } //=================== // Start Phase2 //=================== clock.AdvanceTime(tc.elapsedTime) // Simulate processor finished processing tasks. for _, msg := range tc.finishedTasks { if err := rdbClient.Done(context.Background(), msg); err != nil { t.Fatalf("RDB.Done failed: %v", err) } finishedCh <- msg } // Wait for heartbeater to write to redis time.Sleep(tc.interval * 2) for qname, wantLease := range tc.wantLease2 { gotLease := h.GetLeaseEntries(t, r, qname) if diff := cmp.Diff(wantLease, gotLease, h.SortZSetEntryOpt); diff != "" { t.Errorf("%s: mismatch found in %q: (-want,+got):\n%s", tc.desc, base.LeaseKey(qname), diff) } } //=================== // Start Phase3 //=================== // Server state change; simulating Server.Shutdown srvState.mu.Lock() srvState.value = srvStateClosed srvState.mu.Unlock() // Wait for heartbeater to write to redis time.Sleep(tc.interval * 2) wantInfo = &base.ServerInfo{ Host: tc.host, PID: tc.pid, Queues: tc.queues, Concurrency: tc.concurrency, Started: now, Status: "closed", ActiveWorkerCount: len(tc.startedWorkers) - len(tc.finishedTasks), } ss, err = rdbClient.ListServers() if err != nil { t.Errorf("%s: could not read server status from redis: %v", tc.desc, err) hb.shutdown() continue } if len(ss) != 1 { t.Errorf("%s: (*RDB).ListServers returned %d server info, want 1", tc.desc, len(ss)) hb.shutdown() continue } if diff := cmp.Diff(wantInfo, ss[0], timeCmpOpt, ignoreOpt, ignoreFieldOpt); diff != "" { t.Errorf("%s: redis stored process status %+v, want %+v; (-want, +got)\n%s", tc.desc, ss[0], wantInfo, diff) hb.shutdown() continue } hb.shutdown() } } func TestHeartbeaterWithRedisDown(t *testing.T) { // Make sure that heartbeater goroutine doesn't panic // if it cannot connect to redis. defer func() { if r := recover(); r != nil { t.Errorf("panic occurred: %v", r) } }() r := rdb.NewRDB(setup(t)) defer r.Close() testBroker := testbroker.NewTestBroker(r) state := &serverState{value: srvStateActive} hb := newHeartbeater(heartbeaterParams{ logger: testLogger, broker: testBroker, interval: time.Second, concurrency: 10, queues: map[string]int{"default": 1}, strictPriority: false, state: state, starting: make(chan *workerInfo), finished: make(chan *base.TaskMessage), }) testBroker.Sleep() var wg sync.WaitGroup hb.start(&wg) // wait for heartbeater to try writing data to redis time.Sleep(2 * time.Second) hb.shutdown() } ================================================ FILE: inspector.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "fmt" "strconv" "strings" "time" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/errors" "github.com/hibiken/asynq/internal/rdb" "github.com/redis/go-redis/v9" ) // Inspector is a client interface to inspect and mutate the state of // queues and tasks. type Inspector struct { rdb *rdb.RDB // When an Inspector has been created with an existing Redis connection, we do // not want to close it. sharedConnection bool } // New returns a new instance of Inspector. func NewInspector(r RedisConnOpt) *Inspector { c, ok := r.MakeRedisClient().(redis.UniversalClient) if !ok { panic(fmt.Sprintf("inspeq: unsupported RedisConnOpt type %T", r)) } inspector := NewInspectorFromRedisClient(c) inspector.sharedConnection = false return inspector } // NewInspectorFromRedisClient returns a new instance of Inspector given a redis.UniversalClient // Warning: The underlying redis connection pool will not be closed by Asynq, you are responsible for closing it. func NewInspectorFromRedisClient(c redis.UniversalClient) *Inspector { return &Inspector{ rdb: rdb.NewRDB(c), sharedConnection: true, } } // Close closes the connection with redis. func (i *Inspector) Close() error { if i.sharedConnection { return fmt.Errorf("redis connection is shared so the Inspector can't be closed through asynq") } return i.rdb.Close() } // Queues returns a list of all queue names. func (i *Inspector) Queues() ([]string, error) { return i.rdb.AllQueues() } // Groups returns a list of all groups within the given queue. func (i *Inspector) Groups(queue string) ([]*GroupInfo, error) { stats, err := i.rdb.GroupStats(queue) if err != nil { return nil, err } var res []*GroupInfo for _, s := range stats { res = append(res, &GroupInfo{ Group: s.Group, Size: s.Size, }) } return res, nil } // GroupInfo represents a state of a group at a certain time. type GroupInfo struct { // Name of the group. Group string // Size is the total number of tasks in the group. Size int } // QueueInfo represents a state of a queue at a certain time. type QueueInfo struct { // Name of the queue. Queue string // Total number of bytes that the queue and its tasks require to be stored in redis. // It is an approximate memory usage value in bytes since the value is computed by sampling. MemoryUsage int64 // Latency of the queue, measured by the oldest pending task in the queue. Latency time.Duration // Size is the total number of tasks in the queue. // The value is the sum of Pending, Active, Scheduled, Retry, Aggregating and Archived. Size int // Groups is the total number of groups in the queue. Groups int // Number of pending tasks. Pending int // Number of active tasks. Active int // Number of scheduled tasks. Scheduled int // Number of retry tasks. Retry int // Number of archived tasks. Archived int // Number of stored completed tasks. Completed int // Number of aggregating tasks. Aggregating int // Total number of tasks being processed within the given date (counter resets daily). // The number includes both succeeded and failed tasks. Processed int // Total number of tasks failed to be processed within the given date (counter resets daily). Failed int // Total number of tasks processed (cumulative). ProcessedTotal int // Total number of tasks failed (cumulative). FailedTotal int // Paused indicates whether the queue is paused. // If true, tasks in the queue will not be processed. Paused bool // Time when this queue info snapshot was taken. Timestamp time.Time } // GetQueueInfo returns current information of the given queue. func (i *Inspector) GetQueueInfo(queue string) (*QueueInfo, error) { if err := base.ValidateQueueName(queue); err != nil { return nil, err } stats, err := i.rdb.CurrentStats(queue) if err != nil { return nil, err } return &QueueInfo{ Queue: stats.Queue, MemoryUsage: stats.MemoryUsage, Latency: stats.Latency, Size: stats.Size, Groups: stats.Groups, Pending: stats.Pending, Active: stats.Active, Scheduled: stats.Scheduled, Retry: stats.Retry, Archived: stats.Archived, Completed: stats.Completed, Aggregating: stats.Aggregating, Processed: stats.Processed, Failed: stats.Failed, ProcessedTotal: stats.ProcessedTotal, FailedTotal: stats.FailedTotal, Paused: stats.Paused, Timestamp: stats.Timestamp, }, nil } // DailyStats holds aggregate data for a given day for a given queue. type DailyStats struct { // Name of the queue. Queue string // Total number of tasks being processed during the given date. // The number includes both succeeded and failed tasks. Processed int // Total number of tasks failed to be processed during the given date. Failed int // Date this stats was taken. Date time.Time } // History returns a list of stats from the last n days. func (i *Inspector) History(queue string, n int) ([]*DailyStats, error) { if err := base.ValidateQueueName(queue); err != nil { return nil, err } stats, err := i.rdb.HistoricalStats(queue, n) if err != nil { return nil, err } var res []*DailyStats for _, s := range stats { res = append(res, &DailyStats{ Queue: s.Queue, Processed: s.Processed, Failed: s.Failed, Date: s.Time, }) } return res, nil } var ( // ErrQueueNotFound indicates that the specified queue does not exist. ErrQueueNotFound = errors.New("queue not found") // ErrQueueNotEmpty indicates that the specified queue is not empty. ErrQueueNotEmpty = errors.New("queue is not empty") // ErrTaskNotFound indicates that the specified task cannot be found in the queue. ErrTaskNotFound = errors.New("task not found") ) // DeleteQueue removes the specified queue. // // If force is set to true, DeleteQueue will remove the queue regardless of // the queue size as long as no tasks are active in the queue. // If force is set to false, DeleteQueue will remove the queue only if // the queue is empty. // // If the specified queue does not exist, DeleteQueue returns ErrQueueNotFound. // If force is set to false and the specified queue is not empty, DeleteQueue // returns ErrQueueNotEmpty. func (i *Inspector) DeleteQueue(queue string, force bool) error { err := i.rdb.RemoveQueue(queue, force) if errors.IsQueueNotFound(err) { return fmt.Errorf("%w: queue=%q", ErrQueueNotFound, queue) } if errors.IsQueueNotEmpty(err) { return fmt.Errorf("%w: queue=%q", ErrQueueNotEmpty, queue) } return err } // GetTaskInfo retrieves task information given a task id and queue name. // // Returns an error wrapping ErrQueueNotFound if a queue with the given name doesn't exist. // Returns an error wrapping ErrTaskNotFound if a task with the given id doesn't exist in the queue. func (i *Inspector) GetTaskInfo(queue, id string) (*TaskInfo, error) { info, err := i.rdb.GetTaskInfo(queue, id) switch { case errors.IsQueueNotFound(err): return nil, fmt.Errorf("asynq: %w", ErrQueueNotFound) case errors.IsTaskNotFound(err): return nil, fmt.Errorf("asynq: %w", ErrTaskNotFound) case err != nil: return nil, fmt.Errorf("asynq: %w", err) } return newTaskInfo(info.Message, info.State, info.NextProcessAt, info.Result), nil } // ListOption specifies behavior of list operation. type ListOption interface{} // Internal list option representations. type ( pageSizeOpt int pageNumOpt int ) type listOption struct { pageSize int pageNum int } const ( // Page size used by default in list operation. defaultPageSize = 30 // Page number used by default in list operation. defaultPageNum = 1 ) func composeListOptions(opts ...ListOption) listOption { res := listOption{ pageSize: defaultPageSize, pageNum: defaultPageNum, } for _, opt := range opts { switch opt := opt.(type) { case pageSizeOpt: res.pageSize = int(opt) case pageNumOpt: res.pageNum = int(opt) default: // ignore unexpected option } } return res } // PageSize returns an option to specify the page size for list operation. // // Negative page size is treated as zero. func PageSize(n int) ListOption { if n < 0 { n = 0 } return pageSizeOpt(n) } // Page returns an option to specify the page number for list operation. // The value 1 fetches the first page. // // Negative page number is treated as one. func Page(n int) ListOption { if n < 0 { n = 1 } return pageNumOpt(n) } // ListPendingTasks retrieves pending tasks from the specified queue. // // By default, it retrieves the first 30 tasks. func (i *Inspector) ListPendingTasks(queue string, opts ...ListOption) ([]*TaskInfo, error) { if err := base.ValidateQueueName(queue); err != nil { return nil, fmt.Errorf("asynq: %w", err) } opt := composeListOptions(opts...) pgn := rdb.Pagination{Size: opt.pageSize, Page: opt.pageNum - 1} infos, err := i.rdb.ListPending(queue, pgn) switch { case errors.IsQueueNotFound(err): return nil, fmt.Errorf("asynq: %w", ErrQueueNotFound) case err != nil: return nil, fmt.Errorf("asynq: %w", err) } var tasks []*TaskInfo for _, i := range infos { tasks = append(tasks, newTaskInfo( i.Message, i.State, i.NextProcessAt, i.Result, )) } return tasks, err } // ListActiveTasks retrieves active tasks from the specified queue. // // By default, it retrieves the first 30 tasks. func (i *Inspector) ListActiveTasks(queue string, opts ...ListOption) ([]*TaskInfo, error) { if err := base.ValidateQueueName(queue); err != nil { return nil, fmt.Errorf("asynq: %w", err) } opt := composeListOptions(opts...) pgn := rdb.Pagination{Size: opt.pageSize, Page: opt.pageNum - 1} infos, err := i.rdb.ListActive(queue, pgn) switch { case errors.IsQueueNotFound(err): return nil, fmt.Errorf("asynq: %w", ErrQueueNotFound) case err != nil: return nil, fmt.Errorf("asynq: %w", err) } expired, err := i.rdb.ListLeaseExpired(time.Now(), queue) if err != nil { return nil, fmt.Errorf("asynq: %w", err) } expiredSet := make(map[string]struct{}) // set of expired message IDs for _, msg := range expired { expiredSet[msg.ID] = struct{}{} } var tasks []*TaskInfo for _, i := range infos { t := newTaskInfo( i.Message, i.State, i.NextProcessAt, i.Result, ) if _, ok := expiredSet[i.Message.ID]; ok { t.IsOrphaned = true } tasks = append(tasks, t) } return tasks, nil } // ListAggregatingTasks retrieves scheduled tasks from the specified group. // // By default, it retrieves the first 30 tasks. func (i *Inspector) ListAggregatingTasks(queue, group string, opts ...ListOption) ([]*TaskInfo, error) { if err := base.ValidateQueueName(queue); err != nil { return nil, fmt.Errorf("asynq: %w", err) } opt := composeListOptions(opts...) pgn := rdb.Pagination{Size: opt.pageSize, Page: opt.pageNum - 1} infos, err := i.rdb.ListAggregating(queue, group, pgn) switch { case errors.IsQueueNotFound(err): return nil, fmt.Errorf("asynq: %w", ErrQueueNotFound) case err != nil: return nil, fmt.Errorf("asynq: %w", err) } var tasks []*TaskInfo for _, i := range infos { tasks = append(tasks, newTaskInfo( i.Message, i.State, i.NextProcessAt, i.Result, )) } return tasks, nil } // ListScheduledTasks retrieves scheduled tasks from the specified queue. // Tasks are sorted by NextProcessAt in ascending order. // // By default, it retrieves the first 30 tasks. func (i *Inspector) ListScheduledTasks(queue string, opts ...ListOption) ([]*TaskInfo, error) { if err := base.ValidateQueueName(queue); err != nil { return nil, fmt.Errorf("asynq: %w", err) } opt := composeListOptions(opts...) pgn := rdb.Pagination{Size: opt.pageSize, Page: opt.pageNum - 1} infos, err := i.rdb.ListScheduled(queue, pgn) switch { case errors.IsQueueNotFound(err): return nil, fmt.Errorf("asynq: %w", ErrQueueNotFound) case err != nil: return nil, fmt.Errorf("asynq: %w", err) } var tasks []*TaskInfo for _, i := range infos { tasks = append(tasks, newTaskInfo( i.Message, i.State, i.NextProcessAt, i.Result, )) } return tasks, nil } // ListRetryTasks retrieves retry tasks from the specified queue. // Tasks are sorted by NextProcessAt in ascending order. // // By default, it retrieves the first 30 tasks. func (i *Inspector) ListRetryTasks(queue string, opts ...ListOption) ([]*TaskInfo, error) { if err := base.ValidateQueueName(queue); err != nil { return nil, fmt.Errorf("asynq: %w", err) } opt := composeListOptions(opts...) pgn := rdb.Pagination{Size: opt.pageSize, Page: opt.pageNum - 1} infos, err := i.rdb.ListRetry(queue, pgn) switch { case errors.IsQueueNotFound(err): return nil, fmt.Errorf("asynq: %w", ErrQueueNotFound) case err != nil: return nil, fmt.Errorf("asynq: %w", err) } var tasks []*TaskInfo for _, i := range infos { tasks = append(tasks, newTaskInfo( i.Message, i.State, i.NextProcessAt, i.Result, )) } return tasks, nil } // ListArchivedTasks retrieves archived tasks from the specified queue. // Tasks are sorted by LastFailedAt in descending order. // // By default, it retrieves the first 30 tasks. func (i *Inspector) ListArchivedTasks(queue string, opts ...ListOption) ([]*TaskInfo, error) { if err := base.ValidateQueueName(queue); err != nil { return nil, fmt.Errorf("asynq: %w", err) } opt := composeListOptions(opts...) pgn := rdb.Pagination{Size: opt.pageSize, Page: opt.pageNum - 1} infos, err := i.rdb.ListArchived(queue, pgn) switch { case errors.IsQueueNotFound(err): return nil, fmt.Errorf("asynq: %w", ErrQueueNotFound) case err != nil: return nil, fmt.Errorf("asynq: %w", err) } var tasks []*TaskInfo for _, i := range infos { tasks = append(tasks, newTaskInfo( i.Message, i.State, i.NextProcessAt, i.Result, )) } return tasks, nil } // ListCompletedTasks retrieves completed tasks from the specified queue. // Tasks are sorted by expiration time (i.e. CompletedAt + Retention) in descending order. // // By default, it retrieves the first 30 tasks. func (i *Inspector) ListCompletedTasks(queue string, opts ...ListOption) ([]*TaskInfo, error) { if err := base.ValidateQueueName(queue); err != nil { return nil, fmt.Errorf("asynq: %w", err) } opt := composeListOptions(opts...) pgn := rdb.Pagination{Size: opt.pageSize, Page: opt.pageNum - 1} infos, err := i.rdb.ListCompleted(queue, pgn) switch { case errors.IsQueueNotFound(err): return nil, fmt.Errorf("asynq: %w", ErrQueueNotFound) case err != nil: return nil, fmt.Errorf("asynq: %w", err) } var tasks []*TaskInfo for _, i := range infos { tasks = append(tasks, newTaskInfo( i.Message, i.State, i.NextProcessAt, i.Result, )) } return tasks, nil } // DeleteAllPendingTasks deletes all pending tasks from the specified queue, // and reports the number tasks deleted. func (i *Inspector) DeleteAllPendingTasks(queue string) (int, error) { if err := base.ValidateQueueName(queue); err != nil { return 0, err } n, err := i.rdb.DeleteAllPendingTasks(queue) return int(n), err } // DeleteAllScheduledTasks deletes all scheduled tasks from the specified queue, // and reports the number tasks deleted. func (i *Inspector) DeleteAllScheduledTasks(queue string) (int, error) { if err := base.ValidateQueueName(queue); err != nil { return 0, err } n, err := i.rdb.DeleteAllScheduledTasks(queue) return int(n), err } // DeleteAllRetryTasks deletes all retry tasks from the specified queue, // and reports the number tasks deleted. func (i *Inspector) DeleteAllRetryTasks(queue string) (int, error) { if err := base.ValidateQueueName(queue); err != nil { return 0, err } n, err := i.rdb.DeleteAllRetryTasks(queue) return int(n), err } // DeleteAllArchivedTasks deletes all archived tasks from the specified queue, // and reports the number tasks deleted. func (i *Inspector) DeleteAllArchivedTasks(queue string) (int, error) { if err := base.ValidateQueueName(queue); err != nil { return 0, err } n, err := i.rdb.DeleteAllArchivedTasks(queue) return int(n), err } // DeleteAllCompletedTasks deletes all completed tasks from the specified queue, // and reports the number tasks deleted. func (i *Inspector) DeleteAllCompletedTasks(queue string) (int, error) { if err := base.ValidateQueueName(queue); err != nil { return 0, err } n, err := i.rdb.DeleteAllCompletedTasks(queue) return int(n), err } // DeleteAllAggregatingTasks deletes all tasks from the specified group, // and reports the number of tasks deleted. func (i *Inspector) DeleteAllAggregatingTasks(queue, group string) (int, error) { if err := base.ValidateQueueName(queue); err != nil { return 0, err } n, err := i.rdb.DeleteAllAggregatingTasks(queue, group) return int(n), err } // UpdateTaskPayload updates a task with the given id from the given queue with given payload. // The task needs to be in scheduled state, // otherwise UpdateTaskPayload will return an error. // // If a queue with the given name doesn't exist, it returns an error wrapping ErrQueueNotFound. // If a task with the given id doesn't exist in the queue, it returns an error wrapping ErrTaskNotFound. // If the task is not in scheduled state, it returns a non-nil error. func (i *Inspector) UpdateTaskPayload(queue, id string, payload []byte) error { if err := base.ValidateQueueName(queue); err != nil { return fmt.Errorf("asynq: %v", err) } err := i.rdb.UpdateTaskPayload(queue, id, payload) switch { case errors.IsQueueNotFound(err): return fmt.Errorf("asynq: %w", ErrQueueNotFound) case errors.IsTaskNotFound(err): return fmt.Errorf("asynq: %w", ErrTaskNotFound) case err != nil: return fmt.Errorf("asynq: %v", err) } return nil } // DeleteTask deletes a task with the given id from the given queue. // The task needs to be in pending, scheduled, retry, or archived state, // otherwise DeleteTask will return an error. // // If a queue with the given name doesn't exist, it returns an error wrapping ErrQueueNotFound. // If a task with the given id doesn't exist in the queue, it returns an error wrapping ErrTaskNotFound. // If the task is in active state, it returns a non-nil error. func (i *Inspector) DeleteTask(queue, id string) error { if err := base.ValidateQueueName(queue); err != nil { return fmt.Errorf("asynq: %w", err) } err := i.rdb.DeleteTask(queue, id) switch { case errors.IsQueueNotFound(err): return fmt.Errorf("asynq: %w", ErrQueueNotFound) case errors.IsTaskNotFound(err): return fmt.Errorf("asynq: %w", ErrTaskNotFound) case err != nil: return fmt.Errorf("asynq: %w", err) } return nil } // RunAllScheduledTasks schedules all scheduled tasks from the given queue to run, // and reports the number of tasks scheduled to run. func (i *Inspector) RunAllScheduledTasks(queue string) (int, error) { if err := base.ValidateQueueName(queue); err != nil { return 0, err } n, err := i.rdb.RunAllScheduledTasks(queue) return int(n), err } // RunAllRetryTasks schedules all retry tasks from the given queue to run, // and reports the number of tasks scheduled to run. func (i *Inspector) RunAllRetryTasks(queue string) (int, error) { if err := base.ValidateQueueName(queue); err != nil { return 0, err } n, err := i.rdb.RunAllRetryTasks(queue) return int(n), err } // RunAllArchivedTasks schedules all archived tasks from the given queue to run, // and reports the number of tasks scheduled to run. func (i *Inspector) RunAllArchivedTasks(queue string) (int, error) { if err := base.ValidateQueueName(queue); err != nil { return 0, err } n, err := i.rdb.RunAllArchivedTasks(queue) return int(n), err } // RunAllAggregatingTasks schedules all tasks from the given grou to run. // and reports the number of tasks scheduled to run. func (i *Inspector) RunAllAggregatingTasks(queue, group string) (int, error) { if err := base.ValidateQueueName(queue); err != nil { return 0, err } n, err := i.rdb.RunAllAggregatingTasks(queue, group) return int(n), err } // RunTask updates the task to pending state given a queue name and task id. // The task needs to be in scheduled, retry, or archived state, otherwise RunTask // will return an error. // // If a queue with the given name doesn't exist, it returns an error wrapping ErrQueueNotFound. // If a task with the given id doesn't exist in the queue, it returns an error wrapping ErrTaskNotFound. // If the task is in pending or active state, it returns a non-nil error. func (i *Inspector) RunTask(queue, id string) error { if err := base.ValidateQueueName(queue); err != nil { return fmt.Errorf("asynq: %w", err) } err := i.rdb.RunTask(queue, id) switch { case errors.IsQueueNotFound(err): return fmt.Errorf("asynq: %w", ErrQueueNotFound) case errors.IsTaskNotFound(err): return fmt.Errorf("asynq: %w", ErrTaskNotFound) case err != nil: return fmt.Errorf("asynq: %w", err) } return nil } // ArchiveAllPendingTasks archives all pending tasks from the given queue, // and reports the number of tasks archived. func (i *Inspector) ArchiveAllPendingTasks(queue string) (int, error) { if err := base.ValidateQueueName(queue); err != nil { return 0, err } n, err := i.rdb.ArchiveAllPendingTasks(queue) return int(n), err } // ArchiveAllScheduledTasks archives all scheduled tasks from the given queue, // and reports the number of tasks archiveed. func (i *Inspector) ArchiveAllScheduledTasks(queue string) (int, error) { if err := base.ValidateQueueName(queue); err != nil { return 0, err } n, err := i.rdb.ArchiveAllScheduledTasks(queue) return int(n), err } // ArchiveAllRetryTasks archives all retry tasks from the given queue, // and reports the number of tasks archiveed. func (i *Inspector) ArchiveAllRetryTasks(queue string) (int, error) { if err := base.ValidateQueueName(queue); err != nil { return 0, err } n, err := i.rdb.ArchiveAllRetryTasks(queue) return int(n), err } // ArchiveAllAggregatingTasks archives all tasks from the given group, // and reports the number of tasks archived. func (i *Inspector) ArchiveAllAggregatingTasks(queue, group string) (int, error) { if err := base.ValidateQueueName(queue); err != nil { return 0, err } n, err := i.rdb.ArchiveAllAggregatingTasks(queue, group) return int(n), err } // ArchiveTask archives a task with the given id in the given queue. // The task needs to be in pending, scheduled, or retry state, otherwise ArchiveTask // will return an error. // // If a queue with the given name doesn't exist, it returns an error wrapping ErrQueueNotFound. // If a task with the given id doesn't exist in the queue, it returns an error wrapping ErrTaskNotFound. // If the task is in already archived, it returns a non-nil error. func (i *Inspector) ArchiveTask(queue, id string) error { if err := base.ValidateQueueName(queue); err != nil { return fmt.Errorf("asynq: err") } err := i.rdb.ArchiveTask(queue, id) switch { case errors.IsQueueNotFound(err): return fmt.Errorf("asynq: %w", ErrQueueNotFound) case errors.IsTaskNotFound(err): return fmt.Errorf("asynq: %w", ErrTaskNotFound) case err != nil: return fmt.Errorf("asynq: %w", err) } return nil } // CancelProcessing sends a signal to cancel processing of the task // given a task id. CancelProcessing is best-effort, which means that it does not // guarantee that the task with the given id will be canceled. The return // value only indicates whether the cancelation signal has been sent. func (i *Inspector) CancelProcessing(id string) error { return i.rdb.PublishCancelation(id) } // PauseQueue pauses task processing on the specified queue. // If the queue is already paused, it will return a non-nil error. func (i *Inspector) PauseQueue(queue string) error { if err := base.ValidateQueueName(queue); err != nil { return err } return i.rdb.Pause(queue) } // UnpauseQueue resumes task processing on the specified queue. // If the queue is not paused, it will return a non-nil error. func (i *Inspector) UnpauseQueue(queue string) error { if err := base.ValidateQueueName(queue); err != nil { return err } return i.rdb.Unpause(queue) } // Servers return a list of running servers' information. func (i *Inspector) Servers() ([]*ServerInfo, error) { servers, err := i.rdb.ListServers() if err != nil { return nil, err } workers, err := i.rdb.ListWorkers() if err != nil { return nil, err } m := make(map[string]*ServerInfo) // ServerInfo keyed by serverID for _, s := range servers { m[s.ServerID] = &ServerInfo{ ID: s.ServerID, Host: s.Host, PID: s.PID, Concurrency: s.Concurrency, Queues: s.Queues, StrictPriority: s.StrictPriority, Started: s.Started, Status: s.Status, ActiveWorkers: make([]*WorkerInfo, 0), } } for _, w := range workers { srvInfo, ok := m[w.ServerID] if !ok { continue } wrkInfo := &WorkerInfo{ TaskID: w.ID, TaskType: w.Type, TaskPayload: w.Payload, Queue: w.Queue, Started: w.Started, Deadline: w.Deadline, } srvInfo.ActiveWorkers = append(srvInfo.ActiveWorkers, wrkInfo) } var out []*ServerInfo for _, srvInfo := range m { out = append(out, srvInfo) } return out, nil } // ServerInfo describes a running Server instance. type ServerInfo struct { // Unique Identifier for the server. ID string // Host machine on which the server is running. Host string // PID of the process in which the server is running. PID int // Server configuration details. // See Config doc for field descriptions. Concurrency int Queues map[string]int StrictPriority bool // Time the server started. Started time.Time // Status indicates the status of the server. // TODO: Update comment with more details. Status string // A List of active workers currently processing tasks. ActiveWorkers []*WorkerInfo } // WorkerInfo describes a running worker processing a task. type WorkerInfo struct { // ID of the task the worker is processing. TaskID string // Type of the task the worker is processing. TaskType string // Payload of the task the worker is processing. TaskPayload []byte // Queue from which the worker got its task. Queue string // Time the worker started processing the task. Started time.Time // Time the worker needs to finish processing the task by. Deadline time.Time } // ClusterKeySlot returns an integer identifying the hash slot the given queue hashes to. func (i *Inspector) ClusterKeySlot(queue string) (int64, error) { return i.rdb.ClusterKeySlot(queue) } // ClusterNode describes a node in redis cluster. type ClusterNode struct { // Node ID in the cluster. ID string // Address of the node. Addr string } // ClusterNodes returns a list of nodes the given queue belongs to. // // Only relevant if task queues are stored in redis cluster. func (i *Inspector) ClusterNodes(queue string) ([]*ClusterNode, error) { nodes, err := i.rdb.ClusterNodes(queue) if err != nil { return nil, err } var res []*ClusterNode for _, node := range nodes { res = append(res, &ClusterNode{ID: node.ID, Addr: node.Addr}) } return res, nil } // SchedulerEntry holds information about a periodic task registered with a scheduler. type SchedulerEntry struct { // Identifier of this entry. ID string // Spec describes the schedule of this entry. Spec string // Periodic Task registered for this entry. Task *Task // Opts is the options for the periodic task. Opts []Option // Next shows the next time the task will be enqueued. Next time.Time // Prev shows the last time the task was enqueued. // Zero time if task was never enqueued. Prev time.Time } // SchedulerEntries returns a list of all entries registered with // currently running schedulers. func (i *Inspector) SchedulerEntries() ([]*SchedulerEntry, error) { var entries []*SchedulerEntry res, err := i.rdb.ListSchedulerEntries() if err != nil { return nil, err } for _, e := range res { task := NewTask(e.Type, e.Payload) var opts []Option for _, s := range e.Opts { if o, err := parseOption(s); err == nil { // ignore bad data opts = append(opts, o) } } entries = append(entries, &SchedulerEntry{ ID: e.ID, Spec: e.Spec, Task: task, Opts: opts, Next: e.Next, Prev: e.Prev, }) } return entries, nil } // parseOption interprets a string s as an Option and returns the Option if parsing is successful, // otherwise returns non-nil error. func parseOption(s string) (Option, error) { fn, arg := parseOptionFunc(s), parseOptionArg(s) switch fn { case "Queue": queue, err := strconv.Unquote(arg) if err != nil { return nil, err } return Queue(queue), nil case "MaxRetry": n, err := strconv.Atoi(arg) if err != nil { return nil, err } return MaxRetry(n), nil case "Timeout": d, err := time.ParseDuration(arg) if err != nil { return nil, err } return Timeout(d), nil case "Deadline": t, err := time.Parse(time.UnixDate, arg) if err != nil { return nil, err } return Deadline(t), nil case "Unique": d, err := time.ParseDuration(arg) if err != nil { return nil, err } return Unique(d), nil case "ProcessAt": t, err := time.Parse(time.UnixDate, arg) if err != nil { return nil, err } return ProcessAt(t), nil case "ProcessIn": d, err := time.ParseDuration(arg) if err != nil { return nil, err } return ProcessIn(d), nil case "Retention": d, err := time.ParseDuration(arg) if err != nil { return nil, err } return Retention(d), nil default: return nil, fmt.Errorf("cannot not parse option string %q", s) } } func parseOptionFunc(s string) string { i := strings.Index(s, "(") return s[:i] } func parseOptionArg(s string) string { i := strings.Index(s, "(") if i >= 0 { j := strings.Index(s, ")") if j > i { return s[i+1 : j] } } return "" } // SchedulerEnqueueEvent holds information about an enqueue event by a scheduler. type SchedulerEnqueueEvent struct { // ID of the task that was enqueued. TaskID string // Time the task was enqueued. EnqueuedAt time.Time } // ListSchedulerEnqueueEvents retrieves a list of enqueue events from the specified scheduler entry. // // By default, it retrieves the first 30 tasks. func (i *Inspector) ListSchedulerEnqueueEvents(entryID string, opts ...ListOption) ([]*SchedulerEnqueueEvent, error) { opt := composeListOptions(opts...) pgn := rdb.Pagination{Size: opt.pageSize, Page: opt.pageNum - 1} data, err := i.rdb.ListSchedulerEnqueueEvents(entryID, pgn) if err != nil { return nil, err } var events []*SchedulerEnqueueEvent for _, e := range data { events = append(events, &SchedulerEnqueueEvent{TaskID: e.TaskID, EnqueuedAt: e.EnqueuedAt}) } return events, nil } ================================================ FILE: inspector_test.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "context" "errors" "fmt" "sort" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/google/uuid" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/rdb" h "github.com/hibiken/asynq/internal/testutil" "github.com/hibiken/asynq/internal/timeutil" "github.com/redis/go-redis/v9" ) func testInspectorQueues(t *testing.T, inspector *Inspector, r redis.UniversalClient) { tests := []struct { queues []string }{ {queues: []string{"default"}}, {queues: []string{"custom1", "custom2"}}, {queues: []string{"default", "custom1", "custom2"}}, {queues: []string{}}, } for _, tc := range tests { h.FlushDB(t, r) for _, qname := range tc.queues { if err := r.SAdd(context.Background(), base.AllQueues, qname).Err(); err != nil { t.Fatalf("could not initialize all queue set: %v", err) } } got, err := inspector.Queues() if err != nil { t.Errorf("Queues() returned an error: %v", err) continue } if diff := cmp.Diff(tc.queues, got, h.SortStringSliceOpt); diff != "" { t.Errorf("Queues() = %v, want %v; (-want, +got)\n%s", got, tc.queues, diff) } } } func TestInspectorQueues(t *testing.T) { r := setup(t) defer r.Close() inspector := NewInspector(getRedisConnOpt(t)) testInspectorQueues(t, inspector, r) } func TestInspectorFromRedisClientQueues(t *testing.T) { r := setup(t) defer r.Close() redisClient := getRedisConnOpt(t).MakeRedisClient().(redis.UniversalClient) inspector := NewInspectorFromRedisClient(redisClient) testInspectorQueues(t, inspector, r) } func TestInspectorDeleteQueue(t *testing.T) { r := setup(t) defer r.Close() inspector := NewInspector(getRedisConnOpt(t)) defer inspector.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") m4 := h.NewTaskMessageWithQueue("task4", nil, "custom") tests := []struct { pending map[string][]*base.TaskMessage active map[string][]*base.TaskMessage scheduled map[string][]base.Z retry map[string][]base.Z archived map[string][]base.Z qname string // queue to remove force bool }{ { pending: map[string][]*base.TaskMessage{ "default": {m1, m2}, "custom": {}, }, active: map[string][]*base.TaskMessage{ "default": {}, "custom": {}, }, scheduled: map[string][]base.Z{ "default": {}, "custom": {}, }, retry: map[string][]base.Z{ "default": {}, "custom": {}, }, archived: map[string][]base.Z{ "default": {}, "custom": {}, }, qname: "custom", force: false, }, { pending: map[string][]*base.TaskMessage{ "default": {m1, m2}, "custom": {m3}, }, active: map[string][]*base.TaskMessage{ "default": {}, "custom": {}, }, scheduled: map[string][]base.Z{ "default": {}, "custom": {{Message: m4, Score: time.Now().Unix()}}, }, retry: map[string][]base.Z{ "default": {}, "custom": {}, }, archived: map[string][]base.Z{ "default": {}, "custom": {}, }, qname: "custom", force: true, // allow removing non-empty queue }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllPendingQueues(t, r, tc.pending) h.SeedAllActiveQueues(t, r, tc.active) h.SeedAllScheduledQueues(t, r, tc.scheduled) h.SeedAllRetryQueues(t, r, tc.retry) h.SeedAllArchivedQueues(t, r, tc.archived) err := inspector.DeleteQueue(tc.qname, tc.force) if err != nil { t.Errorf("DeleteQueue(%q, %t) = %v, want nil", tc.qname, tc.force, err) continue } if r.SIsMember(context.Background(), base.AllQueues, tc.qname).Val() { t.Errorf("%q is a member of %q", tc.qname, base.AllQueues) } } } func TestInspectorDeleteQueueErrorQueueNotEmpty(t *testing.T) { r := setup(t) defer r.Close() inspector := NewInspector(getRedisConnOpt(t)) defer inspector.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") m4 := h.NewTaskMessageWithQueue("task4", nil, "custom") tests := []struct { pending map[string][]*base.TaskMessage active map[string][]*base.TaskMessage scheduled map[string][]base.Z retry map[string][]base.Z archived map[string][]base.Z qname string // queue to remove force bool }{ { pending: map[string][]*base.TaskMessage{ "default": {m1, m2}, }, active: map[string][]*base.TaskMessage{ "default": {m3, m4}, }, scheduled: map[string][]base.Z{ "default": {}, }, retry: map[string][]base.Z{ "default": {}, }, archived: map[string][]base.Z{ "default": {}, }, qname: "default", force: false, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllPendingQueues(t, r, tc.pending) h.SeedAllActiveQueues(t, r, tc.active) h.SeedAllScheduledQueues(t, r, tc.scheduled) h.SeedAllRetryQueues(t, r, tc.retry) h.SeedAllArchivedQueues(t, r, tc.archived) err := inspector.DeleteQueue(tc.qname, tc.force) if !errors.Is(err, ErrQueueNotEmpty) { t.Errorf("DeleteQueue(%v, %t) did not return ErrQueueNotEmpty", tc.qname, tc.force) } } } func TestInspectorDeleteQueueErrorQueueNotFound(t *testing.T) { r := setup(t) defer r.Close() inspector := NewInspector(getRedisConnOpt(t)) defer inspector.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") m4 := h.NewTaskMessageWithQueue("task4", nil, "custom") tests := []struct { pending map[string][]*base.TaskMessage active map[string][]*base.TaskMessage scheduled map[string][]base.Z retry map[string][]base.Z archived map[string][]base.Z qname string // queue to remove force bool }{ { pending: map[string][]*base.TaskMessage{ "default": {m1, m2}, }, active: map[string][]*base.TaskMessage{ "default": {m3, m4}, }, scheduled: map[string][]base.Z{ "default": {}, }, retry: map[string][]base.Z{ "default": {}, }, archived: map[string][]base.Z{ "default": {}, }, qname: "nonexistent", force: false, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllPendingQueues(t, r, tc.pending) h.SeedAllActiveQueues(t, r, tc.active) h.SeedAllScheduledQueues(t, r, tc.scheduled) h.SeedAllRetryQueues(t, r, tc.retry) h.SeedAllArchivedQueues(t, r, tc.archived) err := inspector.DeleteQueue(tc.qname, tc.force) if !errors.Is(err, ErrQueueNotFound) { t.Errorf("DeleteQueue(%v, %t) did not return ErrQueueNotFound", tc.qname, tc.force) } } } func TestInspectorGetQueueInfo(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessage("task3", nil) m4 := h.NewTaskMessage("task4", nil) m5 := h.NewTaskMessageWithQueue("task5", nil, "critical") m6 := h.NewTaskMessageWithQueue("task6", nil, "low") now := time.Now() timeCmpOpt := cmpopts.EquateApproxTime(time.Second) ignoreMemUsg := cmpopts.IgnoreFields(QueueInfo{}, "MemoryUsage") inspector := NewInspector(getRedisConnOpt(t)) inspector.rdb.SetClock(timeutil.NewSimulatedClock(now)) tests := []struct { pending map[string][]*base.TaskMessage active map[string][]*base.TaskMessage scheduled map[string][]base.Z retry map[string][]base.Z archived map[string][]base.Z completed map[string][]base.Z processed map[string]int failed map[string]int processedTotal map[string]int failedTotal map[string]int oldestPendingMessageEnqueueTime map[string]time.Time qname string want *QueueInfo }{ { pending: map[string][]*base.TaskMessage{ "default": {m1}, "critical": {m5}, "low": {m6}, }, active: map[string][]*base.TaskMessage{ "default": {m2}, "critical": {}, "low": {}, }, scheduled: map[string][]base.Z{ "default": { {Message: m3, Score: now.Add(time.Hour).Unix()}, {Message: m4, Score: now.Unix()}, }, "critical": {}, "low": {}, }, retry: map[string][]base.Z{ "default": {}, "critical": {}, "low": {}, }, archived: map[string][]base.Z{ "default": {}, "critical": {}, "low": {}, }, completed: map[string][]base.Z{ "default": {}, "critical": {}, "low": {}, }, processed: map[string]int{ "default": 120, "critical": 100, "low": 42, }, failed: map[string]int{ "default": 2, "critical": 0, "low": 5, }, processedTotal: map[string]int{ "default": 11111, "critical": 22222, "low": 33333, }, failedTotal: map[string]int{ "default": 111, "critical": 222, "low": 333, }, oldestPendingMessageEnqueueTime: map[string]time.Time{ "default": now.Add(-15 * time.Second), "critical": now.Add(-200 * time.Millisecond), "low": now.Add(-30 * time.Second), }, qname: "default", want: &QueueInfo{ Queue: "default", Latency: 15 * time.Second, Size: 4, Pending: 1, Active: 1, Scheduled: 2, Retry: 0, Archived: 0, Completed: 0, Processed: 120, Failed: 2, ProcessedTotal: 11111, FailedTotal: 111, Paused: false, Timestamp: now, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllPendingQueues(t, r, tc.pending) h.SeedAllActiveQueues(t, r, tc.active) h.SeedAllScheduledQueues(t, r, tc.scheduled) h.SeedAllRetryQueues(t, r, tc.retry) h.SeedAllArchivedQueues(t, r, tc.archived) h.SeedAllCompletedQueues(t, r, tc.completed) ctx := context.Background() for qname, n := range tc.processed { r.Set(ctx, base.ProcessedKey(qname, now), n, 0) } for qname, n := range tc.failed { r.Set(ctx, base.FailedKey(qname, now), n, 0) } for qname, n := range tc.processedTotal { r.Set(ctx, base.ProcessedTotalKey(qname), n, 0) } for qname, n := range tc.failedTotal { r.Set(ctx, base.FailedTotalKey(qname), n, 0) } for qname, enqueueTime := range tc.oldestPendingMessageEnqueueTime { if enqueueTime.IsZero() { continue } oldestPendingMessageID := r.LRange(ctx, base.PendingKey(qname), -1, -1).Val()[0] // get the right most msg in the list r.HSet(ctx, base.TaskKey(qname, oldestPendingMessageID), "pending_since", enqueueTime.UnixNano()) } got, err := inspector.GetQueueInfo(tc.qname) if err != nil { t.Errorf("r.GetQueueInfo(%q) = %v, %v, want %v, nil", tc.qname, got, err, tc.want) continue } if diff := cmp.Diff(tc.want, got, timeCmpOpt, ignoreMemUsg); diff != "" { t.Errorf("r.GetQueueInfo(%q) = %v, %v, want %v, nil; (-want, +got)\n%s", tc.qname, got, err, tc.want, diff) continue } } } func TestInspectorHistory(t *testing.T) { r := setup(t) defer r.Close() now := time.Now().UTC() inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { qname string // queue of interest n int // number of days }{ {"default", 90}, {"custom", 7}, {"default", 1}, } for _, tc := range tests { h.FlushDB(t, r) r.SAdd(context.Background(), base.AllQueues, tc.qname) // populate last n days data for i := 0; i < tc.n; i++ { ts := now.Add(-time.Duration(i) * 24 * time.Hour) processedKey := base.ProcessedKey(tc.qname, ts) failedKey := base.FailedKey(tc.qname, ts) r.Set(context.Background(), processedKey, (i+1)*1000, 0) r.Set(context.Background(), failedKey, (i+1)*10, 0) } got, err := inspector.History(tc.qname, tc.n) if err != nil { t.Errorf("Inspector.History(%q, %d) returned error: %v", tc.qname, tc.n, err) continue } if len(got) != tc.n { t.Errorf("Inspector.History(%q, %d) returned %d daily stats, want %d", tc.qname, tc.n, len(got), tc.n) continue } for i := 0; i < tc.n; i++ { want := &DailyStats{ Queue: tc.qname, Processed: (i + 1) * 1000, Failed: (i + 1) * 10, Date: now.Add(-time.Duration(i) * 24 * time.Hour), } // Allow 2 seconds difference in timestamp. timeCmpOpt := cmpopts.EquateApproxTime(2 * time.Second) if diff := cmp.Diff(want, got[i], timeCmpOpt); diff != "" { t.Errorf("Inspector.History %d days ago data; got %+v, want %+v; (-want,+got):\n%s", i, got[i], want, diff) } } } } func createPendingTask(msg *base.TaskMessage) *TaskInfo { return newTaskInfo(msg, base.TaskStatePending, time.Now(), nil) } func TestInspectorGetTaskInfo(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessageWithQueue("task1", nil, "default") m2 := h.NewTaskMessageWithQueue("task2", nil, "default") m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") m4 := h.NewTaskMessageWithQueue("task4", nil, "custom") m5 := h.NewTaskMessageWithQueue("task5", nil, "custom") now := time.Now() fiveMinsFromNow := now.Add(5 * time.Minute) oneHourFromNow := now.Add(1 * time.Hour) twoHoursAgo := now.Add(-2 * time.Hour) fixtures := struct { active map[string][]*base.TaskMessage pending map[string][]*base.TaskMessage scheduled map[string][]base.Z retry map[string][]base.Z archived map[string][]base.Z }{ active: map[string][]*base.TaskMessage{ "default": {m1}, "custom": {}, }, pending: map[string][]*base.TaskMessage{ "default": {}, "custom": {m5}, }, scheduled: map[string][]base.Z{ "default": {{Message: m2, Score: fiveMinsFromNow.Unix()}}, "custom": {}, }, retry: map[string][]base.Z{ "default": {}, "custom": {{Message: m3, Score: oneHourFromNow.Unix()}}, }, archived: map[string][]base.Z{ "default": {}, "custom": {{Message: m4, Score: twoHoursAgo.Unix()}}, }, } h.SeedAllActiveQueues(t, r, fixtures.active) h.SeedAllPendingQueues(t, r, fixtures.pending) h.SeedAllScheduledQueues(t, r, fixtures.scheduled) h.SeedAllRetryQueues(t, r, fixtures.retry) h.SeedAllArchivedQueues(t, r, fixtures.archived) tests := []struct { qname string id string want *TaskInfo }{ { qname: "default", id: m1.ID, want: newTaskInfo( m1, base.TaskStateActive, time.Time{}, // zero value for n/a nil, ), }, { qname: "default", id: m2.ID, want: newTaskInfo( m2, base.TaskStateScheduled, fiveMinsFromNow, nil, ), }, { qname: "custom", id: m3.ID, want: newTaskInfo( m3, base.TaskStateRetry, oneHourFromNow, nil, ), }, { qname: "custom", id: m4.ID, want: newTaskInfo( m4, base.TaskStateArchived, time.Time{}, // zero value for n/a nil, ), }, { qname: "custom", id: m5.ID, want: newTaskInfo( m5, base.TaskStatePending, now, nil, ), }, } inspector := NewInspector(getRedisConnOpt(t)) for _, tc := range tests { got, err := inspector.GetTaskInfo(tc.qname, tc.id) if err != nil { t.Errorf("GetTaskInfo(%q, %q) returned error: %v", tc.qname, tc.id, err) continue } cmpOpts := []cmp.Option{ cmp.AllowUnexported(TaskInfo{}), cmpopts.EquateApproxTime(2 * time.Second), } if diff := cmp.Diff(tc.want, got, cmpOpts...); diff != "" { t.Errorf("GetTaskInfo(%q, %q) = %v, want %v; (-want, +got)\n%s", tc.qname, tc.id, got, tc.want, diff) } } } func TestInspectorGetTaskInfoError(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessageWithQueue("task1", nil, "default") m2 := h.NewTaskMessageWithQueue("task2", nil, "default") m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") m4 := h.NewTaskMessageWithQueue("task4", nil, "custom") m5 := h.NewTaskMessageWithQueue("task5", nil, "custom") now := time.Now() fiveMinsFromNow := now.Add(5 * time.Minute) oneHourFromNow := now.Add(1 * time.Hour) twoHoursAgo := now.Add(-2 * time.Hour) fixtures := struct { active map[string][]*base.TaskMessage pending map[string][]*base.TaskMessage scheduled map[string][]base.Z retry map[string][]base.Z archived map[string][]base.Z }{ active: map[string][]*base.TaskMessage{ "default": {m1}, "custom": {}, }, pending: map[string][]*base.TaskMessage{ "default": {}, "custom": {m5}, }, scheduled: map[string][]base.Z{ "default": {{Message: m2, Score: fiveMinsFromNow.Unix()}}, "custom": {}, }, retry: map[string][]base.Z{ "default": {}, "custom": {{Message: m3, Score: oneHourFromNow.Unix()}}, }, archived: map[string][]base.Z{ "default": {}, "custom": {{Message: m4, Score: twoHoursAgo.Unix()}}, }, } h.SeedAllActiveQueues(t, r, fixtures.active) h.SeedAllPendingQueues(t, r, fixtures.pending) h.SeedAllScheduledQueues(t, r, fixtures.scheduled) h.SeedAllRetryQueues(t, r, fixtures.retry) h.SeedAllArchivedQueues(t, r, fixtures.archived) tests := []struct { qname string id string wantErr error }{ { qname: "nonexistent", id: m1.ID, wantErr: ErrQueueNotFound, }, { qname: "default", id: uuid.NewString(), wantErr: ErrTaskNotFound, }, } inspector := NewInspector(getRedisConnOpt(t)) for _, tc := range tests { info, err := inspector.GetTaskInfo(tc.qname, tc.id) if info != nil { t.Errorf("GetTaskInfo(%q, %q) returned info: %v", tc.qname, tc.id, info) } if !errors.Is(err, tc.wantErr) { t.Errorf("GetTaskInfo(%q, %q) returned unexpected error: %v, want %v", tc.qname, tc.id, err, tc.wantErr) } } } func TestInspectorListPendingTasks(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "critical") m4 := h.NewTaskMessageWithQueue("task4", nil, "low") inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { desc string pending map[string][]*base.TaskMessage qname string want []*TaskInfo }{ { desc: "with default queue", pending: map[string][]*base.TaskMessage{ "default": {m1, m2}, }, qname: "default", want: []*TaskInfo{ createPendingTask(m1), createPendingTask(m2), }, }, { desc: "with named queue", pending: map[string][]*base.TaskMessage{ "default": {m1, m2}, "critical": {m3}, "low": {m4}, }, qname: "critical", want: []*TaskInfo{ createPendingTask(m3), }, }, { desc: "with empty queue", pending: map[string][]*base.TaskMessage{ "default": {}, }, qname: "default", want: []*TaskInfo(nil), }, } for _, tc := range tests { h.FlushDB(t, r) for q, msgs := range tc.pending { h.SeedPendingQueue(t, r, msgs, q) } got, err := inspector.ListPendingTasks(tc.qname) if err != nil { t.Errorf("%s; ListPendingTasks(%q) returned error: %v", tc.desc, tc.qname, err) continue } cmpOpts := []cmp.Option{ cmpopts.EquateApproxTime(2 * time.Second), cmp.AllowUnexported(TaskInfo{}), } if diff := cmp.Diff(tc.want, got, cmpOpts...); diff != "" { t.Errorf("%s; ListPendingTasks(%q) = %v, want %v; (-want,+got)\n%s", tc.desc, tc.qname, got, tc.want, diff) } } } func newOrphanedTaskInfo(msg *base.TaskMessage) *TaskInfo { info := newTaskInfo(msg, base.TaskStateActive, time.Time{}, nil) info.IsOrphaned = true return info } func TestInspectorListActiveTasks(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") m4 := h.NewTaskMessageWithQueue("task4", nil, "custom") inspector := NewInspector(getRedisConnOpt(t)) now := time.Now() tests := []struct { desc string active map[string][]*base.TaskMessage lease map[string][]base.Z qname string want []*TaskInfo }{ { desc: "with a few active tasks", active: map[string][]*base.TaskMessage{ "default": {m1, m2}, "custom": {m3, m4}, }, lease: map[string][]base.Z{ "default": { {Message: m1, Score: now.Add(20 * time.Second).Unix()}, {Message: m2, Score: now.Add(20 * time.Second).Unix()}, }, "custom": { {Message: m3, Score: now.Add(20 * time.Second).Unix()}, {Message: m4, Score: now.Add(20 * time.Second).Unix()}, }, }, qname: "custom", want: []*TaskInfo{ newTaskInfo(m3, base.TaskStateActive, time.Time{}, nil), newTaskInfo(m4, base.TaskStateActive, time.Time{}, nil), }, }, { desc: "with an orphaned task", active: map[string][]*base.TaskMessage{ "default": {m1, m2}, "custom": {m3, m4}, }, lease: map[string][]base.Z{ "default": { {Message: m1, Score: now.Add(20 * time.Second).Unix()}, {Message: m2, Score: now.Add(-10 * time.Second).Unix()}, // orphaned task }, "custom": { {Message: m3, Score: now.Add(20 * time.Second).Unix()}, {Message: m4, Score: now.Add(20 * time.Second).Unix()}, }, }, qname: "default", want: []*TaskInfo{ newTaskInfo(m1, base.TaskStateActive, time.Time{}, nil), newOrphanedTaskInfo(m2), }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllActiveQueues(t, r, tc.active) h.SeedAllLease(t, r, tc.lease) got, err := inspector.ListActiveTasks(tc.qname) if err != nil { t.Errorf("%s; ListActiveTasks(%q) returned error: %v", tc.qname, tc.desc, err) continue } if diff := cmp.Diff(tc.want, got, cmp.AllowUnexported(TaskInfo{})); diff != "" { t.Errorf("%s; ListActiveTask(%q) = %v, want %v; (-want,+got)\n%s", tc.desc, tc.qname, got, tc.want, diff) } } } func createScheduledTask(z base.Z) *TaskInfo { return newTaskInfo( z.Message, base.TaskStateScheduled, time.Unix(z.Score, 0), nil, ) } func TestInspectorListScheduledTasks(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessage("task3", nil) m4 := h.NewTaskMessageWithQueue("task4", nil, "custom") now := time.Now() z1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()} z2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()} z3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()} z4 := base.Z{Message: m4, Score: now.Add(2 * time.Minute).Unix()} inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { desc string scheduled map[string][]base.Z qname string want []*TaskInfo }{ { desc: "with a few scheduled tasks", scheduled: map[string][]base.Z{ "default": {z1, z2, z3}, "custom": {z4}, }, qname: "default", // Should be sorted by NextProcessAt. want: []*TaskInfo{ createScheduledTask(z3), createScheduledTask(z1), createScheduledTask(z2), }, }, { desc: "with empty scheduled queue", scheduled: map[string][]base.Z{ "default": {}, }, qname: "default", want: []*TaskInfo(nil), }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllScheduledQueues(t, r, tc.scheduled) got, err := inspector.ListScheduledTasks(tc.qname) if err != nil { t.Errorf("%s; ListScheduledTasks(%q) returned error: %v", tc.desc, tc.qname, err) continue } if diff := cmp.Diff(tc.want, got, cmp.AllowUnexported(TaskInfo{})); diff != "" { t.Errorf("%s; ListScheduledTask(%q) = %v, want %v; (-want,+got)\n%s", tc.desc, tc.qname, got, tc.want, diff) } } } func createRetryTask(z base.Z) *TaskInfo { return newTaskInfo( z.Message, base.TaskStateRetry, time.Unix(z.Score, 0), nil, ) } func TestInspectorListRetryTasks(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessage("task3", nil) m4 := h.NewTaskMessageWithQueue("task4", nil, "custom") now := time.Now() z1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()} z2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()} z3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()} z4 := base.Z{Message: m4, Score: now.Add(2 * time.Minute).Unix()} inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { desc string retry map[string][]base.Z qname string want []*TaskInfo }{ { desc: "with a few retry tasks", retry: map[string][]base.Z{ "default": {z1, z2, z3}, "custom": {z4}, }, qname: "default", // Should be sorted by NextProcessAt. want: []*TaskInfo{ createRetryTask(z3), createRetryTask(z1), createRetryTask(z2), }, }, { desc: "with empty retry queue", retry: map[string][]base.Z{ "default": {}, }, qname: "default", want: []*TaskInfo(nil), }, // TODO(hibiken): ErrQueueNotFound when queue doesn't exist } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllRetryQueues(t, r, tc.retry) got, err := inspector.ListRetryTasks(tc.qname) if err != nil { t.Errorf("%s; ListRetryTasks(%q) returned error: %v", tc.desc, tc.qname, err) continue } if diff := cmp.Diff(tc.want, got, cmp.AllowUnexported(TaskInfo{})); diff != "" { t.Errorf("%s; ListRetryTask(%q) = %v, want %v; (-want,+got)\n%s", tc.desc, tc.qname, got, tc.want, diff) } } } func createArchivedTask(z base.Z) *TaskInfo { return newTaskInfo( z.Message, base.TaskStateArchived, time.Time{}, // zero value for n/a nil, ) } func TestInspectorListArchivedTasks(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessage("task3", nil) m4 := h.NewTaskMessageWithQueue("task4", nil, "custom") now := time.Now() z1 := base.Z{Message: m1, Score: now.Add(-5 * time.Minute).Unix()} z2 := base.Z{Message: m2, Score: now.Add(-15 * time.Minute).Unix()} z3 := base.Z{Message: m3, Score: now.Add(-2 * time.Minute).Unix()} z4 := base.Z{Message: m4, Score: now.Add(-2 * time.Minute).Unix()} inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { desc string archived map[string][]base.Z qname string want []*TaskInfo }{ { desc: "with a few archived tasks", archived: map[string][]base.Z{ "default": {z1, z2, z3}, "custom": {z4}, }, qname: "default", // Should be sorted by LastFailedAt. want: []*TaskInfo{ createArchivedTask(z2), createArchivedTask(z1), createArchivedTask(z3), }, }, { desc: "with empty archived queue", archived: map[string][]base.Z{ "default": {}, }, qname: "default", want: []*TaskInfo(nil), }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllArchivedQueues(t, r, tc.archived) got, err := inspector.ListArchivedTasks(tc.qname) if err != nil { t.Errorf("%s; ListArchivedTasks(%q) returned error: %v", tc.desc, tc.qname, err) continue } if diff := cmp.Diff(tc.want, got, cmp.AllowUnexported(TaskInfo{})); diff != "" { t.Errorf("%s; ListArchivedTask(%q) = %v, want %v; (-want,+got)\n%s", tc.desc, tc.qname, got, tc.want, diff) } } } func newCompletedTaskMessage(typename, qname string, retention time.Duration, completedAt time.Time) *base.TaskMessage { msg := h.NewTaskMessageWithQueue(typename, nil, qname) msg.Retention = int64(retention.Seconds()) msg.CompletedAt = completedAt.Unix() return msg } func createCompletedTask(z base.Z) *TaskInfo { return newTaskInfo( z.Message, base.TaskStateCompleted, time.Time{}, // zero value for n/a nil, // TODO: Test with result data ) } func TestInspectorListCompletedTasks(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() m1 := newCompletedTaskMessage("task1", "default", 1*time.Hour, now.Add(-3*time.Minute)) // Expires in 57 mins m2 := newCompletedTaskMessage("task2", "default", 30*time.Minute, now.Add(-10*time.Minute)) // Expires in 20 mins m3 := newCompletedTaskMessage("task3", "default", 2*time.Hour, now.Add(-30*time.Minute)) // Expires in 90 mins m4 := newCompletedTaskMessage("task4", "custom", 15*time.Minute, now.Add(-2*time.Minute)) // Expires in 13 mins z1 := base.Z{Message: m1, Score: m1.CompletedAt + m1.Retention} z2 := base.Z{Message: m2, Score: m2.CompletedAt + m2.Retention} z3 := base.Z{Message: m3, Score: m3.CompletedAt + m3.Retention} z4 := base.Z{Message: m4, Score: m4.CompletedAt + m4.Retention} inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { desc string completed map[string][]base.Z qname string want []*TaskInfo }{ { desc: "with a few completed tasks", completed: map[string][]base.Z{ "default": {z1, z2, z3}, "custom": {z4}, }, qname: "default", // Should be sorted by expiration time (CompletedAt + Retention). want: []*TaskInfo{ createCompletedTask(z2), createCompletedTask(z1), createCompletedTask(z3), }, }, { desc: "with empty completed queue", completed: map[string][]base.Z{ "default": {}, }, qname: "default", want: []*TaskInfo(nil), }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllCompletedQueues(t, r, tc.completed) got, err := inspector.ListCompletedTasks(tc.qname) if err != nil { t.Errorf("%s; ListCompletedTasks(%q) returned error: %v", tc.desc, tc.qname, err) continue } if diff := cmp.Diff(tc.want, got, cmp.AllowUnexported(TaskInfo{})); diff != "" { t.Errorf("%s; ListCompletedTasks(%q) = %v, want %v; (-want,+got)\n%s", tc.desc, tc.qname, got, tc.want, diff) } } } func TestInspectorListAggregatingTasks(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() m1 := h.NewTaskMessageBuilder().SetType("task1").SetQueue("default").SetGroup("group1").Build() m2 := h.NewTaskMessageBuilder().SetType("task2").SetQueue("default").SetGroup("group1").Build() m3 := h.NewTaskMessageBuilder().SetType("task3").SetQueue("default").SetGroup("group1").Build() m4 := h.NewTaskMessageBuilder().SetType("task4").SetQueue("default").SetGroup("group2").Build() m5 := h.NewTaskMessageBuilder().SetType("task5").SetQueue("custom").SetGroup("group1").Build() inspector := NewInspector(getRedisConnOpt(t)) fxt := struct { tasks []*h.TaskSeedData allQueues []string allGroups map[string][]string groups map[string][]redis.Z }{ tasks: []*h.TaskSeedData{ {Msg: m1, State: base.TaskStateAggregating}, {Msg: m2, State: base.TaskStateAggregating}, {Msg: m3, State: base.TaskStateAggregating}, {Msg: m4, State: base.TaskStateAggregating}, {Msg: m5, State: base.TaskStateAggregating}, }, allQueues: []string{"default", "custom"}, allGroups: map[string][]string{ base.AllGroups("default"): {"group1", "group2"}, base.AllGroups("custom"): {"group1"}, }, groups: map[string][]redis.Z{ base.GroupKey("default", "group1"): { {Member: m1.ID, Score: float64(now.Add(-30 * time.Second).Unix())}, {Member: m2.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, {Member: m3.ID, Score: float64(now.Add(-10 * time.Second).Unix())}, }, base.GroupKey("default", "group2"): { {Member: m4.ID, Score: float64(now.Add(-30 * time.Second).Unix())}, }, base.GroupKey("custom", "group1"): { {Member: m5.ID, Score: float64(now.Add(-30 * time.Second).Unix())}, }, }, } tests := []struct { desc string qname string gname string want []*TaskInfo }{ { desc: "default queue group1", qname: "default", gname: "group1", want: []*TaskInfo{ createAggregatingTaskInfo(m1), createAggregatingTaskInfo(m2), createAggregatingTaskInfo(m3), }, }, { desc: "custom queue group1", qname: "custom", gname: "group1", want: []*TaskInfo{ createAggregatingTaskInfo(m5), }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedTasks(t, r, fxt.tasks) h.SeedRedisSet(t, r, base.AllQueues, fxt.allQueues) h.SeedRedisSets(t, r, fxt.allGroups) h.SeedRedisZSets(t, r, fxt.groups) t.Run(tc.desc, func(t *testing.T) { got, err := inspector.ListAggregatingTasks(tc.qname, tc.gname) if err != nil { t.Fatalf("ListAggregatingTasks returned error: %v", err) } cmpOpts := []cmp.Option{ cmpopts.EquateApproxTime(2 * time.Second), cmp.AllowUnexported(TaskInfo{}), } if diff := cmp.Diff(tc.want, got, cmpOpts...); diff != "" { t.Errorf("ListAggregatingTasks = %v, want = %v; (-want,+got)\n%s", got, tc.want, diff) } }) } } func createAggregatingTaskInfo(msg *base.TaskMessage) *TaskInfo { return newTaskInfo(msg, base.TaskStateAggregating, time.Time{}, nil) } func TestInspectorListPagination(t *testing.T) { // Create 100 tasks. var msgs []*base.TaskMessage for i := 0; i <= 99; i++ { msgs = append(msgs, h.NewTaskMessage(fmt.Sprintf("task%d", i), nil)) } r := setup(t) defer r.Close() h.SeedPendingQueue(t, r, msgs, base.DefaultQueueName) inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { page int pageSize int want []*TaskInfo }{ { page: 1, pageSize: 5, want: []*TaskInfo{ createPendingTask(msgs[0]), createPendingTask(msgs[1]), createPendingTask(msgs[2]), createPendingTask(msgs[3]), createPendingTask(msgs[4]), }, }, { page: 3, pageSize: 10, want: []*TaskInfo{ createPendingTask(msgs[20]), createPendingTask(msgs[21]), createPendingTask(msgs[22]), createPendingTask(msgs[23]), createPendingTask(msgs[24]), createPendingTask(msgs[25]), createPendingTask(msgs[26]), createPendingTask(msgs[27]), createPendingTask(msgs[28]), createPendingTask(msgs[29]), }, }, } for _, tc := range tests { got, err := inspector.ListPendingTasks("default", Page(tc.page), PageSize(tc.pageSize)) if err != nil { t.Errorf("ListPendingTask('default') returned error: %v", err) continue } cmpOpts := []cmp.Option{ cmpopts.EquateApproxTime(2 * time.Second), cmp.AllowUnexported(TaskInfo{}), } if diff := cmp.Diff(tc.want, got, cmpOpts...); diff != "" { t.Errorf("ListPendingTask('default') = %v, want %v; (-want,+got)\n%s", got, tc.want, diff) } } } func TestInspectorListTasksQueueNotFoundError(t *testing.T) { r := setup(t) defer r.Close() inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { qname string wantErr error }{ { qname: "nonexistent", wantErr: ErrQueueNotFound, }, } for _, tc := range tests { h.FlushDB(t, r) if _, err := inspector.ListActiveTasks(tc.qname); !errors.Is(err, tc.wantErr) { t.Errorf("ListActiveTasks(%q) returned error %v, want %v", tc.qname, err, tc.wantErr) } if _, err := inspector.ListPendingTasks(tc.qname); !errors.Is(err, tc.wantErr) { t.Errorf("ListPendingTasks(%q) returned error %v, want %v", tc.qname, err, tc.wantErr) } if _, err := inspector.ListScheduledTasks(tc.qname); !errors.Is(err, tc.wantErr) { t.Errorf("ListScheduledTasks(%q) returned error %v, want %v", tc.qname, err, tc.wantErr) } if _, err := inspector.ListRetryTasks(tc.qname); !errors.Is(err, tc.wantErr) { t.Errorf("ListRetryTasks(%q) returned error %v, want %v", tc.qname, err, tc.wantErr) } if _, err := inspector.ListArchivedTasks(tc.qname); !errors.Is(err, tc.wantErr) { t.Errorf("ListArchivedTasks(%q) returned error %v, want %v", tc.qname, err, tc.wantErr) } if _, err := inspector.ListCompletedTasks(tc.qname); !errors.Is(err, tc.wantErr) { t.Errorf("ListCompletedTasks(%q) returned error %v, want %v", tc.qname, err, tc.wantErr) } if _, err := inspector.ListAggregatingTasks(tc.qname, "mygroup"); !errors.Is(err, tc.wantErr) { t.Errorf("ListAggregatingTasks(%q, \"mygroup\") returned error %v, want %v", tc.qname, err, tc.wantErr) } } } func TestInspectorDeleteAllPendingTasks(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessage("task3", nil) m4 := h.NewTaskMessageWithQueue("task3", nil, "custom") inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { pending map[string][]*base.TaskMessage qname string want int wantPending map[string][]*base.TaskMessage }{ { pending: map[string][]*base.TaskMessage{ "default": {m1, m2, m3}, "custom": {m4}, }, qname: "default", want: 3, wantPending: map[string][]*base.TaskMessage{ "default": {}, "custom": {m4}, }, }, { pending: map[string][]*base.TaskMessage{ "default": {m1, m2, m3}, "custom": {m4}, }, qname: "custom", want: 1, wantPending: map[string][]*base.TaskMessage{ "default": {m1, m2, m3}, "custom": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllPendingQueues(t, r, tc.pending) got, err := inspector.DeleteAllPendingTasks(tc.qname) if err != nil { t.Errorf("DeleteAllPendingTasks(%q) returned error: %v", tc.qname, err) continue } if got != tc.want { t.Errorf("DeleteAllPendingTasks(%q) = %d, want %d", tc.qname, got, tc.want) } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("unexpected pending tasks in queue %q: (-want, +got)\n%s", qname, diff) } } } } func TestInspectorDeleteAllScheduledTasks(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessage("task3", nil) m4 := h.NewTaskMessageWithQueue("task3", nil, "custom") now := time.Now() z1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()} z2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()} z3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()} z4 := base.Z{Message: m4, Score: now.Add(2 * time.Minute).Unix()} inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { scheduled map[string][]base.Z qname string want int wantScheduled map[string][]base.Z }{ { scheduled: map[string][]base.Z{ "default": {z1, z2, z3}, "custom": {z4}, }, qname: "default", want: 3, wantScheduled: map[string][]base.Z{ "default": {}, "custom": {z4}, }, }, { scheduled: map[string][]base.Z{ "default": {}, }, qname: "default", want: 0, wantScheduled: map[string][]base.Z{ "default": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllScheduledQueues(t, r, tc.scheduled) got, err := inspector.DeleteAllScheduledTasks(tc.qname) if err != nil { t.Errorf("DeleteAllScheduledTasks(%q) returned error: %v", tc.qname, err) continue } if got != tc.want { t.Errorf("DeleteAllScheduledTasks(%q) = %d, want %d", tc.qname, got, tc.want) } for qname, want := range tc.wantScheduled { gotScheduled := h.GetScheduledEntries(t, r, qname) if diff := cmp.Diff(want, gotScheduled, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected scheduled tasks in queue %q: (-want, +got)\n%s", qname, diff) } } } } func TestInspectorDeleteAllRetryTasks(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessage("task3", nil) m4 := h.NewTaskMessageWithQueue("task4", nil, "custom") now := time.Now() z1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()} z2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()} z3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()} z4 := base.Z{Message: m4, Score: now.Add(2 * time.Minute).Unix()} inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { retry map[string][]base.Z qname string want int wantRetry map[string][]base.Z }{ { retry: map[string][]base.Z{ "default": {z1, z2, z3}, "custom": {z4}, }, qname: "default", want: 3, wantRetry: map[string][]base.Z{ "default": {}, "custom": {z4}, }, }, { retry: map[string][]base.Z{ "default": {}, }, qname: "default", want: 0, wantRetry: map[string][]base.Z{ "default": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllRetryQueues(t, r, tc.retry) got, err := inspector.DeleteAllRetryTasks(tc.qname) if err != nil { t.Errorf("DeleteAllRetryTasks(%q) returned error: %v", tc.qname, err) continue } if got != tc.want { t.Errorf("DeleteAllRetryTasks(%q) = %d, want %d", tc.qname, got, tc.want) } for qname, want := range tc.wantRetry { gotRetry := h.GetRetryEntries(t, r, qname) if diff := cmp.Diff(want, gotRetry, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected retry tasks in queue %q: (-want, +got)\n%s", qname, diff) } } } } func TestInspectorDeleteAllArchivedTasks(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessage("task3", nil) m4 := h.NewTaskMessageWithQueue("task4", nil, "custom") now := time.Now() z1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()} z2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()} z3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()} z4 := base.Z{Message: m4, Score: now.Add(2 * time.Minute).Unix()} inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { archived map[string][]base.Z qname string want int wantArchived map[string][]base.Z }{ { archived: map[string][]base.Z{ "default": {z1, z2, z3}, "custom": {z4}, }, qname: "default", want: 3, wantArchived: map[string][]base.Z{ "default": {}, "custom": {z4}, }, }, { archived: map[string][]base.Z{ "default": {}, }, qname: "default", want: 0, wantArchived: map[string][]base.Z{ "default": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllArchivedQueues(t, r, tc.archived) got, err := inspector.DeleteAllArchivedTasks(tc.qname) if err != nil { t.Errorf("DeleteAllArchivedTasks(%q) returned error: %v", tc.qname, err) continue } if got != tc.want { t.Errorf("DeleteAllArchivedTasks(%q) = %d, want %d", tc.qname, got, tc.want) } for qname, want := range tc.wantArchived { gotArchived := h.GetArchivedEntries(t, r, qname) if diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected archived tasks in queue %q: (-want, +got)\n%s", qname, diff) } } } } func TestInspectorDeleteAllCompletedTasks(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() m1 := newCompletedTaskMessage("task1", "default", 30*time.Minute, now.Add(-2*time.Minute)) m2 := newCompletedTaskMessage("task2", "default", 30*time.Minute, now.Add(-5*time.Minute)) m3 := newCompletedTaskMessage("task3", "default", 30*time.Minute, now.Add(-10*time.Minute)) m4 := newCompletedTaskMessage("task4", "custom", 30*time.Minute, now.Add(-3*time.Minute)) z1 := base.Z{Message: m1, Score: m1.CompletedAt + m1.Retention} z2 := base.Z{Message: m2, Score: m2.CompletedAt + m2.Retention} z3 := base.Z{Message: m3, Score: m3.CompletedAt + m3.Retention} z4 := base.Z{Message: m4, Score: m4.CompletedAt + m4.Retention} inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { completed map[string][]base.Z qname string want int wantCompleted map[string][]base.Z }{ { completed: map[string][]base.Z{ "default": {z1, z2, z3}, "custom": {z4}, }, qname: "default", want: 3, wantCompleted: map[string][]base.Z{ "default": {}, "custom": {z4}, }, }, { completed: map[string][]base.Z{ "default": {}, }, qname: "default", want: 0, wantCompleted: map[string][]base.Z{ "default": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllCompletedQueues(t, r, tc.completed) got, err := inspector.DeleteAllCompletedTasks(tc.qname) if err != nil { t.Errorf("DeleteAllCompletedTasks(%q) returned error: %v", tc.qname, err) continue } if got != tc.want { t.Errorf("DeleteAllCompletedTasks(%q) = %d, want %d", tc.qname, got, tc.want) } for qname, want := range tc.wantCompleted { gotCompleted := h.GetCompletedEntries(t, r, qname) if diff := cmp.Diff(want, gotCompleted, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected completed tasks in queue %q: (-want, +got)\n%s", qname, diff) } } } } func TestInspectorArchiveAllPendingTasks(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessage("task3", nil) m4 := h.NewTaskMessageWithQueue("task4", nil, "custom") now := time.Now() z1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()} z2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()} inspector := NewInspector(getRedisConnOpt(t)) inspector.rdb.SetClock(timeutil.NewSimulatedClock(now)) tests := []struct { pending map[string][]*base.TaskMessage archived map[string][]base.Z qname string want int wantPending map[string][]*base.TaskMessage wantArchived map[string][]base.Z }{ { pending: map[string][]*base.TaskMessage{ "default": {m1, m2, m3}, "custom": {m4}, }, archived: map[string][]base.Z{ "default": {}, "custom": {}, }, qname: "default", want: 3, wantPending: map[string][]*base.TaskMessage{ "default": {}, "custom": {m4}, }, wantArchived: map[string][]base.Z{ "default": { base.Z{Message: m1, Score: now.Unix()}, base.Z{Message: m2, Score: now.Unix()}, base.Z{Message: m3, Score: now.Unix()}, }, "custom": {}, }, }, { pending: map[string][]*base.TaskMessage{ "default": {}, }, archived: map[string][]base.Z{ "default": {}, }, qname: "default", want: 0, wantPending: map[string][]*base.TaskMessage{ "default": {}, }, wantArchived: map[string][]base.Z{ "default": {}, }, }, { pending: map[string][]*base.TaskMessage{ "default": {m3}, }, archived: map[string][]base.Z{ "default": {z1, z2}, }, qname: "default", want: 1, wantPending: map[string][]*base.TaskMessage{ "default": {}, }, wantArchived: map[string][]base.Z{ "default": { z1, z2, base.Z{Message: m3, Score: now.Unix()}, }, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllPendingQueues(t, r, tc.pending) h.SeedAllArchivedQueues(t, r, tc.archived) got, err := inspector.ArchiveAllPendingTasks(tc.qname) if err != nil { t.Errorf("ArchiveAllPendingTasks(%q) returned error: %v", tc.qname, err) continue } if got != tc.want { t.Errorf("ArchiveAllPendingTasks(%q) = %d, want %d", tc.qname, got, tc.want) } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("unexpected pending tasks in queue %q: (-want, +got)\n%s", qname, diff) } } for qname, want := range tc.wantArchived { gotArchived := h.GetArchivedEntries(t, r, qname) if diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected archived tasks in queue %q: (-want, +got)\n%s", qname, diff) } } } } func TestInspectorArchiveAllScheduledTasks(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessage("task3", nil) m4 := h.NewTaskMessageWithQueue("task4", nil, "custom") now := time.Now() z1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()} z2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()} z3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()} z4 := base.Z{Message: m4, Score: now.Add(2 * time.Minute).Unix()} inspector := NewInspector(getRedisConnOpt(t)) inspector.rdb.SetClock(timeutil.NewSimulatedClock(now)) tests := []struct { scheduled map[string][]base.Z archived map[string][]base.Z qname string want int wantScheduled map[string][]base.Z wantArchived map[string][]base.Z }{ { scheduled: map[string][]base.Z{ "default": {z1, z2, z3}, "custom": {z4}, }, archived: map[string][]base.Z{ "default": {}, "custom": {}, }, qname: "default", want: 3, wantScheduled: map[string][]base.Z{ "default": {}, "custom": {z4}, }, wantArchived: map[string][]base.Z{ "default": { base.Z{Message: m1, Score: now.Unix()}, base.Z{Message: m2, Score: now.Unix()}, base.Z{Message: m3, Score: now.Unix()}, }, "custom": {}, }, }, { scheduled: map[string][]base.Z{ "default": {z1, z2}, }, archived: map[string][]base.Z{ "default": {z3}, }, qname: "default", want: 2, wantScheduled: map[string][]base.Z{ "default": {}, }, wantArchived: map[string][]base.Z{ "default": { z3, base.Z{Message: m1, Score: now.Unix()}, base.Z{Message: m2, Score: now.Unix()}, }, }, }, { scheduled: map[string][]base.Z{ "default": {}, }, archived: map[string][]base.Z{ "default": {}, }, qname: "default", want: 0, wantScheduled: map[string][]base.Z{ "default": {}, }, wantArchived: map[string][]base.Z{ "default": {}, }, }, { scheduled: map[string][]base.Z{ "default": {}, }, archived: map[string][]base.Z{ "default": {z1, z2}, }, qname: "default", want: 0, wantScheduled: map[string][]base.Z{ "default": {}, }, wantArchived: map[string][]base.Z{ "default": {z1, z2}, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllScheduledQueues(t, r, tc.scheduled) h.SeedAllArchivedQueues(t, r, tc.archived) got, err := inspector.ArchiveAllScheduledTasks(tc.qname) if err != nil { t.Errorf("ArchiveAllScheduledTasks(%q) returned error: %v", tc.qname, err) continue } if got != tc.want { t.Errorf("ArchiveAllScheduledTasks(%q) = %d, want %d", tc.qname, got, tc.want) } for qname, want := range tc.wantScheduled { gotScheduled := h.GetScheduledEntries(t, r, qname) if diff := cmp.Diff(want, gotScheduled, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected scheduled tasks in queue %q: (-want, +got)\n%s", qname, diff) } } for qname, want := range tc.wantArchived { gotArchived := h.GetArchivedEntries(t, r, qname) if diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected archived tasks in queue %q: (-want, +got)\n%s", qname, diff) } } } } func TestInspectorArchiveAllRetryTasks(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessage("task3", nil) m4 := h.NewTaskMessageWithQueue("task4", nil, "custom") now := time.Now() z1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()} z2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()} z3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()} z4 := base.Z{Message: m4, Score: now.Add(2 * time.Minute).Unix()} inspector := NewInspector(getRedisConnOpt(t)) inspector.rdb.SetClock(timeutil.NewSimulatedClock(now)) tests := []struct { retry map[string][]base.Z archived map[string][]base.Z qname string want int wantRetry map[string][]base.Z wantArchived map[string][]base.Z }{ { retry: map[string][]base.Z{ "default": {z1, z2, z3}, "custom": {z4}, }, archived: map[string][]base.Z{ "default": {}, "custom": {}, }, qname: "default", want: 3, wantRetry: map[string][]base.Z{ "default": {}, "custom": {z4}, }, wantArchived: map[string][]base.Z{ "default": { base.Z{Message: m1, Score: now.Unix()}, base.Z{Message: m2, Score: now.Unix()}, base.Z{Message: m3, Score: now.Unix()}, }, "custom": {}, }, }, { retry: map[string][]base.Z{ "default": {z1, z2}, }, archived: map[string][]base.Z{ "default": {z3}, }, qname: "default", want: 2, wantRetry: map[string][]base.Z{ "default": {}, }, wantArchived: map[string][]base.Z{ "default": { z3, base.Z{Message: m1, Score: now.Unix()}, base.Z{Message: m2, Score: now.Unix()}, }, }, }, { retry: map[string][]base.Z{ "default": {}, }, archived: map[string][]base.Z{ "default": {z1, z2}, }, qname: "default", want: 0, wantRetry: map[string][]base.Z{ "default": {}, }, wantArchived: map[string][]base.Z{ "default": {z1, z2}, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllRetryQueues(t, r, tc.retry) h.SeedAllArchivedQueues(t, r, tc.archived) got, err := inspector.ArchiveAllRetryTasks(tc.qname) if err != nil { t.Errorf("ArchiveAllRetryTasks(%q) returned error: %v", tc.qname, err) continue } if got != tc.want { t.Errorf("ArchiveAllRetryTasks(%q) = %d, want %d", tc.qname, got, tc.want) } for qname, want := range tc.wantRetry { gotRetry := h.GetRetryEntries(t, r, qname) if diff := cmp.Diff(want, gotRetry, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected retry tasks in queue %q: (-want, +got)\n%s", qname, diff) } } for qname, want := range tc.wantArchived { wantArchived := h.GetArchivedEntries(t, r, qname) if diff := cmp.Diff(want, wantArchived, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected archived tasks in queue %q: (-want, +got)\n%s", qname, diff) } } } } func TestInspectorRunAllScheduledTasks(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessageWithQueue("task2", nil, "critical") m3 := h.NewTaskMessageWithQueue("task3", nil, "low") m4 := h.NewTaskMessage("task4", nil) now := time.Now() z1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()} z2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()} z3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()} z4 := base.Z{Message: m4, Score: now.Add(2 * time.Minute).Unix()} inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { scheduled map[string][]base.Z pending map[string][]*base.TaskMessage qname string want int wantScheduled map[string][]base.Z wantPending map[string][]*base.TaskMessage }{ { scheduled: map[string][]base.Z{ "default": {z1, z4}, "critical": {z2}, "low": {z3}, }, pending: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, "low": {}, }, qname: "default", want: 2, wantScheduled: map[string][]base.Z{ "default": {}, "critical": {z2}, "low": {z3}, }, wantPending: map[string][]*base.TaskMessage{ "default": {m1, m4}, "critical": {}, "low": {}, }, }, { scheduled: map[string][]base.Z{ "default": {z1}, "critical": {z2}, "low": {z3}, }, pending: map[string][]*base.TaskMessage{ "default": {m4}, "critical": {}, "low": {}, }, qname: "default", want: 1, wantScheduled: map[string][]base.Z{ "default": {}, "critical": {z2}, "low": {z3}, }, wantPending: map[string][]*base.TaskMessage{ "default": {m4, m1}, "critical": {}, "low": {}, }, }, { scheduled: map[string][]base.Z{ "default": {}, }, pending: map[string][]*base.TaskMessage{ "default": {m1, m4}, }, qname: "default", want: 0, wantScheduled: map[string][]base.Z{ "default": {}, }, wantPending: map[string][]*base.TaskMessage{ "default": {m1, m4}, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllScheduledQueues(t, r, tc.scheduled) h.SeedAllPendingQueues(t, r, tc.pending) got, err := inspector.RunAllScheduledTasks(tc.qname) if err != nil { t.Errorf("RunAllScheduledTasks(%q) returned error: %v", tc.qname, err) continue } if got != tc.want { t.Errorf("RunAllScheduledTasks(%q) = %d, want %d", tc.qname, got, tc.want) } for qname, want := range tc.wantScheduled { gotScheduled := h.GetScheduledEntries(t, r, qname) if diff := cmp.Diff(want, gotScheduled, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected scheduled tasks in queue %q: (-want, +got)\n%s", qname, diff) } } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("unexpected pending tasks in queue %q: (-want, +got)\n%s", qname, diff) } } } } func TestInspectorRunAllRetryTasks(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessageWithQueue("task2", nil, "critical") m3 := h.NewTaskMessageWithQueue("task3", nil, "low") m4 := h.NewTaskMessage("task2", nil) now := time.Now() z1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()} z2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()} z3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()} z4 := base.Z{Message: m4, Score: now.Add(2 * time.Minute).Unix()} inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { retry map[string][]base.Z pending map[string][]*base.TaskMessage qname string want int wantRetry map[string][]base.Z wantPending map[string][]*base.TaskMessage }{ { retry: map[string][]base.Z{ "default": {z1, z4}, "critical": {z2}, "low": {z3}, }, pending: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, "low": {}, }, qname: "default", want: 2, wantRetry: map[string][]base.Z{ "default": {}, "critical": {z2}, "low": {z3}, }, wantPending: map[string][]*base.TaskMessage{ "default": {m1, m4}, "critical": {}, "low": {}, }, }, { retry: map[string][]base.Z{ "default": {z1}, "critical": {z2}, "low": {z3}, }, pending: map[string][]*base.TaskMessage{ "default": {m4}, "critical": {}, "low": {}, }, qname: "default", want: 1, wantRetry: map[string][]base.Z{ "default": {}, "critical": {z2}, "low": {z3}, }, wantPending: map[string][]*base.TaskMessage{ "default": {m4, m1}, "critical": {}, "low": {}, }, }, { retry: map[string][]base.Z{ "default": {}, }, pending: map[string][]*base.TaskMessage{ "default": {m1, m4}, }, qname: "default", want: 0, wantRetry: map[string][]base.Z{ "default": {}, }, wantPending: map[string][]*base.TaskMessage{ "default": {m1, m4}, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllRetryQueues(t, r, tc.retry) h.SeedAllPendingQueues(t, r, tc.pending) got, err := inspector.RunAllRetryTasks(tc.qname) if err != nil { t.Errorf("RunAllRetryTasks(%q) returned error: %v", tc.qname, err) continue } if got != tc.want { t.Errorf("RunAllRetryTasks(%q) = %d, want %d", tc.qname, got, tc.want) } for qname, want := range tc.wantRetry { gotRetry := h.GetRetryEntries(t, r, qname) if diff := cmp.Diff(want, gotRetry, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected retry tasks in queue %q: (-want, +got)\n%s", qname, diff) } } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("unexpected pending tasks in queue %q: (-want, +got)\n%s", qname, diff) } } } } func TestInspectorRunAllArchivedTasks(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessageWithQueue("task2", nil, "critical") m3 := h.NewTaskMessageWithQueue("task3", nil, "low") m4 := h.NewTaskMessage("task2", nil) now := time.Now() z1 := base.Z{Message: m1, Score: now.Add(-5 * time.Minute).Unix()} z2 := base.Z{Message: m2, Score: now.Add(-15 * time.Minute).Unix()} z3 := base.Z{Message: m3, Score: now.Add(-2 * time.Minute).Unix()} z4 := base.Z{Message: m4, Score: now.Add(-2 * time.Minute).Unix()} inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { archived map[string][]base.Z pending map[string][]*base.TaskMessage qname string want int wantArchived map[string][]base.Z wantPending map[string][]*base.TaskMessage }{ { archived: map[string][]base.Z{ "default": {z1, z4}, "critical": {z2}, "low": {z3}, }, pending: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, "low": {}, }, qname: "default", want: 2, wantArchived: map[string][]base.Z{ "default": {}, "critical": {z2}, "low": {z3}, }, wantPending: map[string][]*base.TaskMessage{ "default": {m1, m4}, "critical": {}, "low": {}, }, }, { archived: map[string][]base.Z{ "default": {z1}, "critical": {z2}, }, pending: map[string][]*base.TaskMessage{ "default": {m4}, "critical": {}, }, qname: "default", want: 1, wantArchived: map[string][]base.Z{ "default": {}, "critical": {z2}, }, wantPending: map[string][]*base.TaskMessage{ "default": {m4, m1}, "critical": {}, }, }, { archived: map[string][]base.Z{ "default": {}, }, pending: map[string][]*base.TaskMessage{ "default": {m1, m4}, }, qname: "default", want: 0, wantArchived: map[string][]base.Z{ "default": {}, }, wantPending: map[string][]*base.TaskMessage{ "default": {m1, m4}, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllArchivedQueues(t, r, tc.archived) h.SeedAllPendingQueues(t, r, tc.pending) got, err := inspector.RunAllArchivedTasks(tc.qname) if err != nil { t.Errorf("RunAllArchivedTasks(%q) returned error: %v", tc.qname, err) continue } if got != tc.want { t.Errorf("RunAllArchivedTasks(%q) = %d, want %d", tc.qname, got, tc.want) } for qname, want := range tc.wantArchived { wantArchived := h.GetArchivedEntries(t, r, qname) if diff := cmp.Diff(want, wantArchived, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected archived tasks in queue %q: (-want, +got)\n%s", qname, diff) } } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("unexpected pending tasks in queue %q: (-want, +got)\n%s", qname, diff) } } } } func TestInspectorUpdateTaskPayloadUpdatesScheduledTaskPayload(t *testing.T) { r := setup(t) defer r.Close() m1_old := h.NewTaskMessage("task1", []byte("m1_old")) m1_new := h.NewTaskMessage("task1", nil) m1_new.ID = m1_old.ID m2_old := h.NewTaskMessage("task2", nil) m2_new := h.NewTaskMessage("task2", []byte("m2_new")) m2_new.ID = m2_old.ID m3_old := h.NewTaskMessageWithQueue("task3", []byte("m3_old"), "custom") m3_new := h.NewTaskMessageWithQueue("task3", []byte("m3_new"), "custom") m3_new.ID = m3_old.ID now := time.Now() z1_old := base.Z{Message: m1_old, Score: now.Add(5 * time.Minute).Unix()} z1_new := base.Z{Message: m1_new, Score: now.Add(5 * time.Minute).Unix()} z2_old := base.Z{Message: m2_old, Score: now.Add(15 * time.Minute).Unix()} z2_new := base.Z{Message: m2_new, Score: now.Add(15 * time.Minute).Unix()} z3_old := base.Z{Message: m3_old, Score: now.Add(2 * time.Minute).Unix()} z3_new := base.Z{Message: m3_new, Score: now.Add(2 * time.Minute).Unix()} inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { scheduled map[string][]base.Z qname string id string newPayload []byte wantScheduled map[string][]base.Z }{ { scheduled: map[string][]base.Z{ "default": {z1_old, z2_old}, "custom": {z3_old}, }, qname: "default", id: createScheduledTask(z2_old).ID, newPayload: m2_new.Payload, wantScheduled: map[string][]base.Z{ "default": {z1_old, z2_new}, "custom": {z3_old}, }, }, { scheduled: map[string][]base.Z{ "default": {z1_old, z2_old}, "custom": {z3_old}, }, qname: "default", id: createScheduledTask(z1_old).ID, newPayload: m1_new.Payload, wantScheduled: map[string][]base.Z{ "default": {z1_new, z2_old}, "custom": {z3_old}, }, }, { scheduled: map[string][]base.Z{ "default": {z1_old, z2_old}, "custom": {z3_old}, }, qname: "custom", id: createScheduledTask(z3_old).ID, newPayload: m3_new.Payload, wantScheduled: map[string][]base.Z{ "default": {z1_old, z2_old}, "custom": {z3_new}, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllScheduledQueues(t, r, tc.scheduled) if err := inspector.UpdateTaskPayload(tc.qname, tc.id, tc.newPayload); err != nil { t.Errorf("UpdateTask(%q, %q) returned error: %v", tc.qname, tc.id, err) } for qname, want := range tc.wantScheduled { gotScheduled := h.GetScheduledEntries(t, r, qname) if diff := cmp.Diff(want, gotScheduled, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected scheduled tasks in queue %q: (-want, +got)\n%s", qname, diff) } } } } func TestInspectorUpdateTaskPayloadError(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") now := time.Now() z1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()} z2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()} z3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()} inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { tasks map[string][]base.Z qname string id string newPayload []byte wantErr error }{ { tasks: map[string][]base.Z{ "default": {z1, z2}, "custom": {z3}, }, qname: "nonexistent", id: createScheduledTask(z2).ID, newPayload: nil, wantErr: ErrQueueNotFound, }, { tasks: map[string][]base.Z{ "default": {z1, z2}, "custom": {z3}, }, qname: "default", id: uuid.NewString(), newPayload: nil, wantErr: ErrTaskNotFound, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllScheduledQueues(t, r, tc.tasks) if err := inspector.UpdateTaskPayload(tc.qname, tc.id, tc.newPayload); !errors.Is(err, tc.wantErr) { t.Errorf("UpdateTask(%q, %q) = %v, want %v", tc.qname, tc.id, err, tc.wantErr) continue } } } func TestInspectorDeleteTaskDeletesPendingTask(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { pending map[string][]*base.TaskMessage qname string id string wantPending map[string][]*base.TaskMessage }{ { pending: map[string][]*base.TaskMessage{ "default": {m1, m2}, "custom": {m3}, }, qname: "default", id: createPendingTask(m2).ID, wantPending: map[string][]*base.TaskMessage{ "default": {m1}, "custom": {m3}, }, }, { pending: map[string][]*base.TaskMessage{ "default": {m1, m2}, "custom": {m3}, }, qname: "custom", id: createPendingTask(m3).ID, wantPending: map[string][]*base.TaskMessage{ "default": {m1, m2}, "custom": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllPendingQueues(t, r, tc.pending) if err := inspector.DeleteTask(tc.qname, tc.id); err != nil { t.Errorf("DeleteTask(%q, %q) returned error: %v", tc.qname, tc.id, err) continue } for qname, want := range tc.wantPending { got := h.GetPendingMessages(t, r, qname) if diff := cmp.Diff(want, got, h.SortMsgOpt); diff != "" { t.Errorf("unspected pending tasks in queue %q: (-want,+got):\n%s", qname, diff) continue } } } } func TestInspectorDeleteTaskDeletesScheduledTask(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") now := time.Now() z1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()} z2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()} z3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()} inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { scheduled map[string][]base.Z qname string id string wantScheduled map[string][]base.Z }{ { scheduled: map[string][]base.Z{ "default": {z1, z2}, "custom": {z3}, }, qname: "default", id: createScheduledTask(z2).ID, wantScheduled: map[string][]base.Z{ "default": {z1}, "custom": {z3}, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllScheduledQueues(t, r, tc.scheduled) if err := inspector.DeleteTask(tc.qname, tc.id); err != nil { t.Errorf("DeleteTask(%q, %q) returned error: %v", tc.qname, tc.id, err) } for qname, want := range tc.wantScheduled { gotScheduled := h.GetScheduledEntries(t, r, qname) if diff := cmp.Diff(want, gotScheduled, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected scheduled tasks in queue %q: (-want, +got)\n%s", qname, diff) } } } } func TestInspectorDeleteTaskDeletesRetryTask(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") now := time.Now() z1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()} z2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()} z3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()} inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { retry map[string][]base.Z qname string id string wantRetry map[string][]base.Z }{ { retry: map[string][]base.Z{ "default": {z1, z2}, "custom": {z3}, }, qname: "default", id: createRetryTask(z2).ID, wantRetry: map[string][]base.Z{ "default": {z1}, "custom": {z3}, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllRetryQueues(t, r, tc.retry) if err := inspector.DeleteTask(tc.qname, tc.id); err != nil { t.Errorf("DeleteTask(%q, %q) returned error: %v", tc.qname, tc.id, err) continue } for qname, want := range tc.wantRetry { gotRetry := h.GetRetryEntries(t, r, qname) if diff := cmp.Diff(want, gotRetry, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected retry tasks in queue %q: (-want, +got)\n%s", qname, diff) } } } } func TestInspectorDeleteTaskDeletesArchivedTask(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") now := time.Now() z1 := base.Z{Message: m1, Score: now.Add(-5 * time.Minute).Unix()} z2 := base.Z{Message: m2, Score: now.Add(-15 * time.Minute).Unix()} z3 := base.Z{Message: m3, Score: now.Add(-2 * time.Minute).Unix()} inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { archived map[string][]base.Z qname string id string wantArchived map[string][]base.Z }{ { archived: map[string][]base.Z{ "default": {z1, z2}, "custom": {z3}, }, qname: "default", id: createArchivedTask(z2).ID, wantArchived: map[string][]base.Z{ "default": {z1}, "custom": {z3}, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllArchivedQueues(t, r, tc.archived) if err := inspector.DeleteTask(tc.qname, tc.id); err != nil { t.Errorf("DeleteTask(%q, %q) returned error: %v", tc.qname, tc.id, err) continue } for qname, want := range tc.wantArchived { wantArchived := h.GetArchivedEntries(t, r, qname) if diff := cmp.Diff(want, wantArchived, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected archived tasks in queue %q: (-want, +got)\n%s", qname, diff) } } } } func TestInspectorDeleteTaskError(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") now := time.Now() z1 := base.Z{Message: m1, Score: now.Add(-5 * time.Minute).Unix()} z2 := base.Z{Message: m2, Score: now.Add(-15 * time.Minute).Unix()} z3 := base.Z{Message: m3, Score: now.Add(-2 * time.Minute).Unix()} inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { archived map[string][]base.Z qname string id string wantErr error wantArchived map[string][]base.Z }{ { archived: map[string][]base.Z{ "default": {z1, z2}, "custom": {z3}, }, qname: "nonexistent", id: createArchivedTask(z2).ID, wantErr: ErrQueueNotFound, wantArchived: map[string][]base.Z{ "default": {z1, z2}, "custom": {z3}, }, }, { archived: map[string][]base.Z{ "default": {z1, z2}, "custom": {z3}, }, qname: "default", id: uuid.NewString(), wantErr: ErrTaskNotFound, wantArchived: map[string][]base.Z{ "default": {z1, z2}, "custom": {z3}, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllArchivedQueues(t, r, tc.archived) if err := inspector.DeleteTask(tc.qname, tc.id); !errors.Is(err, tc.wantErr) { t.Errorf("DeleteTask(%q, %q) = %v, want %v", tc.qname, tc.id, err, tc.wantErr) continue } for qname, want := range tc.wantArchived { wantArchived := h.GetArchivedEntries(t, r, qname) if diff := cmp.Diff(want, wantArchived, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected archived tasks in queue %q: (-want, +got)\n%s", qname, diff) } } } } func TestInspectorRunTaskRunsScheduledTask(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") now := time.Now() z1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()} z2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()} z3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()} inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { scheduled map[string][]base.Z pending map[string][]*base.TaskMessage qname string id string wantScheduled map[string][]base.Z wantPending map[string][]*base.TaskMessage }{ { scheduled: map[string][]base.Z{ "default": {z1, z2}, "custom": {z3}, }, pending: map[string][]*base.TaskMessage{ "default": {}, "custom": {}, }, qname: "default", id: createScheduledTask(z2).ID, wantScheduled: map[string][]base.Z{ "default": {z1}, "custom": {z3}, }, wantPending: map[string][]*base.TaskMessage{ "default": {m2}, "custom": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllScheduledQueues(t, r, tc.scheduled) h.SeedAllPendingQueues(t, r, tc.pending) if err := inspector.RunTask(tc.qname, tc.id); err != nil { t.Errorf("RunTask(%q, %q) returned error: %v", tc.qname, tc.id, err) continue } for qname, want := range tc.wantScheduled { gotScheduled := h.GetScheduledEntries(t, r, qname) if diff := cmp.Diff(want, gotScheduled, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected scheduled tasks in queue %q: (-want, +got)\n%s", qname, diff) } } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("unexpected pending tasks in queue %q: (-want, +got)\n%s", qname, diff) } } } } func TestInspectorRunTaskRunsRetryTask(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessageWithQueue("task2", nil, "custom") m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") now := time.Now() z1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()} z2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()} z3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()} inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { retry map[string][]base.Z pending map[string][]*base.TaskMessage qname string id string wantRetry map[string][]base.Z wantPending map[string][]*base.TaskMessage }{ { retry: map[string][]base.Z{ "default": {z1}, "custom": {z2, z3}, }, pending: map[string][]*base.TaskMessage{ "default": {}, "custom": {}, }, qname: "custom", id: createRetryTask(z2).ID, wantRetry: map[string][]base.Z{ "default": {z1}, "custom": {z3}, }, wantPending: map[string][]*base.TaskMessage{ "default": {}, "custom": {m2}, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllRetryQueues(t, r, tc.retry) h.SeedAllPendingQueues(t, r, tc.pending) if err := inspector.RunTask(tc.qname, tc.id); err != nil { t.Errorf("RunTaskBy(%q, %q) returned error: %v", tc.qname, tc.id, err) continue } for qname, want := range tc.wantRetry { gotRetry := h.GetRetryEntries(t, r, qname) if diff := cmp.Diff(want, gotRetry, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected retry tasks in queue %q: (-want, +got)\n%s", qname, diff) } } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("unexpected pending tasks in queue %q: (-want, +got)\n%s", qname, diff) } } } } func TestInspectorRunTaskRunsArchivedTask(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessageWithQueue("task2", nil, "critical") m3 := h.NewTaskMessageWithQueue("task3", nil, "low") now := time.Now() z1 := base.Z{Message: m1, Score: now.Add(-5 * time.Minute).Unix()} z2 := base.Z{Message: m2, Score: now.Add(-15 * time.Minute).Unix()} z3 := base.Z{Message: m3, Score: now.Add(-2 * time.Minute).Unix()} inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { archived map[string][]base.Z pending map[string][]*base.TaskMessage qname string id string wantArchived map[string][]base.Z wantPending map[string][]*base.TaskMessage }{ { archived: map[string][]base.Z{ "default": {z1}, "critical": {z2}, "low": {z3}, }, pending: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, "low": {}, }, qname: "critical", id: createArchivedTask(z2).ID, wantArchived: map[string][]base.Z{ "default": {z1}, "critical": {}, "low": {z3}, }, wantPending: map[string][]*base.TaskMessage{ "default": {}, "critical": {m2}, "low": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllArchivedQueues(t, r, tc.archived) h.SeedAllPendingQueues(t, r, tc.pending) if err := inspector.RunTask(tc.qname, tc.id); err != nil { t.Errorf("RunTask(%q, %q) returned error: %v", tc.qname, tc.id, err) continue } for qname, want := range tc.wantArchived { wantArchived := h.GetArchivedEntries(t, r, qname) if diff := cmp.Diff(want, wantArchived, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected archived tasks in queue %q: (-want, +got)\n%s", qname, diff) } } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("unexpected pending tasks in queue %q: (-want, +got)\n%s", qname, diff) } } } } func TestInspectorRunTaskError(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessageWithQueue("task2", nil, "critical") m3 := h.NewTaskMessageWithQueue("task3", nil, "low") now := time.Now() z1 := base.Z{Message: m1, Score: now.Add(-5 * time.Minute).Unix()} z2 := base.Z{Message: m2, Score: now.Add(-15 * time.Minute).Unix()} z3 := base.Z{Message: m3, Score: now.Add(-2 * time.Minute).Unix()} inspector := NewInspector(getRedisConnOpt(t)) tests := []struct { archived map[string][]base.Z pending map[string][]*base.TaskMessage qname string id string wantErr error wantArchived map[string][]base.Z wantPending map[string][]*base.TaskMessage }{ { archived: map[string][]base.Z{ "default": {z1}, "critical": {z2}, "low": {z3}, }, pending: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, "low": {}, }, qname: "nonexistent", id: createArchivedTask(z2).ID, wantErr: ErrQueueNotFound, wantArchived: map[string][]base.Z{ "default": {z1}, "critical": {z2}, "low": {z3}, }, wantPending: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, "low": {}, }, }, { archived: map[string][]base.Z{ "default": {z1}, "critical": {z2}, "low": {z3}, }, pending: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, "low": {}, }, qname: "default", id: uuid.NewString(), wantErr: ErrTaskNotFound, wantArchived: map[string][]base.Z{ "default": {z1}, "critical": {z2}, "low": {z3}, }, wantPending: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, "low": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllArchivedQueues(t, r, tc.archived) h.SeedAllPendingQueues(t, r, tc.pending) if err := inspector.RunTask(tc.qname, tc.id); !errors.Is(err, tc.wantErr) { t.Errorf("RunTask(%q, %q) = %v, want %v", tc.qname, tc.id, err, tc.wantErr) continue } for qname, want := range tc.wantArchived { wantArchived := h.GetArchivedEntries(t, r, qname) if diff := cmp.Diff(want, wantArchived, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected archived tasks in queue %q: (-want, +got)\n%s", qname, diff) } } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("unexpected pending tasks in queue %q: (-want, +got)\n%s", qname, diff) } } } } func TestInspectorArchiveTaskArchivesPendingTask(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessageWithQueue("task2", nil, "custom") m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") now := time.Now() inspector := NewInspector(getRedisConnOpt(t)) inspector.rdb.SetClock(timeutil.NewSimulatedClock(now)) tests := []struct { pending map[string][]*base.TaskMessage archived map[string][]base.Z qname string id string wantPending map[string][]*base.TaskMessage wantArchived map[string][]base.Z }{ { pending: map[string][]*base.TaskMessage{ "default": {m1}, "custom": {m2, m3}, }, archived: map[string][]base.Z{ "default": {}, "custom": {}, }, qname: "default", id: createPendingTask(m1).ID, wantPending: map[string][]*base.TaskMessage{ "default": {}, "custom": {m2, m3}, }, wantArchived: map[string][]base.Z{ "default": { {Message: m1, Score: now.Unix()}, }, "custom": {}, }, }, { pending: map[string][]*base.TaskMessage{ "default": {m1}, "custom": {m2, m3}, }, archived: map[string][]base.Z{ "default": {}, "custom": {}, }, qname: "custom", id: createPendingTask(m2).ID, wantPending: map[string][]*base.TaskMessage{ "default": {m1}, "custom": {m3}, }, wantArchived: map[string][]base.Z{ "default": {}, "custom": { {Message: m2, Score: now.Unix()}, }, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllPendingQueues(t, r, tc.pending) h.SeedAllArchivedQueues(t, r, tc.archived) if err := inspector.ArchiveTask(tc.qname, tc.id); err != nil { t.Errorf("ArchiveTask(%q, %q) returned error: %v", tc.qname, tc.id, err) continue } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("unexpected pending tasks in queue %q: (-want,+got)\n%s", qname, diff) } } for qname, want := range tc.wantArchived { wantArchived := h.GetArchivedEntries(t, r, qname) if diff := cmp.Diff(want, wantArchived, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected archived tasks in queue %q: (-want,+got)\n%s", qname, diff) } } } } func TestInspectorArchiveTaskArchivesScheduledTask(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessageWithQueue("task2", nil, "custom") m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") now := time.Now() z1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()} z2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()} z3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()} inspector := NewInspector(getRedisConnOpt(t)) inspector.rdb.SetClock(timeutil.NewSimulatedClock(now)) tests := []struct { scheduled map[string][]base.Z archived map[string][]base.Z qname string id string want string wantScheduled map[string][]base.Z wantArchived map[string][]base.Z }{ { scheduled: map[string][]base.Z{ "default": {z1}, "custom": {z2, z3}, }, archived: map[string][]base.Z{ "default": {}, "custom": {}, }, qname: "custom", id: createScheduledTask(z2).ID, wantScheduled: map[string][]base.Z{ "default": {z1}, "custom": {z3}, }, wantArchived: map[string][]base.Z{ "default": {}, "custom": { { Message: m2, Score: now.Unix(), }, }, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllScheduledQueues(t, r, tc.scheduled) h.SeedAllArchivedQueues(t, r, tc.archived) if err := inspector.ArchiveTask(tc.qname, tc.id); err != nil { t.Errorf("ArchiveTask(%q, %q) returned error: %v", tc.qname, tc.id, err) continue } for qname, want := range tc.wantScheduled { gotScheduled := h.GetScheduledEntries(t, r, qname) if diff := cmp.Diff(want, gotScheduled, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected scheduled tasks in queue %q: (-want, +got)\n%s", qname, diff) } } for qname, want := range tc.wantArchived { wantArchived := h.GetArchivedEntries(t, r, qname) if diff := cmp.Diff(want, wantArchived, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected archived tasks in queue %q: (-want, +got)\n%s", qname, diff) } } } } func TestInspectorArchiveTaskArchivesRetryTask(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessageWithQueue("task2", nil, "custom") m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") now := time.Now() z1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()} z2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()} z3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()} inspector := NewInspector(getRedisConnOpt(t)) inspector.rdb.SetClock(timeutil.NewSimulatedClock(now)) tests := []struct { retry map[string][]base.Z archived map[string][]base.Z qname string id string wantRetry map[string][]base.Z wantArchived map[string][]base.Z }{ { retry: map[string][]base.Z{ "default": {z1}, "custom": {z2, z3}, }, archived: map[string][]base.Z{ "default": {}, "custom": {}, }, qname: "custom", id: createRetryTask(z2).ID, wantRetry: map[string][]base.Z{ "default": {z1}, "custom": {z3}, }, wantArchived: map[string][]base.Z{ "default": {}, "custom": { { Message: m2, Score: now.Unix(), }, }, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllRetryQueues(t, r, tc.retry) h.SeedAllArchivedQueues(t, r, tc.archived) if err := inspector.ArchiveTask(tc.qname, tc.id); err != nil { t.Errorf("ArchiveTask(%q, %q) returned error: %v", tc.qname, tc.id, err) continue } for qname, want := range tc.wantRetry { gotRetry := h.GetRetryEntries(t, r, qname) if diff := cmp.Diff(want, gotRetry, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected retry tasks in queue %q: (-want, +got)\n%s", qname, diff) } } for qname, want := range tc.wantArchived { wantArchived := h.GetArchivedEntries(t, r, qname) if diff := cmp.Diff(want, wantArchived, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected archived tasks in queue %q: (-want, +got)\n%s", qname, diff) } } } } func TestInspectorArchiveTaskError(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessageWithQueue("task2", nil, "custom") m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") now := time.Now() z1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()} z2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()} z3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()} inspector := NewInspector(getRedisConnOpt(t)) inspector.rdb.SetClock(timeutil.NewSimulatedClock(now)) tests := []struct { retry map[string][]base.Z archived map[string][]base.Z qname string id string wantErr error wantRetry map[string][]base.Z wantArchived map[string][]base.Z }{ { retry: map[string][]base.Z{ "default": {z1}, "custom": {z2, z3}, }, archived: map[string][]base.Z{ "default": {}, "custom": {}, }, qname: "nonexistent", id: createRetryTask(z2).ID, wantErr: ErrQueueNotFound, wantRetry: map[string][]base.Z{ "default": {z1}, "custom": {z2, z3}, }, wantArchived: map[string][]base.Z{ "default": {}, "custom": {}, }, }, { retry: map[string][]base.Z{ "default": {z1}, "custom": {z2, z3}, }, archived: map[string][]base.Z{ "default": {}, "custom": {}, }, qname: "custom", id: uuid.NewString(), wantErr: ErrTaskNotFound, wantRetry: map[string][]base.Z{ "default": {z1}, "custom": {z2, z3}, }, wantArchived: map[string][]base.Z{ "default": {}, "custom": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllRetryQueues(t, r, tc.retry) h.SeedAllArchivedQueues(t, r, tc.archived) if err := inspector.ArchiveTask(tc.qname, tc.id); !errors.Is(err, tc.wantErr) { t.Errorf("ArchiveTask(%q, %q) = %v, want %v", tc.qname, tc.id, err, tc.wantErr) continue } for qname, want := range tc.wantRetry { gotRetry := h.GetRetryEntries(t, r, qname) if diff := cmp.Diff(want, gotRetry, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected retry tasks in queue %q: (-want, +got)\n%s", qname, diff) } } for qname, want := range tc.wantArchived { wantArchived := h.GetArchivedEntries(t, r, qname) if diff := cmp.Diff(want, wantArchived, h.SortZSetEntryOpt); diff != "" { t.Errorf("unexpected archived tasks in queue %q: (-want, +got)\n%s", qname, diff) } } } } var sortSchedulerEntry = cmp.Transformer("SortSchedulerEntry", func(in []*SchedulerEntry) []*SchedulerEntry { out := append([]*SchedulerEntry(nil), in...) sort.Slice(out, func(i, j int) bool { return out[i].Spec < out[j].Spec }) return out }) func TestInspectorSchedulerEntries(t *testing.T) { r := setup(t) rdbClient := rdb.NewRDB(r) inspector := NewInspector(getRedisConnOpt(t)) now := time.Now().UTC() schedulerID := "127.0.0.1:9876:abc123" tests := []struct { data []*base.SchedulerEntry // data to seed redis want []*SchedulerEntry }{ { data: []*base.SchedulerEntry{ { Spec: "* * * * *", Type: "foo", Payload: nil, Opts: nil, Next: now.Add(5 * time.Hour), Prev: now.Add(-2 * time.Hour), }, { Spec: "@every 20m", Type: "bar", Payload: h.JSON(map[string]interface{}{"fiz": "baz"}), Opts: []string{`Queue("bar")`, `MaxRetry(20)`}, Next: now.Add(1 * time.Minute), Prev: now.Add(-19 * time.Minute), }, }, want: []*SchedulerEntry{ { Spec: "* * * * *", Task: NewTask("foo", nil), Opts: nil, Next: now.Add(5 * time.Hour), Prev: now.Add(-2 * time.Hour), }, { Spec: "@every 20m", Task: NewTask("bar", h.JSON(map[string]interface{}{"fiz": "baz"})), Opts: []Option{Queue("bar"), MaxRetry(20)}, Next: now.Add(1 * time.Minute), Prev: now.Add(-19 * time.Minute), }, }, }, } for _, tc := range tests { h.FlushDB(t, r) err := rdbClient.WriteSchedulerEntries(schedulerID, tc.data, time.Minute) if err != nil { t.Fatalf("could not write data: %v", err) } got, err := inspector.SchedulerEntries() if err != nil { t.Errorf("SchedulerEntries() returned error: %v", err) continue } ignoreOpt := cmpopts.IgnoreUnexported(Task{}) if diff := cmp.Diff(tc.want, got, sortSchedulerEntry, ignoreOpt); diff != "" { t.Errorf("SchedulerEntries() = %v, want %v; (-want,+got)\n%s", got, tc.want, diff) } } } func TestParseOption(t *testing.T) { oneHourFromNow := time.Now().Add(1 * time.Hour) tests := []struct { s string wantType OptionType wantVal interface{} }{ {`MaxRetry(10)`, MaxRetryOpt, 10}, {`Queue("email")`, QueueOpt, "email"}, {`Timeout(3m)`, TimeoutOpt, 3 * time.Minute}, {Deadline(oneHourFromNow).String(), DeadlineOpt, oneHourFromNow}, {`Unique(1h)`, UniqueOpt, 1 * time.Hour}, {ProcessAt(oneHourFromNow).String(), ProcessAtOpt, oneHourFromNow}, {`ProcessIn(10m)`, ProcessInOpt, 10 * time.Minute}, {`Retention(24h)`, RetentionOpt, 24 * time.Hour}, } for _, tc := range tests { t.Run(tc.s, func(t *testing.T) { got, err := parseOption(tc.s) if err != nil { t.Fatalf("returned error: %v", err) } if got == nil { t.Fatal("returned nil") } if got.Type() != tc.wantType { t.Fatalf("got type %v, want type %v ", got.Type(), tc.wantType) } switch tc.wantType { case QueueOpt: gotVal, ok := got.Value().(string) if !ok { t.Fatal("returned Option with non-string value") } if gotVal != tc.wantVal.(string) { t.Fatalf("got value %v, want %v", gotVal, tc.wantVal) } case MaxRetryOpt: gotVal, ok := got.Value().(int) if !ok { t.Fatal("returned Option with non-int value") } if gotVal != tc.wantVal.(int) { t.Fatalf("got value %v, want %v", gotVal, tc.wantVal) } case TimeoutOpt, UniqueOpt, ProcessInOpt, RetentionOpt: gotVal, ok := got.Value().(time.Duration) if !ok { t.Fatal("returned Option with non duration value") } if gotVal != tc.wantVal.(time.Duration) { t.Fatalf("got value %v, want %v", gotVal, tc.wantVal) } case DeadlineOpt, ProcessAtOpt: gotVal, ok := got.Value().(time.Time) if !ok { t.Fatal("returned Option with non time value") } if cmp.Equal(gotVal, tc.wantVal.(time.Time)) { t.Fatalf("got value %v, want %v", gotVal, tc.wantVal) } default: t.Fatalf("returned Option with unexpected type: %v", got.Type()) } }) } } func TestInspectorGroups(t *testing.T) { r := setup(t) defer r.Close() inspector := NewInspector(getRedisConnOpt(t)) m1 := h.NewTaskMessageBuilder().SetGroup("group1").Build() m2 := h.NewTaskMessageBuilder().SetGroup("group1").Build() m3 := h.NewTaskMessageBuilder().SetGroup("group1").Build() m4 := h.NewTaskMessageBuilder().SetGroup("group2").Build() m5 := h.NewTaskMessageBuilder().SetQueue("custom").SetGroup("group1").Build() m6 := h.NewTaskMessageBuilder().SetQueue("custom").SetGroup("group1").Build() now := time.Now() fixtures := struct { tasks []*h.TaskSeedData allGroups map[string][]string groups map[string][]redis.Z }{ tasks: []*h.TaskSeedData{ {Msg: m1, State: base.TaskStateAggregating}, {Msg: m2, State: base.TaskStateAggregating}, {Msg: m3, State: base.TaskStateAggregating}, {Msg: m4, State: base.TaskStateAggregating}, {Msg: m5, State: base.TaskStateAggregating}, }, allGroups: map[string][]string{ base.AllGroups("default"): {"group1", "group2"}, base.AllGroups("custom"): {"group1"}, }, groups: map[string][]redis.Z{ base.GroupKey("default", "group1"): { {Member: m1.ID, Score: float64(now.Add(-10 * time.Second).Unix())}, {Member: m2.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, {Member: m3.ID, Score: float64(now.Add(-30 * time.Second).Unix())}, }, base.GroupKey("default", "group2"): { {Member: m4.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, }, base.GroupKey("custom", "group1"): { {Member: m5.ID, Score: float64(now.Add(-10 * time.Second).Unix())}, {Member: m6.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, }, }, } tests := []struct { desc string qname string want []*GroupInfo }{ { desc: "default queue groups", qname: "default", want: []*GroupInfo{ {Group: "group1", Size: 3}, {Group: "group2", Size: 1}, }, }, { desc: "custom queue groups", qname: "custom", want: []*GroupInfo{ {Group: "group1", Size: 2}, }, }, } var sortGroupInfosOpt = cmp.Transformer( "SortGroupInfos", func(in []*GroupInfo) []*GroupInfo { out := append([]*GroupInfo(nil), in...) sort.Slice(out, func(i, j int) bool { return out[i].Group < out[j].Group }) return out }) for _, tc := range tests { h.FlushDB(t, r) h.SeedTasks(t, r, fixtures.tasks) h.SeedRedisSets(t, r, fixtures.allGroups) h.SeedRedisZSets(t, r, fixtures.groups) t.Run(tc.desc, func(t *testing.T) { got, err := inspector.Groups(tc.qname) if err != nil { t.Fatalf("Groups returned error: %v", err) } if diff := cmp.Diff(tc.want, got, sortGroupInfosOpt); diff != "" { t.Errorf("Groups = %v, want %v; (-want,+got)\n%s", got, tc.want, diff) } }) } } ================================================ FILE: internal/base/base.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. // Package base defines foundational types and constants used in asynq package. package base import ( "context" "crypto/md5" "encoding/hex" "fmt" "strings" "sync" "time" "github.com/hibiken/asynq/internal/errors" pb "github.com/hibiken/asynq/internal/proto" "github.com/hibiken/asynq/internal/timeutil" "github.com/redis/go-redis/v9" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" ) // Version of asynq library and CLI. const Version = "0.26.0" // DefaultQueueName is the queue name used if none are specified by user. const DefaultQueueName = "default" // DefaultQueue is the redis key for the default queue. var DefaultQueue = PendingKey(DefaultQueueName) // Global Redis keys. const ( AllServers = "asynq:servers" // ZSET AllWorkers = "asynq:workers" // ZSET AllSchedulers = "asynq:schedulers" // ZSET AllQueues = "asynq:queues" // SET CancelChannel = "asynq:cancel" // PubSub channel ) // TaskState denotes the state of a task. type TaskState int const ( TaskStateActive TaskState = iota + 1 TaskStatePending TaskStateScheduled TaskStateRetry TaskStateArchived TaskStateCompleted TaskStateAggregating // describes a state where task is waiting in a group to be aggregated ) func (s TaskState) String() string { switch s { case TaskStateActive: return "active" case TaskStatePending: return "pending" case TaskStateScheduled: return "scheduled" case TaskStateRetry: return "retry" case TaskStateArchived: return "archived" case TaskStateCompleted: return "completed" case TaskStateAggregating: return "aggregating" } panic(fmt.Sprintf("internal error: unknown task state %d", s)) } func TaskStateFromString(s string) (TaskState, error) { switch s { case "active": return TaskStateActive, nil case "pending": return TaskStatePending, nil case "scheduled": return TaskStateScheduled, nil case "retry": return TaskStateRetry, nil case "archived": return TaskStateArchived, nil case "completed": return TaskStateCompleted, nil case "aggregating": return TaskStateAggregating, nil } return 0, errors.E(errors.FailedPrecondition, fmt.Sprintf("%q is not supported task state", s)) } // ValidateQueueName validates a given qname to be used as a queue name. // Returns nil if valid, otherwise returns non-nil error. func ValidateQueueName(qname string) error { if len(strings.TrimSpace(qname)) == 0 { return fmt.Errorf("queue name must contain one or more characters") } return nil } // QueueKeyPrefix returns a prefix for all keys in the given queue. func QueueKeyPrefix(qname string) string { return "asynq:{" + qname + "}:" } // TaskKeyPrefix returns a prefix for task key. func TaskKeyPrefix(qname string) string { return QueueKeyPrefix(qname) + "t:" } // TaskKey returns a redis key for the given task message. func TaskKey(qname, id string) string { return TaskKeyPrefix(qname) + id } // PendingKey returns a redis key for the given queue name. func PendingKey(qname string) string { return QueueKeyPrefix(qname) + "pending" } // ActiveKey returns a redis key for the active tasks. func ActiveKey(qname string) string { return QueueKeyPrefix(qname) + "active" } // ScheduledKey returns a redis key for the scheduled tasks. func ScheduledKey(qname string) string { return QueueKeyPrefix(qname) + "scheduled" } // RetryKey returns a redis key for the retry tasks. func RetryKey(qname string) string { return QueueKeyPrefix(qname) + "retry" } // ArchivedKey returns a redis key for the archived tasks. func ArchivedKey(qname string) string { return QueueKeyPrefix(qname) + "archived" } // LeaseKey returns a redis key for the lease. func LeaseKey(qname string) string { return QueueKeyPrefix(qname) + "lease" } func CompletedKey(qname string) string { return QueueKeyPrefix(qname) + "completed" } // PausedKey returns a redis key to indicate that the given queue is paused. func PausedKey(qname string) string { return QueueKeyPrefix(qname) + "paused" } // ProcessedTotalKey returns a redis key for total processed count for the given queue. func ProcessedTotalKey(qname string) string { return QueueKeyPrefix(qname) + "processed" } // FailedTotalKey returns a redis key for total failure count for the given queue. func FailedTotalKey(qname string) string { return QueueKeyPrefix(qname) + "failed" } // ProcessedKey returns a redis key for processed count for the given day for the queue. func ProcessedKey(qname string, t time.Time) string { return QueueKeyPrefix(qname) + "processed:" + t.UTC().Format("2006-01-02") } // FailedKey returns a redis key for failure count for the given day for the queue. func FailedKey(qname string, t time.Time) string { return QueueKeyPrefix(qname) + "failed:" + t.UTC().Format("2006-01-02") } // ServerInfoKey returns a redis key for process info. func ServerInfoKey(hostname string, pid int, serverID string) string { return fmt.Sprintf("asynq:servers:{%s:%d:%s}", hostname, pid, serverID) } // WorkersKey returns a redis key for the workers given hostname, pid, and server ID. func WorkersKey(hostname string, pid int, serverID string) string { return fmt.Sprintf("asynq:workers:{%s:%d:%s}", hostname, pid, serverID) } // SchedulerEntriesKey returns a redis key for the scheduler entries given scheduler ID. func SchedulerEntriesKey(schedulerID string) string { return "asynq:schedulers:{" + schedulerID + "}" } // SchedulerHistoryKey returns a redis key for the scheduler's history for the given entry. func SchedulerHistoryKey(entryID string) string { return "asynq:scheduler_history:" + entryID } // UniqueKey returns a redis key with the given type, payload, and queue name. func UniqueKey(qname, tasktype string, payload []byte) string { if payload == nil { return QueueKeyPrefix(qname) + "unique:" + tasktype + ":" } checksum := md5.Sum(payload) return QueueKeyPrefix(qname) + "unique:" + tasktype + ":" + hex.EncodeToString(checksum[:]) } // GroupKeyPrefix returns a prefix for group key. func GroupKeyPrefix(qname string) string { return QueueKeyPrefix(qname) + "g:" } // GroupKey returns a redis key used to group tasks belong in the same group. func GroupKey(qname, gkey string) string { return GroupKeyPrefix(qname) + gkey } // AggregationSetKey returns a redis key used for an aggregation set. func AggregationSetKey(qname, gname, setID string) string { return GroupKey(qname, gname) + ":" + setID } // AllGroups return a redis key used to store all group keys used in a given queue. func AllGroups(qname string) string { return QueueKeyPrefix(qname) + "groups" } // AllAggregationSets returns a redis key used to store all aggregation sets (set of tasks staged to be aggregated) // in a given queue. func AllAggregationSets(qname string) string { return QueueKeyPrefix(qname) + "aggregation_sets" } // TaskMessage is the internal representation of a task with additional metadata fields. // Serialized data of this type gets written to redis. type TaskMessage struct { // Type indicates the kind of the task to be performed. Type string // Payload holds data needed to process the task. Payload []byte // Headers holds additional metadata for the task. Headers map[string]string // ID is a unique identifier for each task. ID string // Queue is a name this message should be enqueued to. Queue string // Retry is the max number of retry for this task. Retry int // Retried is the number of times we've retried this task so far. Retried int // ErrorMsg holds the error message from the last failure. ErrorMsg string // Time of last failure in Unix time, // the number of seconds elapsed since January 1, 1970 UTC. // // Use zero to indicate no last failure LastFailedAt int64 // Timeout specifies timeout in seconds. // If task processing doesn't complete within the timeout, the task will be retried // if retry count is remaining. Otherwise it will be moved to the archive. // // Use zero to indicate no timeout. Timeout int64 // Deadline specifies the deadline for the task in Unix time, // the number of seconds elapsed since January 1, 1970 UTC. // If task processing doesn't complete before the deadline, the task will be retried // if retry count is remaining. Otherwise it will be moved to the archive. // // Use zero to indicate no deadline. Deadline int64 // UniqueKey holds the redis key used for uniqueness lock for this task. // // Empty string indicates that no uniqueness lock was used. UniqueKey string // GroupKey holds the group key used for task aggregation. // // Empty string indicates no aggregation is used for this task. GroupKey string // Retention specifies the number of seconds the task should be retained after completion. Retention int64 // CompletedAt is the time the task was processed successfully in Unix time, // the number of seconds elapsed since January 1, 1970 UTC. // // Use zero to indicate no value. CompletedAt int64 } // EncodeMessage marshals the given task message and returns an encoded bytes. func EncodeMessage(msg *TaskMessage) ([]byte, error) { if msg == nil { return nil, fmt.Errorf("cannot encode nil message") } return proto.Marshal(&pb.TaskMessage{ Type: msg.Type, Payload: msg.Payload, Headers: msg.Headers, Id: msg.ID, Queue: msg.Queue, Retry: int32(msg.Retry), Retried: int32(msg.Retried), ErrorMsg: msg.ErrorMsg, LastFailedAt: msg.LastFailedAt, Timeout: msg.Timeout, Deadline: msg.Deadline, UniqueKey: msg.UniqueKey, GroupKey: msg.GroupKey, Retention: msg.Retention, CompletedAt: msg.CompletedAt, }) } // DecodeMessage unmarshals the given bytes and returns a decoded task message. func DecodeMessage(data []byte) (*TaskMessage, error) { var pbmsg pb.TaskMessage if err := proto.Unmarshal(data, &pbmsg); err != nil { return nil, err } return &TaskMessage{ Type: pbmsg.GetType(), Payload: pbmsg.GetPayload(), Headers: pbmsg.GetHeaders(), ID: pbmsg.GetId(), Queue: pbmsg.GetQueue(), Retry: int(pbmsg.GetRetry()), Retried: int(pbmsg.GetRetried()), ErrorMsg: pbmsg.GetErrorMsg(), LastFailedAt: pbmsg.GetLastFailedAt(), Timeout: pbmsg.GetTimeout(), Deadline: pbmsg.GetDeadline(), UniqueKey: pbmsg.GetUniqueKey(), GroupKey: pbmsg.GetGroupKey(), Retention: pbmsg.GetRetention(), CompletedAt: pbmsg.GetCompletedAt(), }, nil } // TaskInfo describes a task message and its metadata. type TaskInfo struct { Message *TaskMessage State TaskState NextProcessAt time.Time Result []byte } // Z represents sorted set member. type Z struct { Message *TaskMessage Score int64 } // ServerInfo holds information about a running server. type ServerInfo struct { Host string PID int ServerID string Concurrency int Queues map[string]int StrictPriority bool Status string Started time.Time ActiveWorkerCount int } // EncodeServerInfo marshals the given ServerInfo and returns the encoded bytes. func EncodeServerInfo(info *ServerInfo) ([]byte, error) { if info == nil { return nil, fmt.Errorf("cannot encode nil server info") } queues := make(map[string]int32, len(info.Queues)) for q, p := range info.Queues { queues[q] = int32(p) } started := timestamppb.New(info.Started) return proto.Marshal(&pb.ServerInfo{ Host: info.Host, Pid: int32(info.PID), ServerId: info.ServerID, Concurrency: int32(info.Concurrency), Queues: queues, StrictPriority: info.StrictPriority, Status: info.Status, StartTime: started, ActiveWorkerCount: int32(info.ActiveWorkerCount), }) } // DecodeServerInfo decodes the given bytes into ServerInfo. func DecodeServerInfo(b []byte) (*ServerInfo, error) { var pbmsg pb.ServerInfo if err := proto.Unmarshal(b, &pbmsg); err != nil { return nil, err } queues := make(map[string]int, len(pbmsg.GetQueues())) for q, p := range pbmsg.GetQueues() { queues[q] = int(p) } startTime := pbmsg.GetStartTime() return &ServerInfo{ Host: pbmsg.GetHost(), PID: int(pbmsg.GetPid()), ServerID: pbmsg.GetServerId(), Concurrency: int(pbmsg.GetConcurrency()), Queues: queues, StrictPriority: pbmsg.GetStrictPriority(), Status: pbmsg.GetStatus(), Started: startTime.AsTime(), ActiveWorkerCount: int(pbmsg.GetActiveWorkerCount()), }, nil } // WorkerInfo holds information about a running worker. type WorkerInfo struct { Host string PID int ServerID string ID string Type string Payload []byte Queue string Started time.Time Deadline time.Time } // EncodeWorkerInfo marshals the given WorkerInfo and returns the encoded bytes. func EncodeWorkerInfo(info *WorkerInfo) ([]byte, error) { if info == nil { return nil, fmt.Errorf("cannot encode nil worker info") } startTime := timestamppb.New(info.Started) deadline := timestamppb.New(info.Deadline) return proto.Marshal(&pb.WorkerInfo{ Host: info.Host, Pid: int32(info.PID), ServerId: info.ServerID, TaskId: info.ID, TaskType: info.Type, TaskPayload: info.Payload, Queue: info.Queue, StartTime: startTime, Deadline: deadline, }) } // DecodeWorkerInfo decodes the given bytes into WorkerInfo. func DecodeWorkerInfo(b []byte) (*WorkerInfo, error) { var pbmsg pb.WorkerInfo if err := proto.Unmarshal(b, &pbmsg); err != nil { return nil, err } startTime := pbmsg.GetStartTime() deadline := pbmsg.GetDeadline() return &WorkerInfo{ Host: pbmsg.GetHost(), PID: int(pbmsg.GetPid()), ServerID: pbmsg.GetServerId(), ID: pbmsg.GetTaskId(), Type: pbmsg.GetTaskType(), Payload: pbmsg.GetTaskPayload(), Queue: pbmsg.GetQueue(), Started: startTime.AsTime(), Deadline: deadline.AsTime(), }, nil } // SchedulerEntry holds information about a periodic task registered with a scheduler. type SchedulerEntry struct { // Identifier of this entry. ID string // Spec describes the schedule of this entry. Spec string // Type is the task type of the periodic task. Type string // Payload is the payload of the periodic task. Payload []byte // Opts is the options for the periodic task. Opts []string // Next shows the next time the task will be enqueued. Next time.Time // Prev shows the last time the task was enqueued. // Zero time if task was never enqueued. Prev time.Time } // EncodeSchedulerEntry marshals the given entry and returns an encoded bytes. func EncodeSchedulerEntry(entry *SchedulerEntry) ([]byte, error) { if entry == nil { return nil, fmt.Errorf("cannot encode nil scheduler entry") } next := timestamppb.New(entry.Next) prev := timestamppb.New(entry.Prev) return proto.Marshal(&pb.SchedulerEntry{ Id: entry.ID, Spec: entry.Spec, TaskType: entry.Type, TaskPayload: entry.Payload, EnqueueOptions: entry.Opts, NextEnqueueTime: next, PrevEnqueueTime: prev, }) } // DecodeSchedulerEntry unmarshals the given bytes and returns a decoded SchedulerEntry. func DecodeSchedulerEntry(b []byte) (*SchedulerEntry, error) { var pbmsg pb.SchedulerEntry if err := proto.Unmarshal(b, &pbmsg); err != nil { return nil, err } next := pbmsg.GetNextEnqueueTime() prev := pbmsg.GetPrevEnqueueTime() return &SchedulerEntry{ ID: pbmsg.GetId(), Spec: pbmsg.GetSpec(), Type: pbmsg.GetTaskType(), Payload: pbmsg.GetTaskPayload(), Opts: pbmsg.GetEnqueueOptions(), Next: next.AsTime(), Prev: prev.AsTime(), }, nil } // SchedulerEnqueueEvent holds information about an enqueue event by a scheduler. type SchedulerEnqueueEvent struct { // ID of the task that was enqueued. TaskID string // Time the task was enqueued. EnqueuedAt time.Time } // EncodeSchedulerEnqueueEvent marshals the given event // and returns an encoded bytes. func EncodeSchedulerEnqueueEvent(event *SchedulerEnqueueEvent) ([]byte, error) { if event == nil { return nil, fmt.Errorf("cannot encode nil enqueue event") } enqueuedAt := timestamppb.New(event.EnqueuedAt) return proto.Marshal(&pb.SchedulerEnqueueEvent{ TaskId: event.TaskID, EnqueueTime: enqueuedAt, }) } // DecodeSchedulerEnqueueEvent unmarshals the given bytes // and returns a decoded SchedulerEnqueueEvent. func DecodeSchedulerEnqueueEvent(b []byte) (*SchedulerEnqueueEvent, error) { var pbmsg pb.SchedulerEnqueueEvent if err := proto.Unmarshal(b, &pbmsg); err != nil { return nil, err } enqueuedAt := pbmsg.GetEnqueueTime() return &SchedulerEnqueueEvent{ TaskID: pbmsg.GetTaskId(), EnqueuedAt: enqueuedAt.AsTime(), }, nil } // Cancelations is a collection that holds cancel functions for all active tasks. // // Cancelations are safe for concurrent use by multiple goroutines. type Cancelations struct { mu sync.Mutex cancelFuncs map[string]context.CancelFunc } // NewCancelations returns a Cancelations instance. func NewCancelations() *Cancelations { return &Cancelations{ cancelFuncs: make(map[string]context.CancelFunc), } } // Add adds a new cancel func to the collection. func (c *Cancelations) Add(id string, fn context.CancelFunc) { c.mu.Lock() defer c.mu.Unlock() c.cancelFuncs[id] = fn } // Delete deletes a cancel func from the collection given an id. func (c *Cancelations) Delete(id string) { c.mu.Lock() defer c.mu.Unlock() delete(c.cancelFuncs, id) } // Get returns a cancel func given an id. func (c *Cancelations) Get(id string) (fn context.CancelFunc, ok bool) { c.mu.Lock() defer c.mu.Unlock() fn, ok = c.cancelFuncs[id] return fn, ok } // Lease is a time bound lease for worker to process task. // It provides a communication channel between lessor and lessee about lease expiration. type Lease struct { once sync.Once ch chan struct{} Clock timeutil.Clock mu sync.Mutex expireAt time.Time // guarded by mu } func NewLease(expirationTime time.Time) *Lease { return &Lease{ ch: make(chan struct{}), expireAt: expirationTime, Clock: timeutil.NewRealClock(), } } // Reset changes the lease to expire at the given time. // It returns true if the lease is still valid and reset operation was successful, false if the lease had been expired. func (l *Lease) Reset(expirationTime time.Time) bool { if !l.IsValid() { return false } l.mu.Lock() defer l.mu.Unlock() l.expireAt = expirationTime return true } // Sends a notification to lessee about expired lease // Returns true if notification was sent, returns false if the lease is still valid and notification was not sent. func (l *Lease) NotifyExpiration() bool { if l.IsValid() { return false } l.once.Do(l.closeCh) return true } func (l *Lease) closeCh() { close(l.ch) } // Done returns a communication channel from which the lessee can read to get notified when lessor notifies about lease expiration. func (l *Lease) Done() <-chan struct{} { return l.ch } // Deadline returns the expiration time of the lease. func (l *Lease) Deadline() time.Time { l.mu.Lock() defer l.mu.Unlock() return l.expireAt } // IsValid returns true if the lease's expiration time is in the future or equals to the current time, // returns false otherwise. func (l *Lease) IsValid() bool { now := l.Clock.Now() l.mu.Lock() defer l.mu.Unlock() return l.expireAt.After(now) || l.expireAt.Equal(now) } // Broker is a message broker that supports operations to manage task queues. // // See rdb.RDB as a reference implementation. type Broker interface { Ping() error Close() error Enqueue(ctx context.Context, msg *TaskMessage) error EnqueueUnique(ctx context.Context, msg *TaskMessage, ttl time.Duration) error Dequeue(qnames ...string) (*TaskMessage, time.Time, error) Done(ctx context.Context, msg *TaskMessage) error MarkAsComplete(ctx context.Context, msg *TaskMessage) error Requeue(ctx context.Context, msg *TaskMessage) error Schedule(ctx context.Context, msg *TaskMessage, processAt time.Time) error ScheduleUnique(ctx context.Context, msg *TaskMessage, processAt time.Time, ttl time.Duration) error Retry(ctx context.Context, msg *TaskMessage, processAt time.Time, errMsg string, isFailure bool) error Archive(ctx context.Context, msg *TaskMessage, errMsg string) error ForwardIfReady(qnames ...string) error // Group aggregation related methods AddToGroup(ctx context.Context, msg *TaskMessage, gname string) error AddToGroupUnique(ctx context.Context, msg *TaskMessage, groupKey string, ttl time.Duration) error ListGroups(qname string) ([]string, error) AggregationCheck(qname, gname string, t time.Time, gracePeriod, maxDelay time.Duration, maxSize int) (aggregationSetID string, err error) ReadAggregationSet(qname, gname, aggregationSetID string) ([]*TaskMessage, time.Time, error) DeleteAggregationSet(ctx context.Context, qname, gname, aggregationSetID string) error ReclaimStaleAggregationSets(qname string) error // Task retention related method DeleteExpiredCompletedTasks(qname string, batchSize int) error // Lease related methods ListLeaseExpired(cutoff time.Time, qnames ...string) ([]*TaskMessage, error) ExtendLease(qname string, ids ...string) (time.Time, error) // State snapshot related methods WriteServerState(info *ServerInfo, workers []*WorkerInfo, ttl time.Duration) error ClearServerState(host string, pid int, serverID string) error // Cancelation related methods CancelationPubSub() (*redis.PubSub, error) // TODO: Need to decouple from redis to support other brokers PublishCancelation(id string) error WriteResult(qname, id string, data []byte) (n int, err error) } ================================================ FILE: internal/base/base_test.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package base import ( "context" "crypto/md5" "encoding/hex" "encoding/json" "fmt" "sync" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/google/uuid" "github.com/hibiken/asynq/internal/timeutil" ) func TestTaskKey(t *testing.T) { id := uuid.NewString() tests := []struct { qname string id string want string }{ {"default", id, fmt.Sprintf("asynq:{default}:t:%s", id)}, } for _, tc := range tests { got := TaskKey(tc.qname, tc.id) if got != tc.want { t.Errorf("TaskKey(%q, %s) = %q, want %q", tc.qname, tc.id, got, tc.want) } } } func TestQueueKey(t *testing.T) { tests := []struct { qname string want string }{ {"default", "asynq:{default}:pending"}, {"custom", "asynq:{custom}:pending"}, } for _, tc := range tests { got := PendingKey(tc.qname) if got != tc.want { t.Errorf("QueueKey(%q) = %q, want %q", tc.qname, got, tc.want) } } } func TestActiveKey(t *testing.T) { tests := []struct { qname string want string }{ {"default", "asynq:{default}:active"}, {"custom", "asynq:{custom}:active"}, } for _, tc := range tests { got := ActiveKey(tc.qname) if got != tc.want { t.Errorf("ActiveKey(%q) = %q, want %q", tc.qname, got, tc.want) } } } func TestLeaseKey(t *testing.T) { tests := []struct { qname string want string }{ {"default", "asynq:{default}:lease"}, {"custom", "asynq:{custom}:lease"}, } for _, tc := range tests { got := LeaseKey(tc.qname) if got != tc.want { t.Errorf("LeaseKey(%q) = %q, want %q", tc.qname, got, tc.want) } } } func TestScheduledKey(t *testing.T) { tests := []struct { qname string want string }{ {"default", "asynq:{default}:scheduled"}, {"custom", "asynq:{custom}:scheduled"}, } for _, tc := range tests { got := ScheduledKey(tc.qname) if got != tc.want { t.Errorf("ScheduledKey(%q) = %q, want %q", tc.qname, got, tc.want) } } } func TestRetryKey(t *testing.T) { tests := []struct { qname string want string }{ {"default", "asynq:{default}:retry"}, {"custom", "asynq:{custom}:retry"}, } for _, tc := range tests { got := RetryKey(tc.qname) if got != tc.want { t.Errorf("RetryKey(%q) = %q, want %q", tc.qname, got, tc.want) } } } func TestArchivedKey(t *testing.T) { tests := []struct { qname string want string }{ {"default", "asynq:{default}:archived"}, {"custom", "asynq:{custom}:archived"}, } for _, tc := range tests { got := ArchivedKey(tc.qname) if got != tc.want { t.Errorf("ArchivedKey(%q) = %q, want %q", tc.qname, got, tc.want) } } } func TestCompletedKey(t *testing.T) { tests := []struct { qname string want string }{ {"default", "asynq:{default}:completed"}, {"custom", "asynq:{custom}:completed"}, } for _, tc := range tests { got := CompletedKey(tc.qname) if got != tc.want { t.Errorf("CompletedKey(%q) = %q, want %q", tc.qname, got, tc.want) } } } func TestPausedKey(t *testing.T) { tests := []struct { qname string want string }{ {"default", "asynq:{default}:paused"}, {"custom", "asynq:{custom}:paused"}, } for _, tc := range tests { got := PausedKey(tc.qname) if got != tc.want { t.Errorf("PausedKey(%q) = %q, want %q", tc.qname, got, tc.want) } } } func TestProcessedTotalKey(t *testing.T) { tests := []struct { qname string want string }{ {"default", "asynq:{default}:processed"}, {"custom", "asynq:{custom}:processed"}, } for _, tc := range tests { got := ProcessedTotalKey(tc.qname) if got != tc.want { t.Errorf("ProcessedTotalKey(%q) = %q, want %q", tc.qname, got, tc.want) } } } func TestFailedTotalKey(t *testing.T) { tests := []struct { qname string want string }{ {"default", "asynq:{default}:failed"}, {"custom", "asynq:{custom}:failed"}, } for _, tc := range tests { got := FailedTotalKey(tc.qname) if got != tc.want { t.Errorf("FailedTotalKey(%q) = %q, want %q", tc.qname, got, tc.want) } } } func TestProcessedKey(t *testing.T) { tests := []struct { qname string input time.Time want string }{ {"default", time.Date(2019, 11, 14, 10, 30, 1, 1, time.UTC), "asynq:{default}:processed:2019-11-14"}, {"critical", time.Date(2020, 12, 1, 1, 0, 1, 1, time.UTC), "asynq:{critical}:processed:2020-12-01"}, {"default", time.Date(2020, 1, 6, 15, 02, 1, 1, time.UTC), "asynq:{default}:processed:2020-01-06"}, } for _, tc := range tests { got := ProcessedKey(tc.qname, tc.input) if got != tc.want { t.Errorf("ProcessedKey(%v) = %q, want %q", tc.input, got, tc.want) } } } func TestFailedKey(t *testing.T) { tests := []struct { qname string input time.Time want string }{ {"default", time.Date(2019, 11, 14, 10, 30, 1, 1, time.UTC), "asynq:{default}:failed:2019-11-14"}, {"custom", time.Date(2020, 12, 1, 1, 0, 1, 1, time.UTC), "asynq:{custom}:failed:2020-12-01"}, {"low", time.Date(2020, 1, 6, 15, 02, 1, 1, time.UTC), "asynq:{low}:failed:2020-01-06"}, } for _, tc := range tests { got := FailedKey(tc.qname, tc.input) if got != tc.want { t.Errorf("FailureKey(%v) = %q, want %q", tc.input, got, tc.want) } } } func TestServerInfoKey(t *testing.T) { tests := []struct { hostname string pid int sid string want string }{ {"localhost", 9876, "server123", "asynq:servers:{localhost:9876:server123}"}, {"127.0.0.1", 1234, "server987", "asynq:servers:{127.0.0.1:1234:server987}"}, } for _, tc := range tests { got := ServerInfoKey(tc.hostname, tc.pid, tc.sid) if got != tc.want { t.Errorf("ServerInfoKey(%q, %d, %q) = %q, want %q", tc.hostname, tc.pid, tc.sid, got, tc.want) } } } func TestWorkersKey(t *testing.T) { tests := []struct { hostname string pid int sid string want string }{ {"localhost", 9876, "server1", "asynq:workers:{localhost:9876:server1}"}, {"127.0.0.1", 1234, "server2", "asynq:workers:{127.0.0.1:1234:server2}"}, } for _, tc := range tests { got := WorkersKey(tc.hostname, tc.pid, tc.sid) if got != tc.want { t.Errorf("WorkersKey(%q, %d, %q) = %q, want = %q", tc.hostname, tc.pid, tc.sid, got, tc.want) } } } func TestSchedulerEntriesKey(t *testing.T) { tests := []struct { schedulerID string want string }{ {"localhost:9876:scheduler123", "asynq:schedulers:{localhost:9876:scheduler123}"}, {"127.0.0.1:1234:scheduler987", "asynq:schedulers:{127.0.0.1:1234:scheduler987}"}, } for _, tc := range tests { got := SchedulerEntriesKey(tc.schedulerID) if got != tc.want { t.Errorf("SchedulerEntriesKey(%q) = %q, want %q", tc.schedulerID, got, tc.want) } } } func TestSchedulerHistoryKey(t *testing.T) { tests := []struct { entryID string want string }{ {"entry876", "asynq:scheduler_history:entry876"}, {"entry345", "asynq:scheduler_history:entry345"}, } for _, tc := range tests { got := SchedulerHistoryKey(tc.entryID) if got != tc.want { t.Errorf("SchedulerHistoryKey(%q) = %q, want %q", tc.entryID, got, tc.want) } } } func toBytes(m map[string]interface{}) []byte { b, err := json.Marshal(m) if err != nil { panic(err) } return b } func TestUniqueKey(t *testing.T) { payload1 := toBytes(map[string]interface{}{"a": 123, "b": "hello", "c": true}) payload2 := toBytes(map[string]interface{}{"b": "hello", "c": true, "a": 123}) payload3 := toBytes(map[string]interface{}{ "address": map[string]string{"line": "123 Main St", "city": "Boston", "state": "MA"}, "names": []string{"bob", "mike", "rob"}}) payload4 := toBytes(map[string]interface{}{ "time": time.Date(2020, time.July, 28, 0, 0, 0, 0, time.UTC), "duration": time.Hour}) checksum := func(data []byte) string { sum := md5.Sum(data) return hex.EncodeToString(sum[:]) } tests := []struct { desc string qname string tasktype string payload []byte want string }{ { "with primitive types", "default", "email:send", payload1, fmt.Sprintf("asynq:{default}:unique:email:send:%s", checksum(payload1)), }, { "with unsorted keys", "default", "email:send", payload2, fmt.Sprintf("asynq:{default}:unique:email:send:%s", checksum(payload2)), }, { "with composite types", "default", "email:send", payload3, fmt.Sprintf("asynq:{default}:unique:email:send:%s", checksum(payload3)), }, { "with complex types", "default", "email:send", payload4, fmt.Sprintf("asynq:{default}:unique:email:send:%s", checksum(payload4)), }, { "with nil payload", "default", "reindex", nil, "asynq:{default}:unique:reindex:", }, } for _, tc := range tests { got := UniqueKey(tc.qname, tc.tasktype, tc.payload) if got != tc.want { t.Errorf("%s: UniqueKey(%q, %q, %v) = %q, want %q", tc.desc, tc.qname, tc.tasktype, tc.payload, got, tc.want) } } } func TestGroupKey(t *testing.T) { tests := []struct { qname string gkey string want string }{ { qname: "default", gkey: "mygroup", want: "asynq:{default}:g:mygroup", }, { qname: "custom", gkey: "foo", want: "asynq:{custom}:g:foo", }, } for _, tc := range tests { got := GroupKey(tc.qname, tc.gkey) if got != tc.want { t.Errorf("GroupKey(%q, %q) = %q, want %q", tc.qname, tc.gkey, got, tc.want) } } } func TestAggregationSetKey(t *testing.T) { tests := []struct { qname string gname string setID string want string }{ { qname: "default", gname: "mygroup", setID: "12345", want: "asynq:{default}:g:mygroup:12345", }, { qname: "custom", gname: "foo", setID: "98765", want: "asynq:{custom}:g:foo:98765", }, } for _, tc := range tests { got := AggregationSetKey(tc.qname, tc.gname, tc.setID) if got != tc.want { t.Errorf("AggregationSetKey(%q, %q, %q) = %q, want %q", tc.qname, tc.gname, tc.setID, got, tc.want) } } } func TestAllGroups(t *testing.T) { tests := []struct { qname string want string }{ { qname: "default", want: "asynq:{default}:groups", }, { qname: "custom", want: "asynq:{custom}:groups", }, } for _, tc := range tests { got := AllGroups(tc.qname) if got != tc.want { t.Errorf("AllGroups(%q) = %q, want %q", tc.qname, got, tc.want) } } } func TestAllAggregationSets(t *testing.T) { tests := []struct { qname string want string }{ { qname: "default", want: "asynq:{default}:aggregation_sets", }, { qname: "custom", want: "asynq:{custom}:aggregation_sets", }, } for _, tc := range tests { got := AllAggregationSets(tc.qname) if got != tc.want { t.Errorf("AllAggregationSets(%q) = %q, want %q", tc.qname, got, tc.want) } } } func TestMessageEncoding(t *testing.T) { id := uuid.NewString() tests := []struct { in *TaskMessage out *TaskMessage }{ { in: &TaskMessage{ Type: "task1", Payload: toBytes(map[string]interface{}{"a": 1, "b": "hello!", "c": true}), ID: id, Queue: "default", GroupKey: "mygroup", Retry: 10, Retried: 0, Timeout: 1800, Deadline: 1692311100, Retention: 3600, }, out: &TaskMessage{ Type: "task1", Payload: toBytes(map[string]interface{}{"a": json.Number("1"), "b": "hello!", "c": true}), ID: id, Queue: "default", GroupKey: "mygroup", Retry: 10, Retried: 0, Timeout: 1800, Deadline: 1692311100, Retention: 3600, }, }, } for _, tc := range tests { encoded, err := EncodeMessage(tc.in) if err != nil { t.Errorf("EncodeMessage(msg) returned error: %v", err) continue } decoded, err := DecodeMessage(encoded) if err != nil { t.Errorf("DecodeMessage(encoded) returned error: %v", err) continue } if diff := cmp.Diff(tc.out, decoded); diff != "" { t.Errorf("Decoded message == %+v, want %+v;(-want,+got)\n%s", decoded, tc.out, diff) } } } func TestServerInfoEncoding(t *testing.T) { tests := []struct { info ServerInfo }{ { info: ServerInfo{ Host: "127.0.0.1", PID: 9876, ServerID: "abc123", Concurrency: 10, Queues: map[string]int{"default": 1, "critical": 2}, StrictPriority: false, Status: "active", Started: time.Now().Add(-3 * time.Hour), ActiveWorkerCount: 8, }, }, } for _, tc := range tests { encoded, err := EncodeServerInfo(&tc.info) if err != nil { t.Errorf("EncodeServerInfo(info) returned error: %v", err) continue } decoded, err := DecodeServerInfo(encoded) if err != nil { t.Errorf("DecodeServerInfo(encoded) returned error: %v", err) continue } if diff := cmp.Diff(&tc.info, decoded); diff != "" { t.Errorf("Decoded ServerInfo == %+v, want %+v;(-want,+got)\n%s", decoded, tc.info, diff) } } } func TestWorkerInfoEncoding(t *testing.T) { tests := []struct { info WorkerInfo }{ { info: WorkerInfo{ Host: "127.0.0.1", PID: 9876, ServerID: "abc123", ID: uuid.NewString(), Type: "taskA", Payload: toBytes(map[string]interface{}{"foo": "bar"}), Queue: "default", Started: time.Now().Add(-3 * time.Hour), Deadline: time.Now().Add(30 * time.Second), }, }, } for _, tc := range tests { encoded, err := EncodeWorkerInfo(&tc.info) if err != nil { t.Errorf("EncodeWorkerInfo(info) returned error: %v", err) continue } decoded, err := DecodeWorkerInfo(encoded) if err != nil { t.Errorf("DecodeWorkerInfo(encoded) returned error: %v", err) continue } if diff := cmp.Diff(&tc.info, decoded); diff != "" { t.Errorf("Decoded WorkerInfo == %+v, want %+v;(-want,+got)\n%s", decoded, tc.info, diff) } } } func TestSchedulerEntryEncoding(t *testing.T) { tests := []struct { entry SchedulerEntry }{ { entry: SchedulerEntry{ ID: uuid.NewString(), Spec: "* * * * *", Type: "task_A", Payload: toBytes(map[string]interface{}{"foo": "bar"}), Opts: []string{"Queue('email')"}, Next: time.Now().Add(30 * time.Second).UTC(), Prev: time.Now().Add(-2 * time.Minute).UTC(), }, }, } for _, tc := range tests { encoded, err := EncodeSchedulerEntry(&tc.entry) if err != nil { t.Errorf("EncodeSchedulerEntry(entry) returned error: %v", err) continue } decoded, err := DecodeSchedulerEntry(encoded) if err != nil { t.Errorf("DecodeSchedulerEntry(encoded) returned error: %v", err) continue } if diff := cmp.Diff(&tc.entry, decoded); diff != "" { t.Errorf("Decoded SchedulerEntry == %+v, want %+v;(-want,+got)\n%s", decoded, tc.entry, diff) } } } func TestSchedulerEnqueueEventEncoding(t *testing.T) { tests := []struct { event SchedulerEnqueueEvent }{ { event: SchedulerEnqueueEvent{ TaskID: uuid.NewString(), EnqueuedAt: time.Now().Add(-30 * time.Second).UTC(), }, }, } for _, tc := range tests { encoded, err := EncodeSchedulerEnqueueEvent(&tc.event) if err != nil { t.Errorf("EncodeSchedulerEnqueueEvent(event) returned error: %v", err) continue } decoded, err := DecodeSchedulerEnqueueEvent(encoded) if err != nil { t.Errorf("DecodeSchedulerEnqueueEvent(encoded) returned error: %v", err) continue } if diff := cmp.Diff(&tc.event, decoded); diff != "" { t.Errorf("Decoded SchedulerEnqueueEvent == %+v, want %+v;(-want,+got)\n%s", decoded, tc.event, diff) } } } // Test for cancelations being accessed by multiple goroutines. // Run with -race flag to check for data race. func TestCancelationsConcurrentAccess(t *testing.T) { c := NewCancelations() _, cancel1 := context.WithCancel(context.Background()) _, cancel2 := context.WithCancel(context.Background()) _, cancel3 := context.WithCancel(context.Background()) var key1, key2, key3 = "key1", "key2", "key3" var wg sync.WaitGroup wg.Add(1) go func() { defer wg.Done() c.Add(key1, cancel1) }() wg.Add(1) go func() { defer wg.Done() c.Add(key2, cancel2) time.Sleep(200 * time.Millisecond) c.Delete(key2) }() wg.Add(1) go func() { defer wg.Done() c.Add(key3, cancel3) }() wg.Wait() _, ok := c.Get(key1) if !ok { t.Errorf("(*Cancelations).Get(%q) = _, false, want , true", key1) } _, ok = c.Get(key2) if ok { t.Errorf("(*Cancelations).Get(%q) = _, true, want , false", key2) } } func TestLeaseReset(t *testing.T) { now := time.Now() clock := timeutil.NewSimulatedClock(now) l := NewLease(now.Add(30 * time.Second)) l.Clock = clock // Check initial state if !l.IsValid() { t.Errorf("lease should be valid when expiration is set to a future time") } if want := now.Add(30 * time.Second); l.Deadline() != want { t.Errorf("Lease.Deadline() = %v, want %v", l.Deadline(), want) } // Test Reset if !l.Reset(now.Add(45 * time.Second)) { t.Fatalf("Lease.Reset returned false when extending") } if want := now.Add(45 * time.Second); l.Deadline() != want { t.Errorf("After Reset: Lease.Deadline() = %v, want %v", l.Deadline(), want) } clock.AdvanceTime(1 * time.Minute) // simulate lease expiration if l.IsValid() { t.Errorf("lease should be invalid after expiration") } // Reset should return false if lease is expired. if l.Reset(time.Now().Add(20 * time.Second)) { t.Errorf("Lease.Reset should return false after expiration") } } func TestLeaseNotifyExpiration(t *testing.T) { now := time.Now() clock := timeutil.NewSimulatedClock(now) l := NewLease(now.Add(30 * time.Second)) l.Clock = clock select { case <-l.Done(): t.Fatalf("Lease.Done() did not block") default: } if l.NotifyExpiration() { t.Fatalf("Lease.NotifyExpiration() should return false when lease is still valid") } clock.AdvanceTime(1 * time.Minute) // simulate lease expiration if l.IsValid() { t.Errorf("Lease should be invalid after expiration") } if !l.NotifyExpiration() { t.Errorf("Lease.NotifyExpiration() return return true after expiration") } if !l.NotifyExpiration() { t.Errorf("It should be leagal to call Lease.NotifyExpiration multiple times") } select { case <-l.Done(): // expected default: t.Errorf("Lease.Done() blocked after call to Lease.NotifyExpiration()") } } ================================================ FILE: internal/context/context.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package context import ( "context" "time" "github.com/hibiken/asynq/internal/base" ) // A taskMetadata holds task scoped data to put in context. type taskMetadata struct { id string maxRetry int retryCount int qname string } // ctxKey type is unexported to prevent collisions with context keys defined in // other packages. type ctxKey int // metadataCtxKey is the context key for the task metadata. // Its value of zero is arbitrary. const metadataCtxKey ctxKey = 0 // New returns a context and cancel function for a given task message. func New(base context.Context, msg *base.TaskMessage, deadline time.Time) (context.Context, context.CancelFunc) { metadata := taskMetadata{ id: msg.ID, maxRetry: msg.Retry, retryCount: msg.Retried, qname: msg.Queue, } ctx := context.WithValue(base, metadataCtxKey, metadata) return context.WithDeadline(ctx, deadline) } // GetTaskID extracts a task ID from a context, if any. // // ID of a task is guaranteed to be unique. // ID of a task doesn't change if the task is being retried. func GetTaskID(ctx context.Context) (id string, ok bool) { metadata, ok := ctx.Value(metadataCtxKey).(taskMetadata) if !ok { return "", false } return metadata.id, true } // GetRetryCount extracts retry count from a context, if any. // // Return value n indicates the number of times associated task has been // retried so far. func GetRetryCount(ctx context.Context) (n int, ok bool) { metadata, ok := ctx.Value(metadataCtxKey).(taskMetadata) if !ok { return 0, false } return metadata.retryCount, true } // GetMaxRetry extracts maximum retry from a context, if any. // // Return value n indicates the maximum number of times the associated task // can be retried if ProcessTask returns a non-nil error. func GetMaxRetry(ctx context.Context) (n int, ok bool) { metadata, ok := ctx.Value(metadataCtxKey).(taskMetadata) if !ok { return 0, false } return metadata.maxRetry, true } // GetQueueName extracts queue name from a context, if any. // // Return value qname indicates which queue the task was pulled from. func GetQueueName(ctx context.Context) (qname string, ok bool) { metadata, ok := ctx.Value(metadataCtxKey).(taskMetadata) if !ok { return "", false } return metadata.qname, true } ================================================ FILE: internal/context/context_test.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package context import ( "context" "fmt" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/google/uuid" "github.com/hibiken/asynq/internal/base" ) func TestCreateContextWithFutureDeadline(t *testing.T) { tests := []struct { deadline time.Time }{ {time.Now().Add(time.Hour)}, } for _, tc := range tests { msg := &base.TaskMessage{ Type: "something", ID: uuid.NewString(), Payload: nil, } ctx, cancel := New(context.Background(), msg, tc.deadline) select { case x := <-ctx.Done(): t.Errorf("<-ctx.Done() == %v, want nothing (it should block)", x) default: } got, ok := ctx.Deadline() if !ok { t.Errorf("ctx.Deadline() returned false, want deadline to be set") } if !cmp.Equal(tc.deadline, got) { t.Errorf("ctx.Deadline() returned %v, want %v", got, tc.deadline) } cancel() select { case <-ctx.Done(): default: t.Errorf("ctx.Done() blocked, want it to be non-blocking") } } } func TestCreateContextWithBaseContext(t *testing.T) { type ctxKey string type ctxValue string var key ctxKey = "key" var value ctxValue = "value" tests := []struct { baseCtx context.Context validate func(ctx context.Context, t *testing.T) error }{ { baseCtx: context.WithValue(context.Background(), key, value), validate: func(ctx context.Context, t *testing.T) error { got, ok := ctx.Value(key).(ctxValue) if !ok { return fmt.Errorf("ctx.Value().(ctxValue) returned false, expected to be true") } if want := value; got != want { return fmt.Errorf("ctx.Value().(ctxValue) returned unknown value (%v), expected to be %s", got, value) } return nil }, }, } for _, tc := range tests { msg := &base.TaskMessage{ Type: "something", ID: uuid.NewString(), Payload: nil, } ctx, cancel := New(tc.baseCtx, msg, time.Now().Add(30*time.Minute)) defer cancel() select { case x := <-ctx.Done(): t.Errorf("<-ctx.Done() == %v, want nothing (it should block)", x) default: } if err := tc.validate(ctx, t); err != nil { t.Errorf("%v", err) } } } func TestCreateContextWithPastDeadline(t *testing.T) { tests := []struct { deadline time.Time }{ {time.Now().Add(-2 * time.Hour)}, } for _, tc := range tests { msg := &base.TaskMessage{ Type: "something", ID: uuid.NewString(), Payload: nil, } ctx, cancel := New(context.Background(), msg, tc.deadline) defer cancel() select { case <-ctx.Done(): default: t.Errorf("ctx.Done() blocked, want it to be non-blocking") } got, ok := ctx.Deadline() if !ok { t.Errorf("ctx.Deadline() returned false, want deadline to be set") } if !cmp.Equal(tc.deadline, got) { t.Errorf("ctx.Deadline() returned %v, want %v", got, tc.deadline) } } } func TestGetTaskMetadataFromContext(t *testing.T) { tests := []struct { desc string msg *base.TaskMessage }{ {"with zero retried message", &base.TaskMessage{Type: "something", ID: uuid.NewString(), Retry: 25, Retried: 0, Timeout: 1800, Queue: "default"}}, {"with non-zero retried message", &base.TaskMessage{Type: "something", ID: uuid.NewString(), Retry: 10, Retried: 5, Timeout: 1800, Queue: "default"}}, {"with custom queue name", &base.TaskMessage{Type: "something", ID: uuid.NewString(), Retry: 25, Retried: 0, Timeout: 1800, Queue: "custom"}}, } for _, tc := range tests { ctx, cancel := New(context.Background(), tc.msg, time.Now().Add(30*time.Minute)) defer cancel() id, ok := GetTaskID(ctx) if !ok { t.Errorf("%s: GetTaskID(ctx) returned ok == false", tc.desc) } if ok && id != tc.msg.ID { t.Errorf("%s: GetTaskID(ctx) returned id == %q, want %q", tc.desc, id, tc.msg.ID) } retried, ok := GetRetryCount(ctx) if !ok { t.Errorf("%s: GetRetryCount(ctx) returned ok == false", tc.desc) } if ok && retried != tc.msg.Retried { t.Errorf("%s: GetRetryCount(ctx) returned n == %d want %d", tc.desc, retried, tc.msg.Retried) } maxRetry, ok := GetMaxRetry(ctx) if !ok { t.Errorf("%s: GetMaxRetry(ctx) returned ok == false", tc.desc) } if ok && maxRetry != tc.msg.Retry { t.Errorf("%s: GetMaxRetry(ctx) returned n == %d want %d", tc.desc, maxRetry, tc.msg.Retry) } qname, ok := GetQueueName(ctx) if !ok { t.Errorf("%s: GetQueueName(ctx) returned ok == false", tc.desc) } if ok && qname != tc.msg.Queue { t.Errorf("%s: GetQueueName(ctx) returned qname == %q, want %q", tc.desc, qname, tc.msg.Queue) } } } func TestGetTaskMetadataFromContextError(t *testing.T) { tests := []struct { desc string ctx context.Context }{ {"with background context", context.Background()}, } for _, tc := range tests { if _, ok := GetTaskID(tc.ctx); ok { t.Errorf("%s: GetTaskID(ctx) returned ok == true", tc.desc) } if _, ok := GetRetryCount(tc.ctx); ok { t.Errorf("%s: GetRetryCount(ctx) returned ok == true", tc.desc) } if _, ok := GetMaxRetry(tc.ctx); ok { t.Errorf("%s: GetMaxRetry(ctx) returned ok == true", tc.desc) } if _, ok := GetQueueName(tc.ctx); ok { t.Errorf("%s: GetQueueName(ctx) returned ok == true", tc.desc) } } } ================================================ FILE: internal/errors/errors.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. // Package errors defines the error type and functions used by // asynq and its internal packages. package errors // Note: This package is inspired by a blog post about error handling in project Upspin // https://commandcenter.blogspot.com/2017/12/error-handling-in-upspin.html. import ( "errors" "fmt" "log" "runtime" "strings" ) // Error is the type that implements the error interface. // It contains a number of fields, each of different type. // An Error value may leave some values unset. type Error struct { Code Code Op Op Err error } func (e *Error) DebugString() string { var b strings.Builder if e.Op != "" { b.WriteString(string(e.Op)) } if e.Code != Unspecified { if b.Len() > 0 { b.WriteString(": ") } b.WriteString(e.Code.String()) } if e.Err != nil { if b.Len() > 0 { b.WriteString(": ") } b.WriteString(e.Err.Error()) } return b.String() } func (e *Error) Error() string { var b strings.Builder if e.Code != Unspecified { b.WriteString(e.Code.String()) } if e.Err != nil { if b.Len() > 0 { b.WriteString(": ") } b.WriteString(e.Err.Error()) } return b.String() } func (e *Error) Unwrap() error { return e.Err } // Code defines the canonical error code. type Code uint8 // List of canonical error codes. const ( Unspecified Code = iota NotFound FailedPrecondition Internal AlreadyExists Unknown // Note: If you add a new value here, make sure to update String method. ) func (c Code) String() string { switch c { case Unspecified: return "ERROR_CODE_UNSPECIFIED" case NotFound: return "NOT_FOUND" case FailedPrecondition: return "FAILED_PRECONDITION" case Internal: return "INTERNAL_ERROR" case AlreadyExists: return "ALREADY_EXISTS" case Unknown: return "UNKNOWN" } panic(fmt.Sprintf("unknown error code %d", c)) } // Op describes an operation, usually as the package and method, // such as "rdb.Enqueue". type Op string // E builds an error value from its arguments. // There must be at least one argument or E panics. // The type of each argument determines its meaning. // If more than one argument of a given type is presented, // only the last one is recorded. // // The types are: // // errors.Op // The operation being performed, usually the method // being invoked (Get, Put, etc.). // errors.Code // The canonical error code, such as NOT_FOUND. // string // Treated as an error message and assigned to the // Err field after a call to errors.New. // error // The underlying error that triggered this one. // // If the error is printed, only those items that have been // set to non-zero values will appear in the result. func E(args ...interface{}) error { if len(args) == 0 { panic("call to errors.E with no arguments") } e := &Error{} for _, arg := range args { switch arg := arg.(type) { case Op: e.Op = arg case Code: e.Code = arg case error: e.Err = arg case string: e.Err = errors.New(arg) default: _, file, line, _ := runtime.Caller(1) log.Printf("errors.E: bad call from %s:%d: %v", file, line, args) return fmt.Errorf("unknown type %T, value %v in error call", arg, arg) } } return e } // CanonicalCode returns the canonical code of the given error if one is present. // Otherwise it returns Unspecified. func CanonicalCode(err error) Code { if err == nil { return Unspecified } e, ok := err.(*Error) if !ok { return Unspecified } if e.Code == Unspecified { return CanonicalCode(e.Err) } return e.Code } /****************************************** Domain Specific Error Types & Values *******************************************/ var ( // ErrNoProcessableTask indicates that there are no tasks ready to be processed. ErrNoProcessableTask = errors.New("no tasks are ready for processing") // ErrDuplicateTask indicates that another task with the same unique key holds the uniqueness lock. ErrDuplicateTask = errors.New("task already exists") // ErrTaskIdConflict indicates that another task with the same task ID already exist ErrTaskIdConflict = errors.New("task id conflicts with another task") ) // TaskNotFoundError indicates that a task with the given ID does not exist // in the given queue. type TaskNotFoundError struct { Queue string // queue name ID string // task id } func (e *TaskNotFoundError) Error() string { return fmt.Sprintf("cannot find task with id=%s in queue %q", e.ID, e.Queue) } // IsTaskNotFound reports whether any error in err's chain is of type TaskNotFoundError. func IsTaskNotFound(err error) bool { var target *TaskNotFoundError return As(err, &target) } // QueueNotFoundError indicates that a queue with the given name does not exist. type QueueNotFoundError struct { Queue string // queue name } func (e *QueueNotFoundError) Error() string { return fmt.Sprintf("queue %q does not exist", e.Queue) } // IsQueueNotFound reports whether any error in err's chain is of type QueueNotFoundError. func IsQueueNotFound(err error) bool { var target *QueueNotFoundError return As(err, &target) } // QueueNotEmptyError indicates that the given queue is not empty. type QueueNotEmptyError struct { Queue string // queue name } func (e *QueueNotEmptyError) Error() string { return fmt.Sprintf("queue %q is not empty", e.Queue) } // IsQueueNotEmpty reports whether any error in err's chain is of type QueueNotEmptyError. func IsQueueNotEmpty(err error) bool { var target *QueueNotEmptyError return As(err, &target) } // TaskAlreadyArchivedError indicates that the task in question is already archived. type TaskAlreadyArchivedError struct { Queue string // queue name ID string // task id } func (e *TaskAlreadyArchivedError) Error() string { return fmt.Sprintf("task is already archived: id=%s, queue=%s", e.ID, e.Queue) } // IsTaskAlreadyArchived reports whether any error in err's chain is of type TaskAlreadyArchivedError. func IsTaskAlreadyArchived(err error) bool { var target *TaskAlreadyArchivedError return As(err, &target) } // RedisCommandError indicates that the given redis command returned error. type RedisCommandError struct { Command string // redis command (e.g. LRANGE, ZADD, etc) Err error // underlying error } func (e *RedisCommandError) Error() string { return fmt.Sprintf("redis command error: %s failed: %v", strings.ToUpper(e.Command), e.Err) } func (e *RedisCommandError) Unwrap() error { return e.Err } // IsRedisCommandError reports whether any error in err's chain is of type RedisCommandError. func IsRedisCommandError(err error) bool { var target *RedisCommandError return As(err, &target) } // PanicError defines an error when occurred a panic error. type PanicError struct { ErrMsg string } func (e *PanicError) Error() string { return fmt.Sprintf("panic error cause by: %s", e.ErrMsg) } // IsPanicError reports whether any error in err's chain is of type PanicError. func IsPanicError(err error) bool { var target *PanicError return As(err, &target) } /************************************************* Standard Library errors package functions *************************************************/ // New returns an error that formats as the given text. // Each call to New returns a distinct error value even if the text is identical. // // This function is the errors.New function from the standard library (https://golang.org/pkg/errors/#New). // It is exported from this package for import convenience. func New(text string) error { return errors.New(text) } // Is reports whether any error in err's chain matches target. // // This function is the errors.Is function from the standard library (https://golang.org/pkg/errors/#Is). // It is exported from this package for import convenience. func Is(err, target error) bool { return errors.Is(err, target) } // As finds the first error in err's chain that matches target, and if so, sets target to that error value and returns true. // Otherwise, it returns false. // // This function is the errors.As function from the standard library (https://golang.org/pkg/errors/#As). // It is exported from this package for import convenience. func As(err error, target interface{}) bool { return errors.As(err, target) } // Unwrap returns the result of calling the Unwrap method on err, if err's type contains an Unwrap method returning error. // Otherwise, Unwrap returns nil. // // This function is the errors.Unwrap function from the standard library (https://golang.org/pkg/errors/#Unwrap). // It is exported from this package for import convenience. func Unwrap(err error) error { return errors.Unwrap(err) } ================================================ FILE: internal/errors/errors_test.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package errors import "testing" func TestErrorDebugString(t *testing.T) { // DebugString should include Op since its meant to be used by // maintainers/contributors of the asynq package. tests := []struct { desc string err error want string }{ { desc: "With Op, Code, and string", err: E(Op("rdb.DeleteTask"), NotFound, "cannot find task with id=123"), want: "rdb.DeleteTask: NOT_FOUND: cannot find task with id=123", }, { desc: "With Op, Code and error", err: E(Op("rdb.DeleteTask"), NotFound, &TaskNotFoundError{Queue: "default", ID: "123"}), want: `rdb.DeleteTask: NOT_FOUND: cannot find task with id=123 in queue "default"`, }, } for _, tc := range tests { if got := tc.err.(*Error).DebugString(); got != tc.want { t.Errorf("%s: got=%q, want=%q", tc.desc, got, tc.want) } } } func TestErrorString(t *testing.T) { // String method should omit Op since op is an internal detail // and we don't want to provide it to users of the package. tests := []struct { desc string err error want string }{ { desc: "With Op, Code, and string", err: E(Op("rdb.DeleteTask"), NotFound, "cannot find task with id=123"), want: "NOT_FOUND: cannot find task with id=123", }, { desc: "With Op, Code and error", err: E(Op("rdb.DeleteTask"), NotFound, &TaskNotFoundError{Queue: "default", ID: "123"}), want: `NOT_FOUND: cannot find task with id=123 in queue "default"`, }, } for _, tc := range tests { if got := tc.err.Error(); got != tc.want { t.Errorf("%s: got=%q, want=%q", tc.desc, got, tc.want) } } } func TestErrorIs(t *testing.T) { var ErrCustom = New("custom sentinel error") tests := []struct { desc string err error target error want bool }{ { desc: "should unwrap one level", err: E(Op("rdb.DeleteTask"), ErrCustom), target: ErrCustom, want: true, }, } for _, tc := range tests { if got := Is(tc.err, tc.target); got != tc.want { t.Errorf("%s: got=%t, want=%t", tc.desc, got, tc.want) } } } func TestErrorAs(t *testing.T) { tests := []struct { desc string err error target interface{} want bool }{ { desc: "should unwrap one level", err: E(Op("rdb.DeleteTask"), NotFound, &QueueNotFoundError{Queue: "email"}), target: &QueueNotFoundError{}, want: true, }, } for _, tc := range tests { if got := As(tc.err, &tc.target); got != tc.want { t.Errorf("%s: got=%t, want=%t", tc.desc, got, tc.want) } } } func TestErrorPredicates(t *testing.T) { tests := []struct { desc string fn func(err error) bool err error want bool }{ { desc: "IsTaskNotFound should detect presence of TaskNotFoundError in err's chain", fn: IsTaskNotFound, err: E(Op("rdb.ArchiveTask"), NotFound, &TaskNotFoundError{Queue: "default", ID: "9876"}), want: true, }, { desc: "IsTaskNotFound should detect absence of TaskNotFoundError in err's chain", fn: IsTaskNotFound, err: E(Op("rdb.ArchiveTask"), NotFound, &QueueNotFoundError{Queue: "default"}), want: false, }, { desc: "IsQueueNotFound should detect presence of QueueNotFoundError in err's chain", fn: IsQueueNotFound, err: E(Op("rdb.ArchiveTask"), NotFound, &QueueNotFoundError{Queue: "default"}), want: true, }, { desc: "IsPanicError should detect presence of PanicError in err's chain", fn: IsPanicError, err: E(Op("unknown"), Unknown, &PanicError{ErrMsg: "Something went wrong"}), want: true, }, } for _, tc := range tests { if got := tc.fn(tc.err); got != tc.want { t.Errorf("%s: got=%t, want=%t", tc.desc, got, tc.want) } } } func TestCanonicalCode(t *testing.T) { tests := []struct { desc string err error want Code }{ { desc: "without nesting", err: E(Op("rdb.DeleteTask"), NotFound, &TaskNotFoundError{Queue: "default", ID: "123"}), want: NotFound, }, { desc: "with nesting", err: E(FailedPrecondition, E(NotFound)), want: FailedPrecondition, }, { desc: "returns Unspecified if err is not *Error", err: New("some other error"), want: Unspecified, }, { desc: "returns Unspecified if err is nil", err: nil, want: Unspecified, }, } for _, tc := range tests { if got := CanonicalCode(tc.err); got != tc.want { t.Errorf("%s: got=%s, want=%s", tc.desc, got, tc.want) } } } ================================================ FILE: internal/log/log.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. // Package log exports logging related types and functions. package log import ( "fmt" "io" stdlog "log" "os" "sync" ) // Base supports logging at various log levels. type Base interface { // Debug logs a message at Debug level. Debug(args ...interface{}) // Info logs a message at Info level. Info(args ...interface{}) // Warn logs a message at Warning level. Warn(args ...interface{}) // Error logs a message at Error level. Error(args ...interface{}) // Fatal logs a message at Fatal level // and process will exit with status set to 1. Fatal(args ...interface{}) } // baseLogger is a wrapper object around log.Logger from the standard library. // It supports logging at various log levels. type baseLogger struct { *stdlog.Logger } // Debug logs a message at Debug level. func (l *baseLogger) Debug(args ...interface{}) { l.prefixPrint("DEBUG: ", args...) } // Info logs a message at Info level. func (l *baseLogger) Info(args ...interface{}) { l.prefixPrint("INFO: ", args...) } // Warn logs a message at Warning level. func (l *baseLogger) Warn(args ...interface{}) { l.prefixPrint("WARN: ", args...) } // Error logs a message at Error level. func (l *baseLogger) Error(args ...interface{}) { l.prefixPrint("ERROR: ", args...) } // Fatal logs a message at Fatal level // and process will exit with status set to 1. func (l *baseLogger) Fatal(args ...interface{}) { l.prefixPrint("FATAL: ", args...) os.Exit(1) } func (l *baseLogger) prefixPrint(prefix string, args ...interface{}) { args = append([]interface{}{prefix}, args...) l.Print(args...) } // newBase creates and returns a new instance of baseLogger. func newBase(out io.Writer) *baseLogger { prefix := fmt.Sprintf("asynq: pid=%d ", os.Getpid()) return &baseLogger{ stdlog.New(out, prefix, stdlog.Ldate|stdlog.Ltime|stdlog.Lmicroseconds|stdlog.LUTC), } } // NewLogger creates and returns a new instance of Logger. // Log level is set to DebugLevel by default. func NewLogger(base Base) *Logger { if base == nil { base = newBase(os.Stderr) } return &Logger{base: base, level: DebugLevel} } // Logger logs message to io.Writer at various log levels. type Logger struct { base Base mu sync.Mutex // Minimum log level for this logger. // Message with level lower than this level won't be outputted. level Level } // Level represents a log level. type Level int32 const ( // DebugLevel is the lowest level of logging. // Debug logs are intended for debugging and development purposes. DebugLevel Level = iota // InfoLevel is used for general informational log messages. InfoLevel // WarnLevel is used for undesired but relatively expected events, // which may indicate a problem. WarnLevel // ErrorLevel is used for undesired and unexpected events that // the program can recover from. ErrorLevel // FatalLevel is used for undesired and unexpected events that // the program cannot recover from. FatalLevel ) // String is part of the fmt.Stringer interface. // // Used for testing and debugging purposes. func (l Level) String() string { switch l { case DebugLevel: return "debug" case InfoLevel: return "info" case WarnLevel: return "warning" case ErrorLevel: return "error" case FatalLevel: return "fatal" default: return "unknown" } } // canLogAt reports whether logger can log at level v. func (l *Logger) canLogAt(v Level) bool { l.mu.Lock() defer l.mu.Unlock() return v >= l.level } func (l *Logger) Debug(args ...interface{}) { if !l.canLogAt(DebugLevel) { return } l.base.Debug(args...) } func (l *Logger) Info(args ...interface{}) { if !l.canLogAt(InfoLevel) { return } l.base.Info(args...) } func (l *Logger) Warn(args ...interface{}) { if !l.canLogAt(WarnLevel) { return } l.base.Warn(args...) } func (l *Logger) Error(args ...interface{}) { if !l.canLogAt(ErrorLevel) { return } l.base.Error(args...) } func (l *Logger) Fatal(args ...interface{}) { if !l.canLogAt(FatalLevel) { return } l.base.Fatal(args...) } func (l *Logger) Debugf(format string, args ...interface{}) { l.Debug(fmt.Sprintf(format, args...)) } func (l *Logger) Infof(format string, args ...interface{}) { l.Info(fmt.Sprintf(format, args...)) } func (l *Logger) Warnf(format string, args ...interface{}) { l.Warn(fmt.Sprintf(format, args...)) } func (l *Logger) Errorf(format string, args ...interface{}) { l.Error(fmt.Sprintf(format, args...)) } func (l *Logger) Fatalf(format string, args ...interface{}) { l.Fatal(fmt.Sprintf(format, args...)) } // SetLevel sets the logger level. // It panics if v is less than DebugLevel or greater than FatalLevel. func (l *Logger) SetLevel(v Level) { l.mu.Lock() defer l.mu.Unlock() if v < DebugLevel || v > FatalLevel { panic("log: invalid log level") } l.level = v } ================================================ FILE: internal/log/log_test.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package log import ( "bytes" "fmt" "regexp" "testing" ) // regexp for timestamps const ( rgxPID = `[0-9]+` rgxdate = `[0-9][0-9][0-9][0-9]/[0-9][0-9]/[0-9][0-9]` rgxtime = `[0-9][0-9]:[0-9][0-9]:[0-9][0-9]` rgxmicroseconds = `\.[0-9][0-9][0-9][0-9][0-9][0-9]` ) type tester struct { desc string message string wantPattern string // regexp that log output must match } func TestLoggerDebug(t *testing.T) { tests := []tester{ { desc: "without trailing newline, logger adds newline", message: "hello, world!", wantPattern: fmt.Sprintf("^asynq: pid=%s %s %s%s DEBUG: hello, world!\n$", rgxPID, rgxdate, rgxtime, rgxmicroseconds), }, { desc: "with trailing newline, logger preserves newline", message: "hello, world!\n", wantPattern: fmt.Sprintf("^asynq: pid=%s %s %s%s DEBUG: hello, world!\n$", rgxPID, rgxdate, rgxtime, rgxmicroseconds), }, } for _, tc := range tests { var buf bytes.Buffer logger := NewLogger(newBase(&buf)) logger.Debug(tc.message) got := buf.String() matched, err := regexp.MatchString(tc.wantPattern, got) if err != nil { t.Fatal("pattern did not compile:", err) } if !matched { t.Errorf("logger.Debug(%q) outputted %q, should match pattern %q", tc.message, got, tc.wantPattern) } } } func TestLoggerInfo(t *testing.T) { tests := []tester{ { desc: "without trailing newline, logger adds newline", message: "hello, world!", wantPattern: fmt.Sprintf("^asynq: pid=%s %s %s%s INFO: hello, world!\n$", rgxPID, rgxdate, rgxtime, rgxmicroseconds), }, { desc: "with trailing newline, logger preserves newline", message: "hello, world!\n", wantPattern: fmt.Sprintf("^asynq: pid=%s %s %s%s INFO: hello, world!\n$", rgxPID, rgxdate, rgxtime, rgxmicroseconds), }, } for _, tc := range tests { var buf bytes.Buffer logger := NewLogger(newBase(&buf)) logger.Info(tc.message) got := buf.String() matched, err := regexp.MatchString(tc.wantPattern, got) if err != nil { t.Fatal("pattern did not compile:", err) } if !matched { t.Errorf("logger.Info(%q) outputted %q, should match pattern %q", tc.message, got, tc.wantPattern) } } } func TestLoggerWarn(t *testing.T) { tests := []tester{ { desc: "without trailing newline, logger adds newline", message: "hello, world!", wantPattern: fmt.Sprintf("^asynq: pid=%s %s %s%s WARN: hello, world!\n$", rgxPID, rgxdate, rgxtime, rgxmicroseconds), }, { desc: "with trailing newline, logger preserves newline", message: "hello, world!\n", wantPattern: fmt.Sprintf("^asynq: pid=%s %s %s%s WARN: hello, world!\n$", rgxPID, rgxdate, rgxtime, rgxmicroseconds), }, } for _, tc := range tests { var buf bytes.Buffer logger := NewLogger(newBase(&buf)) logger.Warn(tc.message) got := buf.String() matched, err := regexp.MatchString(tc.wantPattern, got) if err != nil { t.Fatal("pattern did not compile:", err) } if !matched { t.Errorf("logger.Warn(%q) outputted %q, should match pattern %q", tc.message, got, tc.wantPattern) } } } func TestLoggerError(t *testing.T) { tests := []tester{ { desc: "without trailing newline, logger adds newline", message: "hello, world!", wantPattern: fmt.Sprintf("^asynq: pid=%s %s %s%s ERROR: hello, world!\n$", rgxPID, rgxdate, rgxtime, rgxmicroseconds), }, { desc: "with trailing newline, logger preserves newline", message: "hello, world!\n", wantPattern: fmt.Sprintf("^asynq: pid=%s %s %s%s ERROR: hello, world!\n$", rgxPID, rgxdate, rgxtime, rgxmicroseconds), }, } for _, tc := range tests { var buf bytes.Buffer logger := NewLogger(newBase(&buf)) logger.Error(tc.message) got := buf.String() matched, err := regexp.MatchString(tc.wantPattern, got) if err != nil { t.Fatal("pattern did not compile:", err) } if !matched { t.Errorf("logger.Error(%q) outputted %q, should match pattern %q", tc.message, got, tc.wantPattern) } } } type formatTester struct { desc string format string args []interface{} wantPattern string // regexp that log output must match } func TestLoggerDebugf(t *testing.T) { tests := []formatTester{ { desc: "Formats message with DEBUG prefix", format: "hello, %s!", args: []interface{}{"Gopher"}, wantPattern: fmt.Sprintf("^asynq: pid=%s %s %s%s DEBUG: hello, Gopher!\n$", rgxPID, rgxdate, rgxtime, rgxmicroseconds), }, } for _, tc := range tests { var buf bytes.Buffer logger := NewLogger(newBase(&buf)) logger.Debugf(tc.format, tc.args...) got := buf.String() matched, err := regexp.MatchString(tc.wantPattern, got) if err != nil { t.Fatal("pattern did not compile:", err) } if !matched { t.Errorf("logger.Debugf(%q, %v) outputted %q, should match pattern %q", tc.format, tc.args, got, tc.wantPattern) } } } func TestLoggerInfof(t *testing.T) { tests := []formatTester{ { desc: "Formats message with INFO prefix", format: "%d,%d,%d", args: []interface{}{1, 2, 3}, wantPattern: fmt.Sprintf("^asynq: pid=%s %s %s%s INFO: 1,2,3\n$", rgxPID, rgxdate, rgxtime, rgxmicroseconds), }, } for _, tc := range tests { var buf bytes.Buffer logger := NewLogger(newBase(&buf)) logger.Infof(tc.format, tc.args...) got := buf.String() matched, err := regexp.MatchString(tc.wantPattern, got) if err != nil { t.Fatal("pattern did not compile:", err) } if !matched { t.Errorf("logger.Infof(%q, %v) outputted %q, should match pattern %q", tc.format, tc.args, got, tc.wantPattern) } } } func TestLoggerWarnf(t *testing.T) { tests := []formatTester{ { desc: "Formats message with WARN prefix", format: "hello, %s", args: []interface{}{"Gophers"}, wantPattern: fmt.Sprintf("^asynq: pid=%s %s %s%s WARN: hello, Gophers\n$", rgxPID, rgxdate, rgxtime, rgxmicroseconds), }, } for _, tc := range tests { var buf bytes.Buffer logger := NewLogger(newBase(&buf)) logger.Warnf(tc.format, tc.args...) got := buf.String() matched, err := regexp.MatchString(tc.wantPattern, got) if err != nil { t.Fatal("pattern did not compile:", err) } if !matched { t.Errorf("logger.Warnf(%q, %v) outputted %q, should match pattern %q", tc.format, tc.args, got, tc.wantPattern) } } } func TestLoggerErrorf(t *testing.T) { tests := []formatTester{ { desc: "Formats message with ERROR prefix", format: "hello, %s", args: []interface{}{"Gophers"}, wantPattern: fmt.Sprintf("^asynq: pid=%s %s %s%s ERROR: hello, Gophers\n$", rgxPID, rgxdate, rgxtime, rgxmicroseconds), }, } for _, tc := range tests { var buf bytes.Buffer logger := NewLogger(newBase(&buf)) logger.Errorf(tc.format, tc.args...) got := buf.String() matched, err := regexp.MatchString(tc.wantPattern, got) if err != nil { t.Fatal("pattern did not compile:", err) } if !matched { t.Errorf("logger.Errorf(%q, %v) outputted %q, should match pattern %q", tc.format, tc.args, got, tc.wantPattern) } } } func TestLoggerWithLowerLevels(t *testing.T) { // Logger should not log messages at a level // lower than the specified level. tests := []struct { level Level op string }{ // with level one above {InfoLevel, "Debug"}, {InfoLevel, "Debugf"}, {WarnLevel, "Info"}, {WarnLevel, "Infof"}, {ErrorLevel, "Warn"}, {ErrorLevel, "Warnf"}, {FatalLevel, "Error"}, {FatalLevel, "Errorf"}, // with skip level {WarnLevel, "Debug"}, {ErrorLevel, "Infof"}, } for _, tc := range tests { var buf bytes.Buffer logger := NewLogger(newBase(&buf)) logger.SetLevel(tc.level) switch tc.op { case "Debug": logger.Debug("hello") case "Debugf": logger.Debugf("hello, %s", "world") case "Info": logger.Info("hello") case "Infof": logger.Infof("hello, %s", "world") case "Warn": logger.Warn("hello") case "Warnf": logger.Warnf("hello, %s", "world") case "Error": logger.Error("hello") case "Errorf": logger.Errorf("hello, %s", "world") default: t.Fatalf("unexpected op: %q", tc.op) } if buf.String() != "" { t.Errorf("logger.%s outputted log message when level is set to %v", tc.op, tc.level) } } } func TestLoggerWithSameOrHigherLevels(t *testing.T) { // Logger should log messages at a level // same as or higher than the specified level. tests := []struct { level Level op string }{ // same level {DebugLevel, "Debug"}, {InfoLevel, "Infof"}, {WarnLevel, "Warn"}, {ErrorLevel, "Errorf"}, // higher level {DebugLevel, "Info"}, {InfoLevel, "Warnf"}, {WarnLevel, "Error"}, } for _, tc := range tests { var buf bytes.Buffer logger := NewLogger(newBase(&buf)) logger.SetLevel(tc.level) switch tc.op { case "Debug": logger.Debug("hello") case "Debugf": logger.Debugf("hello, %s", "world") case "Info": logger.Info("hello") case "Infof": logger.Infof("hello, %s", "world") case "Warn": logger.Warn("hello") case "Warnf": logger.Warnf("hello, %s", "world") case "Error": logger.Error("hello") case "Errorf": logger.Errorf("hello, %s", "world") default: t.Fatalf("unexpected op: %q", tc.op) } if buf.String() == "" { t.Errorf("logger.%s did not output log message when level is set to %v", tc.op, tc.level) } } } ================================================ FILE: internal/proto/asynq.pb.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.6 // protoc v5.29.3 // source: asynq.proto package proto import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" timestamppb "google.golang.org/protobuf/types/known/timestamppb" reflect "reflect" sync "sync" unsafe "unsafe" ) const ( // Verify that this generated code is sufficiently up-to-date. _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) // Verify that runtime/protoimpl is sufficiently up-to-date. _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) // TaskMessage is the internal representation of a task with additional // metadata fields. // Next ID: 16 type TaskMessage struct { state protoimpl.MessageState `protogen:"open.v1"` // Type indicates the kind of the task to be performed. Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // Payload holds data needed to process the task. Payload []byte `protobuf:"bytes,2,opt,name=payload,proto3" json:"payload,omitempty"` // Headers holds additional metadata for the task. Headers map[string]string `protobuf:"bytes,15,rep,name=headers,proto3" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // Unique identifier for the task. Id string `protobuf:"bytes,3,opt,name=id,proto3" json:"id,omitempty"` // Name of the queue to which this task belongs. Queue string `protobuf:"bytes,4,opt,name=queue,proto3" json:"queue,omitempty"` // Max number of retries for this task. Retry int32 `protobuf:"varint,5,opt,name=retry,proto3" json:"retry,omitempty"` // Number of times this task has been retried so far. Retried int32 `protobuf:"varint,6,opt,name=retried,proto3" json:"retried,omitempty"` // Error message from the last failure. ErrorMsg string `protobuf:"bytes,7,opt,name=error_msg,json=errorMsg,proto3" json:"error_msg,omitempty"` // Time of last failure in Unix time, // the number of seconds elapsed since January 1, 1970 UTC. // Use zero to indicate no last failure. LastFailedAt int64 `protobuf:"varint,11,opt,name=last_failed_at,json=lastFailedAt,proto3" json:"last_failed_at,omitempty"` // Timeout specifies timeout in seconds. // Use zero to indicate no timeout. Timeout int64 `protobuf:"varint,8,opt,name=timeout,proto3" json:"timeout,omitempty"` // Deadline specifies the deadline for the task in Unix time, // the number of seconds elapsed since January 1, 1970 UTC. // Use zero to indicate no deadline. Deadline int64 `protobuf:"varint,9,opt,name=deadline,proto3" json:"deadline,omitempty"` // UniqueKey holds the redis key used for uniqueness lock for this task. // Empty string indicates that no uniqueness lock was used. UniqueKey string `protobuf:"bytes,10,opt,name=unique_key,json=uniqueKey,proto3" json:"unique_key,omitempty"` // GroupKey is a name of the group used for task aggregation. // This field is optional and empty value means no aggregation for the task. GroupKey string `protobuf:"bytes,14,opt,name=group_key,json=groupKey,proto3" json:"group_key,omitempty"` // Retention period specified in a number of seconds. // The task will be stored in redis as a completed task until the TTL // expires. Retention int64 `protobuf:"varint,12,opt,name=retention,proto3" json:"retention,omitempty"` // Time when the task completed in success in Unix time, // the number of seconds elapsed since January 1, 1970 UTC. // This field is populated if result_ttl > 0 upon completion. CompletedAt int64 `protobuf:"varint,13,opt,name=completed_at,json=completedAt,proto3" json:"completed_at,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *TaskMessage) Reset() { *x = TaskMessage{} mi := &file_asynq_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } func (x *TaskMessage) String() string { return protoimpl.X.MessageStringOf(x) } func (*TaskMessage) ProtoMessage() {} func (x *TaskMessage) ProtoReflect() protoreflect.Message { mi := &file_asynq_proto_msgTypes[0] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use TaskMessage.ProtoReflect.Descriptor instead. func (*TaskMessage) Descriptor() ([]byte, []int) { return file_asynq_proto_rawDescGZIP(), []int{0} } func (x *TaskMessage) GetType() string { if x != nil { return x.Type } return "" } func (x *TaskMessage) GetPayload() []byte { if x != nil { return x.Payload } return nil } func (x *TaskMessage) GetHeaders() map[string]string { if x != nil { return x.Headers } return nil } func (x *TaskMessage) GetId() string { if x != nil { return x.Id } return "" } func (x *TaskMessage) GetQueue() string { if x != nil { return x.Queue } return "" } func (x *TaskMessage) GetRetry() int32 { if x != nil { return x.Retry } return 0 } func (x *TaskMessage) GetRetried() int32 { if x != nil { return x.Retried } return 0 } func (x *TaskMessage) GetErrorMsg() string { if x != nil { return x.ErrorMsg } return "" } func (x *TaskMessage) GetLastFailedAt() int64 { if x != nil { return x.LastFailedAt } return 0 } func (x *TaskMessage) GetTimeout() int64 { if x != nil { return x.Timeout } return 0 } func (x *TaskMessage) GetDeadline() int64 { if x != nil { return x.Deadline } return 0 } func (x *TaskMessage) GetUniqueKey() string { if x != nil { return x.UniqueKey } return "" } func (x *TaskMessage) GetGroupKey() string { if x != nil { return x.GroupKey } return "" } func (x *TaskMessage) GetRetention() int64 { if x != nil { return x.Retention } return 0 } func (x *TaskMessage) GetCompletedAt() int64 { if x != nil { return x.CompletedAt } return 0 } // ServerInfo holds information about a running server. type ServerInfo struct { state protoimpl.MessageState `protogen:"open.v1"` // Host machine the server is running on. Host string `protobuf:"bytes,1,opt,name=host,proto3" json:"host,omitempty"` // PID of the server process. Pid int32 `protobuf:"varint,2,opt,name=pid,proto3" json:"pid,omitempty"` // Unique identifier for this server. ServerId string `protobuf:"bytes,3,opt,name=server_id,json=serverId,proto3" json:"server_id,omitempty"` // Maximum number of concurrency this server will use. Concurrency int32 `protobuf:"varint,4,opt,name=concurrency,proto3" json:"concurrency,omitempty"` // List of queue names with their priorities. // The server will consume tasks from the queues and prioritize // queues with higher priority numbers. Queues map[string]int32 `protobuf:"bytes,5,rep,name=queues,proto3" json:"queues,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"` // If set, the server will always consume tasks from a queue with higher // priority. StrictPriority bool `protobuf:"varint,6,opt,name=strict_priority,json=strictPriority,proto3" json:"strict_priority,omitempty"` // Status indicates the status of the server. Status string `protobuf:"bytes,7,opt,name=status,proto3" json:"status,omitempty"` // Time this server was started. StartTime *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=start_time,json=startTime,proto3" json:"start_time,omitempty"` // Number of workers currently processing tasks. ActiveWorkerCount int32 `protobuf:"varint,9,opt,name=active_worker_count,json=activeWorkerCount,proto3" json:"active_worker_count,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *ServerInfo) Reset() { *x = ServerInfo{} mi := &file_asynq_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } func (x *ServerInfo) String() string { return protoimpl.X.MessageStringOf(x) } func (*ServerInfo) ProtoMessage() {} func (x *ServerInfo) ProtoReflect() protoreflect.Message { mi := &file_asynq_proto_msgTypes[1] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use ServerInfo.ProtoReflect.Descriptor instead. func (*ServerInfo) Descriptor() ([]byte, []int) { return file_asynq_proto_rawDescGZIP(), []int{1} } func (x *ServerInfo) GetHost() string { if x != nil { return x.Host } return "" } func (x *ServerInfo) GetPid() int32 { if x != nil { return x.Pid } return 0 } func (x *ServerInfo) GetServerId() string { if x != nil { return x.ServerId } return "" } func (x *ServerInfo) GetConcurrency() int32 { if x != nil { return x.Concurrency } return 0 } func (x *ServerInfo) GetQueues() map[string]int32 { if x != nil { return x.Queues } return nil } func (x *ServerInfo) GetStrictPriority() bool { if x != nil { return x.StrictPriority } return false } func (x *ServerInfo) GetStatus() string { if x != nil { return x.Status } return "" } func (x *ServerInfo) GetStartTime() *timestamppb.Timestamp { if x != nil { return x.StartTime } return nil } func (x *ServerInfo) GetActiveWorkerCount() int32 { if x != nil { return x.ActiveWorkerCount } return 0 } // WorkerInfo holds information about a running worker. type WorkerInfo struct { state protoimpl.MessageState `protogen:"open.v1"` // Host matchine this worker is running on. Host string `protobuf:"bytes,1,opt,name=host,proto3" json:"host,omitempty"` // PID of the process in which this worker is running. Pid int32 `protobuf:"varint,2,opt,name=pid,proto3" json:"pid,omitempty"` // ID of the server in which this worker is running. ServerId string `protobuf:"bytes,3,opt,name=server_id,json=serverId,proto3" json:"server_id,omitempty"` // ID of the task this worker is processing. TaskId string `protobuf:"bytes,4,opt,name=task_id,json=taskId,proto3" json:"task_id,omitempty"` // Type of the task this worker is processing. TaskType string `protobuf:"bytes,5,opt,name=task_type,json=taskType,proto3" json:"task_type,omitempty"` // Payload of the task this worker is processing. TaskPayload []byte `protobuf:"bytes,6,opt,name=task_payload,json=taskPayload,proto3" json:"task_payload,omitempty"` // Name of the queue the task the worker is processing belongs. Queue string `protobuf:"bytes,7,opt,name=queue,proto3" json:"queue,omitempty"` // Time this worker started processing the task. StartTime *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=start_time,json=startTime,proto3" json:"start_time,omitempty"` // Deadline by which the worker needs to complete processing // the task. If worker exceeds the deadline, the task will fail. Deadline *timestamppb.Timestamp `protobuf:"bytes,9,opt,name=deadline,proto3" json:"deadline,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *WorkerInfo) Reset() { *x = WorkerInfo{} mi := &file_asynq_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } func (x *WorkerInfo) String() string { return protoimpl.X.MessageStringOf(x) } func (*WorkerInfo) ProtoMessage() {} func (x *WorkerInfo) ProtoReflect() protoreflect.Message { mi := &file_asynq_proto_msgTypes[2] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use WorkerInfo.ProtoReflect.Descriptor instead. func (*WorkerInfo) Descriptor() ([]byte, []int) { return file_asynq_proto_rawDescGZIP(), []int{2} } func (x *WorkerInfo) GetHost() string { if x != nil { return x.Host } return "" } func (x *WorkerInfo) GetPid() int32 { if x != nil { return x.Pid } return 0 } func (x *WorkerInfo) GetServerId() string { if x != nil { return x.ServerId } return "" } func (x *WorkerInfo) GetTaskId() string { if x != nil { return x.TaskId } return "" } func (x *WorkerInfo) GetTaskType() string { if x != nil { return x.TaskType } return "" } func (x *WorkerInfo) GetTaskPayload() []byte { if x != nil { return x.TaskPayload } return nil } func (x *WorkerInfo) GetQueue() string { if x != nil { return x.Queue } return "" } func (x *WorkerInfo) GetStartTime() *timestamppb.Timestamp { if x != nil { return x.StartTime } return nil } func (x *WorkerInfo) GetDeadline() *timestamppb.Timestamp { if x != nil { return x.Deadline } return nil } // SchedulerEntry holds information about a periodic task registered // with a scheduler. type SchedulerEntry struct { state protoimpl.MessageState `protogen:"open.v1"` // Identifier of the scheduler entry. Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // Periodic schedule spec of the entry. Spec string `protobuf:"bytes,2,opt,name=spec,proto3" json:"spec,omitempty"` // Task type of the periodic task. TaskType string `protobuf:"bytes,3,opt,name=task_type,json=taskType,proto3" json:"task_type,omitempty"` // Task payload of the periodic task. TaskPayload []byte `protobuf:"bytes,4,opt,name=task_payload,json=taskPayload,proto3" json:"task_payload,omitempty"` // Options used to enqueue the periodic task. EnqueueOptions []string `protobuf:"bytes,5,rep,name=enqueue_options,json=enqueueOptions,proto3" json:"enqueue_options,omitempty"` // Next time the task will be enqueued. NextEnqueueTime *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=next_enqueue_time,json=nextEnqueueTime,proto3" json:"next_enqueue_time,omitempty"` // Last time the task was enqueued. // Zero time if task was never enqueued. PrevEnqueueTime *timestamppb.Timestamp `protobuf:"bytes,7,opt,name=prev_enqueue_time,json=prevEnqueueTime,proto3" json:"prev_enqueue_time,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *SchedulerEntry) Reset() { *x = SchedulerEntry{} mi := &file_asynq_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } func (x *SchedulerEntry) String() string { return protoimpl.X.MessageStringOf(x) } func (*SchedulerEntry) ProtoMessage() {} func (x *SchedulerEntry) ProtoReflect() protoreflect.Message { mi := &file_asynq_proto_msgTypes[3] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use SchedulerEntry.ProtoReflect.Descriptor instead. func (*SchedulerEntry) Descriptor() ([]byte, []int) { return file_asynq_proto_rawDescGZIP(), []int{3} } func (x *SchedulerEntry) GetId() string { if x != nil { return x.Id } return "" } func (x *SchedulerEntry) GetSpec() string { if x != nil { return x.Spec } return "" } func (x *SchedulerEntry) GetTaskType() string { if x != nil { return x.TaskType } return "" } func (x *SchedulerEntry) GetTaskPayload() []byte { if x != nil { return x.TaskPayload } return nil } func (x *SchedulerEntry) GetEnqueueOptions() []string { if x != nil { return x.EnqueueOptions } return nil } func (x *SchedulerEntry) GetNextEnqueueTime() *timestamppb.Timestamp { if x != nil { return x.NextEnqueueTime } return nil } func (x *SchedulerEntry) GetPrevEnqueueTime() *timestamppb.Timestamp { if x != nil { return x.PrevEnqueueTime } return nil } // SchedulerEnqueueEvent holds information about an enqueue event // by a scheduler. type SchedulerEnqueueEvent struct { state protoimpl.MessageState `protogen:"open.v1"` // ID of the task that was enqueued. TaskId string `protobuf:"bytes,1,opt,name=task_id,json=taskId,proto3" json:"task_id,omitempty"` // Time the task was enqueued. EnqueueTime *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=enqueue_time,json=enqueueTime,proto3" json:"enqueue_time,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *SchedulerEnqueueEvent) Reset() { *x = SchedulerEnqueueEvent{} mi := &file_asynq_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } func (x *SchedulerEnqueueEvent) String() string { return protoimpl.X.MessageStringOf(x) } func (*SchedulerEnqueueEvent) ProtoMessage() {} func (x *SchedulerEnqueueEvent) ProtoReflect() protoreflect.Message { mi := &file_asynq_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use SchedulerEnqueueEvent.ProtoReflect.Descriptor instead. func (*SchedulerEnqueueEvent) Descriptor() ([]byte, []int) { return file_asynq_proto_rawDescGZIP(), []int{4} } func (x *SchedulerEnqueueEvent) GetTaskId() string { if x != nil { return x.TaskId } return "" } func (x *SchedulerEnqueueEvent) GetEnqueueTime() *timestamppb.Timestamp { if x != nil { return x.EnqueueTime } return nil } var File_asynq_proto protoreflect.FileDescriptor const file_asynq_proto_rawDesc = "" + "\n" + "\vasynq.proto\x12\x05asynq\x1a\x1fgoogle/protobuf/timestamp.proto\"\xfe\x03\n" + "\vTaskMessage\x12\x12\n" + "\x04type\x18\x01 \x01(\tR\x04type\x12\x18\n" + "\apayload\x18\x02 \x01(\fR\apayload\x129\n" + "\aheaders\x18\x0f \x03(\v2\x1f.asynq.TaskMessage.HeadersEntryR\aheaders\x12\x0e\n" + "\x02id\x18\x03 \x01(\tR\x02id\x12\x14\n" + "\x05queue\x18\x04 \x01(\tR\x05queue\x12\x14\n" + "\x05retry\x18\x05 \x01(\x05R\x05retry\x12\x18\n" + "\aretried\x18\x06 \x01(\x05R\aretried\x12\x1b\n" + "\terror_msg\x18\a \x01(\tR\berrorMsg\x12$\n" + "\x0elast_failed_at\x18\v \x01(\x03R\flastFailedAt\x12\x18\n" + "\atimeout\x18\b \x01(\x03R\atimeout\x12\x1a\n" + "\bdeadline\x18\t \x01(\x03R\bdeadline\x12\x1d\n" + "\n" + "unique_key\x18\n" + " \x01(\tR\tuniqueKey\x12\x1b\n" + "\tgroup_key\x18\x0e \x01(\tR\bgroupKey\x12\x1c\n" + "\tretention\x18\f \x01(\x03R\tretention\x12!\n" + "\fcompleted_at\x18\r \x01(\x03R\vcompletedAt\x1a:\n" + "\fHeadersEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x8f\x03\n" + "\n" + "ServerInfo\x12\x12\n" + "\x04host\x18\x01 \x01(\tR\x04host\x12\x10\n" + "\x03pid\x18\x02 \x01(\x05R\x03pid\x12\x1b\n" + "\tserver_id\x18\x03 \x01(\tR\bserverId\x12 \n" + "\vconcurrency\x18\x04 \x01(\x05R\vconcurrency\x125\n" + "\x06queues\x18\x05 \x03(\v2\x1d.asynq.ServerInfo.QueuesEntryR\x06queues\x12'\n" + "\x0fstrict_priority\x18\x06 \x01(\bR\x0estrictPriority\x12\x16\n" + "\x06status\x18\a \x01(\tR\x06status\x129\n" + "\n" + "start_time\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\tstartTime\x12.\n" + "\x13active_worker_count\x18\t \x01(\x05R\x11activeWorkerCount\x1a9\n" + "\vQueuesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\x05R\x05value:\x028\x01\"\xb1\x02\n" + "\n" + "WorkerInfo\x12\x12\n" + "\x04host\x18\x01 \x01(\tR\x04host\x12\x10\n" + "\x03pid\x18\x02 \x01(\x05R\x03pid\x12\x1b\n" + "\tserver_id\x18\x03 \x01(\tR\bserverId\x12\x17\n" + "\atask_id\x18\x04 \x01(\tR\x06taskId\x12\x1b\n" + "\ttask_type\x18\x05 \x01(\tR\btaskType\x12!\n" + "\ftask_payload\x18\x06 \x01(\fR\vtaskPayload\x12\x14\n" + "\x05queue\x18\a \x01(\tR\x05queue\x129\n" + "\n" + "start_time\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\tstartTime\x126\n" + "\bdeadline\x18\t \x01(\v2\x1a.google.protobuf.TimestampR\bdeadline\"\xad\x02\n" + "\x0eSchedulerEntry\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n" + "\x04spec\x18\x02 \x01(\tR\x04spec\x12\x1b\n" + "\ttask_type\x18\x03 \x01(\tR\btaskType\x12!\n" + "\ftask_payload\x18\x04 \x01(\fR\vtaskPayload\x12'\n" + "\x0fenqueue_options\x18\x05 \x03(\tR\x0eenqueueOptions\x12F\n" + "\x11next_enqueue_time\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\x0fnextEnqueueTime\x12F\n" + "\x11prev_enqueue_time\x18\a \x01(\v2\x1a.google.protobuf.TimestampR\x0fprevEnqueueTime\"o\n" + "\x15SchedulerEnqueueEvent\x12\x17\n" + "\atask_id\x18\x01 \x01(\tR\x06taskId\x12=\n" + "\fenqueue_time\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\venqueueTimeB)Z'github.com/hibiken/asynq/internal/protob\x06proto3" var ( file_asynq_proto_rawDescOnce sync.Once file_asynq_proto_rawDescData []byte ) func file_asynq_proto_rawDescGZIP() []byte { file_asynq_proto_rawDescOnce.Do(func() { file_asynq_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_asynq_proto_rawDesc), len(file_asynq_proto_rawDesc))) }) return file_asynq_proto_rawDescData } var file_asynq_proto_msgTypes = make([]protoimpl.MessageInfo, 7) var file_asynq_proto_goTypes = []any{ (*TaskMessage)(nil), // 0: asynq.TaskMessage (*ServerInfo)(nil), // 1: asynq.ServerInfo (*WorkerInfo)(nil), // 2: asynq.WorkerInfo (*SchedulerEntry)(nil), // 3: asynq.SchedulerEntry (*SchedulerEnqueueEvent)(nil), // 4: asynq.SchedulerEnqueueEvent nil, // 5: asynq.TaskMessage.HeadersEntry nil, // 6: asynq.ServerInfo.QueuesEntry (*timestamppb.Timestamp)(nil), // 7: google.protobuf.Timestamp } var file_asynq_proto_depIdxs = []int32{ 5, // 0: asynq.TaskMessage.headers:type_name -> asynq.TaskMessage.HeadersEntry 6, // 1: asynq.ServerInfo.queues:type_name -> asynq.ServerInfo.QueuesEntry 7, // 2: asynq.ServerInfo.start_time:type_name -> google.protobuf.Timestamp 7, // 3: asynq.WorkerInfo.start_time:type_name -> google.protobuf.Timestamp 7, // 4: asynq.WorkerInfo.deadline:type_name -> google.protobuf.Timestamp 7, // 5: asynq.SchedulerEntry.next_enqueue_time:type_name -> google.protobuf.Timestamp 7, // 6: asynq.SchedulerEntry.prev_enqueue_time:type_name -> google.protobuf.Timestamp 7, // 7: asynq.SchedulerEnqueueEvent.enqueue_time:type_name -> google.protobuf.Timestamp 8, // [8:8] is the sub-list for method output_type 8, // [8:8] is the sub-list for method input_type 8, // [8:8] is the sub-list for extension type_name 8, // [8:8] is the sub-list for extension extendee 0, // [0:8] is the sub-list for field type_name } func init() { file_asynq_proto_init() } func file_asynq_proto_init() { if File_asynq_proto != nil { return } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_asynq_proto_rawDesc), len(file_asynq_proto_rawDesc)), NumEnums: 0, NumMessages: 7, NumExtensions: 0, NumServices: 0, }, GoTypes: file_asynq_proto_goTypes, DependencyIndexes: file_asynq_proto_depIdxs, MessageInfos: file_asynq_proto_msgTypes, }.Build() File_asynq_proto = out.File file_asynq_proto_goTypes = nil file_asynq_proto_depIdxs = nil } ================================================ FILE: internal/proto/asynq.proto ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. syntax = "proto3"; package asynq; import "google/protobuf/timestamp.proto"; option go_package = "github.com/hibiken/asynq/internal/proto"; // TaskMessage is the internal representation of a task with additional // metadata fields. // Next ID: 16 message TaskMessage { // Type indicates the kind of the task to be performed. string type = 1; // Payload holds data needed to process the task. bytes payload = 2; // Headers holds additional metadata for the task. map headers = 15; // Unique identifier for the task. string id = 3; // Name of the queue to which this task belongs. string queue = 4; // Max number of retries for this task. int32 retry = 5; // Number of times this task has been retried so far. int32 retried = 6; // Error message from the last failure. string error_msg = 7; // Time of last failure in Unix time, // the number of seconds elapsed since January 1, 1970 UTC. // Use zero to indicate no last failure. int64 last_failed_at = 11; // Timeout specifies timeout in seconds. // Use zero to indicate no timeout. int64 timeout = 8; // Deadline specifies the deadline for the task in Unix time, // the number of seconds elapsed since January 1, 1970 UTC. // Use zero to indicate no deadline. int64 deadline = 9; // UniqueKey holds the redis key used for uniqueness lock for this task. // Empty string indicates that no uniqueness lock was used. string unique_key = 10; // GroupKey is a name of the group used for task aggregation. // This field is optional and empty value means no aggregation for the task. string group_key = 14; // Retention period specified in a number of seconds. // The task will be stored in redis as a completed task until the TTL // expires. int64 retention = 12; // Time when the task completed in success in Unix time, // the number of seconds elapsed since January 1, 1970 UTC. // This field is populated if result_ttl > 0 upon completion. int64 completed_at = 13; }; // ServerInfo holds information about a running server. message ServerInfo { // Host machine the server is running on. string host = 1; // PID of the server process. int32 pid = 2; // Unique identifier for this server. string server_id = 3; // Maximum number of concurrency this server will use. int32 concurrency = 4; // List of queue names with their priorities. // The server will consume tasks from the queues and prioritize // queues with higher priority numbers. map queues = 5; // If set, the server will always consume tasks from a queue with higher // priority. bool strict_priority = 6; // Status indicates the status of the server. string status = 7; // Time this server was started. google.protobuf.Timestamp start_time = 8; // Number of workers currently processing tasks. int32 active_worker_count = 9; }; // WorkerInfo holds information about a running worker. message WorkerInfo { // Host matchine this worker is running on. string host = 1; // PID of the process in which this worker is running. int32 pid = 2; // ID of the server in which this worker is running. string server_id = 3; // ID of the task this worker is processing. string task_id = 4; // Type of the task this worker is processing. string task_type = 5; // Payload of the task this worker is processing. bytes task_payload = 6; // Name of the queue the task the worker is processing belongs. string queue = 7; // Time this worker started processing the task. google.protobuf.Timestamp start_time = 8; // Deadline by which the worker needs to complete processing // the task. If worker exceeds the deadline, the task will fail. google.protobuf.Timestamp deadline = 9; }; // SchedulerEntry holds information about a periodic task registered // with a scheduler. message SchedulerEntry { // Identifier of the scheduler entry. string id = 1; // Periodic schedule spec of the entry. string spec = 2; // Task type of the periodic task. string task_type = 3; // Task payload of the periodic task. bytes task_payload = 4; // Options used to enqueue the periodic task. repeated string enqueue_options = 5; // Next time the task will be enqueued. google.protobuf.Timestamp next_enqueue_time = 6; // Last time the task was enqueued. // Zero time if task was never enqueued. google.protobuf.Timestamp prev_enqueue_time = 7; }; // SchedulerEnqueueEvent holds information about an enqueue event // by a scheduler. message SchedulerEnqueueEvent { // ID of the task that was enqueued. string task_id = 1; // Time the task was enqueued. google.protobuf.Timestamp enqueue_time = 2; }; ================================================ FILE: internal/rdb/benchmark_test.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package rdb import ( "context" "fmt" "testing" "time" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/testutil" ) func BenchmarkEnqueue(b *testing.B) { r := setup(b) ctx := context.Background() msg := testutil.NewTaskMessage("task1", nil) b.ResetTimer() for i := 0; i < b.N; i++ { b.StopTimer() testutil.FlushDB(b, r.client) b.StartTimer() if err := r.Enqueue(ctx, msg); err != nil { b.Fatalf("Enqueue failed: %v", err) } } } func BenchmarkEnqueueUnique(b *testing.B) { r := setup(b) ctx := context.Background() msg := &base.TaskMessage{ Type: "task1", Payload: nil, Queue: base.DefaultQueueName, UniqueKey: base.UniqueKey("default", "task1", nil), } uniqueTTL := 5 * time.Minute b.ResetTimer() for i := 0; i < b.N; i++ { b.StopTimer() testutil.FlushDB(b, r.client) b.StartTimer() if err := r.EnqueueUnique(ctx, msg, uniqueTTL); err != nil { b.Fatalf("EnqueueUnique failed: %v", err) } } } func BenchmarkSchedule(b *testing.B) { r := setup(b) ctx := context.Background() msg := testutil.NewTaskMessage("task1", nil) processAt := time.Now().Add(3 * time.Minute) b.ResetTimer() for i := 0; i < b.N; i++ { b.StopTimer() testutil.FlushDB(b, r.client) b.StartTimer() if err := r.Schedule(ctx, msg, processAt); err != nil { b.Fatalf("Schedule failed: %v", err) } } } func BenchmarkScheduleUnique(b *testing.B) { r := setup(b) ctx := context.Background() msg := &base.TaskMessage{ Type: "task1", Payload: nil, Queue: base.DefaultQueueName, UniqueKey: base.UniqueKey("default", "task1", nil), } processAt := time.Now().Add(3 * time.Minute) uniqueTTL := 5 * time.Minute b.ResetTimer() for i := 0; i < b.N; i++ { b.StopTimer() testutil.FlushDB(b, r.client) b.StartTimer() if err := r.ScheduleUnique(ctx, msg, processAt, uniqueTTL); err != nil { b.Fatalf("EnqueueUnique failed: %v", err) } } } func BenchmarkDequeueSingleQueue(b *testing.B) { r := setup(b) ctx := context.Background() b.ResetTimer() for i := 0; i < b.N; i++ { b.StopTimer() testutil.FlushDB(b, r.client) for i := 0; i < 10; i++ { m := testutil.NewTaskMessageWithQueue( fmt.Sprintf("task%d", i), nil, base.DefaultQueueName) if err := r.Enqueue(ctx, m); err != nil { b.Fatalf("Enqueue failed: %v", err) } } b.StartTimer() if _, _, err := r.Dequeue(base.DefaultQueueName); err != nil { b.Fatalf("Dequeue failed: %v", err) } } } func BenchmarkDequeueMultipleQueues(b *testing.B) { qnames := []string{"critical", "default", "low"} r := setup(b) ctx := context.Background() b.ResetTimer() for i := 0; i < b.N; i++ { b.StopTimer() testutil.FlushDB(b, r.client) for i := 0; i < 10; i++ { for _, qname := range qnames { m := testutil.NewTaskMessageWithQueue( fmt.Sprintf("%s_task%d", qname, i), nil, qname) if err := r.Enqueue(ctx, m); err != nil { b.Fatalf("Enqueue failed: %v", err) } } } b.StartTimer() if _, _, err := r.Dequeue(qnames...); err != nil { b.Fatalf("Dequeue failed: %v", err) } } } func BenchmarkDone(b *testing.B) { r := setup(b) m1 := testutil.NewTaskMessage("task1", nil) m2 := testutil.NewTaskMessage("task2", nil) m3 := testutil.NewTaskMessage("task3", nil) msgs := []*base.TaskMessage{m1, m2, m3} zs := []base.Z{ {Message: m1, Score: time.Now().Add(10 * time.Second).Unix()}, {Message: m2, Score: time.Now().Add(20 * time.Second).Unix()}, {Message: m3, Score: time.Now().Add(30 * time.Second).Unix()}, } ctx := context.Background() b.ResetTimer() for i := 0; i < b.N; i++ { b.StopTimer() testutil.FlushDB(b, r.client) testutil.SeedActiveQueue(b, r.client, msgs, base.DefaultQueueName) testutil.SeedLease(b, r.client, zs, base.DefaultQueueName) b.StartTimer() if err := r.Done(ctx, msgs[0]); err != nil { b.Fatalf("Done failed: %v", err) } } } func BenchmarkRetry(b *testing.B) { r := setup(b) m1 := testutil.NewTaskMessage("task1", nil) m2 := testutil.NewTaskMessage("task2", nil) m3 := testutil.NewTaskMessage("task3", nil) msgs := []*base.TaskMessage{m1, m2, m3} zs := []base.Z{ {Message: m1, Score: time.Now().Add(10 * time.Second).Unix()}, {Message: m2, Score: time.Now().Add(20 * time.Second).Unix()}, {Message: m3, Score: time.Now().Add(30 * time.Second).Unix()}, } ctx := context.Background() b.ResetTimer() for i := 0; i < b.N; i++ { b.StopTimer() testutil.FlushDB(b, r.client) testutil.SeedActiveQueue(b, r.client, msgs, base.DefaultQueueName) testutil.SeedLease(b, r.client, zs, base.DefaultQueueName) b.StartTimer() if err := r.Retry(ctx, msgs[0], time.Now().Add(1*time.Minute), "error", true /*isFailure*/); err != nil { b.Fatalf("Retry failed: %v", err) } } } func BenchmarkArchive(b *testing.B) { r := setup(b) m1 := testutil.NewTaskMessage("task1", nil) m2 := testutil.NewTaskMessage("task2", nil) m3 := testutil.NewTaskMessage("task3", nil) msgs := []*base.TaskMessage{m1, m2, m3} zs := []base.Z{ {Message: m1, Score: time.Now().Add(10 * time.Second).Unix()}, {Message: m2, Score: time.Now().Add(20 * time.Second).Unix()}, {Message: m3, Score: time.Now().Add(30 * time.Second).Unix()}, } ctx := context.Background() b.ResetTimer() for i := 0; i < b.N; i++ { b.StopTimer() testutil.FlushDB(b, r.client) testutil.SeedActiveQueue(b, r.client, msgs, base.DefaultQueueName) testutil.SeedLease(b, r.client, zs, base.DefaultQueueName) b.StartTimer() if err := r.Archive(ctx, msgs[0], "error"); err != nil { b.Fatalf("Archive failed: %v", err) } } } func BenchmarkRequeue(b *testing.B) { r := setup(b) m1 := testutil.NewTaskMessage("task1", nil) m2 := testutil.NewTaskMessage("task2", nil) m3 := testutil.NewTaskMessage("task3", nil) msgs := []*base.TaskMessage{m1, m2, m3} zs := []base.Z{ {Message: m1, Score: time.Now().Add(10 * time.Second).Unix()}, {Message: m2, Score: time.Now().Add(20 * time.Second).Unix()}, {Message: m3, Score: time.Now().Add(30 * time.Second).Unix()}, } ctx := context.Background() b.ResetTimer() for i := 0; i < b.N; i++ { b.StopTimer() testutil.FlushDB(b, r.client) testutil.SeedActiveQueue(b, r.client, msgs, base.DefaultQueueName) testutil.SeedLease(b, r.client, zs, base.DefaultQueueName) b.StartTimer() if err := r.Requeue(ctx, msgs[0]); err != nil { b.Fatalf("Requeue failed: %v", err) } } } func BenchmarkCheckAndEnqueue(b *testing.B) { r := setup(b) now := time.Now() var zs []base.Z for i := -100; i < 100; i++ { msg := testutil.NewTaskMessage(fmt.Sprintf("task%d", i), nil) score := now.Add(time.Duration(i) * time.Second).Unix() zs = append(zs, base.Z{Message: msg, Score: score}) } b.ResetTimer() for i := 0; i < b.N; i++ { b.StopTimer() testutil.FlushDB(b, r.client) testutil.SeedScheduledQueue(b, r.client, zs, base.DefaultQueueName) b.StartTimer() if err := r.ForwardIfReady(base.DefaultQueueName); err != nil { b.Fatalf("ForwardIfReady failed: %v", err) } } } ================================================ FILE: internal/rdb/inspect.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package rdb import ( "context" "fmt" "strings" "time" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/errors" "github.com/redis/go-redis/v9" "github.com/spf13/cast" ) // AllQueues returns a list of all queue names. func (r *RDB) AllQueues() ([]string, error) { return r.client.SMembers(context.Background(), base.AllQueues).Result() } // Stats represents a state of queues at a certain time. type Stats struct { // Name of the queue (e.g. "default", "critical"). Queue string // MemoryUsage is the total number of bytes the queue and its tasks require // to be stored in redis. It is an approximate memory usage value in bytes // since the value is computed by sampling. MemoryUsage int64 // Paused indicates whether the queue is paused. // If true, tasks in the queue should not be processed. Paused bool // Size is the total number of tasks in the queue. Size int // Groups is the total number of groups in the queue. Groups int // Number of tasks in each state. Pending int Active int Scheduled int Retry int Archived int Completed int Aggregating int // Number of tasks processed within the current date. // The number includes both succeeded and failed tasks. Processed int // Number of tasks failed within the current date. Failed int // Total number of tasks processed (both succeeded and failed) from this queue. ProcessedTotal int // Total number of tasks failed. FailedTotal int // Latency of the queue, measured by the oldest pending task in the queue. Latency time.Duration // Time this stats was taken. Timestamp time.Time } // DailyStats holds aggregate data for a given day. type DailyStats struct { // Name of the queue (e.g. "default", "critical"). Queue string // Total number of tasks processed during the given day. // The number includes both succeeded and failed tasks. Processed int // Total number of tasks failed during the given day. Failed int // Date this stats was taken. Time time.Time } // KEYS[1] -> asynq::pending // KEYS[2] -> asynq::active // KEYS[3] -> asynq::scheduled // KEYS[4] -> asynq::retry // KEYS[5] -> asynq::archived // KEYS[6] -> asynq::completed // KEYS[7] -> asynq::processed: // KEYS[8] -> asynq::failed: // KEYS[9] -> asynq::processed // KEYS[10] -> asynq::failed // KEYS[11] -> asynq::paused // KEYS[12] -> asynq::groups // -------- // ARGV[1] -> task key prefix // ARGV[2] -> group key prefix var currentStatsCmd = redis.NewScript(` local res = {} local pendingTaskCount = redis.call("LLEN", KEYS[1]) table.insert(res, KEYS[1]) table.insert(res, pendingTaskCount) table.insert(res, KEYS[2]) table.insert(res, redis.call("LLEN", KEYS[2])) table.insert(res, KEYS[3]) table.insert(res, redis.call("ZCARD", KEYS[3])) table.insert(res, KEYS[4]) table.insert(res, redis.call("ZCARD", KEYS[4])) table.insert(res, KEYS[5]) table.insert(res, redis.call("ZCARD", KEYS[5])) table.insert(res, KEYS[6]) table.insert(res, redis.call("ZCARD", KEYS[6])) for i=7,10 do local count = 0 local n = redis.call("GET", KEYS[i]) if n then count = tonumber(n) end table.insert(res, KEYS[i]) table.insert(res, count) end table.insert(res, KEYS[11]) table.insert(res, redis.call("EXISTS", KEYS[11])) table.insert(res, "oldest_pending_since") if pendingTaskCount > 0 then local id = redis.call("LRANGE", KEYS[1], -1, -1)[1] table.insert(res, redis.call("HGET", ARGV[1] .. id, "pending_since")) else table.insert(res, 0) end local group_names = redis.call("SMEMBERS", KEYS[12]) table.insert(res, "group_size") table.insert(res, table.getn(group_names)) local aggregating_count = 0 for _, gname in ipairs(group_names) do aggregating_count = aggregating_count + redis.call("ZCARD", ARGV[2] .. gname) end table.insert(res, "aggregating_count") table.insert(res, aggregating_count) return res`) // CurrentStats returns a current state of the queues. func (r *RDB) CurrentStats(qname string) (*Stats, error) { var op errors.Op = "rdb.CurrentStats" exists, err := r.queueExists(qname) if err != nil { return nil, errors.E(op, errors.Unknown, err) } if !exists { return nil, errors.E(op, errors.NotFound, &errors.QueueNotFoundError{Queue: qname}) } now := r.clock.Now() keys := []string{ base.PendingKey(qname), base.ActiveKey(qname), base.ScheduledKey(qname), base.RetryKey(qname), base.ArchivedKey(qname), base.CompletedKey(qname), base.ProcessedKey(qname, now), base.FailedKey(qname, now), base.ProcessedTotalKey(qname), base.FailedTotalKey(qname), base.PausedKey(qname), base.AllGroups(qname), } argv := []interface{}{ base.TaskKeyPrefix(qname), base.GroupKeyPrefix(qname), } res, err := currentStatsCmd.Run(context.Background(), r.client, keys, argv...).Result() if err != nil { return nil, errors.E(op, errors.Unknown, err) } data, err := cast.ToSliceE(res) if err != nil { return nil, errors.E(op, errors.Internal, "cast error: unexpected return value from Lua script") } stats := &Stats{ Queue: qname, Timestamp: now, } size := 0 for i := 0; i < len(data); i += 2 { key := cast.ToString(data[i]) val := cast.ToInt(data[i+1]) switch key { case base.PendingKey(qname): stats.Pending = val size += val case base.ActiveKey(qname): stats.Active = val size += val case base.ScheduledKey(qname): stats.Scheduled = val size += val case base.RetryKey(qname): stats.Retry = val size += val case base.ArchivedKey(qname): stats.Archived = val size += val case base.CompletedKey(qname): stats.Completed = val size += val case base.ProcessedKey(qname, now): stats.Processed = val case base.FailedKey(qname, now): stats.Failed = val case base.ProcessedTotalKey(qname): stats.ProcessedTotal = val case base.FailedTotalKey(qname): stats.FailedTotal = val case base.PausedKey(qname): if val == 0 { stats.Paused = false } else { stats.Paused = true } case "oldest_pending_since": if val == 0 { stats.Latency = 0 } else { stats.Latency = r.clock.Now().Sub(time.Unix(0, int64(val))) } case "group_size": stats.Groups = val case "aggregating_count": stats.Aggregating = val size += val } } stats.Size = size memusg, err := r.memoryUsage(qname) if err != nil { return nil, errors.E(op, errors.CanonicalCode(err), err) } stats.MemoryUsage = memusg return stats, nil } // Computes memory usage for the given queue by sampling tasks // from each redis list/zset. Returns approximate memory usage value // in bytes. // // KEYS[1] -> asynq:{qname}:active // KEYS[2] -> asynq:{qname}:pending // KEYS[3] -> asynq:{qname}:scheduled // KEYS[4] -> asynq:{qname}:retry // KEYS[5] -> asynq:{qname}:archived // KEYS[6] -> asynq:{qname}:completed // KEYS[7] -> asynq:{qname}:groups // ------- // ARGV[1] -> asynq:{qname}:t: (task key prefix) // ARGV[2] -> task sample size per redis list/zset (e.g 20) // ARGV[3] -> group sample size // ARGV[4] -> asynq:{qname}:g: (group key prefix) var memoryUsageCmd = redis.NewScript(` local sample_size = tonumber(ARGV[2]) if sample_size <= 0 then return redis.error_reply("sample size must be a positive number") end local memusg = 0 for i=1,2 do local ids = redis.call("LRANGE", KEYS[i], 0, sample_size - 1) local sample_total = 0 if (table.getn(ids) > 0) then for _, id in ipairs(ids) do local bytes = redis.call("MEMORY", "USAGE", ARGV[1] .. id) sample_total = sample_total + bytes end local n = redis.call("LLEN", KEYS[i]) local avg = sample_total / table.getn(ids) memusg = memusg + (avg * n) end local m = redis.call("MEMORY", "USAGE", KEYS[i]) if (m) then memusg = memusg + m end end for i=3,6 do local ids = redis.call("ZRANGE", KEYS[i], 0, sample_size - 1) local sample_total = 0 if (table.getn(ids) > 0) then for _, id in ipairs(ids) do local bytes = redis.call("MEMORY", "USAGE", ARGV[1] .. id) sample_total = sample_total + bytes end local n = redis.call("ZCARD", KEYS[i]) local avg = sample_total / table.getn(ids) memusg = memusg + (avg * n) end local m = redis.call("MEMORY", "USAGE", KEYS[i]) if (m) then memusg = memusg + m end end local groups = redis.call("SMEMBERS", KEYS[7]) if table.getn(groups) > 0 then local agg_task_count = 0 local agg_task_sample_total = 0 local agg_task_sample_size = 0 for i, gname in ipairs(groups) do local group_key = ARGV[4] .. gname agg_task_count = agg_task_count + redis.call("ZCARD", group_key) if i <= tonumber(ARGV[3]) then local ids = redis.call("ZRANGE", group_key, 0, sample_size - 1) for _, id in ipairs(ids) do local bytes = redis.call("MEMORY", "USAGE", ARGV[1] .. id) agg_task_sample_total = agg_task_sample_total + bytes agg_task_sample_size = agg_task_sample_size + 1 end end end local avg = agg_task_sample_total / agg_task_sample_size memusg = memusg + (avg * agg_task_count) end return memusg `) func (r *RDB) memoryUsage(qname string) (int64, error) { var op errors.Op = "rdb.memoryUsage" const ( taskSampleSize = 20 groupSampleSize = 5 ) keys := []string{ base.ActiveKey(qname), base.PendingKey(qname), base.ScheduledKey(qname), base.RetryKey(qname), base.ArchivedKey(qname), base.CompletedKey(qname), base.AllGroups(qname), } argv := []interface{}{ base.TaskKeyPrefix(qname), taskSampleSize, groupSampleSize, base.GroupKeyPrefix(qname), } res, err := memoryUsageCmd.Run(context.Background(), r.client, keys, argv...).Result() if err != nil { return 0, errors.E(op, errors.Unknown, fmt.Sprintf("redis eval error: %v", err)) } usg, err := cast.ToInt64E(res) if err != nil { return 0, errors.E(op, errors.Internal, "could not cast script return value to int64") } return usg, nil } var historicalStatsCmd = redis.NewScript(` local res = {} for _, key in ipairs(KEYS) do local n = redis.call("GET", key) if not n then n = 0 end table.insert(res, tonumber(n)) end return res`) // HistoricalStats returns a list of stats from the last n days for the given queue. func (r *RDB) HistoricalStats(qname string, n int) ([]*DailyStats, error) { var op errors.Op = "rdb.HistoricalStats" if n < 1 { return nil, errors.E(op, errors.FailedPrecondition, "the number of days must be positive") } exists, err := r.queueExists(qname) if err != nil { return nil, errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: "sismember", Err: err}) } if !exists { return nil, errors.E(op, errors.NotFound, &errors.QueueNotFoundError{Queue: qname}) } const day = 24 * time.Hour now := r.clock.Now().UTC() var days []time.Time var keys []string for i := 0; i < n; i++ { ts := now.Add(-time.Duration(i) * day) days = append(days, ts) keys = append(keys, base.ProcessedKey(qname, ts)) keys = append(keys, base.FailedKey(qname, ts)) } res, err := historicalStatsCmd.Run(context.Background(), r.client, keys).Result() if err != nil { return nil, errors.E(op, errors.Unknown, fmt.Sprintf("redis eval error: %v", err)) } data, err := cast.ToIntSliceE(res) if err != nil { return nil, errors.E(op, errors.Internal, fmt.Sprintf("cast error: unexpected return value from Lua script: %v", res)) } var stats []*DailyStats for i := 0; i < len(data); i += 2 { stats = append(stats, &DailyStats{ Queue: qname, Processed: data[i], Failed: data[i+1], Time: days[i/2], }) } return stats, nil } // RedisInfo returns a map of redis info. func (r *RDB) RedisInfo() (map[string]string, error) { res, err := r.client.Info(context.Background()).Result() if err != nil { return nil, err } return parseInfo(res) } // RedisClusterInfo returns a map of redis cluster info. func (r *RDB) RedisClusterInfo() (map[string]string, error) { res, err := r.client.ClusterInfo(context.Background()).Result() if err != nil { return nil, err } return parseInfo(res) } func parseInfo(infoStr string) (map[string]string, error) { info := make(map[string]string) lines := strings.Split(infoStr, "\r\n") for _, l := range lines { kv := strings.Split(l, ":") if len(kv) == 2 { info[kv[0]] = kv[1] } } return info, nil } // TODO: Use generics once available. func reverse(x []*base.TaskInfo) { for i := len(x)/2 - 1; i >= 0; i-- { opp := len(x) - 1 - i x[i], x[opp] = x[opp], x[i] } } // checkQueueExists verifies whether the queue exists. // It returns QueueNotFoundError if queue doesn't exist. func (r *RDB) checkQueueExists(qname string) error { exists, err := r.queueExists(qname) if err != nil { return errors.E(errors.Unknown, &errors.RedisCommandError{Command: "sismember", Err: err}) } if !exists { return errors.E(errors.Internal, &errors.QueueNotFoundError{Queue: qname}) } return nil } // Input: // KEYS[1] -> task key (asynq:{}:t:) // ARGV[1] -> task id // ARGV[2] -> current time in Unix time (seconds) // ARGV[3] -> queue key prefix (asynq:{}:) // // Output: // Tuple of {msg, state, nextProcessAt, result} // msg: encoded task message // state: string describing the state of the task // nextProcessAt: unix time in seconds, zero if not applicable. // result: result data associated with the task // // If the task key doesn't exist, it returns error with a message "NOT FOUND" var getTaskInfoCmd = redis.NewScript(` if redis.call("EXISTS", KEYS[1]) == 0 then return redis.error_reply("NOT FOUND") end local msg, state, result = unpack(redis.call("HMGET", KEYS[1], "msg", "state", "result")) if state == "scheduled" or state == "retry" then return {msg, state, redis.call("ZSCORE", ARGV[3] .. state, ARGV[1]), result} end if state == "pending" then return {msg, state, ARGV[2], result} end return {msg, state, 0, result} `) // GetTaskInfo returns a TaskInfo describing the task from the given queue. func (r *RDB) GetTaskInfo(qname, id string) (*base.TaskInfo, error) { var op errors.Op = "rdb.GetTaskInfo" if err := r.checkQueueExists(qname); err != nil { return nil, errors.E(op, errors.CanonicalCode(err), err) } keys := []string{base.TaskKey(qname, id)} argv := []interface{}{ id, r.clock.Now().Unix(), base.QueueKeyPrefix(qname), } res, err := getTaskInfoCmd.Run(context.Background(), r.client, keys, argv...).Result() if err != nil { if err.Error() == "NOT FOUND" { return nil, errors.E(op, errors.NotFound, &errors.TaskNotFoundError{Queue: qname, ID: id}) } return nil, errors.E(op, errors.Unknown, err) } vals, err := cast.ToSliceE(res) if err != nil { return nil, errors.E(op, errors.Internal, "unexpected value returned from Lua script") } if len(vals) != 4 { return nil, errors.E(op, errors.Internal, "unepxected number of values returned from Lua script") } encoded, err := cast.ToStringE(vals[0]) if err != nil { return nil, errors.E(op, errors.Internal, "unexpected value returned from Lua script") } stateStr, err := cast.ToStringE(vals[1]) if err != nil { return nil, errors.E(op, errors.Internal, "unexpected value returned from Lua script") } processAtUnix, err := cast.ToInt64E(vals[2]) if err != nil { return nil, errors.E(op, errors.Internal, "unexpected value returned from Lua script") } resultStr, err := cast.ToStringE(vals[3]) if err != nil { return nil, errors.E(op, errors.Internal, "unexpected value returned from Lua script") } msg, err := base.DecodeMessage([]byte(encoded)) if err != nil { return nil, errors.E(op, errors.Internal, "could not decode task message") } state, err := base.TaskStateFromString(stateStr) if err != nil { return nil, errors.E(op, errors.CanonicalCode(err), err) } var nextProcessAt time.Time if processAtUnix != 0 { nextProcessAt = time.Unix(processAtUnix, 0) } var result []byte if len(resultStr) > 0 { result = []byte(resultStr) } return &base.TaskInfo{ Message: msg, State: state, NextProcessAt: nextProcessAt, Result: result, }, nil } type GroupStat struct { // Name of the group. Group string // Size of the group. Size int } // KEYS[1] -> asynq:{}:groups // ------- // ARGV[1] -> group key prefix // // Output: // list of group name and size (e.g. group1 size1 group2 size2 ...) // // Time Complexity: // O(N) where N being the number of groups in the given queue. var groupStatsCmd = redis.NewScript(` local res = {} local group_names = redis.call("SMEMBERS", KEYS[1]) for _, gname in ipairs(group_names) do local size = redis.call("ZCARD", ARGV[1] .. gname) table.insert(res, gname) table.insert(res, size) end return res `) func (r *RDB) GroupStats(qname string) ([]*GroupStat, error) { var op errors.Op = "RDB.GroupStats" keys := []string{base.AllGroups(qname)} argv := []interface{}{base.GroupKeyPrefix(qname)} res, err := groupStatsCmd.Run(context.Background(), r.client, keys, argv...).Result() if err != nil { return nil, errors.E(op, errors.Unknown, err) } data, err := cast.ToSliceE(res) if err != nil { return nil, errors.E(op, errors.Internal, "cast error: unexpected return value from Lua script") } var stats []*GroupStat for i := 0; i < len(data); i += 2 { stats = append(stats, &GroupStat{ Group: data[i].(string), Size: int(data[i+1].(int64)), }) } return stats, nil } // Pagination specifies the page size and page number // for the list operation. type Pagination struct { // Number of items in the page. Size int // Page number starting from zero. Page int } func (p Pagination) start() int64 { return int64(p.Size * p.Page) } func (p Pagination) stop() int64 { return int64(p.Size*p.Page + p.Size - 1) } // ListPending returns pending tasks that are ready to be processed. func (r *RDB) ListPending(qname string, pgn Pagination) ([]*base.TaskInfo, error) { var op errors.Op = "rdb.ListPending" exists, err := r.queueExists(qname) if err != nil { return nil, errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: "sismember", Err: err}) } if !exists { return nil, errors.E(op, errors.NotFound, &errors.QueueNotFoundError{Queue: qname}) } res, err := r.listMessages(qname, base.TaskStatePending, pgn) if err != nil { return nil, errors.E(op, errors.CanonicalCode(err), err) } return res, nil } // ListActive returns all tasks that are currently being processed for the given queue. func (r *RDB) ListActive(qname string, pgn Pagination) ([]*base.TaskInfo, error) { var op errors.Op = "rdb.ListActive" exists, err := r.queueExists(qname) if err != nil { return nil, errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: "sismember", Err: err}) } if !exists { return nil, errors.E(op, errors.NotFound, &errors.QueueNotFoundError{Queue: qname}) } res, err := r.listMessages(qname, base.TaskStateActive, pgn) if err != nil { return nil, errors.E(op, errors.CanonicalCode(err), err) } return res, nil } // KEYS[1] -> key for id list (e.g. asynq:{}:pending) // ARGV[1] -> start offset // ARGV[2] -> stop offset // ARGV[3] -> task key prefix var listMessagesCmd = redis.NewScript(` local ids = redis.call("LRange", KEYS[1], ARGV[1], ARGV[2]) local data = {} for _, id in ipairs(ids) do local key = ARGV[3] .. id local msg, result = unpack(redis.call("HMGET", key, "msg","result")) table.insert(data, msg) table.insert(data, result) end return data `) // listMessages returns a list of TaskInfo in Redis list with the given key. func (r *RDB) listMessages(qname string, state base.TaskState, pgn Pagination) ([]*base.TaskInfo, error) { var key string switch state { case base.TaskStateActive: key = base.ActiveKey(qname) case base.TaskStatePending: key = base.PendingKey(qname) default: panic(fmt.Sprintf("unsupported task state: %v", state)) } // Note: Because we use LPUSH to redis list, we need to calculate the // correct range and reverse the list to get the tasks with pagination. stop := -pgn.start() - 1 start := -pgn.stop() - 1 res, err := listMessagesCmd.Run(context.Background(), r.client, []string{key}, start, stop, base.TaskKeyPrefix(qname)).Result() if err != nil { return nil, errors.E(errors.Unknown, err) } data, err := cast.ToStringSliceE(res) if err != nil { return nil, errors.E(errors.Internal, fmt.Errorf("cast error: Lua script returned unexpected value: %v", res)) } var infos []*base.TaskInfo for i := 0; i < len(data); i += 2 { m, err := base.DecodeMessage([]byte(data[i])) if err != nil { continue // bad data, ignore and continue } var res []byte if len(data[i+1]) > 0 { res = []byte(data[i+1]) } var nextProcessAt time.Time if state == base.TaskStatePending { nextProcessAt = r.clock.Now() } infos = append(infos, &base.TaskInfo{ Message: m, State: state, NextProcessAt: nextProcessAt, Result: res, }) } reverse(infos) return infos, nil } // ListScheduled returns all tasks from the given queue that are scheduled // to be processed in the future. func (r *RDB) ListScheduled(qname string, pgn Pagination) ([]*base.TaskInfo, error) { var op errors.Op = "rdb.ListScheduled" exists, err := r.queueExists(qname) if err != nil { return nil, errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: "sismember", Err: err}) } if !exists { return nil, errors.E(op, errors.NotFound, &errors.QueueNotFoundError{Queue: qname}) } res, err := r.listZSetEntries(qname, base.TaskStateScheduled, base.ScheduledKey(qname), pgn) if err != nil { return nil, errors.E(op, errors.CanonicalCode(err), err) } return res, nil } // ListRetry returns all tasks from the given queue that have failed before // and willl be retried in the future. func (r *RDB) ListRetry(qname string, pgn Pagination) ([]*base.TaskInfo, error) { var op errors.Op = "rdb.ListRetry" exists, err := r.queueExists(qname) if err != nil { return nil, errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: "sismember", Err: err}) } if !exists { return nil, errors.E(op, errors.NotFound, &errors.QueueNotFoundError{Queue: qname}) } res, err := r.listZSetEntries(qname, base.TaskStateRetry, base.RetryKey(qname), pgn) if err != nil { return nil, errors.E(op, errors.CanonicalCode(err), err) } return res, nil } // ListArchived returns all tasks from the given queue that have exhausted its retry limit. func (r *RDB) ListArchived(qname string, pgn Pagination) ([]*base.TaskInfo, error) { var op errors.Op = "rdb.ListArchived" exists, err := r.queueExists(qname) if err != nil { return nil, errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: "sismember", Err: err}) } if !exists { return nil, errors.E(op, errors.NotFound, &errors.QueueNotFoundError{Queue: qname}) } zs, err := r.listZSetEntries(qname, base.TaskStateArchived, base.ArchivedKey(qname), pgn) if err != nil { return nil, errors.E(op, errors.CanonicalCode(err), err) } return zs, nil } // ListCompleted returns all tasks from the given queue that have completed successfully. func (r *RDB) ListCompleted(qname string, pgn Pagination) ([]*base.TaskInfo, error) { var op errors.Op = "rdb.ListCompleted" exists, err := r.queueExists(qname) if err != nil { return nil, errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: "sismember", Err: err}) } if !exists { return nil, errors.E(op, errors.NotFound, &errors.QueueNotFoundError{Queue: qname}) } zs, err := r.listZSetEntries(qname, base.TaskStateCompleted, base.CompletedKey(qname), pgn) if err != nil { return nil, errors.E(op, errors.CanonicalCode(err), err) } return zs, nil } // ListAggregating returns all tasks from the given group. func (r *RDB) ListAggregating(qname, gname string, pgn Pagination) ([]*base.TaskInfo, error) { var op errors.Op = "rdb.ListAggregating" exists, err := r.queueExists(qname) if err != nil { return nil, errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: "sismember", Err: err}) } if !exists { return nil, errors.E(op, errors.NotFound, &errors.QueueNotFoundError{Queue: qname}) } zs, err := r.listZSetEntries(qname, base.TaskStateAggregating, base.GroupKey(qname, gname), pgn) if err != nil { return nil, errors.E(op, errors.CanonicalCode(err), err) } return zs, nil } // Reports whether a queue with the given name exists. func (r *RDB) queueExists(qname string) (bool, error) { return r.client.SIsMember(context.Background(), base.AllQueues, qname).Result() } // KEYS[1] -> key for ids set (e.g. asynq:{}:scheduled) // ARGV[1] -> min // ARGV[2] -> max // ARGV[3] -> task key prefix // // Returns an array populated with // [msg1, score1, result1, msg2, score2, result2, ..., msgN, scoreN, resultN] var listZSetEntriesCmd = redis.NewScript(` local data = {} local id_score_pairs = redis.call("ZRANGE", KEYS[1], ARGV[1], ARGV[2], "WITHSCORES") for i = 1, table.getn(id_score_pairs), 2 do local id = id_score_pairs[i] local score = id_score_pairs[i+1] local key = ARGV[3] .. id local msg, res = unpack(redis.call("HMGET", key, "msg", "result")) table.insert(data, msg) table.insert(data, score) table.insert(data, res) end return data `) // listZSetEntries returns a list of message and score pairs in Redis sorted-set // with the given key. func (r *RDB) listZSetEntries(qname string, state base.TaskState, key string, pgn Pagination) ([]*base.TaskInfo, error) { res, err := listZSetEntriesCmd.Run(context.Background(), r.client, []string{key}, pgn.start(), pgn.stop(), base.TaskKeyPrefix(qname)).Result() if err != nil { return nil, errors.E(errors.Unknown, err) } data, err := cast.ToSliceE(res) if err != nil { return nil, errors.E(errors.Internal, fmt.Errorf("cast error: Lua script returned unexpected value: %v", res)) } var infos []*base.TaskInfo for i := 0; i < len(data); i += 3 { s, err := cast.ToStringE(data[i]) if err != nil { return nil, errors.E(errors.Internal, fmt.Errorf("cast error: Lua script returned unexpected value: %v", res)) } score, err := cast.ToInt64E(data[i+1]) if err != nil { return nil, errors.E(errors.Internal, fmt.Errorf("cast error: Lua script returned unexpected value: %v", res)) } resStr, err := cast.ToStringE(data[i+2]) if err != nil { return nil, errors.E(errors.Internal, fmt.Errorf("cast error: Lua script returned unexpected value: %v", res)) } msg, err := base.DecodeMessage([]byte(s)) if err != nil { continue // bad data, ignore and continue } var nextProcessAt time.Time if state == base.TaskStateScheduled || state == base.TaskStateRetry { nextProcessAt = time.Unix(score, 0) } var resBytes []byte if len(resStr) > 0 { resBytes = []byte(resStr) } infos = append(infos, &base.TaskInfo{ Message: msg, State: state, NextProcessAt: nextProcessAt, Result: resBytes, }) } return infos, nil } // RunAllScheduledTasks enqueues all scheduled tasks from the given queue // and returns the number of tasks enqueued. // If a queue with the given name doesn't exist, it returns QueueNotFoundError. func (r *RDB) RunAllScheduledTasks(qname string) (int64, error) { var op errors.Op = "rdb.RunAllScheduledTasks" n, err := r.runAll(base.ScheduledKey(qname), qname) if errors.IsQueueNotFound(err) { return 0, errors.E(op, errors.NotFound, err) } if err != nil { return 0, errors.E(op, errors.Unknown, err) } return n, nil } // RunAllRetryTasks enqueues all retry tasks from the given queue // and returns the number of tasks enqueued. // If a queue with the given name doesn't exist, it returns QueueNotFoundError. func (r *RDB) RunAllRetryTasks(qname string) (int64, error) { var op errors.Op = "rdb.RunAllRetryTasks" n, err := r.runAll(base.RetryKey(qname), qname) if errors.IsQueueNotFound(err) { return 0, errors.E(op, errors.NotFound, err) } if err != nil { return 0, errors.E(op, errors.Unknown, err) } return n, nil } // RunAllArchivedTasks enqueues all archived tasks from the given queue // and returns the number of tasks enqueued. // If a queue with the given name doesn't exist, it returns QueueNotFoundError. func (r *RDB) RunAllArchivedTasks(qname string) (int64, error) { var op errors.Op = "rdb.RunAllArchivedTasks" n, err := r.runAll(base.ArchivedKey(qname), qname) if errors.IsQueueNotFound(err) { return 0, errors.E(op, errors.NotFound, err) } if err != nil { return 0, errors.E(op, errors.Unknown, err) } return n, nil } // runAllAggregatingCmd schedules all tasks in the group to run individually. // // Input: // KEYS[1] -> asynq:{}:g: // KEYS[2] -> asynq:{}:pending // KEYS[3] -> asynq:{}:groups // ------- // ARGV[1] -> task key prefix // ARGV[2] -> group name // // Output: // integer: number of tasks scheduled to run var runAllAggregatingCmd = redis.NewScript(` local ids = redis.call("ZRANGE", KEYS[1], 0, -1) for _, id in ipairs(ids) do redis.call("LPUSH", KEYS[2], id) redis.call("HSET", ARGV[1] .. id, "state", "pending") end redis.call("DEL", KEYS[1]) redis.call("SREM", KEYS[3], ARGV[2]) return table.getn(ids) `) // RunAllAggregatingTasks schedules all tasks from the given queue to run // and returns the number of tasks scheduled to run. // If a queue with the given name doesn't exist, it returns QueueNotFoundError. func (r *RDB) RunAllAggregatingTasks(qname, gname string) (int64, error) { var op errors.Op = "rdb.RunAllAggregatingTasks" if err := r.checkQueueExists(qname); err != nil { return 0, errors.E(op, errors.CanonicalCode(err), err) } keys := []string{ base.GroupKey(qname, gname), base.PendingKey(qname), base.AllGroups(qname), } argv := []interface{}{ base.TaskKeyPrefix(qname), gname, } res, err := runAllAggregatingCmd.Run(context.Background(), r.client, keys, argv...).Result() if err != nil { return 0, errors.E(op, errors.Internal, err) } n, ok := res.(int64) if !ok { return 0, errors.E(op, errors.Internal, fmt.Sprintf("unexpected return value from script %v", res)) } return n, nil } // runTaskCmd is a Lua script that updates the given task to pending state. // // Input: // KEYS[1] -> asynq:{}:t: // KEYS[2] -> asynq:{}:pending // KEYS[3] -> asynq:{}:groups // -- // ARGV[1] -> task ID // ARGV[2] -> queue key prefix; asynq:{}: // ARGV[3] -> group key prefix // // Output: // Numeric code indicating the status: // Returns 1 if task is successfully updated. // Returns 0 if task is not found. // Returns -1 if task is in active state. // Returns -2 if task is in pending state. // Returns error reply if unexpected error occurs. var runTaskCmd = redis.NewScript(` if redis.call("EXISTS", KEYS[1]) == 0 then return 0 end local state, group = unpack(redis.call("HMGET", KEYS[1], "state", "group")) if state == "active" then return -1 elseif state == "pending" then return -2 elseif state == "aggregating" then local n = redis.call("ZREM", ARGV[3] .. group, ARGV[1]) if n == 0 then return redis.error_reply("internal error: task id not found in zset " .. tostring(ARGV[3] .. group)) end if redis.call("ZCARD", ARGV[3] .. group) == 0 then redis.call("SREM", KEYS[3], group) end else local n = redis.call("ZREM", ARGV[2] .. state, ARGV[1]) if n == 0 then return redis.error_reply("internal error: task id not found in zset " .. tostring(ARGV[2] .. state)) end end redis.call("LPUSH", KEYS[2], ARGV[1]) redis.call("HSET", KEYS[1], "state", "pending") return 1 `) // RunTask finds a task that matches the id from the given queue and updates it to pending state. // It returns nil if it successfully updated the task. // // If a queue with the given name doesn't exist, it returns QueueNotFoundError. // If a task with the given id doesn't exist in the queue, it returns TaskNotFoundError // If a task is in active or pending state it returns non-nil error with Code FailedPrecondition. func (r *RDB) RunTask(qname, id string) error { var op errors.Op = "rdb.RunTask" if err := r.checkQueueExists(qname); err != nil { return errors.E(op, errors.CanonicalCode(err), err) } keys := []string{ base.TaskKey(qname, id), base.PendingKey(qname), base.AllGroups(qname), } argv := []interface{}{ id, base.QueueKeyPrefix(qname), base.GroupKeyPrefix(qname), } res, err := runTaskCmd.Run(context.Background(), r.client, keys, argv...).Result() if err != nil { return errors.E(op, errors.Unknown, err) } n, ok := res.(int64) if !ok { return errors.E(op, errors.Internal, fmt.Sprintf("cast error: unexpected return value from Lua script: %v", res)) } switch n { case 1: return nil case 0: return errors.E(op, errors.NotFound, &errors.TaskNotFoundError{Queue: qname, ID: id}) case -1: return errors.E(op, errors.FailedPrecondition, "task is already running") case -2: return errors.E(op, errors.FailedPrecondition, "task is already in pending state") default: return errors.E(op, errors.Internal, fmt.Sprintf("unexpected return value from Lua script %d", n)) } } // runAllCmd is a Lua script that moves all tasks in the given state // (one of: scheduled, retry, archived) to pending state. // // Input: // KEYS[1] -> zset which holds task ids (e.g. asynq:{}:scheduled) // KEYS[2] -> asynq:{}:pending // -- // ARGV[1] -> task key prefix // // Output: // integer: number of tasks updated to pending state. var runAllCmd = redis.NewScript(` local ids = redis.call("ZRANGE", KEYS[1], 0, -1) for _, id in ipairs(ids) do redis.call("LPUSH", KEYS[2], id) redis.call("HSET", ARGV[1] .. id, "state", "pending") end redis.call("DEL", KEYS[1]) return table.getn(ids)`) func (r *RDB) runAll(zset, qname string) (int64, error) { if err := r.checkQueueExists(qname); err != nil { return 0, err } keys := []string{ zset, base.PendingKey(qname), } argv := []interface{}{ base.TaskKeyPrefix(qname), } res, err := runAllCmd.Run(context.Background(), r.client, keys, argv...).Result() if err != nil { return 0, err } n, ok := res.(int64) if !ok { return 0, fmt.Errorf("could not cast %v to int64", res) } if n == -1 { return 0, &errors.QueueNotFoundError{Queue: qname} } return n, nil } // ArchiveAllRetryTasks archives all retry tasks from the given queue and // returns the number of tasks that were moved. // If a queue with the given name doesn't exist, it returns QueueNotFoundError. func (r *RDB) ArchiveAllRetryTasks(qname string) (int64, error) { var op errors.Op = "rdb.ArchiveAllRetryTasks" n, err := r.archiveAll(base.RetryKey(qname), base.ArchivedKey(qname), qname) if errors.IsQueueNotFound(err) { return 0, errors.E(op, errors.NotFound, err) } if err != nil { return 0, errors.E(op, errors.Internal, err) } return n, nil } // ArchiveAllScheduledTasks archives all scheduled tasks from the given queue and // returns the number of tasks that were moved. // If a queue with the given name doesn't exist, it returns QueueNotFoundError. func (r *RDB) ArchiveAllScheduledTasks(qname string) (int64, error) { var op errors.Op = "rdb.ArchiveAllScheduledTasks" n, err := r.archiveAll(base.ScheduledKey(qname), base.ArchivedKey(qname), qname) if errors.IsQueueNotFound(err) { return 0, errors.E(op, errors.NotFound, err) } if err != nil { return 0, errors.E(op, errors.Internal, err) } return n, nil } // archiveAllAggregatingCmd archives all tasks in the given group. // // Input: // KEYS[1] -> asynq:{}:g: // KEYS[2] -> asynq:{}:archived // KEYS[3] -> asynq:{}:groups // ------- // ARGV[1] -> current timestamp // ARGV[2] -> cutoff timestamp (e.g., 90 days ago) // ARGV[3] -> max number of tasks in archive (e.g., 100) // ARGV[4] -> task key prefix (asynq:{}:t:) // ARGV[5] -> group name // // Output: // integer: Number of tasks archived var archiveAllAggregatingCmd = redis.NewScript(` local ids = redis.call("ZRANGE", KEYS[1], 0, -1) for _, id in ipairs(ids) do redis.call("ZADD", KEYS[2], ARGV[1], id) redis.call("HSET", ARGV[4] .. id, "state", "archived") end redis.call("ZREMRANGEBYSCORE", KEYS[2], "-inf", ARGV[2]) redis.call("ZREMRANGEBYRANK", KEYS[2], 0, -ARGV[3]) redis.call("DEL", KEYS[1]) redis.call("SREM", KEYS[3], ARGV[5]) return table.getn(ids) `) // ArchiveAllAggregatingTasks archives all aggregating tasks from the given group // and returns the number of tasks archived. // If a queue with the given name doesn't exist, it returns QueueNotFoundError. func (r *RDB) ArchiveAllAggregatingTasks(qname, gname string) (int64, error) { var op errors.Op = "rdb.ArchiveAllAggregatingTasks" if err := r.checkQueueExists(qname); err != nil { return 0, errors.E(op, errors.CanonicalCode(err), err) } keys := []string{ base.GroupKey(qname, gname), base.ArchivedKey(qname), base.AllGroups(qname), } now := r.clock.Now() argv := []interface{}{ now.Unix(), now.AddDate(0, 0, -archivedExpirationInDays).Unix(), maxArchiveSize, base.TaskKeyPrefix(qname), gname, } res, err := archiveAllAggregatingCmd.Run(context.Background(), r.client, keys, argv...).Result() if err != nil { return 0, errors.E(op, errors.Internal, err) } n, ok := res.(int64) if !ok { return 0, errors.E(op, errors.Internal, fmt.Sprintf("unexpected return value from script %v", res)) } return n, nil } // archiveAllPendingCmd is a Lua script that moves all pending tasks from // the given queue to archived state. // // Input: // KEYS[1] -> asynq:{}:pending // KEYS[2] -> asynq:{}:archived // -- // ARGV[1] -> current timestamp // ARGV[2] -> cutoff timestamp (e.g., 90 days ago) // ARGV[3] -> max number of tasks in archive (e.g., 100) // ARGV[4] -> task key prefix (asynq:{}:t:) // // Output: // integer: Number of tasks archived var archiveAllPendingCmd = redis.NewScript(` local ids = redis.call("LRANGE", KEYS[1], 0, -1) for _, id in ipairs(ids) do redis.call("ZADD", KEYS[2], ARGV[1], id) redis.call("HSET", ARGV[4] .. id, "state", "archived") end redis.call("ZREMRANGEBYSCORE", KEYS[2], "-inf", ARGV[2]) redis.call("ZREMRANGEBYRANK", KEYS[2], 0, -ARGV[3]) redis.call("DEL", KEYS[1]) return table.getn(ids)`) // ArchiveAllPendingTasks archives all pending tasks from the given queue and // returns the number of tasks moved. // If a queue with the given name doesn't exist, it returns QueueNotFoundError. func (r *RDB) ArchiveAllPendingTasks(qname string) (int64, error) { var op errors.Op = "rdb.ArchiveAllPendingTasks" if err := r.checkQueueExists(qname); err != nil { return 0, errors.E(op, errors.CanonicalCode(err), err) } keys := []string{ base.PendingKey(qname), base.ArchivedKey(qname), } now := r.clock.Now() argv := []interface{}{ now.Unix(), now.AddDate(0, 0, -archivedExpirationInDays).Unix(), maxArchiveSize, base.TaskKeyPrefix(qname), } res, err := archiveAllPendingCmd.Run(context.Background(), r.client, keys, argv...).Result() if err != nil { return 0, errors.E(op, errors.Internal, err) } n, ok := res.(int64) if !ok { return 0, errors.E(op, errors.Internal, fmt.Sprintf("unexpected return value from script %v", res)) } return n, nil } // archiveTaskCmd is a Lua script that archives a task given a task id. // // Input: // KEYS[1] -> task key (asynq:{}:t:) // KEYS[2] -> archived key (asynq:{}:archived) // KEYS[3] -> all groups key (asynq:{}:groups) // -- // ARGV[1] -> id of the task to archive // ARGV[2] -> current timestamp // ARGV[3] -> cutoff timestamp (e.g., 90 days ago) // ARGV[4] -> max number of tasks in archived state (e.g., 100) // ARGV[5] -> queue key prefix (asynq:{}:) // ARGV[6] -> group key prefix (asynq:{}:g:) // // Output: // Numeric code indicating the status: // Returns 1 if task is successfully archived. // Returns 0 if task is not found. // Returns -1 if task is already archived. // Returns -2 if task is in active state. // Returns error reply if unexpected error occurs. var archiveTaskCmd = redis.NewScript(` if redis.call("EXISTS", KEYS[1]) == 0 then return 0 end local state, group = unpack(redis.call("HMGET", KEYS[1], "state", "group")) if state == "active" then return -2 end if state == "archived" then return -1 end if state == "pending" then if redis.call("LREM", ARGV[5] .. state, 1, ARGV[1]) == 0 then return redis.error_reply("task id not found in list " .. tostring(ARGV[5] .. state)) end elseif state == "aggregating" then if redis.call("ZREM", ARGV[6] .. group, ARGV[1]) == 0 then return redis.error_reply("task id not found in zset " .. tostring(ARGV[6] .. group)) end if redis.call("ZCARD", ARGV[6] .. group) == 0 then redis.call("SREM", KEYS[3], group) end else if redis.call("ZREM", ARGV[5] .. state, ARGV[1]) == 0 then return redis.error_reply("task id not found in zset " .. tostring(ARGV[5] .. state)) end end redis.call("ZADD", KEYS[2], ARGV[2], ARGV[1]) redis.call("HSET", KEYS[1], "state", "archived") redis.call("ZREMRANGEBYSCORE", KEYS[2], "-inf", ARGV[3]) redis.call("ZREMRANGEBYRANK", KEYS[2], 0, -ARGV[4]) return 1 `) // ArchiveTask finds a task that matches the id from the given queue and archives it. // It returns nil if it successfully archived the task. // // If a queue with the given name doesn't exist, it returns QueueNotFoundError. // If a task with the given id doesn't exist in the queue, it returns TaskNotFoundError // If a task is already archived, it returns TaskAlreadyArchivedError. // If a task is in active state it returns non-nil error with FailedPrecondition code. func (r *RDB) ArchiveTask(qname, id string) error { var op errors.Op = "rdb.ArchiveTask" if err := r.checkQueueExists(qname); err != nil { return errors.E(op, errors.CanonicalCode(err), err) } keys := []string{ base.TaskKey(qname, id), base.ArchivedKey(qname), base.AllGroups(qname), } now := r.clock.Now() argv := []interface{}{ id, now.Unix(), now.AddDate(0, 0, -archivedExpirationInDays).Unix(), maxArchiveSize, base.QueueKeyPrefix(qname), base.GroupKeyPrefix(qname), } res, err := archiveTaskCmd.Run(context.Background(), r.client, keys, argv...).Result() if err != nil { return errors.E(op, errors.Unknown, err) } n, ok := res.(int64) if !ok { return errors.E(op, errors.Internal, fmt.Sprintf("could not cast the return value %v from archiveTaskCmd to int64.", res)) } switch n { case 1: return nil case 0: return errors.E(op, errors.NotFound, &errors.TaskNotFoundError{Queue: qname, ID: id}) case -1: return errors.E(op, errors.FailedPrecondition, &errors.TaskAlreadyArchivedError{Queue: qname, ID: id}) case -2: return errors.E(op, errors.FailedPrecondition, "cannot archive task in active state. use CancelProcessing instead.") case -3: return errors.E(op, errors.NotFound, &errors.QueueNotFoundError{Queue: qname}) default: return errors.E(op, errors.Internal, fmt.Sprintf("unexpected return value from archiveTaskCmd script: %d", n)) } } // archiveAllCmd is a Lua script that archives all tasks in either scheduled // or retry state from the given queue. // // Input: // KEYS[1] -> ZSET to move task from (e.g., asynq:{}:retry) // KEYS[2] -> asynq:{}:archived // -- // ARGV[1] -> current timestamp // ARGV[2] -> cutoff timestamp (e.g., 90 days ago) // ARGV[3] -> max number of tasks in archive (e.g., 100) // ARGV[4] -> task key prefix (asynq:{}:t:) // // Output: // integer: number of tasks archived var archiveAllCmd = redis.NewScript(` local ids = redis.call("ZRANGE", KEYS[1], 0, -1) for _, id in ipairs(ids) do redis.call("ZADD", KEYS[2], ARGV[1], id) redis.call("HSET", ARGV[4] .. id, "state", "archived") end redis.call("ZREMRANGEBYSCORE", KEYS[2], "-inf", ARGV[2]) redis.call("ZREMRANGEBYRANK", KEYS[2], 0, -ARGV[3]) redis.call("DEL", KEYS[1]) return table.getn(ids)`) func (r *RDB) archiveAll(src, dst, qname string) (int64, error) { if err := r.checkQueueExists(qname); err != nil { return 0, err } keys := []string{ src, dst, } now := r.clock.Now() argv := []interface{}{ now.Unix(), now.AddDate(0, 0, -archivedExpirationInDays).Unix(), maxArchiveSize, base.TaskKeyPrefix(qname), qname, } res, err := archiveAllCmd.Run(context.Background(), r.client, keys, argv...).Result() if err != nil { return 0, err } n, ok := res.(int64) if !ok { return 0, fmt.Errorf("unexpected return value from script: %v", res) } if n == -1 { return 0, &errors.QueueNotFoundError{Queue: qname} } return n, nil } // Input: // KEYS[1] -> asynq:{}:t: // -- // ARGV[1] -> task message data // // Output: // Numeric code indicating the status: // Returns 1 if task is successfully updated. // Returns 0 if task is not found. // Returns -1 if task is not in scheduled state. var updateTaskPayloadCmd = redis.NewScript(` -- Check if given taks exists if redis.call("EXISTS", KEYS[1]) == 0 then return 0 end local state, pending_since, group, unique_key = unpack(redis.call("HMGET", KEYS[1], "state", "pending_since", "group", "unique_key")) if state ~= "scheduled" then return -1 end local redis_call_args = {"state", state} if pending_since then table.insert(redis_call_args, "pending_since") table.insert(redis_call_args, pending_since) end if group then table.insert(redis_call_args, "group") table.insert(redis_call_args, group) end if unique_key then table.insert(redis_call_args, "unique_key") table.insert(redis_call_args, unique_key) end redis.call("HSET", KEYS[1], "msg", ARGV[1], unpack(redis_call_args)) return 1 `) // UpdateTaskPayload finds a task that matches the id from the given queue and updates it's payload. // It returns nil if it successfully updated the task payload. // // If a queue with the given name doesn't exist, it returns QueueNotFoundError. // If a task with the given id doesn't exist in the queue, it returns TaskNotFoundError // If a task is in active state it returns non-nil error with Code FailedPrecondition. func (r *RDB) UpdateTaskPayload(qname, id string, payload []byte) error { var op errors.Op = "rdb.UpdateTask" if err := r.checkQueueExists(qname); err != nil { return errors.E(op, errors.CanonicalCode(err), err) } taskInfo, err := r.GetTaskInfo(qname, id) if err != nil { return errors.E(op, errors.Unknown, err) } taskInfo.Message.Payload = payload encoded, err := base.EncodeMessage(taskInfo.Message) if err != nil { return errors.E(op, errors.Unknown, fmt.Sprintf("cannot encode message: %v", err)) } keys := []string{ base.TaskKey(qname, id), } argv := []interface{}{ encoded, } res, err := updateTaskPayloadCmd.Run(context.Background(), r.client, keys, argv...).Result() if err != nil { return errors.E(op, errors.Unknown, err) } n, ok := res.(int64) if !ok { return errors.E(op, errors.Internal, fmt.Sprintf("cast error: updateTaskCmd script returned unexported value %v", res)) } switch n { case 1: return nil case 0: return errors.E(op, errors.NotFound, &errors.TaskNotFoundError{Queue: qname, ID: id}) case -1: return errors.E(op, errors.FailedPrecondition, "cannot update task that is not in scheduled state.") default: return errors.E(op, errors.Internal, fmt.Sprintf("unexpected return value from updateTaskCmd script: %d", n)) } } // Input: // KEYS[1] -> asynq:{}:t: // KEYS[2] -> asynq:{}:groups // -- // ARGV[1] -> task ID // ARGV[2] -> queue key prefix // ARGV[3] -> group key prefix // // Output: // Numeric code indicating the status: // Returns 1 if task is successfully deleted. // Returns 0 if task is not found. // Returns -1 if task is in active state. var deleteTaskCmd = redis.NewScript(` if redis.call("EXISTS", KEYS[1]) == 0 then return 0 end local state, group = unpack(redis.call("HMGET", KEYS[1], "state", "group")) if state == "active" then return -1 end if state == "pending" then if redis.call("LREM", ARGV[2] .. state, 0, ARGV[1]) == 0 then return redis.error_reply("task is not found in list: " .. tostring(ARGV[2] .. state)) end elseif state == "aggregating" then if redis.call("ZREM", ARGV[3] .. group, ARGV[1]) == 0 then return redis.error_reply("task is not found in zset: " .. tostring(ARGV[3] .. group)) end if redis.call("ZCARD", ARGV[3] .. group) == 0 then redis.call("SREM", KEYS[2], group) end else if redis.call("ZREM", ARGV[2] .. state, ARGV[1]) == 0 then return redis.error_reply("task is not found in zset: " .. tostring(ARGV[2] .. state)) end end local unique_key = redis.call("HGET", KEYS[1], "unique_key") if unique_key and unique_key ~= "" and redis.call("GET", unique_key) == ARGV[1] then redis.call("DEL", unique_key) end return redis.call("DEL", KEYS[1]) `) // DeleteTask finds a task that matches the id from the given queue and deletes it. // It returns nil if it successfully archived the task. // // If a queue with the given name doesn't exist, it returns QueueNotFoundError. // If a task with the given id doesn't exist in the queue, it returns TaskNotFoundError // If a task is in active state it returns non-nil error with Code FailedPrecondition. func (r *RDB) DeleteTask(qname, id string) error { var op errors.Op = "rdb.DeleteTask" if err := r.checkQueueExists(qname); err != nil { return errors.E(op, errors.CanonicalCode(err), err) } keys := []string{ base.TaskKey(qname, id), base.AllGroups(qname), } argv := []interface{}{ id, base.QueueKeyPrefix(qname), base.GroupKeyPrefix(qname), } res, err := deleteTaskCmd.Run(context.Background(), r.client, keys, argv...).Result() if err != nil { return errors.E(op, errors.Unknown, err) } n, ok := res.(int64) if !ok { return errors.E(op, errors.Internal, fmt.Sprintf("cast error: deleteTaskCmd script returned unexported value %v", res)) } switch n { case 1: return nil case 0: return errors.E(op, errors.NotFound, &errors.TaskNotFoundError{Queue: qname, ID: id}) case -1: return errors.E(op, errors.FailedPrecondition, "cannot delete task in active state. use CancelProcessing instead.") default: return errors.E(op, errors.Internal, fmt.Sprintf("unexpected return value from deleteTaskCmd script: %d", n)) } } // DeleteAllArchivedTasks deletes all archived tasks from the given queue // and returns the number of tasks deleted. func (r *RDB) DeleteAllArchivedTasks(qname string) (int64, error) { var op errors.Op = "rdb.DeleteAllArchivedTasks" n, err := r.deleteAll(base.ArchivedKey(qname), qname) if errors.IsQueueNotFound(err) { return 0, errors.E(op, errors.NotFound, err) } if err != nil { return 0, errors.E(op, errors.Unknown, err) } return n, nil } // DeleteAllRetryTasks deletes all retry tasks from the given queue // and returns the number of tasks deleted. func (r *RDB) DeleteAllRetryTasks(qname string) (int64, error) { var op errors.Op = "rdb.DeleteAllRetryTasks" n, err := r.deleteAll(base.RetryKey(qname), qname) if errors.IsQueueNotFound(err) { return 0, errors.E(op, errors.NotFound, err) } if err != nil { return 0, errors.E(op, errors.Unknown, err) } return n, nil } // DeleteAllScheduledTasks deletes all scheduled tasks from the given queue // and returns the number of tasks deleted. func (r *RDB) DeleteAllScheduledTasks(qname string) (int64, error) { var op errors.Op = "rdb.DeleteAllScheduledTasks" n, err := r.deleteAll(base.ScheduledKey(qname), qname) if errors.IsQueueNotFound(err) { return 0, errors.E(op, errors.NotFound, err) } if err != nil { return 0, errors.E(op, errors.Unknown, err) } return n, nil } // DeleteAllCompletedTasks deletes all completed tasks from the given queue // and returns the number of tasks deleted. func (r *RDB) DeleteAllCompletedTasks(qname string) (int64, error) { var op errors.Op = "rdb.DeleteAllCompletedTasks" n, err := r.deleteAll(base.CompletedKey(qname), qname) if errors.IsQueueNotFound(err) { return 0, errors.E(op, errors.NotFound, err) } if err != nil { return 0, errors.E(op, errors.Unknown, err) } return n, nil } // deleteAllCmd deletes tasks from the given zset. // // Input: // KEYS[1] -> zset holding the task ids. // -- // ARGV[1] -> task key prefix // // Output: // integer: number of tasks deleted var deleteAllCmd = redis.NewScript(` local ids = redis.call("ZRANGE", KEYS[1], 0, -1) for _, id in ipairs(ids) do local task_key = ARGV[1] .. id local unique_key = redis.call("HGET", task_key, "unique_key") if unique_key and unique_key ~= "" and redis.call("GET", unique_key) == id then redis.call("DEL", unique_key) end redis.call("DEL", task_key) end redis.call("DEL", KEYS[1]) return table.getn(ids)`) func (r *RDB) deleteAll(key, qname string) (int64, error) { if err := r.checkQueueExists(qname); err != nil { return 0, err } argv := []interface{}{ base.TaskKeyPrefix(qname), qname, } res, err := deleteAllCmd.Run(context.Background(), r.client, []string{key}, argv...).Result() if err != nil { return 0, err } n, ok := res.(int64) if !ok { return 0, fmt.Errorf("unexpected return value from Lua script: %v", res) } return n, nil } // deleteAllAggregatingCmd deletes all tasks from the given group. // // Input: // KEYS[1] -> asynq:{}:g: // KEYS[2] -> asynq:{}:groups // ------- // ARGV[1] -> task key prefix // ARGV[2] -> group name var deleteAllAggregatingCmd = redis.NewScript(` local ids = redis.call("ZRANGE", KEYS[1], 0, -1) for _, id in ipairs(ids) do redis.call("DEL", ARGV[1] .. id) end redis.call("SREM", KEYS[2], ARGV[2]) redis.call("DEL", KEYS[1]) return table.getn(ids) `) // DeleteAllAggregatingTasks deletes all aggregating tasks from the given group // and returns the number of tasks deleted. func (r *RDB) DeleteAllAggregatingTasks(qname, gname string) (int64, error) { var op errors.Op = "rdb.DeleteAllAggregatingTasks" if err := r.checkQueueExists(qname); err != nil { return 0, errors.E(op, errors.CanonicalCode(err), err) } keys := []string{ base.GroupKey(qname, gname), base.AllGroups(qname), } argv := []interface{}{ base.TaskKeyPrefix(qname), gname, } res, err := deleteAllAggregatingCmd.Run(context.Background(), r.client, keys, argv...).Result() if err != nil { return 0, errors.E(op, errors.Unknown, err) } n, ok := res.(int64) if !ok { return 0, errors.E(op, errors.Internal, "command error: unexpected return value %v", res) } return n, nil } // deleteAllPendingCmd deletes all pending tasks from the given queue. // // Input: // KEYS[1] -> asynq:{}:pending // -- // ARGV[1] -> task key prefix // // Output: // integer: number of tasks deleted var deleteAllPendingCmd = redis.NewScript(` local ids = redis.call("LRANGE", KEYS[1], 0, -1) for _, id in ipairs(ids) do redis.call("DEL", ARGV[1] .. id) end redis.call("DEL", KEYS[1]) return table.getn(ids)`) // DeleteAllPendingTasks deletes all pending tasks from the given queue // and returns the number of tasks deleted. func (r *RDB) DeleteAllPendingTasks(qname string) (int64, error) { var op errors.Op = "rdb.DeleteAllPendingTasks" if err := r.checkQueueExists(qname); err != nil { return 0, errors.E(op, errors.CanonicalCode(err), err) } keys := []string{ base.PendingKey(qname), } argv := []interface{}{ base.TaskKeyPrefix(qname), } res, err := deleteAllPendingCmd.Run(context.Background(), r.client, keys, argv...).Result() if err != nil { return 0, errors.E(op, errors.Unknown, err) } n, ok := res.(int64) if !ok { return 0, errors.E(op, errors.Internal, "command error: unexpected return value %v", res) } return n, nil } // removeQueueForceCmd removes the given queue regardless of // whether the queue is empty. // It only check whether active queue is empty before removing. // // Input: // KEYS[1] -> asynq:{} // KEYS[2] -> asynq:{}:active // KEYS[3] -> asynq:{}:scheduled // KEYS[4] -> asynq:{}:retry // KEYS[5] -> asynq:{}:archived // KEYS[6] -> asynq:{}:lease // -- // ARGV[1] -> task key prefix // // Output: // Numeric code to indicate the status. // Returns 1 if successfully removed. // Returns -2 if the queue has active tasks. var removeQueueForceCmd = redis.NewScript(` local active = redis.call("LLEN", KEYS[2]) if active > 0 then return -2 end for _, id in ipairs(redis.call("LRANGE", KEYS[1], 0, -1)) do redis.call("DEL", ARGV[1] .. id) end for _, id in ipairs(redis.call("LRANGE", KEYS[2], 0, -1)) do redis.call("DEL", ARGV[1] .. id) end for _, id in ipairs(redis.call("ZRANGE", KEYS[3], 0, -1)) do redis.call("DEL", ARGV[1] .. id) end for _, id in ipairs(redis.call("ZRANGE", KEYS[4], 0, -1)) do redis.call("DEL", ARGV[1] .. id) end for _, id in ipairs(redis.call("ZRANGE", KEYS[5], 0, -1)) do redis.call("DEL", ARGV[1] .. id) end for _, id in ipairs(redis.call("LRANGE", KEYS[1], 0, -1)) do redis.call("DEL", ARGV[1] .. id) end for _, id in ipairs(redis.call("LRANGE", KEYS[2], 0, -1)) do redis.call("DEL", ARGV[1] .. id) end for _, id in ipairs(redis.call("ZRANGE", KEYS[3], 0, -1)) do redis.call("DEL", ARGV[1] .. id) end for _, id in ipairs(redis.call("ZRANGE", KEYS[4], 0, -1)) do redis.call("DEL", ARGV[1] .. id) end for _, id in ipairs(redis.call("ZRANGE", KEYS[5], 0, -1)) do redis.call("DEL", ARGV[1] .. id) end redis.call("DEL", KEYS[1]) redis.call("DEL", KEYS[2]) redis.call("DEL", KEYS[3]) redis.call("DEL", KEYS[4]) redis.call("DEL", KEYS[5]) redis.call("DEL", KEYS[6]) return 1`) // removeQueueCmd removes the given queue. // It checks whether queue is empty before removing. // // Input: // KEYS[1] -> asynq:{}:pending // KEYS[2] -> asynq:{}:active // KEYS[3] -> asynq:{}:scheduled // KEYS[4] -> asynq:{}:retry // KEYS[5] -> asynq:{}:archived // KEYS[6] -> asynq:{}:lease // -- // ARGV[1] -> task key prefix // // Output: // Numeric code to indicate the status // Returns 1 if successfully removed. // Returns -1 if queue is not empty var removeQueueCmd = redis.NewScript(` local ids = {} for _, id in ipairs(redis.call("LRANGE", KEYS[1], 0, -1)) do table.insert(ids, id) end for _, id in ipairs(redis.call("LRANGE", KEYS[2], 0, -1)) do table.insert(ids, id) end for _, id in ipairs(redis.call("ZRANGE", KEYS[3], 0, -1)) do table.insert(ids, id) end for _, id in ipairs(redis.call("ZRANGE", KEYS[4], 0, -1)) do table.insert(ids, id) end for _, id in ipairs(redis.call("ZRANGE", KEYS[5], 0, -1)) do table.insert(ids, id) end if table.getn(ids) > 0 then return -1 end for _, id in ipairs(ids) do redis.call("DEL", ARGV[1] .. id) end for _, id in ipairs(ids) do redis.call("DEL", ARGV[1] .. id) end redis.call("DEL", KEYS[1]) redis.call("DEL", KEYS[2]) redis.call("DEL", KEYS[3]) redis.call("DEL", KEYS[4]) redis.call("DEL", KEYS[5]) redis.call("DEL", KEYS[6]) return 1`) // RemoveQueue removes the specified queue. // // If force is set to true, it will remove the queue regardless // as long as no tasks are active for the queue. // If force is set to false, it will only remove the queue if // the queue is empty. func (r *RDB) RemoveQueue(qname string, force bool) error { var op errors.Op = "rdb.RemoveQueue" exists, err := r.queueExists(qname) if err != nil { return err } if !exists { return errors.E(op, errors.NotFound, &errors.QueueNotFoundError{Queue: qname}) } var script *redis.Script if force { script = removeQueueForceCmd } else { script = removeQueueCmd } keys := []string{ base.PendingKey(qname), base.ActiveKey(qname), base.ScheduledKey(qname), base.RetryKey(qname), base.ArchivedKey(qname), base.LeaseKey(qname), } res, err := script.Run(context.Background(), r.client, keys, base.TaskKeyPrefix(qname)).Result() if err != nil { return errors.E(op, errors.Unknown, err) } n, ok := res.(int64) if !ok { return errors.E(op, errors.Internal, fmt.Sprintf("unexpeced return value from Lua script: %v", res)) } switch n { case 1: if err := r.client.SRem(context.Background(), base.AllQueues, qname).Err(); err != nil { return errors.E(op, errors.Unknown, err) } r.queuesPublished.Delete(qname) return nil case -1: return errors.E(op, errors.NotFound, &errors.QueueNotEmptyError{Queue: qname}) case -2: return errors.E(op, errors.FailedPrecondition, "cannot remove queue with active tasks") default: return errors.E(op, errors.Unknown, fmt.Sprintf("unexpected return value from Lua script: %d", n)) } } // Note: Script also removes stale keys. var listServerKeysCmd = redis.NewScript(` local now = tonumber(ARGV[1]) local keys = redis.call("ZRANGEBYSCORE", KEYS[1], now, "+inf") redis.call("ZREMRANGEBYSCORE", KEYS[1], "-inf", now-1) return keys`) // ListServers returns the list of server info. func (r *RDB) ListServers() ([]*base.ServerInfo, error) { now := r.clock.Now() res, err := listServerKeysCmd.Run(context.Background(), r.client, []string{base.AllServers}, now.Unix()).Result() if err != nil { return nil, err } keys, err := cast.ToStringSliceE(res) if err != nil { return nil, err } var servers []*base.ServerInfo for _, key := range keys { data, err := r.client.Get(context.Background(), key).Result() if err != nil { continue // skip bad data } info, err := base.DecodeServerInfo([]byte(data)) if err != nil { continue // skip bad data } servers = append(servers, info) } return servers, nil } // Note: Script also removes stale keys. var listWorkersCmd = redis.NewScript(` local now = tonumber(ARGV[1]) local keys = redis.call("ZRANGEBYSCORE", KEYS[1], now, "+inf") redis.call("ZREMRANGEBYSCORE", KEYS[1], "-inf", now-1) return keys`) // ListWorkers returns the list of worker stats. func (r *RDB) ListWorkers() ([]*base.WorkerInfo, error) { var op errors.Op = "rdb.ListWorkers" now := r.clock.Now() res, err := listWorkersCmd.Run(context.Background(), r.client, []string{base.AllWorkers}, now.Unix()).Result() if err != nil { return nil, errors.E(op, errors.Unknown, err) } keys, err := cast.ToStringSliceE(res) if err != nil { return nil, errors.E(op, errors.Internal, fmt.Sprintf("unexpeced return value from Lua script: %v", res)) } var workers []*base.WorkerInfo for _, key := range keys { data, err := r.client.HVals(context.Background(), key).Result() if err != nil { continue // skip bad data } for _, s := range data { w, err := base.DecodeWorkerInfo([]byte(s)) if err != nil { continue // skip bad data } workers = append(workers, w) } } return workers, nil } // Note: Script also removes stale keys. var listSchedulerKeysCmd = redis.NewScript(` local now = tonumber(ARGV[1]) local keys = redis.call("ZRANGEBYSCORE", KEYS[1], now, "+inf") redis.call("ZREMRANGEBYSCORE", KEYS[1], "-inf", now-1) return keys`) // ListSchedulerEntries returns the list of scheduler entries. func (r *RDB) ListSchedulerEntries() ([]*base.SchedulerEntry, error) { now := r.clock.Now() res, err := listSchedulerKeysCmd.Run(context.Background(), r.client, []string{base.AllSchedulers}, now.Unix()).Result() if err != nil { return nil, err } keys, err := cast.ToStringSliceE(res) if err != nil { return nil, err } var entries []*base.SchedulerEntry for _, key := range keys { data, err := r.client.LRange(context.Background(), key, 0, -1).Result() if err != nil { continue // skip bad data } for _, s := range data { e, err := base.DecodeSchedulerEntry([]byte(s)) if err != nil { continue // skip bad data } entries = append(entries, e) } } return entries, nil } // ListSchedulerEnqueueEvents returns the list of scheduler enqueue events. func (r *RDB) ListSchedulerEnqueueEvents(entryID string, pgn Pagination) ([]*base.SchedulerEnqueueEvent, error) { key := base.SchedulerHistoryKey(entryID) zs, err := r.client.ZRevRangeWithScores(context.Background(), key, pgn.start(), pgn.stop()).Result() if err != nil { return nil, err } var events []*base.SchedulerEnqueueEvent for _, z := range zs { data, err := cast.ToStringE(z.Member) if err != nil { return nil, err } e, err := base.DecodeSchedulerEnqueueEvent([]byte(data)) if err != nil { return nil, err } events = append(events, e) } return events, nil } // Pause pauses processing of tasks from the given queue. func (r *RDB) Pause(qname string) error { key := base.PausedKey(qname) ok, err := r.client.SetNX(context.Background(), key, r.clock.Now().Unix(), 0).Result() if err != nil { return err } if !ok { return fmt.Errorf("queue %q is already paused", qname) } return nil } // Unpause resumes processing of tasks from the given queue. func (r *RDB) Unpause(qname string) error { key := base.PausedKey(qname) deleted, err := r.client.Del(context.Background(), key).Result() if err != nil { return err } if deleted == 0 { return fmt.Errorf("queue %q is not paused", qname) } return nil } // ClusterKeySlot returns an integer identifying the hash slot the given queue hashes to. func (r *RDB) ClusterKeySlot(qname string) (int64, error) { key := base.PendingKey(qname) return r.client.ClusterKeySlot(context.Background(), key).Result() } // ClusterNodes returns a list of nodes the given queue belongs to. func (r *RDB) ClusterNodes(qname string) ([]redis.ClusterNode, error) { keyslot, err := r.ClusterKeySlot(qname) if err != nil { return nil, err } clusterSlots, err := r.client.ClusterSlots(context.Background()).Result() if err != nil { return nil, err } for _, slotRange := range clusterSlots { if int64(slotRange.Start) <= keyslot && keyslot <= int64(slotRange.End) { return slotRange.Nodes, nil } } return nil, fmt.Errorf("nodes not found") } ================================================ FILE: internal/rdb/inspect_test.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package rdb import ( "context" "encoding/json" "fmt" "sort" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/google/uuid" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/errors" h "github.com/hibiken/asynq/internal/testutil" "github.com/hibiken/asynq/internal/timeutil" "github.com/redis/go-redis/v9" ) func TestAllQueues(t *testing.T) { r := setup(t) defer r.Close() tests := []struct { queues []string }{ {queues: []string{"default"}}, {queues: []string{"custom1", "custom2"}}, {queues: []string{"default", "custom1", "custom2"}}, {queues: []string{}}, } for _, tc := range tests { h.FlushDB(t, r.client) for _, qname := range tc.queues { if err := r.client.SAdd(context.Background(), base.AllQueues, qname).Err(); err != nil { t.Fatalf("could not initialize all queue set: %v", err) } } got, err := r.AllQueues() if err != nil { t.Errorf("AllQueues() returned an error: %v", err) continue } if diff := cmp.Diff(tc.queues, got, h.SortStringSliceOpt); diff != "" { t.Errorf("AllQueues() = %v, want %v; (-want, +got)\n%s", got, tc.queues, diff) } } } func TestCurrentStats(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessageBuilder().SetType("send_email").Build() m2 := h.NewTaskMessageBuilder().SetType("reindex").Build() m3 := h.NewTaskMessageBuilder().SetType("gen_thumbnail").Build() m4 := h.NewTaskMessageBuilder().SetType("sync").Build() m5 := h.NewTaskMessageBuilder().SetType("important_notification").SetQueue("critical").Build() m6 := h.NewTaskMessageBuilder().SetType("minor_notification").SetQueue("low").Build() m7 := h.NewTaskMessageBuilder().SetType("send_sms").Build() now := time.Now() r.SetClock(timeutil.NewSimulatedClock(now)) tests := []struct { tasks []*h.TaskSeedData allQueues []string allGroups map[string][]string pending map[string][]string active map[string][]string scheduled map[string][]redis.Z retry map[string][]redis.Z archived map[string][]redis.Z completed map[string][]redis.Z groups map[string][]redis.Z processed map[string]int failed map[string]int processedTotal map[string]int failedTotal map[string]int paused []string oldestPendingMessageEnqueueTime map[string]time.Time qname string want *Stats }{ { tasks: []*h.TaskSeedData{ {Msg: m1, State: base.TaskStatePending}, {Msg: m2, State: base.TaskStateActive}, {Msg: m3, State: base.TaskStateScheduled}, {Msg: m4, State: base.TaskStateScheduled}, {Msg: m5, State: base.TaskStatePending}, {Msg: m6, State: base.TaskStatePending}, {Msg: m7, State: base.TaskStateAggregating}, }, allQueues: []string{"default", "critical", "low"}, allGroups: map[string][]string{ base.AllGroups("default"): {"sms:user1"}, }, pending: map[string][]string{ base.PendingKey("default"): {m1.ID}, base.PendingKey("critical"): {m5.ID}, base.PendingKey("low"): {m6.ID}, }, active: map[string][]string{ base.ActiveKey("default"): {m2.ID}, base.ActiveKey("critical"): {}, base.ActiveKey("low"): {}, }, scheduled: map[string][]redis.Z{ base.ScheduledKey("default"): { {Member: m3.ID, Score: float64(now.Add(time.Hour).Unix())}, {Member: m4.ID, Score: float64(now.Unix())}, }, base.ScheduledKey("critical"): {}, base.ScheduledKey("low"): {}, }, retry: map[string][]redis.Z{ base.RetryKey("default"): {}, base.RetryKey("critical"): {}, base.RetryKey("low"): {}, }, archived: map[string][]redis.Z{ base.ArchivedKey("default"): {}, base.ArchivedKey("critical"): {}, base.ArchivedKey("low"): {}, }, completed: map[string][]redis.Z{ base.CompletedKey("default"): {}, base.CompletedKey("critical"): {}, base.CompletedKey("low"): {}, }, groups: map[string][]redis.Z{ base.GroupKey("default", "sms:user1"): { {Member: m7.ID, Score: float64(now.Add(-3 * time.Second).Unix())}, }, }, processed: map[string]int{ "default": 120, "critical": 100, "low": 50, }, failed: map[string]int{ "default": 2, "critical": 0, "low": 1, }, processedTotal: map[string]int{ "default": 11111, "critical": 22222, "low": 33333, }, failedTotal: map[string]int{ "default": 111, "critical": 222, "low": 333, }, oldestPendingMessageEnqueueTime: map[string]time.Time{ "default": now.Add(-15 * time.Second), "critical": now.Add(-200 * time.Millisecond), "low": now.Add(-30 * time.Second), }, paused: []string{}, qname: "default", want: &Stats{ Queue: "default", Paused: false, Size: 5, Groups: 1, Pending: 1, Active: 1, Scheduled: 2, Retry: 0, Archived: 0, Completed: 0, Aggregating: 1, Processed: 120, Failed: 2, ProcessedTotal: 11111, FailedTotal: 111, Latency: 15 * time.Second, Timestamp: now, }, }, { tasks: []*h.TaskSeedData{ {Msg: m1, State: base.TaskStatePending}, {Msg: m2, State: base.TaskStateActive}, {Msg: m3, State: base.TaskStateScheduled}, {Msg: m4, State: base.TaskStateScheduled}, {Msg: m6, State: base.TaskStatePending}, }, allQueues: []string{"default", "critical", "low"}, pending: map[string][]string{ base.PendingKey("default"): {m1.ID}, base.PendingKey("critical"): {}, base.PendingKey("low"): {m6.ID}, }, active: map[string][]string{ base.ActiveKey("default"): {m2.ID}, base.ActiveKey("critical"): {}, base.ActiveKey("low"): {}, }, scheduled: map[string][]redis.Z{ base.ScheduledKey("default"): { {Member: m3.ID, Score: float64(now.Add(time.Hour).Unix())}, {Member: m4.ID, Score: float64(now.Unix())}, }, base.ScheduledKey("critical"): {}, base.ScheduledKey("low"): {}, }, retry: map[string][]redis.Z{ base.RetryKey("default"): {}, base.RetryKey("critical"): {}, base.RetryKey("low"): {}, }, archived: map[string][]redis.Z{ base.ArchivedKey("default"): {}, base.ArchivedKey("critical"): {}, base.ArchivedKey("low"): {}, }, completed: map[string][]redis.Z{ base.CompletedKey("default"): {}, base.CompletedKey("critical"): {}, base.CompletedKey("low"): {}, }, processed: map[string]int{ "default": 120, "critical": 100, "low": 50, }, failed: map[string]int{ "default": 2, "critical": 0, "low": 1, }, processedTotal: map[string]int{ "default": 11111, "critical": 22222, "low": 33333, }, failedTotal: map[string]int{ "default": 111, "critical": 222, "low": 333, }, oldestPendingMessageEnqueueTime: map[string]time.Time{ "default": now.Add(-15 * time.Second), "critical": {}, // zero value since there's no pending task in this queue "low": now.Add(-30 * time.Second), }, paused: []string{"critical", "low"}, qname: "critical", want: &Stats{ Queue: "critical", Paused: true, Size: 0, Groups: 0, Pending: 0, Active: 0, Scheduled: 0, Retry: 0, Archived: 0, Completed: 0, Aggregating: 0, Processed: 100, Failed: 0, ProcessedTotal: 22222, FailedTotal: 222, Latency: 0, Timestamp: now, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case for _, qname := range tc.paused { if err := r.Pause(qname); err != nil { t.Fatal(err) } } h.SeedRedisSet(t, r.client, base.AllQueues, tc.allQueues) h.SeedRedisSets(t, r.client, tc.allGroups) h.SeedTasks(t, r.client, tc.tasks) h.SeedRedisLists(t, r.client, tc.pending) h.SeedRedisLists(t, r.client, tc.active) h.SeedRedisZSets(t, r.client, tc.scheduled) h.SeedRedisZSets(t, r.client, tc.retry) h.SeedRedisZSets(t, r.client, tc.archived) h.SeedRedisZSets(t, r.client, tc.completed) h.SeedRedisZSets(t, r.client, tc.groups) ctx := context.Background() for qname, n := range tc.processed { r.client.Set(ctx, base.ProcessedKey(qname, now), n, 0) } for qname, n := range tc.failed { r.client.Set(ctx, base.FailedKey(qname, now), n, 0) } for qname, n := range tc.processedTotal { r.client.Set(ctx, base.ProcessedTotalKey(qname), n, 0) } for qname, n := range tc.failedTotal { r.client.Set(ctx, base.FailedTotalKey(qname), n, 0) } for qname, enqueueTime := range tc.oldestPendingMessageEnqueueTime { if enqueueTime.IsZero() { continue } oldestPendingMessageID := r.client.LRange(ctx, base.PendingKey(qname), -1, -1).Val()[0] // get the right most msg in the list r.client.HSet(ctx, base.TaskKey(qname, oldestPendingMessageID), "pending_since", enqueueTime.UnixNano()) } got, err := r.CurrentStats(tc.qname) if err != nil { t.Errorf("r.CurrentStats(%q) = %v, %v, want %v, nil", tc.qname, got, err, tc.want) continue } ignoreMemUsg := cmpopts.IgnoreFields(Stats{}, "MemoryUsage") if diff := cmp.Diff(tc.want, got, timeCmpOpt, ignoreMemUsg); diff != "" { t.Errorf("r.CurrentStats(%q) = %v, %v, want %v, nil; (-want, +got)\n%s", tc.qname, got, err, tc.want, diff) continue } } } func TestCurrentStatsWithNonExistentQueue(t *testing.T) { r := setup(t) defer r.Close() qname := "non-existent" got, err := r.CurrentStats(qname) if !errors.IsQueueNotFound(err) { t.Fatalf("r.CurrentStats(%q) = %v, %v, want nil, %v", qname, got, err, &errors.QueueNotFoundError{Queue: qname}) } } func TestHistoricalStats(t *testing.T) { r := setup(t) defer r.Close() now := time.Now().UTC() tests := []struct { qname string // queue of interest n int // number of days }{ {"default", 90}, {"custom", 7}, {"default", 1}, } for _, tc := range tests { h.FlushDB(t, r.client) r.client.SAdd(context.Background(), base.AllQueues, tc.qname) // populate last n days data for i := 0; i < tc.n; i++ { ts := now.Add(-time.Duration(i) * 24 * time.Hour) processedKey := base.ProcessedKey(tc.qname, ts) failedKey := base.FailedKey(tc.qname, ts) r.client.Set(context.Background(), processedKey, (i+1)*1000, 0) r.client.Set(context.Background(), failedKey, (i+1)*10, 0) } got, err := r.HistoricalStats(tc.qname, tc.n) if err != nil { t.Errorf("RDB.HistoricalStats(%q, %d) returned error: %v", tc.qname, tc.n, err) continue } if len(got) != tc.n { t.Errorf("RDB.HistorycalStats(%q, %d) returned %d daily stats, want %d", tc.qname, tc.n, len(got), tc.n) continue } for i := 0; i < tc.n; i++ { want := &DailyStats{ Queue: tc.qname, Processed: (i + 1) * 1000, Failed: (i + 1) * 10, Time: now.Add(-time.Duration(i) * 24 * time.Hour), } // Allow 2 seconds difference in timestamp. cmpOpt := cmpopts.EquateApproxTime(2 * time.Second) if diff := cmp.Diff(want, got[i], cmpOpt); diff != "" { t.Errorf("RDB.HistoricalStats for the last %d days; got %+v, want %+v; (-want,+got):\n%s", i, got[i], want, diff) } } } } func TestRedisInfo(t *testing.T) { r := setup(t) defer r.Close() info, err := r.RedisInfo() if err != nil { t.Fatalf("RDB.RedisInfo() returned error: %v", err) } wantKeys := []string{ "redis_version", "uptime_in_days", "connected_clients", "used_memory_human", "used_memory_peak_human", "used_memory_peak_perc", } for _, key := range wantKeys { if _, ok := info[key]; !ok { t.Errorf("RDB.RedisInfo() = %v is missing entry for %q", info, key) } } } func TestGroupStats(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessageBuilder().SetGroup("group1").Build() m2 := h.NewTaskMessageBuilder().SetGroup("group1").Build() m3 := h.NewTaskMessageBuilder().SetGroup("group1").Build() m4 := h.NewTaskMessageBuilder().SetGroup("group2").Build() m5 := h.NewTaskMessageBuilder().SetQueue("custom").SetGroup("group1").Build() m6 := h.NewTaskMessageBuilder().SetQueue("custom").SetGroup("group1").Build() now := time.Now() fixtures := struct { tasks []*h.TaskSeedData allGroups map[string][]string groups map[string][]redis.Z }{ tasks: []*h.TaskSeedData{ {Msg: m1, State: base.TaskStateAggregating}, {Msg: m2, State: base.TaskStateAggregating}, {Msg: m3, State: base.TaskStateAggregating}, {Msg: m4, State: base.TaskStateAggregating}, {Msg: m5, State: base.TaskStateAggregating}, }, allGroups: map[string][]string{ base.AllGroups("default"): {"group1", "group2"}, base.AllGroups("custom"): {"group1"}, }, groups: map[string][]redis.Z{ base.GroupKey("default", "group1"): { {Member: m1.ID, Score: float64(now.Add(-10 * time.Second).Unix())}, {Member: m2.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, {Member: m3.ID, Score: float64(now.Add(-30 * time.Second).Unix())}, }, base.GroupKey("default", "group2"): { {Member: m4.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, }, base.GroupKey("custom", "group1"): { {Member: m5.ID, Score: float64(now.Add(-10 * time.Second).Unix())}, {Member: m6.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, }, }, } tests := []struct { desc string qname string want []*GroupStat }{ { desc: "default queue groups", qname: "default", want: []*GroupStat{ {Group: "group1", Size: 3}, {Group: "group2", Size: 1}, }, }, { desc: "custom queue groups", qname: "custom", want: []*GroupStat{ {Group: "group1", Size: 2}, }, }, } sortGroupStatsOpt := cmp.Transformer( "SortGroupStats", func(in []*GroupStat) []*GroupStat { out := append([]*GroupStat(nil), in...) sort.Slice(out, func(i, j int) bool { return out[i].Group < out[j].Group }) return out }) for _, tc := range tests { h.FlushDB(t, r.client) h.SeedTasks(t, r.client, fixtures.tasks) h.SeedRedisSets(t, r.client, fixtures.allGroups) h.SeedRedisZSets(t, r.client, fixtures.groups) t.Run(tc.desc, func(t *testing.T) { got, err := r.GroupStats(tc.qname) if err != nil { t.Fatalf("GroupStats returned error: %v", err) } if diff := cmp.Diff(tc.want, got, sortGroupStatsOpt); diff != "" { t.Errorf("GroupStats = %v, want %v; (-want,+got)\n%s", got, tc.want, diff) } }) } } func TestGetTaskInfo(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() fiveMinsFromNow := now.Add(5 * time.Minute) oneHourFromNow := now.Add(1 * time.Hour) twoHoursAgo := now.Add(-2 * time.Hour) m1 := h.NewTaskMessageWithQueue("task1", nil, "default") m2 := h.NewTaskMessageWithQueue("task2", nil, "default") m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") m4 := h.NewTaskMessageWithQueue("task4", nil, "custom") m5 := h.NewTaskMessageWithQueue("task5", nil, "custom") m6 := h.NewTaskMessageWithQueue("task5", nil, "custom") m6.CompletedAt = twoHoursAgo.Unix() m6.Retention = int64((24 * time.Hour).Seconds()) fixtures := struct { active map[string][]*base.TaskMessage pending map[string][]*base.TaskMessage scheduled map[string][]base.Z retry map[string][]base.Z archived map[string][]base.Z completed map[string][]base.Z }{ active: map[string][]*base.TaskMessage{ "default": {m1}, "custom": {}, }, pending: map[string][]*base.TaskMessage{ "default": {}, "custom": {m5}, }, scheduled: map[string][]base.Z{ "default": {{Message: m2, Score: fiveMinsFromNow.Unix()}}, "custom": {}, }, retry: map[string][]base.Z{ "default": {}, "custom": {{Message: m3, Score: oneHourFromNow.Unix()}}, }, archived: map[string][]base.Z{ "default": {}, "custom": {{Message: m4, Score: twoHoursAgo.Unix()}}, }, completed: map[string][]base.Z{ "default": {}, "custom": {{Message: m6, Score: m6.CompletedAt + m6.Retention}}, }, } h.SeedAllActiveQueues(t, r.client, fixtures.active) h.SeedAllPendingQueues(t, r.client, fixtures.pending) h.SeedAllScheduledQueues(t, r.client, fixtures.scheduled) h.SeedAllRetryQueues(t, r.client, fixtures.retry) h.SeedAllArchivedQueues(t, r.client, fixtures.archived) h.SeedAllCompletedQueues(t, r.client, fixtures.completed) // Write result data for the completed task. if err := r.client.HSet(context.Background(), base.TaskKey(m6.Queue, m6.ID), "result", "foobar").Err(); err != nil { t.Fatalf("Failed to write result data under task key: %v", err) } tests := []struct { qname string id string want *base.TaskInfo }{ { qname: "default", id: m1.ID, want: &base.TaskInfo{ Message: m1, State: base.TaskStateActive, NextProcessAt: time.Time{}, // zero value for N/A Result: nil, }, }, { qname: "default", id: m2.ID, want: &base.TaskInfo{ Message: m2, State: base.TaskStateScheduled, NextProcessAt: fiveMinsFromNow, Result: nil, }, }, { qname: "custom", id: m3.ID, want: &base.TaskInfo{ Message: m3, State: base.TaskStateRetry, NextProcessAt: oneHourFromNow, Result: nil, }, }, { qname: "custom", id: m4.ID, want: &base.TaskInfo{ Message: m4, State: base.TaskStateArchived, NextProcessAt: time.Time{}, // zero value for N/A Result: nil, }, }, { qname: "custom", id: m5.ID, want: &base.TaskInfo{ Message: m5, State: base.TaskStatePending, NextProcessAt: now, Result: nil, }, }, { qname: "custom", id: m6.ID, want: &base.TaskInfo{ Message: m6, State: base.TaskStateCompleted, NextProcessAt: time.Time{}, // zero value for N/A Result: []byte("foobar"), }, }, } for _, tc := range tests { got, err := r.GetTaskInfo(tc.qname, tc.id) if err != nil { t.Errorf("GetTaskInfo(%q, %v) returned error: %v", tc.qname, tc.id, err) continue } if diff := cmp.Diff(tc.want, got, cmpopts.EquateApproxTime(2*time.Second)); diff != "" { t.Errorf("GetTaskInfo(%q, %v) = %v, want %v; (-want,+got)\n%s", tc.qname, tc.id, got, tc.want, diff) } } } func TestGetTaskInfoError(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessageWithQueue("task1", nil, "default") m2 := h.NewTaskMessageWithQueue("task2", nil, "default") m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") m4 := h.NewTaskMessageWithQueue("task4", nil, "custom") m5 := h.NewTaskMessageWithQueue("task5", nil, "custom") now := time.Now() fiveMinsFromNow := now.Add(5 * time.Minute) oneHourFromNow := now.Add(1 * time.Hour) twoHoursAgo := now.Add(-2 * time.Hour) fixtures := struct { active map[string][]*base.TaskMessage pending map[string][]*base.TaskMessage scheduled map[string][]base.Z retry map[string][]base.Z archived map[string][]base.Z }{ active: map[string][]*base.TaskMessage{ "default": {m1}, "custom": {}, }, pending: map[string][]*base.TaskMessage{ "default": {}, "custom": {m5}, }, scheduled: map[string][]base.Z{ "default": {{Message: m2, Score: fiveMinsFromNow.Unix()}}, "custom": {}, }, retry: map[string][]base.Z{ "default": {}, "custom": {{Message: m3, Score: oneHourFromNow.Unix()}}, }, archived: map[string][]base.Z{ "default": {}, "custom": {{Message: m4, Score: twoHoursAgo.Unix()}}, }, } h.SeedAllActiveQueues(t, r.client, fixtures.active) h.SeedAllPendingQueues(t, r.client, fixtures.pending) h.SeedAllScheduledQueues(t, r.client, fixtures.scheduled) h.SeedAllRetryQueues(t, r.client, fixtures.retry) h.SeedAllArchivedQueues(t, r.client, fixtures.archived) tests := []struct { qname string id string match func(err error) bool }{ { qname: "nonexistent", id: m1.ID, match: errors.IsQueueNotFound, }, { qname: "default", id: uuid.NewString(), match: errors.IsTaskNotFound, }, } for _, tc := range tests { info, err := r.GetTaskInfo(tc.qname, tc.id) if info != nil { t.Errorf("GetTaskInfo(%q, %v) returned info: %v", tc.qname, tc.id, info) } if !tc.match(err) { t.Errorf("GetTaskInfo(%q, %v) returned unexpected error: %v", tc.qname, tc.id, err) } } } func TestListPending(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("send_email", h.JSON(map[string]interface{}{"subject": "hello"})) m2 := h.NewTaskMessage("reindex", nil) m3 := h.NewTaskMessageWithQueue("important_notification", nil, "critical") m4 := h.NewTaskMessageWithQueue("minor_notification", nil, "low") tests := []struct { pending map[string][]*base.TaskMessage qname string want []*base.TaskInfo }{ { pending: map[string][]*base.TaskMessage{ base.DefaultQueueName: {m1, m2}, }, qname: base.DefaultQueueName, want: []*base.TaskInfo{ {Message: m1, State: base.TaskStatePending, NextProcessAt: time.Now(), Result: nil}, {Message: m2, State: base.TaskStatePending, NextProcessAt: time.Now(), Result: nil}, }, }, { pending: map[string][]*base.TaskMessage{ base.DefaultQueueName: nil, }, qname: base.DefaultQueueName, want: []*base.TaskInfo(nil), }, { pending: map[string][]*base.TaskMessage{ base.DefaultQueueName: {m1, m2}, "critical": {m3}, "low": {m4}, }, qname: base.DefaultQueueName, want: []*base.TaskInfo{ {Message: m1, State: base.TaskStatePending, NextProcessAt: time.Now(), Result: nil}, {Message: m2, State: base.TaskStatePending, NextProcessAt: time.Now(), Result: nil}, }, }, { pending: map[string][]*base.TaskMessage{ base.DefaultQueueName: {m1, m2}, "critical": {m3}, "low": {m4}, }, qname: "critical", want: []*base.TaskInfo{ {Message: m3, State: base.TaskStatePending, NextProcessAt: time.Now(), Result: nil}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllPendingQueues(t, r.client, tc.pending) got, err := r.ListPending(tc.qname, Pagination{Size: 20, Page: 0}) op := fmt.Sprintf("r.ListPending(%q, Pagination{Size: 20, Page: 0})", tc.qname) if err != nil { t.Errorf("%s = %v, %v, want %v, nil", op, got, err, tc.want) continue } if diff := cmp.Diff(tc.want, got, cmpopts.EquateApproxTime(2*time.Second)); diff != "" { t.Errorf("%s = %v, %v, want %v, nil; (-want, +got)\n%s", op, got, err, tc.want, diff) continue } } } func TestListPendingPagination(t *testing.T) { r := setup(t) defer r.Close() var msgs []*base.TaskMessage for i := 0; i < 100; i++ { msg := h.NewTaskMessage(fmt.Sprintf("task %d", i), nil) msgs = append(msgs, msg) } // create 100 tasks in default queue h.SeedPendingQueue(t, r.client, msgs, "default") msgs = []*base.TaskMessage(nil) // empty list for i := 0; i < 100; i++ { msg := h.NewTaskMessageWithQueue(fmt.Sprintf("custom %d", i), nil, "custom") msgs = append(msgs, msg) } // create 100 tasks in custom queue h.SeedPendingQueue(t, r.client, msgs, "custom") tests := []struct { desc string qname string page int size int wantSize int wantFirst string wantLast string }{ {"first page", "default", 0, 20, 20, "task 0", "task 19"}, {"second page", "default", 1, 20, 20, "task 20", "task 39"}, {"different page size", "default", 2, 30, 30, "task 60", "task 89"}, {"last page", "default", 3, 30, 10, "task 90", "task 99"}, {"out of range", "default", 4, 30, 0, "", ""}, {"second page with custom queue", "custom", 1, 20, 20, "custom 20", "custom 39"}, } for _, tc := range tests { got, err := r.ListPending(tc.qname, Pagination{Size: tc.size, Page: tc.page}) op := fmt.Sprintf("r.ListPending(%q, Pagination{Size: %d, Page: %d})", tc.qname, tc.size, tc.page) if err != nil { t.Errorf("%s; %s returned error %v", tc.desc, op, err) continue } if len(got) != tc.wantSize { t.Errorf("%s; %s returned a list of size %d, want %d", tc.desc, op, len(got), tc.wantSize) continue } if tc.wantSize == 0 { continue } first := got[0].Message if first.Type != tc.wantFirst { t.Errorf("%s; %s returned a list with first message %q, want %q", tc.desc, op, first.Type, tc.wantFirst) } last := got[len(got)-1].Message if last.Type != tc.wantLast { t.Errorf("%s; %s returned a list with the last message %q, want %q", tc.desc, op, last.Type, tc.wantLast) } } } func TestListActive(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task2", nil, "critical") m4 := h.NewTaskMessageWithQueue("task2", nil, "low") tests := []struct { inProgress map[string][]*base.TaskMessage qname string want []*base.TaskInfo }{ { inProgress: map[string][]*base.TaskMessage{ "default": {m1, m2}, "critical": {m3}, "low": {m4}, }, qname: "default", want: []*base.TaskInfo{ {Message: m1, State: base.TaskStateActive, NextProcessAt: time.Time{}, Result: nil}, {Message: m2, State: base.TaskStateActive, NextProcessAt: time.Time{}, Result: nil}, }, }, { inProgress: map[string][]*base.TaskMessage{ "default": {}, }, qname: "default", want: []*base.TaskInfo(nil), }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllActiveQueues(t, r.client, tc.inProgress) got, err := r.ListActive(tc.qname, Pagination{Size: 20, Page: 0}) op := fmt.Sprintf("r.ListActive(%q, Pagination{Size: 20, Page: 0})", tc.qname) if err != nil { t.Errorf("%s = %v, %v, want %v, nil", op, got, err, tc.inProgress) continue } if diff := cmp.Diff(tc.want, got, cmpopts.EquateApproxTime(1*time.Second)); diff != "" { t.Errorf("%s = %v, %v, want %v, nil; (-want, +got)\n%s", op, got, err, tc.want, diff) continue } } } func TestListActivePagination(t *testing.T) { r := setup(t) defer r.Close() var msgs []*base.TaskMessage for i := 0; i < 100; i++ { msg := h.NewTaskMessage(fmt.Sprintf("task %d", i), nil) msgs = append(msgs, msg) } h.SeedActiveQueue(t, r.client, msgs, "default") tests := []struct { desc string qname string page int size int wantSize int wantFirst string wantLast string }{ {"first page", "default", 0, 20, 20, "task 0", "task 19"}, {"second page", "default", 1, 20, 20, "task 20", "task 39"}, {"different page size", "default", 2, 30, 30, "task 60", "task 89"}, {"last page", "default", 3, 30, 10, "task 90", "task 99"}, {"out of range", "default", 4, 30, 0, "", ""}, } for _, tc := range tests { got, err := r.ListActive(tc.qname, Pagination{Size: tc.size, Page: tc.page}) op := fmt.Sprintf("r.ListActive(%q, Pagination{Size: %d, Page: %d})", tc.qname, tc.size, tc.page) if err != nil { t.Errorf("%s; %s returned error %v", tc.desc, op, err) continue } if len(got) != tc.wantSize { t.Errorf("%s; %s returned list of size %d, want %d", tc.desc, op, len(got), tc.wantSize) continue } if tc.wantSize == 0 { continue } first := got[0].Message if first.Type != tc.wantFirst { t.Errorf("%s; %s returned a list with first message %q, want %q", tc.desc, op, first.Type, tc.wantFirst) } last := got[len(got)-1].Message if last.Type != tc.wantLast { t.Errorf("%s; %s returned a list with the last message %q, want %q", tc.desc, op, last.Type, tc.wantLast) } } } func TestListScheduled(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessage("task3", nil) m4 := h.NewTaskMessageWithQueue("task3", nil, "custom") p1 := time.Now().Add(30 * time.Minute) p2 := time.Now().Add(24 * time.Hour) p3 := time.Now().Add(5 * time.Minute) p4 := time.Now().Add(2 * time.Minute) tests := []struct { scheduled map[string][]base.Z qname string want []*base.TaskInfo }{ { scheduled: map[string][]base.Z{ "default": { {Message: m1, Score: p1.Unix()}, {Message: m2, Score: p2.Unix()}, {Message: m3, Score: p3.Unix()}, }, "custom": { {Message: m4, Score: p4.Unix()}, }, }, qname: "default", // should be sorted by score in ascending order want: []*base.TaskInfo{ {Message: m3, NextProcessAt: p3, State: base.TaskStateScheduled, Result: nil}, {Message: m1, NextProcessAt: p1, State: base.TaskStateScheduled, Result: nil}, {Message: m2, NextProcessAt: p2, State: base.TaskStateScheduled, Result: nil}, }, }, { scheduled: map[string][]base.Z{ "default": { {Message: m1, Score: p1.Unix()}, {Message: m2, Score: p2.Unix()}, {Message: m3, Score: p3.Unix()}, }, "custom": { {Message: m4, Score: p4.Unix()}, }, }, qname: "custom", want: []*base.TaskInfo{ {Message: m4, NextProcessAt: p4, State: base.TaskStateScheduled, Result: nil}, }, }, { scheduled: map[string][]base.Z{ "default": {}, }, qname: "default", want: []*base.TaskInfo(nil), }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllScheduledQueues(t, r.client, tc.scheduled) got, err := r.ListScheduled(tc.qname, Pagination{Size: 20, Page: 0}) op := fmt.Sprintf("r.ListScheduled(%q, Pagination{Size: 20, Page: 0})", tc.qname) if err != nil { t.Errorf("%s = %v, %v, want %v, nil", op, got, err, tc.want) continue } if diff := cmp.Diff(tc.want, got, cmpopts.EquateApproxTime(1*time.Second)); diff != "" { t.Errorf("%s = %v, %v, want %v, nil; (-want, +got)\n%s", op, got, err, tc.want, diff) continue } } } func TestListScheduledPagination(t *testing.T) { r := setup(t) defer r.Close() // create 100 tasks with an increasing number of wait time. for i := 0; i < 100; i++ { msg := h.NewTaskMessage(fmt.Sprintf("task %d", i), nil) if err := r.Schedule(context.Background(), msg, time.Now().Add(time.Duration(i)*time.Second)); err != nil { t.Fatal(err) } } tests := []struct { desc string qname string page int size int wantSize int wantFirst string wantLast string }{ {"first page", "default", 0, 20, 20, "task 0", "task 19"}, {"second page", "default", 1, 20, 20, "task 20", "task 39"}, {"different page size", "default", 2, 30, 30, "task 60", "task 89"}, {"last page", "default", 3, 30, 10, "task 90", "task 99"}, {"out of range", "default", 4, 30, 0, "", ""}, } for _, tc := range tests { got, err := r.ListScheduled(tc.qname, Pagination{Size: tc.size, Page: tc.page}) op := fmt.Sprintf("r.ListScheduled(%q, Pagination{Size: %d, Page: %d})", tc.qname, tc.size, tc.page) if err != nil { t.Errorf("%s; %s returned error %v", tc.desc, op, err) continue } if len(got) != tc.wantSize { t.Errorf("%s; %s returned list of size %d, want %d", tc.desc, op, len(got), tc.wantSize) continue } if tc.wantSize == 0 { continue } first := got[0].Message if first.Type != tc.wantFirst { t.Errorf("%s; %s returned a list with first message %q, want %q", tc.desc, op, first.Type, tc.wantFirst) } last := got[len(got)-1].Message if last.Type != tc.wantLast { t.Errorf("%s; %s returned a list with the last message %q, want %q", tc.desc, op, last.Type, tc.wantLast) } } } func TestListRetry(t *testing.T) { r := setup(t) defer r.Close() m1 := &base.TaskMessage{ ID: uuid.NewString(), Type: "task1", Queue: "default", Payload: nil, ErrorMsg: "some error occurred", Retry: 25, Retried: 10, } m2 := &base.TaskMessage{ ID: uuid.NewString(), Type: "task2", Queue: "default", Payload: nil, ErrorMsg: "some error occurred", Retry: 25, Retried: 2, } m3 := &base.TaskMessage{ ID: uuid.NewString(), Type: "task3", Queue: "custom", Payload: nil, ErrorMsg: "some error occurred", Retry: 25, Retried: 3, } p1 := time.Now().Add(5 * time.Minute) p2 := time.Now().Add(24 * time.Hour) p3 := time.Now().Add(24 * time.Hour) tests := []struct { retry map[string][]base.Z qname string want []*base.TaskInfo }{ { retry: map[string][]base.Z{ "default": { {Message: m1, Score: p1.Unix()}, {Message: m2, Score: p2.Unix()}, }, "custom": { {Message: m3, Score: p3.Unix()}, }, }, qname: "default", want: []*base.TaskInfo{ {Message: m1, NextProcessAt: p1, State: base.TaskStateRetry, Result: nil}, {Message: m2, NextProcessAt: p2, State: base.TaskStateRetry, Result: nil}, }, }, { retry: map[string][]base.Z{ "default": { {Message: m1, Score: p1.Unix()}, {Message: m2, Score: p2.Unix()}, }, "custom": { {Message: m3, Score: p3.Unix()}, }, }, qname: "custom", want: []*base.TaskInfo{ {Message: m3, NextProcessAt: p3, State: base.TaskStateRetry, Result: nil}, }, }, { retry: map[string][]base.Z{ "default": {}, }, qname: "default", want: []*base.TaskInfo(nil), }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllRetryQueues(t, r.client, tc.retry) got, err := r.ListRetry(tc.qname, Pagination{Size: 20, Page: 0}) op := fmt.Sprintf("r.ListRetry(%q, Pagination{Size: 20, Page: 0})", tc.qname) if err != nil { t.Errorf("%s = %v, %v, want %v, nil", op, got, err, tc.want) continue } if diff := cmp.Diff(tc.want, got, cmpopts.EquateApproxTime(1*time.Second)); diff != "" { t.Errorf("%s = %v, %v, want %v, nil; (-want, +got)\n%s", op, got, err, tc.want, diff) continue } } } func TestListRetryPagination(t *testing.T) { r := setup(t) defer r.Close() // create 100 tasks with an increasing number of wait time. now := time.Now() var seed []base.Z for i := 0; i < 100; i++ { msg := h.NewTaskMessage(fmt.Sprintf("task %d", i), nil) processAt := now.Add(time.Duration(i) * time.Second) seed = append(seed, base.Z{Message: msg, Score: processAt.Unix()}) } h.SeedRetryQueue(t, r.client, seed, "default") tests := []struct { desc string qname string page int size int wantSize int wantFirst string wantLast string }{ {"first page", "default", 0, 20, 20, "task 0", "task 19"}, {"second page", "default", 1, 20, 20, "task 20", "task 39"}, {"different page size", "default", 2, 30, 30, "task 60", "task 89"}, {"last page", "default", 3, 30, 10, "task 90", "task 99"}, {"out of range", "default", 4, 30, 0, "", ""}, } for _, tc := range tests { got, err := r.ListRetry(tc.qname, Pagination{Size: tc.size, Page: tc.page}) op := fmt.Sprintf("r.ListRetry(%q, Pagination{Size: %d, Page: %d})", tc.qname, tc.size, tc.page) if err != nil { t.Errorf("%s; %s returned error %v", tc.desc, op, err) continue } if len(got) != tc.wantSize { t.Errorf("%s; %s returned list of size %d, want %d", tc.desc, op, len(got), tc.wantSize) continue } if tc.wantSize == 0 { continue } first := got[0].Message if first.Type != tc.wantFirst { t.Errorf("%s; %s returned a list with first message %q, want %q", tc.desc, op, first.Type, tc.wantFirst) } last := got[len(got)-1].Message if last.Type != tc.wantLast { t.Errorf("%s; %s returned a list with the last message %q, want %q", tc.desc, op, last.Type, tc.wantLast) } } } func TestListArchived(t *testing.T) { r := setup(t) defer r.Close() m1 := &base.TaskMessage{ ID: uuid.NewString(), Type: "task1", Queue: "default", Payload: nil, ErrorMsg: "some error occurred", } m2 := &base.TaskMessage{ ID: uuid.NewString(), Type: "task2", Queue: "default", Payload: nil, ErrorMsg: "some error occurred", } m3 := &base.TaskMessage{ ID: uuid.NewString(), Type: "task3", Queue: "custom", Payload: nil, ErrorMsg: "some error occurred", } f1 := time.Now().Add(-5 * time.Minute) f2 := time.Now().Add(-24 * time.Hour) f3 := time.Now().Add(-4 * time.Hour) tests := []struct { archived map[string][]base.Z qname string want []*base.TaskInfo }{ { archived: map[string][]base.Z{ "default": { {Message: m1, Score: f1.Unix()}, {Message: m2, Score: f2.Unix()}, }, "custom": { {Message: m3, Score: f3.Unix()}, }, }, qname: "default", want: []*base.TaskInfo{ {Message: m2, NextProcessAt: time.Time{}, State: base.TaskStateArchived, Result: nil}, // FIXME: shouldn't be sorted in the other order? {Message: m1, NextProcessAt: time.Time{}, State: base.TaskStateArchived, Result: nil}, }, }, { archived: map[string][]base.Z{ "default": { {Message: m1, Score: f1.Unix()}, {Message: m2, Score: f2.Unix()}, }, "custom": { {Message: m3, Score: f3.Unix()}, }, }, qname: "custom", want: []*base.TaskInfo{ {Message: m3, NextProcessAt: time.Time{}, State: base.TaskStateArchived, Result: nil}, }, }, { archived: map[string][]base.Z{ "default": {}, }, qname: "default", want: []*base.TaskInfo(nil), }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllArchivedQueues(t, r.client, tc.archived) got, err := r.ListArchived(tc.qname, Pagination{Size: 20, Page: 0}) op := fmt.Sprintf("r.ListArchived(%q, Pagination{Size: 20, Page: 0})", tc.qname) if err != nil { t.Errorf("%s = %v, %v, want %v, nil", op, got, err, tc.want) continue } if diff := cmp.Diff(tc.want, got); diff != "" { t.Errorf("%s = %v, %v, want %v, nil; (-want, +got)\n%s", op, got, err, tc.want, diff) continue } } } func TestListArchivedPagination(t *testing.T) { r := setup(t) defer r.Close() var entries []base.Z for i := 0; i < 100; i++ { msg := h.NewTaskMessage(fmt.Sprintf("task %d", i), nil) entries = append(entries, base.Z{Message: msg, Score: int64(i)}) } h.SeedArchivedQueue(t, r.client, entries, "default") tests := []struct { desc string qname string page int size int wantSize int wantFirst string wantLast string }{ {"first page", "default", 0, 20, 20, "task 0", "task 19"}, {"second page", "default", 1, 20, 20, "task 20", "task 39"}, {"different page size", "default", 2, 30, 30, "task 60", "task 89"}, {"last page", "default", 3, 30, 10, "task 90", "task 99"}, {"out of range", "default", 4, 30, 0, "", ""}, } for _, tc := range tests { got, err := r.ListArchived(tc.qname, Pagination{Size: tc.size, Page: tc.page}) op := fmt.Sprintf("r.ListArchived(Pagination{Size: %d, Page: %d})", tc.size, tc.page) if err != nil { t.Errorf("%s; %s returned error %v", tc.desc, op, err) continue } if len(got) != tc.wantSize { t.Errorf("%s; %s returned list of size %d, want %d", tc.desc, op, len(got), tc.wantSize) continue } if tc.wantSize == 0 { continue } first := got[0].Message if first.Type != tc.wantFirst { t.Errorf("%s; %s returned a list with first message %q, want %q", tc.desc, op, first.Type, tc.wantFirst) } last := got[len(got)-1].Message if last.Type != tc.wantLast { t.Errorf("%s; %s returned a list with the last message %q, want %q", tc.desc, op, last.Type, tc.wantLast) } } } func TestListCompleted(t *testing.T) { r := setup(t) defer r.Close() msg1 := &base.TaskMessage{ ID: uuid.NewString(), Type: "foo", Queue: "default", CompletedAt: time.Now().Add(-2 * time.Hour).Unix(), } msg2 := &base.TaskMessage{ ID: uuid.NewString(), Type: "foo", Queue: "default", CompletedAt: time.Now().Add(-5 * time.Hour).Unix(), } msg3 := &base.TaskMessage{ ID: uuid.NewString(), Type: "foo", Queue: "custom", CompletedAt: time.Now().Add(-5 * time.Hour).Unix(), } expireAt1 := time.Now().Add(3 * time.Hour) expireAt2 := time.Now().Add(4 * time.Hour) expireAt3 := time.Now().Add(5 * time.Hour) tests := []struct { completed map[string][]base.Z qname string want []*base.TaskInfo }{ { completed: map[string][]base.Z{ "default": { {Message: msg1, Score: expireAt1.Unix()}, {Message: msg2, Score: expireAt2.Unix()}, }, "custom": { {Message: msg3, Score: expireAt3.Unix()}, }, }, qname: "default", want: []*base.TaskInfo{ {Message: msg1, NextProcessAt: time.Time{}, State: base.TaskStateCompleted, Result: nil}, {Message: msg2, NextProcessAt: time.Time{}, State: base.TaskStateCompleted, Result: nil}, }, }, { completed: map[string][]base.Z{ "default": { {Message: msg1, Score: expireAt1.Unix()}, {Message: msg2, Score: expireAt2.Unix()}, }, "custom": { {Message: msg3, Score: expireAt3.Unix()}, }, }, qname: "custom", want: []*base.TaskInfo{ {Message: msg3, NextProcessAt: time.Time{}, State: base.TaskStateCompleted, Result: nil}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllCompletedQueues(t, r.client, tc.completed) got, err := r.ListCompleted(tc.qname, Pagination{Size: 20, Page: 0}) op := fmt.Sprintf("r.ListCompleted(%q, Pagination{Size: 20, Page: 0})", tc.qname) if err != nil { t.Errorf("%s = %v, %v, want %v, nil", op, got, err, tc.want) continue } if diff := cmp.Diff(tc.want, got); diff != "" { t.Errorf("%s = %v, %v, want %v, nil; (-want, +got)\n%s", op, got, err, tc.want, diff) continue } } } func TestListCompletedPagination(t *testing.T) { r := setup(t) defer r.Close() var entries []base.Z for i := 0; i < 100; i++ { msg := h.NewTaskMessage(fmt.Sprintf("task %d", i), nil) entries = append(entries, base.Z{Message: msg, Score: int64(i)}) } h.SeedCompletedQueue(t, r.client, entries, "default") tests := []struct { desc string qname string page int size int wantSize int wantFirst string wantLast string }{ {"first page", "default", 0, 20, 20, "task 0", "task 19"}, {"second page", "default", 1, 20, 20, "task 20", "task 39"}, {"different page size", "default", 2, 30, 30, "task 60", "task 89"}, {"last page", "default", 3, 30, 10, "task 90", "task 99"}, {"out of range", "default", 4, 30, 0, "", ""}, } for _, tc := range tests { got, err := r.ListCompleted(tc.qname, Pagination{Size: tc.size, Page: tc.page}) op := fmt.Sprintf("r.ListCompleted(Pagination{Size: %d, Page: %d})", tc.size, tc.page) if err != nil { t.Errorf("%s; %s returned error %v", tc.desc, op, err) continue } if len(got) != tc.wantSize { t.Errorf("%s; %s returned list of size %d, want %d", tc.desc, op, len(got), tc.wantSize) continue } if tc.wantSize == 0 { continue } first := got[0].Message if first.Type != tc.wantFirst { t.Errorf("%s; %s returned a list with first message %q, want %q", tc.desc, op, first.Type, tc.wantFirst) } last := got[len(got)-1].Message if last.Type != tc.wantLast { t.Errorf("%s; %s returned a list with the last message %q, want %q", tc.desc, op, last.Type, tc.wantLast) } } } func TestListAggregating(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() m1 := h.NewTaskMessageBuilder().SetType("task1").SetQueue("default").SetGroup("group1").Build() m2 := h.NewTaskMessageBuilder().SetType("task2").SetQueue("default").SetGroup("group1").Build() m3 := h.NewTaskMessageBuilder().SetType("task3").SetQueue("default").SetGroup("group2").Build() m4 := h.NewTaskMessageBuilder().SetType("task4").SetQueue("custom").SetGroup("group3").Build() fxt := struct { tasks []*h.TaskSeedData allQueues []string allGroups map[string][]string groups map[string][]redis.Z }{ tasks: []*h.TaskSeedData{ {Msg: m1, State: base.TaskStateAggregating}, {Msg: m2, State: base.TaskStateAggregating}, {Msg: m3, State: base.TaskStateAggregating}, {Msg: m4, State: base.TaskStateAggregating}, }, allQueues: []string{"default", "custom"}, allGroups: map[string][]string{ base.AllGroups("default"): {"group1", "group2"}, base.AllGroups("custom"): {"group3"}, }, groups: map[string][]redis.Z{ base.GroupKey("default", "group1"): { {Member: m1.ID, Score: float64(now.Add(-30 * time.Second).Unix())}, {Member: m2.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, }, base.GroupKey("default", "group2"): { {Member: m3.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, }, base.GroupKey("custom", "group3"): { {Member: m4.ID, Score: float64(now.Add(-40 * time.Second).Unix())}, }, }, } tests := []struct { desc string qname string gname string want []*base.TaskInfo }{ { desc: "with group1 in default queue", qname: "default", gname: "group1", want: []*base.TaskInfo{ {Message: m1, State: base.TaskStateAggregating, NextProcessAt: time.Time{}, Result: nil}, {Message: m2, State: base.TaskStateAggregating, NextProcessAt: time.Time{}, Result: nil}, }, }, { desc: "with group3 in custom queue", qname: "custom", gname: "group3", want: []*base.TaskInfo{ {Message: m4, State: base.TaskStateAggregating, NextProcessAt: time.Time{}, Result: nil}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) h.SeedRedisSet(t, r.client, base.AllQueues, fxt.allQueues) h.SeedRedisSets(t, r.client, fxt.allGroups) h.SeedTasks(t, r.client, fxt.tasks) h.SeedRedisZSets(t, r.client, fxt.groups) t.Run(tc.desc, func(t *testing.T) { got, err := r.ListAggregating(tc.qname, tc.gname, Pagination{}) if err != nil { t.Fatalf("ListAggregating returned error: %v", err) } if diff := cmp.Diff(tc.want, got); diff != "" { t.Errorf("ListAggregating = %v, want %v; (-want,+got)\n%s", got, tc.want, diff) } }) } } func TestListAggregatingPagination(t *testing.T) { r := setup(t) defer r.Close() groupkey := base.GroupKey("default", "mygroup") fxt := struct { tasks []*h.TaskSeedData allQueues []string allGroups map[string][]string groups map[string][]redis.Z }{ tasks: []*h.TaskSeedData{}, // will be populated below allQueues: []string{"default"}, allGroups: map[string][]string{ base.AllGroups("default"): {"mygroup"}, }, groups: map[string][]redis.Z{ groupkey: {}, // will be populated below }, } now := time.Now() for i := 0; i < 100; i++ { msg := h.NewTaskMessageBuilder().SetType(fmt.Sprintf("task%d", i)).SetGroup("mygroup").Build() fxt.tasks = append(fxt.tasks, &h.TaskSeedData{ Msg: msg, State: base.TaskStateAggregating, }) fxt.groups[groupkey] = append(fxt.groups[groupkey], redis.Z{ Member: msg.ID, Score: float64(now.Add(-time.Duration(100-i) * time.Second).Unix()), }) } tests := []struct { desc string qname string gname string page int size int wantSize int wantFirst string wantLast string }{ { desc: "first page", qname: "default", gname: "mygroup", page: 0, size: 20, wantSize: 20, wantFirst: "task0", wantLast: "task19", }, { desc: "second page", qname: "default", gname: "mygroup", page: 1, size: 20, wantSize: 20, wantFirst: "task20", wantLast: "task39", }, { desc: "with different page size", qname: "default", gname: "mygroup", page: 2, size: 30, wantSize: 30, wantFirst: "task60", wantLast: "task89", }, { desc: "last page", qname: "default", gname: "mygroup", page: 3, size: 30, wantSize: 10, wantFirst: "task90", wantLast: "task99", }, { desc: "out of range", qname: "default", gname: "mygroup", page: 4, size: 30, wantSize: 0, wantFirst: "", wantLast: "", }, } for _, tc := range tests { h.FlushDB(t, r.client) h.SeedRedisSet(t, r.client, base.AllQueues, fxt.allQueues) h.SeedRedisSets(t, r.client, fxt.allGroups) h.SeedTasks(t, r.client, fxt.tasks) h.SeedRedisZSets(t, r.client, fxt.groups) t.Run(tc.desc, func(t *testing.T) { got, err := r.ListAggregating(tc.qname, tc.gname, Pagination{Page: tc.page, Size: tc.size}) if err != nil { t.Fatalf("ListAggregating returned error: %v", err) } if len(got) != tc.wantSize { t.Errorf("got %d results, want %d", len(got), tc.wantSize) } if len(got) == 0 { return } first := got[0].Message if first.Type != tc.wantFirst { t.Errorf("First message %q, want %q", first.Type, tc.wantFirst) } last := got[len(got)-1].Message if last.Type != tc.wantLast { t.Errorf("Last message %q, want %q", last.Type, tc.wantLast) } }) } } func TestListTasksError(t *testing.T) { r := setup(t) defer r.Close() tests := []struct { desc string qname string match func(err error) bool }{ { desc: "It returns QueueNotFoundError if queue doesn't exist", qname: "nonexistent", match: errors.IsQueueNotFound, }, } for _, tc := range tests { pgn := Pagination{Page: 0, Size: 20} if _, got := r.ListActive(tc.qname, pgn); !tc.match(got) { t.Errorf("%s: ListActive returned %v", tc.desc, got) } if _, got := r.ListPending(tc.qname, pgn); !tc.match(got) { t.Errorf("%s: ListPending returned %v", tc.desc, got) } if _, got := r.ListScheduled(tc.qname, pgn); !tc.match(got) { t.Errorf("%s: ListScheduled returned %v", tc.desc, got) } if _, got := r.ListRetry(tc.qname, pgn); !tc.match(got) { t.Errorf("%s: ListRetry returned %v", tc.desc, got) } if _, got := r.ListArchived(tc.qname, pgn); !tc.match(got) { t.Errorf("%s: ListArchived returned %v", tc.desc, got) } } } var ( timeCmpOpt = cmpopts.EquateApproxTime(2 * time.Second) // allow for 2 seconds margin in time.Time zScoreCmpOpt = h.EquateInt64Approx(2) // allow for 2 seconds margin in Z.Score ) func TestRunArchivedTask(t *testing.T) { r := setup(t) defer r.Close() t1 := h.NewTaskMessage("send_email", nil) t2 := h.NewTaskMessage("gen_thumbnail", nil) t3 := h.NewTaskMessageWithQueue("send_notification", nil, "critical") s1 := time.Now().Add(-5 * time.Minute).Unix() s2 := time.Now().Add(-time.Hour).Unix() tests := []struct { archived map[string][]base.Z qname string id string wantArchived map[string][]*base.TaskMessage wantPending map[string][]*base.TaskMessage }{ { archived: map[string][]base.Z{ "default": { {Message: t1, Score: s1}, {Message: t2, Score: s2}, }, }, qname: "default", id: t2.ID, wantArchived: map[string][]*base.TaskMessage{ "default": {t1}, }, wantPending: map[string][]*base.TaskMessage{ "default": {t2}, }, }, { archived: map[string][]base.Z{ "default": { {Message: t1, Score: s1}, {Message: t2, Score: s2}, }, "critical": { {Message: t3, Score: s1}, }, }, qname: "critical", id: t3.ID, wantArchived: map[string][]*base.TaskMessage{ "default": {t1, t2}, "critical": {}, }, wantPending: map[string][]*base.TaskMessage{ "default": {}, "critical": {t3}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllArchivedQueues(t, r.client, tc.archived) if got := r.RunTask(tc.qname, tc.id); got != nil { t.Errorf("r.RunTask(%q, %s) returned error: %v", tc.qname, tc.id, got) continue } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r.client, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.PendingKey(qname), diff) } } for qname, want := range tc.wantArchived { gotArchived := h.GetArchivedMessages(t, r.client, qname) if diff := cmp.Diff(want, gotArchived, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q, (-want, +got)\n%s", base.ArchivedKey(qname), diff) } } } } func TestRunRetryTask(t *testing.T) { r := setup(t) defer r.Close() t1 := h.NewTaskMessage("send_email", nil) t2 := h.NewTaskMessage("gen_thumbnail", nil) t3 := h.NewTaskMessageWithQueue("send_notification", nil, "low") s1 := time.Now().Add(-5 * time.Minute).Unix() s2 := time.Now().Add(-time.Hour).Unix() tests := []struct { retry map[string][]base.Z qname string id string wantRetry map[string][]*base.TaskMessage wantPending map[string][]*base.TaskMessage }{ { retry: map[string][]base.Z{ "default": { {Message: t1, Score: s1}, {Message: t2, Score: s2}, }, }, qname: "default", id: t2.ID, wantRetry: map[string][]*base.TaskMessage{ "default": {t1}, }, wantPending: map[string][]*base.TaskMessage{ "default": {t2}, }, }, { retry: map[string][]base.Z{ "default": { {Message: t1, Score: s1}, {Message: t2, Score: s2}, }, "low": { {Message: t3, Score: s2}, }, }, qname: "low", id: t3.ID, wantRetry: map[string][]*base.TaskMessage{ "default": {t1, t2}, "low": {}, }, wantPending: map[string][]*base.TaskMessage{ "default": {}, "low": {t3}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllRetryQueues(t, r.client, tc.retry) // initialize retry queue if got := r.RunTask(tc.qname, tc.id); got != nil { t.Errorf("r.RunTask(%q, %s) returned error: %v", tc.qname, tc.id, got) continue } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r.client, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.PendingKey(qname), diff) } } for qname, want := range tc.wantRetry { gotRetry := h.GetRetryMessages(t, r.client, qname) if diff := cmp.Diff(want, gotRetry, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q, (-want, +got)\n%s", base.RetryKey(qname), diff) } } } } func TestRunAggregatingTask(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() r.SetClock(timeutil.NewSimulatedClock(now)) m1 := h.NewTaskMessageBuilder().SetQueue("default").SetType("task1").SetGroup("group1").Build() m2 := h.NewTaskMessageBuilder().SetQueue("default").SetType("task2").SetGroup("group1").Build() m3 := h.NewTaskMessageBuilder().SetQueue("custom").SetType("task3").SetGroup("group1").Build() fxt := struct { tasks []*h.TaskSeedData allQueues []string allGroups map[string][]string groups map[string][]redis.Z }{ tasks: []*h.TaskSeedData{ {Msg: m1, State: base.TaskStateAggregating}, {Msg: m2, State: base.TaskStateAggregating}, {Msg: m3, State: base.TaskStateAggregating}, }, allQueues: []string{"default", "custom"}, allGroups: map[string][]string{ base.AllGroups("default"): {"group1"}, base.AllGroups("custom"): {"group1"}, }, groups: map[string][]redis.Z{ base.GroupKey("default", "group1"): { {Member: m1.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, {Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())}, }, base.GroupKey("custom", "group1"): { {Member: m3.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, }, }, } tests := []struct { desc string qname string id string wantPending map[string][]string wantAllGroups map[string][]string wantGroups map[string][]redis.Z }{ { desc: "schedules task from a group with multiple tasks", qname: "default", id: m1.ID, wantPending: map[string][]string{ base.PendingKey("default"): {m1.ID}, }, wantAllGroups: map[string][]string{ base.AllGroups("default"): {"group1"}, base.AllGroups("custom"): {"group1"}, }, wantGroups: map[string][]redis.Z{ base.GroupKey("default", "group1"): { {Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())}, }, base.GroupKey("custom", "group1"): { {Member: m3.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, }, }, }, { desc: "schedules task from a group with a single task", qname: "custom", id: m3.ID, wantPending: map[string][]string{ base.PendingKey("custom"): {m3.ID}, }, wantAllGroups: map[string][]string{ base.AllGroups("default"): {"group1"}, base.AllGroups("custom"): {}, }, wantGroups: map[string][]redis.Z{ base.GroupKey("default", "group1"): { {Member: m1.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, {Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())}, }, base.GroupKey("custom", "group1"): {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) h.SeedTasks(t, r.client, fxt.tasks) h.SeedRedisSet(t, r.client, base.AllQueues, fxt.allQueues) h.SeedRedisSets(t, r.client, fxt.allGroups) h.SeedRedisZSets(t, r.client, fxt.groups) t.Run(tc.desc, func(t *testing.T) { err := r.RunTask(tc.qname, tc.id) if err != nil { t.Fatalf("RunTask returned error: %v", err) } h.AssertRedisLists(t, r.client, tc.wantPending) h.AssertRedisZSets(t, r.client, tc.wantGroups) h.AssertRedisSets(t, r.client, tc.wantAllGroups) }) } } func TestRunScheduledTask(t *testing.T) { r := setup(t) defer r.Close() t1 := h.NewTaskMessage("send_email", nil) t2 := h.NewTaskMessage("gen_thumbnail", nil) t3 := h.NewTaskMessageWithQueue("send_notification", nil, "notifications") s1 := time.Now().Add(-5 * time.Minute).Unix() s2 := time.Now().Add(-time.Hour).Unix() tests := []struct { scheduled map[string][]base.Z qname string id string wantScheduled map[string][]*base.TaskMessage wantPending map[string][]*base.TaskMessage }{ { scheduled: map[string][]base.Z{ "default": { {Message: t1, Score: s1}, {Message: t2, Score: s2}, }, }, qname: "default", id: t2.ID, wantScheduled: map[string][]*base.TaskMessage{ "default": {t1}, }, wantPending: map[string][]*base.TaskMessage{ "default": {t2}, }, }, { scheduled: map[string][]base.Z{ "default": { {Message: t1, Score: s1}, {Message: t2, Score: s2}, }, "notifications": { {Message: t3, Score: s1}, }, }, qname: "notifications", id: t3.ID, wantScheduled: map[string][]*base.TaskMessage{ "default": {t1, t2}, "notifications": {}, }, wantPending: map[string][]*base.TaskMessage{ "default": {}, "notifications": {t3}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllScheduledQueues(t, r.client, tc.scheduled) if got := r.RunTask(tc.qname, tc.id); got != nil { t.Errorf("r.RunTask(%q, %s) returned error: %v", tc.qname, tc.id, got) continue } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r.client, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.PendingKey(qname), diff) } } for qname, want := range tc.wantScheduled { gotScheduled := h.GetScheduledMessages(t, r.client, qname) if diff := cmp.Diff(want, gotScheduled, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q, (-want, +got)\n%s", base.ScheduledKey(qname), diff) } } } } func TestRunTaskError(t *testing.T) { r := setup(t) defer r.Close() t1 := h.NewTaskMessage("send_email", nil) s1 := time.Now().Add(-5 * time.Minute).Unix() tests := []struct { desc string active map[string][]*base.TaskMessage pending map[string][]*base.TaskMessage scheduled map[string][]base.Z qname string id string match func(err error) bool wantActive map[string][]*base.TaskMessage wantPending map[string][]*base.TaskMessage wantScheduled map[string][]*base.TaskMessage }{ { desc: "It should return QueueNotFoundError if the queue doesn't exist", active: map[string][]*base.TaskMessage{ "default": {}, }, pending: map[string][]*base.TaskMessage{ "default": {}, }, scheduled: map[string][]base.Z{ "default": { {Message: t1, Score: s1}, }, }, qname: "nonexistent", id: t1.ID, match: errors.IsQueueNotFound, wantActive: map[string][]*base.TaskMessage{ "default": {}, }, wantPending: map[string][]*base.TaskMessage{ "default": {}, }, wantScheduled: map[string][]*base.TaskMessage{ "default": {t1}, }, }, { desc: "It should return TaskNotFound if the task is not found in the queue", active: map[string][]*base.TaskMessage{ "default": {}, }, pending: map[string][]*base.TaskMessage{ "default": {}, }, scheduled: map[string][]base.Z{ "default": { {Message: t1, Score: s1}, }, }, qname: "default", id: uuid.NewString(), match: errors.IsTaskNotFound, wantActive: map[string][]*base.TaskMessage{ "default": {}, }, wantPending: map[string][]*base.TaskMessage{ "default": {}, }, wantScheduled: map[string][]*base.TaskMessage{ "default": {t1}, }, }, { desc: "It should return FailedPrecondition error if task is already active", active: map[string][]*base.TaskMessage{ "default": {t1}, }, pending: map[string][]*base.TaskMessage{ "default": {}, }, scheduled: map[string][]base.Z{ "default": {}, }, qname: "default", id: t1.ID, match: func(err error) bool { return errors.CanonicalCode(err) == errors.FailedPrecondition }, wantActive: map[string][]*base.TaskMessage{ "default": {t1}, }, wantPending: map[string][]*base.TaskMessage{ "default": {}, }, wantScheduled: map[string][]*base.TaskMessage{ "default": {}, }, }, { desc: "It should return FailedPrecondition error if task is already pending", active: map[string][]*base.TaskMessage{ "default": {}, }, pending: map[string][]*base.TaskMessage{ "default": {t1}, }, scheduled: map[string][]base.Z{ "default": {}, }, qname: "default", id: t1.ID, match: func(err error) bool { return errors.CanonicalCode(err) == errors.FailedPrecondition }, wantActive: map[string][]*base.TaskMessage{ "default": {}, }, wantPending: map[string][]*base.TaskMessage{ "default": {t1}, }, wantScheduled: map[string][]*base.TaskMessage{ "default": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllActiveQueues(t, r.client, tc.active) h.SeedAllPendingQueues(t, r.client, tc.pending) h.SeedAllScheduledQueues(t, r.client, tc.scheduled) got := r.RunTask(tc.qname, tc.id) if !tc.match(got) { t.Errorf("%s: unexpected return value %v", tc.desc, got) continue } for qname, want := range tc.wantActive { gotActive := h.GetActiveMessages(t, r.client, qname) if diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q: (-want, +got)\n%s", base.ActiveKey(qname), diff) } } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r.client, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.PendingKey(qname), diff) } } for qname, want := range tc.wantScheduled { gotScheduled := h.GetScheduledMessages(t, r.client, qname) if diff := cmp.Diff(want, gotScheduled, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q, (-want, +got)\n%s", base.ScheduledKey(qname), diff) } } } } func TestRunAllScheduledTasks(t *testing.T) { r := setup(t) defer r.Close() t1 := h.NewTaskMessage("send_email", nil) t2 := h.NewTaskMessage("gen_thumbnail", nil) t3 := h.NewTaskMessage("reindex", nil) t4 := h.NewTaskMessageWithQueue("important_notification", nil, "custom") t5 := h.NewTaskMessageWithQueue("minor_notification", nil, "custom") tests := []struct { desc string scheduled map[string][]base.Z qname string want int64 wantPending map[string][]*base.TaskMessage wantScheduled map[string][]*base.TaskMessage }{ { desc: "with tasks in scheduled queue", scheduled: map[string][]base.Z{ "default": { {Message: t1, Score: time.Now().Add(time.Hour).Unix()}, {Message: t2, Score: time.Now().Add(time.Hour).Unix()}, {Message: t3, Score: time.Now().Add(time.Hour).Unix()}, }, }, qname: "default", want: 3, wantPending: map[string][]*base.TaskMessage{ "default": {t1, t2, t3}, }, wantScheduled: map[string][]*base.TaskMessage{ "default": {}, }, }, { desc: "with empty scheduled queue", scheduled: map[string][]base.Z{ "default": {}, }, qname: "default", want: 0, wantPending: map[string][]*base.TaskMessage{ "default": {}, }, wantScheduled: map[string][]*base.TaskMessage{ "default": {}, }, }, { desc: "with custom queues", scheduled: map[string][]base.Z{ "default": { {Message: t1, Score: time.Now().Add(time.Hour).Unix()}, {Message: t2, Score: time.Now().Add(time.Hour).Unix()}, {Message: t3, Score: time.Now().Add(time.Hour).Unix()}, }, "custom": { {Message: t4, Score: time.Now().Add(time.Hour).Unix()}, {Message: t5, Score: time.Now().Add(time.Hour).Unix()}, }, }, qname: "custom", want: 2, wantPending: map[string][]*base.TaskMessage{ "default": {}, "custom": {t4, t5}, }, wantScheduled: map[string][]*base.TaskMessage{ "default": {t1, t2, t3}, "custom": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllScheduledQueues(t, r.client, tc.scheduled) got, err := r.RunAllScheduledTasks(tc.qname) if err != nil { t.Errorf("%s; r.RunAllScheduledTasks(%q) = %v, %v; want %v, nil", tc.desc, tc.qname, got, err, tc.want) continue } if got != tc.want { t.Errorf("%s; r.RunAllScheduledTasks(%q) = %v, %v; want %v, nil", tc.desc, tc.qname, got, err, tc.want) } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r.client, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("%s; mismatch found in %q; (-want, +got)\n%s", tc.desc, base.PendingKey(qname), diff) } } for qname, want := range tc.wantScheduled { gotScheduled := h.GetScheduledMessages(t, r.client, qname) if diff := cmp.Diff(want, gotScheduled, h.SortMsgOpt); diff != "" { t.Errorf("%s; mismatch found in %q; (-want, +got)\n%s", tc.desc, base.ScheduledKey(qname), diff) } } } } func TestRunAllRetryTasks(t *testing.T) { r := setup(t) defer r.Close() t1 := h.NewTaskMessage("send_email", nil) t2 := h.NewTaskMessage("gen_thumbnail", nil) t3 := h.NewTaskMessage("reindex", nil) t4 := h.NewTaskMessageWithQueue("important_notification", nil, "custom") t5 := h.NewTaskMessageWithQueue("minor_notification", nil, "custom") tests := []struct { desc string retry map[string][]base.Z qname string want int64 wantPending map[string][]*base.TaskMessage wantRetry map[string][]*base.TaskMessage }{ { desc: "with tasks in retry queue", retry: map[string][]base.Z{ "default": { {Message: t1, Score: time.Now().Add(time.Hour).Unix()}, {Message: t2, Score: time.Now().Add(time.Hour).Unix()}, {Message: t3, Score: time.Now().Add(time.Hour).Unix()}, }, }, qname: "default", want: 3, wantPending: map[string][]*base.TaskMessage{ "default": {t1, t2, t3}, }, wantRetry: map[string][]*base.TaskMessage{ "default": {}, }, }, { desc: "with empty retry queue", retry: map[string][]base.Z{ "default": {}, }, qname: "default", want: 0, wantPending: map[string][]*base.TaskMessage{ "default": {}, }, wantRetry: map[string][]*base.TaskMessage{ "default": {}, }, }, { desc: "with custom queues", retry: map[string][]base.Z{ "default": { {Message: t1, Score: time.Now().Add(time.Hour).Unix()}, {Message: t2, Score: time.Now().Add(time.Hour).Unix()}, {Message: t3, Score: time.Now().Add(time.Hour).Unix()}, }, "custom": { {Message: t4, Score: time.Now().Add(time.Hour).Unix()}, {Message: t5, Score: time.Now().Add(time.Hour).Unix()}, }, }, qname: "custom", want: 2, wantPending: map[string][]*base.TaskMessage{ "default": {}, "custom": {t4, t5}, }, wantRetry: map[string][]*base.TaskMessage{ "default": {t1, t2, t3}, "custom": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllRetryQueues(t, r.client, tc.retry) got, err := r.RunAllRetryTasks(tc.qname) if err != nil { t.Errorf("%s; r.RunAllRetryTasks(%q) = %v, %v; want %v, nil", tc.desc, tc.qname, got, err, tc.want) continue } if got != tc.want { t.Errorf("%s; r.RunAllRetryTasks(%q) = %v, %v; want %v, nil", tc.desc, tc.qname, got, err, tc.want) } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r.client, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("%s; mismatch found in %q; (-want, +got)\n%s", tc.desc, base.PendingKey(qname), diff) } } for qname, want := range tc.wantRetry { gotRetry := h.GetRetryMessages(t, r.client, qname) if diff := cmp.Diff(want, gotRetry, h.SortMsgOpt); diff != "" { t.Errorf("%s; mismatch found in %q; (-want, +got)\n%s", tc.desc, base.RetryKey(qname), diff) } } } } func TestRunAllArchivedTasks(t *testing.T) { r := setup(t) defer r.Close() t1 := h.NewTaskMessage("send_email", nil) t2 := h.NewTaskMessage("gen_thumbnail", nil) t3 := h.NewTaskMessage("reindex", nil) t4 := h.NewTaskMessageWithQueue("important_notification", nil, "custom") t5 := h.NewTaskMessageWithQueue("minor_notification", nil, "custom") tests := []struct { desc string archived map[string][]base.Z qname string want int64 wantPending map[string][]*base.TaskMessage wantArchived map[string][]*base.TaskMessage }{ { desc: "with tasks in archived queue", archived: map[string][]base.Z{ "default": { {Message: t1, Score: time.Now().Add(-time.Minute).Unix()}, {Message: t2, Score: time.Now().Add(-time.Minute).Unix()}, {Message: t3, Score: time.Now().Add(-time.Minute).Unix()}, }, }, qname: "default", want: 3, wantPending: map[string][]*base.TaskMessage{ "default": {t1, t2, t3}, }, wantArchived: map[string][]*base.TaskMessage{ "default": {}, }, }, { desc: "with empty archived queue", archived: map[string][]base.Z{ "default": {}, }, qname: "default", want: 0, wantPending: map[string][]*base.TaskMessage{ "default": {}, }, wantArchived: map[string][]*base.TaskMessage{ "default": {}, }, }, { desc: "with custom queues", archived: map[string][]base.Z{ "default": { {Message: t1, Score: time.Now().Add(-time.Minute).Unix()}, {Message: t2, Score: time.Now().Add(-time.Minute).Unix()}, {Message: t3, Score: time.Now().Add(-time.Minute).Unix()}, }, "custom": { {Message: t4, Score: time.Now().Add(-time.Minute).Unix()}, {Message: t5, Score: time.Now().Add(-time.Minute).Unix()}, }, }, qname: "custom", want: 2, wantPending: map[string][]*base.TaskMessage{ "default": {}, "custom": {t4, t5}, }, wantArchived: map[string][]*base.TaskMessage{ "default": {t1, t2, t3}, "custom": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllArchivedQueues(t, r.client, tc.archived) got, err := r.RunAllArchivedTasks(tc.qname) if err != nil { t.Errorf("%s; r.RunAllArchivedTasks(%q) = %v, %v; want %v, nil", tc.desc, tc.qname, got, err, tc.want) continue } if got != tc.want { t.Errorf("%s; r.RunAllArchivedTasks(%q) = %v, %v; want %v, nil", tc.desc, tc.qname, got, err, tc.want) } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r.client, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("%s; mismatch found in %q; (-want, +got)\n%s", tc.desc, base.PendingKey(qname), diff) } } for qname, want := range tc.wantArchived { gotArchived := h.GetArchivedMessages(t, r.client, qname) if diff := cmp.Diff(want, gotArchived, h.SortMsgOpt); diff != "" { t.Errorf("%s; mismatch found in %q; (-want, +got)\n%s", tc.desc, base.ArchivedKey(qname), diff) } } } } func TestRunAllTasksError(t *testing.T) { r := setup(t) defer r.Close() tests := []struct { desc string qname string match func(err error) bool }{ { desc: "It returns QueueNotFoundError if queue doesn't exist", qname: "nonexistent", match: errors.IsQueueNotFound, }, } for _, tc := range tests { if _, got := r.RunAllScheduledTasks(tc.qname); !tc.match(got) { t.Errorf("%s: RunAllScheduledTasks returned %v", tc.desc, got) } if _, got := r.RunAllRetryTasks(tc.qname); !tc.match(got) { t.Errorf("%s: RunAllRetryTasks returned %v", tc.desc, got) } if _, got := r.RunAllArchivedTasks(tc.qname); !tc.match(got) { t.Errorf("%s: RunAllArchivedTasks returned %v", tc.desc, got) } if _, got := r.RunAllAggregatingTasks(tc.qname, "mygroup"); !tc.match(got) { t.Errorf("%s: RunAllAggregatingTasks returned %v", tc.desc, got) } } } func TestRunAllAggregatingTasks(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() r.SetClock(timeutil.NewSimulatedClock(now)) m1 := h.NewTaskMessageBuilder().SetQueue("default").SetType("task1").SetGroup("group1").Build() m2 := h.NewTaskMessageBuilder().SetQueue("default").SetType("task2").SetGroup("group1").Build() m3 := h.NewTaskMessageBuilder().SetQueue("custom").SetType("task3").SetGroup("group2").Build() fxt := struct { tasks []*h.TaskSeedData allQueues []string allGroups map[string][]string groups map[string][]redis.Z }{ tasks: []*h.TaskSeedData{ {Msg: m1, State: base.TaskStateAggregating}, {Msg: m2, State: base.TaskStateAggregating}, {Msg: m3, State: base.TaskStateAggregating}, }, allQueues: []string{"default", "custom"}, allGroups: map[string][]string{ base.AllGroups("default"): {"group1"}, base.AllGroups("custom"): {"group2"}, }, groups: map[string][]redis.Z{ base.GroupKey("default", "group1"): { {Member: m1.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, {Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())}, }, base.GroupKey("custom", "group2"): { {Member: m3.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, }, }, } tests := []struct { desc string qname string gname string want int64 wantPending map[string][]string wantGroups map[string][]redis.Z wantAllGroups map[string][]string }{ { desc: "schedules tasks in a group with multiple tasks", qname: "default", gname: "group1", want: 2, wantPending: map[string][]string{ base.PendingKey("default"): {m1.ID, m2.ID}, }, wantGroups: map[string][]redis.Z{ base.GroupKey("default", "group1"): {}, base.GroupKey("custom", "group2"): { {Member: m3.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, }, }, wantAllGroups: map[string][]string{ base.AllGroups("default"): {}, base.AllGroups("custom"): {"group2"}, }, }, { desc: "schedules tasks in a group with a single task", qname: "custom", gname: "group2", want: 1, wantPending: map[string][]string{ base.PendingKey("custom"): {m3.ID}, }, wantGroups: map[string][]redis.Z{ base.GroupKey("default", "group1"): { {Member: m1.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, {Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())}, }, base.GroupKey("custom", "group2"): {}, }, wantAllGroups: map[string][]string{ base.AllGroups("default"): {"group1"}, base.AllGroups("custom"): {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) h.SeedTasks(t, r.client, fxt.tasks) h.SeedRedisSet(t, r.client, base.AllQueues, fxt.allQueues) h.SeedRedisSets(t, r.client, fxt.allGroups) h.SeedRedisZSets(t, r.client, fxt.groups) t.Run(tc.desc, func(t *testing.T) { got, err := r.RunAllAggregatingTasks(tc.qname, tc.gname) if err != nil { t.Fatalf("RunAllAggregatingTasks returned error: %v", err) } if got != tc.want { t.Errorf("RunAllAggregatingTasks = %d, want %d", got, tc.want) } h.AssertRedisLists(t, r.client, tc.wantPending) h.AssertRedisZSets(t, r.client, tc.wantGroups) h.AssertRedisSets(t, r.client, tc.wantAllGroups) }) } } func TestArchiveRetryTask(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") m4 := h.NewTaskMessageWithQueue("task4", nil, "custom") t1 := time.Now().Add(1 * time.Minute) t2 := time.Now().Add(1 * time.Hour) t3 := time.Now().Add(2 * time.Hour) t4 := time.Now().Add(3 * time.Hour) tests := []struct { retry map[string][]base.Z archived map[string][]base.Z qname string id string wantRetry map[string][]base.Z wantArchived map[string][]base.Z }{ { retry: map[string][]base.Z{ "default": { {Message: m1, Score: t1.Unix()}, {Message: m2, Score: t2.Unix()}, }, }, archived: map[string][]base.Z{ "default": {}, }, qname: "default", id: m1.ID, wantRetry: map[string][]base.Z{ "default": {{Message: m2, Score: t2.Unix()}}, }, wantArchived: map[string][]base.Z{ "default": {{Message: m1, Score: time.Now().Unix()}}, }, }, { retry: map[string][]base.Z{ "default": { {Message: m1, Score: t1.Unix()}, {Message: m2, Score: t2.Unix()}, }, "custom": { {Message: m3, Score: t3.Unix()}, {Message: m4, Score: t4.Unix()}, }, }, archived: map[string][]base.Z{ "default": {}, "custom": {}, }, qname: "custom", id: m3.ID, wantRetry: map[string][]base.Z{ "default": { {Message: m1, Score: t1.Unix()}, {Message: m2, Score: t2.Unix()}, }, "custom": { {Message: m4, Score: t4.Unix()}, }, }, wantArchived: map[string][]base.Z{ "default": {}, "custom": {{Message: m3, Score: time.Now().Unix()}}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) h.SeedAllRetryQueues(t, r.client, tc.retry) h.SeedAllArchivedQueues(t, r.client, tc.archived) if got := r.ArchiveTask(tc.qname, tc.id); got != nil { t.Errorf("(*RDB).ArchiveTask(%q, %v) returned error: %v", tc.qname, tc.id, got) continue } for qname, want := range tc.wantRetry { gotRetry := h.GetRetryEntries(t, r.client, qname) if diff := cmp.Diff(want, gotRetry, h.SortZSetEntryOpt, zScoreCmpOpt); diff != "" { t.Errorf("mismatch found in %q; (-want,+got)\n%s", base.RetryKey(qname), diff) } } for qname, want := range tc.wantArchived { gotArchived := h.GetArchivedEntries(t, r.client, qname) if diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt, zScoreCmpOpt); diff != "" { t.Errorf("mismatch found in %q; (-want,+got)\n%s", base.ArchivedKey(qname), diff) } } } } func TestArchiveScheduledTask(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") m4 := h.NewTaskMessageWithQueue("task4", nil, "custom") t1 := time.Now().Add(1 * time.Minute) t2 := time.Now().Add(1 * time.Hour) t3 := time.Now().Add(2 * time.Hour) t4 := time.Now().Add(3 * time.Hour) tests := []struct { scheduled map[string][]base.Z archived map[string][]base.Z qname string id string wantScheduled map[string][]base.Z wantArchived map[string][]base.Z }{ { scheduled: map[string][]base.Z{ "default": { {Message: m1, Score: t1.Unix()}, {Message: m2, Score: t2.Unix()}, }, }, archived: map[string][]base.Z{ "default": {}, }, qname: "default", id: m1.ID, wantScheduled: map[string][]base.Z{ "default": {{Message: m2, Score: t2.Unix()}}, }, wantArchived: map[string][]base.Z{ "default": {{Message: m1, Score: time.Now().Unix()}}, }, }, { scheduled: map[string][]base.Z{ "default": { {Message: m1, Score: t1.Unix()}, {Message: m2, Score: t2.Unix()}, }, "custom": { {Message: m3, Score: t3.Unix()}, {Message: m4, Score: t4.Unix()}, }, }, archived: map[string][]base.Z{ "default": {}, "custom": {}, }, qname: "custom", id: m3.ID, wantScheduled: map[string][]base.Z{ "default": { {Message: m1, Score: t1.Unix()}, {Message: m2, Score: t2.Unix()}, }, "custom": { {Message: m4, Score: t4.Unix()}, }, }, wantArchived: map[string][]base.Z{ "default": {}, "custom": {{Message: m3, Score: time.Now().Unix()}}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) h.SeedAllScheduledQueues(t, r.client, tc.scheduled) h.SeedAllArchivedQueues(t, r.client, tc.archived) if got := r.ArchiveTask(tc.qname, tc.id); got != nil { t.Errorf("(*RDB).ArchiveTask(%q, %v) returned error: %v", tc.qname, tc.id, got) continue } for qname, want := range tc.wantScheduled { gotScheduled := h.GetScheduledEntries(t, r.client, qname) if diff := cmp.Diff(want, gotScheduled, h.SortZSetEntryOpt, zScoreCmpOpt); diff != "" { t.Errorf("mismatch found in %q; (-want,+got)\n%s", base.ScheduledKey(qname), diff) } } for qname, want := range tc.wantArchived { gotArchived := h.GetArchivedEntries(t, r.client, qname) if diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt, zScoreCmpOpt); diff != "" { t.Errorf("mismatch found in %q; (-want,+got)\n%s", base.ArchivedKey(qname), diff) } } } } func TestArchiveAggregatingTask(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() r.SetClock(timeutil.NewSimulatedClock(now)) m1 := h.NewTaskMessageBuilder().SetQueue("default").SetType("task1").SetGroup("group1").Build() m2 := h.NewTaskMessageBuilder().SetQueue("default").SetType("task2").SetGroup("group1").Build() m3 := h.NewTaskMessageBuilder().SetQueue("custom").SetType("task3").SetGroup("group1").Build() fxt := struct { tasks []*h.TaskSeedData allQueues []string allGroups map[string][]string groups map[string][]redis.Z }{ tasks: []*h.TaskSeedData{ {Msg: m1, State: base.TaskStateAggregating}, {Msg: m2, State: base.TaskStateAggregating}, {Msg: m3, State: base.TaskStateAggregating}, }, allQueues: []string{"default", "custom"}, allGroups: map[string][]string{ base.AllGroups("default"): {"group1"}, base.AllGroups("custom"): {"group1"}, }, groups: map[string][]redis.Z{ base.GroupKey("default", "group1"): { {Member: m1.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, {Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())}, }, base.GroupKey("custom", "group1"): { {Member: m3.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, }, }, } tests := []struct { desc string qname string id string wantArchived map[string][]redis.Z wantAllGroups map[string][]string wantGroups map[string][]redis.Z }{ { desc: "archive task from a group with multiple tasks", qname: "default", id: m1.ID, wantArchived: map[string][]redis.Z{ base.ArchivedKey("default"): { {Member: m1.ID, Score: float64(now.Unix())}, }, }, wantAllGroups: map[string][]string{ base.AllGroups("default"): {"group1"}, base.AllGroups("custom"): {"group1"}, }, wantGroups: map[string][]redis.Z{ base.GroupKey("default", "group1"): { {Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())}, }, base.GroupKey("custom", "group1"): { {Member: m3.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, }, }, }, { desc: "archive task from a group with a single task", qname: "custom", id: m3.ID, wantArchived: map[string][]redis.Z{ base.ArchivedKey("custom"): { {Member: m3.ID, Score: float64(now.Unix())}, }, }, wantAllGroups: map[string][]string{ base.AllGroups("default"): {"group1"}, base.AllGroups("custom"): {}, }, wantGroups: map[string][]redis.Z{ base.GroupKey("default", "group1"): { {Member: m1.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, {Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())}, }, base.GroupKey("custom", "group1"): {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) h.SeedTasks(t, r.client, fxt.tasks) h.SeedRedisSet(t, r.client, base.AllQueues, fxt.allQueues) h.SeedRedisSets(t, r.client, fxt.allGroups) h.SeedRedisZSets(t, r.client, fxt.groups) t.Run(tc.desc, func(t *testing.T) { err := r.ArchiveTask(tc.qname, tc.id) if err != nil { t.Fatalf("ArchiveTask returned error: %v", err) } h.AssertRedisZSets(t, r.client, tc.wantArchived) h.AssertRedisZSets(t, r.client, tc.wantGroups) h.AssertRedisSets(t, r.client, tc.wantAllGroups) }) } } func TestArchivePendingTask(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") m4 := h.NewTaskMessageWithQueue("task4", nil, "custom") tests := []struct { pending map[string][]*base.TaskMessage archived map[string][]base.Z qname string id string wantPending map[string][]*base.TaskMessage wantArchived map[string][]base.Z }{ { pending: map[string][]*base.TaskMessage{ "default": {m1, m2}, }, archived: map[string][]base.Z{ "default": {}, }, qname: "default", id: m1.ID, wantPending: map[string][]*base.TaskMessage{ "default": {m2}, }, wantArchived: map[string][]base.Z{ "default": {{Message: m1, Score: time.Now().Unix()}}, }, }, { pending: map[string][]*base.TaskMessage{ "default": {m1, m2}, "custom": {m3, m4}, }, archived: map[string][]base.Z{ "default": {}, "custom": {}, }, qname: "custom", id: m3.ID, wantPending: map[string][]*base.TaskMessage{ "default": {m1, m2}, "custom": {m4}, }, wantArchived: map[string][]base.Z{ "default": {}, "custom": {{Message: m3, Score: time.Now().Unix()}}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) h.SeedAllPendingQueues(t, r.client, tc.pending) h.SeedAllArchivedQueues(t, r.client, tc.archived) if got := r.ArchiveTask(tc.qname, tc.id); got != nil { t.Errorf("(*RDB).ArchiveTask(%q, %v) returned error: %v", tc.qname, tc.id, got) continue } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r.client, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want,+got)\n%s", base.PendingKey(qname), diff) } } for qname, want := range tc.wantArchived { gotArchived := h.GetArchivedEntries(t, r.client, qname) if diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt, zScoreCmpOpt); diff != "" { t.Errorf("mismatch found in %q; (-want,+got)\n%s", base.ArchivedKey(qname), diff) } } } } func TestArchiveTaskError(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) t1 := time.Now().Add(1 * time.Minute) t2 := time.Now().Add(1 * time.Hour) tests := []struct { desc string active map[string][]*base.TaskMessage scheduled map[string][]base.Z archived map[string][]base.Z qname string id string match func(err error) bool wantActive map[string][]*base.TaskMessage wantScheduled map[string][]base.Z wantArchived map[string][]base.Z }{ { desc: "It should return QueueNotFoundError if provided queue name doesn't exist", active: map[string][]*base.TaskMessage{ "default": {}, }, scheduled: map[string][]base.Z{ "default": {{Message: m1, Score: t1.Unix()}}, }, archived: map[string][]base.Z{ "default": {{Message: m2, Score: t2.Unix()}}, }, qname: "nonexistent", id: m2.ID, match: errors.IsQueueNotFound, wantActive: map[string][]*base.TaskMessage{ "default": {}, }, wantScheduled: map[string][]base.Z{ "default": {{Message: m1, Score: t1.Unix()}}, }, wantArchived: map[string][]base.Z{ "default": {{Message: m2, Score: t2.Unix()}}, }, }, { desc: "It should return TaskNotFoundError if provided task ID doesn't exist in the queue", active: map[string][]*base.TaskMessage{ "default": {}, }, scheduled: map[string][]base.Z{ "default": {{Message: m1, Score: t1.Unix()}}, }, archived: map[string][]base.Z{ "default": {{Message: m2, Score: t2.Unix()}}, }, qname: "default", id: uuid.NewString(), match: errors.IsTaskNotFound, wantActive: map[string][]*base.TaskMessage{ "default": {}, }, wantScheduled: map[string][]base.Z{ "default": {{Message: m1, Score: t1.Unix()}}, }, wantArchived: map[string][]base.Z{ "default": {{Message: m2, Score: t2.Unix()}}, }, }, { desc: "It should return TaskAlreadyArchivedError if task is already in archived state", active: map[string][]*base.TaskMessage{ "default": {}, }, scheduled: map[string][]base.Z{ "default": {{Message: m1, Score: t1.Unix()}}, }, archived: map[string][]base.Z{ "default": {{Message: m2, Score: t2.Unix()}}, }, qname: "default", id: m2.ID, match: errors.IsTaskAlreadyArchived, wantActive: map[string][]*base.TaskMessage{ "default": {}, }, wantScheduled: map[string][]base.Z{ "default": {{Message: m1, Score: t1.Unix()}}, }, wantArchived: map[string][]base.Z{ "default": {{Message: m2, Score: t2.Unix()}}, }, }, { desc: "It should return FailedPrecondition error if task is active", active: map[string][]*base.TaskMessage{ "default": {m1}, }, scheduled: map[string][]base.Z{ "default": {}, }, archived: map[string][]base.Z{ "default": {}, }, qname: "default", id: m1.ID, match: func(err error) bool { return errors.CanonicalCode(err) == errors.FailedPrecondition }, wantActive: map[string][]*base.TaskMessage{ "default": {m1}, }, wantScheduled: map[string][]base.Z{ "default": {}, }, wantArchived: map[string][]base.Z{ "default": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) h.SeedAllActiveQueues(t, r.client, tc.active) h.SeedAllScheduledQueues(t, r.client, tc.scheduled) h.SeedAllArchivedQueues(t, r.client, tc.archived) got := r.ArchiveTask(tc.qname, tc.id) if !tc.match(got) { t.Errorf("%s: returned error didn't match: got=%v", tc.desc, got) continue } for qname, want := range tc.wantActive { gotActive := h.GetActiveMessages(t, r.client, qname) if diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.ActiveKey(qname), diff) } } for qname, want := range tc.wantScheduled { gotScheduled := h.GetScheduledEntries(t, r.client, qname) if diff := cmp.Diff(want, gotScheduled, h.SortZSetEntryOpt, zScoreCmpOpt); diff != "" { t.Errorf("mismatch found in %q; (-want,+got)\n%s", base.ScheduledKey(qname), diff) } } for qname, want := range tc.wantArchived { gotArchived := h.GetArchivedEntries(t, r.client, qname) if diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt, zScoreCmpOpt); diff != "" { t.Errorf("mismatch found in %q; (-want,+got)\n%s", base.ArchivedKey(qname), diff) } } } } func TestArchiveAllPendingTasks(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") m4 := h.NewTaskMessageWithQueue("task4", nil, "custom") now := time.Now() t1 := now.Add(1 * time.Minute) t2 := now.Add(1 * time.Hour) r.SetClock(timeutil.NewSimulatedClock(now)) tests := []struct { pending map[string][]*base.TaskMessage archived map[string][]base.Z qname string want int64 wantPending map[string][]*base.TaskMessage wantArchived map[string][]base.Z }{ { pending: map[string][]*base.TaskMessage{ "default": {m1, m2}, }, archived: map[string][]base.Z{ "default": {}, }, qname: "default", want: 2, wantPending: map[string][]*base.TaskMessage{ "default": {}, }, wantArchived: map[string][]base.Z{ "default": { {Message: m1, Score: now.Unix()}, {Message: m2, Score: now.Unix()}, }, }, }, { pending: map[string][]*base.TaskMessage{ "default": {m1}, }, archived: map[string][]base.Z{ "default": {{Message: m2, Score: t2.Unix()}}, }, qname: "default", want: 1, wantPending: map[string][]*base.TaskMessage{ "default": {}, }, wantArchived: map[string][]base.Z{ "default": { {Message: m1, Score: now.Unix()}, {Message: m2, Score: t2.Unix()}, }, }, }, { pending: map[string][]*base.TaskMessage{ "default": {}, }, archived: map[string][]base.Z{ "default": { {Message: m1, Score: t1.Unix()}, {Message: m2, Score: t2.Unix()}, }, }, qname: "default", want: 0, wantPending: map[string][]*base.TaskMessage{ "default": {}, }, wantArchived: map[string][]base.Z{ "default": { {Message: m1, Score: t1.Unix()}, {Message: m2, Score: t2.Unix()}, }, }, }, { pending: map[string][]*base.TaskMessage{ "default": {m1, m2}, "custom": {m3, m4}, }, archived: map[string][]base.Z{ "default": {}, "custom": {}, }, qname: "custom", want: 2, wantPending: map[string][]*base.TaskMessage{ "default": {m1, m2}, "custom": {}, }, wantArchived: map[string][]base.Z{ "default": {}, "custom": { {Message: m3, Score: now.Unix()}, {Message: m4, Score: now.Unix()}, }, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) h.SeedAllPendingQueues(t, r.client, tc.pending) h.SeedAllArchivedQueues(t, r.client, tc.archived) got, err := r.ArchiveAllPendingTasks(tc.qname) if got != tc.want || err != nil { t.Errorf("(*RDB).KillAllRetryTasks(%q) = %v, %v; want %v, nil", tc.qname, got, err, tc.want) continue } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r.client, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want,+got)\n%s", base.PendingKey(qname), diff) } } for qname, want := range tc.wantArchived { gotArchived := h.GetArchivedEntries(t, r.client, qname) if diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt); diff != "" { t.Errorf("mismatch found in %q; (-want,+got)\n%s", base.ArchivedKey(qname), diff) } } } } func TestArchiveAllAggregatingTasks(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() r.SetClock(timeutil.NewSimulatedClock(now)) m1 := h.NewTaskMessageBuilder().SetQueue("default").SetType("task1").SetGroup("group1").Build() m2 := h.NewTaskMessageBuilder().SetQueue("default").SetType("task2").SetGroup("group1").Build() m3 := h.NewTaskMessageBuilder().SetQueue("custom").SetType("task3").SetGroup("group2").Build() fxt := struct { tasks []*h.TaskSeedData allQueues []string allGroups map[string][]string groups map[string][]redis.Z }{ tasks: []*h.TaskSeedData{ {Msg: m1, State: base.TaskStateAggregating}, {Msg: m2, State: base.TaskStateAggregating}, {Msg: m3, State: base.TaskStateAggregating}, }, allQueues: []string{"default", "custom"}, allGroups: map[string][]string{ base.AllGroups("default"): {"group1"}, base.AllGroups("custom"): {"group2"}, }, groups: map[string][]redis.Z{ base.GroupKey("default", "group1"): { {Member: m1.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, {Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())}, }, base.GroupKey("custom", "group2"): { {Member: m3.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, }, }, } tests := []struct { desc string qname string gname string want int64 wantArchived map[string][]redis.Z wantGroups map[string][]redis.Z wantAllGroups map[string][]string }{ { desc: "archive tasks in a group with multiple tasks", qname: "default", gname: "group1", want: 2, wantArchived: map[string][]redis.Z{ base.ArchivedKey("default"): { {Member: m1.ID, Score: float64(now.Unix())}, {Member: m2.ID, Score: float64(now.Unix())}, }, }, wantGroups: map[string][]redis.Z{ base.GroupKey("default", "group1"): {}, base.GroupKey("custom", "group2"): { {Member: m3.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, }, }, wantAllGroups: map[string][]string{ base.AllGroups("default"): {}, base.AllGroups("custom"): {"group2"}, }, }, { desc: "archive tasks in a group with a single task", qname: "custom", gname: "group2", want: 1, wantArchived: map[string][]redis.Z{ base.ArchivedKey("custom"): { {Member: m3.ID, Score: float64(now.Unix())}, }, }, wantGroups: map[string][]redis.Z{ base.GroupKey("default", "group1"): { {Member: m1.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, {Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())}, }, base.GroupKey("custom", "group2"): {}, }, wantAllGroups: map[string][]string{ base.AllGroups("default"): {"group1"}, base.AllGroups("custom"): {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) h.SeedTasks(t, r.client, fxt.tasks) h.SeedRedisSet(t, r.client, base.AllQueues, fxt.allQueues) h.SeedRedisSets(t, r.client, fxt.allGroups) h.SeedRedisZSets(t, r.client, fxt.groups) t.Run(tc.desc, func(t *testing.T) { got, err := r.ArchiveAllAggregatingTasks(tc.qname, tc.gname) if err != nil { t.Fatalf("ArchiveAllAggregatingTasks returned error: %v", err) } if got != tc.want { t.Errorf("ArchiveAllAggregatingTasks = %d, want %d", got, tc.want) } h.AssertRedisZSets(t, r.client, tc.wantArchived) h.AssertRedisZSets(t, r.client, tc.wantGroups) h.AssertRedisSets(t, r.client, tc.wantAllGroups) }) } } func TestArchiveAllRetryTasks(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") m4 := h.NewTaskMessageWithQueue("task4", nil, "custom") now := time.Now() t1 := now.Add(1 * time.Minute) t2 := now.Add(1 * time.Hour) t3 := now.Add(2 * time.Hour) t4 := now.Add(3 * time.Hour) r.SetClock(timeutil.NewSimulatedClock(now)) tests := []struct { retry map[string][]base.Z archived map[string][]base.Z qname string want int64 wantRetry map[string][]base.Z wantArchived map[string][]base.Z }{ { retry: map[string][]base.Z{ "default": { {Message: m1, Score: t1.Unix()}, {Message: m2, Score: t2.Unix()}, }, }, archived: map[string][]base.Z{ "default": {}, }, qname: "default", want: 2, wantRetry: map[string][]base.Z{ "default": {}, }, wantArchived: map[string][]base.Z{ "default": { {Message: m1, Score: now.Unix()}, {Message: m2, Score: now.Unix()}, }, }, }, { retry: map[string][]base.Z{ "default": {{Message: m1, Score: t1.Unix()}}, }, archived: map[string][]base.Z{ "default": {{Message: m2, Score: t2.Unix()}}, }, qname: "default", want: 1, wantRetry: map[string][]base.Z{ "default": {}, }, wantArchived: map[string][]base.Z{ "default": { {Message: m1, Score: now.Unix()}, {Message: m2, Score: t2.Unix()}, }, }, }, { retry: map[string][]base.Z{ "default": {}, }, archived: map[string][]base.Z{ "default": { {Message: m1, Score: t1.Unix()}, {Message: m2, Score: t2.Unix()}, }, }, qname: "default", want: 0, wantRetry: map[string][]base.Z{ "default": {}, }, wantArchived: map[string][]base.Z{ "default": { {Message: m1, Score: t1.Unix()}, {Message: m2, Score: t2.Unix()}, }, }, }, { retry: map[string][]base.Z{ "default": { {Message: m1, Score: t1.Unix()}, {Message: m2, Score: t2.Unix()}, }, "custom": { {Message: m3, Score: t3.Unix()}, {Message: m4, Score: t4.Unix()}, }, }, archived: map[string][]base.Z{ "default": {}, "custom": {}, }, qname: "custom", want: 2, wantRetry: map[string][]base.Z{ "default": { {Message: m1, Score: t1.Unix()}, {Message: m2, Score: t2.Unix()}, }, "custom": {}, }, wantArchived: map[string][]base.Z{ "default": {}, "custom": { {Message: m3, Score: now.Unix()}, {Message: m4, Score: now.Unix()}, }, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) h.SeedAllRetryQueues(t, r.client, tc.retry) h.SeedAllArchivedQueues(t, r.client, tc.archived) got, err := r.ArchiveAllRetryTasks(tc.qname) if got != tc.want || err != nil { t.Errorf("(*RDB).KillAllRetryTasks(%q) = %v, %v; want %v, nil", tc.qname, got, err, tc.want) continue } for qname, want := range tc.wantRetry { gotRetry := h.GetRetryEntries(t, r.client, qname) if diff := cmp.Diff(want, gotRetry, h.SortZSetEntryOpt); diff != "" { t.Errorf("mismatch found in %q; (-want,+got)\n%s", base.RetryKey(qname), diff) } } for qname, want := range tc.wantArchived { gotArchived := h.GetArchivedEntries(t, r.client, qname) if diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt); diff != "" { t.Errorf("mismatch found in %q; (-want,+got)\n%s", base.ArchivedKey(qname), diff) } } } } func TestArchiveAllScheduledTasks(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") m4 := h.NewTaskMessageWithQueue("task4", nil, "custom") now := time.Now() t1 := now.Add(time.Minute) t2 := now.Add(time.Hour) t3 := now.Add(time.Hour) t4 := now.Add(time.Hour) r.SetClock(timeutil.NewSimulatedClock(now)) tests := []struct { scheduled map[string][]base.Z archived map[string][]base.Z qname string want int64 wantScheduled map[string][]base.Z wantArchived map[string][]base.Z }{ { scheduled: map[string][]base.Z{ "default": { {Message: m1, Score: t1.Unix()}, {Message: m2, Score: t2.Unix()}, }, }, archived: map[string][]base.Z{ "default": {}, }, qname: "default", want: 2, wantScheduled: map[string][]base.Z{ "default": {}, }, wantArchived: map[string][]base.Z{ "default": { {Message: m1, Score: now.Unix()}, {Message: m2, Score: now.Unix()}, }, }, }, { scheduled: map[string][]base.Z{ "default": {{Message: m1, Score: t1.Unix()}}, }, archived: map[string][]base.Z{ "default": {{Message: m2, Score: t2.Unix()}}, }, qname: "default", want: 1, wantScheduled: map[string][]base.Z{ "default": {}, }, wantArchived: map[string][]base.Z{ "default": { {Message: m1, Score: now.Unix()}, {Message: m2, Score: t2.Unix()}, }, }, }, { scheduled: map[string][]base.Z{ "default": {}, }, archived: map[string][]base.Z{ "default": { {Message: m1, Score: t1.Unix()}, {Message: m2, Score: t2.Unix()}, }, }, qname: "default", want: 0, wantScheduled: map[string][]base.Z{ "default": {}, }, wantArchived: map[string][]base.Z{ "default": { {Message: m1, Score: t1.Unix()}, {Message: m2, Score: t2.Unix()}, }, }, }, { scheduled: map[string][]base.Z{ "default": { {Message: m1, Score: t1.Unix()}, {Message: m2, Score: t2.Unix()}, }, "custom": { {Message: m3, Score: t3.Unix()}, {Message: m4, Score: t4.Unix()}, }, }, archived: map[string][]base.Z{ "default": {}, "custom": {}, }, qname: "custom", want: 2, wantScheduled: map[string][]base.Z{ "default": { {Message: m1, Score: t1.Unix()}, {Message: m2, Score: t2.Unix()}, }, "custom": {}, }, wantArchived: map[string][]base.Z{ "default": {}, "custom": { {Message: m3, Score: now.Unix()}, {Message: m4, Score: now.Unix()}, }, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) h.SeedAllScheduledQueues(t, r.client, tc.scheduled) h.SeedAllArchivedQueues(t, r.client, tc.archived) got, err := r.ArchiveAllScheduledTasks(tc.qname) if got != tc.want || err != nil { t.Errorf("(*RDB).KillAllScheduledTasks(%q) = %v, %v; want %v, nil", tc.qname, got, err, tc.want) continue } for qname, want := range tc.wantScheduled { gotScheduled := h.GetScheduledEntries(t, r.client, qname) if diff := cmp.Diff(want, gotScheduled, h.SortZSetEntryOpt); diff != "" { t.Errorf("mismatch found in %q; (-want,+got)\n%s", base.ScheduledKey(qname), diff) } } for qname, want := range tc.wantArchived { gotArchived := h.GetArchivedEntries(t, r.client, qname) if diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt); diff != "" { t.Errorf("mismatch found in %q; (-want,+got)\n%s", base.ArchivedKey(qname), diff) } } } } func TestArchiveAllTasksError(t *testing.T) { r := setup(t) defer r.Close() tests := []struct { desc string qname string match func(err error) bool }{ { desc: "It returns QueueNotFoundError if queue doesn't exist", qname: "nonexistent", match: errors.IsQueueNotFound, }, } for _, tc := range tests { if _, got := r.ArchiveAllPendingTasks(tc.qname); !tc.match(got) { t.Errorf("%s: ArchiveAllPendingTasks returned %v", tc.desc, got) } if _, got := r.ArchiveAllScheduledTasks(tc.qname); !tc.match(got) { t.Errorf("%s: ArchiveAllScheduledTasks returned %v", tc.desc, got) } if _, got := r.ArchiveAllRetryTasks(tc.qname); !tc.match(got) { t.Errorf("%s: ArchiveAllRetryTasks returned %v", tc.desc, got) } } } func TestDeleteArchivedTask(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") t1 := time.Now().Add(-5 * time.Minute) t2 := time.Now().Add(-time.Hour) t3 := time.Now().Add(-time.Hour) tests := []struct { archived map[string][]base.Z qname string id string wantArchived map[string][]*base.TaskMessage }{ { archived: map[string][]base.Z{ "default": { {Message: m1, Score: t1.Unix()}, {Message: m2, Score: t2.Unix()}, }, }, qname: "default", id: m1.ID, wantArchived: map[string][]*base.TaskMessage{ "default": {m2}, }, }, { archived: map[string][]base.Z{ "default": { {Message: m1, Score: t1.Unix()}, {Message: m2, Score: t2.Unix()}, }, "custom": { {Message: m3, Score: t3.Unix()}, }, }, qname: "custom", id: m3.ID, wantArchived: map[string][]*base.TaskMessage{ "default": {m1, m2}, "custom": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllArchivedQueues(t, r.client, tc.archived) if got := r.DeleteTask(tc.qname, tc.id); got != nil { t.Errorf("r.DeleteTask(%q, %v) returned error: %v", tc.qname, tc.id, got) continue } for qname, want := range tc.wantArchived { gotArchived := h.GetArchivedMessages(t, r.client, qname) if diff := cmp.Diff(want, gotArchived, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.ArchivedKey(qname), diff) } } } } func TestDeleteRetryTask(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") t1 := time.Now().Add(5 * time.Minute) t2 := time.Now().Add(time.Hour) t3 := time.Now().Add(time.Hour) tests := []struct { retry map[string][]base.Z qname string id string wantRetry map[string][]*base.TaskMessage }{ { retry: map[string][]base.Z{ "default": { {Message: m1, Score: t1.Unix()}, {Message: m2, Score: t2.Unix()}, }, }, qname: "default", id: m1.ID, wantRetry: map[string][]*base.TaskMessage{ "default": {m2}, }, }, { retry: map[string][]base.Z{ "default": { {Message: m1, Score: t1.Unix()}, {Message: m2, Score: t2.Unix()}, }, "custom": { {Message: m3, Score: t3.Unix()}, }, }, qname: "custom", id: m3.ID, wantRetry: map[string][]*base.TaskMessage{ "default": {m1, m2}, "custom": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllRetryQueues(t, r.client, tc.retry) if got := r.DeleteTask(tc.qname, tc.id); got != nil { t.Errorf("r.DeleteTask(%q, %v) returned error: %v", tc.qname, tc.id, got) continue } for qname, want := range tc.wantRetry { gotRetry := h.GetRetryMessages(t, r.client, qname) if diff := cmp.Diff(want, gotRetry, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.RetryKey(qname), diff) } } } } func TestDeleteScheduledTask(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") t1 := time.Now().Add(5 * time.Minute) t2 := time.Now().Add(time.Hour) t3 := time.Now().Add(time.Hour) tests := []struct { scheduled map[string][]base.Z qname string id string wantScheduled map[string][]*base.TaskMessage }{ { scheduled: map[string][]base.Z{ "default": { {Message: m1, Score: t1.Unix()}, {Message: m2, Score: t2.Unix()}, }, }, qname: "default", id: m1.ID, wantScheduled: map[string][]*base.TaskMessage{ "default": {m2}, }, }, { scheduled: map[string][]base.Z{ "default": { {Message: m1, Score: t1.Unix()}, {Message: m2, Score: t2.Unix()}, }, "custom": { {Message: m3, Score: t3.Unix()}, }, }, qname: "custom", id: m3.ID, wantScheduled: map[string][]*base.TaskMessage{ "default": {m1, m2}, "custom": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllScheduledQueues(t, r.client, tc.scheduled) if got := r.DeleteTask(tc.qname, tc.id); got != nil { t.Errorf("r.DeleteTask(%q, %v) returned error: %v", tc.qname, tc.id, got) continue } for qname, want := range tc.wantScheduled { gotScheduled := h.GetScheduledMessages(t, r.client, qname) if diff := cmp.Diff(want, gotScheduled, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.ScheduledKey(qname), diff) } } } } func TestDeleteAggregatingTask(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() m1 := h.NewTaskMessageBuilder().SetQueue("default").SetType("task1").SetGroup("group1").Build() m2 := h.NewTaskMessageBuilder().SetQueue("default").SetType("task2").SetGroup("group1").Build() m3 := h.NewTaskMessageBuilder().SetQueue("custom").SetType("task3").SetGroup("group1").Build() fxt := struct { tasks []*h.TaskSeedData allQueues []string allGroups map[string][]string groups map[string][]redis.Z }{ tasks: []*h.TaskSeedData{ {Msg: m1, State: base.TaskStateAggregating}, {Msg: m2, State: base.TaskStateAggregating}, {Msg: m3, State: base.TaskStateAggregating}, }, allQueues: []string{"default", "custom"}, allGroups: map[string][]string{ base.AllGroups("default"): {"group1"}, base.AllGroups("custom"): {"group1"}, }, groups: map[string][]redis.Z{ base.GroupKey("default", "group1"): { {Member: m1.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, {Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())}, }, base.GroupKey("custom", "group1"): { {Member: m3.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, }, }, } tests := []struct { desc string qname string id string wantAllGroups map[string][]string wantGroups map[string][]redis.Z }{ { desc: "deletes a task from group with multiple tasks", qname: "default", id: m1.ID, wantAllGroups: map[string][]string{ base.AllGroups("default"): {"group1"}, base.AllGroups("custom"): {"group1"}, }, wantGroups: map[string][]redis.Z{ base.GroupKey("default", "group1"): { {Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())}, }, base.GroupKey("custom", "group1"): { {Member: m3.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, }, }, }, { desc: "deletes a task from group with single task", qname: "custom", id: m3.ID, wantAllGroups: map[string][]string{ base.AllGroups("default"): {"group1"}, base.AllGroups("custom"): {}, // should be clear out group from all-groups set }, wantGroups: map[string][]redis.Z{ base.GroupKey("default", "group1"): { {Member: m1.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, {Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())}, }, base.GroupKey("custom", "group1"): {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) h.SeedTasks(t, r.client, fxt.tasks) h.SeedRedisSet(t, r.client, base.AllQueues, fxt.allQueues) h.SeedRedisSets(t, r.client, fxt.allGroups) h.SeedRedisZSets(t, r.client, fxt.groups) t.Run(tc.desc, func(t *testing.T) { err := r.DeleteTask(tc.qname, tc.id) if err != nil { t.Fatalf("DeleteTask returned error: %v", err) } h.AssertRedisSets(t, r.client, tc.wantAllGroups) h.AssertRedisZSets(t, r.client, tc.wantGroups) }) } } func TestDeletePendingTask(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") tests := []struct { pending map[string][]*base.TaskMessage qname string id string wantPending map[string][]*base.TaskMessage }{ { pending: map[string][]*base.TaskMessage{ "default": {m1, m2}, }, qname: "default", id: m1.ID, wantPending: map[string][]*base.TaskMessage{ "default": {m2}, }, }, { pending: map[string][]*base.TaskMessage{ "default": {m1, m2}, "custom": {m3}, }, qname: "custom", id: m3.ID, wantPending: map[string][]*base.TaskMessage{ "default": {m1, m2}, "custom": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) h.SeedAllPendingQueues(t, r.client, tc.pending) if got := r.DeleteTask(tc.qname, tc.id); got != nil { t.Errorf("r.DeleteTask(%q, %v) returned error: %v", tc.qname, tc.id, got) continue } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r.client, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.PendingKey(qname), diff) } } } } func TestDeleteTaskWithUniqueLock(t *testing.T) { r := setup(t) defer r.Close() m1 := &base.TaskMessage{ ID: uuid.NewString(), Type: "email", Payload: h.JSON(map[string]interface{}{"user_id": json.Number("123")}), Queue: base.DefaultQueueName, UniqueKey: base.UniqueKey(base.DefaultQueueName, "email", h.JSON(map[string]interface{}{"user_id": 123})), } t1 := time.Now().Add(3 * time.Hour) tests := []struct { scheduled map[string][]base.Z qname string id string uniqueKey string wantScheduled map[string][]*base.TaskMessage }{ { scheduled: map[string][]base.Z{ "default": { {Message: m1, Score: t1.Unix()}, }, }, qname: "default", id: m1.ID, uniqueKey: m1.UniqueKey, wantScheduled: map[string][]*base.TaskMessage{ "default": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllScheduledQueues(t, r.client, tc.scheduled) if got := r.DeleteTask(tc.qname, tc.id); got != nil { t.Errorf("r.DeleteTask(%q, %v) returned error: %v", tc.qname, tc.id, got) continue } for qname, want := range tc.wantScheduled { gotScheduled := h.GetScheduledMessages(t, r.client, qname) if diff := cmp.Diff(want, gotScheduled, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.ScheduledKey(qname), diff) } } if r.client.Exists(context.Background(), tc.uniqueKey).Val() != 0 { t.Errorf("Uniqueness lock %q still exists", tc.uniqueKey) } } } func TestDeleteTaskError(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) t1 := time.Now().Add(5 * time.Minute) tests := []struct { desc string active map[string][]*base.TaskMessage scheduled map[string][]base.Z qname string id string match func(err error) bool wantActive map[string][]*base.TaskMessage wantScheduled map[string][]*base.TaskMessage }{ { desc: "It should return TaskNotFoundError if task doesn't exist the queue", active: map[string][]*base.TaskMessage{ "default": {}, }, scheduled: map[string][]base.Z{ "default": {{Message: m1, Score: t1.Unix()}}, }, qname: "default", id: uuid.NewString(), match: errors.IsTaskNotFound, wantActive: map[string][]*base.TaskMessage{ "default": {}, }, wantScheduled: map[string][]*base.TaskMessage{ "default": {m1}, }, }, { desc: "It should return QueueNotFoundError if the queue doesn't exist", active: map[string][]*base.TaskMessage{ "default": {}, }, scheduled: map[string][]base.Z{ "default": {{Message: m1, Score: t1.Unix()}}, }, qname: "nonexistent", id: uuid.NewString(), match: errors.IsQueueNotFound, wantActive: map[string][]*base.TaskMessage{ "default": {}, }, wantScheduled: map[string][]*base.TaskMessage{ "default": {m1}, }, }, { desc: "It should return FailedPrecondition error if task is active", active: map[string][]*base.TaskMessage{ "default": {m1}, }, scheduled: map[string][]base.Z{ "default": {}, }, qname: "default", id: m1.ID, match: func(err error) bool { return errors.CanonicalCode(err) == errors.FailedPrecondition }, wantActive: map[string][]*base.TaskMessage{ "default": {m1}, }, wantScheduled: map[string][]*base.TaskMessage{ "default": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllActiveQueues(t, r.client, tc.active) h.SeedAllScheduledQueues(t, r.client, tc.scheduled) got := r.DeleteTask(tc.qname, tc.id) if !tc.match(got) { t.Errorf("%s: r.DeleteTask(qname, id) returned %v", tc.desc, got) continue } for qname, want := range tc.wantActive { gotActive := h.GetActiveMessages(t, r.client, qname) if diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.ActiveKey(qname), diff) } } for qname, want := range tc.wantScheduled { gotScheduled := h.GetScheduledMessages(t, r.client, qname) if diff := cmp.Diff(want, gotScheduled, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.ScheduledKey(qname), diff) } } } } func TestDeleteAllArchivedTasks(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") tests := []struct { archived map[string][]base.Z qname string want int64 wantArchived map[string][]*base.TaskMessage }{ { archived: map[string][]base.Z{ "default": { {Message: m1, Score: time.Now().Unix()}, {Message: m2, Score: time.Now().Unix()}, }, "custom": { {Message: m3, Score: time.Now().Unix()}, }, }, qname: "default", want: 2, wantArchived: map[string][]*base.TaskMessage{ "default": {}, "custom": {m3}, }, }, { archived: map[string][]base.Z{ "default": {}, }, qname: "default", want: 0, wantArchived: map[string][]*base.TaskMessage{ "default": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllArchivedQueues(t, r.client, tc.archived) got, err := r.DeleteAllArchivedTasks(tc.qname) if err != nil { t.Errorf("r.DeleteAllArchivedTasks(%q) returned error: %v", tc.qname, err) } if got != tc.want { t.Errorf("r.DeleteAllArchivedTasks(%q) = %d, nil, want %d, nil", tc.qname, got, tc.want) } for qname, want := range tc.wantArchived { gotArchived := h.GetArchivedMessages(t, r.client, qname) if diff := cmp.Diff(want, gotArchived, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.ArchivedKey(qname), diff) } } } } func newCompletedTaskMessage(qname, typename string, retention time.Duration, completedAt time.Time) *base.TaskMessage { msg := h.NewTaskMessageWithQueue(typename, nil, qname) msg.Retention = int64(retention.Seconds()) msg.CompletedAt = completedAt.Unix() return msg } func TestDeleteAllCompletedTasks(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() m1 := newCompletedTaskMessage("default", "task1", 30*time.Minute, now.Add(-2*time.Minute)) m2 := newCompletedTaskMessage("default", "task2", 30*time.Minute, now.Add(-5*time.Minute)) m3 := newCompletedTaskMessage("custom", "task3", 30*time.Minute, now.Add(-5*time.Minute)) tests := []struct { completed map[string][]base.Z qname string want int64 wantCompleted map[string][]*base.TaskMessage }{ { completed: map[string][]base.Z{ "default": { {Message: m1, Score: m1.CompletedAt + m1.Retention}, {Message: m2, Score: m2.CompletedAt + m2.Retention}, }, "custom": { {Message: m3, Score: m2.CompletedAt + m3.Retention}, }, }, qname: "default", want: 2, wantCompleted: map[string][]*base.TaskMessage{ "default": {}, "custom": {m3}, }, }, { completed: map[string][]base.Z{ "default": {}, }, qname: "default", want: 0, wantCompleted: map[string][]*base.TaskMessage{ "default": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllCompletedQueues(t, r.client, tc.completed) got, err := r.DeleteAllCompletedTasks(tc.qname) if err != nil { t.Errorf("r.DeleteAllCompletedTasks(%q) returned error: %v", tc.qname, err) } if got != tc.want { t.Errorf("r.DeleteAllCompletedTasks(%q) = %d, nil, want %d, nil", tc.qname, got, tc.want) } for qname, want := range tc.wantCompleted { gotCompleted := h.GetCompletedMessages(t, r.client, qname) if diff := cmp.Diff(want, gotCompleted, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.CompletedKey(qname), diff) } } } } func TestDeleteAllArchivedTasksWithUniqueKey(t *testing.T) { r := setup(t) defer r.Close() m1 := &base.TaskMessage{ ID: uuid.NewString(), Type: "task1", Payload: nil, Timeout: 1800, Deadline: 0, UniqueKey: "asynq:{default}:unique:task1:nil", Queue: "default", } m2 := &base.TaskMessage{ ID: uuid.NewString(), Type: "task2", Payload: nil, Timeout: 1800, Deadline: 0, UniqueKey: "asynq:{default}:unique:task2:nil", Queue: "default", } m3 := h.NewTaskMessage("task3", nil) tests := []struct { archived map[string][]base.Z qname string want int64 uniqueKeys []string // list of unique keys that should be cleared wantArchived map[string][]*base.TaskMessage }{ { archived: map[string][]base.Z{ "default": { {Message: m1, Score: time.Now().Unix()}, {Message: m2, Score: time.Now().Unix()}, {Message: m3, Score: time.Now().Unix()}, }, }, qname: "default", want: 3, uniqueKeys: []string{m1.UniqueKey, m2.UniqueKey}, wantArchived: map[string][]*base.TaskMessage{ "default": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllArchivedQueues(t, r.client, tc.archived) got, err := r.DeleteAllArchivedTasks(tc.qname) if err != nil { t.Errorf("r.DeleteAllArchivedTasks(%q) returned error: %v", tc.qname, err) } if got != tc.want { t.Errorf("r.DeleteAllArchivedTasks(%q) = %d, nil, want %d, nil", tc.qname, got, tc.want) } for qname, want := range tc.wantArchived { gotArchived := h.GetArchivedMessages(t, r.client, qname) if diff := cmp.Diff(want, gotArchived, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.ArchivedKey(qname), diff) } } for _, uniqueKey := range tc.uniqueKeys { if r.client.Exists(context.Background(), uniqueKey).Val() != 0 { t.Errorf("Uniqueness lock %q still exists", uniqueKey) } } } } func TestDeleteAllRetryTasks(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") tests := []struct { retry map[string][]base.Z qname string want int64 wantRetry map[string][]*base.TaskMessage }{ { retry: map[string][]base.Z{ "default": { {Message: m1, Score: time.Now().Unix()}, {Message: m2, Score: time.Now().Unix()}, }, "custom": { {Message: m3, Score: time.Now().Unix()}, }, }, qname: "custom", want: 1, wantRetry: map[string][]*base.TaskMessage{ "default": {m1, m2}, "custom": {}, }, }, { retry: map[string][]base.Z{ "default": {}, }, qname: "default", want: 0, wantRetry: map[string][]*base.TaskMessage{ "default": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllRetryQueues(t, r.client, tc.retry) got, err := r.DeleteAllRetryTasks(tc.qname) if err != nil { t.Errorf("r.DeleteAllRetryTasks(%q) returned error: %v", tc.qname, err) } if got != tc.want { t.Errorf("r.DeleteAllRetryTasks(%q) = %d, nil, want %d, nil", tc.qname, got, tc.want) } for qname, want := range tc.wantRetry { gotRetry := h.GetRetryMessages(t, r.client, qname) if diff := cmp.Diff(want, gotRetry, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.RetryKey(qname), diff) } } } } func TestDeleteAllScheduledTasks(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") tests := []struct { scheduled map[string][]base.Z qname string want int64 wantScheduled map[string][]*base.TaskMessage }{ { scheduled: map[string][]base.Z{ "default": { {Message: m1, Score: time.Now().Add(time.Minute).Unix()}, {Message: m2, Score: time.Now().Add(time.Minute).Unix()}, }, "custom": { {Message: m3, Score: time.Now().Add(time.Minute).Unix()}, }, }, qname: "default", want: 2, wantScheduled: map[string][]*base.TaskMessage{ "default": {}, "custom": {m3}, }, }, { scheduled: map[string][]base.Z{ "custom": {}, }, qname: "custom", want: 0, wantScheduled: map[string][]*base.TaskMessage{ "custom": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllScheduledQueues(t, r.client, tc.scheduled) got, err := r.DeleteAllScheduledTasks(tc.qname) if err != nil { t.Errorf("r.DeleteAllScheduledTasks(%q) returned error: %v", tc.qname, err) } if got != tc.want { t.Errorf("r.DeleteAllScheduledTasks(%q) = %d, nil, want %d, nil", tc.qname, got, tc.want) } for qname, want := range tc.wantScheduled { gotScheduled := h.GetScheduledMessages(t, r.client, qname) if diff := cmp.Diff(want, gotScheduled, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.ScheduledKey(qname), diff) } } } } func TestDeleteAllAggregatingTasks(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() m1 := h.NewTaskMessageBuilder().SetQueue("default").SetType("task1").SetGroup("group1").Build() m2 := h.NewTaskMessageBuilder().SetQueue("default").SetType("task2").SetGroup("group1").Build() m3 := h.NewTaskMessageBuilder().SetQueue("custom").SetType("task3").SetGroup("group1").Build() fxt := struct { tasks []*h.TaskSeedData allQueues []string allGroups map[string][]string groups map[string][]redis.Z }{ tasks: []*h.TaskSeedData{ {Msg: m1, State: base.TaskStateAggregating}, {Msg: m2, State: base.TaskStateAggregating}, {Msg: m3, State: base.TaskStateAggregating}, }, allQueues: []string{"default", "custom"}, allGroups: map[string][]string{ base.AllGroups("default"): {"group1"}, base.AllGroups("custom"): {"group1"}, }, groups: map[string][]redis.Z{ base.GroupKey("default", "group1"): { {Member: m1.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, {Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())}, }, base.GroupKey("custom", "group1"): { {Member: m3.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, }, }, } tests := []struct { desc string qname string gname string want int64 wantAllGroups map[string][]string wantGroups map[string][]redis.Z }{ { desc: "default queue group1", qname: "default", gname: "group1", want: 2, wantAllGroups: map[string][]string{ base.AllGroups("default"): {}, base.AllGroups("custom"): {"group1"}, }, wantGroups: map[string][]redis.Z{ base.GroupKey("default", "group1"): nil, base.GroupKey("custom", "group1"): { {Member: m3.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, }, }, }, { desc: "custom queue group1", qname: "custom", gname: "group1", want: 1, wantAllGroups: map[string][]string{ base.AllGroups("default"): {"group1"}, base.AllGroups("custom"): {}, }, wantGroups: map[string][]redis.Z{ base.GroupKey("default", "group1"): { {Member: m1.ID, Score: float64(now.Add(-20 * time.Second).Unix())}, {Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())}, }, base.GroupKey("custom", "group1"): nil, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) h.SeedTasks(t, r.client, fxt.tasks) h.SeedRedisSet(t, r.client, base.AllQueues, fxt.allQueues) h.SeedRedisSets(t, r.client, fxt.allGroups) h.SeedRedisZSets(t, r.client, fxt.groups) t.Run(tc.desc, func(t *testing.T) { got, err := r.DeleteAllAggregatingTasks(tc.qname, tc.gname) if err != nil { t.Fatalf("DeleteAllAggregatingTasks returned error: %v", err) } if got != tc.want { t.Errorf("DeleteAllAggregatingTasks = %d, want %d", got, tc.want) } h.AssertRedisSets(t, r.client, tc.wantAllGroups) h.AssertRedisZSets(t, r.client, tc.wantGroups) }) } } func TestDeleteAllPendingTasks(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") tests := []struct { pending map[string][]*base.TaskMessage qname string want int64 wantPending map[string][]*base.TaskMessage }{ { pending: map[string][]*base.TaskMessage{ "default": {m1, m2}, "custom": {m3}, }, qname: "default", want: 2, wantPending: map[string][]*base.TaskMessage{ "default": {}, "custom": {m3}, }, }, { pending: map[string][]*base.TaskMessage{ "custom": {}, }, qname: "custom", want: 0, wantPending: map[string][]*base.TaskMessage{ "custom": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllPendingQueues(t, r.client, tc.pending) got, err := r.DeleteAllPendingTasks(tc.qname) if err != nil { t.Errorf("r.DeleteAllPendingTasks(%q) returned error: %v", tc.qname, err) } if got != tc.want { t.Errorf("r.DeleteAllPendingTasks(%q) = %d, nil, want %d, nil", tc.qname, got, tc.want) } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r.client, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.PendingKey(qname), diff) } } } } func TestDeleteAllTasksError(t *testing.T) { r := setup(t) defer r.Close() tests := []struct { desc string qname string match func(err error) bool }{ { desc: "It returns QueueNotFoundError if queue doesn't exist", qname: "nonexistent", match: errors.IsQueueNotFound, }, } for _, tc := range tests { if _, got := r.DeleteAllPendingTasks(tc.qname); !tc.match(got) { t.Errorf("%s: DeleteAllPendingTasks returned %v", tc.desc, got) } if _, got := r.DeleteAllScheduledTasks(tc.qname); !tc.match(got) { t.Errorf("%s: DeleteAllScheduledTasks returned %v", tc.desc, got) } if _, got := r.DeleteAllRetryTasks(tc.qname); !tc.match(got) { t.Errorf("%s: DeleteAllRetryTasks returned %v", tc.desc, got) } if _, got := r.DeleteAllArchivedTasks(tc.qname); !tc.match(got) { t.Errorf("%s: DeleteAllArchivedTasks returned %v", tc.desc, got) } } } func TestRemoveQueue(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") m4 := h.NewTaskMessageWithQueue("task4", nil, "custom") tests := []struct { pending map[string][]*base.TaskMessage inProgress map[string][]*base.TaskMessage scheduled map[string][]base.Z retry map[string][]base.Z archived map[string][]base.Z qname string // queue to remove force bool }{ { pending: map[string][]*base.TaskMessage{ "default": {m1, m2}, "custom": {}, }, inProgress: map[string][]*base.TaskMessage{ "default": {}, "custom": {}, }, scheduled: map[string][]base.Z{ "default": {}, "custom": {}, }, retry: map[string][]base.Z{ "default": {}, "custom": {}, }, archived: map[string][]base.Z{ "default": {}, "custom": {}, }, qname: "custom", force: false, }, { pending: map[string][]*base.TaskMessage{ "default": {m1, m2}, "custom": {m3}, }, inProgress: map[string][]*base.TaskMessage{ "default": {}, "custom": {}, }, scheduled: map[string][]base.Z{ "default": {}, "custom": {{Message: m4, Score: time.Now().Unix()}}, }, retry: map[string][]base.Z{ "default": {}, "custom": {}, }, archived: map[string][]base.Z{ "default": {}, "custom": {}, }, qname: "custom", force: true, // allow removing non-empty queue }, } for _, tc := range tests { h.FlushDB(t, r.client) h.SeedAllPendingQueues(t, r.client, tc.pending) h.SeedAllActiveQueues(t, r.client, tc.inProgress) h.SeedAllScheduledQueues(t, r.client, tc.scheduled) h.SeedAllRetryQueues(t, r.client, tc.retry) h.SeedAllArchivedQueues(t, r.client, tc.archived) err := r.RemoveQueue(tc.qname, tc.force) if err != nil { t.Errorf("(*RDB).RemoveQueue(%q, %t) = %v, want nil", tc.qname, tc.force, err) continue } if r.client.SIsMember(context.Background(), base.AllQueues, tc.qname).Val() { t.Errorf("%q is a member of %q", tc.qname, base.AllQueues) } keys := []string{ base.PendingKey(tc.qname), base.ActiveKey(tc.qname), base.LeaseKey(tc.qname), base.ScheduledKey(tc.qname), base.RetryKey(tc.qname), base.ArchivedKey(tc.qname), } for _, key := range keys { if r.client.Exists(context.Background(), key).Val() != 0 { t.Errorf("key %q still exists", key) } } if n := len(r.client.Keys(context.Background(), base.TaskKeyPrefix(tc.qname)+"*").Val()); n != 0 { t.Errorf("%d keys still exists for tasks", n) } } } func TestRemoveQueueError(t *testing.T) { r := setup(t) defer r.Close() m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessageWithQueue("task3", nil, "custom") m4 := h.NewTaskMessageWithQueue("task4", nil, "custom") tests := []struct { desc string pending map[string][]*base.TaskMessage inProgress map[string][]*base.TaskMessage scheduled map[string][]base.Z retry map[string][]base.Z archived map[string][]base.Z qname string // queue to remove force bool match func(err error) bool }{ { desc: "removing non-existent queue", pending: map[string][]*base.TaskMessage{ "default": {m1, m2}, "custom": {m3}, }, inProgress: map[string][]*base.TaskMessage{ "default": {}, "custom": {}, }, scheduled: map[string][]base.Z{ "default": {}, "custom": {}, }, retry: map[string][]base.Z{ "default": {}, "custom": {}, }, archived: map[string][]base.Z{ "default": {}, "custom": {}, }, qname: "nonexistent", force: false, match: errors.IsQueueNotFound, }, { desc: "removing non-empty queue", pending: map[string][]*base.TaskMessage{ "default": {m1, m2}, "custom": {m3}, }, inProgress: map[string][]*base.TaskMessage{ "default": {}, "custom": {}, }, scheduled: map[string][]base.Z{ "default": {}, "custom": {{Message: m4, Score: time.Now().Unix()}}, }, retry: map[string][]base.Z{ "default": {}, "custom": {}, }, archived: map[string][]base.Z{ "default": {}, "custom": {}, }, qname: "custom", force: false, match: errors.IsQueueNotEmpty, }, { desc: "force removing queue with active tasks", pending: map[string][]*base.TaskMessage{ "default": {m1, m2}, "custom": {m3}, }, inProgress: map[string][]*base.TaskMessage{ "default": {}, "custom": {m4}, }, scheduled: map[string][]base.Z{ "default": {}, "custom": {}, }, retry: map[string][]base.Z{ "default": {}, "custom": {}, }, archived: map[string][]base.Z{ "default": {}, "custom": {}, }, qname: "custom", // Even with force=true, it should error if there are active tasks. force: true, match: func(err error) bool { return errors.CanonicalCode(err) == errors.FailedPrecondition }, }, } for _, tc := range tests { h.FlushDB(t, r.client) h.SeedAllPendingQueues(t, r.client, tc.pending) h.SeedAllActiveQueues(t, r.client, tc.inProgress) h.SeedAllScheduledQueues(t, r.client, tc.scheduled) h.SeedAllRetryQueues(t, r.client, tc.retry) h.SeedAllArchivedQueues(t, r.client, tc.archived) got := r.RemoveQueue(tc.qname, tc.force) if !tc.match(got) { t.Errorf("%s; returned error didn't match expected value; got=%v", tc.desc, got) continue } // Make sure that nothing changed for qname, want := range tc.pending { gotPending := h.GetPendingMessages(t, r.client, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("%s;mismatch found in %q; (-want,+got):\n%s", tc.desc, base.PendingKey(qname), diff) } } for qname, want := range tc.inProgress { gotActive := h.GetActiveMessages(t, r.client, qname) if diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != "" { t.Errorf("%s;mismatch found in %q; (-want,+got):\n%s", tc.desc, base.ActiveKey(qname), diff) } } for qname, want := range tc.scheduled { gotScheduled := h.GetScheduledEntries(t, r.client, qname) if diff := cmp.Diff(want, gotScheduled, h.SortZSetEntryOpt); diff != "" { t.Errorf("%s;mismatch found in %q; (-want,+got):\n%s", tc.desc, base.ScheduledKey(qname), diff) } } for qname, want := range tc.retry { gotRetry := h.GetRetryEntries(t, r.client, qname) if diff := cmp.Diff(want, gotRetry, h.SortZSetEntryOpt); diff != "" { t.Errorf("%s;mismatch found in %q; (-want,+got):\n%s", tc.desc, base.RetryKey(qname), diff) } } for qname, want := range tc.archived { gotArchived := h.GetArchivedEntries(t, r.client, qname) if diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt); diff != "" { t.Errorf("%s;mismatch found in %q; (-want,+got):\n%s", tc.desc, base.ArchivedKey(qname), diff) } } } } func TestListServers(t *testing.T) { r := setup(t) defer r.Close() started1 := time.Now().Add(-time.Hour) info1 := &base.ServerInfo{ Host: "do.droplet1", PID: 1234, ServerID: "server123", Concurrency: 10, Queues: map[string]int{"default": 1}, Status: "active", Started: started1, ActiveWorkerCount: 0, } started2 := time.Now().Add(-2 * time.Hour) info2 := &base.ServerInfo{ Host: "do.droplet2", PID: 9876, ServerID: "server456", Concurrency: 20, Queues: map[string]int{"email": 1}, Status: "stopped", Started: started2, ActiveWorkerCount: 1, } tests := []struct { data []*base.ServerInfo }{ { data: []*base.ServerInfo{}, }, { data: []*base.ServerInfo{info1}, }, { data: []*base.ServerInfo{info1, info2}, }, } for _, tc := range tests { h.FlushDB(t, r.client) for _, info := range tc.data { if err := r.WriteServerState(info, []*base.WorkerInfo{}, 5*time.Second); err != nil { t.Fatal(err) } } got, err := r.ListServers() if err != nil { t.Errorf("r.ListServers returned an error: %v", err) } if diff := cmp.Diff(tc.data, got, h.SortServerInfoOpt); diff != "" { t.Errorf("r.ListServers returned %v, want %v; (-want,+got)\n%s", got, tc.data, diff) } } } func TestListWorkers(t *testing.T) { r := setup(t) defer r.Close() var ( host = "127.0.0.1" pid = 4567 serverID = "server123" m1 = h.NewTaskMessage("send_email", h.JSON(map[string]interface{}{"user_id": "abc123"})) m2 = h.NewTaskMessage("gen_thumbnail", h.JSON(map[string]interface{}{"path": "some/path/to/image/file"})) m3 = h.NewTaskMessage("reindex", h.JSON(map[string]interface{}{})) ) tests := []struct { data []*base.WorkerInfo }{ { data: []*base.WorkerInfo{ { Host: host, PID: pid, ServerID: serverID, ID: m1.ID, Type: m1.Type, Queue: m1.Queue, Payload: m1.Payload, Started: time.Now().Add(-1 * time.Second), Deadline: time.Now().Add(30 * time.Second), }, { Host: host, PID: pid, ServerID: serverID, ID: m2.ID, Type: m2.Type, Queue: m2.Queue, Payload: m2.Payload, Started: time.Now().Add(-5 * time.Second), Deadline: time.Now().Add(10 * time.Minute), }, { Host: host, PID: pid, ServerID: serverID, ID: m3.ID, Type: m3.Type, Queue: m3.Queue, Payload: m3.Payload, Started: time.Now().Add(-30 * time.Second), Deadline: time.Now().Add(30 * time.Minute), }, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) err := r.WriteServerState(&base.ServerInfo{}, tc.data, time.Minute) if err != nil { t.Errorf("could not write server state to redis: %v", err) continue } got, err := r.ListWorkers() if err != nil { t.Errorf("(*RDB).ListWorkers() returned an error: %v", err) continue } if diff := cmp.Diff(tc.data, got, h.SortWorkerInfoOpt); diff != "" { t.Errorf("(*RDB).ListWorkers() = %v, want = %v; (-want,+got)\n%s", got, tc.data, diff) } } } func TestWriteListClearSchedulerEntries(t *testing.T) { r := setup(t) now := time.Now().UTC() schedulerID := "127.0.0.1:9876:abc123" data := []*base.SchedulerEntry{ { Spec: "* * * * *", Type: "foo", Payload: nil, Opts: nil, Next: now.Add(5 * time.Hour), Prev: now.Add(-2 * time.Hour), }, { Spec: "@every 20m", Type: "bar", Payload: h.JSON(map[string]interface{}{"fiz": "baz"}), Opts: nil, Next: now.Add(1 * time.Minute), Prev: now.Add(-19 * time.Minute), }, } if err := r.WriteSchedulerEntries(schedulerID, data, 30*time.Second); err != nil { t.Fatalf("WriteSchedulerEnties failed: %v", err) } entries, err := r.ListSchedulerEntries() if err != nil { t.Fatalf("ListSchedulerEntries failed: %v", err) } if diff := cmp.Diff(data, entries, h.SortSchedulerEntryOpt); diff != "" { t.Errorf("ListSchedulerEntries() = %v, want %v; (-want,+got)\n%s", entries, data, diff) } if err := r.ClearSchedulerEntries(schedulerID); err != nil { t.Fatalf("ClearSchedulerEntries failed: %v", err) } entries, err = r.ListSchedulerEntries() if err != nil { t.Fatalf("ListSchedulerEntries() after clear failed: %v", err) } if len(entries) != 0 { t.Errorf("found %d entries, want 0 after clearing", len(entries)) } } func TestSchedulerEnqueueEvents(t *testing.T) { r := setup(t) var ( now = time.Now() oneDayAgo = now.Add(-24 * time.Hour) fiveHoursAgo = now.Add(-5 * time.Hour) oneHourAgo = now.Add(-1 * time.Hour) ) tests := []struct { entryID string events []*base.SchedulerEnqueueEvent want []*base.SchedulerEnqueueEvent }{ { entryID: "entry123", events: []*base.SchedulerEnqueueEvent{ {TaskID: "task123", EnqueuedAt: oneDayAgo}, {TaskID: "task789", EnqueuedAt: oneHourAgo}, {TaskID: "task456", EnqueuedAt: fiveHoursAgo}, }, // Recent events first want: []*base.SchedulerEnqueueEvent{ {TaskID: "task789", EnqueuedAt: oneHourAgo}, {TaskID: "task456", EnqueuedAt: fiveHoursAgo}, {TaskID: "task123", EnqueuedAt: oneDayAgo}, }, }, { entryID: "entry456", events: nil, want: nil, }, } loop: for _, tc := range tests { h.FlushDB(t, r.client) for _, e := range tc.events { if err := r.RecordSchedulerEnqueueEvent(tc.entryID, e); err != nil { t.Errorf("RecordSchedulerEnqueueEvent(%q, %v) failed: %v", tc.entryID, e, err) continue loop } } got, err := r.ListSchedulerEnqueueEvents(tc.entryID, Pagination{Size: 20, Page: 0}) if err != nil { t.Errorf("ListSchedulerEnqueueEvents(%q) failed: %v", tc.entryID, err) continue } if diff := cmp.Diff(tc.want, got, timeCmpOpt); diff != "" { t.Errorf("ListSchedulerEnqueueEvent(%q) = %v, want %v; (-want,+got)\n%s", tc.entryID, got, tc.want, diff) } } } func TestRecordSchedulerEnqueueEventTrimsDataSet(t *testing.T) { r := setup(t) var ( entryID = "entry123" now = time.Now() key = base.SchedulerHistoryKey(entryID) ) // Record maximum number of events. for i := 1; i <= maxEvents; i++ { event := base.SchedulerEnqueueEvent{ TaskID: fmt.Sprintf("task%d", i), EnqueuedAt: now.Add(-time.Duration(i) * time.Second), } if err := r.RecordSchedulerEnqueueEvent(entryID, &event); err != nil { t.Fatalf("RecordSchedulerEnqueueEvent failed: %v", err) } } // Make sure the set is full. if n := r.client.ZCard(context.Background(), key).Val(); n != maxEvents { t.Fatalf("unexpected number of events; got %d, want %d", n, maxEvents) } // Record one more event, should evict the oldest event. event := base.SchedulerEnqueueEvent{ TaskID: "latest", EnqueuedAt: now, } if err := r.RecordSchedulerEnqueueEvent(entryID, &event); err != nil { t.Fatalf("RecordSchedulerEnqueueEvent failed: %v", err) } if n := r.client.ZCard(context.Background(), key).Val(); n != maxEvents { t.Fatalf("unexpected number of events; got %d, want %d", n, maxEvents) } events, err := r.ListSchedulerEnqueueEvents(entryID, Pagination{Size: maxEvents}) if err != nil { t.Fatalf("ListSchedulerEnqueueEvents failed: %v", err) } if first := events[0]; first.TaskID != "latest" { t.Errorf("unexpected first event; got %q, want %q", first.TaskID, "latest") } if last := events[maxEvents-1]; last.TaskID != fmt.Sprintf("task%d", maxEvents-1) { t.Errorf("unexpected last event; got %q, want %q", last.TaskID, fmt.Sprintf("task%d", maxEvents-1)) } } func TestPause(t *testing.T) { r := setup(t) tests := []struct { qname string // name of the queue to pause }{ {qname: "default"}, {qname: "custom"}, } for _, tc := range tests { h.FlushDB(t, r.client) err := r.Pause(tc.qname) if err != nil { t.Errorf("Pause(%q) returned error: %v", tc.qname, err) } key := base.PausedKey(tc.qname) if r.client.Exists(context.Background(), key).Val() == 0 { t.Errorf("key %q does not exist", key) } } } func TestPauseError(t *testing.T) { r := setup(t) tests := []struct { desc string // test case description paused []string // already paused queues qname string // name of the queue to pause }{ {"queue already paused", []string{"default", "custom"}, "default"}, } for _, tc := range tests { h.FlushDB(t, r.client) for _, qname := range tc.paused { if err := r.Pause(qname); err != nil { t.Fatalf("could not pause %q: %v", qname, err) } } err := r.Pause(tc.qname) if err == nil { t.Errorf("%s; Pause(%q) returned nil: want error", tc.desc, tc.qname) } } } func TestUnpause(t *testing.T) { r := setup(t) tests := []struct { paused []string // already paused queues qname string // name of the queue to unpause }{ {[]string{"default", "custom"}, "default"}, } for _, tc := range tests { h.FlushDB(t, r.client) for _, qname := range tc.paused { if err := r.Pause(qname); err != nil { t.Fatalf("could not pause %q: %v", qname, err) } } err := r.Unpause(tc.qname) if err != nil { t.Errorf("Unpause(%q) returned error: %v", tc.qname, err) } key := base.PausedKey(tc.qname) if r.client.Exists(context.Background(), key).Val() == 1 { t.Errorf("key %q exists", key) } } } func TestUnpauseError(t *testing.T) { r := setup(t) tests := []struct { desc string // test case description paused []string // already paused queues qname string // name of the queue to unpause }{ {"queue is not paused", []string{"default"}, "custom"}, } for _, tc := range tests { h.FlushDB(t, r.client) for _, qname := range tc.paused { if err := r.Pause(qname); err != nil { t.Fatalf("could not pause %q: %v", qname, err) } } err := r.Unpause(tc.qname) if err == nil { t.Errorf("%s; Unpause(%q) returned nil: want error", tc.desc, tc.qname) } } } ================================================ FILE: internal/rdb/rdb.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. // Package rdb encapsulates the interactions with redis. package rdb import ( "context" "fmt" "math" "sync" "time" "github.com/google/uuid" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/errors" "github.com/hibiken/asynq/internal/timeutil" "github.com/redis/go-redis/v9" "github.com/spf13/cast" ) const statsTTL = 90 * 24 * time.Hour // 90 days // LeaseDuration is the duration used to initially create a lease and to extend it thereafter. const LeaseDuration = 30 * time.Second // RDB is a client interface to query and mutate task queues. type RDB struct { client redis.UniversalClient clock timeutil.Clock queuesPublished sync.Map } // NewRDB returns a new instance of RDB. func NewRDB(client redis.UniversalClient) *RDB { return &RDB{ client: client, clock: timeutil.NewRealClock(), } } // Close closes the connection with redis server. func (r *RDB) Close() error { return r.client.Close() } // Client returns the reference to underlying redis client. func (r *RDB) Client() redis.UniversalClient { return r.client } // SetClock sets the clock used by RDB to the given clock. // // Use this function to set the clock to SimulatedClock in tests. func (r *RDB) SetClock(c timeutil.Clock) { r.clock = c } // Ping checks the connection with redis server. func (r *RDB) Ping() error { return r.client.Ping(context.Background()).Err() } func (r *RDB) runScript(ctx context.Context, op errors.Op, script *redis.Script, keys []string, args ...interface{}) error { if err := script.Run(ctx, r.client, keys, args...).Err(); err != nil { return errors.E(op, errors.Internal, fmt.Sprintf("redis eval error: %v", err)) } return nil } // Runs the given script with keys and args and returns the script's return value as int64. func (r *RDB) runScriptWithErrorCode(ctx context.Context, op errors.Op, script *redis.Script, keys []string, args ...interface{}) (int64, error) { res, err := script.Run(ctx, r.client, keys, args...).Result() if err != nil { return 0, errors.E(op, errors.Unknown, fmt.Sprintf("redis eval error: %v", err)) } n, ok := res.(int64) if !ok { return 0, errors.E(op, errors.Internal, fmt.Sprintf("unexpected return value from Lua script: %v", res)) } return n, nil } // enqueueCmd enqueues a given task message. // // Input: // KEYS[1] -> asynq:{}:t: // KEYS[2] -> asynq:{}:pending // -- // ARGV[1] -> task message data // ARGV[2] -> task ID // ARGV[3] -> current unix time in nsec // // Output: // Returns 1 if successfully enqueued // Returns 0 if task ID already exists var enqueueCmd = redis.NewScript(` if redis.call("EXISTS", KEYS[1]) == 1 then return 0 end redis.call("HSET", KEYS[1], "msg", ARGV[1], "state", "pending", "pending_since", ARGV[3]) redis.call("LPUSH", KEYS[2], ARGV[2]) return 1 `) // Enqueue adds the given task to the pending list of the queue. func (r *RDB) Enqueue(ctx context.Context, msg *base.TaskMessage) error { var op errors.Op = "rdb.Enqueue" encoded, err := base.EncodeMessage(msg) if err != nil { return errors.E(op, errors.Unknown, fmt.Sprintf("cannot encode message: %v", err)) } if _, found := r.queuesPublished.Load(msg.Queue); !found { if err := r.client.SAdd(ctx, base.AllQueues, msg.Queue).Err(); err != nil { return errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: "sadd", Err: err}) } r.queuesPublished.Store(msg.Queue, true) } keys := []string{ base.TaskKey(msg.Queue, msg.ID), base.PendingKey(msg.Queue), } argv := []interface{}{ encoded, msg.ID, r.clock.Now().UnixNano(), } n, err := r.runScriptWithErrorCode(ctx, op, enqueueCmd, keys, argv...) if err != nil { return err } if n == 0 { return errors.E(op, errors.AlreadyExists, errors.ErrTaskIdConflict) } return nil } // enqueueUniqueCmd enqueues the task message if the task is unique. // // KEYS[1] -> unique key // KEYS[2] -> asynq:{}:t: // KEYS[3] -> asynq:{}:pending // -- // ARGV[1] -> task ID // ARGV[2] -> uniqueness lock TTL // ARGV[3] -> task message data // ARGV[4] -> current unix time in nsec // // Output: // Returns 1 if successfully enqueued // Returns 0 if task ID conflicts with another task // Returns -1 if task unique key already exists var enqueueUniqueCmd = redis.NewScript(` local ok = redis.call("SET", KEYS[1], ARGV[1], "NX", "EX", ARGV[2]) if not ok then return -1 end if redis.call("EXISTS", KEYS[2]) == 1 then return 0 end redis.call("HSET", KEYS[2], "msg", ARGV[3], "state", "pending", "pending_since", ARGV[4], "unique_key", KEYS[1]) redis.call("LPUSH", KEYS[3], ARGV[1]) return 1 `) // EnqueueUnique inserts the given task if the task's uniqueness lock can be acquired. // It returns ErrDuplicateTask if the lock cannot be acquired. func (r *RDB) EnqueueUnique(ctx context.Context, msg *base.TaskMessage, ttl time.Duration) error { var op errors.Op = "rdb.EnqueueUnique" encoded, err := base.EncodeMessage(msg) if err != nil { return errors.E(op, errors.Internal, "cannot encode task message: %v", err) } if _, found := r.queuesPublished.Load(msg.Queue); !found { if err := r.client.SAdd(ctx, base.AllQueues, msg.Queue).Err(); err != nil { return errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: "sadd", Err: err}) } r.queuesPublished.Store(msg.Queue, true) } keys := []string{ msg.UniqueKey, base.TaskKey(msg.Queue, msg.ID), base.PendingKey(msg.Queue), } argv := []interface{}{ msg.ID, int(ttl.Seconds()), encoded, r.clock.Now().UnixNano(), } n, err := r.runScriptWithErrorCode(ctx, op, enqueueUniqueCmd, keys, argv...) if err != nil { return err } if n == -1 { return errors.E(op, errors.AlreadyExists, errors.ErrDuplicateTask) } if n == 0 { return errors.E(op, errors.AlreadyExists, errors.ErrTaskIdConflict) } return nil } // Input: // KEYS[1] -> asynq:{}:pending // KEYS[2] -> asynq:{}:paused // KEYS[3] -> asynq:{}:active // KEYS[4] -> asynq:{}:lease // -- // ARGV[1] -> initial lease expiration Unix time // ARGV[2] -> task key prefix // // Output: // Returns nil if no processable task is found in the given queue. // Returns an encoded TaskMessage. // // Note: dequeueCmd checks whether a queue is paused first, before // calling RPOPLPUSH to pop a task from the queue. var dequeueCmd = redis.NewScript(` if redis.call("EXISTS", KEYS[2]) == 0 then local id = redis.call("RPOPLPUSH", KEYS[1], KEYS[3]) if id then local key = ARGV[2] .. id redis.call("HSET", key, "state", "active") redis.call("HDEL", key, "pending_since") redis.call("ZADD", KEYS[4], ARGV[1], id) return redis.call("HGET", key, "msg") end end return nil`) // Dequeue queries given queues in order and pops a task message // off a queue if one exists and returns the message and its lease expiration time. // Dequeue skips a queue if the queue is paused. // If all queues are empty, ErrNoProcessableTask error is returned. func (r *RDB) Dequeue(qnames ...string) (msg *base.TaskMessage, leaseExpirationTime time.Time, err error) { var op errors.Op = "rdb.Dequeue" for _, qname := range qnames { keys := []string{ base.PendingKey(qname), base.PausedKey(qname), base.ActiveKey(qname), base.LeaseKey(qname), } leaseExpirationTime = r.clock.Now().Add(LeaseDuration) argv := []interface{}{ leaseExpirationTime.Unix(), base.TaskKeyPrefix(qname), } res, err := dequeueCmd.Run(context.Background(), r.client, keys, argv...).Result() if err == redis.Nil { continue } else if err != nil { return nil, time.Time{}, errors.E(op, errors.Unknown, fmt.Sprintf("redis eval error: %v", err)) } encoded, err := cast.ToStringE(res) if err != nil { return nil, time.Time{}, errors.E(op, errors.Internal, fmt.Sprintf("cast error: unexpected return value from Lua script: %v", res)) } if msg, err = base.DecodeMessage([]byte(encoded)); err != nil { return nil, time.Time{}, errors.E(op, errors.Internal, fmt.Sprintf("cannot decode message: %v", err)) } return msg, leaseExpirationTime, nil } return nil, time.Time{}, errors.E(op, errors.NotFound, errors.ErrNoProcessableTask) } // KEYS[1] -> asynq:{}:active // KEYS[2] -> asynq:{}:lease // KEYS[3] -> asynq:{}:t: // KEYS[4] -> asynq:{}:processed: // KEYS[5] -> asynq:{}:processed // ------- // ARGV[1] -> task ID // ARGV[2] -> stats expiration timestamp // ARGV[3] -> max int64 value var doneCmd = redis.NewScript(` if redis.call("LREM", KEYS[1], 0, ARGV[1]) == 0 then return redis.error_reply("NOT FOUND") end if redis.call("ZREM", KEYS[2], ARGV[1]) == 0 then return redis.error_reply("NOT FOUND") end if redis.call("DEL", KEYS[3]) == 0 then return redis.error_reply("NOT FOUND") end local n = redis.call("INCR", KEYS[4]) if tonumber(n) == 1 then redis.call("EXPIREAT", KEYS[4], ARGV[2]) end local total = redis.call("GET", KEYS[5]) if tonumber(total) == tonumber(ARGV[3]) then redis.call("SET", KEYS[5], 1) else redis.call("INCR", KEYS[5]) end return redis.status_reply("OK") `) // KEYS[1] -> asynq:{}:active // KEYS[2] -> asynq:{}:lease // KEYS[3] -> asynq:{}:t: // KEYS[4] -> asynq:{}:processed: // KEYS[5] -> asynq:{}:processed // KEYS[6] -> unique key // ------- // ARGV[1] -> task ID // ARGV[2] -> stats expiration timestamp // ARGV[3] -> max int64 value var doneUniqueCmd = redis.NewScript(` if redis.call("LREM", KEYS[1], 0, ARGV[1]) == 0 then return redis.error_reply("NOT FOUND") end if redis.call("ZREM", KEYS[2], ARGV[1]) == 0 then return redis.error_reply("NOT FOUND") end if redis.call("DEL", KEYS[3]) == 0 then return redis.error_reply("NOT FOUND") end local n = redis.call("INCR", KEYS[4]) if tonumber(n) == 1 then redis.call("EXPIREAT", KEYS[4], ARGV[2]) end local total = redis.call("GET", KEYS[5]) if tonumber(total) == tonumber(ARGV[3]) then redis.call("SET", KEYS[5], 1) else redis.call("INCR", KEYS[5]) end if redis.call("GET", KEYS[6]) == ARGV[1] then redis.call("DEL", KEYS[6]) end return redis.status_reply("OK") `) // Done removes the task from active queue and deletes the task. // It removes a uniqueness lock acquired by the task, if any. func (r *RDB) Done(ctx context.Context, msg *base.TaskMessage) error { var op errors.Op = "rdb.Done" now := r.clock.Now() expireAt := now.Add(statsTTL) keys := []string{ base.ActiveKey(msg.Queue), base.LeaseKey(msg.Queue), base.TaskKey(msg.Queue, msg.ID), base.ProcessedKey(msg.Queue, now), base.ProcessedTotalKey(msg.Queue), } argv := []interface{}{ msg.ID, expireAt.Unix(), int64(math.MaxInt64), } // Note: We cannot pass empty unique key when running this script in redis-cluster. if len(msg.UniqueKey) > 0 { keys = append(keys, msg.UniqueKey) return r.runScript(ctx, op, doneUniqueCmd, keys, argv...) } return r.runScript(ctx, op, doneCmd, keys, argv...) } // KEYS[1] -> asynq:{}:active // KEYS[2] -> asynq:{}:lease // KEYS[3] -> asynq:{}:completed // KEYS[4] -> asynq:{}:t: // KEYS[5] -> asynq:{}:processed: // KEYS[6] -> asynq:{}:processed // // ARGV[1] -> task ID // ARGV[2] -> stats expiration timestamp // ARGV[3] -> task expiration time in unix time // ARGV[4] -> task message data // ARGV[5] -> max int64 value var markAsCompleteCmd = redis.NewScript(` if redis.call("LREM", KEYS[1], 0, ARGV[1]) == 0 then return redis.error_reply("NOT FOUND") end if redis.call("ZREM", KEYS[2], ARGV[1]) == 0 then return redis.error_reply("NOT FOUND") end if redis.call("ZADD", KEYS[3], ARGV[3], ARGV[1]) ~= 1 then return redis.error_reply("INTERNAL") end redis.call("HSET", KEYS[4], "msg", ARGV[4], "state", "completed") local n = redis.call("INCR", KEYS[5]) if tonumber(n) == 1 then redis.call("EXPIREAT", KEYS[5], ARGV[2]) end local total = redis.call("GET", KEYS[6]) if tonumber(total) == tonumber(ARGV[5]) then redis.call("SET", KEYS[6], 1) else redis.call("INCR", KEYS[6]) end return redis.status_reply("OK") `) // KEYS[1] -> asynq:{}:active // KEYS[2] -> asynq:{}:lease // KEYS[3] -> asynq:{}:completed // KEYS[4] -> asynq:{}:t: // KEYS[5] -> asynq:{}:processed: // KEYS[6] -> asynq:{}:processed // KEYS[7] -> asynq:{}:unique:{} // // ARGV[1] -> task ID // ARGV[2] -> stats expiration timestamp // ARGV[3] -> task expiration time in unix time // ARGV[4] -> task message data // ARGV[5] -> max int64 value var markAsCompleteUniqueCmd = redis.NewScript(` if redis.call("LREM", KEYS[1], 0, ARGV[1]) == 0 then return redis.error_reply("NOT FOUND") end if redis.call("ZREM", KEYS[2], ARGV[1]) == 0 then return redis.error_reply("NOT FOUND") end if redis.call("ZADD", KEYS[3], ARGV[3], ARGV[1]) ~= 1 then return redis.error_reply("INTERNAL") end redis.call("HSET", KEYS[4], "msg", ARGV[4], "state", "completed") local n = redis.call("INCR", KEYS[5]) if tonumber(n) == 1 then redis.call("EXPIREAT", KEYS[5], ARGV[2]) end local total = redis.call("GET", KEYS[6]) if tonumber(total) == tonumber(ARGV[5]) then redis.call("SET", KEYS[6], 1) else redis.call("INCR", KEYS[6]) end if redis.call("GET", KEYS[7]) == ARGV[1] then redis.call("DEL", KEYS[7]) end return redis.status_reply("OK") `) // MarkAsComplete removes the task from active queue to mark the task as completed. // It removes a uniqueness lock acquired by the task, if any. func (r *RDB) MarkAsComplete(ctx context.Context, msg *base.TaskMessage) error { var op errors.Op = "rdb.MarkAsComplete" now := r.clock.Now() statsExpireAt := now.Add(statsTTL) msg.CompletedAt = now.Unix() encoded, err := base.EncodeMessage(msg) if err != nil { return errors.E(op, errors.Unknown, fmt.Sprintf("cannot encode message: %v", err)) } keys := []string{ base.ActiveKey(msg.Queue), base.LeaseKey(msg.Queue), base.CompletedKey(msg.Queue), base.TaskKey(msg.Queue, msg.ID), base.ProcessedKey(msg.Queue, now), base.ProcessedTotalKey(msg.Queue), } argv := []interface{}{ msg.ID, statsExpireAt.Unix(), now.Unix() + msg.Retention, encoded, int64(math.MaxInt64), } // Note: We cannot pass empty unique key when running this script in redis-cluster. if len(msg.UniqueKey) > 0 { keys = append(keys, msg.UniqueKey) return r.runScript(ctx, op, markAsCompleteUniqueCmd, keys, argv...) } return r.runScript(ctx, op, markAsCompleteCmd, keys, argv...) } // KEYS[1] -> asynq:{}:active // KEYS[2] -> asynq:{}:lease // KEYS[3] -> asynq:{}:pending // KEYS[4] -> asynq:{}:t: // ARGV[1] -> task ID // Note: Use RPUSH to push to the head of the queue. var requeueCmd = redis.NewScript(` if redis.call("LREM", KEYS[1], 0, ARGV[1]) == 0 then return redis.error_reply("NOT FOUND") end if redis.call("ZREM", KEYS[2], ARGV[1]) == 0 then return redis.error_reply("NOT FOUND") end redis.call("RPUSH", KEYS[3], ARGV[1]) redis.call("HSET", KEYS[4], "state", "pending") return redis.status_reply("OK")`) // Requeue moves the task from active queue to the specified queue. func (r *RDB) Requeue(ctx context.Context, msg *base.TaskMessage) error { var op errors.Op = "rdb.Requeue" keys := []string{ base.ActiveKey(msg.Queue), base.LeaseKey(msg.Queue), base.PendingKey(msg.Queue), base.TaskKey(msg.Queue, msg.ID), } return r.runScript(ctx, op, requeueCmd, keys, msg.ID) } // KEYS[1] -> asynq:{}:t: // KEYS[2] -> asynq:{}:g: // KEYS[3] -> asynq:{}:groups // ------- // ARGV[1] -> task message data // ARGV[2] -> task ID // ARGV[3] -> current time in Unix time // ARGV[4] -> group key // // Output: // Returns 1 if successfully added // Returns 0 if task ID already exists var addToGroupCmd = redis.NewScript(` if redis.call("EXISTS", KEYS[1]) == 1 then return 0 end redis.call("HSET", KEYS[1], "msg", ARGV[1], "state", "aggregating", "group", ARGV[4]) redis.call("ZADD", KEYS[2], ARGV[3], ARGV[2]) redis.call("SADD", KEYS[3], ARGV[4]) return 1 `) func (r *RDB) AddToGroup(ctx context.Context, msg *base.TaskMessage, groupKey string) error { var op errors.Op = "rdb.AddToGroup" encoded, err := base.EncodeMessage(msg) if err != nil { return errors.E(op, errors.Unknown, fmt.Sprintf("cannot encode message: %v", err)) } if _, found := r.queuesPublished.Load(msg.Queue); !found { if err := r.client.SAdd(ctx, base.AllQueues, msg.Queue).Err(); err != nil { return errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: "sadd", Err: err}) } r.queuesPublished.Store(msg.Queue, true) } keys := []string{ base.TaskKey(msg.Queue, msg.ID), base.GroupKey(msg.Queue, groupKey), base.AllGroups(msg.Queue), } argv := []interface{}{ encoded, msg.ID, r.clock.Now().Unix(), groupKey, } n, err := r.runScriptWithErrorCode(ctx, op, addToGroupCmd, keys, argv...) if err != nil { return err } if n == 0 { return errors.E(op, errors.AlreadyExists, errors.ErrTaskIdConflict) } return nil } // KEYS[1] -> asynq:{}:t: // KEYS[2] -> asynq:{}:g: // KEYS[3] -> asynq:{}:groups // KEYS[4] -> unique key // ------- // ARGV[1] -> task message data // ARGV[2] -> task ID // ARGV[3] -> current time in Unix time // ARGV[4] -> group key // ARGV[5] -> uniqueness lock TTL // // Output: // Returns 1 if successfully added // Returns 0 if task ID already exists // Returns -1 if task unique key already exists var addToGroupUniqueCmd = redis.NewScript(` local ok = redis.call("SET", KEYS[4], ARGV[2], "NX", "EX", ARGV[5]) if not ok then return -1 end if redis.call("EXISTS", KEYS[1]) == 1 then return 0 end redis.call("HSET", KEYS[1], "msg", ARGV[1], "state", "aggregating", "group", ARGV[4]) redis.call("ZADD", KEYS[2], ARGV[3], ARGV[2]) redis.call("SADD", KEYS[3], ARGV[4]) return 1 `) func (r *RDB) AddToGroupUnique(ctx context.Context, msg *base.TaskMessage, groupKey string, ttl time.Duration) error { var op errors.Op = "rdb.AddToGroupUnique" encoded, err := base.EncodeMessage(msg) if err != nil { return errors.E(op, errors.Unknown, fmt.Sprintf("cannot encode message: %v", err)) } if _, found := r.queuesPublished.Load(msg.Queue); !found { if err := r.client.SAdd(ctx, base.AllQueues, msg.Queue).Err(); err != nil { return errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: "sadd", Err: err}) } r.queuesPublished.Store(msg.Queue, true) } keys := []string{ base.TaskKey(msg.Queue, msg.ID), base.GroupKey(msg.Queue, groupKey), base.AllGroups(msg.Queue), base.UniqueKey(msg.Queue, msg.Type, msg.Payload), } argv := []interface{}{ encoded, msg.ID, r.clock.Now().Unix(), groupKey, int(ttl.Seconds()), } n, err := r.runScriptWithErrorCode(ctx, op, addToGroupUniqueCmd, keys, argv...) if err != nil { return err } if n == -1 { return errors.E(op, errors.AlreadyExists, errors.ErrDuplicateTask) } if n == 0 { return errors.E(op, errors.AlreadyExists, errors.ErrTaskIdConflict) } return nil } // KEYS[1] -> asynq:{}:t: // KEYS[2] -> asynq:{}:scheduled // ------- // ARGV[1] -> task message data // ARGV[2] -> process_at time in Unix time // ARGV[3] -> task ID // // Output: // Returns 1 if successfully enqueued // Returns 0 if task ID already exists var scheduleCmd = redis.NewScript(` if redis.call("EXISTS", KEYS[1]) == 1 then return 0 end redis.call("HSET", KEYS[1], "msg", ARGV[1], "state", "scheduled") redis.call("ZADD", KEYS[2], ARGV[2], ARGV[3]) return 1 `) // Schedule adds the task to the scheduled set to be processed in the future. func (r *RDB) Schedule(ctx context.Context, msg *base.TaskMessage, processAt time.Time) error { var op errors.Op = "rdb.Schedule" encoded, err := base.EncodeMessage(msg) if err != nil { return errors.E(op, errors.Unknown, fmt.Sprintf("cannot encode message: %v", err)) } if _, found := r.queuesPublished.Load(msg.Queue); !found { if err := r.client.SAdd(ctx, base.AllQueues, msg.Queue).Err(); err != nil { return errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: "sadd", Err: err}) } r.queuesPublished.Store(msg.Queue, true) } keys := []string{ base.TaskKey(msg.Queue, msg.ID), base.ScheduledKey(msg.Queue), } argv := []interface{}{ encoded, processAt.Unix(), msg.ID, } n, err := r.runScriptWithErrorCode(ctx, op, scheduleCmd, keys, argv...) if err != nil { return err } if n == 0 { return errors.E(op, errors.AlreadyExists, errors.ErrTaskIdConflict) } return nil } // KEYS[1] -> unique key // KEYS[2] -> asynq:{}:t: // KEYS[3] -> asynq:{}:scheduled // ------- // ARGV[1] -> task ID // ARGV[2] -> uniqueness lock TTL // ARGV[3] -> score (process_at timestamp) // ARGV[4] -> task message // // Output: // Returns 1 if successfully scheduled // Returns 0 if task ID already exists // Returns -1 if task unique key already exists var scheduleUniqueCmd = redis.NewScript(` local ok = redis.call("SET", KEYS[1], ARGV[1], "NX", "EX", ARGV[2]) if not ok then return -1 end if redis.call("EXISTS", KEYS[2]) == 1 then return 0 end redis.call("HSET", KEYS[2], "msg", ARGV[4], "state", "scheduled", "unique_key", KEYS[1]) redis.call("ZADD", KEYS[3], ARGV[3], ARGV[1]) return 1 `) // ScheduleUnique adds the task to the backlog queue to be processed in the future if the uniqueness lock can be acquired. // It returns ErrDuplicateTask if the lock cannot be acquired. func (r *RDB) ScheduleUnique(ctx context.Context, msg *base.TaskMessage, processAt time.Time, ttl time.Duration) error { var op errors.Op = "rdb.ScheduleUnique" encoded, err := base.EncodeMessage(msg) if err != nil { return errors.E(op, errors.Internal, fmt.Sprintf("cannot encode task message: %v", err)) } if _, found := r.queuesPublished.Load(msg.Queue); !found { if err := r.client.SAdd(ctx, base.AllQueues, msg.Queue).Err(); err != nil { return errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: "sadd", Err: err}) } r.queuesPublished.Store(msg.Queue, true) } keys := []string{ msg.UniqueKey, base.TaskKey(msg.Queue, msg.ID), base.ScheduledKey(msg.Queue), } argv := []interface{}{ msg.ID, int(ttl.Seconds()), processAt.Unix(), encoded, } n, err := r.runScriptWithErrorCode(ctx, op, scheduleUniqueCmd, keys, argv...) if err != nil { return err } if n == -1 { return errors.E(op, errors.AlreadyExists, errors.ErrDuplicateTask) } if n == 0 { return errors.E(op, errors.AlreadyExists, errors.ErrTaskIdConflict) } return nil } // KEYS[1] -> asynq:{}:t: // KEYS[2] -> asynq:{}:active // KEYS[3] -> asynq:{}:lease // KEYS[4] -> asynq:{}:retry // KEYS[5] -> asynq:{}:processed: // KEYS[6] -> asynq:{}:failed: // KEYS[7] -> asynq:{}:processed // KEYS[8] -> asynq:{}:failed // ------- // ARGV[1] -> task ID // ARGV[2] -> updated base.TaskMessage value // ARGV[3] -> retry_at UNIX timestamp // ARGV[4] -> stats expiration timestamp // ARGV[5] -> is_failure (bool) // ARGV[6] -> max int64 value var retryCmd = redis.NewScript(` if redis.call("LREM", KEYS[2], 0, ARGV[1]) == 0 then return redis.error_reply("NOT FOUND") end if redis.call("ZREM", KEYS[3], ARGV[1]) == 0 then return redis.error_reply("NOT FOUND") end redis.call("ZADD", KEYS[4], ARGV[3], ARGV[1]) redis.call("HSET", KEYS[1], "msg", ARGV[2], "state", "retry") if tonumber(ARGV[5]) == 1 then local n = redis.call("INCR", KEYS[5]) if tonumber(n) == 1 then redis.call("EXPIREAT", KEYS[5], ARGV[4]) end local m = redis.call("INCR", KEYS[6]) if tonumber(m) == 1 then redis.call("EXPIREAT", KEYS[6], ARGV[4]) end local total = redis.call("GET", KEYS[7]) if tonumber(total) == tonumber(ARGV[6]) then redis.call("SET", KEYS[7], 1) redis.call("SET", KEYS[8], 1) else redis.call("INCR", KEYS[7]) redis.call("INCR", KEYS[8]) end end return redis.status_reply("OK")`) // Retry moves the task from active to retry queue. // It also annotates the message with the given error message and // if isFailure is true increments the retried counter. func (r *RDB) Retry(ctx context.Context, msg *base.TaskMessage, processAt time.Time, errMsg string, isFailure bool) error { var op errors.Op = "rdb.Retry" now := r.clock.Now() modified := *msg if isFailure { modified.Retried++ } modified.ErrorMsg = errMsg modified.LastFailedAt = now.Unix() encoded, err := base.EncodeMessage(&modified) if err != nil { return errors.E(op, errors.Internal, fmt.Sprintf("cannot encode message: %v", err)) } expireAt := now.Add(statsTTL) keys := []string{ base.TaskKey(msg.Queue, msg.ID), base.ActiveKey(msg.Queue), base.LeaseKey(msg.Queue), base.RetryKey(msg.Queue), base.ProcessedKey(msg.Queue, now), base.FailedKey(msg.Queue, now), base.ProcessedTotalKey(msg.Queue), base.FailedTotalKey(msg.Queue), } argv := []interface{}{ msg.ID, encoded, processAt.Unix(), expireAt.Unix(), isFailure, int64(math.MaxInt64), } return r.runScript(ctx, op, retryCmd, keys, argv...) } const ( maxArchiveSize = 10000 // maximum number of tasks in archive archivedExpirationInDays = 90 // number of days before an archived task gets deleted permanently ) // KEYS[1] -> asynq:{}:t: // KEYS[2] -> asynq:{}:active // KEYS[3] -> asynq:{}:lease // KEYS[4] -> asynq:{}:archived // KEYS[5] -> asynq:{}:processed: // KEYS[6] -> asynq:{}:failed: // KEYS[7] -> asynq:{}:processed // KEYS[8] -> asynq:{}:failed // KEYS[9] -> asynq:{}:t: // ------- // ARGV[1] -> task ID // ARGV[2] -> updated base.TaskMessage value // ARGV[3] -> died_at UNIX timestamp // ARGV[4] -> cutoff timestamp (e.g., 90 days ago) // ARGV[5] -> max number of tasks in archive (e.g., 100) // ARGV[6] -> stats expiration timestamp // ARGV[7] -> max int64 value var archiveCmd = redis.NewScript(` if redis.call("LREM", KEYS[2], 0, ARGV[1]) == 0 then return redis.error_reply("NOT FOUND") end if redis.call("ZREM", KEYS[3], ARGV[1]) == 0 then return redis.error_reply("NOT FOUND") end redis.call("ZADD", KEYS[4], ARGV[3], ARGV[1]) local old = redis.call("ZRANGE", KEYS[4], "-inf", ARGV[4], "BYSCORE") if #old > 0 then for _, id in ipairs(old) do redis.call("DEL", KEYS[9] .. id) end redis.call("ZREM", KEYS[4], unpack(old)) end local extra = redis.call("ZRANGE", KEYS[4], 0, -ARGV[5]) if #extra > 0 then for _, id in ipairs(extra) do redis.call("DEL", KEYS[9] .. id) end redis.call("ZREM", KEYS[4], unpack(extra)) end redis.call("HSET", KEYS[1], "msg", ARGV[2], "state", "archived") local n = redis.call("INCR", KEYS[5]) if tonumber(n) == 1 then redis.call("EXPIREAT", KEYS[5], ARGV[6]) end local m = redis.call("INCR", KEYS[6]) if tonumber(m) == 1 then redis.call("EXPIREAT", KEYS[6], ARGV[6]) end local total = redis.call("GET", KEYS[7]) if tonumber(total) == tonumber(ARGV[7]) then redis.call("SET", KEYS[7], 1) redis.call("SET", KEYS[8], 1) else redis.call("INCR", KEYS[7]) redis.call("INCR", KEYS[8]) end return redis.status_reply("OK")`) // Archive sends the given task to archive, attaching the error message to the task. // It also trims the archive by timestamp and set size. func (r *RDB) Archive(ctx context.Context, msg *base.TaskMessage, errMsg string) error { var op errors.Op = "rdb.Archive" now := r.clock.Now() modified := *msg modified.ErrorMsg = errMsg modified.LastFailedAt = now.Unix() encoded, err := base.EncodeMessage(&modified) if err != nil { return errors.E(op, errors.Internal, fmt.Sprintf("cannot encode message: %v", err)) } cutoff := now.AddDate(0, 0, -archivedExpirationInDays) expireAt := now.Add(statsTTL) keys := []string{ base.TaskKey(msg.Queue, msg.ID), base.ActiveKey(msg.Queue), base.LeaseKey(msg.Queue), base.ArchivedKey(msg.Queue), base.ProcessedKey(msg.Queue, now), base.FailedKey(msg.Queue, now), base.ProcessedTotalKey(msg.Queue), base.FailedTotalKey(msg.Queue), base.TaskKeyPrefix(msg.Queue), } argv := []interface{}{ msg.ID, encoded, now.Unix(), cutoff.Unix(), maxArchiveSize, expireAt.Unix(), int64(math.MaxInt64), } return r.runScript(ctx, op, archiveCmd, keys, argv...) } // ForwardIfReady checks scheduled and retry sets of the given queues // and move any tasks that are ready to be processed to the pending set. func (r *RDB) ForwardIfReady(qnames ...string) error { var op errors.Op = "rdb.ForwardIfReady" for _, qname := range qnames { if err := r.forwardAll(qname); err != nil { return errors.E(op, errors.CanonicalCode(err), err) } } return nil } // KEYS[1] -> source queue (e.g. asynq:{:scheduled or asynq:{}:retry}) // KEYS[2] -> asynq:{}:pending // ARGV[1] -> current unix time in seconds // ARGV[2] -> task key prefix // ARGV[3] -> current unix time in nsec // ARGV[4] -> group key prefix // Note: Script moves tasks up to 100 at a time to keep the runtime of script short. var forwardCmd = redis.NewScript(` local ids = redis.call("ZRANGEBYSCORE", KEYS[1], "-inf", ARGV[1], "LIMIT", 0, 100) for _, id in ipairs(ids) do local taskKey = ARGV[2] .. id local group = redis.call("HGET", taskKey, "group") if group and group ~= '' then redis.call("ZADD", ARGV[4] .. group, ARGV[1], id) redis.call("ZREM", KEYS[1], id) redis.call("HSET", taskKey, "state", "aggregating") else redis.call("LPUSH", KEYS[2], id) redis.call("ZREM", KEYS[1], id) redis.call("HSET", taskKey, "state", "pending", "pending_since", ARGV[3]) end end return table.getn(ids)`) // forward moves tasks with a score less than the current unix time from the delayed (i.e. scheduled | retry) zset // to the pending list or group set. // It returns the number of tasks moved. func (r *RDB) forward(delayedKey, pendingKey, taskKeyPrefix, groupKeyPrefix string) (int, error) { now := r.clock.Now() keys := []string{delayedKey, pendingKey} argv := []interface{}{ now.Unix(), taskKeyPrefix, now.UnixNano(), groupKeyPrefix, } res, err := forwardCmd.Run(context.Background(), r.client, keys, argv...).Result() if err != nil { return 0, errors.E(errors.Internal, fmt.Sprintf("redis eval error: %v", err)) } n, err := cast.ToIntE(res) if err != nil { return 0, errors.E(errors.Internal, fmt.Sprintf("cast error: Lua script returned unexpected value: %v", res)) } return n, nil } // forwardAll checks for tasks in scheduled/retry state that are ready to be run, and updates // their state to "pending" or "aggregating". func (r *RDB) forwardAll(qname string) (err error) { delayedKeys := []string{base.ScheduledKey(qname), base.RetryKey(qname)} pendingKey := base.PendingKey(qname) taskKeyPrefix := base.TaskKeyPrefix(qname) groupKeyPrefix := base.GroupKeyPrefix(qname) for _, delayedKey := range delayedKeys { n := 1 for n != 0 { n, err = r.forward(delayedKey, pendingKey, taskKeyPrefix, groupKeyPrefix) if err != nil { return err } } } return nil } // ListGroups returns a list of all known groups in the given queue. func (r *RDB) ListGroups(qname string) ([]string, error) { var op errors.Op = "RDB.ListGroups" groups, err := r.client.SMembers(context.Background(), base.AllGroups(qname)).Result() if err != nil { return nil, errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: "smembers", Err: err}) } return groups, nil } // aggregationCheckCmd checks the given group for whether to create an aggregation set. // An aggregation set is created if one of the aggregation criteria is met: // 1) group has reached or exceeded its max size // 2) group's oldest task has reached or exceeded its max delay // 3) group's latest task has reached or exceeded its grace period // if aggreation criteria is met, the command moves those tasks from the group // and put them in an aggregation set. Additionally, if the creation of aggregation set // empties the group, it will clear the group name from the all groups set. // // KEYS[1] -> asynq:{}:g: // KEYS[2] -> asynq:{}:g:: // KEYS[3] -> asynq:{}:aggregation_sets // KEYS[4] -> asynq:{}:groups // ------- // ARGV[1] -> max group size // ARGV[2] -> max group delay in unix time // ARGV[3] -> start time of the grace period // ARGV[4] -> aggregation set expire time // ARGV[5] -> current time in unix time // ARGV[6] -> group name // // Output: // Returns 0 if no aggregation set was created // Returns 1 if an aggregation set was created // // Time Complexity: // O(log(N) + M) with N being the number tasks in the group zset // and M being the max size. var aggregationCheckCmd = redis.NewScript(` local size = redis.call("ZCARD", KEYS[1]) if size == 0 then return 0 end local maxSize = tonumber(ARGV[1]) if maxSize ~= 0 and size >= maxSize then local res = redis.call("ZRANGE", KEYS[1], 0, maxSize-1, "WITHSCORES") for i=1, table.getn(res)-1, 2 do redis.call("ZADD", KEYS[2], tonumber(res[i+1]), res[i]) end redis.call("ZREMRANGEBYRANK", KEYS[1], 0, maxSize-1) redis.call("ZADD", KEYS[3], ARGV[4], KEYS[2]) if size == maxSize then redis.call("SREM", KEYS[4], ARGV[6]) end return 1 end local maxDelay = tonumber(ARGV[2]) local currentTime = tonumber(ARGV[5]) if maxDelay ~= 0 then local oldestEntry = redis.call("ZRANGE", KEYS[1], 0, 0, "WITHSCORES") local oldestEntryScore = tonumber(oldestEntry[2]) local maxDelayTime = currentTime - maxDelay if oldestEntryScore <= maxDelayTime then local res = redis.call("ZRANGE", KEYS[1], 0, maxSize-1, "WITHSCORES") for i=1, table.getn(res)-1, 2 do redis.call("ZADD", KEYS[2], tonumber(res[i+1]), res[i]) end redis.call("ZREMRANGEBYRANK", KEYS[1], 0, maxSize-1) redis.call("ZADD", KEYS[3], ARGV[4], KEYS[2]) if size <= maxSize or maxSize == 0 then redis.call("SREM", KEYS[4], ARGV[6]) end return 1 end end local latestEntry = redis.call("ZREVRANGE", KEYS[1], 0, 0, "WITHSCORES") local latestEntryScore = tonumber(latestEntry[2]) local gracePeriodStartTime = currentTime - tonumber(ARGV[3]) if latestEntryScore <= gracePeriodStartTime then local res = redis.call("ZRANGE", KEYS[1], 0, maxSize-1, "WITHSCORES") for i=1, table.getn(res)-1, 2 do redis.call("ZADD", KEYS[2], tonumber(res[i+1]), res[i]) end redis.call("ZREMRANGEBYRANK", KEYS[1], 0, maxSize-1) redis.call("ZADD", KEYS[3], ARGV[4], KEYS[2]) if size <= maxSize or maxSize == 0 then redis.call("SREM", KEYS[4], ARGV[6]) end return 1 end return 0 `) // Task aggregation should finish within this timeout. // Otherwise an aggregation set should be reclaimed by the recoverer. const aggregationTimeout = 2 * time.Minute // AggregationCheck checks the group identified by the given queue and group name to see if the tasks in the // group are ready to be aggregated. If so, it moves the tasks to be aggregated to a aggregation set and returns // the set ID. If not, it returns an empty string for the set ID. // The time for gracePeriod and maxDelay is computed relative to the time t. // // Note: It assumes that this function is called at frequency less than or equal to the gracePeriod. In other words, // the function only checks the most recently added task against the given gracePeriod. func (r *RDB) AggregationCheck(qname, gname string, t time.Time, gracePeriod, maxDelay time.Duration, maxSize int) (string, error) { var op errors.Op = "RDB.AggregationCheck" aggregationSetID := uuid.NewString() expireTime := r.clock.Now().Add(aggregationTimeout) keys := []string{ base.GroupKey(qname, gname), base.AggregationSetKey(qname, gname, aggregationSetID), base.AllAggregationSets(qname), base.AllGroups(qname), } argv := []interface{}{ maxSize, int64(maxDelay.Seconds()), int64(gracePeriod.Seconds()), expireTime.Unix(), t.Unix(), gname, } n, err := r.runScriptWithErrorCode(context.Background(), op, aggregationCheckCmd, keys, argv...) if err != nil { return "", err } switch n { case 0: return "", nil case 1: return aggregationSetID, nil default: return "", errors.E(op, errors.Internal, fmt.Sprintf("unexpected return value from lua script: %d", n)) } } // KEYS[1] -> asynq:{}:g:: // ------ // ARGV[1] -> task key prefix // // Output: // Array of encoded task messages // // Time Complexity: // O(N) with N being the number of tasks in the aggregation set. var readAggregationSetCmd = redis.NewScript(` local msgs = {} local ids = redis.call("ZRANGE", KEYS[1], 0, -1) for _, id in ipairs(ids) do local key = ARGV[1] .. id table.insert(msgs, redis.call("HGET", key, "msg")) end return msgs `) // ReadAggregationSet retrieves members of an aggregation set and returns a list of tasks in the set and // the deadline for aggregating those tasks. func (r *RDB) ReadAggregationSet(qname, gname, setID string) ([]*base.TaskMessage, time.Time, error) { var op errors.Op = "RDB.ReadAggregationSet" ctx := context.Background() aggSetKey := base.AggregationSetKey(qname, gname, setID) res, err := readAggregationSetCmd.Run(ctx, r.client, []string{aggSetKey}, base.TaskKeyPrefix(qname)).Result() if err != nil { return nil, time.Time{}, errors.E(op, errors.Unknown, fmt.Sprintf("redis eval error: %v", err)) } data, err := cast.ToStringSliceE(res) if err != nil { return nil, time.Time{}, errors.E(op, errors.Internal, fmt.Sprintf("cast error: Lua script returned unexpected value: %v", res)) } var msgs []*base.TaskMessage for _, s := range data { msg, err := base.DecodeMessage([]byte(s)) if err != nil { return nil, time.Time{}, errors.E(op, errors.Internal, fmt.Sprintf("cannot decode message: %v", err)) } msgs = append(msgs, msg) } deadlineUnix, err := r.client.ZScore(ctx, base.AllAggregationSets(qname), aggSetKey).Result() if err != nil { return nil, time.Time{}, errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: "zscore", Err: err}) } return msgs, time.Unix(int64(deadlineUnix), 0), nil } // KEYS[1] -> asynq:{}:g:: // KEYS[2] -> asynq:{}:aggregation_sets // ------- // ARGV[1] -> task key prefix // // Output: // Redis status reply // // Time Complexity: // max(O(N), O(log(M))) with N being the number of tasks in the aggregation set // and M being the number of elements in the all-aggregation-sets list. var deleteAggregationSetCmd = redis.NewScript(` local ids = redis.call("ZRANGE", KEYS[1], 0, -1) for _, id in ipairs(ids) do redis.call("DEL", ARGV[1] .. id) end redis.call("DEL", KEYS[1]) redis.call("ZREM", KEYS[2], KEYS[1]) return redis.status_reply("OK") `) // DeleteAggregationSet deletes the aggregation set and its members identified by the parameters. func (r *RDB) DeleteAggregationSet(ctx context.Context, qname, gname, setID string) error { var op errors.Op = "RDB.DeleteAggregationSet" keys := []string{ base.AggregationSetKey(qname, gname, setID), base.AllAggregationSets(qname), } return r.runScript(ctx, op, deleteAggregationSetCmd, keys, base.TaskKeyPrefix(qname)) } // KEYS[1] -> asynq:{}:aggregation_sets // ------- // ARGV[1] -> current time in unix time var reclaimStateAggregationSetsCmd = redis.NewScript(` local staleSetKeys = redis.call("ZRANGEBYSCORE", KEYS[1], "-inf", ARGV[1]) for _, key in ipairs(staleSetKeys) do local idx = string.find(key, ":[^:]*$") local groupKey = string.sub(key, 1, idx-1) local res = redis.call("ZRANGE", key, 0, -1, "WITHSCORES") for i=1, table.getn(res)-1, 2 do redis.call("ZADD", groupKey, tonumber(res[i+1]), res[i]) end redis.call("DEL", key) end redis.call("ZREMRANGEBYSCORE", KEYS[1], "-inf", ARGV[1]) return redis.status_reply("OK") `) // ReclaimStaleAggregationSets checks for any stale aggregation sets in the given queue, and // reclaim tasks in the stale aggregation set by putting them back in the group. func (r *RDB) ReclaimStaleAggregationSets(qname string) error { var op errors.Op = "RDB.ReclaimStaleAggregationSets" return r.runScript(context.Background(), op, reclaimStateAggregationSetsCmd, []string{base.AllAggregationSets(qname)}, r.clock.Now().Unix()) } // KEYS[1] -> asynq:{}:completed // ARGV[1] -> current time in unix time // ARGV[2] -> task key prefix // ARGV[3] -> batch size (i.e. maximum number of tasks to delete) // // Returns the number of tasks deleted. var deleteExpiredCompletedTasksCmd = redis.NewScript(` local ids = redis.call("ZRANGEBYSCORE", KEYS[1], "-inf", ARGV[1], "LIMIT", 0, tonumber(ARGV[3])) for _, id in ipairs(ids) do redis.call("DEL", ARGV[2] .. id) redis.call("ZREM", KEYS[1], id) end return table.getn(ids)`) // DeleteExpiredCompletedTasks checks for any expired tasks in the given queue's completed set, // and delete all expired tasks. func (r *RDB) DeleteExpiredCompletedTasks(qname string, batchSize int) error { for { n, err := r.deleteExpiredCompletedTasks(qname, batchSize) if err != nil { return err } if n == 0 { return nil } } } // deleteExpiredCompletedTasks runs the lua script to delete expired deleted task with the specified // batch size. It reports the number of tasks deleted. func (r *RDB) deleteExpiredCompletedTasks(qname string, batchSize int) (int64, error) { var op errors.Op = "rdb.DeleteExpiredCompletedTasks" keys := []string{base.CompletedKey(qname)} argv := []interface{}{ r.clock.Now().Unix(), base.TaskKeyPrefix(qname), batchSize, } res, err := deleteExpiredCompletedTasksCmd.Run(context.Background(), r.client, keys, argv...).Result() if err != nil { return 0, errors.E(op, errors.Internal, fmt.Sprintf("redis eval error: %v", err)) } n, ok := res.(int64) if !ok { return 0, errors.E(op, errors.Internal, fmt.Sprintf("unexpected return value from Lua script: %v", res)) } return n, nil } // KEYS[1] -> asynq:{}:lease // ARGV[1] -> cutoff in unix time // ARGV[2] -> task key prefix var listLeaseExpiredCmd = redis.NewScript(` local res = {} local ids = redis.call("ZRANGEBYSCORE", KEYS[1], "-inf", ARGV[1]) for _, id in ipairs(ids) do local key = ARGV[2] .. id local v = redis.call("HGET", key, "msg") if v then table.insert(res, v) end end return res `) // ListLeaseExpired returns a list of task messages with an expired lease from the given queues. func (r *RDB) ListLeaseExpired(cutoff time.Time, qnames ...string) ([]*base.TaskMessage, error) { var op errors.Op = "rdb.ListLeaseExpired" var msgs []*base.TaskMessage for _, qname := range qnames { res, err := listLeaseExpiredCmd.Run(context.Background(), r.client, []string{base.LeaseKey(qname)}, cutoff.Unix(), base.TaskKeyPrefix(qname)).Result() if err != nil { return nil, errors.E(op, errors.Internal, fmt.Sprintf("redis eval error: %v", err)) } data, err := cast.ToStringSliceE(res) if err != nil { return nil, errors.E(op, errors.Internal, fmt.Sprintf("cast error: Lua script returned unexpected value: %v", res)) } for _, s := range data { msg, err := base.DecodeMessage([]byte(s)) if err != nil { return nil, errors.E(op, errors.Internal, fmt.Sprintf("cannot decode message: %v", err)) } msgs = append(msgs, msg) } } return msgs, nil } // ExtendLease extends the lease for the given tasks by LeaseDuration (30s). // It returns a new expiration time if the operation was successful. func (r *RDB) ExtendLease(qname string, ids ...string) (expirationTime time.Time, err error) { expireAt := r.clock.Now().Add(LeaseDuration) var zs []redis.Z for _, id := range ids { zs = append(zs, redis.Z{Member: id, Score: float64(expireAt.Unix())}) } // Use XX option to only update elements that already exist; Don't add new elements // TODO: Consider adding GT option to ensure we only "extend" the lease. Ceveat is that GT is supported from redis v6.2.0 or above. err = r.client.ZAddXX(context.Background(), base.LeaseKey(qname), zs...).Err() if err != nil { return time.Time{}, err } return expireAt, nil } // KEYS[1] -> asynq:servers:{} // KEYS[2] -> asynq:workers:{} // ARGV[1] -> TTL in seconds // ARGV[2] -> server info // ARGV[3:] -> alternate key-value pair of (worker id, worker data) // Note: Add key to ZSET with expiration time as score. // ref: https://github.com/antirez/redis/issues/135#issuecomment-2361996 var writeServerStateCmd = redis.NewScript(` redis.call("SETEX", KEYS[1], ARGV[1], ARGV[2]) redis.call("DEL", KEYS[2]) for i = 3, table.getn(ARGV)-1, 2 do redis.call("HSET", KEYS[2], ARGV[i], ARGV[i+1]) end redis.call("EXPIRE", KEYS[2], ARGV[1]) return redis.status_reply("OK")`) // WriteServerState writes server state data to redis with expiration set to the value ttl. func (r *RDB) WriteServerState(info *base.ServerInfo, workers []*base.WorkerInfo, ttl time.Duration) error { var op errors.Op = "rdb.WriteServerState" ctx := context.Background() bytes, err := base.EncodeServerInfo(info) if err != nil { return errors.E(op, errors.Internal, fmt.Sprintf("cannot encode server info: %v", err)) } exp := r.clock.Now().Add(ttl).UTC() args := []interface{}{ttl.Seconds(), bytes} // args to the lua script for _, w := range workers { bytes, err := base.EncodeWorkerInfo(w) if err != nil { continue // skip bad data } args = append(args, w.ID, bytes) } skey := base.ServerInfoKey(info.Host, info.PID, info.ServerID) wkey := base.WorkersKey(info.Host, info.PID, info.ServerID) if err := r.client.ZAdd(ctx, base.AllServers, redis.Z{Score: float64(exp.Unix()), Member: skey}).Err(); err != nil { return errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: "sadd", Err: err}) } if err := r.client.ZAdd(ctx, base.AllWorkers, redis.Z{Score: float64(exp.Unix()), Member: wkey}).Err(); err != nil { return errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: "zadd", Err: err}) } return r.runScript(ctx, op, writeServerStateCmd, []string{skey, wkey}, args...) } // KEYS[1] -> asynq:servers:{} // KEYS[2] -> asynq:workers:{} var clearServerStateCmd = redis.NewScript(` redis.call("DEL", KEYS[1]) redis.call("DEL", KEYS[2]) return redis.status_reply("OK")`) // ClearServerState deletes server state data from redis. func (r *RDB) ClearServerState(host string, pid int, serverID string) error { var op errors.Op = "rdb.ClearServerState" ctx := context.Background() skey := base.ServerInfoKey(host, pid, serverID) wkey := base.WorkersKey(host, pid, serverID) if err := r.client.ZRem(ctx, base.AllServers, skey).Err(); err != nil { return errors.E(op, errors.Internal, &errors.RedisCommandError{Command: "zrem", Err: err}) } if err := r.client.ZRem(ctx, base.AllWorkers, wkey).Err(); err != nil { return errors.E(op, errors.Internal, &errors.RedisCommandError{Command: "zrem", Err: err}) } return r.runScript(ctx, op, clearServerStateCmd, []string{skey, wkey}) } // KEYS[1] -> asynq:schedulers:{} // ARGV[1] -> TTL in seconds // ARGV[2:] -> scheduler entries var writeSchedulerEntriesCmd = redis.NewScript(` redis.call("DEL", KEYS[1]) for i = 2, #ARGV do redis.call("LPUSH", KEYS[1], ARGV[i]) end redis.call("EXPIRE", KEYS[1], ARGV[1]) return redis.status_reply("OK")`) // WriteSchedulerEntries writes scheduler entries data to redis with expiration set to the value ttl. func (r *RDB) WriteSchedulerEntries(schedulerID string, entries []*base.SchedulerEntry, ttl time.Duration) error { var op errors.Op = "rdb.WriteSchedulerEntries" ctx := context.Background() args := []interface{}{ttl.Seconds()} for _, e := range entries { bytes, err := base.EncodeSchedulerEntry(e) if err != nil { continue // skip bad data } args = append(args, bytes) } exp := r.clock.Now().Add(ttl).UTC() key := base.SchedulerEntriesKey(schedulerID) err := r.client.ZAdd(ctx, base.AllSchedulers, redis.Z{Score: float64(exp.Unix()), Member: key}).Err() if err != nil { return errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: "zadd", Err: err}) } return r.runScript(ctx, op, writeSchedulerEntriesCmd, []string{key}, args...) } // ClearSchedulerEntries deletes scheduler entries data from redis. func (r *RDB) ClearSchedulerEntries(schedulerID string) error { var op errors.Op = "rdb.ClearSchedulerEntries" ctx := context.Background() key := base.SchedulerEntriesKey(schedulerID) if err := r.client.ZRem(ctx, base.AllSchedulers, key).Err(); err != nil { return errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: "zrem", Err: err}) } if err := r.client.Del(ctx, key).Err(); err != nil { return errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: "del", Err: err}) } return nil } // CancelationPubSub returns a pubsub for cancelation messages. func (r *RDB) CancelationPubSub() (*redis.PubSub, error) { var op errors.Op = "rdb.CancelationPubSub" ctx := context.Background() pubsub := r.client.Subscribe(ctx, base.CancelChannel) _, err := pubsub.Receive(ctx) if err != nil { return nil, errors.E(op, errors.Unknown, fmt.Sprintf("redis pubsub receive error: %v", err)) } return pubsub, nil } // PublishCancelation publish cancelation message to all subscribers. // The message is the ID for the task to be canceled. func (r *RDB) PublishCancelation(id string) error { var op errors.Op = "rdb.PublishCancelation" ctx := context.Background() if err := r.client.Publish(ctx, base.CancelChannel, id).Err(); err != nil { return errors.E(op, errors.Unknown, fmt.Sprintf("redis pubsub publish error: %v", err)) } return nil } // KEYS[1] -> asynq:scheduler_history: // ARGV[1] -> enqueued_at timestamp // ARGV[2] -> serialized SchedulerEnqueueEvent data // ARGV[3] -> max number of events to be persisted var recordSchedulerEnqueueEventCmd = redis.NewScript(` redis.call("ZREMRANGEBYRANK", KEYS[1], 0, -ARGV[3]) redis.call("ZADD", KEYS[1], ARGV[1], ARGV[2]) return redis.status_reply("OK")`) // Maximum number of enqueue events to store per entry. const maxEvents = 1000 // RecordSchedulerEnqueueEvent records the time when the given task was enqueued. func (r *RDB) RecordSchedulerEnqueueEvent(entryID string, event *base.SchedulerEnqueueEvent) error { var op errors.Op = "rdb.RecordSchedulerEnqueueEvent" ctx := context.Background() data, err := base.EncodeSchedulerEnqueueEvent(event) if err != nil { return errors.E(op, errors.Internal, fmt.Sprintf("cannot encode scheduler enqueue event: %v", err)) } keys := []string{ base.SchedulerHistoryKey(entryID), } argv := []interface{}{ event.EnqueuedAt.Unix(), data, maxEvents, } return r.runScript(ctx, op, recordSchedulerEnqueueEventCmd, keys, argv...) } // ClearSchedulerHistory deletes the enqueue event history for the given scheduler entry. func (r *RDB) ClearSchedulerHistory(entryID string) error { var op errors.Op = "rdb.ClearSchedulerHistory" ctx := context.Background() key := base.SchedulerHistoryKey(entryID) if err := r.client.Del(ctx, key).Err(); err != nil { return errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: "del", Err: err}) } return nil } // WriteResult writes the given result data for the specified task. func (r *RDB) WriteResult(qname, taskID string, data []byte) (int, error) { var op errors.Op = "rdb.WriteResult" ctx := context.Background() taskKey := base.TaskKey(qname, taskID) if err := r.client.HSet(ctx, taskKey, "result", data).Err(); err != nil { return 0, errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: "hset", Err: err}) } return len(data), nil } ================================================ FILE: internal/rdb/rdb_test.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package rdb import ( "context" "encoding/json" "flag" "math" "strconv" "strings" "sync" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/google/uuid" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/errors" h "github.com/hibiken/asynq/internal/testutil" "github.com/hibiken/asynq/internal/timeutil" "github.com/redis/go-redis/v9" ) // variables used for package testing. var ( redisAddr string redisDB int useRedisCluster bool redisClusterAddrs string // comma-separated list of host:port ) func init() { flag.StringVar(&redisAddr, "redis_addr", "localhost:6379", "redis address to use in testing") flag.IntVar(&redisDB, "redis_db", 15, "redis db number to use in testing") flag.BoolVar(&useRedisCluster, "redis_cluster", false, "use redis cluster as a broker in testing") flag.StringVar(&redisClusterAddrs, "redis_cluster_addrs", "localhost:7000,localhost:7001,localhost:7002", "comma separated list of redis server addresses") } func setup(tb testing.TB) (r *RDB) { tb.Helper() if useRedisCluster { addrs := strings.Split(redisClusterAddrs, ",") if len(addrs) == 0 { tb.Fatal("No redis cluster addresses provided. Please set addresses using --redis_cluster_addrs flag.") } r = NewRDB(redis.NewClusterClient(&redis.ClusterOptions{ Addrs: addrs, })) } else { r = NewRDB(redis.NewClient(&redis.Options{ Addr: redisAddr, DB: redisDB, })) } // Start each test with a clean slate. h.FlushDB(tb, r.client) return r } func TestEnqueue(t *testing.T) { r := setup(t) defer r.Close() t1 := h.NewTaskMessage("send_email", h.JSON(map[string]interface{}{"to": "exampleuser@gmail.com", "from": "noreply@example.com"})) t2 := h.NewTaskMessageWithQueue("generate_csv", h.JSON(map[string]interface{}{}), "csv") t3 := h.NewTaskMessageWithQueue("sync", nil, "low") enqueueTime := time.Now() r.SetClock(timeutil.NewSimulatedClock(enqueueTime)) tests := []struct { msg *base.TaskMessage }{ {t1}, {t2}, {t3}, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case. err := r.Enqueue(context.Background(), tc.msg) if err != nil { t.Errorf("(*RDB).Enqueue(msg) = %v, want nil", err) continue } // Check Pending list has task ID. pendingKey := base.PendingKey(tc.msg.Queue) pendingIDs := r.client.LRange(context.Background(), pendingKey, 0, -1).Val() if n := len(pendingIDs); n != 1 { t.Errorf("Redis LIST %q contains %d IDs, want 1", pendingKey, n) continue } if pendingIDs[0] != tc.msg.ID { t.Errorf("Redis LIST %q: got %v, want %v", pendingKey, pendingIDs, []string{tc.msg.ID}) continue } // Check the value under the task key. taskKey := base.TaskKey(tc.msg.Queue, tc.msg.ID) encoded := r.client.HGet(context.Background(), taskKey, "msg").Val() // "msg" field decoded := h.MustUnmarshal(t, encoded) if diff := cmp.Diff(tc.msg, decoded); diff != "" { t.Errorf("persisted message was %v, want %v; (-want, +got)\n%s", decoded, tc.msg, diff) } state := r.client.HGet(context.Background(), taskKey, "state").Val() // "state" field if state != "pending" { t.Errorf("state field under task-key is set to %q, want %q", state, "pending") } pendingSince := r.client.HGet(context.Background(), taskKey, "pending_since").Val() // "pending_since" field if want := strconv.Itoa(int(enqueueTime.UnixNano())); pendingSince != want { t.Errorf("pending_since field under task-key is set to %v, want %v", pendingSince, want) } // Check queue is in the AllQueues set. if !r.client.SIsMember(context.Background(), base.AllQueues, tc.msg.Queue).Val() { t.Errorf("%q is not a member of SET %q", tc.msg.Queue, base.AllQueues) } } } func TestEnqueueTaskIdConflictError(t *testing.T) { r := setup(t) defer r.Close() m1 := base.TaskMessage{ ID: "custom_id", Type: "foo", Payload: nil, } m2 := base.TaskMessage{ ID: "custom_id", Type: "bar", Payload: nil, } tests := []struct { firstMsg *base.TaskMessage secondMsg *base.TaskMessage }{ {firstMsg: &m1, secondMsg: &m2}, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case. if err := r.Enqueue(context.Background(), tc.firstMsg); err != nil { t.Errorf("First message: Enqueue failed: %v", err) continue } if err := r.Enqueue(context.Background(), tc.secondMsg); !errors.Is(err, errors.ErrTaskIdConflict) { t.Errorf("Second message: Enqueue returned %v, want %v", err, errors.ErrTaskIdConflict) continue } } } func TestEnqueueQueueCache(t *testing.T) { r := setup(t) defer r.Close() t1 := h.NewTaskMessageWithQueue("sync1", nil, "low") enqueueTime := time.Now() clock := timeutil.NewSimulatedClock(enqueueTime) r.SetClock(clock) err := r.Enqueue(context.Background(), t1) if err != nil { t.Fatalf("(*RDB).Enqueue(msg) = %v, want nil", err) } // Check queue is in the AllQueues set. if !r.client.SIsMember(context.Background(), base.AllQueues, t1.Queue).Val() { t.Fatalf("%q is not a member of SET %q", t1.Queue, base.AllQueues) } if _, ok := r.queuesPublished.Load(t1.Queue); !ok { t.Fatalf("%q is not cached in queuesPublished", t1.Queue) } t.Run("remove-queue", func(t *testing.T) { err := r.RemoveQueue(t1.Queue, true) if err != nil { t.Errorf("(*RDB).RemoveQueue(%q, %t) = %v, want nil", t1.Queue, true, err) } if _, ok := r.queuesPublished.Load(t1.Queue); ok { t.Fatalf("%q is still cached in queuesPublished", t1.Queue) } if r.client.SIsMember(context.Background(), base.AllQueues, t1.Queue).Val() { t.Fatalf("%q is a member of SET %q", t1.Queue, base.AllQueues) } err = r.Enqueue(context.Background(), t1) if err != nil { t.Fatalf("(*RDB).Enqueue(msg) = %v, want nil", err) } // Check queue is in the AllQueues set. if !r.client.SIsMember(context.Background(), base.AllQueues, t1.Queue).Val() { t.Fatalf("%q is not a member of SET %q", t1.Queue, base.AllQueues) } if _, ok := r.queuesPublished.Load(t1.Queue); !ok { t.Fatalf("%q is not cached in queuesPublished", t1.Queue) } }) } func TestEnqueueUnique(t *testing.T) { r := setup(t) defer r.Close() m1 := base.TaskMessage{ ID: uuid.NewString(), Type: "email", Payload: h.JSON(map[string]interface{}{"user_id": json.Number("123")}), Queue: base.DefaultQueueName, UniqueKey: base.UniqueKey(base.DefaultQueueName, "email", h.JSON(map[string]interface{}{"user_id": 123})), } enqueueTime := time.Now() r.SetClock(timeutil.NewSimulatedClock(enqueueTime)) tests := []struct { msg *base.TaskMessage ttl time.Duration // uniqueness ttl }{ {&m1, time.Minute}, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case. // Enqueue the first message, should succeed. err := r.EnqueueUnique(context.Background(), tc.msg, tc.ttl) if err != nil { t.Errorf("First message: (*RDB).EnqueueUnique(%v, %v) = %v, want nil", tc.msg, tc.ttl, err) continue } gotPending := h.GetPendingMessages(t, r.client, tc.msg.Queue) if len(gotPending) != 1 { t.Errorf("%q has length %d, want 1", base.PendingKey(tc.msg.Queue), len(gotPending)) continue } if diff := cmp.Diff(tc.msg, gotPending[0]); diff != "" { t.Errorf("persisted data differed from the original input (-want, +got)\n%s", diff) } if !r.client.SIsMember(context.Background(), base.AllQueues, tc.msg.Queue).Val() { t.Errorf("%q is not a member of SET %q", tc.msg.Queue, base.AllQueues) } // Check Pending list has task ID. pendingKey := base.PendingKey(tc.msg.Queue) pendingIDs := r.client.LRange(context.Background(), pendingKey, 0, -1).Val() if len(pendingIDs) != 1 { t.Errorf("Redis LIST %q contains %d IDs, want 1", pendingKey, len(pendingIDs)) continue } if pendingIDs[0] != tc.msg.ID { t.Errorf("Redis LIST %q: got %v, want %v", pendingKey, pendingIDs, []string{tc.msg.ID}) continue } // Check the value under the task key. taskKey := base.TaskKey(tc.msg.Queue, tc.msg.ID) encoded := r.client.HGet(context.Background(), taskKey, "msg").Val() // "msg" field decoded := h.MustUnmarshal(t, encoded) if diff := cmp.Diff(tc.msg, decoded); diff != "" { t.Errorf("persisted message was %v, want %v; (-want, +got)\n%s", decoded, tc.msg, diff) } state := r.client.HGet(context.Background(), taskKey, "state").Val() // "state" field if state != "pending" { t.Errorf("state field under task-key is set to %q, want %q", state, "pending") } pendingSince := r.client.HGet(context.Background(), taskKey, "pending_since").Val() // "pending_since" field if want := strconv.Itoa(int(enqueueTime.UnixNano())); pendingSince != want { t.Errorf("pending_since field under task-key is set to %v, want %v", pendingSince, want) } uniqueKey := r.client.HGet(context.Background(), taskKey, "unique_key").Val() // "unique_key" field if uniqueKey != tc.msg.UniqueKey { t.Errorf("uniqueue_key field under task key is set to %q, want %q", uniqueKey, tc.msg.UniqueKey) } // Check queue is in the AllQueues set. if !r.client.SIsMember(context.Background(), base.AllQueues, tc.msg.Queue).Val() { t.Errorf("%q is not a member of SET %q", tc.msg.Queue, base.AllQueues) } // Enqueue the second message, should fail. got := r.EnqueueUnique(context.Background(), tc.msg, tc.ttl) if !errors.Is(got, errors.ErrDuplicateTask) { t.Errorf("Second message: (*RDB).EnqueueUnique(msg, ttl) = %v, want %v", got, errors.ErrDuplicateTask) continue } gotTTL := r.client.TTL(context.Background(), tc.msg.UniqueKey).Val() if !cmp.Equal(tc.ttl.Seconds(), gotTTL.Seconds(), cmpopts.EquateApprox(0, 2)) { t.Errorf("TTL %q = %v, want %v", tc.msg.UniqueKey, gotTTL, tc.ttl) continue } } } func TestEnqueueUniqueTaskIdConflictError(t *testing.T) { r := setup(t) defer r.Close() m1 := base.TaskMessage{ ID: "custom_id", Type: "foo", Payload: nil, UniqueKey: "unique_key_one", } m2 := base.TaskMessage{ ID: "custom_id", Type: "bar", Payload: nil, UniqueKey: "unique_key_two", } const ttl = 30 * time.Second tests := []struct { firstMsg *base.TaskMessage secondMsg *base.TaskMessage }{ {firstMsg: &m1, secondMsg: &m2}, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case. if err := r.EnqueueUnique(context.Background(), tc.firstMsg, ttl); err != nil { t.Errorf("First message: EnqueueUnique failed: %v", err) continue } if err := r.EnqueueUnique(context.Background(), tc.secondMsg, ttl); !errors.Is(err, errors.ErrTaskIdConflict) { t.Errorf("Second message: EnqueueUnique returned %v, want %v", err, errors.ErrTaskIdConflict) continue } } } func TestDequeue(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() r.SetClock(timeutil.NewSimulatedClock(now)) t1 := &base.TaskMessage{ ID: uuid.NewString(), Type: "send_email", Payload: h.JSON(map[string]interface{}{"subject": "hello!"}), Queue: "default", Timeout: 1800, Deadline: 0, } t2 := &base.TaskMessage{ ID: uuid.NewString(), Type: "export_csv", Payload: nil, Queue: "critical", Timeout: 0, Deadline: 1593021600, } t3 := &base.TaskMessage{ ID: uuid.NewString(), Type: "reindex", Payload: nil, Queue: "low", Timeout: int64((5 * time.Minute).Seconds()), Deadline: time.Now().Add(10 * time.Minute).Unix(), } tests := []struct { pending map[string][]*base.TaskMessage qnames []string // list of queues to query wantMsg *base.TaskMessage wantExpirationTime time.Time wantPending map[string][]*base.TaskMessage wantActive map[string][]*base.TaskMessage wantLease map[string][]base.Z }{ { pending: map[string][]*base.TaskMessage{ "default": {t1}, }, qnames: []string{"default"}, wantMsg: t1, wantExpirationTime: now.Add(LeaseDuration), wantPending: map[string][]*base.TaskMessage{ "default": {}, }, wantActive: map[string][]*base.TaskMessage{ "default": {t1}, }, wantLease: map[string][]base.Z{ "default": {{Message: t1, Score: now.Add(LeaseDuration).Unix()}}, }, }, { pending: map[string][]*base.TaskMessage{ "default": {t1}, "critical": {t2}, "low": {t3}, }, qnames: []string{"critical", "default", "low"}, wantMsg: t2, wantExpirationTime: now.Add(LeaseDuration), wantPending: map[string][]*base.TaskMessage{ "default": {t1}, "critical": {}, "low": {t3}, }, wantActive: map[string][]*base.TaskMessage{ "default": {}, "critical": {t2}, "low": {}, }, wantLease: map[string][]base.Z{ "default": {}, "critical": {{Message: t2, Score: now.Add(LeaseDuration).Unix()}}, "low": {}, }, }, { pending: map[string][]*base.TaskMessage{ "default": {t1}, "critical": {}, "low": {t3}, }, qnames: []string{"critical", "default", "low"}, wantMsg: t1, wantExpirationTime: now.Add(LeaseDuration), wantPending: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, "low": {t3}, }, wantActive: map[string][]*base.TaskMessage{ "default": {t1}, "critical": {}, "low": {}, }, wantLease: map[string][]base.Z{ "default": {{Message: t1, Score: now.Add(LeaseDuration).Unix()}}, "critical": {}, "low": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllPendingQueues(t, r.client, tc.pending) gotMsg, gotExpirationTime, err := r.Dequeue(tc.qnames...) if err != nil { t.Errorf("(*RDB).Dequeue(%v) returned error %v", tc.qnames, err) continue } if !cmp.Equal(gotMsg, tc.wantMsg) { t.Errorf("(*RDB).Dequeue(%v) returned message %v; want %v", tc.qnames, gotMsg, tc.wantMsg) continue } if gotExpirationTime != tc.wantExpirationTime { t.Errorf("(*RDB).Dequeue(%v) returned expiration time %v, want %v", tc.qnames, gotExpirationTime, tc.wantExpirationTime) } for queue, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r.client, queue) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q: (-want,+got):\n%s", base.PendingKey(queue), diff) } } for queue, want := range tc.wantActive { gotActive := h.GetActiveMessages(t, r.client, queue) if diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q: (-want,+got):\n%s", base.ActiveKey(queue), diff) } } for queue, want := range tc.wantLease { gotLease := h.GetLeaseEntries(t, r.client, queue) if diff := cmp.Diff(want, gotLease, h.SortZSetEntryOpt); diff != "" { t.Errorf("mismatch found in %q: (-want,+got):\n%s", base.LeaseKey(queue), diff) } } } } func TestDequeueError(t *testing.T) { r := setup(t) defer r.Close() tests := []struct { pending map[string][]*base.TaskMessage qnames []string // list of queues to query wantErr error wantPending map[string][]*base.TaskMessage wantActive map[string][]*base.TaskMessage wantLease map[string][]base.Z }{ { pending: map[string][]*base.TaskMessage{ "default": {}, }, qnames: []string{"default"}, wantErr: errors.ErrNoProcessableTask, wantPending: map[string][]*base.TaskMessage{ "default": {}, }, wantActive: map[string][]*base.TaskMessage{ "default": {}, }, wantLease: map[string][]base.Z{ "default": {}, }, }, { pending: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, "low": {}, }, qnames: []string{"critical", "default", "low"}, wantErr: errors.ErrNoProcessableTask, wantPending: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, "low": {}, }, wantActive: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, "low": {}, }, wantLease: map[string][]base.Z{ "default": {}, "critical": {}, "low": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllPendingQueues(t, r.client, tc.pending) gotMsg, _, gotErr := r.Dequeue(tc.qnames...) if !errors.Is(gotErr, tc.wantErr) { t.Errorf("(*RDB).Dequeue(%v) returned error %v; want %v", tc.qnames, gotErr, tc.wantErr) continue } if gotMsg != nil { t.Errorf("(*RDB).Dequeue(%v) returned message %v; want nil", tc.qnames, gotMsg) continue } for queue, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r.client, queue) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q: (-want,+got):\n%s", base.PendingKey(queue), diff) } } for queue, want := range tc.wantActive { gotActive := h.GetActiveMessages(t, r.client, queue) if diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q: (-want,+got):\n%s", base.ActiveKey(queue), diff) } } for queue, want := range tc.wantLease { gotLease := h.GetLeaseEntries(t, r.client, queue) if diff := cmp.Diff(want, gotLease, h.SortZSetEntryOpt); diff != "" { t.Errorf("mismatch found in %q: (-want,+got):\n%s", base.LeaseKey(queue), diff) } } } } func TestDequeueIgnoresPausedQueues(t *testing.T) { r := setup(t) defer r.Close() t1 := &base.TaskMessage{ ID: uuid.NewString(), Type: "send_email", Payload: h.JSON(map[string]interface{}{"subject": "hello!"}), Queue: "default", Timeout: 1800, Deadline: 0, } t2 := &base.TaskMessage{ ID: uuid.NewString(), Type: "export_csv", Payload: nil, Queue: "critical", Timeout: 1800, Deadline: 0, } tests := []struct { paused []string // list of paused queues pending map[string][]*base.TaskMessage qnames []string // list of queues to query wantMsg *base.TaskMessage wantErr error wantPending map[string][]*base.TaskMessage wantActive map[string][]*base.TaskMessage }{ { paused: []string{"default"}, pending: map[string][]*base.TaskMessage{ "default": {t1}, "critical": {t2}, }, qnames: []string{"default", "critical"}, wantMsg: t2, wantErr: nil, wantPending: map[string][]*base.TaskMessage{ "default": {t1}, "critical": {}, }, wantActive: map[string][]*base.TaskMessage{ "default": {}, "critical": {t2}, }, }, { paused: []string{"default"}, pending: map[string][]*base.TaskMessage{ "default": {t1}, }, qnames: []string{"default"}, wantMsg: nil, wantErr: errors.ErrNoProcessableTask, wantPending: map[string][]*base.TaskMessage{ "default": {t1}, }, wantActive: map[string][]*base.TaskMessage{ "default": {}, }, }, { paused: []string{"critical", "default"}, pending: map[string][]*base.TaskMessage{ "default": {t1}, "critical": {t2}, }, qnames: []string{"default", "critical"}, wantMsg: nil, wantErr: errors.ErrNoProcessableTask, wantPending: map[string][]*base.TaskMessage{ "default": {t1}, "critical": {t2}, }, wantActive: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case for _, qname := range tc.paused { if err := r.Pause(qname); err != nil { t.Fatal(err) } } h.SeedAllPendingQueues(t, r.client, tc.pending) got, _, err := r.Dequeue(tc.qnames...) if !cmp.Equal(got, tc.wantMsg) || !errors.Is(err, tc.wantErr) { t.Errorf("Dequeue(%v) = %v, %v; want %v, %v", tc.qnames, got, err, tc.wantMsg, tc.wantErr) continue } for queue, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r.client, queue) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q: (-want,+got):\n%s", base.PendingKey(queue), diff) } } for queue, want := range tc.wantActive { gotActive := h.GetActiveMessages(t, r.client, queue) if diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q: (-want,+got):\n%s", base.ActiveKey(queue), diff) } } } } func TestDone(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() t1 := &base.TaskMessage{ ID: uuid.NewString(), Type: "send_email", Payload: nil, Timeout: 1800, Deadline: 0, Queue: "default", } t2 := &base.TaskMessage{ ID: uuid.NewString(), Type: "export_csv", Payload: nil, Timeout: 0, Deadline: 1592485787, Queue: "custom", } t3 := &base.TaskMessage{ ID: uuid.NewString(), Type: "reindex", Payload: nil, Timeout: 1800, Deadline: 0, UniqueKey: "asynq:{default}:unique:b0804ec967f48520697662a204f5fe72", Queue: "default", } tests := []struct { desc string active map[string][]*base.TaskMessage // initial state of the active list lease map[string][]base.Z // initial state of the lease set target *base.TaskMessage // task to remove wantActive map[string][]*base.TaskMessage // final state of the active list wantLease map[string][]base.Z // final state of the lease set }{ { desc: "removes message from the correct queue", active: map[string][]*base.TaskMessage{ "default": {t1}, "custom": {t2}, }, lease: map[string][]base.Z{ "default": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}}, "custom": {{Message: t2, Score: now.Add(20 * time.Second).Unix()}}, }, target: t1, wantActive: map[string][]*base.TaskMessage{ "default": {}, "custom": {t2}, }, wantLease: map[string][]base.Z{ "default": {}, "custom": {{Message: t2, Score: now.Add(20 * time.Second).Unix()}}, }, }, { desc: "with one queue", active: map[string][]*base.TaskMessage{ "default": {t1}, }, lease: map[string][]base.Z{ "default": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}}, }, target: t1, wantActive: map[string][]*base.TaskMessage{ "default": {}, }, wantLease: map[string][]base.Z{ "default": {}, }, }, { desc: "with multiple messages in a queue", active: map[string][]*base.TaskMessage{ "default": {t1, t3}, "custom": {t2}, }, lease: map[string][]base.Z{ "default": {{Message: t1, Score: now.Add(15 * time.Second).Unix()}, {Message: t3, Score: now.Add(10 * time.Second).Unix()}}, "custom": {{Message: t2, Score: now.Add(20 * time.Second).Unix()}}, }, target: t3, wantActive: map[string][]*base.TaskMessage{ "default": {t1}, "custom": {t2}, }, wantLease: map[string][]base.Z{ "default": {{Message: t1, Score: now.Add(15 * time.Second).Unix()}}, "custom": {{Message: t2, Score: now.Add(20 * time.Second).Unix()}}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllLease(t, r.client, tc.lease) h.SeedAllActiveQueues(t, r.client, tc.active) for _, msgs := range tc.active { for _, msg := range msgs { // Set uniqueness lock if unique key is present. if len(msg.UniqueKey) > 0 { err := r.client.SetNX(context.Background(), msg.UniqueKey, msg.ID, time.Minute).Err() if err != nil { t.Fatal(err) } } } } err := r.Done(context.Background(), tc.target) if err != nil { t.Errorf("%s; (*RDB).Done(task) = %v, want nil", tc.desc, err) continue } for queue, want := range tc.wantActive { gotActive := h.GetActiveMessages(t, r.client, queue) if diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != "" { t.Errorf("%s; mismatch found in %q: (-want, +got):\n%s", tc.desc, base.ActiveKey(queue), diff) continue } } for queue, want := range tc.wantLease { gotLease := h.GetLeaseEntries(t, r.client, queue) if diff := cmp.Diff(want, gotLease, h.SortZSetEntryOpt); diff != "" { t.Errorf("%s; mismatch found in %q: (-want, +got):\n%s", tc.desc, base.LeaseKey(queue), diff) continue } } processedKey := base.ProcessedKey(tc.target.Queue, time.Now()) gotProcessed := r.client.Get(context.Background(), processedKey).Val() if gotProcessed != "1" { t.Errorf("%s; GET %q = %q, want 1", tc.desc, processedKey, gotProcessed) } gotTTL := r.client.TTL(context.Background(), processedKey).Val() if gotTTL > statsTTL { t.Errorf("%s; TTL %q = %v, want less than or equal to %v", tc.desc, processedKey, gotTTL, statsTTL) } processedTotalKey := base.ProcessedTotalKey(tc.target.Queue) gotProcessedTotal := r.client.Get(context.Background(), processedTotalKey).Val() if gotProcessedTotal != "1" { t.Errorf("%s; GET %q = %q, want 1", tc.desc, processedTotalKey, gotProcessedTotal) } if len(tc.target.UniqueKey) > 0 && r.client.Exists(context.Background(), tc.target.UniqueKey).Val() != 0 { t.Errorf("%s; Uniqueness lock %q still exists", tc.desc, tc.target.UniqueKey) } } } // Make sure that processed_total counter wraps to 1 when reaching int64 max value. func TestDoneWithMaxCounter(t *testing.T) { r := setup(t) defer r.Close() msg := &base.TaskMessage{ ID: uuid.NewString(), Type: "foo", Payload: nil, Timeout: 1800, Deadline: 0, Queue: "default", } z := base.Z{ Message: msg, Score: time.Now().Add(15 * time.Second).Unix(), } h.SeedLease(t, r.client, []base.Z{z}, msg.Queue) h.SeedActiveQueue(t, r.client, []*base.TaskMessage{msg}, msg.Queue) processedTotalKey := base.ProcessedTotalKey(msg.Queue) ctx := context.Background() if err := r.client.Set(ctx, processedTotalKey, math.MaxInt64, 0).Err(); err != nil { t.Fatalf("Redis command failed: SET %q %v", processedTotalKey, math.MaxInt64) } if err := r.Done(context.Background(), msg); err != nil { t.Fatalf("RDB.Done failed: %v", err) } gotProcessedTotal := r.client.Get(ctx, processedTotalKey).Val() if gotProcessedTotal != "1" { t.Errorf("GET %q = %v, want 1", processedTotalKey, gotProcessedTotal) } } func TestMarkAsComplete(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() r.SetClock(timeutil.NewSimulatedClock(now)) t1 := &base.TaskMessage{ ID: uuid.NewString(), Type: "send_email", Payload: nil, Timeout: 1800, Deadline: 0, Queue: "default", Retention: 3600, } t2 := &base.TaskMessage{ ID: uuid.NewString(), Type: "export_csv", Payload: nil, Timeout: 0, Deadline: now.Add(2 * time.Hour).Unix(), Queue: "custom", Retention: 7200, } t3 := &base.TaskMessage{ ID: uuid.NewString(), Type: "reindex", Payload: nil, Timeout: 1800, Deadline: 0, UniqueKey: "asynq:{default}:unique:b0804ec967f48520697662a204f5fe72", Queue: "default", Retention: 1800, } tests := []struct { desc string active map[string][]*base.TaskMessage // initial state of the active list lease map[string][]base.Z // initial state of the lease set completed map[string][]base.Z // initial state of the completed set target *base.TaskMessage // task to mark as completed wantActive map[string][]*base.TaskMessage // final state of the active list wantLease map[string][]base.Z // final state of the lease set wantCompleted map[string][]base.Z // final state of the completed set }{ { desc: "select a message from the correct queue", active: map[string][]*base.TaskMessage{ "default": {t1}, "custom": {t2}, }, lease: map[string][]base.Z{ "default": {{Message: t1, Score: now.Add(30 * time.Second).Unix()}}, "custom": {{Message: t2, Score: now.Add(20 * time.Second).Unix()}}, }, completed: map[string][]base.Z{ "default": {}, "custom": {}, }, target: t1, wantActive: map[string][]*base.TaskMessage{ "default": {}, "custom": {t2}, }, wantLease: map[string][]base.Z{ "default": {}, "custom": {{Message: t2, Score: now.Add(20 * time.Second).Unix()}}, }, wantCompleted: map[string][]base.Z{ "default": {{Message: h.TaskMessageWithCompletedAt(*t1, now), Score: now.Unix() + t1.Retention}}, "custom": {}, }, }, { desc: "with one queue", active: map[string][]*base.TaskMessage{ "default": {t1}, }, lease: map[string][]base.Z{ "default": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}}, }, completed: map[string][]base.Z{ "default": {}, }, target: t1, wantActive: map[string][]*base.TaskMessage{ "default": {}, }, wantLease: map[string][]base.Z{ "default": {}, }, wantCompleted: map[string][]base.Z{ "default": {{Message: h.TaskMessageWithCompletedAt(*t1, now), Score: now.Unix() + t1.Retention}}, }, }, { desc: "with multiple messages in a queue", active: map[string][]*base.TaskMessage{ "default": {t1, t3}, "custom": {t2}, }, lease: map[string][]base.Z{ "default": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}, {Message: t3, Score: now.Add(12 * time.Second).Unix()}}, "custom": {{Message: t2, Score: now.Add(12 * time.Second).Unix()}}, }, completed: map[string][]base.Z{ "default": {}, "custom": {}, }, target: t3, wantActive: map[string][]*base.TaskMessage{ "default": {t1}, "custom": {t2}, }, wantLease: map[string][]base.Z{ "default": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}}, "custom": {{Message: t2, Score: now.Add(12 * time.Second).Unix()}}, }, wantCompleted: map[string][]base.Z{ "default": {{Message: h.TaskMessageWithCompletedAt(*t3, now), Score: now.Unix() + t3.Retention}}, "custom": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllLease(t, r.client, tc.lease) h.SeedAllActiveQueues(t, r.client, tc.active) h.SeedAllCompletedQueues(t, r.client, tc.completed) for _, msgs := range tc.active { for _, msg := range msgs { // Set uniqueness lock if unique key is present. if len(msg.UniqueKey) > 0 { err := r.client.SetNX(context.Background(), msg.UniqueKey, msg.ID, time.Minute).Err() if err != nil { t.Fatal(err) } } } } err := r.MarkAsComplete(context.Background(), tc.target) if err != nil { t.Errorf("%s; (*RDB).MarkAsCompleted(task) = %v, want nil", tc.desc, err) continue } for queue, want := range tc.wantActive { gotActive := h.GetActiveMessages(t, r.client, queue) if diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != "" { t.Errorf("%s; mismatch found in %q: (-want, +got):\n%s", tc.desc, base.ActiveKey(queue), diff) continue } } for queue, want := range tc.wantLease { gotLease := h.GetLeaseEntries(t, r.client, queue) if diff := cmp.Diff(want, gotLease, h.SortZSetEntryOpt); diff != "" { t.Errorf("%s; mismatch found in %q: (-want, +got):\n%s", tc.desc, base.LeaseKey(queue), diff) continue } } for queue, want := range tc.wantCompleted { gotCompleted := h.GetCompletedEntries(t, r.client, queue) if diff := cmp.Diff(want, gotCompleted, h.SortZSetEntryOpt); diff != "" { t.Errorf("%s; mismatch found in %q: (-want, +got):\n%s", tc.desc, base.CompletedKey(queue), diff) continue } } processedKey := base.ProcessedKey(tc.target.Queue, time.Now()) gotProcessed := r.client.Get(context.Background(), processedKey).Val() if gotProcessed != "1" { t.Errorf("%s; GET %q = %q, want 1", tc.desc, processedKey, gotProcessed) } gotTTL := r.client.TTL(context.Background(), processedKey).Val() if gotTTL > statsTTL { t.Errorf("%s; TTL %q = %v, want less than or equal to %v", tc.desc, processedKey, gotTTL, statsTTL) } if len(tc.target.UniqueKey) > 0 && r.client.Exists(context.Background(), tc.target.UniqueKey).Val() != 0 { t.Errorf("%s; Uniqueness lock %q still exists", tc.desc, tc.target.UniqueKey) } } } func TestRequeue(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() t1 := &base.TaskMessage{ ID: uuid.NewString(), Type: "send_email", Payload: nil, Queue: "default", Timeout: 1800, } t2 := &base.TaskMessage{ ID: uuid.NewString(), Type: "export_csv", Payload: nil, Queue: "default", Timeout: 3000, } t3 := &base.TaskMessage{ ID: uuid.NewString(), Type: "send_email", Payload: nil, Queue: "critical", Timeout: 80, } tests := []struct { pending map[string][]*base.TaskMessage // initial state of queues active map[string][]*base.TaskMessage // initial state of the active list lease map[string][]base.Z // initial state of the lease set target *base.TaskMessage // task to requeue wantPending map[string][]*base.TaskMessage // final state of queues wantActive map[string][]*base.TaskMessage // final state of the active list wantLease map[string][]base.Z // final state of the lease set }{ { pending: map[string][]*base.TaskMessage{ "default": {}, }, active: map[string][]*base.TaskMessage{ "default": {t1, t2}, }, lease: map[string][]base.Z{ "default": { {Message: t1, Score: now.Add(10 * time.Second).Unix()}, {Message: t2, Score: now.Add(10 * time.Second).Unix()}, }, }, target: t1, wantPending: map[string][]*base.TaskMessage{ "default": {t1}, }, wantActive: map[string][]*base.TaskMessage{ "default": {t2}, }, wantLease: map[string][]base.Z{ "default": { {Message: t2, Score: now.Add(10 * time.Second).Unix()}, }, }, }, { pending: map[string][]*base.TaskMessage{ "default": {t1}, }, active: map[string][]*base.TaskMessage{ "default": {t2}, }, lease: map[string][]base.Z{ "default": { {Message: t2, Score: now.Add(20 * time.Second).Unix()}, }, }, target: t2, wantPending: map[string][]*base.TaskMessage{ "default": {t1, t2}, }, wantActive: map[string][]*base.TaskMessage{ "default": {}, }, wantLease: map[string][]base.Z{ "default": {}, }, }, { pending: map[string][]*base.TaskMessage{ "default": {t1}, "critical": {}, }, active: map[string][]*base.TaskMessage{ "default": {t2}, "critical": {t3}, }, lease: map[string][]base.Z{ "default": {{Message: t2, Score: now.Add(10 * time.Second).Unix()}}, "critical": {{Message: t3, Score: now.Add(10 * time.Second).Unix()}}, }, target: t3, wantPending: map[string][]*base.TaskMessage{ "default": {t1}, "critical": {t3}, }, wantActive: map[string][]*base.TaskMessage{ "default": {t2}, "critical": {}, }, wantLease: map[string][]base.Z{ "default": {{Message: t2, Score: now.Add(10 * time.Second).Unix()}}, "critical": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllPendingQueues(t, r.client, tc.pending) h.SeedAllActiveQueues(t, r.client, tc.active) h.SeedAllLease(t, r.client, tc.lease) err := r.Requeue(context.Background(), tc.target) if err != nil { t.Errorf("(*RDB).Requeue(task) = %v, want nil", err) continue } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r.client, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.PendingKey(qname), diff) } } for qname, want := range tc.wantActive { gotActive := h.GetActiveMessages(t, r.client, qname) if diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q: (-want, +got):\n%s", base.ActiveKey(qname), diff) } } for qname, want := range tc.wantLease { gotLease := h.GetLeaseEntries(t, r.client, qname) if diff := cmp.Diff(want, gotLease, h.SortZSetEntryOpt); diff != "" { t.Errorf("mismatch found in %q: (-want, +got):\n%s", base.LeaseKey(qname), diff) } } } } func TestAddToGroup(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() r.SetClock(timeutil.NewSimulatedClock(now)) msg := h.NewTaskMessage("mytask", []byte("foo")) ctx := context.Background() tests := []struct { msg *base.TaskMessage groupKey string }{ { msg: msg, groupKey: "mygroup", }, } for _, tc := range tests { h.FlushDB(t, r.client) err := r.AddToGroup(ctx, tc.msg, tc.groupKey) if err != nil { t.Errorf("r.AddToGroup(ctx, msg, %q) returned error: %v", tc.groupKey, err) continue } // Check Group zset has task ID gkey := base.GroupKey(tc.msg.Queue, tc.groupKey) zs := r.client.ZRangeWithScores(ctx, gkey, 0, -1).Val() if n := len(zs); n != 1 { t.Errorf("Redis ZSET %q contains %d elements, want 1", gkey, n) continue } if got := zs[0].Member.(string); got != tc.msg.ID { t.Errorf("Redis ZSET %q member: got %v, want %v", gkey, got, tc.msg.ID) continue } if got := int64(zs[0].Score); got != now.Unix() { t.Errorf("Redis ZSET %q score: got %d, want %d", gkey, got, now.Unix()) continue } // Check the values under the task key. taskKey := base.TaskKey(tc.msg.Queue, tc.msg.ID) encoded := r.client.HGet(ctx, taskKey, "msg").Val() // "msg" field decoded := h.MustUnmarshal(t, encoded) if diff := cmp.Diff(tc.msg, decoded); diff != "" { t.Errorf("persisted message was %v, want %v; (-want, +got)\n%s", decoded, tc.msg, diff) } state := r.client.HGet(ctx, taskKey, "state").Val() // "state" field if want := "aggregating"; state != want { t.Errorf("state field under task-key is set to %q, want %q", state, want) } group := r.client.HGet(ctx, taskKey, "group").Val() // "group" field if want := tc.groupKey; group != want { t.Errorf("group field under task-key is set to %q, want %q", group, want) } // Check queue is in the AllQueues set. if !r.client.SIsMember(context.Background(), base.AllQueues, tc.msg.Queue).Val() { t.Errorf("%q is not a member of SET %q", tc.msg.Queue, base.AllQueues) } } } func TestAddToGroupeTaskIdConflictError(t *testing.T) { r := setup(t) defer r.Close() ctx := context.Background() m1 := base.TaskMessage{ ID: "custom_id", Type: "foo", Payload: nil, UniqueKey: "unique_key_one", } m2 := base.TaskMessage{ ID: "custom_id", Type: "bar", Payload: nil, UniqueKey: "unique_key_two", } const groupKey = "mygroup" tests := []struct { firstMsg *base.TaskMessage secondMsg *base.TaskMessage }{ {firstMsg: &m1, secondMsg: &m2}, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case. if err := r.AddToGroup(ctx, tc.firstMsg, groupKey); err != nil { t.Errorf("First message: AddToGroup failed: %v", err) continue } if err := r.AddToGroup(ctx, tc.secondMsg, groupKey); !errors.Is(err, errors.ErrTaskIdConflict) { t.Errorf("Second message: AddToGroup returned %v, want %v", err, errors.ErrTaskIdConflict) continue } } } func TestAddToGroupUnique(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() r.SetClock(timeutil.NewSimulatedClock(now)) msg := h.NewTaskMessage("mytask", []byte("foo")) msg.UniqueKey = base.UniqueKey(msg.Queue, msg.Type, msg.Payload) ctx := context.Background() tests := []struct { msg *base.TaskMessage groupKey string ttl time.Duration }{ { msg: msg, groupKey: "mygroup", ttl: 30 * time.Second, }, } for _, tc := range tests { h.FlushDB(t, r.client) err := r.AddToGroupUnique(ctx, tc.msg, tc.groupKey, tc.ttl) if err != nil { t.Errorf("First message: r.AddToGroupUnique(ctx, msg, %q) returned error: %v", tc.groupKey, err) continue } // Check Group zset has task ID gkey := base.GroupKey(tc.msg.Queue, tc.groupKey) zs := r.client.ZRangeWithScores(ctx, gkey, 0, -1).Val() if n := len(zs); n != 1 { t.Errorf("Redis ZSET %q contains %d elements, want 1", gkey, n) continue } if got := zs[0].Member.(string); got != tc.msg.ID { t.Errorf("Redis ZSET %q member: got %v, want %v", gkey, got, tc.msg.ID) continue } if got := int64(zs[0].Score); got != now.Unix() { t.Errorf("Redis ZSET %q score: got %d, want %d", gkey, got, now.Unix()) continue } // Check the values under the task key. taskKey := base.TaskKey(tc.msg.Queue, tc.msg.ID) encoded := r.client.HGet(ctx, taskKey, "msg").Val() // "msg" field decoded := h.MustUnmarshal(t, encoded) if diff := cmp.Diff(tc.msg, decoded); diff != "" { t.Errorf("persisted message was %v, want %v; (-want, +got)\n%s", decoded, tc.msg, diff) } state := r.client.HGet(ctx, taskKey, "state").Val() // "state" field if want := "aggregating"; state != want { t.Errorf("state field under task-key is set to %q, want %q", state, want) } group := r.client.HGet(ctx, taskKey, "group").Val() // "group" field if want := tc.groupKey; group != want { t.Errorf("group field under task-key is set to %q, want %q", group, want) } // Check queue is in the AllQueues set. if !r.client.SIsMember(context.Background(), base.AllQueues, tc.msg.Queue).Val() { t.Errorf("%q is not a member of SET %q", tc.msg.Queue, base.AllQueues) } got := r.AddToGroupUnique(ctx, tc.msg, tc.groupKey, tc.ttl) if !errors.Is(got, errors.ErrDuplicateTask) { t.Errorf("Second message: r.AddGroupUnique(ctx, msg, %q) = %v, want %v", tc.groupKey, got, errors.ErrDuplicateTask) continue } gotTTL := r.client.TTL(ctx, tc.msg.UniqueKey).Val() if !cmp.Equal(tc.ttl.Seconds(), gotTTL.Seconds(), cmpopts.EquateApprox(0, 1)) { t.Errorf("TTL %q = %v, want %v", tc.msg.UniqueKey, gotTTL, tc.ttl) continue } } } func TestAddToGroupUniqueTaskIdConflictError(t *testing.T) { r := setup(t) defer r.Close() ctx := context.Background() m1 := base.TaskMessage{ ID: "custom_id", Type: "foo", Payload: nil, UniqueKey: "unique_key_one", } m2 := base.TaskMessage{ ID: "custom_id", Type: "bar", Payload: nil, UniqueKey: "unique_key_two", } const groupKey = "mygroup" const ttl = 30 * time.Second tests := []struct { firstMsg *base.TaskMessage secondMsg *base.TaskMessage }{ {firstMsg: &m1, secondMsg: &m2}, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case. if err := r.AddToGroupUnique(ctx, tc.firstMsg, groupKey, ttl); err != nil { t.Errorf("First message: AddToGroupUnique failed: %v", err) continue } if err := r.AddToGroupUnique(ctx, tc.secondMsg, groupKey, ttl); !errors.Is(err, errors.ErrTaskIdConflict) { t.Errorf("Second message: AddToGroupUnique returned %v, want %v", err, errors.ErrTaskIdConflict) continue } } } func TestSchedule(t *testing.T) { r := setup(t) defer r.Close() msg := h.NewTaskMessage("send_email", h.JSON(map[string]interface{}{"subject": "hello"})) tests := []struct { msg *base.TaskMessage processAt time.Time }{ {msg, time.Now().Add(15 * time.Minute)}, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case err := r.Schedule(context.Background(), tc.msg, tc.processAt) if err != nil { t.Errorf("(*RDB).Schedule(%v, %v) = %v, want nil", tc.msg, tc.processAt, err) continue } // Check Scheduled zset has task ID. scheduledKey := base.ScheduledKey(tc.msg.Queue) zs := r.client.ZRangeWithScores(context.Background(), scheduledKey, 0, -1).Val() if n := len(zs); n != 1 { t.Errorf("Redis ZSET %q contains %d elements, want 1", scheduledKey, n) continue } if got := zs[0].Member.(string); got != tc.msg.ID { t.Errorf("Redis ZSET %q member: got %v, want %v", scheduledKey, got, tc.msg.ID) continue } if got := int64(zs[0].Score); got != tc.processAt.Unix() { t.Errorf("Redis ZSET %q score: got %d, want %d", scheduledKey, got, tc.processAt.Unix()) continue } // Check the values under the task key. taskKey := base.TaskKey(tc.msg.Queue, tc.msg.ID) encoded := r.client.HGet(context.Background(), taskKey, "msg").Val() // "msg" field decoded := h.MustUnmarshal(t, encoded) if diff := cmp.Diff(tc.msg, decoded); diff != "" { t.Errorf("persisted message was %v, want %v; (-want, +got)\n%s", decoded, tc.msg, diff) } state := r.client.HGet(context.Background(), taskKey, "state").Val() // "state" field if want := "scheduled"; state != want { t.Errorf("state field under task-key is set to %q, want %q", state, want) } // Check queue is in the AllQueues set. if !r.client.SIsMember(context.Background(), base.AllQueues, tc.msg.Queue).Val() { t.Errorf("%q is not a member of SET %q", tc.msg.Queue, base.AllQueues) } } } func TestScheduleTaskIdConflictError(t *testing.T) { r := setup(t) defer r.Close() m1 := base.TaskMessage{ ID: "custom_id", Type: "foo", Payload: nil, UniqueKey: "unique_key_one", } m2 := base.TaskMessage{ ID: "custom_id", Type: "bar", Payload: nil, UniqueKey: "unique_key_two", } processAt := time.Now().Add(30 * time.Second) tests := []struct { firstMsg *base.TaskMessage secondMsg *base.TaskMessage }{ {firstMsg: &m1, secondMsg: &m2}, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case. if err := r.Schedule(context.Background(), tc.firstMsg, processAt); err != nil { t.Errorf("First message: Schedule failed: %v", err) continue } if err := r.Schedule(context.Background(), tc.secondMsg, processAt); !errors.Is(err, errors.ErrTaskIdConflict) { t.Errorf("Second message: Schedule returned %v, want %v", err, errors.ErrTaskIdConflict) continue } } } func TestScheduleUnique(t *testing.T) { r := setup(t) defer r.Close() m1 := base.TaskMessage{ ID: uuid.NewString(), Type: "email", Payload: h.JSON(map[string]interface{}{"user_id": 123}), Queue: base.DefaultQueueName, UniqueKey: base.UniqueKey(base.DefaultQueueName, "email", h.JSON(map[string]interface{}{"user_id": 123})), } tests := []struct { msg *base.TaskMessage processAt time.Time ttl time.Duration // uniqueness lock ttl }{ {&m1, time.Now().Add(15 * time.Minute), time.Minute}, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case desc := "(*RDB).ScheduleUnique(msg, processAt, ttl)" err := r.ScheduleUnique(context.Background(), tc.msg, tc.processAt, tc.ttl) if err != nil { t.Errorf("First task: %s = %v, want nil", desc, err) continue } // Check Scheduled zset has task ID. scheduledKey := base.ScheduledKey(tc.msg.Queue) zs := r.client.ZRangeWithScores(context.Background(), scheduledKey, 0, -1).Val() if n := len(zs); n != 1 { t.Errorf("Redis ZSET %q contains %d elements, want 1", scheduledKey, n) continue } if got := zs[0].Member.(string); got != tc.msg.ID { t.Errorf("Redis ZSET %q member: got %v, want %v", scheduledKey, got, tc.msg.ID) continue } if got := int64(zs[0].Score); got != tc.processAt.Unix() { t.Errorf("Redis ZSET %q score: got %d, want %d", scheduledKey, got, tc.processAt.Unix()) continue } // Check the values under the task key. taskKey := base.TaskKey(tc.msg.Queue, tc.msg.ID) encoded := r.client.HGet(context.Background(), taskKey, "msg").Val() // "msg" field decoded := h.MustUnmarshal(t, encoded) if diff := cmp.Diff(tc.msg, decoded); diff != "" { t.Errorf("persisted message was %v, want %v; (-want, +got)\n%s", decoded, tc.msg, diff) } state := r.client.HGet(context.Background(), taskKey, "state").Val() // "state" field if want := "scheduled"; state != want { t.Errorf("state field under task-key is set to %q, want %q", state, want) } uniqueKey := r.client.HGet(context.Background(), taskKey, "unique_key").Val() // "unique_key" field if uniqueKey != tc.msg.UniqueKey { t.Errorf("uniqueue_key field under task key is set to %q, want %q", uniqueKey, tc.msg.UniqueKey) } // Check queue is in the AllQueues set. if !r.client.SIsMember(context.Background(), base.AllQueues, tc.msg.Queue).Val() { t.Errorf("%q is not a member of SET %q", tc.msg.Queue, base.AllQueues) } // Enqueue the second message, should fail. got := r.ScheduleUnique(context.Background(), tc.msg, tc.processAt, tc.ttl) if !errors.Is(got, errors.ErrDuplicateTask) { t.Errorf("Second task: %s = %v, want %v", desc, got, errors.ErrDuplicateTask) continue } gotTTL := r.client.TTL(context.Background(), tc.msg.UniqueKey).Val() if !cmp.Equal(tc.ttl.Seconds(), gotTTL.Seconds(), cmpopts.EquateApprox(0, 1)) { t.Errorf("TTL %q = %v, want %v", tc.msg.UniqueKey, gotTTL, tc.ttl) continue } } } func TestScheduleUniqueTaskIdConflictError(t *testing.T) { r := setup(t) defer r.Close() m1 := base.TaskMessage{ ID: "custom_id", Type: "foo", Payload: nil, UniqueKey: "unique_key_one", } m2 := base.TaskMessage{ ID: "custom_id", Type: "bar", Payload: nil, UniqueKey: "unique_key_two", } const ttl = 30 * time.Second processAt := time.Now().Add(30 * time.Second) tests := []struct { firstMsg *base.TaskMessage secondMsg *base.TaskMessage }{ {firstMsg: &m1, secondMsg: &m2}, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case. if err := r.ScheduleUnique(context.Background(), tc.firstMsg, processAt, ttl); err != nil { t.Errorf("First message: ScheduleUnique failed: %v", err) continue } if err := r.ScheduleUnique(context.Background(), tc.secondMsg, processAt, ttl); !errors.Is(err, errors.ErrTaskIdConflict) { t.Errorf("Second message: ScheduleUnique returned %v, want %v", err, errors.ErrTaskIdConflict) continue } } } func TestRetry(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() r.SetClock(timeutil.NewSimulatedClock(now)) t1 := &base.TaskMessage{ ID: uuid.NewString(), Type: "send_email", Payload: h.JSON(map[string]interface{}{"subject": "Hola!"}), Retried: 10, Timeout: 1800, Queue: "default", } t2 := &base.TaskMessage{ ID: uuid.NewString(), Type: "gen_thumbnail", Payload: h.JSON(map[string]interface{}{"path": "some/path/to/image.jpg"}), Timeout: 3000, Queue: "default", } t3 := &base.TaskMessage{ ID: uuid.NewString(), Type: "reindex", Payload: nil, Timeout: 60, Queue: "default", } t4 := &base.TaskMessage{ ID: uuid.NewString(), Type: "send_notification", Payload: nil, Timeout: 1800, Queue: "custom", } errMsg := "SMTP server is not responding" tests := []struct { active map[string][]*base.TaskMessage lease map[string][]base.Z retry map[string][]base.Z msg *base.TaskMessage processAt time.Time errMsg string wantActive map[string][]*base.TaskMessage wantLease map[string][]base.Z wantRetry map[string][]base.Z }{ { active: map[string][]*base.TaskMessage{ "default": {t1, t2}, }, lease: map[string][]base.Z{ "default": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}, {Message: t2, Score: now.Add(10 * time.Second).Unix()}}, }, retry: map[string][]base.Z{ "default": {{Message: t3, Score: now.Add(time.Minute).Unix()}}, }, msg: t1, processAt: now.Add(5 * time.Minute), errMsg: errMsg, wantActive: map[string][]*base.TaskMessage{ "default": {t2}, }, wantLease: map[string][]base.Z{ "default": {{Message: t2, Score: now.Add(10 * time.Second).Unix()}}, }, wantRetry: map[string][]base.Z{ "default": { {Message: h.TaskMessageAfterRetry(*t1, errMsg, now), Score: now.Add(5 * time.Minute).Unix()}, {Message: t3, Score: now.Add(time.Minute).Unix()}, }, }, }, { active: map[string][]*base.TaskMessage{ "default": {t1, t2}, "custom": {t4}, }, lease: map[string][]base.Z{ "default": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}, {Message: t2, Score: now.Add(20 * time.Second).Unix()}}, "custom": {{Message: t4, Score: now.Add(10 * time.Second).Unix()}}, }, retry: map[string][]base.Z{ "default": {}, "custom": {}, }, msg: t4, processAt: now.Add(5 * time.Minute), errMsg: errMsg, wantActive: map[string][]*base.TaskMessage{ "default": {t1, t2}, "custom": {}, }, wantLease: map[string][]base.Z{ "default": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}, {Message: t2, Score: now.Add(20 * time.Second).Unix()}}, "custom": {}, }, wantRetry: map[string][]base.Z{ "default": {}, "custom": { {Message: h.TaskMessageAfterRetry(*t4, errMsg, now), Score: now.Add(5 * time.Minute).Unix()}, }, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) h.SeedAllActiveQueues(t, r.client, tc.active) h.SeedAllLease(t, r.client, tc.lease) h.SeedAllRetryQueues(t, r.client, tc.retry) err := r.Retry(context.Background(), tc.msg, tc.processAt, tc.errMsg, true /*isFailure*/) if err != nil { t.Errorf("(*RDB).Retry = %v, want nil", err) continue } for queue, want := range tc.wantActive { gotActive := h.GetActiveMessages(t, r.client, queue) if diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.ActiveKey(queue), diff) } } for queue, want := range tc.wantLease { gotLease := h.GetLeaseEntries(t, r.client, queue) if diff := cmp.Diff(want, gotLease, h.SortZSetEntryOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.LeaseKey(queue), diff) } } for queue, want := range tc.wantRetry { gotRetry := h.GetRetryEntries(t, r.client, queue) if diff := cmp.Diff(want, gotRetry, h.SortZSetEntryOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.RetryKey(queue), diff) } } processedKey := base.ProcessedKey(tc.msg.Queue, time.Now()) gotProcessed := r.client.Get(context.Background(), processedKey).Val() if gotProcessed != "1" { t.Errorf("GET %q = %q, want 1", processedKey, gotProcessed) } gotTTL := r.client.TTL(context.Background(), processedKey).Val() if gotTTL > statsTTL { t.Errorf("TTL %q = %v, want less than or equal to %v", processedKey, gotTTL, statsTTL) } failedKey := base.FailedKey(tc.msg.Queue, time.Now()) gotFailed := r.client.Get(context.Background(), failedKey).Val() if gotFailed != "1" { t.Errorf("GET %q = %q, want 1", failedKey, gotFailed) } gotTTL = r.client.TTL(context.Background(), failedKey).Val() if gotTTL > statsTTL { t.Errorf("TTL %q = %v, want less than or equal to %v", failedKey, gotTTL, statsTTL) } processedTotalKey := base.ProcessedTotalKey(tc.msg.Queue) gotProcessedTotal := r.client.Get(context.Background(), processedTotalKey).Val() if gotProcessedTotal != "1" { t.Errorf("GET %q = %q, want 1", processedTotalKey, gotProcessedTotal) } failedTotalKey := base.FailedTotalKey(tc.msg.Queue) gotFailedTotal := r.client.Get(context.Background(), failedTotalKey).Val() if gotFailedTotal != "1" { t.Errorf("GET %q = %q, want 1", failedTotalKey, gotFailedTotal) } } } func TestRetryWithNonFailureError(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() r.SetClock(timeutil.NewSimulatedClock(now)) t1 := &base.TaskMessage{ ID: uuid.NewString(), Type: "send_email", Payload: h.JSON(map[string]interface{}{"subject": "Hola!"}), Retried: 10, Timeout: 1800, Queue: "default", } t2 := &base.TaskMessage{ ID: uuid.NewString(), Type: "gen_thumbnail", Payload: h.JSON(map[string]interface{}{"path": "some/path/to/image.jpg"}), Timeout: 3000, Queue: "default", } t3 := &base.TaskMessage{ ID: uuid.NewString(), Type: "reindex", Payload: nil, Timeout: 60, Queue: "default", } t4 := &base.TaskMessage{ ID: uuid.NewString(), Type: "send_notification", Payload: nil, Timeout: 1800, Queue: "custom", } errMsg := "SMTP server is not responding" tests := []struct { active map[string][]*base.TaskMessage lease map[string][]base.Z retry map[string][]base.Z msg *base.TaskMessage processAt time.Time errMsg string wantActive map[string][]*base.TaskMessage wantLease map[string][]base.Z wantRetry map[string][]base.Z }{ { active: map[string][]*base.TaskMessage{ "default": {t1, t2}, }, lease: map[string][]base.Z{ "default": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}, {Message: t2, Score: now.Add(10 * time.Second).Unix()}}, }, retry: map[string][]base.Z{ "default": {{Message: t3, Score: now.Add(time.Minute).Unix()}}, }, msg: t1, processAt: now.Add(5 * time.Minute), errMsg: errMsg, wantActive: map[string][]*base.TaskMessage{ "default": {t2}, }, wantLease: map[string][]base.Z{ "default": {{Message: t2, Score: now.Add(10 * time.Second).Unix()}}, }, wantRetry: map[string][]base.Z{ "default": { // Task message should include the error message but without incrementing the retry count. {Message: h.TaskMessageWithError(*t1, errMsg, now), Score: now.Add(5 * time.Minute).Unix()}, {Message: t3, Score: now.Add(time.Minute).Unix()}, }, }, }, { active: map[string][]*base.TaskMessage{ "default": {t1, t2}, "custom": {t4}, }, lease: map[string][]base.Z{ "default": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}, {Message: t2, Score: now.Add(10 * time.Second).Unix()}}, "custom": {{Message: t4, Score: now.Add(10 * time.Second).Unix()}}, }, retry: map[string][]base.Z{ "default": {}, "custom": {}, }, msg: t4, processAt: now.Add(5 * time.Minute), errMsg: errMsg, wantActive: map[string][]*base.TaskMessage{ "default": {t1, t2}, "custom": {}, }, wantLease: map[string][]base.Z{ "default": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}, {Message: t2, Score: now.Add(10 * time.Second).Unix()}}, "custom": {}, }, wantRetry: map[string][]base.Z{ "default": {}, "custom": { // Task message should include the error message but without incrementing the retry count. {Message: h.TaskMessageWithError(*t4, errMsg, now), Score: now.Add(5 * time.Minute).Unix()}, }, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) h.SeedAllActiveQueues(t, r.client, tc.active) h.SeedAllLease(t, r.client, tc.lease) h.SeedAllRetryQueues(t, r.client, tc.retry) err := r.Retry(context.Background(), tc.msg, tc.processAt, tc.errMsg, false /*isFailure*/) if err != nil { t.Errorf("(*RDB).Retry = %v, want nil", err) continue } for queue, want := range tc.wantActive { gotActive := h.GetActiveMessages(t, r.client, queue) if diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.ActiveKey(queue), diff) } } for queue, want := range tc.wantLease { gotLease := h.GetLeaseEntries(t, r.client, queue) if diff := cmp.Diff(want, gotLease, h.SortZSetEntryOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.LeaseKey(queue), diff) } } for queue, want := range tc.wantRetry { gotRetry := h.GetRetryEntries(t, r.client, queue) if diff := cmp.Diff(want, gotRetry, h.SortZSetEntryOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.RetryKey(queue), diff) } } // If isFailure is set to false, no stats should be recorded to avoid skewing the error rate. processedKey := base.ProcessedKey(tc.msg.Queue, time.Now()) gotProcessed := r.client.Get(context.Background(), processedKey).Val() if gotProcessed != "" { t.Errorf("GET %q = %q, want empty", processedKey, gotProcessed) } // If isFailure is set to false, no stats should be recorded to avoid skewing the error rate. failedKey := base.FailedKey(tc.msg.Queue, time.Now()) gotFailed := r.client.Get(context.Background(), failedKey).Val() if gotFailed != "" { t.Errorf("GET %q = %q, want empty", failedKey, gotFailed) } processedTotalKey := base.ProcessedTotalKey(tc.msg.Queue) gotProcessedTotal := r.client.Get(context.Background(), processedTotalKey).Val() if gotProcessedTotal != "" { t.Errorf("GET %q = %q, want empty", processedTotalKey, gotProcessedTotal) } failedTotalKey := base.FailedTotalKey(tc.msg.Queue) gotFailedTotal := r.client.Get(context.Background(), failedTotalKey).Val() if gotFailedTotal != "" { t.Errorf("GET %q = %q, want empty", failedTotalKey, gotFailedTotal) } } } func TestArchive(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() r.SetClock(timeutil.NewSimulatedClock(now)) t1 := &base.TaskMessage{ ID: uuid.NewString(), Type: "send_email", Payload: nil, Queue: "default", Retry: 25, Retried: 25, Timeout: 1800, } t2 := &base.TaskMessage{ ID: uuid.NewString(), Type: "reindex", Payload: nil, Queue: "default", Retry: 25, Retried: 0, Timeout: 3000, } t3 := &base.TaskMessage{ ID: uuid.NewString(), Type: "generate_csv", Payload: nil, Queue: "default", Retry: 25, Retried: 0, Timeout: 60, } t4 := &base.TaskMessage{ ID: uuid.NewString(), Type: "send_email", Payload: nil, Queue: "custom", Retry: 25, Retried: 25, Timeout: 1800, } errMsg := "SMTP server not responding" tests := []struct { active map[string][]*base.TaskMessage lease map[string][]base.Z archived map[string][]base.Z target *base.TaskMessage // task to archive wantActive map[string][]*base.TaskMessage wantLease map[string][]base.Z wantArchived map[string][]base.Z }{ { active: map[string][]*base.TaskMessage{ "default": {t1, t2}, }, lease: map[string][]base.Z{ "default": { {Message: t1, Score: now.Add(10 * time.Second).Unix()}, {Message: t2, Score: now.Add(10 * time.Second).Unix()}, }, }, archived: map[string][]base.Z{ "default": { {Message: t3, Score: now.Add(-time.Hour).Unix()}, }, }, target: t1, wantActive: map[string][]*base.TaskMessage{ "default": {t2}, }, wantLease: map[string][]base.Z{ "default": {{Message: t2, Score: now.Add(10 * time.Second).Unix()}}, }, wantArchived: map[string][]base.Z{ "default": { {Message: h.TaskMessageWithError(*t1, errMsg, now), Score: now.Unix()}, {Message: t3, Score: now.Add(-time.Hour).Unix()}, }, }, }, { active: map[string][]*base.TaskMessage{ "default": {t1, t2, t3}, }, lease: map[string][]base.Z{ "default": { {Message: t1, Score: now.Add(10 * time.Second).Unix()}, {Message: t2, Score: now.Add(10 * time.Second).Unix()}, {Message: t3, Score: now.Add(10 * time.Second).Unix()}, }, }, archived: map[string][]base.Z{ "default": {}, }, target: t1, wantActive: map[string][]*base.TaskMessage{ "default": {t2, t3}, }, wantLease: map[string][]base.Z{ "default": { {Message: t2, Score: now.Add(10 * time.Second).Unix()}, {Message: t3, Score: now.Add(10 * time.Second).Unix()}, }, }, wantArchived: map[string][]base.Z{ "default": { {Message: h.TaskMessageWithError(*t1, errMsg, now), Score: now.Unix()}, }, }, }, { active: map[string][]*base.TaskMessage{ "default": {t1}, "custom": {t4}, }, lease: map[string][]base.Z{ "default": { {Message: t1, Score: now.Add(10 * time.Second).Unix()}, }, "custom": { {Message: t4, Score: now.Add(10 * time.Second).Unix()}, }, }, archived: map[string][]base.Z{ "default": {}, "custom": {}, }, target: t4, wantActive: map[string][]*base.TaskMessage{ "default": {t1}, "custom": {}, }, wantLease: map[string][]base.Z{ "default": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}}, "custom": {}, }, wantArchived: map[string][]base.Z{ "default": {}, "custom": { {Message: h.TaskMessageWithError(*t4, errMsg, now), Score: now.Unix()}, }, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllActiveQueues(t, r.client, tc.active) h.SeedAllLease(t, r.client, tc.lease) h.SeedAllArchivedQueues(t, r.client, tc.archived) err := r.Archive(context.Background(), tc.target, errMsg) if err != nil { t.Errorf("(*RDB).Archive(%v, %v) = %v, want nil", tc.target, errMsg, err) continue } for queue, want := range tc.wantActive { gotActive := h.GetActiveMessages(t, r.client, queue) if diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q: (-want, +got)\n%s", base.ActiveKey(queue), diff) } } for queue, want := range tc.wantLease { gotLease := h.GetLeaseEntries(t, r.client, queue) if diff := cmp.Diff(want, gotLease, h.SortZSetEntryOpt); diff != "" { t.Errorf("mismatch found in %q after calling (*RDB).Archive: (-want, +got):\n%s", base.LeaseKey(queue), diff) } } for queue, want := range tc.wantArchived { gotArchived := h.GetArchivedEntries(t, r.client, queue) if diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt, zScoreCmpOpt, timeCmpOpt); diff != "" { t.Errorf("mismatch found in %q after calling (*RDB).Archive: (-want, +got):\n%s", base.ArchivedKey(queue), diff) } } processedKey := base.ProcessedKey(tc.target.Queue, time.Now()) gotProcessed := r.client.Get(context.Background(), processedKey).Val() if gotProcessed != "1" { t.Errorf("GET %q = %q, want 1", processedKey, gotProcessed) } gotTTL := r.client.TTL(context.Background(), processedKey).Val() if gotTTL > statsTTL { t.Errorf("TTL %q = %v, want less than or equal to %v", processedKey, gotTTL, statsTTL) } failedKey := base.FailedKey(tc.target.Queue, time.Now()) gotFailed := r.client.Get(context.Background(), failedKey).Val() if gotFailed != "1" { t.Errorf("GET %q = %q, want 1", failedKey, gotFailed) } gotTTL = r.client.TTL(context.Background(), processedKey).Val() if gotTTL > statsTTL { t.Errorf("TTL %q = %v, want less than or equal to %v", failedKey, gotTTL, statsTTL) } processedTotalKey := base.ProcessedTotalKey(tc.target.Queue) gotProcessedTotal := r.client.Get(context.Background(), processedTotalKey).Val() if gotProcessedTotal != "1" { t.Errorf("GET %q = %q, want 1", processedTotalKey, gotProcessedTotal) } failedTotalKey := base.FailedTotalKey(tc.target.Queue) gotFailedTotal := r.client.Get(context.Background(), failedTotalKey).Val() if gotFailedTotal != "1" { t.Errorf("GET %q = %q, want 1", failedTotalKey, gotFailedTotal) } } } func TestArchiveTrim(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() r.SetClock(timeutil.NewSimulatedClock(now)) t1 := &base.TaskMessage{ ID: uuid.NewString(), Type: "send_email", Payload: nil, Queue: "default", Retry: 25, Retried: 25, Timeout: 1800, } t2 := &base.TaskMessage{ ID: uuid.NewString(), Type: "reindex", Payload: nil, Queue: "default", Retry: 25, Retried: 0, Timeout: 3000, } errMsg := "SMTP server not responding" maxArchiveSet := make([]base.Z, 0) for i := 0; i < maxArchiveSize-1; i++ { maxArchiveSet = append(maxArchiveSet, base.Z{Message: &base.TaskMessage{ ID: uuid.NewString(), Type: "generate_csv", Payload: nil, Queue: "default", Retry: 25, Retried: 0, Timeout: 60, }, Score: now.Add(-time.Hour + -time.Second*time.Duration(i)).Unix()}) } wantMaxArchiveSet := make([]base.Z, 0) // newly archived task should be at the front wantMaxArchiveSet = append(wantMaxArchiveSet, base.Z{Message: h.TaskMessageWithError(*t1, errMsg, now), Score: now.Unix()}) // oldest task should be dropped from the set wantMaxArchiveSet = append(wantMaxArchiveSet, maxArchiveSet[:len(maxArchiveSet)-1]...) tests := []struct { toArchive map[string][]*base.TaskMessage lease map[string][]base.Z archived map[string][]base.Z wantArchived map[string][]base.Z }{ { // simple, 1 to be archived, 1 already archived, both are in the archive set toArchive: map[string][]*base.TaskMessage{ "default": {t1}, }, lease: map[string][]base.Z{ "default": { {Message: t1, Score: now.Add(10 * time.Second).Unix()}, }, }, archived: map[string][]base.Z{ "default": { {Message: t2, Score: now.Add(-time.Hour).Unix()}, }, }, wantArchived: map[string][]base.Z{ "default": { {Message: h.TaskMessageWithError(*t1, errMsg, now), Score: now.Unix()}, {Message: t2, Score: now.Add(-time.Hour).Unix()}, }, }, }, { // 1 to be archived, 1 already archived but past expiry, only the newly archived task should be left toArchive: map[string][]*base.TaskMessage{ "default": {t1}, }, lease: map[string][]base.Z{ "default": { {Message: t1, Score: now.Add(10 * time.Second).Unix()}, }, }, archived: map[string][]base.Z{ "default": { {Message: t2, Score: now.Add(-time.Hour * 24 * (archivedExpirationInDays + 1)).Unix()}, }, }, wantArchived: map[string][]base.Z{ "default": { {Message: h.TaskMessageWithError(*t1, errMsg, now), Score: now.Unix()}, }, }, }, { // 1 to be archived, maxArchiveSize in archive set, archive set should be trimmed back to maxArchiveSize and newly archived task should be in the set toArchive: map[string][]*base.TaskMessage{ "default": {t1}, }, lease: map[string][]base.Z{ "default": { {Message: t1, Score: now.Add(10 * time.Second).Unix()}, }, }, archived: map[string][]base.Z{ "default": maxArchiveSet, }, wantArchived: map[string][]base.Z{ "default": wantMaxArchiveSet, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllActiveQueues(t, r.client, tc.toArchive) h.SeedAllLease(t, r.client, tc.lease) h.SeedAllArchivedQueues(t, r.client, tc.archived) for _, tasks := range tc.toArchive { for _, target := range tasks { err := r.Archive(context.Background(), target, errMsg) if err != nil { t.Errorf("(*RDB).Archive(%v, %v) = %v, want nil", target, errMsg, err) continue } } } for queue, want := range tc.wantArchived { gotArchived := h.GetArchivedEntries(t, r.client, queue) if diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt, zScoreCmpOpt, timeCmpOpt); diff != "" { t.Errorf("mismatch found in %q after calling (*RDB).Archive: (-want, +got):\n%s", base.ArchivedKey(queue), diff) } // check that only keys present in the archived set are in rdb vals := r.client.Keys(context.Background(), base.TaskKeyPrefix(queue)+"*").Val() if len(vals) != len(gotArchived) { t.Errorf("len of keys = %v, want %v", len(vals), len(gotArchived)) return } for _, val := range vals { found := false for _, entry := range gotArchived { if strings.Contains(val, entry.Message.ID) { found = true break } } if !found { t.Errorf("key %v not found in archived set (it was orphaned by the archive trim)", val) } } } } } func TestForwardIfReadyWithGroup(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() r.SetClock(timeutil.NewSimulatedClock(now)) t1 := h.NewTaskMessage("send_email", nil) t2 := h.NewTaskMessage("generate_csv", nil) t3 := h.NewTaskMessage("gen_thumbnail", nil) t4 := h.NewTaskMessageWithQueue("important_task", nil, "critical") t5 := h.NewTaskMessageWithQueue("minor_task", nil, "low") // Set group keys for the tasks. t1.GroupKey = "notifications" t2.GroupKey = "csv" t4.GroupKey = "critical_task_group" t5.GroupKey = "minor_task_group" ctx := context.Background() secondAgo := now.Add(-time.Second) tests := []struct { scheduled map[string][]base.Z retry map[string][]base.Z qnames []string wantPending map[string][]*base.TaskMessage wantScheduled map[string][]*base.TaskMessage wantRetry map[string][]*base.TaskMessage wantGroup map[string]map[string][]base.Z }{ { scheduled: map[string][]base.Z{ "default": { {Message: t1, Score: secondAgo.Unix()}, {Message: t2, Score: secondAgo.Unix()}, }, }, retry: map[string][]base.Z{ "default": {{Message: t3, Score: secondAgo.Unix()}}, }, qnames: []string{"default"}, wantPending: map[string][]*base.TaskMessage{ "default": {t3}, }, wantScheduled: map[string][]*base.TaskMessage{ "default": {}, }, wantRetry: map[string][]*base.TaskMessage{ "default": {}, }, wantGroup: map[string]map[string][]base.Z{ "default": { "notifications": {{Message: t1, Score: now.Unix()}}, "csv": {{Message: t2, Score: now.Unix()}}, }, }, }, { scheduled: map[string][]base.Z{ "default": {{Message: t1, Score: secondAgo.Unix()}}, "critical": {{Message: t4, Score: secondAgo.Unix()}}, "low": {}, }, retry: map[string][]base.Z{ "default": {}, "critical": {}, "low": {{Message: t5, Score: secondAgo.Unix()}}, }, qnames: []string{"default", "critical", "low"}, wantPending: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, "low": {}, }, wantScheduled: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, "low": {}, }, wantRetry: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, "low": {}, }, wantGroup: map[string]map[string][]base.Z{ "default": { "notifications": {{Message: t1, Score: now.Unix()}}, }, "critical": { "critical_task_group": {{Message: t4, Score: now.Unix()}}, }, "low": { "minor_task_group": {{Message: t5, Score: now.Unix()}}, }, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllScheduledQueues(t, r.client, tc.scheduled) h.SeedAllRetryQueues(t, r.client, tc.retry) err := r.ForwardIfReady(tc.qnames...) if err != nil { t.Errorf("(*RDB).ForwardIfReady(%v) = %v, want nil", tc.qnames, err) continue } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r.client, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.PendingKey(qname), diff) } // Make sure "pending_since" field is set for _, msg := range gotPending { pendingSince := r.client.HGet(ctx, base.TaskKey(msg.Queue, msg.ID), "pending_since").Val() if want := strconv.Itoa(int(now.UnixNano())); pendingSince != want { t.Error("pending_since field is not set for newly pending message") } } } for qname, want := range tc.wantScheduled { gotScheduled := h.GetScheduledMessages(t, r.client, qname) if diff := cmp.Diff(want, gotScheduled, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.ScheduledKey(qname), diff) } } for qname, want := range tc.wantRetry { gotRetry := h.GetRetryMessages(t, r.client, qname) if diff := cmp.Diff(want, gotRetry, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.RetryKey(qname), diff) } } for qname, groups := range tc.wantGroup { for groupKey, wantGroup := range groups { gotGroup := h.GetGroupEntries(t, r.client, qname, groupKey) if diff := cmp.Diff(wantGroup, gotGroup, h.SortZSetEntryOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.GroupKey(qname, groupKey), diff) } } } } } func TestForwardIfReady(t *testing.T) { r := setup(t) defer r.Close() t1 := h.NewTaskMessage("send_email", nil) t2 := h.NewTaskMessage("generate_csv", nil) t3 := h.NewTaskMessage("gen_thumbnail", nil) t4 := h.NewTaskMessageWithQueue("important_task", nil, "critical") t5 := h.NewTaskMessageWithQueue("minor_task", nil, "low") secondAgo := time.Now().Add(-time.Second) hourFromNow := time.Now().Add(time.Hour) tests := []struct { scheduled map[string][]base.Z retry map[string][]base.Z qnames []string wantPending map[string][]*base.TaskMessage wantScheduled map[string][]*base.TaskMessage wantRetry map[string][]*base.TaskMessage }{ { scheduled: map[string][]base.Z{ "default": { {Message: t1, Score: secondAgo.Unix()}, {Message: t2, Score: secondAgo.Unix()}, }, }, retry: map[string][]base.Z{ "default": {{Message: t3, Score: secondAgo.Unix()}}, }, qnames: []string{"default"}, wantPending: map[string][]*base.TaskMessage{ "default": {t1, t2, t3}, }, wantScheduled: map[string][]*base.TaskMessage{ "default": {}, }, wantRetry: map[string][]*base.TaskMessage{ "default": {}, }, }, { scheduled: map[string][]base.Z{ "default": { {Message: t1, Score: hourFromNow.Unix()}, {Message: t2, Score: secondAgo.Unix()}, }, }, retry: map[string][]base.Z{ "default": {{Message: t3, Score: secondAgo.Unix()}}, }, qnames: []string{"default"}, wantPending: map[string][]*base.TaskMessage{ "default": {t2, t3}, }, wantScheduled: map[string][]*base.TaskMessage{ "default": {t1}, }, wantRetry: map[string][]*base.TaskMessage{ "default": {}, }, }, { scheduled: map[string][]base.Z{ "default": { {Message: t1, Score: hourFromNow.Unix()}, {Message: t2, Score: hourFromNow.Unix()}, }, }, retry: map[string][]base.Z{ "default": {{Message: t3, Score: hourFromNow.Unix()}}, }, qnames: []string{"default"}, wantPending: map[string][]*base.TaskMessage{ "default": {}, }, wantScheduled: map[string][]*base.TaskMessage{ "default": {t1, t2}, }, wantRetry: map[string][]*base.TaskMessage{ "default": {t3}, }, }, { scheduled: map[string][]base.Z{ "default": {{Message: t1, Score: secondAgo.Unix()}}, "critical": {{Message: t4, Score: secondAgo.Unix()}}, "low": {}, }, retry: map[string][]base.Z{ "default": {}, "critical": {}, "low": {{Message: t5, Score: secondAgo.Unix()}}, }, qnames: []string{"default", "critical", "low"}, wantPending: map[string][]*base.TaskMessage{ "default": {t1}, "critical": {t4}, "low": {t5}, }, wantScheduled: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, "low": {}, }, wantRetry: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, "low": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) // clean up db before each test case h.SeedAllScheduledQueues(t, r.client, tc.scheduled) h.SeedAllRetryQueues(t, r.client, tc.retry) now := time.Now() r.SetClock(timeutil.NewSimulatedClock(now)) err := r.ForwardIfReady(tc.qnames...) if err != nil { t.Errorf("(*RDB).ForwardIfReady(%v) = %v, want nil", tc.qnames, err) continue } for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r.client, qname) if diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.PendingKey(qname), diff) } // Make sure "pending_since" field is set for _, msg := range gotPending { pendingSince := r.client.HGet(context.Background(), base.TaskKey(msg.Queue, msg.ID), "pending_since").Val() if want := strconv.Itoa(int(now.UnixNano())); pendingSince != want { t.Error("pending_since field is not set for newly pending message") } } } for qname, want := range tc.wantScheduled { gotScheduled := h.GetScheduledMessages(t, r.client, qname) if diff := cmp.Diff(want, gotScheduled, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.ScheduledKey(qname), diff) } } for qname, want := range tc.wantRetry { gotRetry := h.GetRetryMessages(t, r.client, qname) if diff := cmp.Diff(want, gotRetry, h.SortMsgOpt); diff != "" { t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.RetryKey(qname), diff) } } } } func newCompletedTask(qname, typename string, payload []byte, completedAt time.Time) *base.TaskMessage { msg := h.NewTaskMessageWithQueue(typename, payload, qname) msg.CompletedAt = completedAt.Unix() return msg } func TestDeleteExpiredCompletedTasks(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() secondAgo := now.Add(-time.Second) hourFromNow := now.Add(time.Hour) hourAgo := now.Add(-time.Hour) minuteAgo := now.Add(-time.Minute) t1 := newCompletedTask("default", "task1", nil, hourAgo) t2 := newCompletedTask("default", "task2", nil, minuteAgo) t3 := newCompletedTask("default", "task3", nil, secondAgo) t4 := newCompletedTask("critical", "critical_task", nil, hourAgo) t5 := newCompletedTask("low", "low_priority_task", nil, hourAgo) tests := []struct { desc string completed map[string][]base.Z qname string wantCompleted map[string][]base.Z }{ { desc: "deletes expired task from default queue", completed: map[string][]base.Z{ "default": { {Message: t1, Score: secondAgo.Unix()}, {Message: t2, Score: hourFromNow.Unix()}, {Message: t3, Score: now.Unix()}, }, }, qname: "default", wantCompleted: map[string][]base.Z{ "default": { {Message: t2, Score: hourFromNow.Unix()}, }, }, }, { desc: "deletes expired task from specified queue", completed: map[string][]base.Z{ "default": { {Message: t2, Score: secondAgo.Unix()}, }, "critical": { {Message: t4, Score: secondAgo.Unix()}, }, "low": { {Message: t5, Score: now.Unix()}, }, }, qname: "critical", wantCompleted: map[string][]base.Z{ "default": { {Message: t2, Score: secondAgo.Unix()}, }, "critical": {}, "low": { {Message: t5, Score: now.Unix()}, }, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) h.SeedAllCompletedQueues(t, r.client, tc.completed) if err := r.DeleteExpiredCompletedTasks(tc.qname, 100); err != nil { t.Errorf("DeleteExpiredCompletedTasks(%q, 100) failed: %v", tc.qname, err) continue } for qname, want := range tc.wantCompleted { got := h.GetCompletedEntries(t, r.client, qname) if diff := cmp.Diff(want, got, h.SortZSetEntryOpt); diff != "" { t.Errorf("%s: diff found in %q completed set: want=%v, got=%v\n%s", tc.desc, qname, want, got, diff) } } } } func TestListLeaseExpired(t *testing.T) { t1 := h.NewTaskMessageWithQueue("task1", nil, "default") t2 := h.NewTaskMessageWithQueue("task2", nil, "default") t3 := h.NewTaskMessageWithQueue("task3", nil, "critical") now := time.Now() tests := []struct { desc string lease map[string][]base.Z qnames []string cutoff time.Time want []*base.TaskMessage }{ { desc: "with a single active task", lease: map[string][]base.Z{ "default": {{Message: t1, Score: now.Add(-10 * time.Second).Unix()}}, }, qnames: []string{"default"}, cutoff: now, want: []*base.TaskMessage{t1}, }, { desc: "with multiple active tasks, and one expired", lease: map[string][]base.Z{ "default": { {Message: t1, Score: now.Add(-5 * time.Minute).Unix()}, {Message: t2, Score: now.Add(20 * time.Second).Unix()}, }, "critical": { {Message: t3, Score: now.Add(10 * time.Second).Unix()}, }, }, qnames: []string{"default", "critical"}, cutoff: now, want: []*base.TaskMessage{t1}, }, { desc: "with multiple expired active tasks", lease: map[string][]base.Z{ "default": { {Message: t1, Score: now.Add(-2 * time.Minute).Unix()}, {Message: t2, Score: now.Add(20 * time.Second).Unix()}, }, "critical": { {Message: t3, Score: now.Add(-30 * time.Second).Unix()}, }, }, qnames: []string{"default", "critical"}, cutoff: now, want: []*base.TaskMessage{t1, t3}, }, { desc: "with empty active queue", lease: map[string][]base.Z{ "default": {}, "critical": {}, }, qnames: []string{"default", "critical"}, cutoff: now, want: []*base.TaskMessage{}, }, } r := setup(t) defer r.Close() for _, tc := range tests { h.FlushDB(t, r.client) h.SeedAllLease(t, r.client, tc.lease) got, err := r.ListLeaseExpired(tc.cutoff, tc.qnames...) if err != nil { t.Errorf("%s; ListLeaseExpired(%v) returned error: %v", tc.desc, tc.cutoff, err) continue } if diff := cmp.Diff(tc.want, got, h.SortMsgOpt); diff != "" { t.Errorf("%s; ListLeaseExpired(%v) returned %v, want %v;(-want,+got)\n%s", tc.desc, tc.cutoff, got, tc.want, diff) } } } func TestExtendLease(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() r.SetClock(timeutil.NewSimulatedClock(now)) t1 := h.NewTaskMessageWithQueue("task1", nil, "default") t2 := h.NewTaskMessageWithQueue("task2", nil, "default") t3 := h.NewTaskMessageWithQueue("task3", nil, "critical") t4 := h.NewTaskMessageWithQueue("task4", nil, "default") tests := []struct { desc string lease map[string][]base.Z qname string ids []string wantExpirationTime time.Time wantLease map[string][]base.Z }{ { desc: "Should extends lease for a single message in a queue", lease: map[string][]base.Z{ "default": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}}, "critical": {{Message: t3, Score: now.Add(10 * time.Second).Unix()}}, }, qname: "default", ids: []string{t1.ID}, wantExpirationTime: now.Add(LeaseDuration), wantLease: map[string][]base.Z{ "default": {{Message: t1, Score: now.Add(LeaseDuration).Unix()}}, "critical": {{Message: t3, Score: now.Add(10 * time.Second).Unix()}}, }, }, { desc: "Should extends lease for multiple message in a queue", lease: map[string][]base.Z{ "default": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}, {Message: t2, Score: now.Add(10 * time.Second).Unix()}}, "critical": {{Message: t3, Score: now.Add(10 * time.Second).Unix()}}, }, qname: "default", ids: []string{t1.ID, t2.ID}, wantExpirationTime: now.Add(LeaseDuration), wantLease: map[string][]base.Z{ "default": {{Message: t1, Score: now.Add(LeaseDuration).Unix()}, {Message: t2, Score: now.Add(LeaseDuration).Unix()}}, "critical": {{Message: t3, Score: now.Add(10 * time.Second).Unix()}}, }, }, { desc: "Should selectively extends lease for messages in a queue", lease: map[string][]base.Z{ "default": { {Message: t1, Score: now.Add(10 * time.Second).Unix()}, {Message: t2, Score: now.Add(10 * time.Second).Unix()}, {Message: t4, Score: now.Add(10 * time.Second).Unix()}, }, "critical": {{Message: t3, Score: now.Add(10 * time.Second).Unix()}}, }, qname: "default", ids: []string{t2.ID, t4.ID}, wantExpirationTime: now.Add(LeaseDuration), wantLease: map[string][]base.Z{ "default": { {Message: t1, Score: now.Add(10 * time.Second).Unix()}, {Message: t2, Score: now.Add(LeaseDuration).Unix()}, {Message: t4, Score: now.Add(LeaseDuration).Unix()}, }, "critical": {{Message: t3, Score: now.Add(10 * time.Second).Unix()}}, }, }, { desc: "Should not add a new entry in the lease set", lease: map[string][]base.Z{ "default": { {Message: t1, Score: now.Add(10 * time.Second).Unix()}, }, }, qname: "default", ids: []string{t1.ID, t2.ID}, wantExpirationTime: now.Add(LeaseDuration), wantLease: map[string][]base.Z{ "default": { {Message: t1, Score: now.Add(LeaseDuration).Unix()}, }, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) h.SeedAllLease(t, r.client, tc.lease) gotExpirationTime, err := r.ExtendLease(tc.qname, tc.ids...) if err != nil { t.Fatalf("%s: ExtendLease(%q, %v) returned error: %v", tc.desc, tc.qname, tc.ids, err) } if gotExpirationTime != tc.wantExpirationTime { t.Errorf("%s: ExtendLease(%q, %v) returned expirationTime %v, want %v", tc.desc, tc.qname, tc.ids, gotExpirationTime, tc.wantExpirationTime) } for qname, want := range tc.wantLease { gotLease := h.GetLeaseEntries(t, r.client, qname) if diff := cmp.Diff(want, gotLease, h.SortZSetEntryOpt); diff != "" { t.Errorf("%s: mismatch found in %q: (-want,+got):\n%s", tc.desc, base.LeaseKey(qname), diff) } } } } func TestWriteServerState(t *testing.T) { r := setup(t) defer r.Close() var ( host = "localhost" pid = 4242 serverID = "server123" ttl = 5 * time.Second ) info := base.ServerInfo{ Host: host, PID: pid, ServerID: serverID, Concurrency: 10, Queues: map[string]int{"default": 2, "email": 5, "low": 1}, StrictPriority: false, Started: time.Now().UTC(), Status: "active", ActiveWorkerCount: 0, } err := r.WriteServerState(&info, nil /* workers */, ttl) if err != nil { t.Errorf("r.WriteServerState returned an error: %v", err) } // Check ServerInfo was written correctly. skey := base.ServerInfoKey(host, pid, serverID) data := r.client.Get(context.Background(), skey).Val() got, err := base.DecodeServerInfo([]byte(data)) if err != nil { t.Fatalf("could not decode server info: %v", err) } if diff := cmp.Diff(info, *got); diff != "" { t.Errorf("persisted ServerInfo was %v, want %v; (-want,+got)\n%s", got, info, diff) } // Check ServerInfo TTL was set correctly. gotTTL := r.client.TTL(context.Background(), skey).Val() if !cmp.Equal(ttl.Seconds(), gotTTL.Seconds(), cmpopts.EquateApprox(0, 1)) { t.Errorf("TTL of %q was %v, want %v", skey, gotTTL, ttl) } // Check ServerInfo key was added to the set all server keys correctly. gotServerKeys := r.client.ZRange(context.Background(), base.AllServers, 0, -1).Val() wantServerKeys := []string{skey} if diff := cmp.Diff(wantServerKeys, gotServerKeys); diff != "" { t.Errorf("%q contained %v, want %v", base.AllServers, gotServerKeys, wantServerKeys) } // Check WorkersInfo was written correctly. wkey := base.WorkersKey(host, pid, serverID) workerExist := r.client.Exists(context.Background(), wkey).Val() if workerExist != 0 { t.Errorf("%q key exists", wkey) } // Check WorkersInfo key was added to the set correctly. gotWorkerKeys := r.client.ZRange(context.Background(), base.AllWorkers, 0, -1).Val() wantWorkerKeys := []string{wkey} if diff := cmp.Diff(wantWorkerKeys, gotWorkerKeys); diff != "" { t.Errorf("%q contained %v, want %v", base.AllWorkers, gotWorkerKeys, wantWorkerKeys) } } func TestWriteServerStateWithWorkers(t *testing.T) { r := setup(t) defer r.Close() var ( host = "127.0.0.1" pid = 4242 serverID = "server123" msg1 = h.NewTaskMessage("send_email", h.JSON(map[string]interface{}{"user_id": "123"})) msg2 = h.NewTaskMessage("gen_thumbnail", h.JSON(map[string]interface{}{"path": "some/path/to/imgfile"})) ttl = 5 * time.Second ) workers := []*base.WorkerInfo{ { Host: host, PID: pid, ID: msg1.ID, Type: msg1.Type, Queue: msg1.Queue, Payload: msg1.Payload, Started: time.Now().Add(-10 * time.Second), }, { Host: host, PID: pid, ID: msg2.ID, Type: msg2.Type, Queue: msg2.Queue, Payload: msg2.Payload, Started: time.Now().Add(-2 * time.Minute), }, } serverInfo := base.ServerInfo{ Host: host, PID: pid, ServerID: serverID, Concurrency: 10, Queues: map[string]int{"default": 2, "email": 5, "low": 1}, StrictPriority: false, Started: time.Now().Add(-10 * time.Minute).UTC(), Status: "active", ActiveWorkerCount: len(workers), } err := r.WriteServerState(&serverInfo, workers, ttl) if err != nil { t.Fatalf("r.WriteServerState returned an error: %v", err) } // Check ServerInfo was written correctly. skey := base.ServerInfoKey(host, pid, serverID) data := r.client.Get(context.Background(), skey).Val() got, err := base.DecodeServerInfo([]byte(data)) if err != nil { t.Fatalf("could not decode server info: %v", err) } if diff := cmp.Diff(serverInfo, *got); diff != "" { t.Errorf("persisted ServerInfo was %v, want %v; (-want,+got)\n%s", got, serverInfo, diff) } // Check ServerInfo TTL was set correctly. gotTTL := r.client.TTL(context.Background(), skey).Val() if !cmp.Equal(ttl.Seconds(), gotTTL.Seconds(), cmpopts.EquateApprox(0, 1)) { t.Errorf("TTL of %q was %v, want %v", skey, gotTTL, ttl) } // Check ServerInfo key was added to the set correctly. gotServerKeys := r.client.ZRange(context.Background(), base.AllServers, 0, -1).Val() wantServerKeys := []string{skey} if diff := cmp.Diff(wantServerKeys, gotServerKeys); diff != "" { t.Errorf("%q contained %v, want %v", base.AllServers, gotServerKeys, wantServerKeys) } // Check WorkersInfo was written correctly. wkey := base.WorkersKey(host, pid, serverID) wdata := r.client.HGetAll(context.Background(), wkey).Val() if len(wdata) != 2 { t.Fatalf("HGETALL %q returned a hash of size %d, want 2", wkey, len(wdata)) } var gotWorkers []*base.WorkerInfo for _, val := range wdata { w, err := base.DecodeWorkerInfo([]byte(val)) if err != nil { t.Fatalf("could not unmarshal worker's data: %v", err) } gotWorkers = append(gotWorkers, w) } if diff := cmp.Diff(workers, gotWorkers, h.SortWorkerInfoOpt); diff != "" { t.Errorf("persisted workers info was %v, want %v; (-want,+got)\n%s", gotWorkers, workers, diff) } // Check WorkersInfo TTL was set correctly. gotTTL = r.client.TTL(context.Background(), wkey).Val() if !cmp.Equal(ttl.Seconds(), gotTTL.Seconds(), cmpopts.EquateApprox(0, 1)) { t.Errorf("TTL of %q was %v, want %v", wkey, gotTTL, ttl) } // Check WorkersInfo key was added to the set correctly. gotWorkerKeys := r.client.ZRange(context.Background(), base.AllWorkers, 0, -1).Val() wantWorkerKeys := []string{wkey} if diff := cmp.Diff(wantWorkerKeys, gotWorkerKeys); diff != "" { t.Errorf("%q contained %v, want %v", base.AllWorkers, gotWorkerKeys, wantWorkerKeys) } } func TestClearServerState(t *testing.T) { r := setup(t) defer r.Close() var ( host = "127.0.0.1" pid = 1234 serverID = "server123" otherHost = "127.0.0.2" otherPID = 9876 otherServerID = "server987" msg1 = h.NewTaskMessage("send_email", h.JSON(map[string]interface{}{"user_id": "123"})) msg2 = h.NewTaskMessage("gen_thumbnail", h.JSON(map[string]interface{}{"path": "some/path/to/imgfile"})) ttl = 5 * time.Second ) workers1 := []*base.WorkerInfo{ { Host: host, PID: pid, ID: msg1.ID, Type: msg1.Type, Queue: msg1.Queue, Payload: msg1.Payload, Started: time.Now().Add(-10 * time.Second), }, } serverInfo1 := base.ServerInfo{ Host: host, PID: pid, ServerID: serverID, Concurrency: 10, Queues: map[string]int{"default": 2, "email": 5, "low": 1}, StrictPriority: false, Started: time.Now().Add(-10 * time.Minute), Status: "active", ActiveWorkerCount: len(workers1), } workers2 := []*base.WorkerInfo{ { Host: otherHost, PID: otherPID, ID: msg2.ID, Type: msg2.Type, Queue: msg2.Queue, Payload: msg2.Payload, Started: time.Now().Add(-30 * time.Second), }, } serverInfo2 := base.ServerInfo{ Host: otherHost, PID: otherPID, ServerID: otherServerID, Concurrency: 10, Queues: map[string]int{"default": 2, "email": 5, "low": 1}, StrictPriority: false, Started: time.Now().Add(-15 * time.Minute), Status: "active", ActiveWorkerCount: len(workers2), } // Write server and workers data. if err := r.WriteServerState(&serverInfo1, workers1, ttl); err != nil { t.Fatalf("could not write server state: %v", err) } if err := r.WriteServerState(&serverInfo2, workers2, ttl); err != nil { t.Fatalf("could not write server state: %v", err) } err := r.ClearServerState(host, pid, serverID) if err != nil { t.Fatalf("(*RDB).ClearServerState failed: %v", err) } skey := base.ServerInfoKey(host, pid, serverID) wkey := base.WorkersKey(host, pid, serverID) otherSKey := base.ServerInfoKey(otherHost, otherPID, otherServerID) otherWKey := base.WorkersKey(otherHost, otherPID, otherServerID) // Check all keys are cleared. if r.client.Exists(context.Background(), skey).Val() != 0 { t.Errorf("Redis key %q exists", skey) } if r.client.Exists(context.Background(), wkey).Val() != 0 { t.Errorf("Redis key %q exists", wkey) } gotServerKeys := r.client.ZRange(context.Background(), base.AllServers, 0, -1).Val() wantServerKeys := []string{otherSKey} if diff := cmp.Diff(wantServerKeys, gotServerKeys); diff != "" { t.Errorf("%q contained %v, want %v", base.AllServers, gotServerKeys, wantServerKeys) } gotWorkerKeys := r.client.ZRange(context.Background(), base.AllWorkers, 0, -1).Val() wantWorkerKeys := []string{otherWKey} if diff := cmp.Diff(wantWorkerKeys, gotWorkerKeys); diff != "" { t.Errorf("%q contained %v, want %v", base.AllWorkers, gotWorkerKeys, wantWorkerKeys) } } func TestCancelationPubSub(t *testing.T) { r := setup(t) defer r.Close() pubsub, err := r.CancelationPubSub() if err != nil { t.Fatalf("(*RDB).CancelationPubSub() returned an error: %v", err) } cancelCh := pubsub.Channel() var ( mu sync.Mutex received []string ) go func() { for msg := range cancelCh { mu.Lock() received = append(received, msg.Payload) mu.Unlock() } }() publish := []string{"one", "two", "three"} for _, msg := range publish { _ = r.PublishCancelation(msg) } // allow for message to reach subscribers. time.Sleep(time.Second) pubsub.Close() mu.Lock() if diff := cmp.Diff(publish, received, h.SortStringSliceOpt); diff != "" { t.Errorf("subscriber received %v, want %v; (-want,+got)\n%s", received, publish, diff) } mu.Unlock() } func TestWriteResult(t *testing.T) { r := setup(t) defer r.Close() tests := []struct { qname string taskID string data []byte }{ { qname: "default", taskID: uuid.NewString(), data: []byte("hello"), }, } for _, tc := range tests { h.FlushDB(t, r.client) n, err := r.WriteResult(tc.qname, tc.taskID, tc.data) if err != nil { t.Errorf("WriteResult failed: %v", err) continue } if n != len(tc.data) { t.Errorf("WriteResult returned %d, want %d", n, len(tc.data)) } taskKey := base.TaskKey(tc.qname, tc.taskID) got := r.client.HGet(context.Background(), taskKey, "result").Val() if got != string(tc.data) { t.Errorf("`result` field under %q key is set to %q, want %q", taskKey, got, string(tc.data)) } } } func TestAggregationCheck(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() r.SetClock(timeutil.NewSimulatedClock(now)) ctx := context.Background() msg1 := h.NewTaskMessageBuilder().SetType("task1").SetGroup("mygroup").Build() msg2 := h.NewTaskMessageBuilder().SetType("task2").SetGroup("mygroup").Build() msg3 := h.NewTaskMessageBuilder().SetType("task3").SetGroup("mygroup").Build() msg4 := h.NewTaskMessageBuilder().SetType("task4").SetGroup("mygroup").Build() msg5 := h.NewTaskMessageBuilder().SetType("task5").SetGroup("mygroup").Build() tests := []struct { desc string // initial data tasks []*h.TaskSeedData groups map[string][]redis.Z allGroups map[string][]string // args qname string gname string gracePeriod time.Duration maxDelay time.Duration maxSize int // expectaions shouldCreateSet bool // whether the check should create a new aggregation set wantAggregationSet []*base.TaskMessage wantGroups map[string][]redis.Z shouldClearGroup bool // whether the check should clear the group from redis }{ { desc: "with an empty group", tasks: []*h.TaskSeedData{}, groups: map[string][]redis.Z{ base.GroupKey("default", "mygroup"): {}, }, allGroups: map[string][]string{ base.AllGroups("default"): {}, }, qname: "default", gname: "mygroup", gracePeriod: 1 * time.Minute, maxDelay: 10 * time.Minute, maxSize: 5, shouldCreateSet: false, wantAggregationSet: nil, wantGroups: map[string][]redis.Z{ base.GroupKey("default", "mygroup"): {}, }, shouldClearGroup: true, }, { desc: "with a group size reaching the max size", tasks: []*h.TaskSeedData{ {Msg: msg1, State: base.TaskStateAggregating}, {Msg: msg2, State: base.TaskStateAggregating}, {Msg: msg3, State: base.TaskStateAggregating}, {Msg: msg4, State: base.TaskStateAggregating}, {Msg: msg5, State: base.TaskStateAggregating}, }, groups: map[string][]redis.Z{ base.GroupKey("default", "mygroup"): { {Member: msg1.ID, Score: float64(now.Add(-5 * time.Minute).Unix())}, {Member: msg2.ID, Score: float64(now.Add(-3 * time.Minute).Unix())}, {Member: msg3.ID, Score: float64(now.Add(-2 * time.Minute).Unix())}, {Member: msg4.ID, Score: float64(now.Add(-1 * time.Minute).Unix())}, {Member: msg5.ID, Score: float64(now.Add(-10 * time.Second).Unix())}, }, }, allGroups: map[string][]string{ base.AllGroups("default"): {"mygroup"}, }, qname: "default", gname: "mygroup", gracePeriod: 1 * time.Minute, maxDelay: 10 * time.Minute, maxSize: 5, shouldCreateSet: true, wantAggregationSet: []*base.TaskMessage{msg1, msg2, msg3, msg4, msg5}, wantGroups: map[string][]redis.Z{ base.GroupKey("default", "mygroup"): {}, }, shouldClearGroup: true, }, { desc: "with group size greater than max size", tasks: []*h.TaskSeedData{ {Msg: msg1, State: base.TaskStateAggregating}, {Msg: msg2, State: base.TaskStateAggregating}, {Msg: msg3, State: base.TaskStateAggregating}, {Msg: msg4, State: base.TaskStateAggregating}, {Msg: msg5, State: base.TaskStateAggregating}, }, groups: map[string][]redis.Z{ base.GroupKey("default", "mygroup"): { {Member: msg1.ID, Score: float64(now.Add(-5 * time.Minute).Unix())}, {Member: msg2.ID, Score: float64(now.Add(-3 * time.Minute).Unix())}, {Member: msg3.ID, Score: float64(now.Add(-2 * time.Minute).Unix())}, {Member: msg4.ID, Score: float64(now.Add(-1 * time.Minute).Unix())}, {Member: msg5.ID, Score: float64(now.Add(-10 * time.Second).Unix())}, }, }, allGroups: map[string][]string{ base.AllGroups("default"): {"mygroup"}, }, qname: "default", gname: "mygroup", gracePeriod: 2 * time.Minute, maxDelay: 10 * time.Minute, maxSize: 3, shouldCreateSet: true, wantAggregationSet: []*base.TaskMessage{msg1, msg2, msg3}, wantGroups: map[string][]redis.Z{ base.GroupKey("default", "mygroup"): { {Member: msg4.ID, Score: float64(now.Add(-1 * time.Minute).Unix())}, {Member: msg5.ID, Score: float64(now.Add(-10 * time.Second).Unix())}, }, }, shouldClearGroup: false, }, { desc: "with the most recent task older than grace period", tasks: []*h.TaskSeedData{ {Msg: msg1, State: base.TaskStateAggregating}, {Msg: msg2, State: base.TaskStateAggregating}, {Msg: msg3, State: base.TaskStateAggregating}, }, groups: map[string][]redis.Z{ base.GroupKey("default", "mygroup"): { {Member: msg1.ID, Score: float64(now.Add(-5 * time.Minute).Unix())}, {Member: msg2.ID, Score: float64(now.Add(-3 * time.Minute).Unix())}, {Member: msg3.ID, Score: float64(now.Add(-2 * time.Minute).Unix())}, }, }, allGroups: map[string][]string{ base.AllGroups("default"): {"mygroup"}, }, qname: "default", gname: "mygroup", gracePeriod: 1 * time.Minute, maxDelay: 10 * time.Minute, maxSize: 5, shouldCreateSet: true, wantAggregationSet: []*base.TaskMessage{msg1, msg2, msg3}, wantGroups: map[string][]redis.Z{ base.GroupKey("default", "mygroup"): {}, }, shouldClearGroup: true, }, { desc: "with the oldest task older than max delay", tasks: []*h.TaskSeedData{ {Msg: msg1, State: base.TaskStateAggregating}, {Msg: msg2, State: base.TaskStateAggregating}, {Msg: msg3, State: base.TaskStateAggregating}, {Msg: msg4, State: base.TaskStateAggregating}, {Msg: msg5, State: base.TaskStateAggregating}, }, groups: map[string][]redis.Z{ base.GroupKey("default", "mygroup"): { {Member: msg1.ID, Score: float64(now.Add(-15 * time.Minute).Unix())}, {Member: msg2.ID, Score: float64(now.Add(-3 * time.Minute).Unix())}, {Member: msg3.ID, Score: float64(now.Add(-2 * time.Minute).Unix())}, {Member: msg4.ID, Score: float64(now.Add(-1 * time.Minute).Unix())}, {Member: msg5.ID, Score: float64(now.Add(-10 * time.Second).Unix())}, }, }, allGroups: map[string][]string{ base.AllGroups("default"): {"mygroup"}, }, qname: "default", gname: "mygroup", gracePeriod: 2 * time.Minute, maxDelay: 10 * time.Minute, maxSize: 30, shouldCreateSet: true, wantAggregationSet: []*base.TaskMessage{msg1, msg2, msg3, msg4, msg5}, wantGroups: map[string][]redis.Z{ base.GroupKey("default", "mygroup"): {}, }, shouldClearGroup: true, }, { desc: "with unlimited size", tasks: []*h.TaskSeedData{ {Msg: msg1, State: base.TaskStateAggregating}, {Msg: msg2, State: base.TaskStateAggregating}, {Msg: msg3, State: base.TaskStateAggregating}, {Msg: msg4, State: base.TaskStateAggregating}, {Msg: msg5, State: base.TaskStateAggregating}, }, groups: map[string][]redis.Z{ base.GroupKey("default", "mygroup"): { {Member: msg1.ID, Score: float64(now.Add(-15 * time.Minute).Unix())}, {Member: msg2.ID, Score: float64(now.Add(-3 * time.Minute).Unix())}, {Member: msg3.ID, Score: float64(now.Add(-2 * time.Minute).Unix())}, {Member: msg4.ID, Score: float64(now.Add(-1 * time.Minute).Unix())}, {Member: msg5.ID, Score: float64(now.Add(-10 * time.Second).Unix())}, }, }, allGroups: map[string][]string{ base.AllGroups("default"): {"mygroup"}, }, qname: "default", gname: "mygroup", gracePeriod: 1 * time.Minute, maxDelay: 30 * time.Minute, maxSize: 0, // maxSize=0 indicates no size limit shouldCreateSet: false, wantAggregationSet: nil, wantGroups: map[string][]redis.Z{ base.GroupKey("default", "mygroup"): { {Member: msg1.ID, Score: float64(now.Add(-15 * time.Minute).Unix())}, {Member: msg2.ID, Score: float64(now.Add(-3 * time.Minute).Unix())}, {Member: msg3.ID, Score: float64(now.Add(-2 * time.Minute).Unix())}, {Member: msg4.ID, Score: float64(now.Add(-1 * time.Minute).Unix())}, {Member: msg5.ID, Score: float64(now.Add(-10 * time.Second).Unix())}, }, }, shouldClearGroup: false, }, { desc: "with unlimited size and passed grace period", tasks: []*h.TaskSeedData{ {Msg: msg1, State: base.TaskStateAggregating}, {Msg: msg2, State: base.TaskStateAggregating}, {Msg: msg3, State: base.TaskStateAggregating}, {Msg: msg4, State: base.TaskStateAggregating}, {Msg: msg5, State: base.TaskStateAggregating}, }, groups: map[string][]redis.Z{ base.GroupKey("default", "mygroup"): { {Member: msg1.ID, Score: float64(now.Add(-15 * time.Minute).Unix())}, {Member: msg2.ID, Score: float64(now.Add(-3 * time.Minute).Unix())}, {Member: msg3.ID, Score: float64(now.Add(-2 * time.Minute).Unix())}, {Member: msg4.ID, Score: float64(now.Add(-1 * time.Minute).Unix())}, {Member: msg5.ID, Score: float64(now.Add(-1 * time.Minute).Unix())}, }, }, allGroups: map[string][]string{ base.AllGroups("default"): {"mygroup"}, }, qname: "default", gname: "mygroup", gracePeriod: 30 * time.Second, maxDelay: 30 * time.Minute, maxSize: 0, // maxSize=0 indicates no size limit shouldCreateSet: true, wantAggregationSet: []*base.TaskMessage{msg1, msg2, msg3, msg4, msg5}, wantGroups: map[string][]redis.Z{ base.GroupKey("default", "mygroup"): {}, }, shouldClearGroup: true, }, { desc: "with unlimited delay", tasks: []*h.TaskSeedData{ {Msg: msg1, State: base.TaskStateAggregating}, {Msg: msg2, State: base.TaskStateAggregating}, {Msg: msg3, State: base.TaskStateAggregating}, {Msg: msg4, State: base.TaskStateAggregating}, {Msg: msg5, State: base.TaskStateAggregating}, }, groups: map[string][]redis.Z{ base.GroupKey("default", "mygroup"): { {Member: msg1.ID, Score: float64(now.Add(-15 * time.Minute).Unix())}, {Member: msg2.ID, Score: float64(now.Add(-3 * time.Minute).Unix())}, {Member: msg3.ID, Score: float64(now.Add(-2 * time.Minute).Unix())}, {Member: msg4.ID, Score: float64(now.Add(-1 * time.Minute).Unix())}, {Member: msg5.ID, Score: float64(now.Add(-10 * time.Second).Unix())}, }, }, allGroups: map[string][]string{ base.AllGroups("default"): {"mygroup"}, }, qname: "default", gname: "mygroup", gracePeriod: 1 * time.Minute, maxDelay: 0, // maxDelay=0 indicates no limit maxSize: 10, shouldCreateSet: false, wantAggregationSet: nil, wantGroups: map[string][]redis.Z{ base.GroupKey("default", "mygroup"): { {Member: msg1.ID, Score: float64(now.Add(-15 * time.Minute).Unix())}, {Member: msg2.ID, Score: float64(now.Add(-3 * time.Minute).Unix())}, {Member: msg3.ID, Score: float64(now.Add(-2 * time.Minute).Unix())}, {Member: msg4.ID, Score: float64(now.Add(-1 * time.Minute).Unix())}, {Member: msg5.ID, Score: float64(now.Add(-10 * time.Second).Unix())}, }, }, shouldClearGroup: false, }, } for _, tc := range tests { h.FlushDB(t, r.client) t.Run(tc.desc, func(t *testing.T) { h.SeedTasks(t, r.client, tc.tasks) h.SeedRedisZSets(t, r.client, tc.groups) h.SeedRedisSets(t, r.client, tc.allGroups) aggregationSetID, err := r.AggregationCheck(tc.qname, tc.gname, now, tc.gracePeriod, tc.maxDelay, tc.maxSize) if err != nil { t.Fatalf("AggregationCheck returned error: %v", err) } if !tc.shouldCreateSet && aggregationSetID != "" { t.Fatal("AggregationCheck returned non empty set ID. want empty ID") } if tc.shouldCreateSet && aggregationSetID == "" { t.Fatal("AggregationCheck returned empty set ID. want non empty ID") } if tc.shouldCreateSet { msgs, deadline, err := r.ReadAggregationSet(tc.qname, tc.gname, aggregationSetID) if err != nil { t.Fatalf("Failed to read aggregation set %q: %v", aggregationSetID, err) } if diff := cmp.Diff(tc.wantAggregationSet, msgs, h.SortMsgOpt); diff != "" { t.Errorf("Mismatch found in aggregation set: (-want,+got)\n%s", diff) } if wantDeadline := now.Add(aggregationTimeout); deadline.Unix() != wantDeadline.Unix() { t.Errorf("ReadAggregationSet returned deadline=%v, want=%v", deadline, wantDeadline) } } h.AssertRedisZSets(t, r.client, tc.wantGroups) if tc.shouldClearGroup { if key := base.GroupKey(tc.qname, tc.gname); r.client.Exists(ctx, key).Val() != 0 { t.Errorf("group key %q still exists", key) } if r.client.SIsMember(ctx, base.AllGroups(tc.qname), tc.gname).Val() { t.Errorf("all-group set %q still contains the group name %q", base.AllGroups(tc.qname), tc.gname) } } else { if key := base.GroupKey(tc.qname, tc.gname); r.client.Exists(ctx, key).Val() == 0 { t.Errorf("group key %q does not exists", key) } if !r.client.SIsMember(ctx, base.AllGroups(tc.qname), tc.gname).Val() { t.Errorf("all-group set %q doesn't contains the group name %q", base.AllGroups(tc.qname), tc.gname) } } }) } } func TestDeleteAggregationSet(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() setID := uuid.NewString() otherSetID := uuid.NewString() m1 := h.NewTaskMessageBuilder().SetQueue("default").SetGroup("mygroup").Build() m2 := h.NewTaskMessageBuilder().SetQueue("default").SetGroup("mygroup").Build() m3 := h.NewTaskMessageBuilder().SetQueue("default").SetGroup("mygroup").Build() tests := []struct { desc string // initial data tasks []*h.TaskSeedData aggregationSets map[string][]redis.Z allAggregationSets map[string][]redis.Z // args ctx context.Context qname string gname string setID string // expectations wantDeletedKeys []string // redis key to check for non existence wantAggregationSets map[string][]redis.Z wantAllAggregationSets map[string][]redis.Z }{ { desc: "with a sigle active aggregation set", tasks: []*h.TaskSeedData{ {Msg: m1, State: base.TaskStateAggregating}, {Msg: m2, State: base.TaskStateAggregating}, {Msg: m3, State: base.TaskStateAggregating}, }, aggregationSets: map[string][]redis.Z{ base.AggregationSetKey("default", "mygroup", setID): { {Member: m1.ID, Score: float64(now.Add(-5 * time.Minute).Unix())}, {Member: m2.ID, Score: float64(now.Add(-4 * time.Minute).Unix())}, {Member: m3.ID, Score: float64(now.Add(-3 * time.Minute).Unix())}, }, }, allAggregationSets: map[string][]redis.Z{ base.AllAggregationSets("default"): { {Member: base.AggregationSetKey("default", "mygroup", setID), Score: float64(now.Add(aggregationTimeout).Unix())}, }, }, ctx: context.Background(), qname: "default", gname: "mygroup", setID: setID, wantDeletedKeys: []string{ base.AggregationSetKey("default", "mygroup", setID), base.TaskKey(m1.Queue, m1.ID), base.TaskKey(m2.Queue, m2.ID), base.TaskKey(m3.Queue, m3.ID), }, wantAggregationSets: map[string][]redis.Z{}, wantAllAggregationSets: map[string][]redis.Z{ base.AllAggregationSets("default"): {}, }, }, { desc: "with multiple active aggregation sets", tasks: []*h.TaskSeedData{ {Msg: m1, State: base.TaskStateAggregating}, {Msg: m2, State: base.TaskStateAggregating}, {Msg: m3, State: base.TaskStateAggregating}, }, aggregationSets: map[string][]redis.Z{ base.AggregationSetKey("default", "mygroup", setID): { {Member: m1.ID, Score: float64(now.Add(-5 * time.Minute).Unix())}, }, base.AggregationSetKey("default", "mygroup", otherSetID): { {Member: m2.ID, Score: float64(now.Add(-4 * time.Minute).Unix())}, {Member: m3.ID, Score: float64(now.Add(-3 * time.Minute).Unix())}, }, }, allAggregationSets: map[string][]redis.Z{ base.AllAggregationSets("default"): { {Member: base.AggregationSetKey("default", "mygroup", setID), Score: float64(now.Add(aggregationTimeout).Unix())}, {Member: base.AggregationSetKey("default", "mygroup", otherSetID), Score: float64(now.Add(aggregationTimeout).Unix())}, }, }, ctx: context.Background(), qname: "default", gname: "mygroup", setID: setID, wantDeletedKeys: []string{ base.AggregationSetKey("default", "mygroup", setID), base.TaskKey(m1.Queue, m1.ID), }, wantAggregationSets: map[string][]redis.Z{ base.AggregationSetKey("default", "mygroup", otherSetID): { {Member: m2.ID, Score: float64(now.Add(-4 * time.Minute).Unix())}, {Member: m3.ID, Score: float64(now.Add(-3 * time.Minute).Unix())}, }, }, wantAllAggregationSets: map[string][]redis.Z{ base.AllAggregationSets("default"): { {Member: base.AggregationSetKey("default", "mygroup", otherSetID), Score: float64(now.Add(aggregationTimeout).Unix())}, }, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) t.Run(tc.desc, func(t *testing.T) { h.SeedTasks(t, r.client, tc.tasks) h.SeedRedisZSets(t, r.client, tc.aggregationSets) h.SeedRedisZSets(t, r.client, tc.allAggregationSets) if err := r.DeleteAggregationSet(tc.ctx, tc.qname, tc.gname, tc.setID); err != nil { t.Fatalf("DeleteAggregationSet returned error: %v", err) } for _, key := range tc.wantDeletedKeys { if r.client.Exists(context.Background(), key).Val() != 0 { t.Errorf("key=%q still exists, want deleted", key) } } h.AssertRedisZSets(t, r.client, tc.wantAllAggregationSets) }) } } func TestDeleteAggregationSetError(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() setID := uuid.NewString() m1 := h.NewTaskMessageBuilder().SetQueue("default").SetGroup("mygroup").Build() m2 := h.NewTaskMessageBuilder().SetQueue("default").SetGroup("mygroup").Build() m3 := h.NewTaskMessageBuilder().SetQueue("default").SetGroup("mygroup").Build() deadlineExceededCtx, cancel := context.WithDeadline(context.Background(), now.Add(-10*time.Second)) defer cancel() tests := []struct { desc string // initial data tasks []*h.TaskSeedData aggregationSets map[string][]redis.Z allAggregationSets map[string][]redis.Z // args ctx context.Context qname string gname string setID string // expectations wantAggregationSets map[string][]redis.Z wantAllAggregationSets map[string][]redis.Z }{ { desc: "with deadline exceeded context", tasks: []*h.TaskSeedData{ {Msg: m1, State: base.TaskStateAggregating}, {Msg: m2, State: base.TaskStateAggregating}, {Msg: m3, State: base.TaskStateAggregating}, }, aggregationSets: map[string][]redis.Z{ base.AggregationSetKey("default", "mygroup", setID): { {Member: m1.ID, Score: float64(now.Add(-5 * time.Minute).Unix())}, {Member: m2.ID, Score: float64(now.Add(-4 * time.Minute).Unix())}, {Member: m3.ID, Score: float64(now.Add(-3 * time.Minute).Unix())}, }, }, allAggregationSets: map[string][]redis.Z{ base.AllAggregationSets("default"): { {Member: base.AggregationSetKey("default", "mygroup", setID), Score: float64(now.Add(aggregationTimeout).Unix())}, }, }, ctx: deadlineExceededCtx, qname: "default", gname: "mygroup", setID: setID, // want data unchanged. wantAggregationSets: map[string][]redis.Z{ base.AggregationSetKey("default", "mygroup", setID): { {Member: m1.ID, Score: float64(now.Add(-5 * time.Minute).Unix())}, {Member: m2.ID, Score: float64(now.Add(-4 * time.Minute).Unix())}, {Member: m3.ID, Score: float64(now.Add(-3 * time.Minute).Unix())}, }, }, // want data unchanged. wantAllAggregationSets: map[string][]redis.Z{ base.AllAggregationSets("default"): { {Member: base.AggregationSetKey("default", "mygroup", setID), Score: float64(now.Add(aggregationTimeout).Unix())}, }, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) t.Run(tc.desc, func(t *testing.T) { h.SeedTasks(t, r.client, tc.tasks) h.SeedRedisZSets(t, r.client, tc.aggregationSets) h.SeedRedisZSets(t, r.client, tc.allAggregationSets) if err := r.DeleteAggregationSet(tc.ctx, tc.qname, tc.gname, tc.setID); err == nil { t.Fatal("DeleteAggregationSet returned nil, want non-nil error") } // Make sure zsets are unchanged. h.AssertRedisZSets(t, r.client, tc.wantAggregationSets) h.AssertRedisZSets(t, r.client, tc.wantAllAggregationSets) }) } } func TestReclaimStaleAggregationSets(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() r.SetClock(timeutil.NewSimulatedClock(now)) m1 := h.NewTaskMessageBuilder().SetQueue("default").SetGroup("foo").Build() m2 := h.NewTaskMessageBuilder().SetQueue("default").SetGroup("foo").Build() m3 := h.NewTaskMessageBuilder().SetQueue("default").SetGroup("bar").Build() m4 := h.NewTaskMessageBuilder().SetQueue("default").SetGroup("qux").Build() // Note: In this test, we're trying out a new way to test RDB by exactly describing how // keys and values are represented in Redis. tests := []struct { groups map[string][]redis.Z // map redis-key to redis-zset aggregationSets map[string][]redis.Z allAggregationSets map[string][]redis.Z qname string wantGroups map[string][]redis.Z wantAggregationSets map[string][]redis.Z wantAllAggregationSets map[string][]redis.Z }{ { groups: map[string][]redis.Z{ base.GroupKey("default", "foo"): {}, base.GroupKey("default", "bar"): {}, base.GroupKey("default", "qux"): { {Member: m4.ID, Score: float64(now.Add(-10 * time.Second).Unix())}, }, }, aggregationSets: map[string][]redis.Z{ base.AggregationSetKey("default", "foo", "set1"): { {Member: m1.ID, Score: float64(now.Add(-3 * time.Minute).Unix())}, {Member: m2.ID, Score: float64(now.Add(-4 * time.Minute).Unix())}, }, base.AggregationSetKey("default", "bar", "set2"): { {Member: m3.ID, Score: float64(now.Add(-1 * time.Minute).Unix())}, }, }, allAggregationSets: map[string][]redis.Z{ base.AllAggregationSets("default"): { {Member: base.AggregationSetKey("default", "foo", "set1"), Score: float64(now.Add(-10 * time.Second).Unix())}, // set1 is expired {Member: base.AggregationSetKey("default", "bar", "set2"), Score: float64(now.Add(40 * time.Second).Unix())}, // set2 is not expired }, }, qname: "default", wantGroups: map[string][]redis.Z{ base.GroupKey("default", "foo"): { {Member: m1.ID, Score: float64(now.Add(-3 * time.Minute).Unix())}, {Member: m2.ID, Score: float64(now.Add(-4 * time.Minute).Unix())}, }, base.GroupKey("default", "bar"): {}, base.GroupKey("default", "qux"): { {Member: m4.ID, Score: float64(now.Add(-10 * time.Second).Unix())}, }, }, wantAggregationSets: map[string][]redis.Z{ base.AggregationSetKey("default", "bar", "set2"): { {Member: m3.ID, Score: float64(now.Add(-1 * time.Minute).Unix())}, }, }, wantAllAggregationSets: map[string][]redis.Z{ base.AllAggregationSets("default"): { {Member: base.AggregationSetKey("default", "bar", "set2"), Score: float64(now.Add(40 * time.Second).Unix())}, }, }, }, } for _, tc := range tests { h.FlushDB(t, r.client) h.SeedRedisZSets(t, r.client, tc.groups) h.SeedRedisZSets(t, r.client, tc.aggregationSets) h.SeedRedisZSets(t, r.client, tc.allAggregationSets) if err := r.ReclaimStaleAggregationSets(tc.qname); err != nil { t.Errorf("ReclaimStaleAggregationSets returned error: %v", err) continue } h.AssertRedisZSets(t, r.client, tc.wantGroups) h.AssertRedisZSets(t, r.client, tc.wantAggregationSets) h.AssertRedisZSets(t, r.client, tc.wantAllAggregationSets) } } func TestListGroups(t *testing.T) { r := setup(t) defer r.Close() now := time.Now() m1 := h.NewTaskMessageBuilder().SetQueue("default").SetGroup("foo").Build() m2 := h.NewTaskMessageBuilder().SetQueue("default").SetGroup("bar").Build() m3 := h.NewTaskMessageBuilder().SetQueue("custom").SetGroup("baz").Build() m4 := h.NewTaskMessageBuilder().SetQueue("custom").SetGroup("qux").Build() tests := []struct { groups map[string]map[string][]base.Z qname string want []string }{ { groups: map[string]map[string][]base.Z{ "default": { "foo": {{Message: m1, Score: now.Add(-10 * time.Second).Unix()}}, "bar": {{Message: m2, Score: now.Add(-10 * time.Second).Unix()}}, }, "custom": { "baz": {{Message: m3, Score: now.Add(-10 * time.Second).Unix()}}, "qux": {{Message: m4, Score: now.Add(-10 * time.Second).Unix()}}, }, }, qname: "default", want: []string{"foo", "bar"}, }, { groups: map[string]map[string][]base.Z{ "default": { "foo": {{Message: m1, Score: now.Add(-10 * time.Second).Unix()}}, "bar": {{Message: m2, Score: now.Add(-10 * time.Second).Unix()}}, }, "custom": { "baz": {{Message: m3, Score: now.Add(-10 * time.Second).Unix()}}, "qux": {{Message: m4, Score: now.Add(-10 * time.Second).Unix()}}, }, }, qname: "custom", want: []string{"baz", "qux"}, }, } for _, tc := range tests { h.FlushDB(t, r.client) h.SeedAllGroups(t, r.client, tc.groups) got, err := r.ListGroups(tc.qname) if err != nil { t.Errorf("ListGroups returned error: %v", err) continue } if diff := cmp.Diff(tc.want, got, h.SortStringSliceOpt); diff != "" { t.Errorf("ListGroups=%v, want=%v; (-want,+got)\n%s", got, tc.want, diff) } } } ================================================ FILE: internal/testbroker/testbroker.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. // Package testbroker exports a broker implementation that should be used in package testing. package testbroker import ( "context" "errors" "sync" "time" "github.com/hibiken/asynq/internal/base" "github.com/redis/go-redis/v9" ) var errRedisDown = errors.New("testutil: redis is down") // TestBroker is a broker implementation which enables // to simulate Redis failure in tests. type TestBroker struct { mu sync.Mutex sleeping bool // real broker real base.Broker } // Make sure TestBroker implements Broker interface at compile time. var _ base.Broker = (*TestBroker)(nil) func NewTestBroker(b base.Broker) *TestBroker { return &TestBroker{real: b} } func (tb *TestBroker) Sleep() { tb.mu.Lock() defer tb.mu.Unlock() tb.sleeping = true } func (tb *TestBroker) Wakeup() { tb.mu.Lock() defer tb.mu.Unlock() tb.sleeping = false } func (tb *TestBroker) Enqueue(ctx context.Context, msg *base.TaskMessage) error { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return errRedisDown } return tb.real.Enqueue(ctx, msg) } func (tb *TestBroker) EnqueueUnique(ctx context.Context, msg *base.TaskMessage, ttl time.Duration) error { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return errRedisDown } return tb.real.EnqueueUnique(ctx, msg, ttl) } func (tb *TestBroker) Dequeue(qnames ...string) (*base.TaskMessage, time.Time, error) { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return nil, time.Time{}, errRedisDown } return tb.real.Dequeue(qnames...) } func (tb *TestBroker) Done(ctx context.Context, msg *base.TaskMessage) error { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return errRedisDown } return tb.real.Done(ctx, msg) } func (tb *TestBroker) MarkAsComplete(ctx context.Context, msg *base.TaskMessage) error { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return errRedisDown } return tb.real.MarkAsComplete(ctx, msg) } func (tb *TestBroker) Requeue(ctx context.Context, msg *base.TaskMessage) error { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return errRedisDown } return tb.real.Requeue(ctx, msg) } func (tb *TestBroker) Schedule(ctx context.Context, msg *base.TaskMessage, processAt time.Time) error { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return errRedisDown } return tb.real.Schedule(ctx, msg, processAt) } func (tb *TestBroker) ScheduleUnique(ctx context.Context, msg *base.TaskMessage, processAt time.Time, ttl time.Duration) error { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return errRedisDown } return tb.real.ScheduleUnique(ctx, msg, processAt, ttl) } func (tb *TestBroker) Retry(ctx context.Context, msg *base.TaskMessage, processAt time.Time, errMsg string, isFailure bool) error { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return errRedisDown } return tb.real.Retry(ctx, msg, processAt, errMsg, isFailure) } func (tb *TestBroker) Archive(ctx context.Context, msg *base.TaskMessage, errMsg string) error { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return errRedisDown } return tb.real.Archive(ctx, msg, errMsg) } func (tb *TestBroker) ForwardIfReady(qnames ...string) error { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return errRedisDown } return tb.real.ForwardIfReady(qnames...) } func (tb *TestBroker) DeleteExpiredCompletedTasks(qname string, batchSize int) error { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return errRedisDown } return tb.real.DeleteExpiredCompletedTasks(qname, batchSize) } func (tb *TestBroker) ListLeaseExpired(cutoff time.Time, qnames ...string) ([]*base.TaskMessage, error) { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return nil, errRedisDown } return tb.real.ListLeaseExpired(cutoff, qnames...) } func (tb *TestBroker) ExtendLease(qname string, ids ...string) (time.Time, error) { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return time.Time{}, errRedisDown } return tb.real.ExtendLease(qname, ids...) } func (tb *TestBroker) WriteServerState(info *base.ServerInfo, workers []*base.WorkerInfo, ttl time.Duration) error { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return errRedisDown } return tb.real.WriteServerState(info, workers, ttl) } func (tb *TestBroker) ClearServerState(host string, pid int, serverID string) error { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return errRedisDown } return tb.real.ClearServerState(host, pid, serverID) } func (tb *TestBroker) CancelationPubSub() (*redis.PubSub, error) { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return nil, errRedisDown } return tb.real.CancelationPubSub() } func (tb *TestBroker) PublishCancelation(id string) error { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return errRedisDown } return tb.real.PublishCancelation(id) } func (tb *TestBroker) WriteResult(qname, id string, data []byte) (int, error) { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return 0, errRedisDown } return tb.real.WriteResult(qname, id, data) } func (tb *TestBroker) Ping() error { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return errRedisDown } return tb.real.Ping() } func (tb *TestBroker) Close() error { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return errRedisDown } return tb.real.Close() } func (tb *TestBroker) AddToGroup(ctx context.Context, msg *base.TaskMessage, gname string) error { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return errRedisDown } return tb.real.AddToGroup(ctx, msg, gname) } func (tb *TestBroker) AddToGroupUnique(ctx context.Context, msg *base.TaskMessage, gname string, ttl time.Duration) error { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return errRedisDown } return tb.real.AddToGroupUnique(ctx, msg, gname, ttl) } func (tb *TestBroker) ListGroups(qname string) ([]string, error) { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return nil, errRedisDown } return tb.real.ListGroups(qname) } func (tb *TestBroker) AggregationCheck(qname, gname string, t time.Time, gracePeriod, maxDelay time.Duration, maxSize int) (aggregationSetID string, err error) { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return "", errRedisDown } return tb.real.AggregationCheck(qname, gname, t, gracePeriod, maxDelay, maxSize) } func (tb *TestBroker) ReadAggregationSet(qname, gname, aggregationSetID string) ([]*base.TaskMessage, time.Time, error) { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return nil, time.Time{}, errRedisDown } return tb.real.ReadAggregationSet(qname, gname, aggregationSetID) } func (tb *TestBroker) DeleteAggregationSet(ctx context.Context, qname, gname, aggregationSetID string) error { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return errRedisDown } return tb.real.DeleteAggregationSet(ctx, qname, gname, aggregationSetID) } func (tb *TestBroker) ReclaimStaleAggregationSets(qname string) error { tb.mu.Lock() defer tb.mu.Unlock() if tb.sleeping { return errRedisDown } return tb.real.ReclaimStaleAggregationSets(qname) } ================================================ FILE: internal/testutil/builder.go ================================================ // Copyright 2022 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package testutil import ( "time" "github.com/google/uuid" "github.com/hibiken/asynq/internal/base" ) func makeDefaultTaskMessage() *base.TaskMessage { return &base.TaskMessage{ ID: uuid.NewString(), Type: "default_task", Queue: "default", Retry: 25, Timeout: 1800, // default timeout of 30 mins Deadline: 0, // no deadline } } type TaskMessageBuilder struct { msg *base.TaskMessage } func NewTaskMessageBuilder() *TaskMessageBuilder { return &TaskMessageBuilder{} } func (b *TaskMessageBuilder) lazyInit() { if b.msg == nil { b.msg = makeDefaultTaskMessage() } } func (b *TaskMessageBuilder) Build() *base.TaskMessage { b.lazyInit() return b.msg } func (b *TaskMessageBuilder) SetType(typename string) *TaskMessageBuilder { b.lazyInit() b.msg.Type = typename return b } func (b *TaskMessageBuilder) SetPayload(payload []byte) *TaskMessageBuilder { b.lazyInit() b.msg.Payload = payload return b } func (b *TaskMessageBuilder) SetQueue(qname string) *TaskMessageBuilder { b.lazyInit() b.msg.Queue = qname return b } func (b *TaskMessageBuilder) SetRetry(n int) *TaskMessageBuilder { b.lazyInit() b.msg.Retry = n return b } func (b *TaskMessageBuilder) SetTimeout(timeout time.Duration) *TaskMessageBuilder { b.lazyInit() b.msg.Timeout = int64(timeout.Seconds()) return b } func (b *TaskMessageBuilder) SetDeadline(deadline time.Time) *TaskMessageBuilder { b.lazyInit() b.msg.Deadline = deadline.Unix() return b } func (b *TaskMessageBuilder) SetGroup(gname string) *TaskMessageBuilder { b.lazyInit() b.msg.GroupKey = gname return b } ================================================ FILE: internal/testutil/builder_test.go ================================================ // Copyright 2022 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package testutil import ( "testing" "time" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/hibiken/asynq/internal/base" ) func TestTaskMessageBuilder(t *testing.T) { tests := []struct { desc string ops func(b *TaskMessageBuilder) // operations to perform on the builder want *base.TaskMessage }{ { desc: "zero value and build", ops: nil, want: &base.TaskMessage{ Type: "default_task", Queue: "default", Payload: nil, Retry: 25, Timeout: 1800, // 30m Deadline: 0, }, }, { desc: "with type, payload, and queue", ops: func(b *TaskMessageBuilder) { b.SetType("foo").SetPayload([]byte("hello")).SetQueue("myqueue") }, want: &base.TaskMessage{ Type: "foo", Queue: "myqueue", Payload: []byte("hello"), Retry: 25, Timeout: 1800, // 30m Deadline: 0, }, }, { desc: "with retry, timeout, and deadline", ops: func(b *TaskMessageBuilder) { b.SetRetry(1). SetTimeout(20 * time.Second). SetDeadline(time.Date(2017, 3, 6, 0, 0, 0, 0, time.UTC)) }, want: &base.TaskMessage{ Type: "default_task", Queue: "default", Payload: nil, Retry: 1, Timeout: 20, Deadline: time.Date(2017, 3, 6, 0, 0, 0, 0, time.UTC).Unix(), }, }, { desc: "with group", ops: func(b *TaskMessageBuilder) { b.SetGroup("mygroup") }, want: &base.TaskMessage{ Type: "default_task", Queue: "default", Payload: nil, Retry: 25, Timeout: 1800, Deadline: 0, GroupKey: "mygroup", }, }, } cmpOpts := []cmp.Option{cmpopts.IgnoreFields(base.TaskMessage{}, "ID")} for _, tc := range tests { var b TaskMessageBuilder if tc.ops != nil { tc.ops(&b) } got := b.Build() if diff := cmp.Diff(tc.want, got, cmpOpts...); diff != "" { t.Errorf("%s: TaskMessageBuilder.Build() = %+v, want %+v;\n(-want,+got)\n%s", tc.desc, got, tc.want, diff) } } } ================================================ FILE: internal/testutil/testutil.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. // Package testutil defines test helpers for asynq and its internal packages. package testutil import ( "context" "encoding/json" "math" "sort" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/google/uuid" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/timeutil" "github.com/redis/go-redis/v9" ) // EquateInt64Approx returns a Comparer option that treats int64 values // to be equal if they are within the given margin. func EquateInt64Approx(margin int64) cmp.Option { return cmp.Comparer(func(a, b int64) bool { return math.Abs(float64(a-b)) <= float64(margin) }) } // SortMsgOpt is a cmp.Option to sort base.TaskMessage for comparing slice of task messages. var SortMsgOpt = cmp.Transformer("SortTaskMessages", func(in []*base.TaskMessage) []*base.TaskMessage { out := append([]*base.TaskMessage(nil), in...) // Copy input to avoid mutating it sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) return out }) // SortZSetEntryOpt is an cmp.Option to sort ZSetEntry for comparing slice of zset entries. var SortZSetEntryOpt = cmp.Transformer("SortZSetEntries", func(in []base.Z) []base.Z { out := append([]base.Z(nil), in...) // Copy input to avoid mutating it sort.Slice(out, func(i, j int) bool { return out[i].Message.ID < out[j].Message.ID }) return out }) // SortServerInfoOpt is a cmp.Option to sort base.ServerInfo for comparing slice of process info. var SortServerInfoOpt = cmp.Transformer("SortServerInfo", func(in []*base.ServerInfo) []*base.ServerInfo { out := append([]*base.ServerInfo(nil), in...) // Copy input to avoid mutating it sort.Slice(out, func(i, j int) bool { if out[i].Host != out[j].Host { return out[i].Host < out[j].Host } return out[i].PID < out[j].PID }) return out }) // SortWorkerInfoOpt is a cmp.Option to sort base.WorkerInfo for comparing slice of worker info. var SortWorkerInfoOpt = cmp.Transformer("SortWorkerInfo", func(in []*base.WorkerInfo) []*base.WorkerInfo { out := append([]*base.WorkerInfo(nil), in...) // Copy input to avoid mutating it sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) return out }) // SortSchedulerEntryOpt is a cmp.Option to sort base.SchedulerEntry for comparing slice of entries. var SortSchedulerEntryOpt = cmp.Transformer("SortSchedulerEntry", func(in []*base.SchedulerEntry) []*base.SchedulerEntry { out := append([]*base.SchedulerEntry(nil), in...) // Copy input to avoid mutating it sort.Slice(out, func(i, j int) bool { return out[i].Spec < out[j].Spec }) return out }) // SortSchedulerEnqueueEventOpt is a cmp.Option to sort base.SchedulerEnqueueEvent for comparing slice of events. var SortSchedulerEnqueueEventOpt = cmp.Transformer("SortSchedulerEnqueueEvent", func(in []*base.SchedulerEnqueueEvent) []*base.SchedulerEnqueueEvent { out := append([]*base.SchedulerEnqueueEvent(nil), in...) sort.Slice(out, func(i, j int) bool { return out[i].EnqueuedAt.Unix() < out[j].EnqueuedAt.Unix() }) return out }) // SortStringSliceOpt is a cmp.Option to sort string slice. var SortStringSliceOpt = cmp.Transformer("SortStringSlice", func(in []string) []string { out := append([]string(nil), in...) sort.Strings(out) return out }) var SortRedisZSetEntryOpt = cmp.Transformer("SortZSetEntries", func(in []redis.Z) []redis.Z { out := append([]redis.Z(nil), in...) // Copy input to avoid mutating it sort.Slice(out, func(i, j int) bool { // TODO: If member is a comparable type (int, string, etc) compare by the member // Use generic comparable type here once update to go1.18 if _, ok := out[i].Member.(string); ok { // If member is a string, compare the member return out[i].Member.(string) < out[j].Member.(string) } return out[i].Score < out[j].Score }) return out }) // IgnoreIDOpt is an cmp.Option to ignore ID field in task messages when comparing. var IgnoreIDOpt = cmpopts.IgnoreFields(base.TaskMessage{}, "ID") // NewTaskMessage returns a new instance of TaskMessage given a task type and payload. func NewTaskMessage(taskType string, payload []byte) *base.TaskMessage { return NewTaskMessageWithQueue(taskType, payload, base.DefaultQueueName) } // NewTaskMessageWithQueue returns a new instance of TaskMessage given a // task type, payload and queue name. func NewTaskMessageWithQueue(taskType string, payload []byte, qname string) *base.TaskMessage { return &base.TaskMessage{ ID: uuid.NewString(), Type: taskType, Queue: qname, Retry: 25, Payload: payload, Timeout: 1800, // default timeout of 30 mins Deadline: 0, // no deadline } } // NewLeaseWithClock returns a new lease with the given expiration time and clock. func NewLeaseWithClock(expirationTime time.Time, clock timeutil.Clock) *base.Lease { l := base.NewLease(expirationTime) l.Clock = clock return l } // JSON serializes the given key-value pairs into stream of bytes in JSON. func JSON(kv map[string]interface{}) []byte { b, err := json.Marshal(kv) if err != nil { panic(err) } return b } // TaskMessageAfterRetry returns an updated copy of t after retry. // It increments retry count and sets the error message and last_failed_at time. func TaskMessageAfterRetry(t base.TaskMessage, errMsg string, failedAt time.Time) *base.TaskMessage { t.Retried = t.Retried + 1 t.ErrorMsg = errMsg t.LastFailedAt = failedAt.Unix() return &t } // TaskMessageWithError returns an updated copy of t with the given error message. func TaskMessageWithError(t base.TaskMessage, errMsg string, failedAt time.Time) *base.TaskMessage { t.ErrorMsg = errMsg t.LastFailedAt = failedAt.Unix() return &t } // TaskMessageWithCompletedAt returns an updated copy of t after completion. func TaskMessageWithCompletedAt(t base.TaskMessage, completedAt time.Time) *base.TaskMessage { t.CompletedAt = completedAt.Unix() return &t } // MustMarshal marshals given task message and returns a json string. // Calling test will fail if marshaling errors out. func MustMarshal(tb testing.TB, msg *base.TaskMessage) string { tb.Helper() data, err := base.EncodeMessage(msg) if err != nil { tb.Fatal(err) } return string(data) } // MustUnmarshal unmarshals given string into task message struct. // Calling test will fail if unmarshaling errors out. func MustUnmarshal(tb testing.TB, data string) *base.TaskMessage { tb.Helper() msg, err := base.DecodeMessage([]byte(data)) if err != nil { tb.Fatal(err) } return msg } // FlushDB deletes all the keys of the currently selected DB. func FlushDB(tb testing.TB, r redis.UniversalClient) { tb.Helper() switch r := r.(type) { case *redis.Client: if err := r.FlushDB(context.Background()).Err(); err != nil { tb.Fatal(err) } case *redis.ClusterClient: err := r.ForEachMaster(context.Background(), func(ctx context.Context, c *redis.Client) error { if err := c.FlushAll(ctx).Err(); err != nil { return err } return nil }) if err != nil { tb.Fatal(err) } } } // SeedPendingQueue initializes the specified queue with the given messages. func SeedPendingQueue(tb testing.TB, r redis.UniversalClient, msgs []*base.TaskMessage, qname string) { tb.Helper() r.SAdd(context.Background(), base.AllQueues, qname) seedRedisList(tb, r, base.PendingKey(qname), msgs, base.TaskStatePending) } // SeedActiveQueue initializes the active queue with the given messages. func SeedActiveQueue(tb testing.TB, r redis.UniversalClient, msgs []*base.TaskMessage, qname string) { tb.Helper() r.SAdd(context.Background(), base.AllQueues, qname) seedRedisList(tb, r, base.ActiveKey(qname), msgs, base.TaskStateActive) } // SeedScheduledQueue initializes the scheduled queue with the given messages. func SeedScheduledQueue(tb testing.TB, r redis.UniversalClient, entries []base.Z, qname string) { tb.Helper() r.SAdd(context.Background(), base.AllQueues, qname) seedRedisZSet(tb, r, base.ScheduledKey(qname), entries, base.TaskStateScheduled) } // SeedRetryQueue initializes the retry queue with the given messages. func SeedRetryQueue(tb testing.TB, r redis.UniversalClient, entries []base.Z, qname string) { tb.Helper() r.SAdd(context.Background(), base.AllQueues, qname) seedRedisZSet(tb, r, base.RetryKey(qname), entries, base.TaskStateRetry) } // SeedArchivedQueue initializes the archived queue with the given messages. func SeedArchivedQueue(tb testing.TB, r redis.UniversalClient, entries []base.Z, qname string) { tb.Helper() r.SAdd(context.Background(), base.AllQueues, qname) seedRedisZSet(tb, r, base.ArchivedKey(qname), entries, base.TaskStateArchived) } // SeedLease initializes the lease set with the given entries. func SeedLease(tb testing.TB, r redis.UniversalClient, entries []base.Z, qname string) { tb.Helper() r.SAdd(context.Background(), base.AllQueues, qname) seedRedisZSet(tb, r, base.LeaseKey(qname), entries, base.TaskStateActive) } // SeedCompletedQueue initializes the completed set with the given entries. func SeedCompletedQueue(tb testing.TB, r redis.UniversalClient, entries []base.Z, qname string) { tb.Helper() r.SAdd(context.Background(), base.AllQueues, qname) seedRedisZSet(tb, r, base.CompletedKey(qname), entries, base.TaskStateCompleted) } // SeedGroup initializes the group with the given entries. func SeedGroup(tb testing.TB, r redis.UniversalClient, entries []base.Z, qname, gname string) { tb.Helper() ctx := context.Background() r.SAdd(ctx, base.AllQueues, qname) r.SAdd(ctx, base.AllGroups(qname), gname) seedRedisZSet(tb, r, base.GroupKey(qname, gname), entries, base.TaskStateAggregating) } func SeedAggregationSet(tb testing.TB, r redis.UniversalClient, entries []base.Z, qname, gname, setID string) { tb.Helper() r.SAdd(context.Background(), base.AllQueues, qname) seedRedisZSet(tb, r, base.AggregationSetKey(qname, gname, setID), entries, base.TaskStateAggregating) } // SeedAllPendingQueues initializes all of the specified queues with the given messages. // // pending maps a queue name to a list of messages. func SeedAllPendingQueues(tb testing.TB, r redis.UniversalClient, pending map[string][]*base.TaskMessage) { tb.Helper() for q, msgs := range pending { SeedPendingQueue(tb, r, msgs, q) } } // SeedAllActiveQueues initializes all of the specified active queues with the given messages. func SeedAllActiveQueues(tb testing.TB, r redis.UniversalClient, active map[string][]*base.TaskMessage) { tb.Helper() for q, msgs := range active { SeedActiveQueue(tb, r, msgs, q) } } // SeedAllScheduledQueues initializes all of the specified scheduled queues with the given entries. func SeedAllScheduledQueues(tb testing.TB, r redis.UniversalClient, scheduled map[string][]base.Z) { tb.Helper() for q, entries := range scheduled { SeedScheduledQueue(tb, r, entries, q) } } // SeedAllRetryQueues initializes all of the specified retry queues with the given entries. func SeedAllRetryQueues(tb testing.TB, r redis.UniversalClient, retry map[string][]base.Z) { tb.Helper() for q, entries := range retry { SeedRetryQueue(tb, r, entries, q) } } // SeedAllArchivedQueues initializes all of the specified archived queues with the given entries. func SeedAllArchivedQueues(tb testing.TB, r redis.UniversalClient, archived map[string][]base.Z) { tb.Helper() for q, entries := range archived { SeedArchivedQueue(tb, r, entries, q) } } // SeedAllLease initializes all of the lease sets with the given entries. func SeedAllLease(tb testing.TB, r redis.UniversalClient, lease map[string][]base.Z) { tb.Helper() for q, entries := range lease { SeedLease(tb, r, entries, q) } } // SeedAllCompletedQueues initializes all of the completed queues with the given entries. func SeedAllCompletedQueues(tb testing.TB, r redis.UniversalClient, completed map[string][]base.Z) { tb.Helper() for q, entries := range completed { SeedCompletedQueue(tb, r, entries, q) } } // SeedAllGroups initializes all groups in all queues. // The map maps queue names to group names which maps to a list of task messages and the time it was // added to the group. func SeedAllGroups(tb testing.TB, r redis.UniversalClient, groups map[string]map[string][]base.Z) { tb.Helper() for qname, g := range groups { for gname, entries := range g { SeedGroup(tb, r, entries, qname, gname) } } } func seedRedisList(tb testing.TB, c redis.UniversalClient, key string, msgs []*base.TaskMessage, state base.TaskState) { tb.Helper() for _, msg := range msgs { encoded := MustMarshal(tb, msg) if err := c.LPush(context.Background(), key, msg.ID).Err(); err != nil { tb.Fatal(err) } taskKey := base.TaskKey(msg.Queue, msg.ID) data := map[string]interface{}{ "msg": encoded, "state": state.String(), "unique_key": msg.UniqueKey, "group": msg.GroupKey, } if err := c.HSet(context.Background(), taskKey, data).Err(); err != nil { tb.Fatal(err) } if len(msg.UniqueKey) > 0 { err := c.SetNX(context.Background(), msg.UniqueKey, msg.ID, 1*time.Minute).Err() if err != nil { tb.Fatalf("Failed to set unique lock in redis: %v", err) } } } } func seedRedisZSet(tb testing.TB, c redis.UniversalClient, key string, items []base.Z, state base.TaskState) { tb.Helper() for _, item := range items { msg := item.Message encoded := MustMarshal(tb, msg) z := redis.Z{Member: msg.ID, Score: float64(item.Score)} if err := c.ZAdd(context.Background(), key, z).Err(); err != nil { tb.Fatal(err) } taskKey := base.TaskKey(msg.Queue, msg.ID) data := map[string]interface{}{ "msg": encoded, "state": state.String(), "unique_key": msg.UniqueKey, "group": msg.GroupKey, } if err := c.HSet(context.Background(), taskKey, data).Err(); err != nil { tb.Fatal(err) } if len(msg.UniqueKey) > 0 { err := c.SetNX(context.Background(), msg.UniqueKey, msg.ID, 1*time.Minute).Err() if err != nil { tb.Fatalf("Failed to set unique lock in redis: %v", err) } } } } // GetPendingMessages returns all pending messages in the given queue. // It also asserts the state field of the task. func GetPendingMessages(tb testing.TB, r redis.UniversalClient, qname string) []*base.TaskMessage { tb.Helper() return getMessagesFromList(tb, r, qname, base.PendingKey, base.TaskStatePending) } // GetActiveMessages returns all active messages in the given queue. // It also asserts the state field of the task. func GetActiveMessages(tb testing.TB, r redis.UniversalClient, qname string) []*base.TaskMessage { tb.Helper() return getMessagesFromList(tb, r, qname, base.ActiveKey, base.TaskStateActive) } // GetScheduledMessages returns all scheduled task messages in the given queue. // It also asserts the state field of the task. func GetScheduledMessages(tb testing.TB, r redis.UniversalClient, qname string) []*base.TaskMessage { tb.Helper() return getMessagesFromZSet(tb, r, qname, base.ScheduledKey, base.TaskStateScheduled) } // GetRetryMessages returns all retry messages in the given queue. // It also asserts the state field of the task. func GetRetryMessages(tb testing.TB, r redis.UniversalClient, qname string) []*base.TaskMessage { tb.Helper() return getMessagesFromZSet(tb, r, qname, base.RetryKey, base.TaskStateRetry) } // GetArchivedMessages returns all archived messages in the given queue. // It also asserts the state field of the task. func GetArchivedMessages(tb testing.TB, r redis.UniversalClient, qname string) []*base.TaskMessage { tb.Helper() return getMessagesFromZSet(tb, r, qname, base.ArchivedKey, base.TaskStateArchived) } // GetCompletedMessages returns all completed task messages in the given queue. // It also asserts the state field of the task. func GetCompletedMessages(tb testing.TB, r redis.UniversalClient, qname string) []*base.TaskMessage { tb.Helper() return getMessagesFromZSet(tb, r, qname, base.CompletedKey, base.TaskStateCompleted) } // GetScheduledEntries returns all scheduled messages and its score in the given queue. // It also asserts the state field of the task. func GetScheduledEntries(tb testing.TB, r redis.UniversalClient, qname string) []base.Z { tb.Helper() return getMessagesFromZSetWithScores(tb, r, qname, base.ScheduledKey, base.TaskStateScheduled) } // GetRetryEntries returns all retry messages and its score in the given queue. // It also asserts the state field of the task. func GetRetryEntries(tb testing.TB, r redis.UniversalClient, qname string) []base.Z { tb.Helper() return getMessagesFromZSetWithScores(tb, r, qname, base.RetryKey, base.TaskStateRetry) } // GetArchivedEntries returns all archived messages and its score in the given queue. // It also asserts the state field of the task. func GetArchivedEntries(tb testing.TB, r redis.UniversalClient, qname string) []base.Z { tb.Helper() return getMessagesFromZSetWithScores(tb, r, qname, base.ArchivedKey, base.TaskStateArchived) } // GetLeaseEntries returns all task IDs and its score in the lease set for the given queue. // It also asserts the state field of the task. func GetLeaseEntries(tb testing.TB, r redis.UniversalClient, qname string) []base.Z { tb.Helper() return getMessagesFromZSetWithScores(tb, r, qname, base.LeaseKey, base.TaskStateActive) } // GetCompletedEntries returns all completed messages and its score in the given queue. // It also asserts the state field of the task. func GetCompletedEntries(tb testing.TB, r redis.UniversalClient, qname string) []base.Z { tb.Helper() return getMessagesFromZSetWithScores(tb, r, qname, base.CompletedKey, base.TaskStateCompleted) } // GetGroupEntries returns all scheduled messages and its score in the given queue. // It also asserts the state field of the task. func GetGroupEntries(tb testing.TB, r redis.UniversalClient, qname, groupKey string) []base.Z { tb.Helper() return getMessagesFromZSetWithScores(tb, r, qname, func(qname string) string { return base.GroupKey(qname, groupKey) }, base.TaskStateAggregating) } // Retrieves all messages stored under `keyFn(qname)` key in redis list. func getMessagesFromList(tb testing.TB, r redis.UniversalClient, qname string, keyFn func(qname string) string, state base.TaskState) []*base.TaskMessage { tb.Helper() ids := r.LRange(context.Background(), keyFn(qname), 0, -1).Val() var msgs []*base.TaskMessage for _, id := range ids { taskKey := base.TaskKey(qname, id) data := r.HGet(context.Background(), taskKey, "msg").Val() msgs = append(msgs, MustUnmarshal(tb, data)) if gotState := r.HGet(context.Background(), taskKey, "state").Val(); gotState != state.String() { tb.Errorf("task (id=%q) is in %q state, want %v", id, gotState, state) } } return msgs } // Retrieves all messages stored under `keyFn(qname)` key in redis zset (sorted-set). func getMessagesFromZSet(tb testing.TB, r redis.UniversalClient, qname string, keyFn func(qname string) string, state base.TaskState) []*base.TaskMessage { tb.Helper() ids := r.ZRange(context.Background(), keyFn(qname), 0, -1).Val() var msgs []*base.TaskMessage for _, id := range ids { taskKey := base.TaskKey(qname, id) msg := r.HGet(context.Background(), taskKey, "msg").Val() msgs = append(msgs, MustUnmarshal(tb, msg)) if gotState := r.HGet(context.Background(), taskKey, "state").Val(); gotState != state.String() { tb.Errorf("task (id=%q) is in %q state, want %v", id, gotState, state) } } return msgs } // Retrieves all messages along with their scores stored under `keyFn(qname)` key in redis zset (sorted-set). func getMessagesFromZSetWithScores(tb testing.TB, r redis.UniversalClient, qname string, keyFn func(qname string) string, state base.TaskState) []base.Z { tb.Helper() zs := r.ZRangeWithScores(context.Background(), keyFn(qname), 0, -1).Val() var res []base.Z for _, z := range zs { taskID := z.Member.(string) taskKey := base.TaskKey(qname, taskID) msg := r.HGet(context.Background(), taskKey, "msg").Val() res = append(res, base.Z{Message: MustUnmarshal(tb, msg), Score: int64(z.Score)}) if gotState := r.HGet(context.Background(), taskKey, "state").Val(); gotState != state.String() { tb.Errorf("task (id=%q) is in %q state, want %v", taskID, gotState, state) } } return res } // TaskSeedData holds the data required to seed tasks under the task key in test. type TaskSeedData struct { Msg *base.TaskMessage State base.TaskState PendingSince time.Time } func SeedTasks(tb testing.TB, r redis.UniversalClient, taskData []*TaskSeedData) { for _, data := range taskData { msg := data.Msg ctx := context.Background() key := base.TaskKey(msg.Queue, msg.ID) v := map[string]interface{}{ "msg": MustMarshal(tb, msg), "state": data.State.String(), "unique_key": msg.UniqueKey, "group": msg.GroupKey, } if !data.PendingSince.IsZero() { v["pending_since"] = data.PendingSince.Unix() } if err := r.HSet(ctx, key, v).Err(); err != nil { tb.Fatalf("Failed to write task data in redis: %v", err) } if len(msg.UniqueKey) > 0 { err := r.SetNX(ctx, msg.UniqueKey, msg.ID, 1*time.Minute).Err() if err != nil { tb.Fatalf("Failed to set unique lock in redis: %v", err) } } } } func SeedRedisZSets(tb testing.TB, r redis.UniversalClient, zsets map[string][]redis.Z) { for key, zs := range zsets { // FIXME: How come we can't simply do ZAdd(ctx, key, zs...) here? for _, z := range zs { if err := r.ZAdd(context.Background(), key, z).Err(); err != nil { tb.Fatalf("Failed to seed zset (key=%q): %v", key, err) } } } } func SeedRedisSets(tb testing.TB, r redis.UniversalClient, sets map[string][]string) { for key, set := range sets { SeedRedisSet(tb, r, key, set) } } func SeedRedisSet(tb testing.TB, r redis.UniversalClient, key string, members []string) { for _, mem := range members { if err := r.SAdd(context.Background(), key, mem).Err(); err != nil { tb.Fatalf("Failed to seed set (key=%q): %v", key, err) } } } func SeedRedisLists(tb testing.TB, r redis.UniversalClient, lists map[string][]string) { for key, vals := range lists { for _, v := range vals { if err := r.LPush(context.Background(), key, v).Err(); err != nil { tb.Fatalf("Failed to seed list (key=%q): %v", key, err) } } } } func AssertRedisLists(t *testing.T, r redis.UniversalClient, wantLists map[string][]string) { for key, want := range wantLists { got, err := r.LRange(context.Background(), key, 0, -1).Result() if err != nil { t.Fatalf("Failed to read list (key=%q): %v", key, err) } if diff := cmp.Diff(want, got, SortStringSliceOpt); diff != "" { t.Errorf("mismatch found in list (key=%q): (-want,+got)\n%s", key, diff) } } } func AssertRedisSets(t *testing.T, r redis.UniversalClient, wantSets map[string][]string) { for key, want := range wantSets { got, err := r.SMembers(context.Background(), key).Result() if err != nil { t.Fatalf("Failed to read set (key=%q): %v", key, err) } if diff := cmp.Diff(want, got, SortStringSliceOpt); diff != "" { t.Errorf("mismatch found in set (key=%q): (-want,+got)\n%s", key, diff) } } } func AssertRedisZSets(t *testing.T, r redis.UniversalClient, wantZSets map[string][]redis.Z) { for key, want := range wantZSets { got, err := r.ZRangeWithScores(context.Background(), key, 0, -1).Result() if err != nil { t.Fatalf("Failed to read zset (key=%q): %v", key, err) } if diff := cmp.Diff(want, got, SortRedisZSetEntryOpt); diff != "" { t.Errorf("mismatch found in zset (key=%q): (-want,+got)\n%s", key, diff) } } } ================================================ FILE: internal/timeutil/timeutil.go ================================================ // Copyright 2022 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. // Package timeutil exports functions and types related to time and date. package timeutil import ( "sync" "time" ) // A Clock is an object that can tell you the current time. // // This interface allows decoupling code that uses time from the code that creates // a point in time. You can use this to your advantage by injecting Clocks into interfaces // rather than having implementations call time.Now() directly. // // Use RealClock() in production. // Use SimulatedClock() in test. type Clock interface { Now() time.Time } func NewRealClock() Clock { return &realTimeClock{} } type realTimeClock struct{} func (_ *realTimeClock) Now() time.Time { return time.Now() } // A SimulatedClock is a concrete Clock implementation that doesn't "tick" on its own. // Time is advanced by explicit call to the AdvanceTime() or SetTime() functions. // This object is concurrency safe. type SimulatedClock struct { mu sync.Mutex t time.Time // guarded by mu } func NewSimulatedClock(t time.Time) *SimulatedClock { return &SimulatedClock{t: t} } func (c *SimulatedClock) Now() time.Time { c.mu.Lock() defer c.mu.Unlock() return c.t } func (c *SimulatedClock) SetTime(t time.Time) { c.mu.Lock() defer c.mu.Unlock() c.t = t } func (c *SimulatedClock) AdvanceTime(d time.Duration) { c.mu.Lock() defer c.mu.Unlock() c.t = c.t.Add(d) } ================================================ FILE: internal/timeutil/timeutil_test.go ================================================ // Copyright 2022 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package timeutil import ( "testing" "time" ) func TestSimulatedClock(t *testing.T) { now := time.Now() tests := []struct { desc string initTime time.Time advanceBy time.Duration wantTime time.Time }{ { desc: "advance time forward", initTime: now, advanceBy: 30 * time.Second, wantTime: now.Add(30 * time.Second), }, { desc: "advance time backward", initTime: now, advanceBy: -10 * time.Second, wantTime: now.Add(-10 * time.Second), }, } for _, tc := range tests { c := NewSimulatedClock(tc.initTime) if c.Now() != tc.initTime { t.Errorf("%s: Before Advance; SimulatedClock.Now() = %v, want %v", tc.desc, c.Now(), tc.initTime) } c.AdvanceTime(tc.advanceBy) if c.Now() != tc.wantTime { t.Errorf("%s: After Advance; SimulatedClock.Now() = %v, want %v", tc.desc, c.Now(), tc.wantTime) } } } ================================================ FILE: janitor.go ================================================ // Copyright 2021 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "sync" "time" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/log" ) // A janitor is responsible for deleting expired completed tasks from the specified // queues. It periodically checks for any expired tasks in the completed set, and // deletes them. type janitor struct { logger *log.Logger broker base.Broker // channel to communicate back to the long running "janitor" goroutine. done chan struct{} // list of queue names to check. queues []string // average interval between checks. avgInterval time.Duration // number of tasks to be deleted when janitor runs to delete the expired completed tasks. batchSize int } type janitorParams struct { logger *log.Logger broker base.Broker queues []string interval time.Duration batchSize int } func newJanitor(params janitorParams) *janitor { return &janitor{ logger: params.logger, broker: params.broker, done: make(chan struct{}), queues: params.queues, avgInterval: params.interval, batchSize: params.batchSize, } } func (j *janitor) shutdown() { j.logger.Debug("Janitor shutting down...") // Signal the janitor goroutine to stop. j.done <- struct{}{} } // start starts the "janitor" goroutine. func (j *janitor) start(wg *sync.WaitGroup) { wg.Add(1) timer := time.NewTimer(j.avgInterval) // randomize this interval with margin of 1s go func() { defer wg.Done() for { select { case <-j.done: j.logger.Debug("Janitor done") return case <-timer.C: j.exec() timer.Reset(j.avgInterval) } } }() } func (j *janitor) exec() { for _, qname := range j.queues { if err := j.broker.DeleteExpiredCompletedTasks(qname, j.batchSize); err != nil { j.logger.Errorf("Failed to delete expired completed tasks from queue %q: %v", qname, err) } } } ================================================ FILE: janitor_test.go ================================================ // Copyright 2021 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "sync" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/rdb" h "github.com/hibiken/asynq/internal/testutil" ) func newCompletedTask(qname, tasktype string, payload []byte, completedAt time.Time) *base.TaskMessage { msg := h.NewTaskMessageWithQueue(tasktype, payload, qname) msg.CompletedAt = completedAt.Unix() return msg } func TestJanitor(t *testing.T) { r := setup(t) defer r.Close() rdbClient := rdb.NewRDB(r) const interval = 1 * time.Second const batchSize = 100 janitor := newJanitor(janitorParams{ logger: testLogger, broker: rdbClient, queues: []string{"default", "custom"}, interval: interval, batchSize: batchSize, }) now := time.Now() hourAgo := now.Add(-1 * time.Hour) minuteAgo := now.Add(-1 * time.Minute) halfHourAgo := now.Add(-30 * time.Minute) halfHourFromNow := now.Add(30 * time.Minute) fiveMinFromNow := now.Add(5 * time.Minute) msg1 := newCompletedTask("default", "task1", nil, hourAgo) msg2 := newCompletedTask("default", "task2", nil, minuteAgo) msg3 := newCompletedTask("custom", "task3", nil, hourAgo) msg4 := newCompletedTask("custom", "task4", nil, minuteAgo) tests := []struct { completed map[string][]base.Z // initial completed sets wantCompleted map[string][]base.Z // expected completed sets after janitor runs }{ { completed: map[string][]base.Z{ "default": { {Message: msg1, Score: halfHourAgo.Unix()}, {Message: msg2, Score: fiveMinFromNow.Unix()}, }, "custom": { {Message: msg3, Score: halfHourFromNow.Unix()}, {Message: msg4, Score: minuteAgo.Unix()}, }, }, wantCompleted: map[string][]base.Z{ "default": { {Message: msg2, Score: fiveMinFromNow.Unix()}, }, "custom": { {Message: msg3, Score: halfHourFromNow.Unix()}, }, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllCompletedQueues(t, r, tc.completed) var wg sync.WaitGroup janitor.start(&wg) time.Sleep(2 * interval) // make sure to let janitor run at least one time janitor.shutdown() for qname, want := range tc.wantCompleted { got := h.GetCompletedEntries(t, r, qname) if diff := cmp.Diff(want, got, h.SortZSetEntryOpt); diff != "" { t.Errorf("diff found in %q after running janitor: (-want, +got)\n%s", base.CompletedKey(qname), diff) } } } } ================================================ FILE: periodic_task_manager.go ================================================ // Copyright 2022 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "crypto/sha256" "fmt" "sort" "sync" "time" "github.com/redis/go-redis/v9" ) // PeriodicTaskManager manages scheduling of periodic tasks. // It syncs scheduler's entries by calling the config provider periodically. type PeriodicTaskManager struct { s *Scheduler p PeriodicTaskConfigProvider syncInterval time.Duration done chan (struct{}) wg sync.WaitGroup m map[string]string // map[hash]entryID } type PeriodicTaskManagerOpts struct { // Required: must be non nil PeriodicTaskConfigProvider PeriodicTaskConfigProvider // Optional: if RedisUniversalClient is nil must be non nil RedisConnOpt RedisConnOpt // Optional: if RedisUniversalClient is non nil, RedisConnOpt is ignored. RedisUniversalClient redis.UniversalClient // Optional: scheduler options *SchedulerOpts // Optional: default is 3m SyncInterval time.Duration } const defaultSyncInterval = 3 * time.Minute // NewPeriodicTaskManager returns a new PeriodicTaskManager instance. // The given opts should specify the RedisConnOp and PeriodicTaskConfigProvider at minimum. func NewPeriodicTaskManager(opts PeriodicTaskManagerOpts) (*PeriodicTaskManager, error) { if opts.PeriodicTaskConfigProvider == nil { return nil, fmt.Errorf("PeriodicTaskConfigProvider cannot be nil") } if opts.RedisConnOpt == nil && opts.RedisUniversalClient == nil { return nil, fmt.Errorf("RedisConnOpt/RedisUniversalClient cannot be nil") } var scheduler *Scheduler if opts.RedisUniversalClient != nil { scheduler = NewSchedulerFromRedisClient(opts.RedisUniversalClient, opts.SchedulerOpts) } else { scheduler = NewScheduler(opts.RedisConnOpt, opts.SchedulerOpts) } syncInterval := opts.SyncInterval if syncInterval == 0 { syncInterval = defaultSyncInterval } return &PeriodicTaskManager{ s: scheduler, p: opts.PeriodicTaskConfigProvider, syncInterval: syncInterval, done: make(chan struct{}), m: make(map[string]string), }, nil } // PeriodicTaskConfigProvider provides configs for periodic tasks. // GetConfigs will be called by a PeriodicTaskManager periodically to // sync the scheduler's entries with the configs returned by the provider. type PeriodicTaskConfigProvider interface { GetConfigs() ([]*PeriodicTaskConfig, error) } // PeriodicTaskConfig specifies the details of a periodic task. type PeriodicTaskConfig struct { Cronspec string // required: must be non empty string Task *Task // required: must be non nil Opts []Option // optional: can be nil } func (c *PeriodicTaskConfig) hash() string { h := sha256.New() _, _ = h.Write([]byte(c.Cronspec)) _, _ = h.Write([]byte(c.Task.Type())) h.Write(c.Task.Payload()) opts := stringifyOptions(c.Opts) sort.Strings(opts) for _, opt := range opts { _, _ = h.Write([]byte(opt)) } return fmt.Sprintf("%x", h.Sum(nil)) } func validatePeriodicTaskConfig(c *PeriodicTaskConfig) error { if c == nil { return fmt.Errorf("PeriodicTaskConfig cannot be nil") } if c.Task == nil { return fmt.Errorf("PeriodicTaskConfig.Task cannot be nil") } if c.Cronspec == "" { return fmt.Errorf("PeriodicTaskConfig.Cronspec cannot be empty") } return nil } // Start starts a scheduler and background goroutine to sync the scheduler with the configs // returned by the provider. // // Start returns any error encountered at start up time. func (mgr *PeriodicTaskManager) Start() error { if mgr.s == nil || mgr.p == nil { panic("asynq: cannot start uninitialized PeriodicTaskManager; use NewPeriodicTaskManager to initialize") } if err := mgr.initialSync(); err != nil { return fmt.Errorf("asynq: %w", err) } if err := mgr.s.Start(); err != nil { return fmt.Errorf("asynq: %w", err) } mgr.wg.Add(1) go func() { defer mgr.wg.Done() ticker := time.NewTicker(mgr.syncInterval) for { select { case <-mgr.done: mgr.s.logger.Debugf("Stopping syncer goroutine") ticker.Stop() return case <-ticker.C: mgr.sync() } } }() return nil } // Shutdown gracefully shuts down the manager. // It notifies a background syncer goroutine to stop and stops scheduler. func (mgr *PeriodicTaskManager) Shutdown() { close(mgr.done) mgr.wg.Wait() mgr.s.Shutdown() } // Run starts the manager and blocks until an os signal to exit the program is received. // Once it receives a signal, it gracefully shuts down the manager. func (mgr *PeriodicTaskManager) Run() error { if err := mgr.Start(); err != nil { return err } mgr.s.waitForSignals() mgr.Shutdown() mgr.s.logger.Debugf("PeriodicTaskManager exiting") return nil } func (mgr *PeriodicTaskManager) initialSync() error { configs, err := mgr.p.GetConfigs() if err != nil { return fmt.Errorf("initial call to GetConfigs failed: %w", err) } for _, c := range configs { if err := validatePeriodicTaskConfig(c); err != nil { return fmt.Errorf("initial call to GetConfigs contained an invalid config: %w", err) } } mgr.add(configs) return nil } func (mgr *PeriodicTaskManager) add(configs []*PeriodicTaskConfig) { for _, c := range configs { entryID, err := mgr.s.Register(c.Cronspec, c.Task, c.Opts...) if err != nil { mgr.s.logger.Errorf("Failed to register periodic task: cronspec=%q task=%q err=%v", c.Cronspec, c.Task.Type(), err) continue } mgr.m[c.hash()] = entryID mgr.s.logger.Infof("Successfully registered periodic task: cronspec=%q task=%q, entryID=%s", c.Cronspec, c.Task.Type(), entryID) } } func (mgr *PeriodicTaskManager) remove(removed map[string]string) { for hash, entryID := range removed { if err := mgr.s.Unregister(entryID); err != nil { mgr.s.logger.Errorf("Failed to unregister periodic task: %v", err) continue } delete(mgr.m, hash) mgr.s.logger.Infof("Successfully unregistered periodic task: entryID=%s", entryID) } } func (mgr *PeriodicTaskManager) sync() { configs, err := mgr.p.GetConfigs() if err != nil { mgr.s.logger.Errorf("Failed to get periodic task configs: %v", err) return } for _, c := range configs { if err := validatePeriodicTaskConfig(c); err != nil { mgr.s.logger.Errorf("Failed to sync: GetConfigs returned an invalid config: %v", err) return } } // Diff and only register/unregister the newly added/removed entries. removed := mgr.diffRemoved(configs) added := mgr.diffAdded(configs) mgr.remove(removed) mgr.add(added) } // diffRemoved diffs the incoming configs with the registered config and returns // a map containing hash and entryID of each config that was removed. func (mgr *PeriodicTaskManager) diffRemoved(configs []*PeriodicTaskConfig) map[string]string { newConfigs := make(map[string]string) for _, c := range configs { newConfigs[c.hash()] = "" // empty value since we don't have entryID yet } removed := make(map[string]string) for k, v := range mgr.m { // test whether existing config is present in the incoming configs if _, found := newConfigs[k]; !found { removed[k] = v } } return removed } // diffAdded diffs the incoming configs with the registered configs and returns // a list of configs that were added. func (mgr *PeriodicTaskManager) diffAdded(configs []*PeriodicTaskConfig) []*PeriodicTaskConfig { var added []*PeriodicTaskConfig for _, c := range configs { if _, found := mgr.m[c.hash()]; !found { added = append(added, c) } } return added } ================================================ FILE: periodic_task_manager_test.go ================================================ // Copyright 2022 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "sort" "sync" "testing" "time" "github.com/google/go-cmp/cmp" ) // Trivial implementation of PeriodicTaskConfigProvider for testing purpose. type FakeConfigProvider struct { mu sync.Mutex cfgs []*PeriodicTaskConfig } func (p *FakeConfigProvider) SetConfigs(cfgs []*PeriodicTaskConfig) { p.mu.Lock() defer p.mu.Unlock() p.cfgs = cfgs } func (p *FakeConfigProvider) GetConfigs() ([]*PeriodicTaskConfig, error) { p.mu.Lock() defer p.mu.Unlock() return p.cfgs, nil } func TestNewPeriodicTaskManager(t *testing.T) { redisConnOpt := getRedisConnOpt(t) cfgs := []*PeriodicTaskConfig{ {Cronspec: "* * * * *", Task: NewTask("foo", nil)}, {Cronspec: "* * * * *", Task: NewTask("bar", nil)}, } tests := []struct { desc string opts PeriodicTaskManagerOpts }{ { desc: "with provider and redisConnOpt", opts: PeriodicTaskManagerOpts{ RedisConnOpt: redisConnOpt, PeriodicTaskConfigProvider: &FakeConfigProvider{cfgs: cfgs}, }, }, { desc: "with sync option", opts: PeriodicTaskManagerOpts{ RedisConnOpt: redisConnOpt, PeriodicTaskConfigProvider: &FakeConfigProvider{cfgs: cfgs}, SyncInterval: 5 * time.Minute, }, }, { desc: "with scheduler option", opts: PeriodicTaskManagerOpts{ RedisConnOpt: redisConnOpt, PeriodicTaskConfigProvider: &FakeConfigProvider{cfgs: cfgs}, SyncInterval: 5 * time.Minute, SchedulerOpts: &SchedulerOpts{ LogLevel: DebugLevel, }, }, }, } for _, tc := range tests { _, err := NewPeriodicTaskManager(tc.opts) if err != nil { t.Errorf("%s; NewPeriodicTaskManager returned error: %v", tc.desc, err) } } t.Run("error", func(t *testing.T) { tests := []struct { desc string opts PeriodicTaskManagerOpts }{ { desc: "without provider", opts: PeriodicTaskManagerOpts{ RedisConnOpt: redisConnOpt, }, }, { desc: "without redisConOpt", opts: PeriodicTaskManagerOpts{ PeriodicTaskConfigProvider: &FakeConfigProvider{cfgs: cfgs}, }, }, } for _, tc := range tests { _, err := NewPeriodicTaskManager(tc.opts) if err == nil { t.Errorf("%s; NewPeriodicTaskManager did not return error", tc.desc) } } }) } func TestPeriodicTaskConfigHash(t *testing.T) { tests := []struct { desc string a *PeriodicTaskConfig b *PeriodicTaskConfig isSame bool }{ { desc: "basic identity test", a: &PeriodicTaskConfig{ Cronspec: "* * * * *", Task: NewTask("foo", nil), }, b: &PeriodicTaskConfig{ Cronspec: "* * * * *", Task: NewTask("foo", nil), }, isSame: true, }, { desc: "with a option", a: &PeriodicTaskConfig{ Cronspec: "* * * * *", Task: NewTask("foo", nil), Opts: []Option{Queue("myqueue")}, }, b: &PeriodicTaskConfig{ Cronspec: "* * * * *", Task: NewTask("foo", nil), Opts: []Option{Queue("myqueue")}, }, isSame: true, }, { desc: "with multiple options (different order)", a: &PeriodicTaskConfig{ Cronspec: "* * * * *", Task: NewTask("foo", nil), Opts: []Option{Unique(5 * time.Minute), Queue("myqueue")}, }, b: &PeriodicTaskConfig{ Cronspec: "* * * * *", Task: NewTask("foo", nil), Opts: []Option{Queue("myqueue"), Unique(5 * time.Minute)}, }, isSame: true, }, { desc: "with payload", a: &PeriodicTaskConfig{ Cronspec: "* * * * *", Task: NewTask("foo", []byte("hello world!")), Opts: []Option{Queue("myqueue")}, }, b: &PeriodicTaskConfig{ Cronspec: "* * * * *", Task: NewTask("foo", []byte("hello world!")), Opts: []Option{Queue("myqueue")}, }, isSame: true, }, { desc: "with different cronspecs", a: &PeriodicTaskConfig{ Cronspec: "* * * * *", Task: NewTask("foo", nil), }, b: &PeriodicTaskConfig{ Cronspec: "5 * * * *", Task: NewTask("foo", nil), }, isSame: false, }, { desc: "with different task type", a: &PeriodicTaskConfig{ Cronspec: "* * * * *", Task: NewTask("foo", nil), }, b: &PeriodicTaskConfig{ Cronspec: "* * * * *", Task: NewTask("bar", nil), }, isSame: false, }, { desc: "with different options", a: &PeriodicTaskConfig{ Cronspec: "* * * * *", Task: NewTask("foo", nil), Opts: []Option{Queue("myqueue")}, }, b: &PeriodicTaskConfig{ Cronspec: "* * * * *", Task: NewTask("foo", nil), Opts: []Option{Unique(10 * time.Minute)}, }, isSame: false, }, { desc: "with different options (one is subset of the other)", a: &PeriodicTaskConfig{ Cronspec: "* * * * *", Task: NewTask("foo", nil), Opts: []Option{Queue("myqueue")}, }, b: &PeriodicTaskConfig{ Cronspec: "* * * * *", Task: NewTask("foo", nil), Opts: []Option{Queue("myqueue"), Unique(10 * time.Minute)}, }, isSame: false, }, { desc: "with different payload", a: &PeriodicTaskConfig{ Cronspec: "* * * * *", Task: NewTask("foo", []byte("hello!")), Opts: []Option{Queue("myqueue")}, }, b: &PeriodicTaskConfig{ Cronspec: "* * * * *", Task: NewTask("foo", []byte("HELLO!")), Opts: []Option{Queue("myqueue"), Unique(10 * time.Minute)}, }, isSame: false, }, } for _, tc := range tests { if tc.isSame && tc.a.hash() != tc.b.hash() { t.Errorf("%s: a.hash=%s b.hash=%s expected to be equal", tc.desc, tc.a.hash(), tc.b.hash()) } if !tc.isSame && tc.a.hash() == tc.b.hash() { t.Errorf("%s: a.hash=%s b.hash=%s expected to be not equal", tc.desc, tc.a.hash(), tc.b.hash()) } } } // Things to test. // - Run the manager // - Change provider to return new configs // - Verify that the scheduler synced with the new config func TestPeriodicTaskManager(t *testing.T) { // Note: In this test, we'll use task type as an ID for each config. cfgs := []*PeriodicTaskConfig{ {Task: NewTask("task1", nil), Cronspec: "* * * * 1"}, {Task: NewTask("task2", nil), Cronspec: "* * * * 2"}, } const syncInterval = 3 * time.Second provider := &FakeConfigProvider{cfgs: cfgs} mgr, err := NewPeriodicTaskManager(PeriodicTaskManagerOpts{ RedisConnOpt: getRedisConnOpt(t), PeriodicTaskConfigProvider: provider, SyncInterval: syncInterval, }) if err != nil { t.Fatalf("Failed to initialize PeriodicTaskManager: %v", err) } if err := mgr.Start(); err != nil { t.Fatalf("Failed to start PeriodicTaskManager: %v", err) } defer mgr.Shutdown() got := extractCronEntries(mgr.s) want := []*cronEntry{ {Cronspec: "* * * * 1", TaskType: "task1"}, {Cronspec: "* * * * 2", TaskType: "task2"}, } if diff := cmp.Diff(want, got, sortCronEntry); diff != "" { t.Errorf("Diff found in scheduler's registered entries: %s", diff) } // Change the underlying configs // - task2 removed // - task3 added provider.SetConfigs([]*PeriodicTaskConfig{ {Task: NewTask("task1", nil), Cronspec: "* * * * 1"}, {Task: NewTask("task3", nil), Cronspec: "* * * * 3"}, }) // Wait for the next sync time.Sleep(syncInterval * 2) // Verify the entries are synced got = extractCronEntries(mgr.s) want = []*cronEntry{ {Cronspec: "* * * * 1", TaskType: "task1"}, {Cronspec: "* * * * 3", TaskType: "task3"}, } if diff := cmp.Diff(want, got, sortCronEntry); diff != "" { t.Errorf("Diff found in scheduler's registered entries: %s", diff) } // Change the underlying configs // All configs removed, empty set. provider.SetConfigs([]*PeriodicTaskConfig{}) // Wait for the next sync time.Sleep(syncInterval * 2) // Verify the entries are synced got = extractCronEntries(mgr.s) want = []*cronEntry{} if diff := cmp.Diff(want, got, sortCronEntry); diff != "" { t.Errorf("Diff found in scheduler's registered entries: %s", diff) } } func extractCronEntries(s *Scheduler) []*cronEntry { var out []*cronEntry for _, e := range s.cron.Entries() { job := e.Job.(*enqueueJob) out = append(out, &cronEntry{Cronspec: job.cronspec, TaskType: job.task.Type()}) } return out } var sortCronEntry = cmp.Transformer("sortCronEntry", func(in []*cronEntry) []*cronEntry { out := append([]*cronEntry(nil), in...) sort.Slice(out, func(i, j int) bool { return out[i].TaskType < out[j].TaskType }) return out }) // A simple struct to allow for simpler comparison in test. type cronEntry struct { Cronspec string TaskType string } ================================================ FILE: processor.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "context" "fmt" "math" "math/rand/v2" "runtime" "runtime/debug" "sort" "strings" "sync" "time" "github.com/hibiken/asynq/internal/base" asynqcontext "github.com/hibiken/asynq/internal/context" "github.com/hibiken/asynq/internal/errors" "github.com/hibiken/asynq/internal/log" "github.com/hibiken/asynq/internal/timeutil" "golang.org/x/time/rate" ) type processor struct { logger *log.Logger broker base.Broker clock timeutil.Clock handler Handler baseCtxFn func() context.Context queueConfig map[string]int // orderedQueues is set only in strict-priority mode. orderedQueues []string taskCheckInterval time.Duration retryDelayFunc RetryDelayFunc isFailureFunc func(error) bool errHandler ErrorHandler shutdownTimeout time.Duration // channel via which to send sync requests to syncer. syncRequestCh chan<- *syncRequest // rate limiter to prevent spamming logs with a bunch of errors. errLogLimiter *rate.Limiter // sema is a counting semaphore to ensure the number of active workers // does not exceed the limit. sema chan struct{} // channel to communicate back to the long running "processor" goroutine. // once is used to send value to the channel only once. done chan struct{} once sync.Once // quit channel is closed when the shutdown of the "processor" goroutine starts. quit chan struct{} // abort channel communicates to the in-flight worker goroutines to stop. abort chan struct{} // cancelations is a set of cancel functions for all active tasks. cancelations *base.Cancelations starting chan<- *workerInfo finished chan<- *base.TaskMessage } type processorParams struct { logger *log.Logger broker base.Broker baseCtxFn func() context.Context retryDelayFunc RetryDelayFunc taskCheckInterval time.Duration isFailureFunc func(error) bool syncCh chan<- *syncRequest cancelations *base.Cancelations concurrency int queues map[string]int strictPriority bool errHandler ErrorHandler shutdownTimeout time.Duration starting chan<- *workerInfo finished chan<- *base.TaskMessage } // newProcessor constructs a new processor. func newProcessor(params processorParams) *processor { queues := normalizeQueues(params.queues) orderedQueues := []string(nil) if params.strictPriority { orderedQueues = sortByPriority(queues) } return &processor{ logger: params.logger, broker: params.broker, baseCtxFn: params.baseCtxFn, clock: timeutil.NewRealClock(), queueConfig: queues, orderedQueues: orderedQueues, taskCheckInterval: params.taskCheckInterval, retryDelayFunc: params.retryDelayFunc, isFailureFunc: params.isFailureFunc, syncRequestCh: params.syncCh, cancelations: params.cancelations, errLogLimiter: rate.NewLimiter(rate.Every(3*time.Second), 1), sema: make(chan struct{}, params.concurrency), done: make(chan struct{}), quit: make(chan struct{}), abort: make(chan struct{}), errHandler: params.errHandler, handler: HandlerFunc(func(ctx context.Context, t *Task) error { return fmt.Errorf("handler not set") }), shutdownTimeout: params.shutdownTimeout, starting: params.starting, finished: params.finished, } } // Note: stops only the "processor" goroutine, does not stop workers. // It's safe to call this method multiple times. func (p *processor) stop() { p.once.Do(func() { p.logger.Debug("Processor shutting down...") // Unblock if processor is waiting for sema token. close(p.quit) // Signal the processor goroutine to stop processing tasks // from the queue. p.done <- struct{}{} }) } // NOTE: once shutdown, processor cannot be re-started. func (p *processor) shutdown() { p.stop() time.AfterFunc(p.shutdownTimeout, func() { close(p.abort) }) p.logger.Info("Waiting for all workers to finish...") // block until all workers have released the token for i := 0; i < cap(p.sema); i++ { p.sema <- struct{}{} } p.logger.Info("All workers have finished") } func (p *processor) start(wg *sync.WaitGroup) { wg.Add(1) go func() { defer wg.Done() for { select { case <-p.done: p.logger.Debug("Processor done") return default: p.exec() } } }() } // exec pulls a task out of the queue and starts a worker goroutine to // process the task. func (p *processor) exec() { select { case <-p.quit: return case p.sema <- struct{}{}: // acquire token qnames := p.queues() msg, leaseExpirationTime, err := p.broker.Dequeue(qnames...) switch { case errors.Is(err, errors.ErrNoProcessableTask): p.logger.Debug("All queues are empty") // Queues are empty, this is a normal behavior. // Sleep to avoid slamming redis and let scheduler move tasks into queues. // Note: We are not using blocking pop operation and polling queues instead. // This adds significant load to redis. jitter := rand.N(p.taskCheckInterval) time.Sleep(p.taskCheckInterval/2 + jitter) <-p.sema // release token return case err != nil: if p.errLogLimiter.Allow() { p.logger.Errorf("Dequeue error: %v", err) } <-p.sema // release token return } lease := base.NewLease(leaseExpirationTime) deadline := p.computeDeadline(msg) p.starting <- &workerInfo{msg, time.Now(), deadline, lease} go func() { defer func() { p.finished <- msg <-p.sema // release token }() ctx, cancel := asynqcontext.New(p.baseCtxFn(), msg, deadline) p.cancelations.Add(msg.ID, cancel) defer func() { cancel() p.cancelations.Delete(msg.ID) }() // check context before starting a worker goroutine. select { case <-ctx.Done(): // already canceled (e.g. deadline exceeded). p.handleFailedMessage(ctx, lease, msg, ctx.Err()) return default: } resCh := make(chan error, 1) go func() { task := newTask( msg.Type, msg.Payload, &ResultWriter{ id: msg.ID, qname: msg.Queue, broker: p.broker, ctx: ctx, }, ) task.headers = msg.Headers resCh <- p.perform(ctx, task) }() select { case <-p.abort: // time is up, push the message back to queue and quit this worker goroutine. p.logger.Warnf("Quitting worker. task id=%s", msg.ID) p.requeue(lease, msg) return case <-lease.Done(): cancel() p.handleFailedMessage(ctx, lease, msg, ErrLeaseExpired) return case <-ctx.Done(): p.handleFailedMessage(ctx, lease, msg, ctx.Err()) return case resErr := <-resCh: if resErr != nil { p.handleFailedMessage(ctx, lease, msg, resErr) return } p.handleSucceededMessage(lease, msg) } }() } } func (p *processor) requeue(l *base.Lease, msg *base.TaskMessage) { if !l.IsValid() { // If lease is not valid, do not write to redis; Let recoverer take care of it. return } ctx, cancel := context.WithDeadline(context.Background(), l.Deadline()) defer cancel() err := p.broker.Requeue(ctx, msg) if err != nil { p.logger.Errorf("Could not push task id=%s back to queue: %v", msg.ID, err) } else { p.logger.Infof("Pushed task id=%s back to queue", msg.ID) } } func (p *processor) handleSucceededMessage(l *base.Lease, msg *base.TaskMessage) { if msg.Retention > 0 { p.markAsComplete(l, msg) } else { p.markAsDone(l, msg) } } func (p *processor) markAsComplete(l *base.Lease, msg *base.TaskMessage) { if !l.IsValid() { // If lease is not valid, do not write to redis; Let recoverer take care of it. return } ctx, cancel := context.WithDeadline(context.Background(), l.Deadline()) defer cancel() err := p.broker.MarkAsComplete(ctx, msg) if err != nil { errMsg := fmt.Sprintf("Could not move task id=%s type=%q from %q to %q: %+v", msg.ID, msg.Type, base.ActiveKey(msg.Queue), base.CompletedKey(msg.Queue), err) p.logger.Warnf("%s; Will retry syncing", errMsg) p.syncRequestCh <- &syncRequest{ fn: func() error { return p.broker.MarkAsComplete(ctx, msg) }, errMsg: errMsg, deadline: l.Deadline(), } } } func (p *processor) markAsDone(l *base.Lease, msg *base.TaskMessage) { if !l.IsValid() { // If lease is not valid, do not write to redis; Let recoverer take care of it. return } ctx, cancel := context.WithDeadline(context.Background(), l.Deadline()) defer cancel() err := p.broker.Done(ctx, msg) if err != nil { errMsg := fmt.Sprintf("Could not remove task id=%s type=%q from %q err: %+v", msg.ID, msg.Type, base.ActiveKey(msg.Queue), err) p.logger.Warnf("%s; Will retry syncing", errMsg) p.syncRequestCh <- &syncRequest{ fn: func() error { return p.broker.Done(ctx, msg) }, errMsg: errMsg, deadline: l.Deadline(), } } } // SkipRetry is used as a return value from Handler.ProcessTask to indicate that // the task should not be retried and should be archived instead. var SkipRetry = errors.New("skip retry for the task") // RevokeTask is used as a return value from Handler.ProcessTask to indicate that // the task should not be retried or archived. var RevokeTask = errors.New("revoke task") func (p *processor) handleFailedMessage(ctx context.Context, l *base.Lease, msg *base.TaskMessage, err error) { if p.errHandler != nil { p.errHandler.HandleError(ctx, NewTaskWithHeaders(msg.Type, msg.Payload, msg.Headers), err) } switch { case errors.Is(err, RevokeTask): p.logger.Warnf("revoke task id=%s", msg.ID) p.markAsDone(l, msg) case msg.Retried >= msg.Retry || errors.Is(err, SkipRetry): p.logger.Warnf("Retry exhausted for task id=%s", msg.ID) p.archive(l, msg, err) default: p.retry(l, msg, err, p.isFailureFunc(err)) } } func (p *processor) retry(l *base.Lease, msg *base.TaskMessage, e error, isFailure bool) { if !l.IsValid() { // If lease is not valid, do not write to redis; Let recoverer take care of it. return } ctx, cancel := context.WithDeadline(context.Background(), l.Deadline()) defer cancel() d := p.retryDelayFunc(msg.Retried, e, NewTaskWithHeaders(msg.Type, msg.Payload, msg.Headers)) retryAt := time.Now().Add(d) err := p.broker.Retry(ctx, msg, retryAt, e.Error(), isFailure) if err != nil { errMsg := fmt.Sprintf("Could not move task id=%s from %q to %q", msg.ID, base.ActiveKey(msg.Queue), base.RetryKey(msg.Queue)) p.logger.Warnf("%s; Will retry syncing", errMsg) p.syncRequestCh <- &syncRequest{ fn: func() error { return p.broker.Retry(ctx, msg, retryAt, e.Error(), isFailure) }, errMsg: errMsg, deadline: l.Deadline(), } } } func (p *processor) archive(l *base.Lease, msg *base.TaskMessage, e error) { if !l.IsValid() { // If lease is not valid, do not write to redis; Let recoverer take care of it. return } ctx, cancel := context.WithDeadline(context.Background(), l.Deadline()) defer cancel() err := p.broker.Archive(ctx, msg, e.Error()) if err != nil { errMsg := fmt.Sprintf("Could not move task id=%s from %q to %q", msg.ID, base.ActiveKey(msg.Queue), base.ArchivedKey(msg.Queue)) p.logger.Warnf("%s; Will retry syncing", errMsg) p.syncRequestCh <- &syncRequest{ fn: func() error { return p.broker.Archive(ctx, msg, e.Error()) }, errMsg: errMsg, deadline: l.Deadline(), } } } // queues returns a list of queues to query. // Order of the queue names is based on the priority of each queue. // Queue names is sorted by their priority level if strict-priority is true. // If strict-priority is false, then the order of queue names are roughly based on // the priority level but randomized in order to avoid starving low priority queues. func (p *processor) queues() []string { // skip the overhead of generating a list of queue names // if we are processing one queue. if len(p.queueConfig) == 1 { for qname := range p.queueConfig { return []string{qname} } } if p.orderedQueues != nil { return p.orderedQueues } var names []string for qname, priority := range p.queueConfig { for i := 0; i < priority; i++ { names = append(names, qname) } } rand.Shuffle(len(names), func(i, j int) { names[i], names[j] = names[j], names[i] }) return uniq(names, len(p.queueConfig)) } // perform calls the handler with the given task. // If the call returns without panic, it simply returns the value, // otherwise, it recovers from panic and returns an error. func (p *processor) perform(ctx context.Context, task *Task) (err error) { defer func() { if x := recover(); x != nil { p.logger.Errorf("recovering from panic. See the stack trace below for details:\n%s", string(debug.Stack())) _, file, line, ok := runtime.Caller(1) // skip the first frame (panic itself) if ok && strings.Contains(file, "runtime/") { // The panic came from the runtime, most likely due to incorrect // map/slice usage. The parent frame should have the real trigger. _, file, line, ok = runtime.Caller(2) } var errMsg string // Include the file and line number info in the error, if runtime.Caller returned ok. if ok { errMsg = fmt.Sprintf("panic [%s:%d]: %v", file, line, x) } else { errMsg = fmt.Sprintf("panic: %v", x) } err = &errors.PanicError{ ErrMsg: errMsg, } } }() return p.handler.ProcessTask(ctx, task) } // uniq dedupes elements and returns a slice of unique names of length l. // Order of the output slice is based on the input list. func uniq(names []string, l int) []string { var res []string seen := make(map[string]struct{}) for _, s := range names { if _, ok := seen[s]; !ok { seen[s] = struct{}{} res = append(res, s) } if len(res) == l { break } } return res } // sortByPriority returns a list of queue names sorted by // their priority level in descending order. func sortByPriority(qcfg map[string]int) []string { var queues []*queue for qname, n := range qcfg { queues = append(queues, &queue{qname, n}) } sort.Sort(sort.Reverse(byPriority(queues))) var res []string for _, q := range queues { res = append(res, q.name) } return res } type queue struct { name string priority int } type byPriority []*queue func (x byPriority) Len() int { return len(x) } func (x byPriority) Less(i, j int) bool { return x[i].priority < x[j].priority } func (x byPriority) Swap(i, j int) { x[i], x[j] = x[j], x[i] } // normalizeQueues divides priority numbers by their greatest common divisor. func normalizeQueues(queues map[string]int) map[string]int { var xs []int for _, x := range queues { xs = append(xs, x) } d := gcd(xs...) res := make(map[string]int) for q, x := range queues { res[q] = x / d } return res } func gcd(xs ...int) int { fn := func(x, y int) int { for y > 0 { x, y = y, x%y } return x } res := xs[0] for i := 0; i < len(xs); i++ { res = fn(xs[i], res) if res == 1 { return 1 } } return res } // computeDeadline returns the given task's deadline, func (p *processor) computeDeadline(msg *base.TaskMessage) time.Time { if msg.Timeout == 0 && msg.Deadline == 0 { p.logger.Errorf("asynq: internal error: both timeout and deadline are not set for the task message: %s", msg.ID) return p.clock.Now().Add(defaultTimeout) } if msg.Timeout != 0 && msg.Deadline != 0 { deadlineUnix := math.Min(float64(p.clock.Now().Unix()+msg.Timeout), float64(msg.Deadline)) return time.Unix(int64(deadlineUnix), 0) } if msg.Timeout != 0 { return p.clock.Now().Add(time.Duration(msg.Timeout) * time.Second) } return time.Unix(msg.Deadline, 0) } func IsPanicError(err error) bool { return errors.IsPanicError(err) } ================================================ FILE: processor_test.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "context" "encoding/json" "fmt" "sort" "strings" "sync" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/errors" "github.com/hibiken/asynq/internal/log" "github.com/hibiken/asynq/internal/rdb" h "github.com/hibiken/asynq/internal/testutil" "github.com/hibiken/asynq/internal/timeutil" ) var taskCmpOpts = []cmp.Option{ sortTaskOpt, // sort the tasks cmp.AllowUnexported(Task{}), // allow typename, payload fields to be compared cmpopts.IgnoreFields(Task{}, "opts", "w"), // ignore opts, w fields } // fakeHeartbeater receives from starting and finished channels and do nothing. func fakeHeartbeater(starting <-chan *workerInfo, finished <-chan *base.TaskMessage, done <-chan struct{}) { for { select { case <-starting: case <-finished: case <-done: return } } } // fakeSyncer receives from sync channel and do nothing. func fakeSyncer(syncCh <-chan *syncRequest, done <-chan struct{}) { for { select { case <-syncCh: case <-done: return } } } // Returns a processor instance configured for testing purpose. func newProcessorForTest(t *testing.T, r *rdb.RDB, h Handler) *processor { starting := make(chan *workerInfo) finished := make(chan *base.TaskMessage) syncCh := make(chan *syncRequest) done := make(chan struct{}) t.Cleanup(func() { close(done) }) go fakeHeartbeater(starting, finished, done) go fakeSyncer(syncCh, done) p := newProcessor(processorParams{ logger: testLogger, broker: r, baseCtxFn: context.Background, retryDelayFunc: DefaultRetryDelayFunc, taskCheckInterval: defaultTaskCheckInterval, isFailureFunc: defaultIsFailureFunc, syncCh: syncCh, cancelations: base.NewCancelations(), concurrency: 10, queues: defaultQueueConfig, strictPriority: false, errHandler: nil, shutdownTimeout: defaultShutdownTimeout, starting: starting, finished: finished, }) p.handler = h return p } func TestProcessorSuccessWithSingleQueue(t *testing.T) { r := setup(t) defer r.Close() rdbClient := rdb.NewRDB(r) m1 := h.NewTaskMessage("task1", nil) m2 := h.NewTaskMessage("task2", nil) m3 := h.NewTaskMessage("task3", nil) m4 := h.NewTaskMessage("task4", nil) t1 := NewTask(m1.Type, m1.Payload) t2 := NewTask(m2.Type, m2.Payload) t3 := NewTask(m3.Type, m3.Payload) t4 := NewTask(m4.Type, m4.Payload) tests := []struct { pending []*base.TaskMessage // initial default queue state incoming []*base.TaskMessage // tasks to be enqueued during run wantProcessed []*Task // tasks to be processed at the end }{ { pending: []*base.TaskMessage{m1}, incoming: []*base.TaskMessage{m2, m3, m4}, wantProcessed: []*Task{t1, t2, t3, t4}, }, { pending: []*base.TaskMessage{}, incoming: []*base.TaskMessage{m1}, wantProcessed: []*Task{t1}, }, } for _, tc := range tests { h.FlushDB(t, r) // clean up db before each test case. h.SeedPendingQueue(t, r, tc.pending, base.DefaultQueueName) // initialize default queue. // instantiate a new processor var mu sync.Mutex var processed []*Task handler := func(ctx context.Context, task *Task) error { mu.Lock() defer mu.Unlock() processed = append(processed, task) return nil } p := newProcessorForTest(t, rdbClient, HandlerFunc(handler)) p.start(&sync.WaitGroup{}) for _, msg := range tc.incoming { err := rdbClient.Enqueue(context.Background(), msg) if err != nil { p.shutdown() t.Fatal(err) } } time.Sleep(2 * time.Second) // wait for two second to allow all pending tasks to be processed. if l := r.LLen(context.Background(), base.ActiveKey(base.DefaultQueueName)).Val(); l != 0 { t.Errorf("%q has %d tasks, want 0", base.ActiveKey(base.DefaultQueueName), l) } p.shutdown() mu.Lock() if diff := cmp.Diff(tc.wantProcessed, processed, taskCmpOpts...); diff != "" { t.Errorf("mismatch found in processed tasks; (-want, +got)\n%s", diff) } mu.Unlock() } } func TestProcessorSuccessWithMultipleQueues(t *testing.T) { var ( r = setup(t) rdbClient = rdb.NewRDB(r) m1 = h.NewTaskMessage("task1", nil) m2 = h.NewTaskMessage("task2", nil) m3 = h.NewTaskMessageWithQueue("task3", nil, "high") m4 = h.NewTaskMessageWithQueue("task4", nil, "low") t1 = NewTask(m1.Type, m1.Payload) t2 = NewTask(m2.Type, m2.Payload) t3 = NewTask(m3.Type, m3.Payload) t4 = NewTask(m4.Type, m4.Payload) ) defer r.Close() tests := []struct { pending map[string][]*base.TaskMessage queues []string // list of queues to consume the tasks from wantProcessed []*Task // tasks to be processed at the end }{ { pending: map[string][]*base.TaskMessage{ "default": {m1, m2}, "high": {m3}, "low": {m4}, }, queues: []string{"default", "high", "low"}, wantProcessed: []*Task{t1, t2, t3, t4}, }, } for _, tc := range tests { // Set up test case. h.FlushDB(t, r) h.SeedAllPendingQueues(t, r, tc.pending) // Instantiate a new processor. var mu sync.Mutex var processed []*Task handler := func(ctx context.Context, task *Task) error { mu.Lock() defer mu.Unlock() processed = append(processed, task) return nil } p := newProcessorForTest(t, rdbClient, HandlerFunc(handler)) p.queueConfig = map[string]int{ "default": 2, "high": 3, "low": 1, } p.start(&sync.WaitGroup{}) // Wait for two second to allow all pending tasks to be processed. time.Sleep(2 * time.Second) // Make sure no messages are stuck in active list. for _, qname := range tc.queues { if l := r.LLen(context.Background(), base.ActiveKey(qname)).Val(); l != 0 { t.Errorf("%q has %d tasks, want 0", base.ActiveKey(qname), l) } } p.shutdown() mu.Lock() if diff := cmp.Diff(tc.wantProcessed, processed, taskCmpOpts...); diff != "" { t.Errorf("mismatch found in processed tasks; (-want, +got)\n%s", diff) } mu.Unlock() } } // https://github.com/hibiken/asynq/issues/166 func TestProcessTasksWithLargeNumberInPayload(t *testing.T) { r := setup(t) defer r.Close() rdbClient := rdb.NewRDB(r) m1 := h.NewTaskMessage("large_number", h.JSON(map[string]interface{}{"data": 111111111111111111})) t1 := NewTask(m1.Type, m1.Payload) tests := []struct { pending []*base.TaskMessage // initial default queue state wantProcessed []*Task // tasks to be processed at the end }{ { pending: []*base.TaskMessage{m1}, wantProcessed: []*Task{t1}, }, } for _, tc := range tests { h.FlushDB(t, r) // clean up db before each test case. h.SeedPendingQueue(t, r, tc.pending, base.DefaultQueueName) // initialize default queue. var mu sync.Mutex var processed []*Task handler := func(ctx context.Context, task *Task) error { mu.Lock() defer mu.Unlock() var payload map[string]int if err := json.Unmarshal(task.Payload(), &payload); err != nil { t.Errorf("coult not decode payload: %v", err) } if data, ok := payload["data"]; ok { t.Logf("data == %d", data) } else { t.Errorf("could not get data from payload") } processed = append(processed, task) return nil } p := newProcessorForTest(t, rdbClient, HandlerFunc(handler)) p.start(&sync.WaitGroup{}) time.Sleep(2 * time.Second) // wait for two second to allow all pending tasks to be processed. if l := r.LLen(context.Background(), base.ActiveKey(base.DefaultQueueName)).Val(); l != 0 { t.Errorf("%q has %d tasks, want 0", base.ActiveKey(base.DefaultQueueName), l) } p.shutdown() mu.Lock() if diff := cmp.Diff(tc.wantProcessed, processed, taskCmpOpts...); diff != "" { t.Errorf("mismatch found in processed tasks; (-want, +got)\n%s", diff) } mu.Unlock() } } func TestProcessorRetry(t *testing.T) { r := setup(t) defer r.Close() rdbClient := rdb.NewRDB(r) m1 := h.NewTaskMessage("send_email", nil) m1.Retried = m1.Retry // m1 has reached its max retry count m2 := h.NewTaskMessage("gen_thumbnail", nil) m3 := h.NewTaskMessage("reindex", nil) m4 := h.NewTaskMessage("sync", nil) errMsg := "something went wrong" wrappedSkipRetry := fmt.Errorf("%s:%w", errMsg, SkipRetry) wrappedRevokeTask := fmt.Errorf("%s:%w", errMsg, RevokeTask) tests := []struct { desc string // test description pending []*base.TaskMessage // initial default queue state delay time.Duration // retry delay duration handler Handler // task handler wait time.Duration // wait duration between starting and stopping processor for this test case wantErrMsg string // error message the task should record wantRetry []*base.TaskMessage // tasks in retry queue at the end wantArchived []*base.TaskMessage // tasks in archived queue at the end wantErrCount int // number of times error handler should be called }{ { desc: "Should automatically retry errored tasks", pending: []*base.TaskMessage{m1, m2, m3, m4}, delay: time.Minute, handler: HandlerFunc(func(ctx context.Context, task *Task) error { return errors.New(errMsg) }), wait: 2 * time.Second, wantErrMsg: errMsg, wantRetry: []*base.TaskMessage{m2, m3, m4}, wantArchived: []*base.TaskMessage{m1}, wantErrCount: 4, }, { desc: "Should skip retry errored tasks", pending: []*base.TaskMessage{m1, m2}, delay: time.Minute, handler: HandlerFunc(func(ctx context.Context, task *Task) error { return SkipRetry // return SkipRetry without wrapping }), wait: 2 * time.Second, wantErrMsg: SkipRetry.Error(), wantRetry: []*base.TaskMessage{}, wantArchived: []*base.TaskMessage{m1, m2}, wantErrCount: 2, // ErrorHandler should still be called with SkipRetry error }, { desc: "Should skip retry errored tasks (with error wrapping)", pending: []*base.TaskMessage{m1, m2}, delay: time.Minute, handler: HandlerFunc(func(ctx context.Context, task *Task) error { return wrappedSkipRetry }), wait: 2 * time.Second, wantErrMsg: wrappedSkipRetry.Error(), wantRetry: []*base.TaskMessage{}, wantArchived: []*base.TaskMessage{m1, m2}, wantErrCount: 2, // ErrorHandler should still be called with SkipRetry error }, { desc: "Should revoke task", pending: []*base.TaskMessage{m1, m2}, delay: time.Minute, handler: HandlerFunc(func(ctx context.Context, task *Task) error { return RevokeTask // return RevokeTask without wrapping }), wait: 2 * time.Second, wantErrMsg: RevokeTask.Error(), wantRetry: []*base.TaskMessage{}, wantArchived: []*base.TaskMessage{}, wantErrCount: 2, // ErrorHandler should still be called with RevokeTask error }, { desc: "Should revoke task (with error wrapping)", pending: []*base.TaskMessage{m1, m2}, delay: time.Minute, handler: HandlerFunc(func(ctx context.Context, task *Task) error { return wrappedRevokeTask }), wait: 2 * time.Second, wantErrMsg: wrappedRevokeTask.Error(), wantRetry: []*base.TaskMessage{}, wantArchived: []*base.TaskMessage{}, wantErrCount: 2, // ErrorHandler should still be called with RevokeTask error }, } for _, tc := range tests { h.FlushDB(t, r) // clean up db before each test case. h.SeedPendingQueue(t, r, tc.pending, base.DefaultQueueName) // initialize default queue. // instantiate a new processor delayFunc := func(n int, e error, t *Task) time.Duration { return tc.delay } var ( mu sync.Mutex // guards n n int // number of times error handler is called ) errHandler := func(ctx context.Context, t *Task, err error) { mu.Lock() defer mu.Unlock() n++ } p := newProcessorForTest(t, rdbClient, tc.handler) p.errHandler = ErrorHandlerFunc(errHandler) p.retryDelayFunc = delayFunc p.start(&sync.WaitGroup{}) runTime := time.Now() // time when processor is running time.Sleep(tc.wait) // FIXME: This makes test flaky. p.shutdown() cmpOpt := h.EquateInt64Approx(int64(tc.wait.Seconds())) // allow up to a wait-second difference in zset score gotRetry := h.GetRetryEntries(t, r, base.DefaultQueueName) var wantRetry []base.Z // Note: construct wantRetry here since `LastFailedAt` and ZSCORE is relative to each test run. for _, msg := range tc.wantRetry { wantRetry = append(wantRetry, base.Z{ Message: h.TaskMessageAfterRetry(*msg, tc.wantErrMsg, runTime), Score: runTime.Add(tc.delay).Unix(), }) } if diff := cmp.Diff(wantRetry, gotRetry, h.SortZSetEntryOpt, cmpOpt); diff != "" { t.Errorf("%s: mismatch found in %q after running processor; (-want, +got)\n%s", tc.desc, base.RetryKey(base.DefaultQueueName), diff) } gotArchived := h.GetArchivedEntries(t, r, base.DefaultQueueName) var wantArchived []base.Z // Note: construct wantArchived here since `LastFailedAt` and ZSCORE is relative to each test run. for _, msg := range tc.wantArchived { wantArchived = append(wantArchived, base.Z{ Message: h.TaskMessageWithError(*msg, tc.wantErrMsg, runTime), Score: runTime.Unix(), }) } if diff := cmp.Diff(wantArchived, gotArchived, h.SortZSetEntryOpt, cmpOpt); diff != "" { t.Errorf("%s: mismatch found in %q after running processor; (-want, +got)\n%s", tc.desc, base.ArchivedKey(base.DefaultQueueName), diff) } if l := r.LLen(context.Background(), base.ActiveKey(base.DefaultQueueName)).Val(); l != 0 { t.Errorf("%s: %q has %d tasks, want 0", base.ActiveKey(base.DefaultQueueName), tc.desc, l) } if n != tc.wantErrCount { t.Errorf("error handler was called %d times, want %d", n, tc.wantErrCount) } } } func TestProcessorMarkAsComplete(t *testing.T) { r := setup(t) defer r.Close() rdbClient := rdb.NewRDB(r) msg1 := h.NewTaskMessage("one", nil) msg2 := h.NewTaskMessage("two", nil) msg3 := h.NewTaskMessageWithQueue("three", nil, "custom") msg1.Retention = 3600 msg3.Retention = 7200 handler := func(ctx context.Context, task *Task) error { return nil } tests := []struct { pending map[string][]*base.TaskMessage completed map[string][]base.Z queueCfg map[string]int wantPending map[string][]*base.TaskMessage wantCompleted func(completedAt time.Time) map[string][]base.Z }{ { pending: map[string][]*base.TaskMessage{ "default": {msg1, msg2}, "custom": {msg3}, }, completed: map[string][]base.Z{ "default": {}, "custom": {}, }, queueCfg: map[string]int{ "default": 1, "custom": 1, }, wantPending: map[string][]*base.TaskMessage{ "default": {}, "custom": {}, }, wantCompleted: func(completedAt time.Time) map[string][]base.Z { return map[string][]base.Z{ "default": {{Message: h.TaskMessageWithCompletedAt(*msg1, completedAt), Score: completedAt.Unix() + msg1.Retention}}, "custom": {{Message: h.TaskMessageWithCompletedAt(*msg3, completedAt), Score: completedAt.Unix() + msg3.Retention}}, } }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllPendingQueues(t, r, tc.pending) h.SeedAllCompletedQueues(t, r, tc.completed) p := newProcessorForTest(t, rdbClient, HandlerFunc(handler)) p.queueConfig = tc.queueCfg p.start(&sync.WaitGroup{}) runTime := time.Now() // time when processor is running time.Sleep(2 * time.Second) p.shutdown() for qname, want := range tc.wantPending { gotPending := h.GetPendingMessages(t, r, qname) if diff := cmp.Diff(want, gotPending, cmpopts.EquateEmpty()); diff != "" { t.Errorf("diff found in %q pending set; want=%v, got=%v\n%s", qname, want, gotPending, diff) } } for qname, want := range tc.wantCompleted(runTime) { gotCompleted := h.GetCompletedEntries(t, r, qname) if diff := cmp.Diff(want, gotCompleted, cmpopts.EquateEmpty()); diff != "" { t.Errorf("diff found in %q completed set; want=%v, got=%v\n%s", qname, want, gotCompleted, diff) } } } } // Test a scenario where the worker server cannot communicate with redis due to a network failure // and the lease expires func TestProcessorWithExpiredLease(t *testing.T) { r := setup(t) defer r.Close() rdbClient := rdb.NewRDB(r) m1 := h.NewTaskMessage("task1", nil) tests := []struct { pending []*base.TaskMessage handler Handler wantErrCount int }{ { pending: []*base.TaskMessage{m1}, handler: HandlerFunc(func(ctx context.Context, task *Task) error { // make sure the task processing time exceeds lease duration // to test expired lease. time.Sleep(rdb.LeaseDuration + 10*time.Second) return nil }), wantErrCount: 1, // ErrorHandler should still be called with ErrLeaseExpired }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedPendingQueue(t, r, tc.pending, base.DefaultQueueName) starting := make(chan *workerInfo) finished := make(chan *base.TaskMessage) syncCh := make(chan *syncRequest) done := make(chan struct{}) t.Cleanup(func() { close(done) }) // fake heartbeater which notifies lease expiration go func() { for { select { case w := <-starting: // simulate expiration by resetting to some time in the past w.lease.Reset(time.Now().Add(-5 * time.Second)) if !w.lease.NotifyExpiration() { panic("Failed to notifiy lease expiration") } case <-finished: // do nothing case <-done: return } } }() go fakeSyncer(syncCh, done) p := newProcessor(processorParams{ logger: testLogger, broker: rdbClient, baseCtxFn: context.Background, taskCheckInterval: defaultTaskCheckInterval, retryDelayFunc: DefaultRetryDelayFunc, isFailureFunc: defaultIsFailureFunc, syncCh: syncCh, cancelations: base.NewCancelations(), concurrency: 10, queues: defaultQueueConfig, strictPriority: false, errHandler: nil, shutdownTimeout: defaultShutdownTimeout, starting: starting, finished: finished, }) p.handler = tc.handler var ( mu sync.Mutex // guards n and errs n int // number of times error handler is called errs []error // error passed to error handler ) p.errHandler = ErrorHandlerFunc(func(ctx context.Context, t *Task, err error) { mu.Lock() defer mu.Unlock() n++ errs = append(errs, err) }) p.start(&sync.WaitGroup{}) time.Sleep(4 * time.Second) p.shutdown() if n != tc.wantErrCount { t.Errorf("Unexpected number of error count: got %d, want %d", n, tc.wantErrCount) continue } for i := 0; i < tc.wantErrCount; i++ { if !errors.Is(errs[i], ErrLeaseExpired) { t.Errorf("Unexpected error was passed to ErrorHandler: got %v want %v", errs[i], ErrLeaseExpired) } } } } func TestProcessorQueues(t *testing.T) { sortOpt := cmp.Transformer("SortStrings", func(in []string) []string { out := append([]string(nil), in...) // Copy input to avoid mutating it sort.Strings(out) return out }) tests := []struct { queueCfg map[string]int want []string }{ { queueCfg: map[string]int{ "high": 6, "default": 3, "low": 1, }, want: []string{"high", "default", "low"}, }, { queueCfg: map[string]int{ "default": 1, }, want: []string{"default"}, }, } for _, tc := range tests { // Note: rdb and handler not needed for this test. p := newProcessorForTest(t, nil, nil) p.queueConfig = tc.queueCfg got := p.queues() if diff := cmp.Diff(tc.want, got, sortOpt); diff != "" { t.Errorf("with queue config: %v\n(*processor).queues() = %v, want %v\n(-want,+got):\n%s", tc.queueCfg, got, tc.want, diff) } } } func TestProcessorWithStrictPriority(t *testing.T) { var ( r = setup(t) rdbClient = rdb.NewRDB(r) m1 = h.NewTaskMessageWithQueue("task1", nil, "critical") m2 = h.NewTaskMessageWithQueue("task2", nil, "critical") m3 = h.NewTaskMessageWithQueue("task3", nil, "critical") m4 = h.NewTaskMessageWithQueue("task4", nil, base.DefaultQueueName) m5 = h.NewTaskMessageWithQueue("task5", nil, base.DefaultQueueName) m6 = h.NewTaskMessageWithQueue("task6", nil, "low") m7 = h.NewTaskMessageWithQueue("task7", nil, "low") t1 = NewTask(m1.Type, m1.Payload) t2 = NewTask(m2.Type, m2.Payload) t3 = NewTask(m3.Type, m3.Payload) t4 = NewTask(m4.Type, m4.Payload) t5 = NewTask(m5.Type, m5.Payload) t6 = NewTask(m6.Type, m6.Payload) t7 = NewTask(m7.Type, m7.Payload) ) defer r.Close() tests := []struct { pending map[string][]*base.TaskMessage // initial queues state queues []string // list of queues to consume tasks from wait time.Duration // wait duration between starting and stopping processor for this test case wantProcessed []*Task // tasks to be processed at the end }{ { pending: map[string][]*base.TaskMessage{ base.DefaultQueueName: {m4, m5}, "critical": {m1, m2, m3}, "low": {m6, m7}, }, queues: []string{base.DefaultQueueName, "critical", "low"}, wait: time.Second, wantProcessed: []*Task{t1, t2, t3, t4, t5, t6, t7}, }, } for _, tc := range tests { h.FlushDB(t, r) // clean up db before each test case. for qname, msgs := range tc.pending { h.SeedPendingQueue(t, r, msgs, qname) } // instantiate a new processor var mu sync.Mutex var processed []*Task handler := func(ctx context.Context, task *Task) error { mu.Lock() defer mu.Unlock() processed = append(processed, task) return nil } queueCfg := map[string]int{ base.DefaultQueueName: 2, "critical": 3, "low": 1, } starting := make(chan *workerInfo) finished := make(chan *base.TaskMessage) syncCh := make(chan *syncRequest) done := make(chan struct{}) defer func() { close(done) }() go fakeHeartbeater(starting, finished, done) go fakeSyncer(syncCh, done) p := newProcessor(processorParams{ logger: testLogger, broker: rdbClient, baseCtxFn: context.Background, taskCheckInterval: defaultTaskCheckInterval, retryDelayFunc: DefaultRetryDelayFunc, isFailureFunc: defaultIsFailureFunc, syncCh: syncCh, cancelations: base.NewCancelations(), concurrency: 1, // Set concurrency to 1 to make sure tasks are processed one at a time. queues: queueCfg, strictPriority: true, errHandler: nil, shutdownTimeout: defaultShutdownTimeout, starting: starting, finished: finished, }) p.handler = HandlerFunc(handler) p.start(&sync.WaitGroup{}) time.Sleep(tc.wait) // Make sure no tasks are stuck in active list. for _, qname := range tc.queues { if l := r.LLen(context.Background(), base.ActiveKey(qname)).Val(); l != 0 { t.Errorf("%q has %d tasks, want 0", base.ActiveKey(qname), l) } } p.shutdown() if diff := cmp.Diff(tc.wantProcessed, processed, taskCmpOpts...); diff != "" { t.Errorf("mismatch found in processed tasks; (-want, +got)\n%s", diff) } } } func TestProcessorPerform(t *testing.T) { tests := []struct { desc string handler HandlerFunc task *Task wantErr bool }{ { desc: "handler returns nil", handler: func(ctx context.Context, t *Task) error { return nil }, task: NewTask("gen_thumbnail", h.JSON(map[string]interface{}{"src": "some/img/path"})), wantErr: false, }, { desc: "handler returns error", handler: func(ctx context.Context, t *Task) error { return fmt.Errorf("something went wrong") }, task: NewTask("gen_thumbnail", h.JSON(map[string]interface{}{"src": "some/img/path"})), wantErr: true, }, { desc: "handler panics", handler: func(ctx context.Context, t *Task) error { panic("something went terribly wrong") }, task: NewTask("gen_thumbnail", h.JSON(map[string]interface{}{"src": "some/img/path"})), wantErr: true, }, } // Note: We don't need to fully initialized the processor since we are only testing // perform method. p := newProcessorForTest(t, nil, nil) for _, tc := range tests { p.handler = tc.handler got := p.perform(context.Background(), tc.task) if !tc.wantErr && got != nil { t.Errorf("%s: perform() = %v, want nil", tc.desc, got) continue } if tc.wantErr && got == nil { t.Errorf("%s: perform() = nil, want non-nil error", tc.desc) continue } } } func TestGCD(t *testing.T) { tests := []struct { input []int want int }{ {[]int{6, 2, 12}, 2}, {[]int{3, 3, 3}, 3}, {[]int{6, 3, 1}, 1}, {[]int{1}, 1}, {[]int{1, 0, 2}, 1}, {[]int{8, 0, 4}, 4}, {[]int{9, 12, 18, 30}, 3}, } for _, tc := range tests { got := gcd(tc.input...) if got != tc.want { t.Errorf("gcd(%v) = %d, want %d", tc.input, got, tc.want) } } } func TestNormalizeQueues(t *testing.T) { tests := []struct { input map[string]int want map[string]int }{ { input: map[string]int{ "high": 100, "default": 20, "low": 5, }, want: map[string]int{ "high": 20, "default": 4, "low": 1, }, }, { input: map[string]int{ "default": 10, }, want: map[string]int{ "default": 1, }, }, { input: map[string]int{ "critical": 5, "default": 1, }, want: map[string]int{ "critical": 5, "default": 1, }, }, { input: map[string]int{ "critical": 6, "default": 3, "low": 0, }, want: map[string]int{ "critical": 2, "default": 1, "low": 0, }, }, } for _, tc := range tests { got := normalizeQueues(tc.input) if diff := cmp.Diff(tc.want, got); diff != "" { t.Errorf("normalizeQueues(%v) = %v, want %v; (-want, +got):\n%s", tc.input, got, tc.want, diff) } } } func TestProcessorComputeDeadline(t *testing.T) { now := time.Now() p := processor{ logger: log.NewLogger(nil), clock: timeutil.NewSimulatedClock(now), } tests := []struct { desc string msg *base.TaskMessage want time.Time }{ { desc: "message with only timeout specified", msg: &base.TaskMessage{ Timeout: int64((30 * time.Minute).Seconds()), }, want: now.Add(30 * time.Minute), }, { desc: "message with only deadline specified", msg: &base.TaskMessage{ Deadline: now.Add(24 * time.Hour).Unix(), }, want: now.Add(24 * time.Hour), }, { desc: "message with both timeout and deadline set (now+timeout < deadline)", msg: &base.TaskMessage{ Deadline: now.Add(24 * time.Hour).Unix(), Timeout: int64((30 * time.Minute).Seconds()), }, want: now.Add(30 * time.Minute), }, { desc: "message with both timeout and deadline set (now+timeout > deadline)", msg: &base.TaskMessage{ Deadline: now.Add(10 * time.Minute).Unix(), Timeout: int64((30 * time.Minute).Seconds()), }, want: now.Add(10 * time.Minute), }, { desc: "message with both timeout and deadline set (now+timeout == deadline)", msg: &base.TaskMessage{ Deadline: now.Add(30 * time.Minute).Unix(), Timeout: int64((30 * time.Minute).Seconds()), }, want: now.Add(30 * time.Minute), }, { desc: "message without timeout and deadline", msg: &base.TaskMessage{}, want: now.Add(defaultTimeout), }, } for _, tc := range tests { got := p.computeDeadline(tc.msg) // Compare the Unix epoch with seconds granularity if got.Unix() != tc.want.Unix() { t.Errorf("%s: got=%v, want=%v", tc.desc, got.Unix(), tc.want.Unix()) } } } func TestReturnPanicError(t *testing.T) { task := NewTask("gen_thumbnail", h.JSON(map[string]interface{}{"src": "some/img/path"})) tests := []struct { name string handler HandlerFunc IsPanicError bool }{ { name: "should return panic error when occurred panic recovery", handler: func(ctx context.Context, t *Task) error { panic("something went terribly wrong") }, IsPanicError: true, }, { name: "should return normal error when don't occur panic recovery", handler: func(ctx context.Context, t *Task) error { return fmt.Errorf("something went terribly wrong") }, IsPanicError: false, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { p := processor{ logger: log.NewLogger(nil), handler: tc.handler, } got := p.perform(context.Background(), task) if tc.IsPanicError != IsPanicError(got) { t.Errorf("%s: got=%t, want=%t", tc.name, IsPanicError(got), tc.IsPanicError) } if tc.IsPanicError && !strings.HasPrefix(got.Error(), "panic error cause by:") { t.Error("wrong text msg for panic error") } }) } } ================================================ FILE: recoverer.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "context" "sync" "time" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/errors" "github.com/hibiken/asynq/internal/log" ) type recoverer struct { logger *log.Logger broker base.Broker retryDelayFunc RetryDelayFunc isFailureFunc func(error) bool // channel to communicate back to the long running "recoverer" goroutine. done chan struct{} // list of queues to check for deadline. queues []string // poll interval. interval time.Duration } type recovererParams struct { logger *log.Logger broker base.Broker queues []string interval time.Duration retryDelayFunc RetryDelayFunc isFailureFunc func(error) bool } func newRecoverer(params recovererParams) *recoverer { return &recoverer{ logger: params.logger, broker: params.broker, done: make(chan struct{}), queues: params.queues, interval: params.interval, retryDelayFunc: params.retryDelayFunc, isFailureFunc: params.isFailureFunc, } } func (r *recoverer) shutdown() { r.logger.Debug("Recoverer shutting down...") // Signal the recoverer goroutine to stop polling. r.done <- struct{}{} } func (r *recoverer) start(wg *sync.WaitGroup) { wg.Add(1) go func() { defer wg.Done() r.recover() timer := time.NewTimer(r.interval) for { select { case <-r.done: r.logger.Debug("Recoverer done") timer.Stop() return case <-timer.C: r.recover() timer.Reset(r.interval) } } }() } // ErrLeaseExpired error indicates that the task failed because the worker working on the task // could not extend its lease due to missing heartbeats. The worker may have crashed or got cutoff from the network. var ErrLeaseExpired = errors.New("asynq: task lease expired") func (r *recoverer) recover() { r.recoverLeaseExpiredTasks() r.recoverStaleAggregationSets() } func (r *recoverer) recoverLeaseExpiredTasks() { // Get all tasks which have expired 30 seconds ago or earlier to accommodate certain amount of clock skew. cutoff := time.Now().Add(-30 * time.Second) msgs, err := r.broker.ListLeaseExpired(cutoff, r.queues...) if err != nil { r.logger.Warnf("recoverer: could not list lease expired tasks: %v", err) return } for _, msg := range msgs { if msg.Retried >= msg.Retry { r.archive(msg, ErrLeaseExpired) } else { r.retry(msg, ErrLeaseExpired) } } } func (r *recoverer) recoverStaleAggregationSets() { for _, qname := range r.queues { if err := r.broker.ReclaimStaleAggregationSets(qname); err != nil { r.logger.Warnf("recoverer: could not reclaim stale aggregation sets in queue %q: %v", qname, err) } } } func (r *recoverer) retry(msg *base.TaskMessage, err error) { delay := r.retryDelayFunc(msg.Retried, err, NewTaskWithHeaders(msg.Type, msg.Payload, msg.Headers)) retryAt := time.Now().Add(delay) if err := r.broker.Retry(context.Background(), msg, retryAt, err.Error(), r.isFailureFunc(err)); err != nil { r.logger.Warnf("recoverer: could not retry lease expired task: %v", err) } } func (r *recoverer) archive(msg *base.TaskMessage, err error) { if err := r.broker.Archive(context.Background(), msg, err.Error()); err != nil { r.logger.Warnf("recoverer: could not move task to archive: %v", err) } } ================================================ FILE: recoverer_test.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "sync" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/rdb" h "github.com/hibiken/asynq/internal/testutil" ) func TestRecoverer(t *testing.T) { r := setup(t) defer r.Close() rdbClient := rdb.NewRDB(r) t1 := h.NewTaskMessageWithQueue("task1", nil, "default") t2 := h.NewTaskMessageWithQueue("task2", nil, "default") t3 := h.NewTaskMessageWithQueue("task3", nil, "critical") t4 := h.NewTaskMessageWithQueue("task4", nil, "default") t4.Retried = t4.Retry // t4 has reached its max retry count now := time.Now() tests := []struct { desc string active map[string][]*base.TaskMessage lease map[string][]base.Z retry map[string][]base.Z archived map[string][]base.Z wantActive map[string][]*base.TaskMessage wantLease map[string][]base.Z wantRetry map[string][]*base.TaskMessage wantArchived map[string][]*base.TaskMessage }{ { desc: "with one active task", active: map[string][]*base.TaskMessage{ "default": {t1}, }, lease: map[string][]base.Z{ "default": {{Message: t1, Score: now.Add(-1 * time.Minute).Unix()}}, }, retry: map[string][]base.Z{ "default": {}, }, archived: map[string][]base.Z{ "default": {}, }, wantActive: map[string][]*base.TaskMessage{ "default": {}, }, wantLease: map[string][]base.Z{ "default": {}, }, wantRetry: map[string][]*base.TaskMessage{ "default": {t1}, }, wantArchived: map[string][]*base.TaskMessage{ "default": {}, }, }, { desc: "with a task with max-retry reached", active: map[string][]*base.TaskMessage{ "default": {t4}, "critical": {}, }, lease: map[string][]base.Z{ "default": {{Message: t4, Score: now.Add(-40 * time.Second).Unix()}}, "critical": {}, }, retry: map[string][]base.Z{ "default": {}, "critical": {}, }, archived: map[string][]base.Z{ "default": {}, "critical": {}, }, wantActive: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, }, wantLease: map[string][]base.Z{ "default": {}, "critical": {}, }, wantRetry: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, }, wantArchived: map[string][]*base.TaskMessage{ "default": {t4}, "critical": {}, }, }, { desc: "with multiple active tasks, and one expired", active: map[string][]*base.TaskMessage{ "default": {t1, t2}, "critical": {t3}, }, lease: map[string][]base.Z{ "default": { {Message: t1, Score: now.Add(-2 * time.Minute).Unix()}, {Message: t2, Score: now.Add(20 * time.Second).Unix()}, }, "critical": { {Message: t3, Score: now.Add(20 * time.Second).Unix()}, }, }, retry: map[string][]base.Z{ "default": {}, "critical": {}, }, archived: map[string][]base.Z{ "default": {}, "critical": {}, }, wantActive: map[string][]*base.TaskMessage{ "default": {t2}, "critical": {t3}, }, wantLease: map[string][]base.Z{ "default": {{Message: t2, Score: now.Add(20 * time.Second).Unix()}}, "critical": {{Message: t3, Score: now.Add(20 * time.Second).Unix()}}, }, wantRetry: map[string][]*base.TaskMessage{ "default": {t1}, "critical": {}, }, wantArchived: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, }, }, { desc: "with multiple expired active tasks", active: map[string][]*base.TaskMessage{ "default": {t1, t2}, "critical": {t3}, }, lease: map[string][]base.Z{ "default": { {Message: t1, Score: now.Add(-1 * time.Minute).Unix()}, {Message: t2, Score: now.Add(10 * time.Second).Unix()}, }, "critical": { {Message: t3, Score: now.Add(-1 * time.Minute).Unix()}, }, }, retry: map[string][]base.Z{ "default": {}, "cricial": {}, }, archived: map[string][]base.Z{ "default": {}, "cricial": {}, }, wantActive: map[string][]*base.TaskMessage{ "default": {t2}, "critical": {}, }, wantLease: map[string][]base.Z{ "default": {{Message: t2, Score: now.Add(10 * time.Second).Unix()}}, }, wantRetry: map[string][]*base.TaskMessage{ "default": {t1}, "critical": {t3}, }, wantArchived: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, }, }, { desc: "with empty active queue", active: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, }, lease: map[string][]base.Z{ "default": {}, "critical": {}, }, retry: map[string][]base.Z{ "default": {}, "critical": {}, }, archived: map[string][]base.Z{ "default": {}, "critical": {}, }, wantActive: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, }, wantLease: map[string][]base.Z{ "default": {}, "critical": {}, }, wantRetry: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, }, wantArchived: map[string][]*base.TaskMessage{ "default": {}, "critical": {}, }, }, } for _, tc := range tests { h.FlushDB(t, r) h.SeedAllActiveQueues(t, r, tc.active) h.SeedAllLease(t, r, tc.lease) h.SeedAllRetryQueues(t, r, tc.retry) h.SeedAllArchivedQueues(t, r, tc.archived) recoverer := newRecoverer(recovererParams{ logger: testLogger, broker: rdbClient, queues: []string{"default", "critical"}, interval: 1 * time.Second, retryDelayFunc: func(n int, err error, task *Task) time.Duration { return 30 * time.Second }, isFailureFunc: defaultIsFailureFunc, }) var wg sync.WaitGroup recoverer.start(&wg) runTime := time.Now() // time when recoverer is running time.Sleep(2 * time.Second) recoverer.shutdown() for qname, want := range tc.wantActive { gotActive := h.GetActiveMessages(t, r, qname) if diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != "" { t.Errorf("%s; mismatch found in %q; (-want,+got)\n%s", tc.desc, base.ActiveKey(qname), diff) } } for qname, want := range tc.wantLease { gotLease := h.GetLeaseEntries(t, r, qname) if diff := cmp.Diff(want, gotLease, h.SortZSetEntryOpt); diff != "" { t.Errorf("%s; mismatch found in %q; (-want,+got)\n%s", tc.desc, base.LeaseKey(qname), diff) } } cmpOpt := h.EquateInt64Approx(2) // allow up to two-second difference in `LastFailedAt` for qname, msgs := range tc.wantRetry { gotRetry := h.GetRetryMessages(t, r, qname) var wantRetry []*base.TaskMessage // Note: construct message here since `LastFailedAt` is relative to each test run for _, msg := range msgs { wantRetry = append(wantRetry, h.TaskMessageAfterRetry(*msg, ErrLeaseExpired.Error(), runTime)) } if diff := cmp.Diff(wantRetry, gotRetry, h.SortMsgOpt, cmpOpt); diff != "" { t.Errorf("%s; mismatch found in %q: (-want, +got)\n%s", tc.desc, base.RetryKey(qname), diff) } } for qname, msgs := range tc.wantArchived { gotArchived := h.GetArchivedMessages(t, r, qname) var wantArchived []*base.TaskMessage for _, msg := range msgs { wantArchived = append(wantArchived, h.TaskMessageWithError(*msg, ErrLeaseExpired.Error(), runTime)) } if diff := cmp.Diff(wantArchived, gotArchived, h.SortMsgOpt, cmpOpt); diff != "" { t.Errorf("%s; mismatch found in %q: (-want, +got)\n%s", tc.desc, base.ArchivedKey(qname), diff) } } } } ================================================ FILE: scheduler.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "fmt" "os" "sync" "time" "github.com/google/uuid" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/log" "github.com/hibiken/asynq/internal/rdb" "github.com/redis/go-redis/v9" "github.com/robfig/cron/v3" ) // A Scheduler kicks off tasks at regular intervals based on the user defined schedule. // // Schedulers are safe for concurrent use by multiple goroutines. type Scheduler struct { id string state *serverState heartbeatInterval time.Duration logger *log.Logger client *Client rdb *rdb.RDB cron *cron.Cron location *time.Location done chan struct{} wg sync.WaitGroup preEnqueueFunc func(task *Task, opts []Option) postEnqueueFunc func(info *TaskInfo, err error) errHandler func(task *Task, opts []Option, err error) // guards idmap mu sync.Mutex // idmap maps Scheduler's entry ID to cron.EntryID // to avoid using cron.EntryID as the public API of // the Scheduler. idmap map[string]cron.EntryID } const defaultHeartbeatInterval = 10 * time.Second // NewScheduler returns a new Scheduler instance given the redis connection option. // The parameter opts is optional, defaults will be used if opts is set to nil func NewScheduler(r RedisConnOpt, opts *SchedulerOpts) *Scheduler { scheduler := newScheduler(opts) redisClient, ok := r.MakeRedisClient().(redis.UniversalClient) if !ok { panic(fmt.Sprintf("asynq: unsupported RedisConnOpt type %T", r)) } rdb := rdb.NewRDB(redisClient) scheduler.rdb = rdb scheduler.client = &Client{broker: rdb, sharedConnection: false} return scheduler } // NewSchedulerFromRedisClient returns a new instance of Scheduler given a redis.UniversalClient // The parameter opts is optional, defaults will be used if opts is set to nil. // Warning: The underlying redis connection pool will not be closed by Asynq, you are responsible for closing it. func NewSchedulerFromRedisClient(c redis.UniversalClient, opts *SchedulerOpts) *Scheduler { scheduler := newScheduler(opts) scheduler.rdb = rdb.NewRDB(c) scheduler.client = NewClientFromRedisClient(c) return scheduler } func newScheduler(opts *SchedulerOpts) *Scheduler { if opts == nil { opts = &SchedulerOpts{} } heartbeatInterval := opts.HeartbeatInterval if heartbeatInterval <= 0 { heartbeatInterval = defaultHeartbeatInterval } logger := log.NewLogger(opts.Logger) loglevel := opts.LogLevel if loglevel == level_unspecified { loglevel = InfoLevel } logger.SetLevel(toInternalLogLevel(loglevel)) loc := opts.Location if loc == nil { loc = time.UTC } return &Scheduler{ id: generateSchedulerID(), state: &serverState{value: srvStateNew}, heartbeatInterval: heartbeatInterval, logger: logger, cron: cron.New(cron.WithLocation(loc)), location: loc, done: make(chan struct{}), preEnqueueFunc: opts.PreEnqueueFunc, postEnqueueFunc: opts.PostEnqueueFunc, errHandler: opts.EnqueueErrorHandler, idmap: make(map[string]cron.EntryID), } } func generateSchedulerID() string { host, err := os.Hostname() if err != nil { host = "unknown-host" } return fmt.Sprintf("%s:%d:%v", host, os.Getpid(), uuid.New()) } // SchedulerOpts specifies scheduler options. type SchedulerOpts struct { // HeartbeatInterval specifies the interval between scheduler heartbeats. // // If unset, zero or a negative value, the interval is set to 10 second. // // Note: Setting this value too low may add significant load to redis. // // By default, HeartbeatInterval is set to 10 seconds. HeartbeatInterval time.Duration // Logger specifies the logger used by the scheduler instance. // // If unset, the default logger is used. Logger Logger // LogLevel specifies the minimum log level to enable. // // If unset, InfoLevel is used by default. LogLevel LogLevel // Location specifies the time zone location. // // If unset, the UTC time zone (time.UTC) is used. Location *time.Location // PreEnqueueFunc, if provided, is called before a task gets enqueued by Scheduler. // The callback function should return quickly to not block the current thread. PreEnqueueFunc func(task *Task, opts []Option) // PostEnqueueFunc, if provided, is called after a task gets enqueued by Scheduler. // The callback function should return quickly to not block the current thread. PostEnqueueFunc func(info *TaskInfo, err error) // Deprecated: Use PostEnqueueFunc instead // EnqueueErrorHandler gets called when scheduler cannot enqueue a registered task // due to an error. EnqueueErrorHandler func(task *Task, opts []Option, err error) } // enqueueJob encapsulates the job of enqueuing a task and recording the event. type enqueueJob struct { id uuid.UUID cronspec string task *Task opts []Option location *time.Location logger *log.Logger client *Client rdb *rdb.RDB preEnqueueFunc func(task *Task, opts []Option) postEnqueueFunc func(info *TaskInfo, err error) errHandler func(task *Task, opts []Option, err error) } func (j *enqueueJob) Run() { if j.preEnqueueFunc != nil { j.preEnqueueFunc(j.task, j.opts) } info, err := j.client.Enqueue(j.task, j.opts...) if j.postEnqueueFunc != nil { j.postEnqueueFunc(info, err) } if err != nil { if j.errHandler != nil { j.errHandler(j.task, j.opts, err) } return } j.logger.Debugf("scheduler enqueued a task: %+v", info) event := &base.SchedulerEnqueueEvent{ TaskID: info.ID, EnqueuedAt: time.Now().In(j.location), } err = j.rdb.RecordSchedulerEnqueueEvent(j.id.String(), event) if err != nil { j.logger.Warnf("scheduler could not record enqueue event of enqueued task %s: %v", info.ID, err) } } // Register registers a task to be enqueued on the given schedule specified by the cronspec. // It returns an ID of the newly registered entry. func (s *Scheduler) Register(cronspec string, task *Task, opts ...Option) (entryID string, err error) { job := &enqueueJob{ id: uuid.New(), cronspec: cronspec, task: task, opts: opts, location: s.location, client: s.client, rdb: s.rdb, logger: s.logger, preEnqueueFunc: s.preEnqueueFunc, postEnqueueFunc: s.postEnqueueFunc, errHandler: s.errHandler, } cronID, err := s.cron.AddJob(cronspec, job) if err != nil { return "", err } s.mu.Lock() s.idmap[job.id.String()] = cronID s.mu.Unlock() return job.id.String(), nil } // Unregister removes a registered entry by entry ID. // Unregister returns a non-nil error if no entries were found for the given entryID. func (s *Scheduler) Unregister(entryID string) error { s.mu.Lock() defer s.mu.Unlock() cronID, ok := s.idmap[entryID] if !ok { return fmt.Errorf("asynq: no scheduler entry found") } delete(s.idmap, entryID) s.cron.Remove(cronID) return nil } // Run starts the scheduler until an os signal to exit the program is received. // It returns an error if scheduler is already running or has been shutdown. func (s *Scheduler) Run() error { if err := s.Start(); err != nil { return err } s.waitForSignals() s.Shutdown() return nil } // Start starts the scheduler. // It returns an error if the scheduler is already running or has been shutdown. func (s *Scheduler) Start() error { if err := s.start(); err != nil { return err } s.logger.Info("Scheduler starting") s.logger.Infof("Scheduler timezone is set to %v", s.location) s.cron.Start() s.wg.Add(1) go s.runHeartbeater() return nil } // Checks server state and returns an error if pre-condition is not met. // Otherwise it sets the server state to active. func (s *Scheduler) start() error { s.state.mu.Lock() defer s.state.mu.Unlock() switch s.state.value { case srvStateActive: return fmt.Errorf("asynq: the scheduler is already running") case srvStateClosed: return fmt.Errorf("asynq: the scheduler has already been stopped") } s.state.value = srvStateActive return nil } // Shutdown stops and shuts down the scheduler. func (s *Scheduler) Shutdown() { s.state.mu.Lock() if s.state.value == srvStateNew || s.state.value == srvStateClosed { // scheduler is not running, do nothing and return. s.state.mu.Unlock() return } s.state.value = srvStateClosed s.state.mu.Unlock() s.logger.Info("Scheduler shutting down") close(s.done) // signal heartbeater to stop ctx := s.cron.Stop() <-ctx.Done() s.wg.Wait() s.clearHistory() if err := s.client.Close(); err != nil { s.logger.Errorf("Failed to close redis client connection: %v", err) } s.logger.Info("Scheduler stopped") } func (s *Scheduler) runHeartbeater() { defer s.wg.Done() ticker := time.NewTicker(s.heartbeatInterval) for { select { case <-s.done: s.logger.Debugf("Scheduler heatbeater shutting down") if err := s.rdb.ClearSchedulerEntries(s.id); err != nil { s.logger.Errorf("Failed to clear the scheduler entries: %v", err) } ticker.Stop() return case <-ticker.C: s.beat() } } } // beat writes a snapshot of entries to redis. func (s *Scheduler) beat() { var entries []*base.SchedulerEntry for _, entry := range s.cron.Entries() { job := entry.Job.(*enqueueJob) e := &base.SchedulerEntry{ ID: job.id.String(), Spec: job.cronspec, Type: job.task.Type(), Payload: job.task.Payload(), Opts: stringifyOptions(job.opts), Next: entry.Next, Prev: entry.Prev, } entries = append(entries, e) } if err := s.rdb.WriteSchedulerEntries(s.id, entries, s.heartbeatInterval*2); err != nil { s.logger.Warnf("Scheduler could not write heartbeat data: %v", err) } } func stringifyOptions(opts []Option) []string { var res []string for _, opt := range opts { res = append(res, opt.String()) } return res } func (s *Scheduler) clearHistory() { for _, entry := range s.cron.Entries() { job := entry.Job.(*enqueueJob) if err := s.rdb.ClearSchedulerHistory(job.id.String()); err != nil { s.logger.Warnf("Could not clear scheduler history for entry %q: %v", job.id.String(), err) } } } // Ping performs a ping against the redis connection. func (s *Scheduler) Ping() error { s.state.mu.Lock() defer s.state.mu.Unlock() if s.state.value == srvStateClosed { return nil } return s.rdb.Ping() } ================================================ FILE: scheduler_test.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "sync" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/redis/go-redis/v9" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/testutil" ) func TestSchedulerRegister(t *testing.T) { tests := []struct { cronspec string task *Task opts []Option wait time.Duration queue string want []*base.TaskMessage }{ { cronspec: "@every 3s", task: NewTask("task1", nil), opts: []Option{MaxRetry(10)}, wait: 10 * time.Second, queue: "default", want: []*base.TaskMessage{ { Type: "task1", Payload: nil, Retry: 10, Timeout: int64(defaultTimeout.Seconds()), Queue: "default", }, { Type: "task1", Payload: nil, Retry: 10, Timeout: int64(defaultTimeout.Seconds()), Queue: "default", }, { Type: "task1", Payload: nil, Retry: 10, Timeout: int64(defaultTimeout.Seconds()), Queue: "default", }, }, }, } r := setup(t) // Tests for new redis connection. for _, tc := range tests { scheduler := NewScheduler(getRedisConnOpt(t), nil) if _, err := scheduler.Register(tc.cronspec, tc.task, tc.opts...); err != nil { t.Fatal(err) } if err := scheduler.Start(); err != nil { t.Fatal(err) } time.Sleep(tc.wait) scheduler.Shutdown() got := testutil.GetPendingMessages(t, r, tc.queue) if diff := cmp.Diff(tc.want, got, testutil.IgnoreIDOpt); diff != "" { t.Errorf("mismatch found in queue %q: (-want,+got)\n%s", tc.queue, diff) } } r = setup(t) // Tests for existing redis connection. for _, tc := range tests { redisClient := getRedisConnOpt(t).MakeRedisClient().(redis.UniversalClient) scheduler := NewSchedulerFromRedisClient(redisClient, nil) if _, err := scheduler.Register(tc.cronspec, tc.task, tc.opts...); err != nil { t.Fatal(err) } if err := scheduler.Start(); err != nil { t.Fatal(err) } time.Sleep(tc.wait) scheduler.Shutdown() got := testutil.GetPendingMessages(t, r, tc.queue) if diff := cmp.Diff(tc.want, got, testutil.IgnoreIDOpt); diff != "" { t.Errorf("mismatch found in queue %q: (-want,+got)\n%s", tc.queue, diff) } } } func TestSchedulerWhenRedisDown(t *testing.T) { var ( mu sync.Mutex counter int ) errorHandler := func(task *Task, opts []Option, err error) { mu.Lock() counter++ mu.Unlock() } // Connect to non-existent redis instance to simulate a redis server being down. scheduler := NewScheduler( RedisClientOpt{Addr: ":9876"}, // no Redis listening to this port. &SchedulerOpts{EnqueueErrorHandler: errorHandler}, ) task := NewTask("test", nil) if _, err := scheduler.Register("@every 3s", task); err != nil { t.Fatal(err) } if err := scheduler.Start(); err != nil { t.Fatal(err) } // Scheduler should attempt to enqueue the task three times (every 3s). time.Sleep(10 * time.Second) scheduler.Shutdown() mu.Lock() if counter != 3 { t.Errorf("EnqueueErrorHandler was called %d times, want 3", counter) } mu.Unlock() } func TestSchedulerUnregister(t *testing.T) { tests := []struct { cronspec string task *Task opts []Option wait time.Duration queue string }{ { cronspec: "@every 3s", task: NewTask("task1", nil), opts: []Option{MaxRetry(10)}, wait: 10 * time.Second, queue: "default", }, } r := setup(t) for _, tc := range tests { scheduler := NewScheduler(getRedisConnOpt(t), nil) entryID, err := scheduler.Register(tc.cronspec, tc.task, tc.opts...) if err != nil { t.Fatal(err) } if err := scheduler.Unregister(entryID); err != nil { t.Fatal(err) } if err := scheduler.Start(); err != nil { t.Fatal(err) } time.Sleep(tc.wait) scheduler.Shutdown() got := testutil.GetPendingMessages(t, r, tc.queue) if len(got) != 0 { t.Errorf("%d tasks were enqueued, want zero", len(got)) } } } func TestSchedulerPostAndPreEnqueueHandler(t *testing.T) { var ( preMu sync.Mutex preCounter int postMu sync.Mutex postCounter int ) preHandler := func(task *Task, opts []Option) { preMu.Lock() preCounter++ preMu.Unlock() } postHandler := func(info *TaskInfo, err error) { postMu.Lock() postCounter++ postMu.Unlock() } // Connect to non-existent redis instance to simulate a redis server being down. scheduler := NewScheduler( getRedisConnOpt(t), &SchedulerOpts{ PreEnqueueFunc: preHandler, PostEnqueueFunc: postHandler, }, ) task := NewTask("test", nil) if _, err := scheduler.Register("@every 3s", task); err != nil { t.Fatal(err) } if err := scheduler.Start(); err != nil { t.Fatal(err) } // Scheduler should attempt to enqueue the task three times (every 3s). time.Sleep(10 * time.Second) scheduler.Shutdown() preMu.Lock() if preCounter != 3 { t.Errorf("PreEnqueueFunc was called %d times, want 3", preCounter) } preMu.Unlock() postMu.Lock() if postCounter != 3 { t.Errorf("PostEnqueueFunc was called %d times, want 3", postCounter) } postMu.Unlock() } ================================================ FILE: servemux.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "context" "errors" "fmt" "sort" "strings" "sync" ) // ErrHandlerNotFound indicates that no task handler was found for a given pattern. var ErrHandlerNotFound = errors.New("handler not found for task") // ServeMux is a multiplexer for asynchronous tasks. // It matches the type of each task against a list of registered patterns // and calls the handler for the pattern that most closely matches the // task's type name. // // Longer patterns take precedence over shorter ones, so that if there are // handlers registered for both "images" and "images:thumbnails", // the latter handler will be called for tasks with a type name beginning with // "images:thumbnails" and the former will receive tasks with type name beginning // with "images". type ServeMux struct { mu sync.RWMutex m map[string]muxEntry es []muxEntry // slice of entries sorted from longest to shortest. mws []MiddlewareFunc } type muxEntry struct { h Handler pattern string } // MiddlewareFunc is a function which receives an asynq.Handler and returns another asynq.Handler. // Typically, the returned handler is a closure which does something with the context and task passed // to it, and then calls the handler passed as parameter to the MiddlewareFunc. type MiddlewareFunc func(Handler) Handler // NewServeMux allocates and returns a new ServeMux. func NewServeMux() *ServeMux { return new(ServeMux) } // ProcessTask dispatches the task to the handler whose // pattern most closely matches the task type. func (mux *ServeMux) ProcessTask(ctx context.Context, task *Task) error { h, _ := mux.Handler(task) return h.ProcessTask(ctx, task) } // Handler returns the handler to use for the given task. // It always return a non-nil handler. // // Handler also returns the registered pattern that matches the task. // // If there is no registered handler that applies to the task, // handler returns a 'not found' handler which returns an error. func (mux *ServeMux) Handler(t *Task) (h Handler, pattern string) { mux.mu.RLock() defer mux.mu.RUnlock() h, pattern = mux.match(t.Type()) if h == nil { h, pattern = NotFoundHandler(), "" } for i := len(mux.mws) - 1; i >= 0; i-- { h = mux.mws[i](h) } return h, pattern } // Find a handler on a handler map given a typename string. // Most-specific (longest) pattern wins. func (mux *ServeMux) match(typename string) (h Handler, pattern string) { // Check for exact match first. v, ok := mux.m[typename] if ok { return v.h, v.pattern } // Check for longest valid match. // mux.es contains all patterns from longest to shortest. for _, e := range mux.es { if strings.HasPrefix(typename, e.pattern) { return e.h, e.pattern } } return nil, "" } // Handle registers the handler for the given pattern. // If a handler already exists for pattern, Handle panics. func (mux *ServeMux) Handle(pattern string, handler Handler) { mux.mu.Lock() defer mux.mu.Unlock() if strings.TrimSpace(pattern) == "" { panic("asynq: invalid pattern") } if handler == nil { panic("asynq: nil handler") } if _, exist := mux.m[pattern]; exist { panic("asynq: multiple registrations for " + pattern) } if mux.m == nil { mux.m = make(map[string]muxEntry) } e := muxEntry{h: handler, pattern: pattern} mux.m[pattern] = e mux.es = appendSorted(mux.es, e) } func appendSorted(es []muxEntry, e muxEntry) []muxEntry { n := len(es) i := sort.Search(n, func(i int) bool { return len(es[i].pattern) < len(e.pattern) }) if i == n { return append(es, e) } // we now know that i points at where we want to insert. es = append(es, muxEntry{}) // try to grow the slice in place, any entry works. copy(es[i+1:], es[i:]) // shift shorter entries down. es[i] = e return es } // HandleFunc registers the handler function for the given pattern. func (mux *ServeMux) HandleFunc(pattern string, handler func(context.Context, *Task) error) { if handler == nil { panic("asynq: nil handler") } mux.Handle(pattern, HandlerFunc(handler)) } // Use appends a MiddlewareFunc to the chain. // Middlewares are executed in the order that they are applied to the ServeMux. func (mux *ServeMux) Use(mws ...MiddlewareFunc) { mux.mu.Lock() defer mux.mu.Unlock() mux.mws = append(mux.mws, mws...) } // NotFound returns an error indicating that the handler was not found for the given task. func NotFound(ctx context.Context, task *Task) error { return fmt.Errorf("%w %q", ErrHandlerNotFound, task.Type()) } // NotFoundHandler returns a simple task handler that returns a “not found“ error. func NotFoundHandler() Handler { return HandlerFunc(NotFound) } ================================================ FILE: servemux_test.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "context" "testing" "github.com/google/go-cmp/cmp" ) var called string // identity of the handler that was called. var invoked []string // list of middlewares in the order they were invoked. // makeFakeHandler returns a handler that updates the global called variable // to the given identity. func makeFakeHandler(identity string) Handler { return HandlerFunc(func(ctx context.Context, t *Task) error { called = identity return nil }) } // makeFakeMiddleware returns a middleware function that appends the given identity // to the global invoked slice. func makeFakeMiddleware(identity string) MiddlewareFunc { return func(next Handler) Handler { return HandlerFunc(func(ctx context.Context, t *Task) error { invoked = append(invoked, identity) return next.ProcessTask(ctx, t) }) } } // A list of pattern, handler pair that is registered with mux. var serveMuxRegister = []struct { pattern string h Handler }{ {"email:", makeFakeHandler("default email handler")}, {"email:signup", makeFakeHandler("signup email handler")}, {"csv:export", makeFakeHandler("csv export handler")}, } var serveMuxTests = []struct { typename string // task's type name want string // identifier of the handler that should be called }{ {"email:signup", "signup email handler"}, {"csv:export", "csv export handler"}, {"email:daily", "default email handler"}, } func TestServeMux(t *testing.T) { mux := NewServeMux() for _, e := range serveMuxRegister { mux.Handle(e.pattern, e.h) } for _, tc := range serveMuxTests { called = "" // reset to zero value task := NewTask(tc.typename, nil) if err := mux.ProcessTask(context.Background(), task); err != nil { t.Fatal(err) } if called != tc.want { t.Errorf("%q handler was called for task %q, want %q to be called", called, task.Type(), tc.want) } } } func TestServeMuxRegisterNilHandler(t *testing.T) { defer func() { if err := recover(); err == nil { t.Error("expected call to mux.HandleFunc to panic") } }() mux := NewServeMux() mux.HandleFunc("email:signup", nil) } func TestServeMuxRegisterEmptyPattern(t *testing.T) { defer func() { if err := recover(); err == nil { t.Error("expected call to mux.HandleFunc to panic") } }() mux := NewServeMux() mux.Handle("", makeFakeHandler("email")) } func TestServeMuxRegisterDuplicatePattern(t *testing.T) { defer func() { if err := recover(); err == nil { t.Error("expected call to mux.HandleFunc to panic") } }() mux := NewServeMux() mux.Handle("email", makeFakeHandler("email")) mux.Handle("email", makeFakeHandler("email:default")) } var notFoundTests = []struct { typename string // task's type name }{ {"image:minimize"}, {"csv:"}, // registered patterns match the task's type prefix, not the other way around. } func TestServeMuxNotFound(t *testing.T) { mux := NewServeMux() for _, e := range serveMuxRegister { mux.Handle(e.pattern, e.h) } for _, tc := range notFoundTests { task := NewTask(tc.typename, nil) err := mux.ProcessTask(context.Background(), task) if err == nil { t.Errorf("ProcessTask did not return error for task %q, should return 'not found' error", task.Type()) } } } var middlewareTests = []struct { typename string // task's type name middlewares []string // middlewares to use. They should be called in this order. want string // identifier of the handler that should be called }{ {"email:signup", []string{"logging", "expiration"}, "signup email handler"}, {"csv:export", []string{}, "csv export handler"}, {"email:daily", []string{"expiration", "logging"}, "default email handler"}, } func TestServeMuxMiddlewares(t *testing.T) { for _, tc := range middlewareTests { mux := NewServeMux() for _, e := range serveMuxRegister { mux.Handle(e.pattern, e.h) } var mws []MiddlewareFunc for _, s := range tc.middlewares { mws = append(mws, makeFakeMiddleware(s)) } mux.Use(mws...) invoked = []string{} // reset to empty slice called = "" // reset to zero value task := NewTask(tc.typename, nil) if err := mux.ProcessTask(context.Background(), task); err != nil { t.Fatal(err) } if diff := cmp.Diff(invoked, tc.middlewares); diff != "" { t.Errorf("invoked middlewares were %v, want %v", invoked, tc.middlewares) } if called != tc.want { t.Errorf("%q handler was called for task %q, want %q to be called", called, task.Type(), tc.want) } } } ================================================ FILE: server.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "context" "errors" "fmt" "math" "math/rand/v2" "runtime" "strings" "sync" "time" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/log" "github.com/hibiken/asynq/internal/rdb" "github.com/redis/go-redis/v9" ) // Server is responsible for task processing and task lifecycle management. // // Server pulls tasks off queues and processes them. // If the processing of a task is unsuccessful, server will schedule it for a retry. // // A task will be retried until either the task gets processed successfully // or until it reaches its max retry count. // // If a task exhausts its retries, it will be moved to the archive and // will be kept in the archive set. // Note that the archive size is finite and once it reaches its max size, // oldest tasks in the archive will be deleted. type Server struct { logger *log.Logger broker base.Broker // When a Server has been created with an existing Redis connection, we do // not want to close it. sharedConnection bool state *serverState // wait group to wait for all goroutines to finish. wg sync.WaitGroup forwarder *forwarder processor *processor syncer *syncer heartbeater *heartbeater subscriber *subscriber recoverer *recoverer healthchecker *healthchecker janitor *janitor aggregator *aggregator } type serverState struct { mu sync.Mutex value serverStateValue } type serverStateValue int const ( // StateNew represents a new server. Server begins in // this state and then transition to StatusActive when // Start or Run is callled. srvStateNew serverStateValue = iota // StateActive indicates the server is up and active. srvStateActive // StateStopped indicates the server is up but no longer processing new tasks. srvStateStopped // StateClosed indicates the server has been shutdown. srvStateClosed ) var serverStates = []string{ "new", "active", "stopped", "closed", } func (s serverStateValue) String() string { if srvStateNew <= s && s <= srvStateClosed { return serverStates[s] } return "unknown status" } // Config specifies the server's background-task processing behavior. type Config struct { // Maximum number of concurrent processing of tasks. // // If set to a zero or negative value, NewServer will overwrite the value // to the number of CPUs usable by the current process. Concurrency int // BaseContext optionally specifies a function that returns the base context for Handler invocations on this server. // // If BaseContext is nil, the default is context.Background(). // If this is defined, then it MUST return a non-nil context BaseContext func() context.Context // TaskCheckInterval specifies the interval between checks for new tasks to process when all queues are empty. // // If unset, zero or a negative value, the interval is set to 1 second. // // Note: Setting this value too low may add significant load to redis. // // By default, TaskCheckInterval is set to 1 seconds. TaskCheckInterval time.Duration // Function to calculate retry delay for a failed task. // // By default, it uses exponential backoff algorithm to calculate the delay. RetryDelayFunc RetryDelayFunc // Predicate function to determine whether the error returned from Handler is a failure. // If the function returns false, Server will not increment the retried counter for the task, // and Server won't record the queue stats (processed and failed stats) to avoid skewing the error // rate of the queue. // // By default, if the given error is non-nil the function returns true. IsFailure func(error) bool // List of queues to process with given priority value. Keys are the names of the // queues and values are associated priority value. // // If set to nil or not specified, the server will process only the "default" queue. // // Priority is treated as follows to avoid starving low priority queues. // // Example: // // Queues: map[string]int{ // "critical": 6, // "default": 3, // "low": 1, // } // // With the above config and given that all queues are not empty, the tasks // in "critical", "default", "low" should be processed 60%, 30%, 10% of // the time respectively. // // If a queue has a zero or negative priority value, the queue will be ignored. Queues map[string]int // StrictPriority indicates whether the queue priority should be treated strictly. // // If set to true, tasks in the queue with the highest priority is processed first. // The tasks in lower priority queues are processed only when those queues with // higher priorities are empty. StrictPriority bool // ErrorHandler handles errors returned by the task handler. // // HandleError is invoked only if the task handler returns a non-nil error. // // Example: // // func reportError(ctx context, task *asynq.Task, err error) { // retried, _ := asynq.GetRetryCount(ctx) // maxRetry, _ := asynq.GetMaxRetry(ctx) // if retried >= maxRetry { // err = fmt.Errorf("retry exhausted for task %s: %w", task.Type, err) // } // errorReportingService.Notify(err) // }) // // ErrorHandler: asynq.ErrorHandlerFunc(reportError) // // we can also handle panic error like: // func reportError(ctx context, task *asynq.Task, err error) { // if asynq.IsPanicError(err) { // errorReportingService.Notify(err) // } // }) // // ErrorHandler: asynq.ErrorHandlerFunc(reportError) ErrorHandler ErrorHandler // Logger specifies the logger used by the server instance. // // If unset, default logger is used. Logger Logger // LogLevel specifies the minimum log level to enable. // // If unset, InfoLevel is used by default. LogLevel LogLevel // ShutdownTimeout specifies the duration to wait to let workers finish their tasks // before forcing them to abort when stopping the server. // // If unset or zero, default timeout of 8 seconds is used. ShutdownTimeout time.Duration // HealthCheckFunc is called periodically with any errors encountered during ping to the // connected redis server. HealthCheckFunc func(error) // HealthCheckInterval specifies the interval between healthchecks. // // If unset or zero, the interval is set to 15 seconds. HealthCheckInterval time.Duration // DelayedTaskCheckInterval specifies the interval between checks run on 'scheduled' and 'retry' // tasks, and forwarding them to 'pending' state if they are ready to be processed. // // If unset or zero, the interval is set to 5 seconds. DelayedTaskCheckInterval time.Duration // GroupGracePeriod specifies the amount of time the server will wait for an incoming task before aggregating // the tasks in a group. If an incoming task is received within this period, the server will wait for another // period of the same length, up to GroupMaxDelay if specified. // // If unset or zero, the grace period is set to 1 minute. // Minimum duration for GroupGracePeriod is 1 second. If value specified is less than a second, the call to // NewServer will panic. GroupGracePeriod time.Duration // GroupMaxDelay specifies the maximum amount of time the server will wait for incoming tasks before aggregating // the tasks in a group. // // If unset or zero, no delay limit is used. GroupMaxDelay time.Duration // GroupMaxSize specifies the maximum number of tasks that can be aggregated into a single task within a group. // If GroupMaxSize is reached, the server will aggregate the tasks into one immediately. // // If unset or zero, no size limit is used. GroupMaxSize int // GroupAggregator specifies the aggregation function used to aggregate multiple tasks in a group into one task. // // If unset or nil, the group aggregation feature will be disabled on the server. GroupAggregator GroupAggregator // JanitorInterval specifies the average interval of janitor checks for expired completed tasks. // // If unset or zero, default interval of 8 seconds is used. JanitorInterval time.Duration // JanitorBatchSize specifies the number of expired completed tasks to be deleted in one run. // // If unset or zero, default batch size of 100 is used. // Make sure to not put a big number as the batch size to prevent a long-running script. JanitorBatchSize int } // GroupAggregator aggregates a group of tasks into one before the tasks are passed to the Handler. type GroupAggregator interface { // Aggregate aggregates the given tasks in a group with the given group name, // and returns a new task which is the aggregation of those tasks. // // Use NewTask(typename, payload, opts...) to set any options for the aggregated task. // The Queue option, if provided, will be ignored and the aggregated task will always be enqueued // to the same queue the group belonged. Aggregate(group string, tasks []*Task) *Task } // The GroupAggregatorFunc type is an adapter to allow the use of ordinary functions as a GroupAggregator. // If f is a function with the appropriate signature, GroupAggregatorFunc(f) is a GroupAggregator that calls f. type GroupAggregatorFunc func(group string, tasks []*Task) *Task // Aggregate calls fn(group, tasks) func (fn GroupAggregatorFunc) Aggregate(group string, tasks []*Task) *Task { return fn(group, tasks) } // An ErrorHandler handles an error occurred during task processing. type ErrorHandler interface { HandleError(ctx context.Context, task *Task, err error) } // The ErrorHandlerFunc type is an adapter to allow the use of ordinary functions as a ErrorHandler. // If f is a function with the appropriate signature, ErrorHandlerFunc(f) is a ErrorHandler that calls f. type ErrorHandlerFunc func(ctx context.Context, task *Task, err error) // HandleError calls fn(ctx, task, err) func (fn ErrorHandlerFunc) HandleError(ctx context.Context, task *Task, err error) { fn(ctx, task, err) } // RetryDelayFunc calculates the retry delay duration for a failed task given // the retry count, error, and the task. // // n is the number of times the task has been retried. // e is the error returned by the task handler. // t is the task in question. type RetryDelayFunc func(n int, e error, t *Task) time.Duration // Logger supports logging at various log levels. type Logger interface { // Debug logs a message at Debug level. Debug(args ...interface{}) // Info logs a message at Info level. Info(args ...interface{}) // Warn logs a message at Warning level. Warn(args ...interface{}) // Error logs a message at Error level. Error(args ...interface{}) // Fatal logs a message at Fatal level // and process will exit with status set to 1. Fatal(args ...interface{}) } // LogLevel represents logging level. // // It satisfies flag.Value interface. type LogLevel int32 const ( // Note: reserving value zero to differentiate unspecified case. level_unspecified LogLevel = iota // DebugLevel is the lowest level of logging. // Debug logs are intended for debugging and development purposes. DebugLevel // InfoLevel is used for general informational log messages. InfoLevel // WarnLevel is used for undesired but relatively expected events, // which may indicate a problem. WarnLevel // ErrorLevel is used for undesired and unexpected events that // the program can recover from. ErrorLevel // FatalLevel is used for undesired and unexpected events that // the program cannot recover from. FatalLevel ) // String is part of the flag.Value interface. func (l *LogLevel) String() string { switch *l { case DebugLevel: return "debug" case InfoLevel: return "info" case WarnLevel: return "warn" case ErrorLevel: return "error" case FatalLevel: return "fatal" } panic(fmt.Sprintf("asynq: unexpected log level: %v", *l)) } // Set is part of the flag.Value interface. func (l *LogLevel) Set(val string) error { switch strings.ToLower(val) { case "debug": *l = DebugLevel case "info": *l = InfoLevel case "warn", "warning": *l = WarnLevel case "error": *l = ErrorLevel case "fatal": *l = FatalLevel default: return fmt.Errorf("asynq: unsupported log level %q", val) } return nil } func toInternalLogLevel(l LogLevel) log.Level { switch l { case DebugLevel: return log.DebugLevel case InfoLevel: return log.InfoLevel case WarnLevel: return log.WarnLevel case ErrorLevel: return log.ErrorLevel case FatalLevel: return log.FatalLevel } panic(fmt.Sprintf("asynq: unexpected log level: %v", l)) } // DefaultRetryDelayFunc is the default RetryDelayFunc used if one is not specified in Config. // It uses exponential back-off strategy to calculate the retry delay. func DefaultRetryDelayFunc(n int, e error, t *Task) time.Duration { // Formula taken from https://github.com/mperham/sidekiq. s := int(math.Pow(float64(n), 4)) + 15 + (rand.IntN(30) * (n + 1)) return time.Duration(s) * time.Second } func defaultIsFailureFunc(err error) bool { return err != nil } var defaultQueueConfig = map[string]int{ base.DefaultQueueName: 1, } const ( defaultTaskCheckInterval = 1 * time.Second defaultShutdownTimeout = 8 * time.Second defaultHealthCheckInterval = 15 * time.Second defaultDelayedTaskCheckInterval = 5 * time.Second defaultGroupGracePeriod = 1 * time.Minute defaultJanitorInterval = 8 * time.Second defaultJanitorBatchSize = 100 ) // NewServer returns a new Server given a redis connection option // and server configuration. func NewServer(r RedisConnOpt, cfg Config) *Server { redisClient, ok := r.MakeRedisClient().(redis.UniversalClient) if !ok { panic(fmt.Sprintf("asynq: unsupported RedisConnOpt type %T", r)) } server := NewServerFromRedisClient(redisClient, cfg) server.sharedConnection = false return server } // NewServerFromRedisClient returns a new instance of Server given a redis.UniversalClient // and server configuration // Warning: The underlying redis connection pool will not be closed by Asynq, you are responsible for closing it. func NewServerFromRedisClient(c redis.UniversalClient, cfg Config) *Server { baseCtxFn := cfg.BaseContext if baseCtxFn == nil { baseCtxFn = context.Background } n := cfg.Concurrency if n < 1 { n = runtime.NumCPU() } taskCheckInterval := cfg.TaskCheckInterval if taskCheckInterval <= 0 { taskCheckInterval = defaultTaskCheckInterval } delayFunc := cfg.RetryDelayFunc if delayFunc == nil { delayFunc = DefaultRetryDelayFunc } isFailureFunc := cfg.IsFailure if isFailureFunc == nil { isFailureFunc = defaultIsFailureFunc } queues := make(map[string]int) for qname, p := range cfg.Queues { if err := base.ValidateQueueName(qname); err != nil { continue // ignore invalid queue names } if p > 0 { queues[qname] = p } } if len(queues) == 0 { queues = defaultQueueConfig } var qnames []string for q := range queues { qnames = append(qnames, q) } shutdownTimeout := cfg.ShutdownTimeout if shutdownTimeout == 0 { shutdownTimeout = defaultShutdownTimeout } healthcheckInterval := cfg.HealthCheckInterval if healthcheckInterval == 0 { healthcheckInterval = defaultHealthCheckInterval } // TODO: Create a helper to check for zero value and fall back to default (e.g. getDurationOrDefault()) groupGracePeriod := cfg.GroupGracePeriod if groupGracePeriod == 0 { groupGracePeriod = defaultGroupGracePeriod } if groupGracePeriod < time.Second { panic("GroupGracePeriod cannot be less than a second") } logger := log.NewLogger(cfg.Logger) loglevel := cfg.LogLevel if loglevel == level_unspecified { loglevel = InfoLevel } logger.SetLevel(toInternalLogLevel(loglevel)) rdb := rdb.NewRDB(c) starting := make(chan *workerInfo) finished := make(chan *base.TaskMessage) syncCh := make(chan *syncRequest) srvState := &serverState{value: srvStateNew} cancels := base.NewCancelations() syncer := newSyncer(syncerParams{ logger: logger, requestsCh: syncCh, interval: 5 * time.Second, }) heartbeater := newHeartbeater(heartbeaterParams{ logger: logger, broker: rdb, interval: 5 * time.Second, concurrency: n, queues: queues, strictPriority: cfg.StrictPriority, state: srvState, starting: starting, finished: finished, }) delayedTaskCheckInterval := cfg.DelayedTaskCheckInterval if delayedTaskCheckInterval == 0 { delayedTaskCheckInterval = defaultDelayedTaskCheckInterval } forwarder := newForwarder(forwarderParams{ logger: logger, broker: rdb, queues: qnames, interval: delayedTaskCheckInterval, }) subscriber := newSubscriber(subscriberParams{ logger: logger, broker: rdb, cancelations: cancels, }) processor := newProcessor(processorParams{ logger: logger, broker: rdb, retryDelayFunc: delayFunc, taskCheckInterval: taskCheckInterval, baseCtxFn: baseCtxFn, isFailureFunc: isFailureFunc, syncCh: syncCh, cancelations: cancels, concurrency: n, queues: queues, strictPriority: cfg.StrictPriority, errHandler: cfg.ErrorHandler, shutdownTimeout: shutdownTimeout, starting: starting, finished: finished, }) recoverer := newRecoverer(recovererParams{ logger: logger, broker: rdb, retryDelayFunc: delayFunc, isFailureFunc: isFailureFunc, queues: qnames, interval: 1 * time.Minute, }) healthchecker := newHealthChecker(healthcheckerParams{ logger: logger, broker: rdb, interval: healthcheckInterval, healthcheckFunc: cfg.HealthCheckFunc, }) janitorInterval := cfg.JanitorInterval if janitorInterval == 0 { janitorInterval = defaultJanitorInterval } janitorBatchSize := cfg.JanitorBatchSize if janitorBatchSize == 0 { janitorBatchSize = defaultJanitorBatchSize } if janitorBatchSize > defaultJanitorBatchSize { logger.Warnf("Janitor batch size of %d is greater than the recommended batch size of %d. "+ "This might cause a long-running script", janitorBatchSize, defaultJanitorBatchSize) } janitor := newJanitor(janitorParams{ logger: logger, broker: rdb, queues: qnames, interval: janitorInterval, batchSize: janitorBatchSize, }) aggregator := newAggregator(aggregatorParams{ logger: logger, broker: rdb, queues: qnames, gracePeriod: groupGracePeriod, maxDelay: cfg.GroupMaxDelay, maxSize: cfg.GroupMaxSize, groupAggregator: cfg.GroupAggregator, }) return &Server{ logger: logger, broker: rdb, sharedConnection: true, state: srvState, forwarder: forwarder, processor: processor, syncer: syncer, heartbeater: heartbeater, subscriber: subscriber, recoverer: recoverer, healthchecker: healthchecker, janitor: janitor, aggregator: aggregator, } } // A Handler processes tasks. // // ProcessTask should return nil if the processing of a task // is successful. // // If ProcessTask returns a non-nil error or panics, the task // will be retried after delay if retry-count is remaining, // otherwise the task will be archived. // // One exception to this rule is when ProcessTask returns a SkipRetry error. // If the returned error is SkipRetry or an error wraps SkipRetry, retry is // skipped and the task will be immediately archived instead. // // Another exception to this rule is when ProcessTask returns a RevokeTask error. // If the returned error is RevokeTask or an error wraps RevokeTask, the task // will not be retried or archived. type Handler interface { ProcessTask(context.Context, *Task) error } // The HandlerFunc type is an adapter to allow the use of // ordinary functions as a Handler. If f is a function // with the appropriate signature, HandlerFunc(f) is a // Handler that calls f. type HandlerFunc func(context.Context, *Task) error // ProcessTask calls fn(ctx, task) func (fn HandlerFunc) ProcessTask(ctx context.Context, task *Task) error { return fn(ctx, task) } // ErrServerClosed indicates that the operation is now illegal because of the server has been shutdown. var ErrServerClosed = errors.New("asynq: Server closed") // Run starts the task processing and blocks until // an os signal to exit the program is received. Once it receives // a signal, it gracefully shuts down all active workers and other // goroutines to process the tasks. // // Run returns any error encountered at server startup time. // If the server has already been shutdown, ErrServerClosed is returned. func (srv *Server) Run(handler Handler) error { if err := srv.Start(handler); err != nil { return err } srv.waitForSignals() srv.Shutdown() return nil } // Start starts the worker server. Once the server has started, // it pulls tasks off queues and starts a worker goroutine for each task // and then call Handler to process it. // Tasks are processed concurrently by the workers up to the number of // concurrency specified in Config.Concurrency. // // Start returns any error encountered at server startup time. // If the server has already been shutdown, ErrServerClosed is returned. func (srv *Server) Start(handler Handler) error { if handler == nil { return fmt.Errorf("asynq: server cannot run with nil handler") } srv.processor.handler = handler if err := srv.start(); err != nil { return err } srv.logger.Info("Starting processing") srv.heartbeater.start(&srv.wg) srv.healthchecker.start(&srv.wg) srv.subscriber.start(&srv.wg) srv.syncer.start(&srv.wg) srv.recoverer.start(&srv.wg) srv.forwarder.start(&srv.wg) srv.processor.start(&srv.wg) srv.janitor.start(&srv.wg) srv.aggregator.start(&srv.wg) return nil } // Checks server state and returns an error if pre-condition is not met. // Otherwise it sets the server state to active. func (srv *Server) start() error { srv.state.mu.Lock() defer srv.state.mu.Unlock() switch srv.state.value { case srvStateActive: return fmt.Errorf("asynq: the server is already running") case srvStateStopped: return fmt.Errorf("asynq: the server is in the stopped state. Waiting for shutdown.") case srvStateClosed: return ErrServerClosed } srv.state.value = srvStateActive return nil } // Shutdown gracefully shuts down the server. // It gracefully closes all active workers. The server will wait for // active workers to finish processing tasks for duration specified in Config.ShutdownTimeout. // If worker didn't finish processing a task during the timeout, the task will be pushed back to Redis. func (srv *Server) Shutdown() { srv.state.mu.Lock() if srv.state.value == srvStateNew || srv.state.value == srvStateClosed { srv.state.mu.Unlock() // server is not running, do nothing and return. return } srv.state.value = srvStateClosed srv.state.mu.Unlock() srv.logger.Info("Starting graceful shutdown") // Note: The order of shutdown is important. // Sender goroutines should be terminated before the receiver goroutines. // processor -> syncer (via syncCh) // processor -> heartbeater (via starting, finished channels) srv.forwarder.shutdown() srv.processor.shutdown() srv.recoverer.shutdown() srv.syncer.shutdown() srv.subscriber.shutdown() srv.janitor.shutdown() srv.aggregator.shutdown() srv.healthchecker.shutdown() srv.heartbeater.shutdown() srv.wg.Wait() if !srv.sharedConnection { srv.broker.Close() } srv.logger.Info("Exiting") } // Stop signals the server to stop pulling new tasks off queues. // Stop can be used before shutting down the server to ensure that all // currently active tasks are processed before server shutdown. // // Stop does not shutdown the server, make sure to call Shutdown before exit. func (srv *Server) Stop() { srv.state.mu.Lock() if srv.state.value != srvStateActive { // Invalid call to Stop, server can only go from Active state to Stopped state. srv.state.mu.Unlock() return } srv.state.value = srvStateStopped srv.state.mu.Unlock() srv.logger.Info("Stopping processor") srv.processor.stop() srv.logger.Info("Processor stopped") } // Ping performs a ping against the redis connection. // // This is an alternative to the HealthCheckFunc available in the Config object. func (srv *Server) Ping() error { srv.state.mu.Lock() defer srv.state.mu.Unlock() if srv.state.value == srvStateClosed { return nil } return srv.broker.Ping() } ================================================ FILE: server_test.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "context" "fmt" "syscall" "testing" "time" "github.com/hibiken/asynq/internal/rdb" "github.com/hibiken/asynq/internal/testbroker" "github.com/hibiken/asynq/internal/testutil" "github.com/redis/go-redis/v9" "go.uber.org/goleak" ) func testServer(t *testing.T, c *Client, srv *Server) { // no-op handler h := func(ctx context.Context, task *Task) error { return nil } err := srv.Start(HandlerFunc(h)) if err != nil { t.Fatal(err) } _, err = c.Enqueue(NewTask("send_email", testutil.JSON(map[string]interface{}{"recipient_id": 123}))) if err != nil { t.Errorf("could not enqueue a task: %v", err) } _, err = c.Enqueue(NewTask("send_email", testutil.JSON(map[string]interface{}{"recipient_id": 456})), ProcessIn(1*time.Hour)) if err != nil { t.Errorf("could not enqueue a task: %v", err) } srv.Shutdown() } func TestServer(t *testing.T) { // https://github.com/go-redis/redis/issues/1029 ignoreOpt := goleak.IgnoreTopFunction("github.com/redis/go-redis/v9/internal/pool.(*ConnPool).reaper") defer goleak.VerifyNone(t, ignoreOpt) redisConnOpt := getRedisConnOpt(t) c := NewClient(redisConnOpt) defer c.Close() srv := NewServer(redisConnOpt, Config{ Concurrency: 10, LogLevel: testLogLevel, }) testServer(t, c, srv) } func TestServerFromRedisClient(t *testing.T) { // https://github.com/go-redis/redis/issues/1029 ignoreOpt := goleak.IgnoreTopFunction("github.com/redis/go-redis/v9/internal/pool.(*ConnPool).reaper") defer goleak.VerifyNone(t, ignoreOpt) redisConnOpt := getRedisConnOpt(t) redisClient := redisConnOpt.MakeRedisClient().(redis.UniversalClient) c := NewClientFromRedisClient(redisClient) srv := NewServerFromRedisClient(redisClient, Config{ Concurrency: 10, LogLevel: testLogLevel, }) testServer(t, c, srv) err := c.Close() if err == nil { t.Error("client.Close() should have failed because of a shared client but it didn't") } } func TestServerRun(t *testing.T) { // https://github.com/go-redis/redis/issues/1029 ignoreOpt := goleak.IgnoreTopFunction("github.com/redis/go-redis/v9/internal/pool.(*ConnPool).reaper") defer goleak.VerifyNone(t, ignoreOpt) srv := NewServer(getRedisConnOpt(t), Config{LogLevel: testLogLevel}) done := make(chan struct{}) // Make sure server exits when receiving TERM signal. go func() { time.Sleep(2 * time.Second) _ = syscall.Kill(syscall.Getpid(), syscall.SIGTERM) done <- struct{}{} }() go func() { select { case <-time.After(10 * time.Second): panic("server did not stop after receiving TERM signal") case <-done: } }() mux := NewServeMux() if err := srv.Run(mux); err != nil { t.Fatal(err) } } func TestServerErrServerClosed(t *testing.T) { srv := NewServer(getRedisConnOpt(t), Config{LogLevel: testLogLevel}) handler := NewServeMux() if err := srv.Start(handler); err != nil { t.Fatal(err) } srv.Shutdown() err := srv.Start(handler) if err != ErrServerClosed { t.Errorf("Restarting server: (*Server).Start(handler) = %v, want ErrServerClosed error", err) } } func TestServerErrNilHandler(t *testing.T) { srv := NewServer(getRedisConnOpt(t), Config{LogLevel: testLogLevel}) err := srv.Start(nil) if err == nil { t.Error("Starting server with nil handler: (*Server).Start(nil) did not return error") srv.Shutdown() } } func TestServerErrServerRunning(t *testing.T) { srv := NewServer(getRedisConnOpt(t), Config{LogLevel: testLogLevel}) handler := NewServeMux() if err := srv.Start(handler); err != nil { t.Fatal(err) } err := srv.Start(handler) if err == nil { t.Error("Calling (*Server).Start(handler) on already running server did not return error") } srv.Shutdown() } func TestServerWithRedisDown(t *testing.T) { // Make sure that server does not panic and exit if redis is down. defer func() { if r := recover(); r != nil { t.Errorf("panic occurred: %v", r) } }() r := rdb.NewRDB(setup(t)) testBroker := testbroker.NewTestBroker(r) srv := NewServer(getRedisConnOpt(t), Config{LogLevel: testLogLevel}) srv.broker = testBroker srv.forwarder.broker = testBroker srv.heartbeater.broker = testBroker srv.processor.broker = testBroker srv.subscriber.broker = testBroker testBroker.Sleep() // no-op handler h := func(ctx context.Context, task *Task) error { return nil } err := srv.Start(HandlerFunc(h)) if err != nil { t.Fatal(err) } time.Sleep(3 * time.Second) srv.Shutdown() } func TestServerWithFlakyBroker(t *testing.T) { // Make sure that server does not panic and exit if redis is down. defer func() { if r := recover(); r != nil { t.Errorf("panic occurred: %v", r) } }() r := rdb.NewRDB(setup(t)) testBroker := testbroker.NewTestBroker(r) redisConnOpt := getRedisConnOpt(t) srv := NewServer(redisConnOpt, Config{LogLevel: testLogLevel}) srv.broker = testBroker srv.forwarder.broker = testBroker srv.heartbeater.broker = testBroker srv.processor.broker = testBroker srv.subscriber.broker = testBroker c := NewClient(redisConnOpt) h := func(ctx context.Context, task *Task) error { // force task retry. if task.Type() == "bad_task" { return fmt.Errorf("could not process %q", task.Type()) } time.Sleep(2 * time.Second) return nil } err := srv.Start(HandlerFunc(h)) if err != nil { t.Fatal(err) } for i := 0; i < 10; i++ { _, err := c.Enqueue(NewTask("enqueued", nil), MaxRetry(i)) if err != nil { t.Fatal(err) } _, err = c.Enqueue(NewTask("bad_task", nil)) if err != nil { t.Fatal(err) } _, err = c.Enqueue(NewTask("scheduled", nil), ProcessIn(time.Duration(i)*time.Second)) if err != nil { t.Fatal(err) } } // simulate redis going down. testBroker.Sleep() time.Sleep(3 * time.Second) // simulate redis comes back online. testBroker.Wakeup() time.Sleep(3 * time.Second) srv.Shutdown() } func TestLogLevel(t *testing.T) { tests := []struct { flagVal string want LogLevel wantStr string }{ {"debug", DebugLevel, "debug"}, {"Info", InfoLevel, "info"}, {"WARN", WarnLevel, "warn"}, {"warning", WarnLevel, "warn"}, {"Error", ErrorLevel, "error"}, {"fatal", FatalLevel, "fatal"}, } for _, tc := range tests { level := new(LogLevel) if err := level.Set(tc.flagVal); err != nil { t.Fatal(err) } if *level != tc.want { t.Errorf("Set(%q): got %v, want %v", tc.flagVal, level, &tc.want) continue } if got := level.String(); got != tc.wantStr { t.Errorf("String() returned %q, want %q", got, tc.wantStr) } } } ================================================ FILE: signals_unix.go ================================================ //go:build linux || dragonfly || freebsd || netbsd || openbsd || darwin package asynq import ( "os" "os/signal" "golang.org/x/sys/unix" ) // waitForSignals waits for signals and handles them. // It handles SIGTERM, SIGINT, and SIGTSTP. // SIGTERM and SIGINT will signal the process to exit. // SIGTSTP will signal the process to stop processing new tasks. func (srv *Server) waitForSignals() { srv.logger.Info("Send signal TSTP to stop processing new tasks") srv.logger.Info("Send signal TERM or INT to terminate the process") sigs := make(chan os.Signal, 1) signal.Notify(sigs, unix.SIGTERM, unix.SIGINT, unix.SIGTSTP) for { sig := <-sigs if sig == unix.SIGTSTP { srv.Stop() continue } else { srv.Stop() break } } } func (s *Scheduler) waitForSignals() { s.logger.Info("Send signal TERM or INT to stop the scheduler") sigs := make(chan os.Signal, 1) signal.Notify(sigs, unix.SIGTERM, unix.SIGINT) <-sigs } ================================================ FILE: signals_windows.go ================================================ //go:build windows package asynq import ( "os" "os/signal" "golang.org/x/sys/windows" ) // waitForSignals waits for signals and handles them. // It handles SIGTERM and SIGINT. // SIGTERM and SIGINT will signal the process to exit. // // Note: Currently SIGTSTP is not supported for windows build. func (srv *Server) waitForSignals() { srv.logger.Info("Send signal TERM or INT to terminate the process") sigs := make(chan os.Signal, 1) signal.Notify(sigs, windows.SIGTERM, windows.SIGINT) <-sigs } func (s *Scheduler) waitForSignals() { s.logger.Info("Send signal TERM or INT to stop the scheduler") sigs := make(chan os.Signal, 1) signal.Notify(sigs, windows.SIGTERM, windows.SIGINT) <-sigs } ================================================ FILE: subscriber.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "sync" "time" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/log" "github.com/redis/go-redis/v9" ) type subscriber struct { logger *log.Logger broker base.Broker // channel to communicate back to the long running "subscriber" goroutine. done chan struct{} // cancelations hold cancel functions for all active tasks. cancelations *base.Cancelations // time to wait before retrying to connect to redis. retryTimeout time.Duration } type subscriberParams struct { logger *log.Logger broker base.Broker cancelations *base.Cancelations } func newSubscriber(params subscriberParams) *subscriber { return &subscriber{ logger: params.logger, broker: params.broker, done: make(chan struct{}), cancelations: params.cancelations, retryTimeout: 5 * time.Second, } } func (s *subscriber) shutdown() { s.logger.Debug("Subscriber shutting down...") // Signal the subscriber goroutine to stop. s.done <- struct{}{} } func (s *subscriber) start(wg *sync.WaitGroup) { wg.Add(1) go func() { defer wg.Done() var ( pubsub *redis.PubSub err error ) // Try until successfully connect to Redis. for { pubsub, err = s.broker.CancelationPubSub() if err != nil { s.logger.Errorf("cannot subscribe to cancelation channel: %v", err) select { case <-time.After(s.retryTimeout): continue case <-s.done: s.logger.Debug("Subscriber done") return } } break } cancelCh := pubsub.Channel() for { select { case <-s.done: pubsub.Close() s.logger.Debug("Subscriber done") return case msg := <-cancelCh: cancel, ok := s.cancelations.Get(msg.Payload) if ok { cancel() } } } }() } ================================================ FILE: subscriber_test.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "sync" "testing" "time" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/rdb" "github.com/hibiken/asynq/internal/testbroker" ) func TestSubscriber(t *testing.T) { r := setup(t) defer r.Close() rdbClient := rdb.NewRDB(r) tests := []struct { registeredID string // ID for which cancel func is registered publishID string // ID to be published wantCalled bool // whether cancel func should be called }{ {"abc123", "abc123", true}, {"abc456", "abc123", false}, } for _, tc := range tests { var mu sync.Mutex called := false fakeCancelFunc := func() { mu.Lock() defer mu.Unlock() called = true } cancelations := base.NewCancelations() cancelations.Add(tc.registeredID, fakeCancelFunc) subscriber := newSubscriber(subscriberParams{ logger: testLogger, broker: rdbClient, cancelations: cancelations, }) var wg sync.WaitGroup subscriber.start(&wg) defer subscriber.shutdown() // wait for subscriber to establish connection to pubsub channel time.Sleep(time.Second) if err := rdbClient.PublishCancelation(tc.publishID); err != nil { t.Fatalf("could not publish cancelation message: %v", err) } // wait for redis to publish message time.Sleep(time.Second) mu.Lock() if called != tc.wantCalled { if tc.wantCalled { t.Errorf("fakeCancelFunc was not called, want the function to be called") } else { t.Errorf("fakeCancelFunc was called, want the function to not be called") } } mu.Unlock() } } func TestSubscriberWithRedisDown(t *testing.T) { defer func() { if r := recover(); r != nil { t.Errorf("panic occurred: %v", r) } }() r := rdb.NewRDB(setup(t)) defer r.Close() testBroker := testbroker.NewTestBroker(r) cancelations := base.NewCancelations() subscriber := newSubscriber(subscriberParams{ logger: testLogger, broker: testBroker, cancelations: cancelations, }) subscriber.retryTimeout = 1 * time.Second // set shorter retry timeout for testing purpose. testBroker.Sleep() // simulate a situation where subscriber cannot connect to redis. var wg sync.WaitGroup subscriber.start(&wg) defer subscriber.shutdown() time.Sleep(2 * time.Second) // subscriber should wait and retry connecting to redis. testBroker.Wakeup() // simulate a situation where redis server is back online. time.Sleep(2 * time.Second) // allow subscriber to establish pubsub channel. const id = "test" var ( mu sync.Mutex called bool ) cancelations.Add(id, func() { mu.Lock() defer mu.Unlock() called = true }) if err := r.PublishCancelation(id); err != nil { t.Fatalf("could not publish cancelation message: %v", err) } time.Sleep(time.Second) // wait for redis to publish message. mu.Lock() if !called { t.Errorf("cancel function was not called") } mu.Unlock() } ================================================ FILE: syncer.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "sync" "time" "github.com/hibiken/asynq/internal/log" ) // syncer is responsible for queuing up failed requests to redis and retry // those requests to sync state between the background process and redis. type syncer struct { logger *log.Logger requestsCh <-chan *syncRequest // channel to communicate back to the long running "syncer" goroutine. done chan struct{} // interval between sync operations. interval time.Duration } type syncRequest struct { fn func() error // sync operation errMsg string // error message deadline time.Time // request should be dropped if deadline has been exceeded } type syncerParams struct { logger *log.Logger requestsCh <-chan *syncRequest interval time.Duration } func newSyncer(params syncerParams) *syncer { return &syncer{ logger: params.logger, requestsCh: params.requestsCh, done: make(chan struct{}), interval: params.interval, } } func (s *syncer) shutdown() { s.logger.Debug("Syncer shutting down...") // Signal the syncer goroutine to stop. s.done <- struct{}{} } func (s *syncer) start(wg *sync.WaitGroup) { wg.Add(1) go func() { defer wg.Done() var requests []*syncRequest for { select { case <-s.done: // Try sync one last time before shutting down. for _, req := range requests { if err := req.fn(); err != nil { s.logger.Error(req.errMsg) } } s.logger.Debug("Syncer done") return case req := <-s.requestsCh: requests = append(requests, req) case <-time.After(s.interval): var temp []*syncRequest for _, req := range requests { if req.deadline.Before(time.Now()) { continue // drop stale request } if err := req.fn(); err != nil { temp = append(temp, req) } } requests = temp } } }() } ================================================ FILE: syncer_test.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package asynq import ( "context" "fmt" "sync" "testing" "time" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/rdb" h "github.com/hibiken/asynq/internal/testutil" ) func TestSyncer(t *testing.T) { inProgress := []*base.TaskMessage{ h.NewTaskMessage("send_email", nil), h.NewTaskMessage("reindex", nil), h.NewTaskMessage("gen_thumbnail", nil), } r := setup(t) defer r.Close() rdbClient := rdb.NewRDB(r) h.SeedActiveQueue(t, r, inProgress, base.DefaultQueueName) const interval = time.Second syncRequestCh := make(chan *syncRequest) syncer := newSyncer(syncerParams{ logger: testLogger, requestsCh: syncRequestCh, interval: interval, }) var wg sync.WaitGroup syncer.start(&wg) defer syncer.shutdown() for _, msg := range inProgress { m := msg syncRequestCh <- &syncRequest{ fn: func() error { return rdbClient.Done(context.Background(), m) }, deadline: time.Now().Add(5 * time.Minute), } } time.Sleep(2 * interval) // ensure that syncer runs at least once gotActive := h.GetActiveMessages(t, r, base.DefaultQueueName) if l := len(gotActive); l != 0 { t.Errorf("%q has length %d; want 0", base.ActiveKey(base.DefaultQueueName), l) } } func TestSyncerRetry(t *testing.T) { const interval = time.Second syncRequestCh := make(chan *syncRequest) syncer := newSyncer(syncerParams{ logger: testLogger, requestsCh: syncRequestCh, interval: interval, }) var wg sync.WaitGroup syncer.start(&wg) defer syncer.shutdown() var ( mu sync.Mutex counter int ) // Increment the counter for each call. // Initial call will fail and second call will succeed. requestFunc := func() error { mu.Lock() defer mu.Unlock() if counter == 0 { counter++ return fmt.Errorf("zero") } counter++ return nil } syncRequestCh <- &syncRequest{ fn: requestFunc, errMsg: "error", deadline: time.Now().Add(5 * time.Minute), } // allow syncer to retry time.Sleep(3 * interval) mu.Lock() if counter != 2 { t.Errorf("counter = %d, want 2", counter) } mu.Unlock() } func TestSyncerDropsStaleRequests(t *testing.T) { const interval = time.Second syncRequestCh := make(chan *syncRequest) syncer := newSyncer(syncerParams{ logger: testLogger, requestsCh: syncRequestCh, interval: interval, }) var wg sync.WaitGroup syncer.start(&wg) var ( mu sync.Mutex n int // number of times request has been processed ) for i := 0; i < 10; i++ { syncRequestCh <- &syncRequest{ fn: func() error { mu.Lock() n++ mu.Unlock() return nil }, deadline: time.Now().Add(time.Duration(-i) * time.Second), // already exceeded deadline } } time.Sleep(2 * interval) // ensure that syncer runs at least once syncer.shutdown() mu.Lock() if n != 0 { t.Errorf("requests has been processed %d times, want 0", n) } mu.Unlock() } ================================================ FILE: tools/asynq/README.md ================================================ # Asynq CLI Asynq CLI is a command line tool to monitor the queues and tasks managed by `asynq` package. ## Table of Contents - [Installation](#installation) - [Usage](#usage) - [Config File](#config-file) ## Installation In order to use the tool, compile it using the following command: go install github.com/hibiken/asynq/tools/asynq@latest This will create the asynq executable under your `$GOPATH/bin` directory. ## Usage ### Commands To view details on any command, use `asynq help `. - `asynq dash` - `asynq stats` - `asynq queue [ls inspect history rm pause unpause]` - `asynq task [ls cancel delete archive run deleteall archiveall runall]` - `asynq server [ls]` ### Global flags Asynq CLI needs to connect to a redis-server to inspect the state of queues and tasks. Use flags to specify the options to connect to the redis-server used by your application. To connect to a redis cluster, pass `--cluster` and `--cluster_addrs` flags. By default, CLI will try to connect to a redis server running at `localhost:6379`. ``` --config string config file to set flag defaut values (default is $HOME/.asynq.yaml) -n, --db int redis database number (default is 0) -h, --help help for asynq -U, --username string username to use when connecting to redis server -p, --password string password to use when connecting to redis server -u, --uri string redis server URI (default "127.0.0.1:6379") --cluster connect to redis cluster --cluster_addrs string list of comma-separated redis server addresses ``` ## Config File You can use a config file to set default values for the flags. By default, `asynq` will try to read config file located in `$HOME/.asynq.(yml|json)`. You can specify the file location via `--config` flag. Config file example: ```yaml uri: 127.0.0.1:6379 db: 2 password: mypassword ``` This will set the default values for `--uri`, `--db`, and `--password` flags. ================================================ FILE: tools/asynq/cmd/cron.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package cmd import ( "fmt" "io" "os" "sort" "time" "github.com/MakeNowJust/heredoc/v2" "github.com/hibiken/asynq" "github.com/spf13/cobra" ) func init() { rootCmd.AddCommand(cronCmd) cronCmd.AddCommand(cronListCmd) cronCmd.AddCommand(cronHistoryCmd) cronHistoryCmd.Flags().Int("page", 1, "page number") cronHistoryCmd.Flags().Int("size", 30, "page size") } var cronCmd = &cobra.Command{ Use: "cron [flags]", Short: "Manage cron", Example: heredoc.Doc(` $ asynq cron ls $ asynq cron history 7837f142-6337-4217-9276-8f27281b67d1`), } var cronListCmd = &cobra.Command{ Use: "list", Aliases: []string{"ls"}, Short: "List cron entries", Run: cronList, } var cronHistoryCmd = &cobra.Command{ Use: "history [...]", Short: "Show history of each cron tasks", Args: cobra.MinimumNArgs(1), Run: cronHistory, Example: heredoc.Doc(` $ asynq cron history 7837f142-6337-4217-9276-8f27281b67d1 $ asynq cron history 7837f142-6337-4217-9276-8f27281b67d1 bf6a8594-cd03-4968-b36a-8572c5e160dd $ asynq cron history 7837f142-6337-4217-9276-8f27281b67d1 --size=100 $ asynq cron history 7837f142-6337-4217-9276-8f27281b67d1 --page=2`), } func cronList(cmd *cobra.Command, args []string) { inspector := createInspector() entries, err := inspector.SchedulerEntries() if err != nil { fmt.Println(err) os.Exit(1) } if len(entries) == 0 { fmt.Println("No scheduler entries") return } // Sort entries by spec. sort.Slice(entries, func(i, j int) bool { x, y := entries[i], entries[j] return x.Spec < y.Spec }) cols := []string{"EntryID", "Spec", "Type", "Payload", "Options", "Next", "Prev"} printRows := func(w io.Writer, tmpl string) { for _, e := range entries { fmt.Fprintf(w, tmpl, e.ID, e.Spec, e.Task.Type(), sprintBytes(e.Task.Payload()), e.Opts, nextEnqueue(e.Next), prevEnqueue(e.Prev)) } } printTable(cols, printRows) } // Returns a string describing when the next enqueue will happen. func nextEnqueue(nextEnqueueAt time.Time) string { d := nextEnqueueAt.Sub(time.Now()).Round(time.Second) if d < 0 { return "Now" } return fmt.Sprintf("In %v", d) } // Returns a string describing when the previous enqueue was. func prevEnqueue(prevEnqueuedAt time.Time) string { if prevEnqueuedAt.IsZero() { return "N/A" } return fmt.Sprintf("%v ago", time.Since(prevEnqueuedAt).Round(time.Second)) } func cronHistory(cmd *cobra.Command, args []string) { pageNum, err := cmd.Flags().GetInt("page") if err != nil { fmt.Println(err) os.Exit(1) } pageSize, err := cmd.Flags().GetInt("size") if err != nil { fmt.Println(err) os.Exit(1) } inspector := createInspector() for i, entryID := range args { if i > 0 { fmt.Printf("\n%s\n", separator) } fmt.Println() fmt.Printf("Entry: %s\n\n", entryID) events, err := inspector.ListSchedulerEnqueueEvents( entryID, asynq.PageSize(pageSize), asynq.Page(pageNum)) if err != nil { fmt.Printf("error: %v\n", err) continue } if len(events) == 0 { fmt.Printf("No scheduler enqueue events found for entry: %s\n", entryID) continue } cols := []string{"TaskID", "EnqueuedAt"} printRows := func(w io.Writer, tmpl string) { for _, e := range events { fmt.Fprintf(w, tmpl, e.TaskID, e.EnqueuedAt) } } printTable(cols, printRows) } } ================================================ FILE: tools/asynq/cmd/dash/dash.go ================================================ // Copyright 2022 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package dash import ( "errors" "fmt" "os" "strings" "time" "github.com/gdamore/tcell/v2" "github.com/hibiken/asynq" ) // viewType is an enum for dashboard views. type viewType int const ( viewTypeQueues viewType = iota viewTypeQueueDetails viewTypeHelp ) // State holds dashboard state. type State struct { queues []*asynq.QueueInfo tasks []*asynq.TaskInfo groups []*asynq.GroupInfo err error // Note: index zero corresponds to the table header; index=1 correctponds to the first element queueTableRowIdx int // highlighted row in queue table taskTableRowIdx int // highlighted row in task table groupTableRowIdx int // highlighted row in group table taskState asynq.TaskState // highlighted task state in queue details view taskID string // selected task ID selectedQueue *asynq.QueueInfo // queue shown on queue details view selectedGroup *asynq.GroupInfo selectedTask *asynq.TaskInfo pageNum int // pagination page number view viewType // current view type prevView viewType // to support "go back" } func (s *State) DebugString() string { var b strings.Builder b.WriteString(fmt.Sprintf("len(queues)=%d ", len(s.queues))) b.WriteString(fmt.Sprintf("len(tasks)=%d ", len(s.tasks))) b.WriteString(fmt.Sprintf("len(groups)=%d ", len(s.groups))) b.WriteString(fmt.Sprintf("err=%v ", s.err)) if s.taskState != 0 { b.WriteString(fmt.Sprintf("taskState=%s ", s.taskState.String())) } else { b.WriteString(fmt.Sprintf("taskState=0")) } b.WriteString(fmt.Sprintf("taskID=%s ", s.taskID)) b.WriteString(fmt.Sprintf("queueTableRowIdx=%d ", s.queueTableRowIdx)) b.WriteString(fmt.Sprintf("taskTableRowIdx=%d ", s.taskTableRowIdx)) b.WriteString(fmt.Sprintf("groupTableRowIdx=%d ", s.groupTableRowIdx)) if s.selectedQueue != nil { b.WriteString(fmt.Sprintf("selectedQueue={Queue:%s} ", s.selectedQueue.Queue)) } else { b.WriteString("selectedQueue=nil ") } if s.selectedGroup != nil { b.WriteString(fmt.Sprintf("selectedGroup={Group:%s} ", s.selectedGroup.Group)) } else { b.WriteString("selectedGroup=nil ") } if s.selectedTask != nil { b.WriteString(fmt.Sprintf("selectedTask={ID:%s} ", s.selectedTask.ID)) } else { b.WriteString("selectedTask=nil ") } b.WriteString(fmt.Sprintf("pageNum=%d", s.pageNum)) return b.String() } type Options struct { DebugMode bool PollInterval time.Duration RedisConnOpt asynq.RedisConnOpt } func Run(opts Options) { s, err := tcell.NewScreen() if err != nil { fmt.Printf("failed to create a screen: %v\n", err) os.Exit(1) } if err := s.Init(); err != nil { fmt.Printf("failed to initialize screen: %v\n", err) os.Exit(1) } s.SetStyle(baseStyle) // set default text style var ( state = State{} // confined in this goroutine only; DO NOT SHARE inspector = asynq.NewInspector(opts.RedisConnOpt) ticker = time.NewTicker(opts.PollInterval) eventCh = make(chan tcell.Event) done = make(chan struct{}) // channels to send/receive data fetched asynchronously errorCh = make(chan error) queueCh = make(chan *asynq.QueueInfo) taskCh = make(chan *asynq.TaskInfo) queuesCh = make(chan []*asynq.QueueInfo) groupsCh = make(chan []*asynq.GroupInfo) tasksCh = make(chan []*asynq.TaskInfo) ) defer ticker.Stop() f := dataFetcher{ inspector, opts, s, errorCh, queueCh, taskCh, queuesCh, groupsCh, tasksCh, } d := dashDrawer{ s, opts, } h := keyEventHandler{ s: s, fetcher: &f, drawer: &d, state: &state, done: done, ticker: ticker, pollInterval: opts.PollInterval, } go fetchQueues(inspector, queuesCh, errorCh, opts) go s.ChannelEvents(eventCh, done) // TODO: Double check that we are not leaking goroutine with this one. d.Draw(&state) // draw initial screen for { // Update screen s.Show() select { case ev := <-eventCh: // Process event switch ev := ev.(type) { case *tcell.EventResize: s.Sync() case *tcell.EventKey: h.HandleKeyEvent(ev) } case <-ticker.C: f.Fetch(&state) case queues := <-queuesCh: state.queues = queues state.err = nil if len(queues) < state.queueTableRowIdx { state.queueTableRowIdx = len(queues) } d.Draw(&state) case q := <-queueCh: state.selectedQueue = q state.err = nil d.Draw(&state) case groups := <-groupsCh: state.groups = groups state.err = nil if len(groups) < state.groupTableRowIdx { state.groupTableRowIdx = len(groups) } d.Draw(&state) case tasks := <-tasksCh: state.tasks = tasks state.err = nil if len(tasks) < state.taskTableRowIdx { state.taskTableRowIdx = len(tasks) } d.Draw(&state) case t := <-taskCh: state.selectedTask = t state.err = nil d.Draw(&state) case err := <-errorCh: if errors.Is(err, asynq.ErrTaskNotFound) { state.selectedTask = nil } else { state.err = err } d.Draw(&state) } } } ================================================ FILE: tools/asynq/cmd/dash/draw.go ================================================ // Copyright 2022 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package dash import ( "fmt" "math" "strconv" "strings" "time" "unicode" "unicode/utf8" "github.com/gdamore/tcell/v2" "github.com/hibiken/asynq" "github.com/mattn/go-runewidth" ) var ( baseStyle = tcell.StyleDefault.Background(tcell.ColorReset).Foreground(tcell.ColorReset) labelStyle = baseStyle.Foreground(tcell.ColorLightGray) // styles for bar graph activeStyle = baseStyle.Foreground(tcell.ColorBlue) pendingStyle = baseStyle.Foreground(tcell.ColorGreen) aggregatingStyle = baseStyle.Foreground(tcell.ColorLightGreen) scheduledStyle = baseStyle.Foreground(tcell.ColorYellow) retryStyle = baseStyle.Foreground(tcell.ColorPink) archivedStyle = baseStyle.Foreground(tcell.ColorPurple) completedStyle = baseStyle.Foreground(tcell.ColorDarkGreen) ) // drawer draws UI with the given state. type drawer interface { Draw(state *State) } type dashDrawer struct { s tcell.Screen opts Options } func (dd *dashDrawer) Draw(state *State) { s, opts := dd.s, dd.opts s.Clear() // Simulate data update on every render d := NewScreenDrawer(s) switch state.view { case viewTypeQueues: d.Println("=== Queues ===", baseStyle.Bold(true)) d.NL() drawQueueSizeGraphs(d, state) d.NL() drawQueueTable(d, baseStyle, state) case viewTypeQueueDetails: d.Println("=== Queue Summary ===", baseStyle.Bold(true)) d.NL() drawQueueSummary(d, state) d.NL() d.NL() d.Println("=== Tasks ===", baseStyle.Bold(true)) d.NL() drawTaskStateBreakdown(d, baseStyle, state) d.NL() drawTaskTable(d, state) drawTaskModal(d, state) case viewTypeHelp: drawHelp(d) } d.GoToBottom() if opts.DebugMode { drawDebugInfo(d, state) } else { drawFooter(d, state) } } func drawQueueSizeGraphs(d *ScreenDrawer, state *State) { var qnames []string var qsizes []string // queue size in strings maxSize := 1 // not zero to avoid division by zero for _, q := range state.queues { qnames = append(qnames, q.Queue) qsizes = append(qsizes, strconv.Itoa(q.Size)) if q.Size > maxSize { maxSize = q.Size } } qnameWidth := maxwidth(qnames) qsizeWidth := maxwidth(qsizes) // Calculate the multipler to scale the graph screenWidth, _ := d.Screen().Size() graphMaxWidth := screenWidth - (qnameWidth + qsizeWidth + 3) // | multipiler := 1.0 if graphMaxWidth < maxSize { multipiler = float64(graphMaxWidth) / float64(maxSize) } const tick = '▇' for _, q := range state.queues { d.Print(q.Queue, baseStyle) d.Print(strings.Repeat(" ", qnameWidth-runewidth.StringWidth(q.Queue)+1), baseStyle) // padding between qname and graph d.Print("|", baseStyle) d.Print(strings.Repeat(string(tick), int(math.Floor(float64(q.Active)*multipiler))), activeStyle) d.Print(strings.Repeat(string(tick), int(math.Floor(float64(q.Pending)*multipiler))), pendingStyle) d.Print(strings.Repeat(string(tick), int(math.Floor(float64(q.Aggregating)*multipiler))), aggregatingStyle) d.Print(strings.Repeat(string(tick), int(math.Floor(float64(q.Scheduled)*multipiler))), scheduledStyle) d.Print(strings.Repeat(string(tick), int(math.Floor(float64(q.Retry)*multipiler))), retryStyle) d.Print(strings.Repeat(string(tick), int(math.Floor(float64(q.Archived)*multipiler))), archivedStyle) d.Print(strings.Repeat(string(tick), int(math.Floor(float64(q.Completed)*multipiler))), completedStyle) d.Print(fmt.Sprintf(" %d", q.Size), baseStyle) d.NL() } d.NL() d.Print("active=", baseStyle) d.Print(string(tick), activeStyle) d.Print(" pending=", baseStyle) d.Print(string(tick), pendingStyle) d.Print(" aggregating=", baseStyle) d.Print(string(tick), aggregatingStyle) d.Print(" scheduled=", baseStyle) d.Print(string(tick), scheduledStyle) d.Print(" retry=", baseStyle) d.Print(string(tick), retryStyle) d.Print(" archived=", baseStyle) d.Print(string(tick), archivedStyle) d.Print(" completed=", baseStyle) d.Print(string(tick), completedStyle) d.NL() } func drawFooter(d *ScreenDrawer, state *State) { if state.err != nil { style := baseStyle.Background(tcell.ColorDarkRed) d.Print(state.err.Error(), style) d.FillLine(' ', style) return } style := baseStyle.Background(tcell.ColorDarkSlateGray).Foreground(tcell.ColorWhite) switch state.view { case viewTypeHelp: d.Print(": GoBack", style) default: d.Print(": Help : Exit ", style) } d.FillLine(' ', style) } // returns the maximum width from the given list of names func maxwidth(names []string) int { max := 0 for _, s := range names { if w := runewidth.StringWidth(s); w > max { max = w } } return max } // rpad adds padding to the right of a string. func rpad(s string, padding int) string { tmpl := fmt.Sprintf("%%-%ds ", padding) return fmt.Sprintf(tmpl, s) } // lpad adds padding to the left of a string. func lpad(s string, padding int) string { tmpl := fmt.Sprintf("%%%ds ", padding) return fmt.Sprintf(tmpl, s) } // byteCount converts the given bytes into human readable string func byteCount(b int64) string { const unit = 1000 if b < unit { return fmt.Sprintf("%d B", b) } div, exp := int64(unit), 0 for n := b / unit; n >= unit; n /= unit { div *= unit exp++ } return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "kMGTPE"[exp]) } var queueColumnConfigs = []*columnConfig[*asynq.QueueInfo]{ {"Queue", alignLeft, func(q *asynq.QueueInfo) string { return q.Queue }}, {"State", alignLeft, func(q *asynq.QueueInfo) string { return formatQueueState(q) }}, {"Size", alignRight, func(q *asynq.QueueInfo) string { return strconv.Itoa(q.Size) }}, {"Latency", alignRight, func(q *asynq.QueueInfo) string { return q.Latency.Round(time.Second).String() }}, {"MemoryUsage", alignRight, func(q *asynq.QueueInfo) string { return byteCount(q.MemoryUsage) }}, {"Processed", alignRight, func(q *asynq.QueueInfo) string { return strconv.Itoa(q.Processed) }}, {"Failed", alignRight, func(q *asynq.QueueInfo) string { return strconv.Itoa(q.Failed) }}, {"ErrorRate", alignRight, func(q *asynq.QueueInfo) string { return formatErrorRate(q.Processed, q.Failed) }}, } func formatQueueState(q *asynq.QueueInfo) string { if q.Paused { return "PAUSED" } return "RUN" } func formatErrorRate(processed, failed int) string { if processed == 0 { return "-" } return fmt.Sprintf("%.2f", float64(failed)/float64(processed)) } func formatNextProcessTime(t time.Time) string { now := time.Now() if t.Before(now) { return "now" } return fmt.Sprintf("in %v", (t.Sub(now).Round(time.Second))) } func formatPastTime(t time.Time) string { now := time.Now() if t.After(now) || t.Equal(now) { return "just now" } return fmt.Sprintf("%v ago", time.Since(t).Round(time.Second)) } func drawQueueTable(d *ScreenDrawer, style tcell.Style, state *State) { drawTable(d, style, queueColumnConfigs, state.queues, state.queueTableRowIdx-1) } func drawQueueSummary(d *ScreenDrawer, state *State) { q := state.selectedQueue if q == nil { d.Println("ERROR: Press q to go back", baseStyle) return } d.Print("Name ", labelStyle) d.Println(q.Queue, baseStyle) d.Print("Size ", labelStyle) d.Println(strconv.Itoa(q.Size), baseStyle) d.Print("Latency ", labelStyle) d.Println(q.Latency.Round(time.Second).String(), baseStyle) d.Print("MemUsage ", labelStyle) d.Println(byteCount(q.MemoryUsage), baseStyle) } // Returns the max number of groups that can be displayed. func groupPageSize(s tcell.Screen) int { _, h := s.Size() return h - 16 // height - (# of rows used) } // Returns the number of tasks to fetch. func taskPageSize(s tcell.Screen) int { _, h := s.Size() return h - 15 // height - (# of rows used) } func shouldShowGroupTable(state *State) bool { return state.taskState == asynq.TaskStateAggregating && state.selectedGroup == nil } func getTaskTableColumnConfig(taskState asynq.TaskState) []*columnConfig[*asynq.TaskInfo] { switch taskState { case asynq.TaskStateActive: return activeTaskTableColumns case asynq.TaskStatePending: return pendingTaskTableColumns case asynq.TaskStateAggregating: return aggregatingTaskTableColumns case asynq.TaskStateScheduled: return scheduledTaskTableColumns case asynq.TaskStateRetry: return retryTaskTableColumns case asynq.TaskStateArchived: return archivedTaskTableColumns case asynq.TaskStateCompleted: return completedTaskTableColumns } panic("unknown task state") } var activeTaskTableColumns = []*columnConfig[*asynq.TaskInfo]{ {"ID", alignLeft, func(t *asynq.TaskInfo) string { return t.ID }}, {"Type", alignLeft, func(t *asynq.TaskInfo) string { return t.Type }}, {"Retried", alignRight, func(t *asynq.TaskInfo) string { return strconv.Itoa(t.Retried) }}, {"Max Retry", alignRight, func(t *asynq.TaskInfo) string { return strconv.Itoa(t.MaxRetry) }}, {"Payload", alignLeft, func(t *asynq.TaskInfo) string { return formatByteSlice(t.Payload) }}, } var pendingTaskTableColumns = []*columnConfig[*asynq.TaskInfo]{ {"ID", alignLeft, func(t *asynq.TaskInfo) string { return t.ID }}, {"Type", alignLeft, func(t *asynq.TaskInfo) string { return t.Type }}, {"Retried", alignRight, func(t *asynq.TaskInfo) string { return strconv.Itoa(t.Retried) }}, {"Max Retry", alignRight, func(t *asynq.TaskInfo) string { return strconv.Itoa(t.MaxRetry) }}, {"Payload", alignLeft, func(t *asynq.TaskInfo) string { return formatByteSlice(t.Payload) }}, } var aggregatingTaskTableColumns = []*columnConfig[*asynq.TaskInfo]{ {"ID", alignLeft, func(t *asynq.TaskInfo) string { return t.ID }}, {"Type", alignLeft, func(t *asynq.TaskInfo) string { return t.Type }}, {"Payload", alignLeft, func(t *asynq.TaskInfo) string { return formatByteSlice(t.Payload) }}, {"Group", alignLeft, func(t *asynq.TaskInfo) string { return t.Group }}, } var scheduledTaskTableColumns = []*columnConfig[*asynq.TaskInfo]{ {"ID", alignLeft, func(t *asynq.TaskInfo) string { return t.ID }}, {"Type", alignLeft, func(t *asynq.TaskInfo) string { return t.Type }}, {"Next Process Time", alignLeft, func(t *asynq.TaskInfo) string { return formatNextProcessTime(t.NextProcessAt) }}, {"Payload", alignLeft, func(t *asynq.TaskInfo) string { return formatByteSlice(t.Payload) }}, } var retryTaskTableColumns = []*columnConfig[*asynq.TaskInfo]{ {"ID", alignLeft, func(t *asynq.TaskInfo) string { return t.ID }}, {"Type", alignLeft, func(t *asynq.TaskInfo) string { return t.Type }}, {"Retry", alignRight, func(t *asynq.TaskInfo) string { return fmt.Sprintf("%d/%d", t.Retried, t.MaxRetry) }}, {"Last Failure", alignLeft, func(t *asynq.TaskInfo) string { return t.LastErr }}, {"Last Failure Time", alignLeft, func(t *asynq.TaskInfo) string { return formatPastTime(t.LastFailedAt) }}, {"Next Process Time", alignLeft, func(t *asynq.TaskInfo) string { return formatNextProcessTime(t.NextProcessAt) }}, {"Payload", alignLeft, func(t *asynq.TaskInfo) string { return formatByteSlice(t.Payload) }}, } var archivedTaskTableColumns = []*columnConfig[*asynq.TaskInfo]{ {"ID", alignLeft, func(t *asynq.TaskInfo) string { return t.ID }}, {"Type", alignLeft, func(t *asynq.TaskInfo) string { return t.Type }}, {"Retry", alignRight, func(t *asynq.TaskInfo) string { return fmt.Sprintf("%d/%d", t.Retried, t.MaxRetry) }}, {"Last Failure", alignLeft, func(t *asynq.TaskInfo) string { return t.LastErr }}, {"Last Failure Time", alignLeft, func(t *asynq.TaskInfo) string { return formatPastTime(t.LastFailedAt) }}, {"Payload", alignLeft, func(t *asynq.TaskInfo) string { return formatByteSlice(t.Payload) }}, } var completedTaskTableColumns = []*columnConfig[*asynq.TaskInfo]{ {"ID", alignLeft, func(t *asynq.TaskInfo) string { return t.ID }}, {"Type", alignLeft, func(t *asynq.TaskInfo) string { return t.Type }}, {"Completion Time", alignLeft, func(t *asynq.TaskInfo) string { return formatPastTime(t.CompletedAt) }}, {"Payload", alignLeft, func(t *asynq.TaskInfo) string { return formatByteSlice(t.Payload) }}, {"Result", alignLeft, func(t *asynq.TaskInfo) string { return formatByteSlice(t.Result) }}, } func drawTaskTable(d *ScreenDrawer, state *State) { if shouldShowGroupTable(state) { drawGroupTable(d, state) return } if len(state.tasks) == 0 { return // print nothing } drawTable(d, baseStyle, getTaskTableColumnConfig(state.taskState), state.tasks, state.taskTableRowIdx-1) // Pagination pageSize := taskPageSize(d.Screen()) totalCount := getTaskCount(state.selectedQueue, state.taskState) if state.taskState == asynq.TaskStateAggregating { // aggregating tasks are scoped to each group when shown in the table. totalCount = state.selectedGroup.Size } if pageSize < totalCount { start := (state.pageNum-1)*pageSize + 1 end := start + len(state.tasks) - 1 paginationStyle := baseStyle.Foreground(tcell.ColorLightGray) d.Print(fmt.Sprintf("Showing %d-%d out of %d", start, end, totalCount), paginationStyle) if isNextTaskPageAvailable(d.Screen(), state) { d.Print(" n=NextPage", paginationStyle) } if state.pageNum > 1 { d.Print(" p=PrevPage", paginationStyle) } d.FillLine(' ', paginationStyle) } } func isNextTaskPageAvailable(s tcell.Screen, state *State) bool { totalCount := getTaskCount(state.selectedQueue, state.taskState) end := (state.pageNum-1)*taskPageSize(s) + len(state.tasks) return end < totalCount } func drawGroupTable(d *ScreenDrawer, state *State) { if len(state.groups) == 0 { return // print nothing } d.Println("<<< Select group >>>", baseStyle) colConfigs := []*columnConfig[*asynq.GroupInfo]{ {"Name", alignLeft, func(g *asynq.GroupInfo) string { return g.Group }}, {"Size", alignRight, func(g *asynq.GroupInfo) string { return strconv.Itoa(g.Size) }}, } // pagination pageSize := groupPageSize(d.Screen()) total := len(state.groups) start := (state.pageNum - 1) * pageSize end := min(start+pageSize, total) drawTable(d, baseStyle, colConfigs, state.groups[start:end], state.groupTableRowIdx-1) if pageSize < total { d.Print(fmt.Sprintf("Showing %d-%d out of %d", start+1, end, total), labelStyle) if end < total { d.Print(" n=NextPage", labelStyle) } if start > 0 { d.Print(" p=PrevPage", labelStyle) } } d.FillLine(' ', labelStyle) } type number interface { int | int64 | float64 } // min returns the smaller of x and y. if x==y, returns x func min[V number](x, y V) V { if x > y { return y } return x } // Define the order of states to show var taskStates = []asynq.TaskState{ asynq.TaskStateActive, asynq.TaskStatePending, asynq.TaskStateAggregating, asynq.TaskStateScheduled, asynq.TaskStateRetry, asynq.TaskStateArchived, asynq.TaskStateCompleted, } func nextTaskState(current asynq.TaskState) asynq.TaskState { for i, ts := range taskStates { if current == ts { if i == len(taskStates)-1 { return taskStates[0] } else { return taskStates[i+1] } } } panic("unknown task state") } func prevTaskState(current asynq.TaskState) asynq.TaskState { for i, ts := range taskStates { if current == ts { if i == 0 { return taskStates[len(taskStates)-1] } else { return taskStates[i-1] } } } panic("unknown task state") } func getTaskCount(queue *asynq.QueueInfo, taskState asynq.TaskState) int { switch taskState { case asynq.TaskStateActive: return queue.Active case asynq.TaskStatePending: return queue.Pending case asynq.TaskStateAggregating: return queue.Aggregating case asynq.TaskStateScheduled: return queue.Scheduled case asynq.TaskStateRetry: return queue.Retry case asynq.TaskStateArchived: return queue.Archived case asynq.TaskStateCompleted: return queue.Completed } panic("unkonwn task state") } func drawTaskStateBreakdown(d *ScreenDrawer, style tcell.Style, state *State) { const pad = " " // padding between states for _, ts := range taskStates { s := style if state.taskState == ts { s = s.Bold(true).Underline(true) } d.Print(fmt.Sprintf("%s:%d", strings.Title(ts.String()), getTaskCount(state.selectedQueue, ts)), s) d.Print(pad, style) } d.NL() } func drawTaskModal(d *ScreenDrawer, state *State) { if state.taskID == "" { return } task := state.selectedTask if task == nil { // task no longer found fns := []func(d *modalRowDrawer){ func(d *modalRowDrawer) { d.Print("=== Task Info ===", baseStyle.Bold(true)) }, func(d *modalRowDrawer) { d.Print("", baseStyle) }, func(d *modalRowDrawer) { d.Print(fmt.Sprintf("Task %q no longer exists", state.taskID), baseStyle) }, } withModal(d, fns) return } fns := []func(d *modalRowDrawer){ func(d *modalRowDrawer) { d.Print("=== Task Info ===", baseStyle.Bold(true)) }, func(d *modalRowDrawer) { d.Print("", baseStyle) }, func(d *modalRowDrawer) { d.Print("ID: ", labelStyle) d.Print(task.ID, baseStyle) }, func(d *modalRowDrawer) { d.Print("Type: ", labelStyle) d.Print(task.Type, baseStyle) }, func(d *modalRowDrawer) { d.Print("State: ", labelStyle) d.Print(task.State.String(), baseStyle) }, func(d *modalRowDrawer) { d.Print("Queue: ", labelStyle) d.Print(task.Queue, baseStyle) }, func(d *modalRowDrawer) { d.Print("Retry: ", labelStyle) d.Print(fmt.Sprintf("%d/%d", task.Retried, task.MaxRetry), baseStyle) }, } if task.LastErr != "" { fns = append(fns, func(d *modalRowDrawer) { d.Print("Last Failure: ", labelStyle) d.Print(task.LastErr, baseStyle) }) fns = append(fns, func(d *modalRowDrawer) { d.Print("Last Failure Time: ", labelStyle) d.Print(fmt.Sprintf("%v (%s)", task.LastFailedAt, formatPastTime(task.LastFailedAt)), baseStyle) }) } if !task.NextProcessAt.IsZero() { fns = append(fns, func(d *modalRowDrawer) { d.Print("Next Process Time: ", labelStyle) d.Print(fmt.Sprintf("%v (%s)", task.NextProcessAt, formatNextProcessTime(task.NextProcessAt)), baseStyle) }) } if !task.CompletedAt.IsZero() { fns = append(fns, func(d *modalRowDrawer) { d.Print("Completion Time: ", labelStyle) d.Print(fmt.Sprintf("%v (%s)", task.CompletedAt, formatPastTime(task.CompletedAt)), baseStyle) }) } fns = append(fns, func(d *modalRowDrawer) { d.Print("Payload: ", labelStyle) d.Print(formatByteSlice(task.Payload), baseStyle) }) if task.Result != nil { fns = append(fns, func(d *modalRowDrawer) { d.Print("Result: ", labelStyle) d.Print(formatByteSlice(task.Result), baseStyle) }) } withModal(d, fns) } // Reports whether the given byte slice is printable (i.e. human readable) func isPrintable(data []byte) bool { if !utf8.Valid(data) { return false } isAllSpace := true for _, r := range string(data) { if !unicode.IsGraphic(r) { return false } if !unicode.IsSpace(r) { isAllSpace = false } } return !isAllSpace } func formatByteSlice(data []byte) string { if data == nil { return "" } if !isPrintable(data) { return "" } return strings.ReplaceAll(string(data), "\n", " ") } type modalRowDrawer struct { d *ScreenDrawer width int // current width occupied by content maxWidth int } // Note: s should not include newline func (d *modalRowDrawer) Print(s string, style tcell.Style) { if d.width >= d.maxWidth { return // no longer write to this row } if d.width+runewidth.StringWidth(s) > d.maxWidth { s = truncate(s, d.maxWidth-d.width) } d.d.Print(s, style) } // withModal draws a modal with the given functions row by row. func withModal(d *ScreenDrawer, rowPrintFns []func(d *modalRowDrawer)) { w, h := d.Screen().Size() var ( modalWidth = int(math.Floor(float64(w) * 0.6)) modalHeight = int(math.Floor(float64(h) * 0.6)) rowOffset = int(math.Floor(float64(h) * 0.2)) // 20% from the top colOffset = int(math.Floor(float64(w) * 0.2)) // 20% from the left ) if modalHeight < 3 { return // no content can be shown } d.Goto(colOffset, rowOffset) d.Print(string(tcell.RuneULCorner), baseStyle) d.Print(strings.Repeat(string(tcell.RuneHLine), modalWidth-2), baseStyle) d.Print(string(tcell.RuneURCorner), baseStyle) d.NL() rowDrawer := modalRowDrawer{ d: d, width: 0, maxWidth: modalWidth - 4, /* borders + paddings */ } for i := 1; i < modalHeight-1; i++ { d.Goto(colOffset, rowOffset+i) d.Print(fmt.Sprintf("%c ", tcell.RuneVLine), baseStyle) if i <= len(rowPrintFns) { rowPrintFns[i-1](&rowDrawer) } d.FillUntil(' ', baseStyle, colOffset+modalWidth-2) d.Print(fmt.Sprintf(" %c", tcell.RuneVLine), baseStyle) d.NL() } d.Goto(colOffset, rowOffset+modalHeight-1) d.Print(string(tcell.RuneLLCorner), baseStyle) d.Print(strings.Repeat(string(tcell.RuneHLine), modalWidth-2), baseStyle) d.Print(string(tcell.RuneLRCorner), baseStyle) d.NL() } func adjustWidth(s string, width int) string { sw := runewidth.StringWidth(s) if sw > width { return truncate(s, width) } var b strings.Builder b.WriteString(s) b.WriteString(strings.Repeat(" ", width-sw)) return b.String() } // truncates s if s exceeds max length. func truncate(s string, max int) string { if runewidth.StringWidth(s) <= max { return s } return string([]rune(s)[:max-1]) + "…" } func drawDebugInfo(d *ScreenDrawer, state *State) { d.Println(state.DebugString(), baseStyle) } func drawHelp(d *ScreenDrawer) { keyStyle := labelStyle.Bold(true) withModal(d, []func(*modalRowDrawer){ func(d *modalRowDrawer) { d.Print("=== Help ===", baseStyle.Bold(true)) }, func(d *modalRowDrawer) { d.Print("", baseStyle) }, func(d *modalRowDrawer) { d.Print("", keyStyle) d.Print(" to select", baseStyle) }, func(d *modalRowDrawer) { d.Print("", keyStyle) d.Print(" or ", baseStyle) d.Print("", keyStyle) d.Print(" to go back", baseStyle) }, func(d *modalRowDrawer) { d.Print("", keyStyle) d.Print(" or ", baseStyle) d.Print("", keyStyle) d.Print(" to move up", baseStyle) }, func(d *modalRowDrawer) { d.Print("", keyStyle) d.Print(" or ", baseStyle) d.Print("", keyStyle) d.Print(" to move down", baseStyle) }, func(d *modalRowDrawer) { d.Print("", keyStyle) d.Print(" or ", baseStyle) d.Print("", keyStyle) d.Print(" to move left", baseStyle) }, func(d *modalRowDrawer) { d.Print("", keyStyle) d.Print(" or ", baseStyle) d.Print("", keyStyle) d.Print(" to move right", baseStyle) }, func(d *modalRowDrawer) { d.Print("", keyStyle) d.Print(" to quit", baseStyle) }, }) } ================================================ FILE: tools/asynq/cmd/dash/draw_test.go ================================================ // Copyright 2022 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package dash import "testing" func TestTruncate(t *testing.T) { tests := []struct { s string max int want string }{ { s: "hello world!", max: 15, want: "hello world!", }, { s: "hello world!", max: 6, want: "hello…", }, } for _, tc := range tests { got := truncate(tc.s, tc.max) if tc.want != got { t.Errorf("truncate(%q, %d) = %q, want %q", tc.s, tc.max, got, tc.want) } } } ================================================ FILE: tools/asynq/cmd/dash/fetch.go ================================================ // Copyright 2022 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package dash import ( "sort" "github.com/gdamore/tcell/v2" "github.com/hibiken/asynq" ) type fetcher interface { // Fetch retries data required by the given state of the dashboard. Fetch(state *State) } type dataFetcher struct { inspector *asynq.Inspector opts Options s tcell.Screen errorCh chan<- error queueCh chan<- *asynq.QueueInfo taskCh chan<- *asynq.TaskInfo queuesCh chan<- []*asynq.QueueInfo groupsCh chan<- []*asynq.GroupInfo tasksCh chan<- []*asynq.TaskInfo } func (f *dataFetcher) Fetch(state *State) { switch state.view { case viewTypeQueues: f.fetchQueues() case viewTypeQueueDetails: if shouldShowGroupTable(state) { f.fetchGroups(state.selectedQueue.Queue) } else if state.taskState == asynq.TaskStateAggregating { f.fetchAggregatingTasks(state.selectedQueue.Queue, state.selectedGroup.Group, taskPageSize(f.s), state.pageNum) } else { f.fetchTasks(state.selectedQueue.Queue, state.taskState, taskPageSize(f.s), state.pageNum) } // if the task modal is open, additionally fetch the selected task's info if state.taskID != "" { f.fetchTaskInfo(state.selectedQueue.Queue, state.taskID) } } } func (f *dataFetcher) fetchQueues() { var ( inspector = f.inspector queuesCh = f.queuesCh errorCh = f.errorCh opts = f.opts ) go fetchQueues(inspector, queuesCh, errorCh, opts) } func fetchQueues(i *asynq.Inspector, queuesCh chan<- []*asynq.QueueInfo, errorCh chan<- error, opts Options) { queues, err := i.Queues() if err != nil { errorCh <- err return } sort.Strings(queues) var res []*asynq.QueueInfo for _, q := range queues { info, err := i.GetQueueInfo(q) if err != nil { errorCh <- err return } res = append(res, info) } queuesCh <- res } func fetchQueueInfo(i *asynq.Inspector, qname string, queueCh chan<- *asynq.QueueInfo, errorCh chan<- error) { q, err := i.GetQueueInfo(qname) if err != nil { errorCh <- err return } queueCh <- q } func (f *dataFetcher) fetchGroups(qname string) { var ( i = f.inspector groupsCh = f.groupsCh errorCh = f.errorCh queueCh = f.queueCh ) go fetchGroups(i, qname, groupsCh, errorCh) go fetchQueueInfo(i, qname, queueCh, errorCh) } func fetchGroups(i *asynq.Inspector, qname string, groupsCh chan<- []*asynq.GroupInfo, errorCh chan<- error) { groups, err := i.Groups(qname) if err != nil { errorCh <- err return } groupsCh <- groups } func (f *dataFetcher) fetchAggregatingTasks(qname, group string, pageSize, pageNum int) { var ( i = f.inspector tasksCh = f.tasksCh errorCh = f.errorCh queueCh = f.queueCh ) go fetchAggregatingTasks(i, qname, group, pageSize, pageNum, tasksCh, errorCh) go fetchQueueInfo(i, qname, queueCh, errorCh) } func fetchAggregatingTasks(i *asynq.Inspector, qname, group string, pageSize, pageNum int, tasksCh chan<- []*asynq.TaskInfo, errorCh chan<- error) { tasks, err := i.ListAggregatingTasks(qname, group, asynq.PageSize(pageSize), asynq.Page(pageNum)) if err != nil { errorCh <- err return } tasksCh <- tasks } func (f *dataFetcher) fetchTasks(qname string, taskState asynq.TaskState, pageSize, pageNum int) { var ( i = f.inspector tasksCh = f.tasksCh errorCh = f.errorCh queueCh = f.queueCh ) go fetchTasks(i, qname, taskState, pageSize, pageNum, tasksCh, errorCh) go fetchQueueInfo(i, qname, queueCh, errorCh) } func fetchTasks(i *asynq.Inspector, qname string, taskState asynq.TaskState, pageSize, pageNum int, tasksCh chan<- []*asynq.TaskInfo, errorCh chan<- error) { var ( tasks []*asynq.TaskInfo err error ) opts := []asynq.ListOption{asynq.PageSize(pageSize), asynq.Page(pageNum)} switch taskState { case asynq.TaskStateActive: tasks, err = i.ListActiveTasks(qname, opts...) case asynq.TaskStatePending: tasks, err = i.ListPendingTasks(qname, opts...) case asynq.TaskStateScheduled: tasks, err = i.ListScheduledTasks(qname, opts...) case asynq.TaskStateRetry: tasks, err = i.ListRetryTasks(qname, opts...) case asynq.TaskStateArchived: tasks, err = i.ListArchivedTasks(qname, opts...) case asynq.TaskStateCompleted: tasks, err = i.ListCompletedTasks(qname, opts...) } if err != nil { errorCh <- err return } tasksCh <- tasks } func (f *dataFetcher) fetchTaskInfo(qname, taskID string) { var ( i = f.inspector taskCh = f.taskCh errorCh = f.errorCh ) go fetchTaskInfo(i, qname, taskID, taskCh, errorCh) } func fetchTaskInfo(i *asynq.Inspector, qname, taskID string, taskCh chan<- *asynq.TaskInfo, errorCh chan<- error) { info, err := i.GetTaskInfo(qname, taskID) if err != nil { errorCh <- err return } taskCh <- info } ================================================ FILE: tools/asynq/cmd/dash/key_event.go ================================================ // Copyright 2022 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package dash import ( "os" "time" "github.com/gdamore/tcell/v2" "github.com/hibiken/asynq" ) // keyEventHandler handles keyboard events and updates the state. // It delegates data fetching to fetcher and UI rendering to drawer. type keyEventHandler struct { s tcell.Screen state *State done chan struct{} fetcher fetcher drawer drawer ticker *time.Ticker pollInterval time.Duration } func (h *keyEventHandler) quit() { h.s.Fini() close(h.done) os.Exit(0) } func (h *keyEventHandler) HandleKeyEvent(ev *tcell.EventKey) { if ev.Key() == tcell.KeyEscape || ev.Rune() == 'q' { h.goBack() // Esc and 'q' key have "go back" semantics } else if ev.Key() == tcell.KeyCtrlC { h.quit() } else if ev.Key() == tcell.KeyCtrlL { h.s.Sync() } else if ev.Key() == tcell.KeyDown || ev.Rune() == 'j' { h.handleDownKey() } else if ev.Key() == tcell.KeyUp || ev.Rune() == 'k' { h.handleUpKey() } else if ev.Key() == tcell.KeyRight || ev.Rune() == 'l' { h.handleRightKey() } else if ev.Key() == tcell.KeyLeft || ev.Rune() == 'h' { h.handleLeftKey() } else if ev.Key() == tcell.KeyEnter { h.handleEnterKey() } else if ev.Rune() == '?' { h.showHelp() } else if ev.Rune() == 'n' { h.nextPage() } else if ev.Rune() == 'p' { h.prevPage() } } func (h *keyEventHandler) goBack() { var ( state = h.state d = h.drawer f = h.fetcher ) if state.view == viewTypeHelp { state.view = state.prevView // exit help f.Fetch(state) h.resetTicker() d.Draw(state) } else if state.view == viewTypeQueueDetails { // if task modal is open close it; otherwise go back to the previous view if state.taskID != "" { state.taskID = "" state.selectedTask = nil d.Draw(state) } else { state.view = viewTypeQueues f.Fetch(state) h.resetTicker() d.Draw(state) } } else { h.quit() } } func (h *keyEventHandler) handleDownKey() { switch h.state.view { case viewTypeQueues: h.downKeyQueues() case viewTypeQueueDetails: h.downKeyQueueDetails() } } func (h *keyEventHandler) downKeyQueues() { if h.state.queueTableRowIdx < len(h.state.queues) { h.state.queueTableRowIdx++ } else { h.state.queueTableRowIdx = 0 // loop back } h.drawer.Draw(h.state) } func (h *keyEventHandler) downKeyQueueDetails() { s, state := h.s, h.state if shouldShowGroupTable(state) { if state.groupTableRowIdx < groupPageSize(s) { state.groupTableRowIdx++ } else { state.groupTableRowIdx = 0 // loop back } } else if state.taskID == "" { if state.taskTableRowIdx < len(state.tasks) { state.taskTableRowIdx++ } else { state.taskTableRowIdx = 0 // loop back } } h.drawer.Draw(state) } func (h *keyEventHandler) handleUpKey() { switch h.state.view { case viewTypeQueues: h.upKeyQueues() case viewTypeQueueDetails: h.upKeyQueueDetails() } } func (h *keyEventHandler) upKeyQueues() { state := h.state if state.queueTableRowIdx == 0 { state.queueTableRowIdx = len(state.queues) } else { state.queueTableRowIdx-- } h.drawer.Draw(state) } func (h *keyEventHandler) upKeyQueueDetails() { s, state := h.s, h.state if shouldShowGroupTable(state) { if state.groupTableRowIdx == 0 { state.groupTableRowIdx = groupPageSize(s) } else { state.groupTableRowIdx-- } } else if state.taskID == "" { if state.taskTableRowIdx == 0 { state.taskTableRowIdx = len(state.tasks) } else { state.taskTableRowIdx-- } } h.drawer.Draw(state) } func (h *keyEventHandler) handleEnterKey() { switch h.state.view { case viewTypeQueues: h.enterKeyQueues() case viewTypeQueueDetails: h.enterKeyQueueDetails() } } func (h *keyEventHandler) resetTicker() { h.ticker.Reset(h.pollInterval) } func (h *keyEventHandler) enterKeyQueues() { var ( state = h.state f = h.fetcher d = h.drawer ) if state.queueTableRowIdx != 0 { state.selectedQueue = state.queues[state.queueTableRowIdx-1] state.view = viewTypeQueueDetails state.taskState = asynq.TaskStateActive state.tasks = nil state.pageNum = 1 f.Fetch(state) h.resetTicker() d.Draw(state) } } func (h *keyEventHandler) enterKeyQueueDetails() { var ( state = h.state f = h.fetcher d = h.drawer ) if shouldShowGroupTable(state) && state.groupTableRowIdx != 0 { state.selectedGroup = state.groups[state.groupTableRowIdx-1] state.tasks = nil state.pageNum = 1 f.Fetch(state) h.resetTicker() d.Draw(state) } else if !shouldShowGroupTable(state) && state.taskTableRowIdx != 0 { task := state.tasks[state.taskTableRowIdx-1] state.selectedTask = task state.taskID = task.ID f.Fetch(state) h.resetTicker() d.Draw(state) } } func (h *keyEventHandler) handleLeftKey() { var ( state = h.state f = h.fetcher d = h.drawer ) if state.view == viewTypeQueueDetails && state.taskID == "" { state.taskState = prevTaskState(state.taskState) state.pageNum = 1 state.taskTableRowIdx = 0 state.tasks = nil state.selectedGroup = nil f.Fetch(state) h.resetTicker() d.Draw(state) } } func (h *keyEventHandler) handleRightKey() { var ( state = h.state f = h.fetcher d = h.drawer ) if state.view == viewTypeQueueDetails && state.taskID == "" { state.taskState = nextTaskState(state.taskState) state.pageNum = 1 state.taskTableRowIdx = 0 state.tasks = nil state.selectedGroup = nil f.Fetch(state) h.resetTicker() d.Draw(state) } } func (h *keyEventHandler) nextPage() { var ( s = h.s state = h.state f = h.fetcher d = h.drawer ) if state.view == viewTypeQueueDetails { if shouldShowGroupTable(state) { pageSize := groupPageSize(s) total := len(state.groups) start := (state.pageNum - 1) * pageSize end := start + pageSize if end <= total { state.pageNum++ d.Draw(state) } } else { pageSize := taskPageSize(s) totalCount := getTaskCount(state.selectedQueue, state.taskState) if (state.pageNum-1)*pageSize+len(state.tasks) < totalCount { state.pageNum++ f.Fetch(state) h.resetTicker() } } } } func (h *keyEventHandler) prevPage() { var ( s = h.s state = h.state f = h.fetcher d = h.drawer ) if state.view == viewTypeQueueDetails { if shouldShowGroupTable(state) { pageSize := groupPageSize(s) start := (state.pageNum - 1) * pageSize if start > 0 { state.pageNum-- d.Draw(state) } } else { if state.pageNum > 1 { state.pageNum-- f.Fetch(state) h.resetTicker() } } } } func (h *keyEventHandler) showHelp() { var ( state = h.state d = h.drawer ) if state.view != viewTypeHelp { state.prevView = state.view state.view = viewTypeHelp d.Draw(state) } } ================================================ FILE: tools/asynq/cmd/dash/key_event_test.go ================================================ // Copyright 2022 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package dash import ( "testing" "time" "github.com/gdamore/tcell/v2" "github.com/google/go-cmp/cmp" "github.com/hibiken/asynq" ) func makeKeyEventHandler(t *testing.T, state *State) *keyEventHandler { ticker := time.NewTicker(time.Second) t.Cleanup(func() { ticker.Stop() }) return &keyEventHandler{ s: tcell.NewSimulationScreen("UTF-8"), state: state, done: make(chan struct{}), fetcher: &fakeFetcher{}, drawer: &fakeDrawer{}, ticker: ticker, pollInterval: time.Second, } } type keyEventHandlerTest struct { desc string // test description state *State // initial state, to be mutated by the handler events []*tcell.EventKey // keyboard events wantState State // expected state after the events } func TestKeyEventHandler(t *testing.T) { tests := []*keyEventHandlerTest{ { desc: "navigates to help view", state: &State{view: viewTypeQueues}, events: []*tcell.EventKey{tcell.NewEventKey(tcell.KeyRune, '?', tcell.ModNone)}, wantState: State{view: viewTypeHelp}, }, { desc: "navigates to queue details view", state: &State{ view: viewTypeQueues, queues: []*asynq.QueueInfo{ {Queue: "default", Size: 100, Active: 10, Pending: 40, Scheduled: 40, Completed: 10}, }, queueTableRowIdx: 0, }, events: []*tcell.EventKey{ tcell.NewEventKey(tcell.KeyRune, 'j', tcell.ModNone), // down tcell.NewEventKey(tcell.KeyEnter, '\n', tcell.ModNone), // Enter }, wantState: State{ view: viewTypeQueueDetails, queues: []*asynq.QueueInfo{ {Queue: "default", Size: 100, Active: 10, Pending: 40, Scheduled: 40, Completed: 10}, }, selectedQueue: &asynq.QueueInfo{Queue: "default", Size: 100, Active: 10, Pending: 40, Scheduled: 40, Completed: 10}, queueTableRowIdx: 1, taskState: asynq.TaskStateActive, pageNum: 1, }, }, { desc: "does nothing if no queues are present", state: &State{ view: viewTypeQueues, queues: []*asynq.QueueInfo{}, // empty queueTableRowIdx: 0, }, events: []*tcell.EventKey{ tcell.NewEventKey(tcell.KeyRune, 'j', tcell.ModNone), // down tcell.NewEventKey(tcell.KeyEnter, '\n', tcell.ModNone), // Enter }, wantState: State{ view: viewTypeQueues, queues: []*asynq.QueueInfo{}, queueTableRowIdx: 0, }, }, { desc: "opens task info modal", state: &State{ view: viewTypeQueueDetails, queues: []*asynq.QueueInfo{ {Queue: "default", Size: 500, Active: 10, Pending: 40}, }, queueTableRowIdx: 1, selectedQueue: &asynq.QueueInfo{Queue: "default", Size: 50, Active: 10, Pending: 40}, taskState: asynq.TaskStatePending, pageNum: 1, tasks: []*asynq.TaskInfo{ {ID: "xxxx", Type: "foo"}, {ID: "yyyy", Type: "bar"}, {ID: "zzzz", Type: "baz"}, }, taskTableRowIdx: 2, }, events: []*tcell.EventKey{ tcell.NewEventKey(tcell.KeyEnter, '\n', tcell.ModNone), // Enter }, wantState: State{ view: viewTypeQueueDetails, queues: []*asynq.QueueInfo{ {Queue: "default", Size: 500, Active: 10, Pending: 40}, }, queueTableRowIdx: 1, selectedQueue: &asynq.QueueInfo{Queue: "default", Size: 50, Active: 10, Pending: 40}, taskState: asynq.TaskStatePending, pageNum: 1, tasks: []*asynq.TaskInfo{ {ID: "xxxx", Type: "foo"}, {ID: "yyyy", Type: "bar"}, {ID: "zzzz", Type: "baz"}, }, taskTableRowIdx: 2, // new states taskID: "yyyy", selectedTask: &asynq.TaskInfo{ID: "yyyy", Type: "bar"}, }, }, { desc: "Esc closes task info modal", state: &State{ view: viewTypeQueueDetails, queues: []*asynq.QueueInfo{ {Queue: "default", Size: 500, Active: 10, Pending: 40}, }, queueTableRowIdx: 1, selectedQueue: &asynq.QueueInfo{Queue: "default", Size: 50, Active: 10, Pending: 40}, taskState: asynq.TaskStatePending, pageNum: 1, tasks: []*asynq.TaskInfo{ {ID: "xxxx", Type: "foo"}, {ID: "yyyy", Type: "bar"}, {ID: "zzzz", Type: "baz"}, }, taskTableRowIdx: 2, taskID: "yyyy", // presence of this field opens the modal }, events: []*tcell.EventKey{ tcell.NewEventKey(tcell.KeyEscape, ' ', tcell.ModNone), // Esc }, wantState: State{ view: viewTypeQueueDetails, queues: []*asynq.QueueInfo{ {Queue: "default", Size: 500, Active: 10, Pending: 40}, }, queueTableRowIdx: 1, selectedQueue: &asynq.QueueInfo{Queue: "default", Size: 50, Active: 10, Pending: 40}, taskState: asynq.TaskStatePending, pageNum: 1, tasks: []*asynq.TaskInfo{ {ID: "xxxx", Type: "foo"}, {ID: "yyyy", Type: "bar"}, {ID: "zzzz", Type: "baz"}, }, taskTableRowIdx: 2, taskID: "", // this field should be unset }, }, { desc: "Arrow keys are disabled while task info modal is open", state: &State{ view: viewTypeQueueDetails, queues: []*asynq.QueueInfo{ {Queue: "default", Size: 500, Active: 10, Pending: 40}, }, queueTableRowIdx: 1, selectedQueue: &asynq.QueueInfo{Queue: "default", Size: 50, Active: 10, Pending: 40}, taskState: asynq.TaskStatePending, pageNum: 1, tasks: []*asynq.TaskInfo{ {ID: "xxxx", Type: "foo"}, {ID: "yyyy", Type: "bar"}, {ID: "zzzz", Type: "baz"}, }, taskTableRowIdx: 2, taskID: "yyyy", // presence of this field opens the modal }, events: []*tcell.EventKey{ tcell.NewEventKey(tcell.KeyLeft, ' ', tcell.ModNone), }, // no change wantState: State{ view: viewTypeQueueDetails, queues: []*asynq.QueueInfo{ {Queue: "default", Size: 500, Active: 10, Pending: 40}, }, queueTableRowIdx: 1, selectedQueue: &asynq.QueueInfo{Queue: "default", Size: 50, Active: 10, Pending: 40}, taskState: asynq.TaskStatePending, pageNum: 1, tasks: []*asynq.TaskInfo{ {ID: "xxxx", Type: "foo"}, {ID: "yyyy", Type: "bar"}, {ID: "zzzz", Type: "baz"}, }, taskTableRowIdx: 2, taskID: "yyyy", // presence of this field opens the modal }, }, // TODO: Add more tests } for _, tc := range tests { t.Run(tc.desc, func(t *testing.T) { h := makeKeyEventHandler(t, tc.state) for _, e := range tc.events { h.HandleKeyEvent(e) } if diff := cmp.Diff(tc.wantState, *tc.state, cmp.AllowUnexported(State{})); diff != "" { t.Errorf("after state was %+v, want %+v: (-want,+got)\n%s", *tc.state, tc.wantState, diff) } }) } } /*** fake implementation for tests ***/ type fakeFetcher struct{} func (f *fakeFetcher) Fetch(s *State) {} type fakeDrawer struct{} func (d *fakeDrawer) Draw(s *State) {} ================================================ FILE: tools/asynq/cmd/dash/screen_drawer.go ================================================ // Copyright 2022 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package dash import ( "strings" "github.com/gdamore/tcell/v2" "github.com/mattn/go-runewidth" ) /*** Screen Drawer ***/ // ScreenDrawer is used to draw contents on screen. // // Usage example: // // d := NewScreenDrawer(s) // d.Println("Hello world", mystyle) // d.NL() // adds newline // d.Print("foo", mystyle.Bold(true)) // d.Print("bar", mystyle.Italic(true)) type ScreenDrawer struct { l *LineDrawer } func NewScreenDrawer(s tcell.Screen) *ScreenDrawer { return &ScreenDrawer{l: NewLineDrawer(0, s)} } func (d *ScreenDrawer) Print(s string, style tcell.Style) { d.l.Draw(s, style) } func (d *ScreenDrawer) Println(s string, style tcell.Style) { d.Print(s, style) d.NL() } // FillLine prints the given rune until the end of the current line // and adds a newline. func (d *ScreenDrawer) FillLine(r rune, style tcell.Style) { w, _ := d.Screen().Size() if w-d.l.col < 0 { d.NL() return } s := strings.Repeat(string(r), w-d.l.col) d.Print(s, style) d.NL() } func (d *ScreenDrawer) FillUntil(r rune, style tcell.Style, limit int) { if d.l.col > limit { return // already passed the limit } s := strings.Repeat(string(r), limit-d.l.col) d.Print(s, style) } // NL adds a newline (i.e., moves to the next line). func (d *ScreenDrawer) NL() { d.l.row++ d.l.col = 0 } func (d *ScreenDrawer) Screen() tcell.Screen { return d.l.s } // Goto moves the screendrawer to the specified cell. func (d *ScreenDrawer) Goto(x, y int) { d.l.row = y d.l.col = x } // Go to the bottom of the screen. func (d *ScreenDrawer) GoToBottom() { _, h := d.Screen().Size() d.l.row = h - 1 d.l.col = 0 } type LineDrawer struct { s tcell.Screen row int col int } func NewLineDrawer(row int, s tcell.Screen) *LineDrawer { return &LineDrawer{row: row, col: 0, s: s} } func (d *LineDrawer) Draw(s string, style tcell.Style) { for _, r := range s { d.s.SetContent(d.col, d.row, r, nil, style) d.col += runewidth.RuneWidth(r) } } ================================================ FILE: tools/asynq/cmd/dash/table.go ================================================ // Copyright 2022 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package dash import ( "github.com/gdamore/tcell/v2" "github.com/mattn/go-runewidth" ) type columnAlignment int const ( alignRight columnAlignment = iota alignLeft ) type columnConfig[V any] struct { name string alignment columnAlignment displayFn func(v V) string } type column[V any] struct { *columnConfig[V] width int } // Helper to draw a table. func drawTable[V any](d *ScreenDrawer, style tcell.Style, configs []*columnConfig[V], data []V, highlightRowIdx int) { const colBuffer = " " // extra buffer between columns cols := make([]*column[V], len(configs)) for i, cfg := range configs { cols[i] = &column[V]{cfg, runewidth.StringWidth(cfg.name)} } // adjust the column width to accommodate the widest value. for _, v := range data { for _, col := range cols { if w := runewidth.StringWidth(col.displayFn(v)); col.width < w { col.width = w } } } // print header headerStyle := style.Background(tcell.ColorDimGray).Foreground(tcell.ColorWhite) for _, col := range cols { if col.alignment == alignLeft { d.Print(rpad(col.name, col.width)+colBuffer, headerStyle) } else { d.Print(lpad(col.name, col.width)+colBuffer, headerStyle) } } d.FillLine(' ', headerStyle) // print body for i, v := range data { rowStyle := style if highlightRowIdx == i { rowStyle = style.Background(tcell.ColorDarkOliveGreen) } for _, col := range cols { if col.alignment == alignLeft { d.Print(rpad(col.displayFn(v), col.width)+colBuffer, rowStyle) } else { d.Print(lpad(col.displayFn(v), col.width)+colBuffer, rowStyle) } } d.FillLine(' ', rowStyle) } } ================================================ FILE: tools/asynq/cmd/dash.go ================================================ // Copyright 2022 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package cmd import ( "fmt" "os" "time" "github.com/MakeNowJust/heredoc/v2" "github.com/hibiken/asynq/tools/asynq/cmd/dash" "github.com/spf13/cobra" ) var ( flagPollInterval = 8 * time.Second ) func init() { rootCmd.AddCommand(dashCmd) dashCmd.Flags().DurationVar(&flagPollInterval, "refresh", 8*time.Second, "Interval between data refresh (default: 8s, min allowed: 1s)") } var dashCmd = &cobra.Command{ Use: "dash", Short: "View dashboard", Long: heredoc.Doc(` Display interactive dashboard.`), Args: cobra.NoArgs, Example: heredoc.Doc(` $ asynq dash $ asynq dash --refresh=3s`), Run: func(cmd *cobra.Command, args []string) { if flagPollInterval < 1*time.Second { fmt.Println("error: --refresh cannot be less than 1s") os.Exit(1) } dash.Run(dash.Options{ PollInterval: flagPollInterval, RedisConnOpt: getRedisConnOpt(), }) }, } ================================================ FILE: tools/asynq/cmd/group.go ================================================ // Copyright 2022 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package cmd import ( "fmt" "os" "github.com/MakeNowJust/heredoc/v2" "github.com/spf13/cobra" ) func init() { rootCmd.AddCommand(groupCmd) groupCmd.AddCommand(groupListCmd) groupListCmd.Flags().StringP("queue", "q", "", "queue to inspect") groupListCmd.MarkFlagRequired("queue") } var groupCmd = &cobra.Command{ Use: "group [flags]", Short: "Manage groups", Example: heredoc.Doc(` $ asynq group list --queue=myqueue`), } var groupListCmd = &cobra.Command{ Use: "list", Aliases: []string{"ls"}, Short: "List groups", Args: cobra.NoArgs, Run: groupLists, } func groupLists(cmd *cobra.Command, args []string) { qname, err := cmd.Flags().GetString("queue") if err != nil { fmt.Println(err) os.Exit(1) } inspector := createInspector() groups, err := inspector.Groups(qname) if len(groups) == 0 { fmt.Printf("No groups found in queue %q\n", qname) return } for _, g := range groups { fmt.Println(g.Group) } } ================================================ FILE: tools/asynq/cmd/queue.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package cmd import ( "fmt" "io" "os" "github.com/MakeNowJust/heredoc/v2" "github.com/fatih/color" "github.com/hibiken/asynq" "github.com/hibiken/asynq/internal/errors" "github.com/spf13/cobra" ) const separator = "=================================================" func init() { rootCmd.AddCommand(queueCmd) queueCmd.AddCommand(queueListCmd) queueCmd.AddCommand(queueInspectCmd) queueCmd.AddCommand(queueHistoryCmd) queueHistoryCmd.Flags().IntP("days", "x", 10, "show data from last x days") queueCmd.AddCommand(queuePauseCmd) queueCmd.AddCommand(queueUnpauseCmd) queueCmd.AddCommand(queueRemoveCmd) queueRemoveCmd.Flags().BoolP("force", "f", false, "remove the queue regardless of its size") } var queueCmd = &cobra.Command{ Use: "queue [flags]", Short: "Manage queues", Example: heredoc.Doc(` $ asynq queue ls $ asynq queue inspect myqueue $ asynq queue pause myqueue`), } var queueListCmd = &cobra.Command{ Use: "list", Short: "List queues", Aliases: []string{"ls"}, // TODO: Use RunE instead? Run: queueList, } var queueInspectCmd = &cobra.Command{ Use: "inspect [...]", Short: "Display detailed information on one or more queues", Args: cobra.MinimumNArgs(1), // TODO: Use RunE instead? Run: queueInspect, Example: heredoc.Doc(` $ asynq queue inspect myqueue $ asynq queue inspect queue1 queue2 queue3`), } var queueHistoryCmd = &cobra.Command{ Use: "history [...]", Short: "Display historical aggregate data from one or more queues", Args: cobra.MinimumNArgs(1), Run: queueHistory, Example: heredoc.Doc(` $ asynq queue history myqueue $ asynq queue history queue1 queue2 queue3 $ asynq queue history myqueue --days=90`), } var queuePauseCmd = &cobra.Command{ Use: "pause [...]", Short: "Pause one or more queues", Args: cobra.MinimumNArgs(1), Run: queuePause, Example: heredoc.Doc(` $ asynq queue pause myqueue $ asynq queue pause queue1 queue2 queue3`), } var queueUnpauseCmd = &cobra.Command{ Use: "resume [...]", Short: "Resume (unpause) one or more queues", Args: cobra.MinimumNArgs(1), Aliases: []string{"unpause"}, Run: queueUnpause, Example: heredoc.Doc(` $ asynq queue resume myqueue $ asynq queue resume queue1 queue2 queue3`), } var queueRemoveCmd = &cobra.Command{ Use: "remove [...]", Short: "Remove one or more queues", Aliases: []string{"rm", "delete"}, Args: cobra.MinimumNArgs(1), Run: queueRemove, Example: heredoc.Doc(` $ asynq queue rm myqueue $ asynq queue rm queue1 queue2 queue3 $ asynq queue rm myqueue --force`), } func queueList(cmd *cobra.Command, args []string) { type queueInfo struct { name string keyslot int64 nodes []*asynq.ClusterNode } inspector := createInspector() queues, err := inspector.Queues() if err != nil { fmt.Printf("error: Could not fetch list of queues: %v\n", err) os.Exit(1) } var qs []*queueInfo for _, qname := range queues { q := queueInfo{name: qname} if useRedisCluster { keyslot, err := inspector.ClusterKeySlot(qname) if err != nil { fmt.Errorf("error: Could not get cluster keyslot for %q\n", qname) continue } q.keyslot = keyslot nodes, err := inspector.ClusterNodes(qname) if err != nil { fmt.Errorf("error: Could not get cluster nodes for %q\n", qname) continue } q.nodes = nodes } qs = append(qs, &q) } if useRedisCluster { printTable( []string{"Queue", "Cluster KeySlot", "Cluster Nodes"}, func(w io.Writer, tmpl string) { for _, q := range qs { fmt.Fprintf(w, tmpl, q.name, q.keyslot, q.nodes) } }, ) } else { for _, q := range qs { fmt.Println(q.name) } } } func queueInspect(cmd *cobra.Command, args []string) { inspector := createInspector() for i, qname := range args { if i > 0 { fmt.Printf("\n%s\n\n", separator) } info, err := inspector.GetQueueInfo(qname) if err != nil { fmt.Printf("error: %v\n", err) continue } printQueueInfo(info) } } func printQueueInfo(info *asynq.QueueInfo) { bold := color.New(color.Bold) bold.Println("Queue Info") fmt.Printf("Name: %s\n", info.Queue) fmt.Printf("Size: %d\n", info.Size) fmt.Printf("Groups: %d\n", info.Groups) fmt.Printf("Paused: %t\n\n", info.Paused) bold.Println("Task Count by State") printTable( []string{"active", "pending", "aggregating", "scheduled", "retry", "archived", "completed"}, func(w io.Writer, tmpl string) { fmt.Fprintf(w, tmpl, info.Active, info.Pending, info.Aggregating, info.Scheduled, info.Retry, info.Archived, info.Completed) }, ) fmt.Println() bold.Printf("Daily Stats %s UTC\n", info.Timestamp.UTC().Format("2006-01-02")) printTable( []string{"processed", "failed", "error rate"}, func(w io.Writer, tmpl string) { var errRate string if info.Processed == 0 { errRate = "N/A" } else { errRate = fmt.Sprintf("%.2f%%", float64(info.Failed)/float64(info.Processed)*100) } fmt.Fprintf(w, tmpl, info.Processed, info.Failed, errRate) }, ) } func queueHistory(cmd *cobra.Command, args []string) { days, err := cmd.Flags().GetInt("days") if err != nil { fmt.Printf("error: Internal error: %v\n", err) os.Exit(1) } inspector := createInspector() for i, qname := range args { if i > 0 { fmt.Printf("\n%s\n\n", separator) } fmt.Printf("Queue: %s\n\n", qname) stats, err := inspector.History(qname, days) if err != nil { fmt.Printf("error: %v\n", err) continue } printDailyStats(stats) } } func printDailyStats(stats []*asynq.DailyStats) { printTable( []string{"date (UTC)", "processed", "failed", "error rate"}, func(w io.Writer, tmpl string) { for _, s := range stats { var errRate string if s.Processed == 0 { errRate = "N/A" } else { errRate = fmt.Sprintf("%.2f%%", float64(s.Failed)/float64(s.Processed)*100) } fmt.Fprintf(w, tmpl, s.Date.Format("2006-01-02"), s.Processed, s.Failed, errRate) } }, ) } func queuePause(cmd *cobra.Command, args []string) { inspector := createInspector() for _, qname := range args { err := inspector.PauseQueue(qname) if err != nil { fmt.Println(err) continue } fmt.Printf("Successfully paused queue %q\n", qname) } } func queueUnpause(cmd *cobra.Command, args []string) { inspector := createInspector() for _, qname := range args { err := inspector.UnpauseQueue(qname) if err != nil { fmt.Println(err) continue } fmt.Printf("Successfully unpaused queue %q\n", qname) } } func queueRemove(cmd *cobra.Command, args []string) { // TODO: Use inspector once RemoveQueue become public API. force, err := cmd.Flags().GetBool("force") if err != nil { fmt.Printf("error: Internal error: %v\n", err) os.Exit(1) } r := createRDB() for _, qname := range args { err = r.RemoveQueue(qname, force) if err != nil { if errors.IsQueueNotEmpty(err) { fmt.Printf("error: %v\nIf you are sure you want to delete it, run 'asynq queue rm --force %s'\n", err, qname) continue } fmt.Printf("error: %v\n", err) continue } fmt.Printf("Successfully removed queue %q\n", qname) } } ================================================ FILE: tools/asynq/cmd/root.go ================================================ // // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package cmd import ( "crypto/tls" "fmt" "io" "os" "strings" "text/tabwriter" "time" "unicode" "unicode/utf8" "github.com/MakeNowJust/heredoc/v2" "github.com/fatih/color" "github.com/hibiken/asynq" "github.com/hibiken/asynq/internal/base" "github.com/hibiken/asynq/internal/rdb" "github.com/redis/go-redis/v9" "github.com/spf13/cobra" "github.com/spf13/pflag" "golang.org/x/exp/utf8string" homedir "github.com/mitchellh/go-homedir" "github.com/spf13/viper" ) var cfgFile string // Global flag variables var ( uri string db int password string username string useRedisCluster bool clusterAddrs string tlsServerName string insecure bool useTLS bool ) // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ Use: "asynq [flags]", Short: "Asynq CLI", Long: `Command line tool to inspect tasks and queues managed by Asynq`, Version: base.Version, SilenceUsage: true, SilenceErrors: true, Example: heredoc.Doc(` $ asynq stats $ asynq queue pause myqueue $ asynq task list --queue=myqueue --state=archived`), Annotations: map[string]string{ "help:feedback": heredoc.Doc(` Open an issue at https://github.com/hibiken/asynq/issues/new/choose`), }, } var versionOutput = fmt.Sprintf("asynq version %s\n", base.Version) var versionCmd = &cobra.Command{ Use: "version", Hidden: true, Run: func(cmd *cobra.Command, args []string) { fmt.Print(versionOutput) }, } // Execute adds all child commands to the root command and sets flags appropriately. // This is called by main.main(). It only needs to happen once to the rootCmd. func Execute() { if err := rootCmd.Execute(); err != nil { fmt.Println(err) os.Exit(1) } } func isRootCmd(cmd *cobra.Command) bool { return cmd != nil && !cmd.HasParent() } // displayLine represents a line displayed in the output as ' ', // where pad is used to pad the name from desc. type displayLine struct { name string desc string pad int // number of rpad } func (l *displayLine) String() string { return rpad(l.name, l.pad) + l.desc } type displayLines []*displayLine func (dls displayLines) String() string { var lines []string for _, dl := range dls { lines = append(lines, dl.String()) } return strings.Join(lines, "\n") } // Capitalize the first word in the given string. func capitalize(s string) string { str := utf8string.NewString(s) if str.RuneCount() == 0 { return "" } var b strings.Builder b.WriteString(strings.ToUpper(string(str.At(0)))) b.WriteString(str.Slice(1, str.RuneCount())) return b.String() } func rootHelpFunc(cmd *cobra.Command, args []string) { // Display helpful error message when user mistypes a subcommand (e.g. 'asynq queue lst'). if isRootCmd(cmd.Parent()) && len(args) >= 2 && args[1] != "--help" && args[1] != "-h" { printSubcommandSuggestions(cmd, args[1]) return } var lines []*displayLine var commands []*displayLine for _, c := range cmd.Commands() { if c.Hidden || c.Short == "" || c.Name() == "help" { continue } l := &displayLine{name: c.Name() + ":", desc: capitalize(c.Short)} commands = append(commands, l) lines = append(lines, l) } var localFlags []*displayLine cmd.LocalFlags().VisitAll(func(f *pflag.Flag) { l := &displayLine{name: "--" + f.Name, desc: capitalize(f.Usage)} localFlags = append(localFlags, l) lines = append(lines, l) }) var inheritedFlags []*displayLine cmd.InheritedFlags().VisitAll(func(f *pflag.Flag) { l := &displayLine{name: "--" + f.Name, desc: capitalize(f.Usage)} inheritedFlags = append(inheritedFlags, l) lines = append(lines, l) }) adjustPadding(lines...) type helpEntry struct { Title string Body string } var helpEntries []*helpEntry desc := cmd.Long if desc == "" { desc = cmd.Short } if desc != "" { helpEntries = append(helpEntries, &helpEntry{"", desc}) } helpEntries = append(helpEntries, &helpEntry{"USAGE", cmd.UseLine()}) if len(commands) > 0 { helpEntries = append(helpEntries, &helpEntry{"COMMANDS", displayLines(commands).String()}) } if cmd.LocalFlags().HasFlags() { helpEntries = append(helpEntries, &helpEntry{"FLAGS", displayLines(localFlags).String()}) } if cmd.InheritedFlags().HasFlags() { helpEntries = append(helpEntries, &helpEntry{"INHERITED FLAGS", displayLines(inheritedFlags).String()}) } if cmd.Example != "" { helpEntries = append(helpEntries, &helpEntry{"EXAMPLES", cmd.Example}) } helpEntries = append(helpEntries, &helpEntry{"LEARN MORE", heredoc.Doc(` Use 'asynq --help' for more information about a command.`)}) if s, ok := cmd.Annotations["help:feedback"]; ok { helpEntries = append(helpEntries, &helpEntry{"FEEDBACK", s}) } out := cmd.OutOrStdout() bold := color.New(color.Bold) for _, e := range helpEntries { if e.Title != "" { // If there is a title, add indentation to each line in the body bold.Fprintln(out, e.Title) fmt.Fprintln(out, indent(e.Body, 2 /* spaces */)) } else { // If there is no title, print the body as is fmt.Fprintln(out, e.Body) } fmt.Fprintln(out) } } func rootUsageFunc(cmd *cobra.Command) error { out := cmd.OutOrStdout() fmt.Fprintf(out, "Usage: %s", cmd.UseLine()) if subcmds := cmd.Commands(); len(subcmds) > 0 { fmt.Fprint(out, "\n\nAvailable commands:\n") for _, c := range subcmds { if c.Hidden { continue } fmt.Fprintf(out, " %s\n", c.Name()) } } var localFlags []*displayLine cmd.LocalFlags().VisitAll(func(f *pflag.Flag) { localFlags = append(localFlags, &displayLine{name: "--" + f.Name, desc: capitalize(f.Usage)}) }) adjustPadding(localFlags...) if len(localFlags) > 0 { fmt.Fprint(out, "\n\nFlags:\n") for _, l := range localFlags { fmt.Fprintf(out, " %s\n", l.String()) } } return nil } func printSubcommandSuggestions(cmd *cobra.Command, arg string) { out := cmd.OutOrStdout() fmt.Fprintf(out, "unknown command %q for %q\n", arg, cmd.CommandPath()) if cmd.SuggestionsMinimumDistance <= 0 { cmd.SuggestionsMinimumDistance = 2 } candidates := cmd.SuggestionsFor(arg) if len(candidates) > 0 { fmt.Fprint(out, "\nDid you mean this?\n") for _, c := range candidates { fmt.Fprintf(out, "\t%s\n", c) } } fmt.Fprintln(out) rootUsageFunc(cmd) } func adjustPadding(lines ...*displayLine) { // find the maximum width of the name max := 0 for _, l := range lines { if n := utf8.RuneCountInString(l.name); n > max { max = n } } for _, l := range lines { l.pad = max } } // rpad adds padding to the right of a string. func rpad(s string, padding int) string { tmpl := fmt.Sprintf("%%-%ds ", padding) return fmt.Sprintf(tmpl, s) } // lpad adds padding to the left of a string. func lpad(s string, padding int) string { tmpl := fmt.Sprintf("%%%ds ", padding) return fmt.Sprintf(tmpl, s) } // indent indents the given text by given spaces. func indent(text string, space int) string { if len(text) == 0 { return "" } var b strings.Builder indentation := strings.Repeat(" ", space) lastRune := '\n' for _, r := range text { if lastRune == '\n' { b.WriteString(indentation) } b.WriteRune(r) lastRune = r } return b.String() } // dedent removes any indentation from the given text. func dedent(text string) string { lines := strings.Split(text, "\n") var b strings.Builder for _, l := range lines { b.WriteString(strings.TrimLeftFunc(l, unicode.IsSpace)) b.WriteRune('\n') } return b.String() } func init() { cobra.OnInitialize(initConfig) rootCmd.SetHelpFunc(rootHelpFunc) rootCmd.SetUsageFunc(rootUsageFunc) rootCmd.AddCommand(versionCmd) rootCmd.SetVersionTemplate(versionOutput) rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "Config file to set flag defaut values (default is $HOME/.asynq.yaml)") rootCmd.PersistentFlags().StringVarP(&uri, "uri", "u", "127.0.0.1:6379", "Redis server URI") rootCmd.PersistentFlags().IntVarP(&db, "db", "n", 0, "Redis database number (default is 0)") rootCmd.PersistentFlags().StringVarP(&password, "password", "p", "", "Password to use when connecting to redis server") rootCmd.PersistentFlags().StringVarP(&username, "username", "U", "", "Username to use when connecting to Redis (ACL username)") rootCmd.PersistentFlags().BoolVar(&useRedisCluster, "cluster", false, "Connect to redis cluster") rootCmd.PersistentFlags().StringVar(&clusterAddrs, "cluster_addrs", "127.0.0.1:7000,127.0.0.1:7001,127.0.0.1:7002,127.0.0.1:7003,127.0.0.1:7004,127.0.0.1:7005", "List of comma-separated redis server addresses") rootCmd.PersistentFlags().BoolVar(&useTLS, "tls", false, "Enable TLS connection") rootCmd.PersistentFlags().StringVar(&tlsServerName, "tls_server", "", "Server name for TLS validation") rootCmd.PersistentFlags().BoolVar(&insecure, "insecure", false, "Allow insecure TLS connection by skipping cert validation") // Bind flags with config. viper.BindPFlag("uri", rootCmd.PersistentFlags().Lookup("uri")) viper.BindPFlag("db", rootCmd.PersistentFlags().Lookup("db")) viper.BindPFlag("password", rootCmd.PersistentFlags().Lookup("password")) viper.BindPFlag("username", rootCmd.PersistentFlags().Lookup("username")) viper.BindPFlag("cluster", rootCmd.PersistentFlags().Lookup("cluster")) viper.BindPFlag("cluster_addrs", rootCmd.PersistentFlags().Lookup("cluster_addrs")) viper.BindPFlag("tls", rootCmd.PersistentFlags().Lookup("tls")) viper.BindPFlag("tls_server", rootCmd.PersistentFlags().Lookup("tls_server")) viper.BindPFlag("insecure", rootCmd.PersistentFlags().Lookup("insecure")) } // initConfig reads in config file and ENV variables if set. func initConfig() { if cfgFile != "" { // Use config file from the flag. viper.SetConfigFile(cfgFile) } else { // Find home directory. home, err := homedir.Dir() if err != nil { fmt.Println(err) os.Exit(1) } // Search config in home directory with name ".asynq" (without extension). viper.AddConfigPath(home) viper.SetConfigName(".asynq") } viper.AutomaticEnv() // read in environment variables that match // If a config file is found, read it in. if err := viper.ReadInConfig(); err == nil { fmt.Println("Using config file:", viper.ConfigFileUsed()) } } // createRDB creates a RDB instance using flag values and returns it. func createRDB() *rdb.RDB { var c redis.UniversalClient if viper.GetBool("cluster") { addrs := strings.Split(viper.GetString("cluster_addrs"), ",") c = redis.NewClusterClient(&redis.ClusterOptions{ Addrs: addrs, Password: viper.GetString("password"), Username: viper.GetString("username"), TLSConfig: getTLSConfig(), }) } else { c = redis.NewClient(&redis.Options{ Addr: viper.GetString("uri"), DB: viper.GetInt("db"), Password: viper.GetString("password"), Username: viper.GetString("username"), TLSConfig: getTLSConfig(), }) } return rdb.NewRDB(c) } // createClient creates a Client instance using flag values and returns it. func createClient() *asynq.Client { return asynq.NewClient(getRedisConnOpt()) } // createInspector creates a Inspector instance using flag values and returns it. func createInspector() *asynq.Inspector { return asynq.NewInspector(getRedisConnOpt()) } func getRedisConnOpt() asynq.RedisConnOpt { if viper.GetBool("cluster") { addrs := strings.Split(viper.GetString("cluster_addrs"), ",") return asynq.RedisClusterClientOpt{ Addrs: addrs, Password: viper.GetString("password"), Username: viper.GetString("username"), TLSConfig: getTLSConfig(), } } return asynq.RedisClientOpt{ Addr: viper.GetString("uri"), DB: viper.GetInt("db"), Password: viper.GetString("password"), Username: viper.GetString("username"), TLSConfig: getTLSConfig(), } } func getTLSConfig() *tls.Config { tlsServer := viper.GetString("tls_server") if tlsServer != "" { return &tls.Config{ServerName: tlsServer, InsecureSkipVerify: viper.GetBool("insecure")} } if viper.GetBool("tls") { return &tls.Config{InsecureSkipVerify: viper.GetBool("insecure")} } return nil } // printTable is a helper function to print data in table format. // // cols is a list of headers and printRow specifies how to print rows. // // Example: // // type User struct { // Name string // Addr string // Age int // } // // data := []*User{{"user1", "addr1", 24}, {"user2", "addr2", 42}, ...} // cols := []string{"Name", "Addr", "Age"} // // printRows := func(w io.Writer, tmpl string) { // for _, u := range data { // fmt.Fprintf(w, tmpl, u.Name, u.Addr, u.Age) // } // } // // printTable(cols, printRows) func printTable(cols []string, printRows func(w io.Writer, tmpl string)) { format := strings.Repeat("%v\t", len(cols)) + "\n" tw := new(tabwriter.Writer).Init(os.Stdout, 0, 8, 2, ' ', 0) var headers []interface{} var seps []interface{} for _, name := range cols { headers = append(headers, name) seps = append(seps, strings.Repeat("-", len(name))) } fmt.Fprintf(tw, format, headers...) fmt.Fprintf(tw, format, seps...) printRows(tw, format) tw.Flush() } // sprintBytes returns a string representation of the given byte slice if data is printable. // If data is not printable, it returns a string describing it is not printable. func sprintBytes(payload []byte) string { if !isPrintable(payload) { return "non-printable bytes" } return string(payload) } func isPrintable(data []byte) bool { if !utf8.Valid(data) { return false } isAllSpace := true for _, r := range string(data) { if !unicode.IsPrint(r) { return false } if !unicode.IsSpace(r) { isAllSpace = false } } return !isAllSpace } // Helper to turn a command line flag into a duration func getDuration(cmd *cobra.Command, arg string) time.Duration { durationStr, err := cmd.Flags().GetString(arg) if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } duration, err := time.ParseDuration(durationStr) if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } return duration } // Helper to turn a command line flag into a time func getTime(cmd *cobra.Command, arg string) time.Time { timeStr, err := cmd.Flags().GetString(arg) if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } timeVal, err := time.Parse(time.RFC3339, timeStr) if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } return timeVal } ================================================ FILE: tools/asynq/cmd/server.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package cmd import ( "fmt" "io" "os" "sort" "strings" "time" "github.com/MakeNowJust/heredoc/v2" "github.com/spf13/cobra" ) func init() { rootCmd.AddCommand(serverCmd) serverCmd.AddCommand(serverListCmd) } var serverCmd = &cobra.Command{ Use: "server [flags]", Short: "Manage servers", Example: heredoc.Doc(` $ asynq server list`), } var serverListCmd = &cobra.Command{ Use: "list", Aliases: []string{"ls"}, Short: "List servers", Long: `Server list (asynq server ls) shows all running worker servers pulling tasks from the given redis instance. The command shows the following for each server: * Host and PID of the process in which the server is running * Number of active workers out of worker pool * Queue configuration * State of the worker server ("active" | "stopped") * Time the server was started A "active" server is pulling tasks from queues and processing them. A "stopped" server is no longer pulling new tasks from queues`, Run: serverList, } func serverList(cmd *cobra.Command, args []string) { r := createRDB() servers, err := r.ListServers() if err != nil { fmt.Println(err) os.Exit(1) } if len(servers) == 0 { fmt.Println("No running servers") return } // sort by hostname and pid sort.Slice(servers, func(i, j int) bool { x, y := servers[i], servers[j] if x.Host != y.Host { return x.Host < y.Host } return x.PID < y.PID }) // print server info cols := []string{"Host", "PID", "State", "Active Workers", "Queues", "Started"} printRows := func(w io.Writer, tmpl string) { for _, info := range servers { fmt.Fprintf(w, tmpl, info.Host, info.PID, info.Status, fmt.Sprintf("%d/%d", info.ActiveWorkerCount, info.Concurrency), formatQueues(info.Queues), timeAgo(info.Started)) } } printTable(cols, printRows) } func formatQueues(qmap map[string]int) string { // sort queues by priority and name type queue struct { name string priority int } var queues []*queue for qname, p := range qmap { queues = append(queues, &queue{qname, p}) } sort.Slice(queues, func(i, j int) bool { x, y := queues[i], queues[j] if x.priority != y.priority { return x.priority > y.priority } return x.name < y.name }) var b strings.Builder l := len(queues) for _, q := range queues { fmt.Fprintf(&b, "%s:%d", q.name, q.priority) l-- if l > 0 { b.WriteString(" ") } } return b.String() } // timeAgo takes a time and returns a string of the format " ago". func timeAgo(since time.Time) string { d := time.Since(since).Round(time.Second) return fmt.Sprintf("%v ago", d) } ================================================ FILE: tools/asynq/cmd/stats.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package cmd import ( "encoding/json" "fmt" "io" "math" "os" "strconv" "strings" "text/tabwriter" "time" "unicode/utf8" "github.com/MakeNowJust/heredoc/v2" "github.com/fatih/color" "github.com/hibiken/asynq/internal/rdb" "github.com/spf13/cobra" ) // statsCmd represents the stats command var statsCmd = &cobra.Command{ Use: "stats", Short: "View current state", Long: heredoc.Doc(` Stats shows the overview of tasks and queues at that instant. The command shows the following: * Number of tasks in each state * Number of tasks in each queue * Aggregate data for the current day * Basic information about the running redis instance`), Args: cobra.NoArgs, Run: stats, } var jsonFlag bool func init() { rootCmd.AddCommand(statsCmd) statsCmd.Flags().BoolVar(&jsonFlag, "json", false, "Output stats in JSON format.") // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // statsCmd.PersistentFlags().String("foo", "", "A help for foo") // Cobra supports local flags which will only run when this command // is called directly, e.g.: // statsCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") } type AggregateStats struct { Active int `json:"active"` Pending int `json:"pending"` Aggregating int `json:"aggregating"` Scheduled int `json:"scheduled"` Retry int `json:"retry"` Archived int `json:"archived"` Completed int `json:"completed"` Processed int `json:"processed"` Failed int `json:"failed"` Timestamp time.Time `json:"timestamp"` } type FullStats struct { Aggregate AggregateStats `json:"aggregate"` QueueStats []*rdb.Stats `json:"queues"` RedisInfo map[string]string `json:"redis"` } func stats(cmd *cobra.Command, args []string) { r := createRDB() queues, err := r.AllQueues() if err != nil { fmt.Println(err) os.Exit(1) } var aggStats AggregateStats var stats []*rdb.Stats for _, qname := range queues { s, err := r.CurrentStats(qname) if err != nil { fmt.Println(err) os.Exit(1) } aggStats.Active += s.Active aggStats.Pending += s.Pending aggStats.Aggregating += s.Aggregating aggStats.Scheduled += s.Scheduled aggStats.Retry += s.Retry aggStats.Archived += s.Archived aggStats.Completed += s.Completed aggStats.Processed += s.Processed aggStats.Failed += s.Failed aggStats.Timestamp = s.Timestamp stats = append(stats, s) } var info map[string]string if useRedisCluster { info, err = r.RedisClusterInfo() } else { info, err = r.RedisInfo() } if err != nil { fmt.Println(err) os.Exit(1) } if jsonFlag { statsJSON, err := json.Marshal(FullStats{ Aggregate: aggStats, QueueStats: stats, RedisInfo: info, }) if err != nil { fmt.Println(err) os.Exit(1) } fmt.Println(string(statsJSON)) return } bold := color.New(color.Bold) bold.Println("Task Count by State") printStatsByState(&aggStats) fmt.Println() bold.Println("Task Count by Queue") printStatsByQueue(stats) fmt.Println() bold.Printf("Daily Stats %s UTC\n", aggStats.Timestamp.UTC().Format("2006-01-02")) printSuccessFailureStats(&aggStats) fmt.Println() if useRedisCluster { bold.Println("Redis Cluster Info") printClusterInfo(info) } else { bold.Println("Redis Info") printInfo(info) } fmt.Println() } func printStatsByState(s *AggregateStats) { format := strings.Repeat("%v\t", 7) + "\n" tw := new(tabwriter.Writer).Init(os.Stdout, 0, 8, 2, ' ', 0) fmt.Fprintf(tw, format, "active", "pending", "aggregating", "scheduled", "retry", "archived", "completed") width := maxInt(9 /* defaultWidth */, maxWidthOf(s.Active, s.Pending, s.Aggregating, s.Scheduled, s.Retry, s.Archived, s.Completed)) // length of widest column sep := strings.Repeat("-", width) fmt.Fprintf(tw, format, sep, sep, sep, sep, sep, sep, sep) fmt.Fprintf(tw, format, s.Active, s.Pending, s.Aggregating, s.Scheduled, s.Retry, s.Archived, s.Completed) tw.Flush() } // numDigits returns the number of digits in n. func numDigits(n int) int { return len(strconv.Itoa(n)) } // maxWidthOf returns the max number of digits amount the provided vals. func maxWidthOf(vals ...int) int { max := 0 for _, v := range vals { if vw := numDigits(v); vw > max { max = vw } } return max } func maxInt(a, b int) int { return int(math.Max(float64(a), float64(b))) } func printStatsByQueue(stats []*rdb.Stats) { var headers, seps, counts []string maxHeaderWidth := 0 for _, s := range stats { title := queueTitle(s) headers = append(headers, title) if w := utf8.RuneCountInString(title); w > maxHeaderWidth { maxHeaderWidth = w } counts = append(counts, strconv.Itoa(s.Size)) } for i := 0; i < len(headers); i++ { seps = append(seps, strings.Repeat("-", maxHeaderWidth)) } format := strings.Repeat("%v\t", len(headers)) + "\n" tw := new(tabwriter.Writer).Init(os.Stdout, 0, 8, 2, ' ', 0) fmt.Fprintf(tw, format, toInterfaceSlice(headers)...) fmt.Fprintf(tw, format, toInterfaceSlice(seps)...) fmt.Fprintf(tw, format, toInterfaceSlice(counts)...) tw.Flush() } func queueTitle(s *rdb.Stats) string { var b strings.Builder b.WriteString(s.Queue) if s.Paused { b.WriteString(" (paused)") } return b.String() } func printSuccessFailureStats(s *AggregateStats) { format := strings.Repeat("%v\t", 3) + "\n" tw := new(tabwriter.Writer).Init(os.Stdout, 0, 8, 2, ' ', 0) fmt.Fprintf(tw, format, "processed", "failed", "error rate") fmt.Fprintf(tw, format, "---------", "------", "----------") var errrate string if s.Processed == 0 { errrate = "N/A" } else { errrate = fmt.Sprintf("%.2f%%", float64(s.Failed)/float64(s.Processed)*100) } fmt.Fprintf(tw, format, s.Processed, s.Failed, errrate) tw.Flush() } func printInfo(info map[string]string) { format := strings.Repeat("%v\t", 5) + "\n" tw := new(tabwriter.Writer).Init(os.Stdout, 0, 8, 2, ' ', 0) fmt.Fprintf(tw, format, "version", "uptime", "connections", "memory usage", "peak memory usage") fmt.Fprintf(tw, format, "-------", "------", "-----------", "------------", "-----------------") fmt.Fprintf(tw, format, info["redis_version"], fmt.Sprintf("%s days", info["uptime_in_days"]), info["connected_clients"], fmt.Sprintf("%sB", info["used_memory_human"]), fmt.Sprintf("%sB", info["used_memory_peak_human"]), ) tw.Flush() } func printClusterInfo(info map[string]string) { printTable( []string{"State", "Known Nodes", "Cluster Size"}, func(w io.Writer, tmpl string) { fmt.Fprintf(w, tmpl, strings.ToUpper(info["cluster_state"]), info["cluster_known_nodes"], info["cluster_size"], ) }, ) } func toInterfaceSlice(strs []string) []interface{} { var res []interface{} for _, s := range strs { res = append(res, s) } return res } ================================================ FILE: tools/asynq/cmd/task.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package cmd import ( "fmt" "io" "os" "time" "github.com/MakeNowJust/heredoc/v2" "github.com/fatih/color" "github.com/hibiken/asynq" "github.com/spf13/cobra" ) func init() { rootCmd.AddCommand(taskCmd) taskCmd.AddCommand(taskListCmd) taskListCmd.Flags().StringP("queue", "q", "", "queue to inspect (required)") taskListCmd.Flags().StringP("state", "s", "", "state of the tasks; one of { active | pending | aggregating | scheduled | retry | archived | completed } (required)") taskListCmd.Flags().Int("page", 1, "page number") taskListCmd.Flags().Int("size", 30, "page size") taskListCmd.Flags().StringP("group", "g", "", "group to inspect (required for listing aggregating tasks)") taskListCmd.MarkFlagRequired("queue") taskListCmd.MarkFlagRequired("state") taskCmd.AddCommand(taskCancelCmd) taskCmd.AddCommand(taskInspectCmd) taskInspectCmd.Flags().StringP("queue", "q", "", "queue to which the task belongs (required)") taskInspectCmd.Flags().StringP("id", "i", "", "id of the task (required)") taskInspectCmd.MarkFlagRequired("queue") taskInspectCmd.MarkFlagRequired("id") taskCmd.AddCommand(taskArchiveCmd) taskArchiveCmd.Flags().StringP("queue", "q", "", "queue to which the task belongs (required)") taskArchiveCmd.Flags().StringP("id", "i", "", "id of the task (required)") taskArchiveCmd.MarkFlagRequired("queue") taskArchiveCmd.MarkFlagRequired("id") taskCmd.AddCommand(taskDeleteCmd) taskDeleteCmd.Flags().StringP("queue", "q", "", "queue to which the task belongs (required)") taskDeleteCmd.Flags().StringP("id", "i", "", "id of the task (required)") taskDeleteCmd.MarkFlagRequired("queue") taskDeleteCmd.MarkFlagRequired("id") taskCmd.AddCommand(taskRunCmd) taskRunCmd.Flags().StringP("queue", "q", "", "queue to which the task belongs (required)") taskRunCmd.Flags().StringP("id", "i", "", "id of the task (required)") taskRunCmd.MarkFlagRequired("queue") taskRunCmd.MarkFlagRequired("id") taskCmd.AddCommand(taskEnqueueCmd) taskEnqueueCmd.Flags().StringP("type_name", "t", "", "type name to enqueue the task as (required)") taskEnqueueCmd.Flags().StringP("payload", "l", "", "payload to enqueue (required)") // The following are the various OptionTypes; if not specified we won't pass them so that composeOptions() // can apply its own defaults taskEnqueueCmd.Flags().Int("retry", 0, "maximum retries") taskEnqueueCmd.Flags().String("queue", "", "queue to enqueue the task to") taskEnqueueCmd.Flags().String("id", "", "id to enqueue the task as") taskEnqueueCmd.Flags().String("timeout", "", "timeout for the task (how long it can run); must be parseable as a time.Duration") taskEnqueueCmd.Flags().String("deadline", "", "deadline for the task; must be in RFC3339 format") taskEnqueueCmd.Flags().String("unique", "", "unique period for the task (duration within which it is guaranteed to be unique); must be parseable as a time.Duration") taskEnqueueCmd.Flags().String("process_at", "", "process at time for the task; must be in RFC3339 format") taskEnqueueCmd.Flags().String("process_in", "", "process in window for the task; must be parseable as a time.Duration") taskEnqueueCmd.Flags().String("retention", "", "retention window for the task; must be parseable as a time.Duration") taskEnqueueCmd.Flags().String("group", "", "group for the task") taskEnqueueCmd.MarkFlagRequired("type_name") taskEnqueueCmd.MarkFlagRequired("payload") taskCmd.AddCommand(taskArchiveAllCmd) taskArchiveAllCmd.Flags().StringP("queue", "q", "", "queue to which the tasks belong (required)") taskArchiveAllCmd.Flags().StringP("state", "s", "", "state of the tasks; one of { pending | aggregating | scheduled | retry } (required)") taskArchiveAllCmd.MarkFlagRequired("queue") taskArchiveAllCmd.MarkFlagRequired("state") taskArchiveAllCmd.Flags().StringP("group", "g", "", "group to which the tasks belong (required for archiving aggregating tasks)") taskCmd.AddCommand(taskDeleteAllCmd) taskDeleteAllCmd.Flags().StringP("queue", "q", "", "queue to which the tasks belong (required)") taskDeleteAllCmd.Flags().StringP("state", "s", "", "state of the tasks; one of { pending | aggregating | scheduled | retry | archived | completed } (required)") taskDeleteAllCmd.MarkFlagRequired("queue") taskDeleteAllCmd.MarkFlagRequired("state") taskDeleteAllCmd.Flags().StringP("group", "g", "", "group to which the tasks belong (required for deleting aggregating tasks)") taskCmd.AddCommand(taskRunAllCmd) taskRunAllCmd.Flags().StringP("queue", "q", "", "queue to which the tasks belong (required)") taskRunAllCmd.Flags().StringP("state", "s", "", "state of the tasks; one of { scheduled | retry | archived } (required)") taskRunAllCmd.MarkFlagRequired("queue") taskRunAllCmd.MarkFlagRequired("state") taskRunAllCmd.Flags().StringP("group", "g", "", "group to which the tasks belong (required for running aggregating tasks)") } var taskCmd = &cobra.Command{ Use: "task [flags]", Short: "Manage tasks", Example: heredoc.Doc(` $ asynq task list --queue=myqueue --state=scheduled $ asynq task inspect --queue=myqueue --id=7837f142-6337-4217-9276-8f27281b67d1 $ asynq task delete --queue=myqueue --id=7837f142-6337-4217-9276-8f27281b67d1 $ asynq task deleteall --queue=myqueue --state=archived`), } var taskListCmd = &cobra.Command{ Use: "list --queue= --state= [flags]", Aliases: []string{"ls"}, Short: "List tasks", Long: heredoc.Doc(` List tasks of the given state from the specified queue. The --queue and --state flags are required. Note: For aggregating tasks, additional --group flag is required. List opeartion paginates the result set. By default, the command fetches the first 30 tasks. Use --page and --size flags to specify the page number and size.`), Example: heredoc.Doc(` $ asynq task list --queue=myqueue --state=pending $ asynq task list --queue=myqueue --state=aggregating --group=mygroup $ asynq task list --queue=myqueue --state=scheduled --page=2`), Run: taskList, } var taskInspectCmd = &cobra.Command{ Use: "inspect --queue= --id=", Short: "Display detailed information on the specified task", Args: cobra.NoArgs, Run: taskInspect, Example: heredoc.Doc(` $ asynq task inspect --queue=myqueue --id=f1720682-f5a6-4db1-8953-4f48ae541d0f`), } var taskCancelCmd = &cobra.Command{ Use: "cancel [...]", Short: "Cancel one or more active tasks", Args: cobra.MinimumNArgs(1), Run: taskCancel, Example: heredoc.Doc(` $ asynq task cancel f1720682-f5a6-4db1-8953-4f48ae541d0f`), } var taskArchiveCmd = &cobra.Command{ Use: "archive --queue= --id=", Short: "Archive a task with the given id", Args: cobra.NoArgs, Run: taskArchive, Example: heredoc.Doc(` $ asynq task archive --queue=myqueue --id=f1720682-f5a6-4db1-8953-4f48ae541d0f`), } var taskDeleteCmd = &cobra.Command{ Use: "delete --queue= --id=", Aliases: []string{"remove", "rm"}, Short: "Delete a task with the given id", Args: cobra.NoArgs, Run: taskDelete, Example: heredoc.Doc(` $ asynq task delete --queue=myqueue --id=f1720682-f5a6-4db1-8953-4f48ae541d0f`), } var taskRunCmd = &cobra.Command{ Use: "run --queue= --id=", Short: "Run a task with the given id", Args: cobra.NoArgs, Run: taskRun, Example: heredoc.Doc(` $ asynq task run --queue=myqueue --id=f1720682-f5a6-4db1-8953-4f48ae541d0f`), } var taskEnqueueCmd = &cobra.Command{ Use: "enqueue --type_name=footype --payload=barpayload", Short: "Enqueue a task", Args: cobra.NoArgs, Run: taskEnqueue, Example: heredoc.Doc(` $ asynq task enqueue -t footype -l barpayload $ asynq task enqueue -t footask -l barpayload --retry 3 --id f1720682-f5a6-4db1-8953-4f48ae541d0f --queue bazqueue --timeout 100s --deadline 2024-12-14T01:23:45Z --unique 100s --process_at 2024-12-14T01:22:05Z --process_in 100s --retention 5h --group baygroup`), } var taskArchiveAllCmd = &cobra.Command{ Use: "archiveall --queue= --state=", Short: "Archive all tasks in the given state", Args: cobra.NoArgs, Run: taskArchiveAll, Example: heredoc.Doc(` $ asynq task archiveall --queue=myqueue --state=retry $ asynq task archiveall --queue=myqueue --state=aggregating --group=mygroup`), } var taskDeleteAllCmd = &cobra.Command{ Use: "deleteall --queue= --state=", Short: "Delete all tasks in the given state", Args: cobra.NoArgs, Run: taskDeleteAll, Example: heredoc.Doc(` $ asynq task deleteall --queue=myqueue --state=archived $ asynq task deleteall --queue=myqueue --state=aggregating --group=mygroup`), } var taskRunAllCmd = &cobra.Command{ Use: "runall --queue= --state=", Short: "Run all tasks in the given state", Args: cobra.NoArgs, Run: taskRunAll, Example: heredoc.Doc(` $ asynq task runall --queue=myqueue --state=retry $ asynq task runall --queue=myqueue --state=aggregating --group=mygroup`), } func taskList(cmd *cobra.Command, args []string) { qname, err := cmd.Flags().GetString("queue") if err != nil { fmt.Println(err) os.Exit(1) } state, err := cmd.Flags().GetString("state") if err != nil { fmt.Println(err) os.Exit(1) } pageNum, err := cmd.Flags().GetInt("page") if err != nil { fmt.Println(err) os.Exit(1) } pageSize, err := cmd.Flags().GetInt("size") if err != nil { fmt.Println(err) os.Exit(1) } switch state { case "active": listActiveTasks(qname, pageNum, pageSize) case "pending": listPendingTasks(qname, pageNum, pageSize) case "scheduled": listScheduledTasks(qname, pageNum, pageSize) case "retry": listRetryTasks(qname, pageNum, pageSize) case "archived": listArchivedTasks(qname, pageNum, pageSize) case "completed": listCompletedTasks(qname, pageNum, pageSize) case "aggregating": group, err := cmd.Flags().GetString("group") if err != nil { fmt.Println(err) os.Exit(1) } if group == "" { fmt.Println("Flag --group is required for listing aggregating tasks") os.Exit(1) } listAggregatingTasks(qname, group, pageNum, pageSize) default: fmt.Printf("error: state=%q is not supported\n", state) os.Exit(1) } } func listActiveTasks(qname string, pageNum, pageSize int) { i := createInspector() tasks, err := i.ListActiveTasks(qname, asynq.PageSize(pageSize), asynq.Page(pageNum)) if err != nil { fmt.Println(err) os.Exit(1) } if len(tasks) == 0 { fmt.Printf("No active tasks in %q queue\n", qname) return } printTable( []string{"ID", "Type", "Payload"}, func(w io.Writer, tmpl string) { for _, t := range tasks { fmt.Fprintf(w, tmpl, t.ID, t.Type, sprintBytes(t.Payload)) } }, ) } func listPendingTasks(qname string, pageNum, pageSize int) { i := createInspector() tasks, err := i.ListPendingTasks(qname, asynq.PageSize(pageSize), asynq.Page(pageNum)) if err != nil { fmt.Println(err) os.Exit(1) } if len(tasks) == 0 { fmt.Printf("No pending tasks in %q queue\n", qname) return } printTable( []string{"ID", "Type", "Payload"}, func(w io.Writer, tmpl string) { for _, t := range tasks { fmt.Fprintf(w, tmpl, t.ID, t.Type, sprintBytes(t.Payload)) } }, ) } func listScheduledTasks(qname string, pageNum, pageSize int) { i := createInspector() tasks, err := i.ListScheduledTasks(qname, asynq.PageSize(pageSize), asynq.Page(pageNum)) if err != nil { fmt.Println(err) os.Exit(1) } if len(tasks) == 0 { fmt.Printf("No scheduled tasks in %q queue\n", qname) return } printTable( []string{"ID", "Type", "Payload", "Process In"}, func(w io.Writer, tmpl string) { for _, t := range tasks { fmt.Fprintf(w, tmpl, t.ID, t.Type, sprintBytes(t.Payload), formatProcessAt(t.NextProcessAt)) } }, ) } // formatProcessAt formats next process at time to human friendly string. // If processAt time is in the past, returns "right now". // If processAt time is in the future, returns "in xxx" where xxx is the duration from now. func formatProcessAt(processAt time.Time) string { d := processAt.Sub(time.Now()) if d < 0 { return "right now" } return fmt.Sprintf("in %v", d.Round(time.Second)) } func listRetryTasks(qname string, pageNum, pageSize int) { i := createInspector() tasks, err := i.ListRetryTasks(qname, asynq.PageSize(pageSize), asynq.Page(pageNum)) if err != nil { fmt.Println(err) os.Exit(1) } if len(tasks) == 0 { fmt.Printf("No retry tasks in %q queue\n", qname) return } printTable( []string{"ID", "Type", "Payload", "Next Retry", "Last Error", "Last Failed", "Retried", "Max Retry"}, func(w io.Writer, tmpl string) { for _, t := range tasks { fmt.Fprintf(w, tmpl, t.ID, t.Type, sprintBytes(t.Payload), formatProcessAt(t.NextProcessAt), t.LastErr, formatPastTime(t.LastFailedAt), t.Retried, t.MaxRetry) } }, ) } func listArchivedTasks(qname string, pageNum, pageSize int) { i := createInspector() tasks, err := i.ListArchivedTasks(qname, asynq.PageSize(pageSize), asynq.Page(pageNum)) if err != nil { fmt.Println(err) os.Exit(1) } if len(tasks) == 0 { fmt.Printf("No archived tasks in %q queue\n", qname) return } printTable( []string{"ID", "Type", "Payload", "Last Failed", "Last Error"}, func(w io.Writer, tmpl string) { for _, t := range tasks { fmt.Fprintf(w, tmpl, t.ID, t.Type, sprintBytes(t.Payload), formatPastTime(t.LastFailedAt), t.LastErr) } }) } func listCompletedTasks(qname string, pageNum, pageSize int) { i := createInspector() tasks, err := i.ListCompletedTasks(qname, asynq.PageSize(pageSize), asynq.Page(pageNum)) if err != nil { fmt.Println(err) os.Exit(1) } if len(tasks) == 0 { fmt.Printf("No completed tasks in %q queue\n", qname) return } printTable( []string{"ID", "Type", "Payload", "CompletedAt", "Result"}, func(w io.Writer, tmpl string) { for _, t := range tasks { fmt.Fprintf(w, tmpl, t.ID, t.Type, sprintBytes(t.Payload), formatPastTime(t.CompletedAt), sprintBytes(t.Result)) } }) } func listAggregatingTasks(qname, group string, pageNum, pageSize int) { i := createInspector() tasks, err := i.ListAggregatingTasks(qname, group, asynq.PageSize(pageSize), asynq.Page(pageNum)) if err != nil { fmt.Println(err) os.Exit(1) } if len(tasks) == 0 { fmt.Printf("No aggregating tasks in group %q \n", group) return } printTable( []string{"ID", "Type", "Payload", "Group"}, func(w io.Writer, tmpl string) { for _, t := range tasks { fmt.Fprintf(w, tmpl, t.ID, t.Type, sprintBytes(t.Payload), t.Group) } }, ) } func taskCancel(cmd *cobra.Command, args []string) { i := createInspector() for _, id := range args { if err := i.CancelProcessing(id); err != nil { fmt.Printf("error: could not send cancelation signal: %v\n", err) continue } fmt.Printf("Sent cancelation signal for task %s\n", id) } } func taskInspect(cmd *cobra.Command, args []string) { qname, err := cmd.Flags().GetString("queue") if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } id, err := cmd.Flags().GetString("id") if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } i := createInspector() info, err := i.GetTaskInfo(qname, id) if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } printTaskInfo(info) } func printTaskInfo(info *asynq.TaskInfo) { bold := color.New(color.Bold) bold.Println("Task Info") fmt.Printf("Queue: %s\n", info.Queue) fmt.Printf("ID: %s\n", info.ID) fmt.Printf("Type: %s\n", info.Type) fmt.Printf("State: %v\n", info.State) fmt.Printf("Retried: %d/%d\n", info.Retried, info.MaxRetry) fmt.Println() fmt.Printf("Next process time: %s\n", formatNextProcessAt(info.NextProcessAt)) if len(info.LastErr) != 0 { fmt.Println() bold.Println("Last Failure") fmt.Printf("Failed at: %s\n", formatPastTime(info.LastFailedAt)) fmt.Printf("Error message: %s\n", info.LastErr) } } func formatNextProcessAt(processAt time.Time) string { if processAt.IsZero() || processAt.Unix() == 0 { return "n/a" } if processAt.Before(time.Now()) { return "now" } return fmt.Sprintf("%s (in %v)", processAt.Format(time.UnixDate), processAt.Sub(time.Now()).Round(time.Second)) } // formatPastTime takes t which is time in the past and returns a user-friendly string. func formatPastTime(t time.Time) string { if t.IsZero() || t.Unix() == 0 { return "" } return t.Format(time.UnixDate) } func taskArchive(cmd *cobra.Command, args []string) { qname, err := cmd.Flags().GetString("queue") if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } id, err := cmd.Flags().GetString("id") if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } i := createInspector() err = i.ArchiveTask(qname, id) if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } fmt.Println("task archived") } func taskDelete(cmd *cobra.Command, args []string) { qname, err := cmd.Flags().GetString("queue") if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } id, err := cmd.Flags().GetString("id") if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } i := createInspector() err = i.DeleteTask(qname, id) if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } fmt.Println("task deleted") } func taskRun(cmd *cobra.Command, args []string) { qname, err := cmd.Flags().GetString("queue") if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } id, err := cmd.Flags().GetString("id") if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } i := createInspector() err = i.RunTask(qname, id) if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } fmt.Println("task is now pending") } func taskEnqueue(cmd *cobra.Command, args []string) { typeName, err := cmd.Flags().GetString("type_name") if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } payload, err := cmd.Flags().GetString("payload") if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } // For all of the optional flags, we need to explicitly check whether they were set or // not; for consistency we want to use the defaults set in composeOptions() rather than // the ones in the flag definitions. opts := []asynq.Option{} if cmd.Flags().Changed("retry") { retry, err := cmd.Flags().GetInt("retry") if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } opts = append(opts, asynq.MaxRetry(retry)) } if cmd.Flags().Changed("queue") { queue, err := cmd.Flags().GetString("queue") if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } opts = append(opts, asynq.Queue(queue)) } if cmd.Flags().Changed("id") { id, err := cmd.Flags().GetString("id") if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } opts = append(opts, asynq.TaskID(id)) } if cmd.Flags().Changed("timeout") { opts = append(opts, asynq.Timeout(getDuration(cmd, "timeout"))) } if cmd.Flags().Changed("deadline") { opts = append(opts, asynq.Deadline(getTime(cmd, "deadline"))) } if cmd.Flags().Changed("unique") { opts = append(opts, asynq.Unique(getDuration(cmd, "unique"))) } if cmd.Flags().Changed("process_at") { opts = append(opts, asynq.ProcessAt(getTime(cmd, "process_at"))) } if cmd.Flags().Changed("process_in") { opts = append(opts, asynq.ProcessIn(getDuration(cmd, "process_in"))) } if cmd.Flags().Changed("retention") { opts = append(opts, asynq.Retention(getDuration(cmd, "retention"))) } if cmd.Flags().Changed("group") { group, err := cmd.Flags().GetString("group") if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } opts = append(opts, asynq.Group(group)) } c := createClient() task := asynq.NewTask(typeName, []byte(payload), opts...) taskInfo, err := c.Enqueue(task) if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } fmt.Printf("Enqueued task %s to queue %s\n", taskInfo.ID, taskInfo.Queue) } func taskArchiveAll(cmd *cobra.Command, args []string) { qname, err := cmd.Flags().GetString("queue") if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } state, err := cmd.Flags().GetString("state") if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } i := createInspector() var n int switch state { case "pending": n, err = i.ArchiveAllPendingTasks(qname) case "scheduled": n, err = i.ArchiveAllScheduledTasks(qname) case "retry": n, err = i.ArchiveAllRetryTasks(qname) case "aggregating": group, err := cmd.Flags().GetString("group") if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } if group == "" { fmt.Println("error: Flag --group is required for aggregating tasks") os.Exit(1) } n, err = i.ArchiveAllAggregatingTasks(qname, group) default: fmt.Printf("error: unsupported state %q\n", state) os.Exit(1) } if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } fmt.Printf("%d tasks archived\n", n) } func taskDeleteAll(cmd *cobra.Command, args []string) { qname, err := cmd.Flags().GetString("queue") if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } state, err := cmd.Flags().GetString("state") if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } i := createInspector() var n int switch state { case "pending": n, err = i.DeleteAllPendingTasks(qname) case "scheduled": n, err = i.DeleteAllScheduledTasks(qname) case "retry": n, err = i.DeleteAllRetryTasks(qname) case "archived": n, err = i.DeleteAllArchivedTasks(qname) case "completed": n, err = i.DeleteAllCompletedTasks(qname) case "aggregating": group, err := cmd.Flags().GetString("group") if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } if group == "" { fmt.Println("error: Flag --group is required for aggregating tasks") os.Exit(1) } n, err = i.DeleteAllAggregatingTasks(qname, group) default: fmt.Printf("error: unsupported state %q\n", state) os.Exit(1) } if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } fmt.Printf("%d tasks deleted\n", n) } func taskRunAll(cmd *cobra.Command, args []string) { qname, err := cmd.Flags().GetString("queue") if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } state, err := cmd.Flags().GetString("state") if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } i := createInspector() var n int switch state { case "scheduled": n, err = i.RunAllScheduledTasks(qname) case "retry": n, err = i.RunAllRetryTasks(qname) case "archived": n, err = i.RunAllArchivedTasks(qname) case "aggregating": group, err := cmd.Flags().GetString("group") if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } if group == "" { fmt.Println("error: Flag --group is required for aggregating tasks") os.Exit(1) } n, err = i.RunAllAggregatingTasks(qname, group) default: fmt.Printf("error: unsupported state %q\n", state) os.Exit(1) } if err != nil { fmt.Printf("error: %v\n", err) os.Exit(1) } fmt.Printf("%d tasks are now pending\n", n) } ================================================ FILE: tools/asynq/main.go ================================================ // Copyright 2020 Kentaro Hibino. All rights reserved. // Use of this source code is governed by a MIT license // that can be found in the LICENSE file. package main import "github.com/hibiken/asynq/tools/asynq/cmd" func main() { cmd.Execute() } ================================================ FILE: tools/go.mod ================================================ module github.com/hibiken/asynq/tools go 1.22 require ( github.com/MakeNowJust/heredoc/v2 v2.0.1 github.com/fatih/color v1.18.0 github.com/gdamore/tcell/v2 v2.5.1 github.com/google/go-cmp v0.6.0 github.com/hibiken/asynq v0.25.0 github.com/hibiken/asynq/x v0.0.0-20220131170841-349f4c50fb1d github.com/mattn/go-runewidth v0.0.16 github.com/mitchellh/go-homedir v1.1.0 github.com/prometheus/client_golang v1.11.1 github.com/redis/go-redis/v9 v9.7.0 github.com/spf13/cobra v1.1.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.7.0 golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136 ) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/gdamore/encoding v1.0.0 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magiconair/properties v1.8.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/mitchellh/mapstructure v1.1.2 // indirect github.com/pelletier/go-toml v1.2.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.26.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/spf13/afero v1.1.2 // indirect github.com/spf13/cast v1.7.0 // indirect github.com/spf13/jwalterweatherman v1.0.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect golang.org/x/sys v0.26.0 // indirect golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect golang.org/x/text v0.3.8 // indirect golang.org/x/time v0.7.0 // indirect google.golang.org/protobuf v1.35.1 // indirect gopkg.in/ini.v1 v1.51.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) ================================================ FILE: tools/go.sum ================================================ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/MakeNowJust/heredoc/v2 v2.0.1 h1:rlCHh70XXXv7toz95ajQWOWQnN4WNLt0TdpZYIR/J6A= github.com/MakeNowJust/heredoc/v2 v2.0.1/go.mod h1:6/2Abh5s+hc3g9nbWLe9ObDIOhaRrqsyY9MWy+4JdRM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko= github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= github.com/gdamore/tcell/v2 v2.5.1 h1:zc3LPdpK184lBW7syF2a5C6MV827KmErk9jGVnmsl/I= github.com/gdamore/tcell/v2 v2.5.1/go.mod h1:wSkrPaXoiIWZqW/g7Px4xc79di6FTcpB8tvaKJ6uGBo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-redis/redis/v8 v8.11.2/go.mod h1:DLomh7y2e3ggQXQLd1YgmvIfecPJoFl7WU5SOQ/r06M= github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hibiken/asynq v0.19.0/go.mod h1:tyc63ojaW8SJ5SBm8mvI4DDONsguP5HE85EEl4Qr5Ig= github.com/hibiken/asynq v0.25.0 h1:VCPyRRrrjFChsTSI8x5OCPu51MlEz6Rk+1p0kHKnZug= github.com/hibiken/asynq v0.25.0/go.mod h1:DYQ1etBEl2Y+uSkqFElGYbk3M0ujLVwCfWE+TlvxtEk= github.com/hibiken/asynq/x v0.0.0-20220131170841-349f4c50fb1d h1:Er+U+9PmnyRHRDQjSjRQ24HoWvOY7w9Pk7bUPYM3Ags= github.com/hibiken/asynq/x v0.0.0-20220131170841-349f4c50fb1d/go.mod h1:VmxwMfMKyb6gyv8xG0oOBMXIhquWKPx+zPtbVBd2Q1s= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s= github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E= github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136 h1:A1gGSx58LAGVHUUsOf7IiR0u8Xb6W51gRwfDBhkdcaw= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220318055525-2edf467146b5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= ================================================ FILE: tools/metrics_exporter/main.go ================================================ package main import ( "flag" "fmt" "log" "net/http" "github.com/hibiken/asynq" "github.com/hibiken/asynq/x/metrics" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" ) // Declare command-line flags. // These variables are binded to flags in init(). var ( flagRedisAddr string flagRedisDB int flagRedisPassword string flagRedisUsername string flagPort int ) func init() { flag.StringVar(&flagRedisAddr, "redis-addr", "127.0.0.1:6379", "host:port of redis server to connect to") flag.IntVar(&flagRedisDB, "redis-db", 0, "redis DB number to use") flag.StringVar(&flagRedisPassword, "redis-password", "", "password used to connect to redis server") flag.StringVar(&flagRedisUsername, "redis-username", "", "username used to connect to redis server") flag.IntVar(&flagPort, "port", 9876, "port to use for the HTTP server") } func main() { flag.Parse() // Using NewPedanticRegistry here to test the implementation of Collectors and Metrics. reg := prometheus.NewPedanticRegistry() inspector := asynq.NewInspector(asynq.RedisClientOpt{ Addr: flagRedisAddr, DB: flagRedisDB, Password: flagRedisPassword, Username: flagRedisUsername, }) reg.MustRegister( metrics.NewQueueMetricsCollector(inspector), // Add the standard process and go metrics to the registry collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), collectors.NewGoCollector(), ) http.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{})) log.Printf("exporter server is listening on port: %d\n", flagPort) log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", flagPort), nil)) } ================================================ FILE: x/go.mod ================================================ module github.com/hibiken/asynq/x go 1.22 require ( github.com/google/uuid v1.6.0 github.com/hibiken/asynq v0.25.0 github.com/prometheus/client_golang v1.20.5 github.com/redis/go-redis/v9 v9.7.0 ) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/spf13/cast v1.7.0 // indirect golang.org/x/sys v0.26.0 // indirect golang.org/x/time v0.7.0 // indirect google.golang.org/protobuf v1.35.1 // indirect ) ================================================ FILE: x/go.sum ================================================ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hibiken/asynq v0.25.0 h1:VCPyRRrrjFChsTSI8x5OCPu51MlEz6Rk+1p0kHKnZug= github.com/hibiken/asynq v0.25.0/go.mod h1:DYQ1etBEl2Y+uSkqFElGYbk3M0ujLVwCfWE+TlvxtEk= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E= github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= ================================================ FILE: x/metrics/metrics.go ================================================ // Package metrics provides implementations of prometheus.Collector to collect Asynq queue metrics. package metrics import ( "fmt" "log" "github.com/hibiken/asynq" "github.com/prometheus/client_golang/prometheus" ) // Namespace used in fully-qualified metrics names. const namespace = "asynq" // QueueMetricsCollector gathers queue metrics. // It implements prometheus.Collector interface. // // All metrics exported from this collector have prefix "asynq". type QueueMetricsCollector struct { inspector *asynq.Inspector } // collectQueueInfo gathers QueueInfo of all queues. // Since this operation is expensive, it must be called once per collection. func (qmc *QueueMetricsCollector) collectQueueInfo() ([]*asynq.QueueInfo, error) { qnames, err := qmc.inspector.Queues() if err != nil { return nil, fmt.Errorf("failed to get queue names: %w", err) } infos := make([]*asynq.QueueInfo, len(qnames)) for i, qname := range qnames { qinfo, err := qmc.inspector.GetQueueInfo(qname) if err != nil { return nil, fmt.Errorf("failed to get queue info: %w", err) } infos[i] = qinfo } return infos, nil } // Descriptors used by QueueMetricsCollector var ( tasksQueuedDesc = prometheus.NewDesc( prometheus.BuildFQName(namespace, "", "tasks_enqueued_total"), "Number of tasks enqueued; broken down by queue and state.", []string{"queue", "state"}, nil, ) queueSizeDesc = prometheus.NewDesc( prometheus.BuildFQName(namespace, "", "queue_size"), "Number of tasks in a queue", []string{"queue"}, nil, ) queueLatencyDesc = prometheus.NewDesc( prometheus.BuildFQName(namespace, "", "queue_latency_seconds"), "Number of seconds the oldest pending task is waiting in pending state to be processed.", []string{"queue"}, nil, ) queueMemUsgDesc = prometheus.NewDesc( prometheus.BuildFQName(namespace, "", "queue_memory_usage_approx_bytes"), "Number of memory used by a given queue (approximated number by sampling).", []string{"queue"}, nil, ) tasksProcessedTotalDesc = prometheus.NewDesc( prometheus.BuildFQName(namespace, "", "tasks_processed_total"), "Number of tasks processed (both succeeded and failed); broken down by queue", []string{"queue"}, nil, ) tasksFailedTotalDesc = prometheus.NewDesc( prometheus.BuildFQName(namespace, "", "tasks_failed_total"), "Number of tasks failed; broken down by queue", []string{"queue"}, nil, ) pausedQueues = prometheus.NewDesc( prometheus.BuildFQName(namespace, "", "queue_paused_total"), "Number of queues paused", []string{"queue"}, nil, ) ) func (qmc *QueueMetricsCollector) Describe(ch chan<- *prometheus.Desc) { prometheus.DescribeByCollect(qmc, ch) } func (qmc *QueueMetricsCollector) Collect(ch chan<- prometheus.Metric) { queueInfos, err := qmc.collectQueueInfo() if err != nil { log.Printf("Failed to collect metrics data: %v", err) } for _, info := range queueInfos { ch <- prometheus.MustNewConstMetric( tasksQueuedDesc, prometheus.GaugeValue, float64(info.Active), info.Queue, "active", ) ch <- prometheus.MustNewConstMetric( tasksQueuedDesc, prometheus.GaugeValue, float64(info.Pending), info.Queue, "pending", ) ch <- prometheus.MustNewConstMetric( tasksQueuedDesc, prometheus.GaugeValue, float64(info.Scheduled), info.Queue, "scheduled", ) ch <- prometheus.MustNewConstMetric( tasksQueuedDesc, prometheus.GaugeValue, float64(info.Retry), info.Queue, "retry", ) ch <- prometheus.MustNewConstMetric( tasksQueuedDesc, prometheus.GaugeValue, float64(info.Archived), info.Queue, "archived", ) ch <- prometheus.MustNewConstMetric( tasksQueuedDesc, prometheus.GaugeValue, float64(info.Completed), info.Queue, "completed", ) ch <- prometheus.MustNewConstMetric( queueSizeDesc, prometheus.GaugeValue, float64(info.Size), info.Queue, ) ch <- prometheus.MustNewConstMetric( queueLatencyDesc, prometheus.GaugeValue, info.Latency.Seconds(), info.Queue, ) ch <- prometheus.MustNewConstMetric( queueMemUsgDesc, prometheus.GaugeValue, float64(info.MemoryUsage), info.Queue, ) ch <- prometheus.MustNewConstMetric( tasksProcessedTotalDesc, prometheus.CounterValue, float64(info.ProcessedTotal), info.Queue, ) ch <- prometheus.MustNewConstMetric( tasksFailedTotalDesc, prometheus.CounterValue, float64(info.FailedTotal), info.Queue, ) pausedValue := 0 // zero to indicate "not paused" if info.Paused { pausedValue = 1 } ch <- prometheus.MustNewConstMetric( pausedQueues, prometheus.GaugeValue, float64(pausedValue), info.Queue, ) } } // NewQueueMetricsCollector returns a collector that exports metrics about Asynq queues. func NewQueueMetricsCollector(inspector *asynq.Inspector) *QueueMetricsCollector { return &QueueMetricsCollector{inspector: inspector} } ================================================ FILE: x/rate/example_test.go ================================================ package rate_test import ( "context" "fmt" "time" "github.com/hibiken/asynq" "github.com/hibiken/asynq/x/rate" ) type RateLimitError struct { RetryIn time.Duration } func (e *RateLimitError) Error() string { return fmt.Sprintf("rate limited (retry in %v)", e.RetryIn) } func ExampleNewSemaphore() { redisConnOpt := asynq.RedisClientOpt{Addr: ":6379"} sema := rate.NewSemaphore(redisConnOpt, "my_queue", 10) // call sema.Close() when appropriate _ = asynq.HandlerFunc(func(ctx context.Context, task *asynq.Task) error { ok, err := sema.Acquire(ctx) if err != nil { return err } if !ok { return &RateLimitError{RetryIn: 30 * time.Second} } // Make sure to release the token once we're done. defer sema.Release(ctx) // Process task return nil }) } ================================================ FILE: x/rate/semaphore.go ================================================ // Package rate contains rate limiting strategies for asynq.Handler(s). package rate import ( "context" "fmt" "strings" "time" "github.com/hibiken/asynq" asynqcontext "github.com/hibiken/asynq/internal/context" "github.com/redis/go-redis/v9" ) // NewSemaphore creates a counting Semaphore for the given scope with the given number of tokens. func NewSemaphore(rco asynq.RedisConnOpt, scope string, maxTokens int) *Semaphore { rc, ok := rco.MakeRedisClient().(redis.UniversalClient) if !ok { panic(fmt.Sprintf("rate.NewSemaphore: unsupported RedisConnOpt type %T", rco)) } if maxTokens < 1 { panic("rate.NewSemaphore: maxTokens cannot be less than 1") } if len(strings.TrimSpace(scope)) == 0 { panic("rate.NewSemaphore: scope should not be empty") } return &Semaphore{ rc: rc, scope: scope, maxTokens: maxTokens, } } // Semaphore is a distributed counting semaphore which can be used to set maxTokens across multiple asynq servers. type Semaphore struct { rc redis.UniversalClient maxTokens int scope string } // KEYS[1] -> asynq:sema: // ARGV[1] -> max concurrency // ARGV[2] -> current time in unix time // ARGV[3] -> deadline in unix time // ARGV[4] -> task ID var acquireCmd = redis.NewScript(` redis.call("ZREMRANGEBYSCORE", KEYS[1], "-inf", tonumber(ARGV[2])-1) local count = redis.call("ZCARD", KEYS[1]) if (count < tonumber(ARGV[1])) then redis.call("ZADD", KEYS[1], ARGV[3], ARGV[4]) return 'true' else return 'false' end `) // Acquire attempts to acquire a token from the semaphore. // - Returns (true, nil), iff semaphore key exists and current value is less than maxTokens // - Returns (false, nil) when token cannot be acquired // - Returns (false, error) otherwise // // The context.Context passed to Acquire must have a deadline set, // this ensures that token is released if the job goroutine crashes and does not call Release. func (s *Semaphore) Acquire(ctx context.Context) (bool, error) { d, ok := ctx.Deadline() if !ok { return false, fmt.Errorf("provided context must have a deadline") } taskID, ok := asynqcontext.GetTaskID(ctx) if !ok { return false, fmt.Errorf("provided context is missing task ID value") } return acquireCmd.Run(ctx, s.rc, []string{semaphoreKey(s.scope)}, s.maxTokens, time.Now().Unix(), d.Unix(), taskID, ).Bool() } // Release will release the token on the counting semaphore. func (s *Semaphore) Release(ctx context.Context) error { taskID, ok := asynqcontext.GetTaskID(ctx) if !ok { return fmt.Errorf("provided context is missing task ID value") } n, err := s.rc.ZRem(ctx, semaphoreKey(s.scope), taskID).Result() if err != nil { return fmt.Errorf("redis command failed: %w", err) } if n == 0 { return fmt.Errorf("no token found for task %q", taskID) } return nil } // Close closes the connection to redis. func (s *Semaphore) Close() error { return s.rc.Close() } func semaphoreKey(scope string) string { return "asynq:sema:" + scope } ================================================ FILE: x/rate/semaphore_test.go ================================================ package rate import ( "context" "flag" "fmt" "strings" "testing" "time" "github.com/google/uuid" "github.com/hibiken/asynq" "github.com/hibiken/asynq/internal/base" asynqcontext "github.com/hibiken/asynq/internal/context" "github.com/redis/go-redis/v9" ) var ( redisAddr string redisDB int useRedisCluster bool redisClusterAddrs string // comma-separated list of host:port ) func init() { flag.StringVar(&redisAddr, "redis_addr", "localhost:6379", "redis address to use in testing") flag.IntVar(&redisDB, "redis_db", 14, "redis db number to use in testing") flag.BoolVar(&useRedisCluster, "redis_cluster", false, "use redis cluster as a broker in testing") flag.StringVar(&redisClusterAddrs, "redis_cluster_addrs", "localhost:7000,localhost:7001,localhost:7002", "comma separated list of redis server addresses") } func TestNewSemaphore(t *testing.T) { tests := []struct { desc string name string maxConcurrency int wantPanic string connOpt asynq.RedisConnOpt }{ { desc: "Bad RedisConnOpt", wantPanic: "rate.NewSemaphore: unsupported RedisConnOpt type *rate.badConnOpt", connOpt: &badConnOpt{}, }, { desc: "Zero maxTokens should panic", wantPanic: "rate.NewSemaphore: maxTokens cannot be less than 1", }, { desc: "Empty scope should panic", maxConcurrency: 2, name: " ", wantPanic: "rate.NewSemaphore: scope should not be empty", }, } for _, tt := range tests { t.Run(tt.desc, func(t *testing.T) { if tt.wantPanic != "" { defer func() { if r := recover(); r.(string) != tt.wantPanic { t.Errorf("%s;\nNewSemaphore should panic with msg: %s, got %s", tt.desc, tt.wantPanic, r.(string)) } }() } opt := tt.connOpt if tt.connOpt == nil { opt = getRedisConnOpt(t) } sema := NewSemaphore(opt, tt.name, tt.maxConcurrency) defer sema.Close() }) } } func TestNewSemaphore_Acquire(t *testing.T) { tests := []struct { desc string name string maxConcurrency int taskIDs []string ctxFunc func(string) (context.Context, context.CancelFunc) want []bool }{ { desc: "Should acquire token when current token count is less than maxTokens", name: "task-1", maxConcurrency: 3, taskIDs: []string{uuid.NewString(), uuid.NewString()}, ctxFunc: func(id string) (context.Context, context.CancelFunc) { return asynqcontext.New(context.Background(), &base.TaskMessage{ ID: id, Queue: "task-1", }, time.Now().Add(time.Second)) }, want: []bool{true, true}, }, { desc: "Should fail acquiring token when current token count is equal to maxTokens", name: "task-2", maxConcurrency: 3, taskIDs: []string{uuid.NewString(), uuid.NewString(), uuid.NewString(), uuid.NewString()}, ctxFunc: func(id string) (context.Context, context.CancelFunc) { return asynqcontext.New(context.Background(), &base.TaskMessage{ ID: id, Queue: "task-2", }, time.Now().Add(time.Second)) }, want: []bool{true, true, true, false}, }, } for _, tt := range tests { t.Run(tt.desc, func(t *testing.T) { opt := getRedisConnOpt(t) rc := opt.MakeRedisClient().(redis.UniversalClient) defer rc.Close() if err := rc.Del(context.Background(), semaphoreKey(tt.name)).Err(); err != nil { t.Errorf("%s;\nredis.UniversalClient.Del() got error %v", tt.desc, err) } sema := NewSemaphore(opt, tt.name, tt.maxConcurrency) defer sema.Close() for i := 0; i < len(tt.taskIDs); i++ { ctx, cancel := tt.ctxFunc(tt.taskIDs[i]) got, err := sema.Acquire(ctx) if err != nil { t.Errorf("%s;\nSemaphore.Acquire() got error %v", tt.desc, err) } if got != tt.want[i] { t.Errorf("%s;\nSemaphore.Acquire(ctx) returned %v, want %v", tt.desc, got, tt.want[i]) } cancel() } }) } } func TestNewSemaphore_Acquire_Error(t *testing.T) { tests := []struct { desc string name string maxConcurrency int taskIDs []string ctxFunc func(string) (context.Context, context.CancelFunc) errStr string }{ { desc: "Should return error if context has no deadline", name: "task-3", maxConcurrency: 1, taskIDs: []string{uuid.NewString(), uuid.NewString()}, ctxFunc: func(id string) (context.Context, context.CancelFunc) { return context.Background(), func() {} }, errStr: "provided context must have a deadline", }, { desc: "Should return error when context is missing taskID", name: "task-4", maxConcurrency: 1, taskIDs: []string{uuid.NewString()}, ctxFunc: func(_ string) (context.Context, context.CancelFunc) { return context.WithTimeout(context.Background(), time.Second) }, errStr: "provided context is missing task ID value", }, } for _, tt := range tests { t.Run(tt.desc, func(t *testing.T) { opt := getRedisConnOpt(t) rc := opt.MakeRedisClient().(redis.UniversalClient) defer rc.Close() if err := rc.Del(context.Background(), semaphoreKey(tt.name)).Err(); err != nil { t.Errorf("%s;\nredis.UniversalClient.Del() got error %v", tt.desc, err) } sema := NewSemaphore(opt, tt.name, tt.maxConcurrency) defer sema.Close() for i := 0; i < len(tt.taskIDs); i++ { ctx, cancel := tt.ctxFunc(tt.taskIDs[i]) _, err := sema.Acquire(ctx) if err == nil || err.Error() != tt.errStr { t.Errorf("%s;\nSemaphore.Acquire() got error %v want error %v", tt.desc, err, tt.errStr) } cancel() } }) } } func TestNewSemaphore_Acquire_StaleToken(t *testing.T) { opt := getRedisConnOpt(t) rc := opt.MakeRedisClient().(redis.UniversalClient) defer rc.Close() taskID := uuid.NewString() // adding a set member to mimic the case where token is acquired but the goroutine crashed, // in which case, the token will not be explicitly removed and should be present already rc.ZAdd(context.Background(), semaphoreKey("stale-token"), redis.Z{ Score: float64(time.Now().Add(-10 * time.Second).Unix()), Member: taskID, }) sema := NewSemaphore(opt, "stale-token", 1) defer sema.Close() ctx, cancel := asynqcontext.New(context.Background(), &base.TaskMessage{ ID: taskID, Queue: "task-1", }, time.Now().Add(time.Second)) defer cancel() got, err := sema.Acquire(ctx) if err != nil { t.Errorf("Acquire_StaleToken;\nSemaphore.Acquire() got error %v", err) } if !got { t.Error("Acquire_StaleToken;\nSemaphore.Acquire() got false want true") } } func TestNewSemaphore_Release(t *testing.T) { tests := []struct { desc string name string taskIDs []string ctxFunc func(string) (context.Context, context.CancelFunc) wantCount int64 }{ { desc: "Should decrease token count", name: "task-5", taskIDs: []string{uuid.NewString()}, ctxFunc: func(id string) (context.Context, context.CancelFunc) { return asynqcontext.New(context.Background(), &base.TaskMessage{ ID: id, Queue: "task-3", }, time.Now().Add(time.Second)) }, }, { desc: "Should decrease token count by 2", name: "task-6", taskIDs: []string{uuid.NewString(), uuid.NewString()}, ctxFunc: func(id string) (context.Context, context.CancelFunc) { return asynqcontext.New(context.Background(), &base.TaskMessage{ ID: id, Queue: "task-4", }, time.Now().Add(time.Second)) }, }, } for _, tt := range tests { t.Run(tt.desc, func(t *testing.T) { opt := getRedisConnOpt(t) rc := opt.MakeRedisClient().(redis.UniversalClient) defer rc.Close() if err := rc.Del(context.Background(), semaphoreKey(tt.name)).Err(); err != nil { t.Errorf("%s;\nredis.UniversalClient.Del() got error %v", tt.desc, err) } var members []redis.Z for i := 0; i < len(tt.taskIDs); i++ { members = append(members, redis.Z{ Score: float64(time.Now().Add(time.Duration(i) * time.Second).Unix()), Member: tt.taskIDs[i], }) } if err := rc.ZAdd(context.Background(), semaphoreKey(tt.name), members...).Err(); err != nil { t.Errorf("%s;\nredis.UniversalClient.ZAdd() got error %v", tt.desc, err) } sema := NewSemaphore(opt, tt.name, 3) defer sema.Close() for i := 0; i < len(tt.taskIDs); i++ { ctx, cancel := tt.ctxFunc(tt.taskIDs[i]) if err := sema.Release(ctx); err != nil { t.Errorf("%s;\nSemaphore.Release() got error %v", tt.desc, err) } cancel() } i, err := rc.ZCount(context.Background(), semaphoreKey(tt.name), "-inf", "+inf").Result() if err != nil { t.Errorf("%s;\nredis.UniversalClient.ZCount() got error %v", tt.desc, err) } if i != tt.wantCount { t.Errorf("%s;\nSemaphore.Release(ctx) didn't release token, got %v want 0", tt.desc, i) } }) } } func TestNewSemaphore_Release_Error(t *testing.T) { testID := uuid.NewString() tests := []struct { desc string name string taskIDs []string ctxFunc func(string) (context.Context, context.CancelFunc) errStr string }{ { desc: "Should return error when context is missing taskID", name: "task-7", taskIDs: []string{uuid.NewString()}, ctxFunc: func(_ string) (context.Context, context.CancelFunc) { return context.WithTimeout(context.Background(), time.Second) }, errStr: "provided context is missing task ID value", }, { desc: "Should return error when context has taskID which never acquired token", name: "task-8", taskIDs: []string{uuid.NewString()}, ctxFunc: func(_ string) (context.Context, context.CancelFunc) { return asynqcontext.New(context.Background(), &base.TaskMessage{ ID: testID, Queue: "task-4", }, time.Now().Add(time.Second)) }, errStr: fmt.Sprintf("no token found for task %q", testID), }, } for _, tt := range tests { t.Run(tt.desc, func(t *testing.T) { opt := getRedisConnOpt(t) rc := opt.MakeRedisClient().(redis.UniversalClient) defer rc.Close() if err := rc.Del(context.Background(), semaphoreKey(tt.name)).Err(); err != nil { t.Errorf("%s;\nredis.UniversalClient.Del() got error %v", tt.desc, err) } var members []redis.Z for i := 0; i < len(tt.taskIDs); i++ { members = append(members, redis.Z{ Score: float64(time.Now().Add(time.Duration(i) * time.Second).Unix()), Member: tt.taskIDs[i], }) } if err := rc.ZAdd(context.Background(), semaphoreKey(tt.name), members...).Err(); err != nil { t.Errorf("%s;\nredis.UniversalClient.ZAdd() got error %v", tt.desc, err) } sema := NewSemaphore(opt, tt.name, 3) defer sema.Close() for i := 0; i < len(tt.taskIDs); i++ { ctx, cancel := tt.ctxFunc(tt.taskIDs[i]) if err := sema.Release(ctx); err == nil || err.Error() != tt.errStr { t.Errorf("%s;\nSemaphore.Release() got error %v want error %v", tt.desc, err, tt.errStr) } cancel() } }) } } func getRedisConnOpt(tb testing.TB) asynq.RedisConnOpt { tb.Helper() if useRedisCluster { addrs := strings.Split(redisClusterAddrs, ",") if len(addrs) == 0 { tb.Fatal("No redis cluster addresses provided. Please set addresses using --redis_cluster_addrs flag.") } return asynq.RedisClusterClientOpt{ Addrs: addrs, } } return asynq.RedisClientOpt{ Addr: redisAddr, DB: redisDB, } } type badConnOpt struct { } func (b badConnOpt) MakeRedisClient() interface{} { return nil }