[ { "path": ".github/FUNDING.yml", "content": "# These are supported funding model platforms\n\ngithub: [hibiken]\nopen_collective: ken-hibino\n" }, { "path": ".github/ISSUE_TEMPLATE/bug_report.md", "content": "---\nname: Bug report\nabout: Create a report to help us improve\ntitle: \"[BUG] Description of the bug\"\nlabels: bug\nassignees:\n - hibiken\n - kamikazechaser\n \n---\n\n**Describe the bug**\nA clear and concise description of what the bug is.\n\n**Environment (please complete the following information):**\n - OS: [e.g. MacOS, Linux]\n - `asynq` package version [e.g. v0.25.0]\n - Redis/Valkey version \n\n**To Reproduce**\nSteps to reproduce the behavior (Code snippets if applicable):\n1. Setup background processing ...\n2. Enqueue tasks ...\n3. See Error ...\n\n**Expected behavior**\nA clear and concise description of what you expected to happen.\n\n**Screenshots**\nIf applicable, add screenshots to help explain your problem.\n\n**Additional context**\nAdd any other context about the problem here.\n" }, { "path": ".github/ISSUE_TEMPLATE/feature_request.md", "content": "---\nname: Feature request\nabout: Suggest an idea for this project\ntitle: \"[FEATURE REQUEST] Description of the feature request\"\nlabels: enhancement\nassignees:\n - hibiken\n - kamikazechaser\n\n---\n\n**Is your feature request related to a problem? Please describe.**\nA clear and concise description of what the problem is. Ex. I'm always frustrated when [...]\n\n**Describe the solution you'd like**\nA clear and concise description of what you want to happen.\n\n**Describe alternatives you've considered**\nA clear and concise description of any alternative solutions or features you've considered.\n\n**Additional context**\nAdd any other context or screenshots about the feature request here.\n" }, { "path": ".github/dependabot.yaml", "content": "version: 2\nupdates:\n - package-ecosystem: \"gomod\"\n directory: \"/\"\n schedule:\n interval: \"weekly\"\n labels:\n - \"pr-deps\"\n - package-ecosystem: \"gomod\"\n directory: \"/tools\"\n schedule:\n interval: \"weekly\"\n labels:\n - \"pr-deps\"\n - package-ecosystem: \"gomod\"\n directory: \"/x\"\n schedule:\n interval: \"weekly\"\n labels:\n - \"pr-deps\"\n - package-ecosystem: \"github-actions\"\n directory: \"/\"\n schedule:\n interval: \"weekly\"\n" }, { "path": ".github/workflows/benchstat.yml", "content": "# This workflow runs benchmarks against the current branch,\n# compares them to benchmarks against master,\n# and uploads the results as an artifact.\n\nname: benchstat\n\non: [pull_request]\n\njobs:\n incoming:\n runs-on: ubuntu-latest\n if: false\n services:\n redis:\n image: redis:7\n ports:\n - 6379:6379\n steps:\n - name: Checkout\n uses: actions/checkout@v4\n - name: Set up Go\n uses: actions/setup-go@v5\n with:\n go-version: 1.23.x\n - name: Benchmark\n run: go test -run=^$ -bench=. -count=5 -timeout=60m ./... | tee -a new.txt\n - name: Upload Benchmark\n uses: actions/upload-artifact@v4\n with:\n name: bench-incoming\n path: new.txt\n\n current:\n runs-on: ubuntu-latest\n if: false\n services:\n redis:\n image: redis:7\n ports:\n - 6379:6379\n steps:\n - name: Checkout\n uses: actions/checkout@v4\n with:\n ref: master\n - name: Set up Go\n uses: actions/setup-go@v5\n with:\n go-version: 1.23.x\n - name: Benchmark\n run: go test -run=^$ -bench=. -count=5 -timeout=60m ./... | tee -a old.txt\n - name: Upload Benchmark\n uses: actions/upload-artifact@v4\n with:\n name: bench-current\n path: old.txt\n\n benchstat:\n needs: [incoming, current]\n if: false\n runs-on: ubuntu-latest\n steps:\n - name: Checkout\n uses: actions/checkout@v4\n - name: Set up Go\n uses: actions/setup-go@v5\n with:\n go-version: 1.23.x\n - name: Install benchstat\n run: go get -u golang.org/x/perf/cmd/benchstat\n - name: Download Incoming\n uses: actions/download-artifact@v4\n with:\n name: bench-incoming\n - name: Download Current\n uses: actions/download-artifact@v4\n with:\n name: bench-current\n - name: Benchstat Results\n run: benchstat old.txt new.txt | tee -a benchstat.txt\n - name: Upload benchstat results\n uses: actions/upload-artifact@v4\n with:\n name: benchstat\n path: benchstat.txt\n" }, { "path": ".github/workflows/build.yml", "content": "name: build\n\non: [push, pull_request]\n\njobs:\n build:\n strategy:\n matrix:\n os: [ubuntu-latest]\n go-version: [1.24.x, 1.25.x]\n runs-on: ${{ matrix.os }}\n services:\n redis:\n image: redis:7\n ports:\n - 6379:6379\n steps:\n - uses: actions/checkout@v4\n\n - name: Set up Go\n uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go-version }}\n cache: false\n\n - name: Build core module\n run: go build -v ./...\n\n - name: Build x module\n run: cd x && go build -v ./... && cd ..\n\n - name: Test core module\n run: go test -race -v -coverprofile=coverage.txt -covermode=atomic ./...\n\n - name: Test x module\n run: cd x && go test -race -v ./... && cd ..\n\n - name: Benchmark Test\n run: go test -run=^$ -bench=. -loglevel=debug ./...\n\n - name: Upload coverage to Codecov\n uses: codecov/codecov-action@v5\n\n build-tool:\n strategy:\n matrix:\n os: [ubuntu-latest]\n go-version: [1.24.x, 1.25.x]\n runs-on: ${{ matrix.os }}\n services:\n redis:\n image: redis:7\n ports:\n - 6379:6379\n steps:\n - uses: actions/checkout@v4\n\n - name: Set up Go\n uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go-version }}\n cache: false\n\n - name: Build tools module\n run: cd tools && go build -v ./... && cd ..\n\n - name: Test tools module\n run: cd tools && go test -race -v ./... && cd ..\n\n golangci:\n name: lint\n runs-on: ubuntu-latest\n if: false\n steps:\n - uses: actions/checkout@v4\n\n - uses: actions/setup-go@v5\n with:\n go-version: stable\n\n - name: golangci-lint\n uses: golangci/golangci-lint-action@v6\n with:\n version: v1.61\n" }, { "path": ".gitignore", "content": "vendor\n# Binaries for programs and plugins\n*.exe\n*.exe~\n*.dll\n*.so\n*.dylib\n\n# Test binary, built with `go test -c`\n*.test\n\n# Output of the go coverage tool, specifically when used with LiteIDE\n*.out\n\n# Ignore examples for now\n/examples\n\n# Ignore tool binaries\n/tools/asynq/asynq\n/tools/metrics_exporter/metrics_exporter\n\n# Ignore asynq config file\n.asynq.*\n\n# Ignore editor config files\n.vscode\n.idea\n" }, { "path": "CHANGELOG.md", "content": "# Changelog\n\nAll notable changes to this project will be documented in this file.\n\nThe format is based on [\"Keep a Changelog\"](https://keepachangelog.com/en/1.0.0/),\nand this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).\n\n## [Unreleased]\n\n## [0.26.0] - 2026-02-03\n\n### Upgrades\n- Prepare CI for Go 1.24.x and 1.25.x (commit: e9037f0)\n\n### Added\n- Add Headers support to tasks (PR: https://github.com/hibiken/asynq/pull/1070)\n- Add `--tls` option to dash command (PR: https://github.com/hibiken/asynq/pull/1073)\n- Add `--username` CLI flag for Redis ACL authentication (PR: https://github.com/hibiken/asynq/pull/1083)\n- Add `UpdateTaskPayload` method for inspector (PR: https://github.com/hibiken/asynq/pull/1042)\n\n### Fixes\n- Fix: Correct error message text in ResultWriter.Write (PR: https://github.com/hibiken/asynq/pull/1054)\n- Fix: Wrap all fmt.Errorf errors with %w (PR: https://github.com/hibiken/asynq/pull/1047)\n- Fix: ServeMux.NotFoundHandler returns ErrHandlerNotFound error (PR: https://github.com/hibiken/asynq/pull/1031)\n\n### Changed\n- Docs: Update server.go documentation (PR: https://github.com/hibiken/asynq/pull/1010)\n- Chore: Fix godoc comment (PR: https://github.com/hibiken/asynq/pull/1009)\n\n## [0.25.1] - 2024-12-11\n\n### Upgrades\n\n* Some packages\n\n### Added\n\n* Add `HeartbeatInterval` option to the scheduler (PR: https://github.com/hibiken/asynq/pull/956)\n* Add `RedisUniversalClient` support to periodic task manager (PR: https://github.com/hibiken/asynq/pull/958)\n* Add `--insecure` flag to CLI dash command (PR: https://github.com/hibiken/asynq/pull/980)\n* Add logging for registration errors (PR: https://github.com/hibiken/asynq/pull/657)\n\n### Fixes\n- Perf: Use string concat inplace of fmt.Sprintf in hotpath (PR: https://github.com/hibiken/asynq/pull/962)\n- Perf: Init map with size (PR: https://github.com/hibiken/asynq/pull/673)\n- Fix: `Scheduler` and `PeriodicTaskManager` graceful shutdown (PR: https://github.com/hibiken/asynq/pull/977)\n- Fix: `Server` graceful shutdown on UNIX systems (PR: https://github.com/hibiken/asynq/pull/982)\n\n## [0.25.0] - 2024-10-29\n\n### Upgrades\n- Minumum go version is set to 1.22 (PR: https://github.com/hibiken/asynq/pull/925)\n- Internal protobuf package is upgraded to address security advisories (PR: https://github.com/hibiken/asynq/pull/925)\n- Most packages are upgraded\n- CI/CD spec upgraded\n\n### Added\n- `IsPanicError` function is introduced to support catching of panic errors when processing tasks (PR: https://github.com/hibiken/asynq/pull/491)\n- `JanitorInterval` and `JanitorBatchSize` are added as Server options (PR: https://github.com/hibiken/asynq/pull/715)\n- `NewClientFromRedisClient` is introduced to allow reusing an existing redis client (PR: https://github.com/hibiken/asynq/pull/742)\n- `TaskCheckInterval` config option is added to specify the interval between checks for new tasks to process when all queues are empty (PR: https://github.com/hibiken/asynq/pull/694)\n- `Ping` method is added to Client, Server and Scheduler ((PR: https://github.com/hibiken/asynq/pull/585))\n- `RevokeTask` error type is introduced to prevent a task from being retried or archived (PR: https://github.com/hibiken/asynq/pull/882)\n- `SentinelUsername` is added as a redis config option (PR: https://github.com/hibiken/asynq/pull/924)\n- Some jitter is introduced to improve latency when fetching jobs in the processor (PR: https://github.com/hibiken/asynq/pull/868)\n- Add task enqueue command to the CLI (PR: https://github.com/hibiken/asynq/pull/918)\n- Add a map cache (concurrent safe) to keep track of queues that ultimately reduces redis load when enqueuing tasks (PR: https://github.com/hibiken/asynq/pull/946)\n\n### Fixes\n- Archived tasks that are trimmed should now be deleted (PR: https://github.com/hibiken/asynq/pull/743)\n- Fix lua script when listing task messages with an expired lease (PR: https://github.com/hibiken/asynq/pull/709)\n- Fix potential context leaks due to cancellation not being called (PR: https://github.com/hibiken/asynq/pull/926)\n- Misc documentation fixes\n- Misc test fixes\n\n\n## [0.24.1] - 2023-05-01\n\n### Changed\n- Updated package version dependency for go-redis \n\n## [0.24.0] - 2023-01-02\n\n### Added\n- `PreEnqueueFunc`, `PostEnqueueFunc` is added in `Scheduler` and deprecated `EnqueueErrorHandler` (PR: https://github.com/hibiken/asynq/pull/476)\n\n### Changed\n- Removed error log when `Scheduler` failed to enqueue a task. Use `PostEnqueueFunc` to check for errors and task actions if needed.\n- Changed log level from ERROR to WARNINING when `Scheduler` failed to record `SchedulerEnqueueEvent`.\n\n## [0.23.0] - 2022-04-11\n\n### Added\n\n- `Group` option is introduced to enqueue task in a group.\n- `GroupAggregator` and related types are introduced for task aggregation feature.\n- `GroupGracePeriod`, `GroupMaxSize`, `GroupMaxDelay`, and `GroupAggregator` fields are added to `Config`.\n- `Inspector` has new methods related to \"aggregating tasks\".\n- `Group` field is added to `TaskInfo`.\n- (CLI): `group ls` command is added\n- (CLI): `task ls` supports listing aggregating tasks via `--state=aggregating --group=` flags\n- Enable rediss url parsing support\n\n### Fixed\n\n- Fixed overflow issue with 32-bit systems (For details, see https://github.com/hibiken/asynq/pull/426)\n\n## [0.22.1] - 2022-02-20\n\n### Fixed\n\n- Fixed Redis version compatibility: Keep support for redis v4.0+\n\n## [0.22.0] - 2022-02-19\n\n### Added\n\n- `BaseContext` is introduced in `Config` to specify callback hook to provide a base `context` from which `Handler` `context` is derived\n- `IsOrphaned` field is added to `TaskInfo` to describe a task left in active state with no worker processing it.\n\n### Changed\n\n- `Server` now recovers tasks with an expired lease. Recovered tasks are retried/archived with `ErrLeaseExpired` error.\n\n## [0.21.0] - 2022-01-22\n\n### Added\n\n- `PeriodicTaskManager` is added. Prefer using this over `Scheduler` as it has better support for dynamic periodic tasks.\n- The `asynq stats` command now supports a `--json` option, making its output a JSON object\n- Introduced new configuration for `DelayedTaskCheckInterval`. See [godoc](https://godoc.org/github.com/hibiken/asynq) for more details.\n\n## [0.20.0] - 2021-12-19\n\n### Added\n\n- Package `x/metrics` is added.\n- Tool `tools/metrics_exporter` binary is added.\n- `ProcessedTotal` and `FailedTotal` fields were added to `QueueInfo` struct.\n\n## [0.19.1] - 2021-12-12\n\n### Added\n\n- `Latency` field is added to `QueueInfo`.\n- `EnqueueContext` method is added to `Client`.\n\n### Fixed\n\n- Fixed an error when user pass a duration less than 1s to `Unique` option\n\n## [0.19.0] - 2021-11-06\n\n### Changed\n\n- `NewTask` takes `Option` as variadic argument\n- Bumped minimum supported go version to 1.14 (i.e. go1.14 or higher is required).\n\n### Added\n\n- `Retention` option is added to allow user to specify task retention duration after completion.\n- `TaskID` option is added to allow user to specify task ID.\n- `ErrTaskIDConflict` sentinel error value is added.\n- `ResultWriter` type is added and provided through `Task.ResultWriter` method.\n- `TaskInfo` has new fields `CompletedAt`, `Result` and `Retention`.\n\n### Removed\n\n- `Client.SetDefaultOptions` is removed. Use `NewTask` instead to pass default options for tasks.\n\n## [0.18.6] - 2021-10-03\n\n### Changed\n\n- Updated `github.com/go-redis/redis` package to v8\n\n## [0.18.5] - 2021-09-01\n\n### Added\n\n- `IsFailure` config option is added to determine whether error returned from Handler counts as a failure.\n\n## [0.18.4] - 2021-08-17\n\n### Fixed\n\n- Scheduler methods are now thread-safe. It's now safe to call `Register` and `Unregister` concurrently.\n\n## [0.18.3] - 2021-08-09\n\n### Changed\n\n- `Client.Enqueue` no longer enqueues tasks with empty typename; Error message is returned.\n\n## [0.18.2] - 2021-07-15\n\n### Changed\n\n- Changed `Queue` function to not to convert the provided queue name to lowercase. Queue names are now case-sensitive.\n- `QueueInfo.MemoryUsage` is now an approximate usage value.\n\n### Fixed\n\n- Fixed latency issue around memory usage (see https://github.com/hibiken/asynq/issues/309).\n\n## [0.18.1] - 2021-07-04\n\n### Changed\n\n- Changed to execute task recovering logic when server starts up; Previously it needed to wait for a minute for task recovering logic to exeucte.\n\n### Fixed\n\n- Fixed task recovering logic to execute every minute\n\n## [0.18.0] - 2021-06-29\n\n### Changed\n\n- NewTask function now takes array of bytes as payload.\n- Task `Type` and `Payload` should be accessed by a method call.\n- `Server` API has changed. Renamed `Quiet` to `Stop`. Renamed `Stop` to `Shutdown`. _Note:_ As a result of this renaming, the behavior of `Stop` has changed. Please update the exising code to call `Shutdown` where it used to call `Stop`.\n- `Scheduler` API has changed. Renamed `Stop` to `Shutdown`.\n- Requires redis v4.0+ for multiple field/value pair support\n- `Client.Enqueue` now returns `TaskInfo`\n- `Inspector.RunTaskByKey` is replaced with `Inspector.RunTask`\n- `Inspector.DeleteTaskByKey` is replaced with `Inspector.DeleteTask`\n- `Inspector.ArchiveTaskByKey` is replaced with `Inspector.ArchiveTask`\n- `inspeq` package is removed. All types and functions from the package is moved to `asynq` package.\n- `WorkerInfo` field names have changed.\n- `Inspector.CancelActiveTask` is renamed to `Inspector.CancelProcessing`\n\n## [0.17.2] - 2021-06-06\n\n### Fixed\n\n- Free unique lock when task is deleted (https://github.com/hibiken/asynq/issues/275).\n\n## [0.17.1] - 2021-04-04\n\n### Fixed\n\n- Fix bug in internal `RDB.memoryUsage` method.\n\n## [0.17.0] - 2021-03-24\n\n### Added\n\n- `DialTimeout`, `ReadTimeout`, and `WriteTimeout` options are added to `RedisConnOpt`.\n\n## [0.16.1] - 2021-03-20\n\n### Fixed\n\n- Replace `KEYS` command with `SCAN` as recommended by [redis doc](https://redis.io/commands/KEYS).\n\n## [0.16.0] - 2021-03-10\n\n### Added\n\n- `Unregister` method is added to `Scheduler` to remove a registered entry.\n\n## [0.15.0] - 2021-01-31\n\n**IMPORTATNT**: All `Inspector` related code are moved to subpackage \"github.com/hibiken/asynq/inspeq\"\n\n### Changed\n\n- `Inspector` related code are moved to subpackage \"github.com/hibken/asynq/inspeq\".\n- `RedisConnOpt` interface has changed slightly. If you have been passing `RedisClientOpt`, `RedisFailoverClientOpt`, or `RedisClusterClientOpt` as a pointer,\n update your code to pass as a value.\n- `ErrorMsg` field in `RetryTask` and `ArchivedTask` was renamed to `LastError`.\n\n### Added\n\n- `MaxRetry`, `Retried`, `LastError` fields were added to all task types returned from `Inspector`.\n- `MemoryUsage` field was added to `QueueStats`.\n- `DeleteAllPendingTasks`, `ArchiveAllPendingTasks` were added to `Inspector`\n- `DeleteTaskByKey` and `ArchiveTaskByKey` now supports deleting/archiving `PendingTask`.\n- asynq CLI now supports deleting/archiving pending tasks.\n\n## [0.14.1] - 2021-01-19\n\n### Fixed\n\n- `go.mod` file for CLI\n\n## [0.14.0] - 2021-01-14\n\n**IMPORTATNT**: Please run `asynq migrate` command to migrate from the previous versions.\n\n### Changed\n\n- Renamed `DeadTask` to `ArchivedTask`.\n- Renamed the operation `Kill` to `Archive` in `Inpsector`.\n- Print stack trace when Handler panics.\n- Include a file name and a line number in the error message when recovering from a panic.\n\n### Added\n\n- `DefaultRetryDelayFunc` is now a public API, which can be used in the custom `RetryDelayFunc`.\n- `SkipRetry` error is added to be used as a return value from `Handler`.\n- `Servers` method is added to `Inspector`\n- `CancelActiveTask` method is added to `Inspector`.\n- `ListSchedulerEnqueueEvents` method is added to `Inspector`.\n- `SchedulerEntries` method is added to `Inspector`.\n- `DeleteQueue` method is added to `Inspector`.\n\n## [0.13.1] - 2020-11-22\n\n### Fixed\n\n- Fixed processor to wait for specified time duration before forcefully shutdown workers.\n\n## [0.13.0] - 2020-10-13\n\n### Added\n\n- `Scheduler` type is added to enable periodic tasks. See the godoc for its APIs and [wiki](https://github.com/hibiken/asynq/wiki/Periodic-Tasks) for the getting-started guide.\n\n### Changed\n\n- interface `Option` has changed. See the godoc for the new interface.\n This change would have no impact as long as you are using exported functions (e.g. `MaxRetry`, `Queue`, etc)\n to create `Option`s.\n\n### Added\n\n- `Payload.String() string` method is added\n- `Payload.MarshalJSON() ([]byte, error)` method is added\n\n## [0.12.0] - 2020-09-12\n\n**IMPORTANT**: If you are upgrading from a previous version, please install the latest version of the CLI `go get -u github.com/hibiken/asynq/tools/asynq` and run `asynq migrate` command. No process should be writing to Redis while you run the migration command.\n\n## The semantics of queue have changed\n\nPreviously, we called tasks that are ready to be processed _\"Enqueued tasks\"_, and other tasks that are scheduled to be processed in the future _\"Scheduled tasks\"_, etc.\nWe changed the semantics of _\"Enqueue\"_ slightly; All tasks that client pushes to Redis are _Enqueued_ to a queue. Within a queue, tasks will transition from one state to another.\nPossible task states are:\n\n- `Pending`: task is ready to be processed (previously called \"Enqueued\")\n- `Active`: tasks is currently being processed (previously called \"InProgress\")\n- `Scheduled`: task is scheduled to be processed in the future\n- `Retry`: task failed to be processed and will be retried again in the future\n- `Dead`: task has exhausted all of its retries and stored for manual inspection purpose\n\n**These semantics change is reflected in the new `Inspector` API and CLI commands.**\n\n---\n\n### Changed\n\n#### `Client`\n\nUse `ProcessIn` or `ProcessAt` option to schedule a task instead of `EnqueueIn` or `EnqueueAt`.\n\n| Previously | v0.12.0 |\n| --------------------------- | ------------------------------------------ |\n| `client.EnqueueAt(t, task)` | `client.Enqueue(task, asynq.ProcessAt(t))` |\n| `client.EnqueueIn(d, task)` | `client.Enqueue(task, asynq.ProcessIn(d))` |\n\n#### `Inspector`\n\nAll Inspector methods are scoped to a queue, and the methods take `qname (string)` as the first argument.\n`EnqueuedTask` is renamed to `PendingTask` and its corresponding methods.\n`InProgressTask` is renamed to `ActiveTask` and its corresponding methods.\nCommand \"Enqueue\" is replaced by the verb \"Run\" (e.g. `EnqueueAllScheduledTasks` --> `RunAllScheduledTasks`)\n\n#### `CLI`\n\nCLI commands are restructured to use subcommands. Commands are organized into a few management commands:\nTo view details on any command, use `asynq help `.\n\n- `asynq stats`\n- `asynq queue [ls inspect history rm pause unpause]`\n- `asynq task [ls cancel delete kill run delete-all kill-all run-all]`\n- `asynq server [ls]`\n\n### Added\n\n#### `RedisConnOpt`\n\n- `RedisClusterClientOpt` is added to connect to Redis Cluster.\n- `Username` field is added to all `RedisConnOpt` types in order to authenticate connection when Redis ACLs are used.\n\n#### `Client`\n\n- `ProcessIn(d time.Duration) Option` and `ProcessAt(t time.Time) Option` are added to replace `EnqueueIn` and `EnqueueAt` functionality.\n\n#### `Inspector`\n\n- `Queues() ([]string, error)` method is added to get all queue names.\n- `ClusterKeySlot(qname string) (int64, error)` method is added to get queue's hash slot in Redis cluster.\n- `ClusterNodes(qname string) ([]ClusterNode, error)` method is added to get a list of Redis cluster nodes for the given queue.\n- `Close() error` method is added to close connection with redis.\n\n### `Handler`\n\n- `GetQueueName(ctx context.Context) (string, bool)` helper is added to extract queue name from a context.\n\n## [0.11.0] - 2020-07-28\n\n### Added\n\n- `Inspector` type was added to monitor and mutate state of queues and tasks.\n- `HealthCheckFunc` and `HealthCheckInterval` fields were added to `Config` to allow user to specify a callback\n function to check for broker connection.\n\n## [0.10.0] - 2020-07-06\n\n### Changed\n\n- All tasks now requires timeout or deadline. By default, timeout is set to 30 mins.\n- Tasks that exceed its deadline are automatically retried.\n- Encoding schema for task message has changed. Please install the latest CLI and run `migrate` command if\n you have tasks enqueued with the previous version of asynq.\n- API of `(*Client).Enqueue`, `(*Client).EnqueueIn`, and `(*Client).EnqueueAt` has changed to return a `*Result`.\n- API of `ErrorHandler` has changed. It now takes context as the first argument and removed `retried`, `maxRetry` from the argument list.\n Use `GetRetryCount` and/or `GetMaxRetry` to get the count values.\n\n## [0.9.4] - 2020-06-13\n\n### Fixed\n\n- Fixes issue of same tasks processed by more than one worker (https://github.com/hibiken/asynq/issues/90).\n\n## [0.9.3] - 2020-06-12\n\n### Fixed\n\n- Fixes the JSON number overflow issue (https://github.com/hibiken/asynq/issues/166).\n\n## [0.9.2] - 2020-06-08\n\n### Added\n\n- The `pause` and `unpause` commands were added to the CLI. See README for the CLI for details.\n\n## [0.9.1] - 2020-05-29\n\n### Added\n\n- `GetTaskID`, `GetRetryCount`, and `GetMaxRetry` functions were added to extract task metadata from context.\n\n## [0.9.0] - 2020-05-16\n\n### Changed\n\n- `Logger` interface has changed. Please see the godoc for the new interface.\n\n### Added\n\n- `LogLevel` type is added. Server's log level can be specified through `LogLevel` field in `Config`.\n\n## [0.8.3] - 2020-05-08\n\n### Added\n\n- `Close` method is added to `Client`.\n\n## [0.8.2] - 2020-05-03\n\n### Fixed\n\n- [Fixed cancelfunc leak](https://github.com/hibiken/asynq/pull/145)\n\n## [0.8.1] - 2020-04-27\n\n### Added\n\n- `ParseRedisURI` helper function is added to create a `RedisConnOpt` from a URI string.\n- `SetDefaultOptions` method is added to `Client`.\n\n## [0.8.0] - 2020-04-19\n\n### Changed\n\n- `Background` type is renamed to `Server`.\n- To upgrade from the previous version, Update `NewBackground` to `NewServer` and pass `Config` by value.\n- CLI is renamed to `asynq`.\n- To upgrade the CLI to the latest version run `go get -u github.com/hibiken/tools/asynq`\n- The `ps` command in CLI is renamed to `servers`\n- `Concurrency` defaults to the number of CPUs when unset or set to a negative value.\n\n### Added\n\n- `ShutdownTimeout` field is added to `Config` to speicfy timeout duration used during graceful shutdown.\n- New `Server` type exposes `Start`, `Stop`, and `Quiet` as well as `Run`.\n\n## [0.7.1] - 2020-04-05\n\n### Fixed\n\n- Fixed signal handling for windows.\n\n## [0.7.0] - 2020-03-22\n\n### Changed\n\n- Support Go v1.13+, dropped support for go v1.12\n\n### Added\n\n- `Unique` option was added to allow client to enqueue a task only if it's unique within a certain time period.\n\n## [0.6.2] - 2020-03-15\n\n### Added\n\n- `Use` method was added to `ServeMux` to apply middlewares to all handlers.\n\n## [0.6.1] - 2020-03-12\n\n### Added\n\n- `Client` can optionally schedule task with `asynq.Deadline(time)` to specify deadline for task's context. Default is no deadline.\n- `Logger` option was added to config, which allows user to specify the logger used by the background instance.\n\n## [0.6.0] - 2020-03-01\n\n### Added\n\n- Added `ServeMux` type to make it easy for users to implement Handler interface.\n- `ErrorHandler` type was added. Allow users to specify error handling function (e.g. Report error to error reporting service such as Honeybadger, Bugsnag, etc)\n\n## [0.5.0] - 2020-02-23\n\n### Changed\n\n- `Client` API has changed. Use `Enqueue`, `EnqueueAt` and `EnqueueIn` to enqueue and schedule tasks.\n\n### Added\n\n- `asynqmon workers` was added to list all running workers information\n\n## [0.4.0] - 2020-02-13\n\n### Changed\n\n- `Handler` interface has changed. `ProcessTask` method takes two arguments `context.Context` and `*asynq.Task`\n- `Queues` field in `Config` has change from `map[string]uint` to `map[string]int`\n\n### Added\n\n- `Client` can optionally schedule task with `asynq.Timeout(duration)` to specify timeout duration for task. Default is no timeout.\n- `asynqmon cancel [task id]` will send a cancelation signal to the goroutine processing the speicified task.\n\n## [0.3.0] - 2020-02-04\n\n### Added\n\n- `asynqmon ps` was added to list all background worker processes\n\n## [0.2.2] - 2020-01-26\n\n### Fixed\n\n- Fixed restoring unfinished tasks back to correct queues.\n\n### Changed\n\n- `asynqmon ls` command is now paginated (default 30 tasks from first page)\n- `asynqmon ls enqueued:[queue name]` requires queue name to be specified\n\n## [0.2.1] - 2020-01-22\n\n### Fixed\n\n- More structured log messages\n- Prevent spamming logs with a bunch of errors when Redis connection is lost\n- Fixed and updated README doc\n\n## [0.2.0] - 2020-01-19\n\n### Added\n\n- NewTask constructor\n- `Queues` option in `Config` to specify mutiple queues with priority level\n- `Client` can schedule a task with `asynq.Queue(name)` to specify which queue to use\n- `StrictPriority` option in `Config` to specify whether the priority should be followed strictly\n- `RedisConnOpt` to abstract away redis client implementation\n- [CLI] `asynqmon rmq` command to remove queue\n\n### Changed\n\n- `Client` and `Background` constructors take `RedisConnOpt` as their first argument.\n- `asynqmon stats` now shows the total of all enqueued tasks under \"Enqueued\"\n- `asynqmon stats` now shows each queue's task count\n- `asynqmon history` now doesn't take any arguments and shows data from the last 10 days by default (use `--days` flag to change the number of days)\n- Task type is now immutable (i.e., Payload is read-only)\n\n## [0.1.0] - 2020-01-04\n\n### Added\n\n- Initial version of asynq package\n- Initial version of asynqmon CLI\n" }, { "path": "CODE_OF_CONDUCT.md", "content": "# Contributor Covenant Code of Conduct\n\n## Our Pledge\n\nWe as members, contributors, and leaders pledge to make participation in our\ncommunity a harassment-free experience for everyone, regardless of age, body\nsize, visible or invisible disability, ethnicity, sex characteristics, gender\nidentity and expression, level of experience, education, socio-economic status,\nnationality, personal appearance, race, religion, or sexual identity\nand orientation.\n\nWe pledge to act and interact in ways that contribute to an open, welcoming,\ndiverse, inclusive, and healthy community.\n\n## Our Standards\n\nExamples of behavior that contributes to a positive environment for our\ncommunity include:\n\n* Demonstrating empathy and kindness toward other people\n* Being respectful of differing opinions, viewpoints, and experiences\n* Giving and gracefully accepting constructive feedback\n* Accepting responsibility and apologizing to those affected by our mistakes,\n and learning from the experience\n* Focusing on what is best not just for us as individuals, but for the\n overall community\n\nExamples of unacceptable behavior include:\n\n* The use of sexualized language or imagery, and sexual attention or\n advances of any kind\n* Trolling, insulting or derogatory comments, and personal or political attacks\n* Public or private harassment\n* Publishing others' private information, such as a physical or email\n address, without their explicit permission\n* Other conduct which could reasonably be considered inappropriate in a\n professional setting\n\n## Enforcement Responsibilities\n\nCommunity leaders are responsible for clarifying and enforcing our standards of\nacceptable behavior and will take appropriate and fair corrective action in\nresponse to any behavior that they deem inappropriate, threatening, offensive,\nor harmful.\n\nCommunity leaders have the right and responsibility to remove, edit, or reject\ncomments, commits, code, wiki edits, issues, and other contributions that are\nnot aligned to this Code of Conduct, and will communicate reasons for moderation\ndecisions when appropriate.\n\n## Scope\n\nThis Code of Conduct applies within all community spaces, and also applies when\nan individual is officially representing the community in public spaces.\nExamples of representing our community include using an official e-mail address,\nposting via an official social media account, or acting as an appointed\nrepresentative at an online or offline event.\n\n## Enforcement\n\nInstances of abusive, harassing, or otherwise unacceptable behavior may be\nreported to the community leaders responsible for enforcement at\nken.hibino7@gmail.com.\nAll complaints will be reviewed and investigated promptly and fairly.\n\nAll community leaders are obligated to respect the privacy and security of the\nreporter of any incident.\n\n## Enforcement Guidelines\n\nCommunity leaders will follow these Community Impact Guidelines in determining\nthe consequences for any action they deem in violation of this Code of Conduct:\n\n### 1. Correction\n\n**Community Impact**: Use of inappropriate language or other behavior deemed\nunprofessional or unwelcome in the community.\n\n**Consequence**: A private, written warning from community leaders, providing\nclarity around the nature of the violation and an explanation of why the\nbehavior was inappropriate. A public apology may be requested.\n\n### 2. Warning\n\n**Community Impact**: A violation through a single incident or series\nof actions.\n\n**Consequence**: A warning with consequences for continued behavior. No\ninteraction with the people involved, including unsolicited interaction with\nthose enforcing the Code of Conduct, for a specified period of time. This\nincludes avoiding interactions in community spaces as well as external channels\nlike social media. Violating these terms may lead to a temporary or\npermanent ban.\n\n### 3. Temporary Ban\n\n**Community Impact**: A serious violation of community standards, including\nsustained inappropriate behavior.\n\n**Consequence**: A temporary ban from any sort of interaction or public\ncommunication with the community for a specified period of time. No public or\nprivate interaction with the people involved, including unsolicited interaction\nwith those enforcing the Code of Conduct, is allowed during this period.\nViolating these terms may lead to a permanent ban.\n\n### 4. Permanent Ban\n\n**Community Impact**: Demonstrating a pattern of violation of community\nstandards, including sustained inappropriate behavior, harassment of an\nindividual, or aggression toward or disparagement of classes of individuals.\n\n**Consequence**: A permanent ban from any sort of public interaction within\nthe community.\n\n## Attribution\n\nThis Code of Conduct is adapted from the [Contributor Covenant][homepage],\nversion 2.0, available at\nhttps://www.contributor-covenant.org/version/2/0/code_of_conduct.html.\n\nCommunity Impact Guidelines were inspired by [Mozilla's code of conduct\nenforcement ladder](https://github.com/mozilla/diversity).\n\n[homepage]: https://www.contributor-covenant.org\n\nFor answers to common questions about this code of conduct, see the FAQ at\nhttps://www.contributor-covenant.org/faq. Translations are available at\nhttps://www.contributor-covenant.org/translations.\n" }, { "path": "CONTRIBUTING.md", "content": "# Contributing\n\nThanks for your interest in contributing to Asynq!\nWe are open to, and grateful for, any contributions made by the community.\n\n## Reporting Bugs\n\nHave a look at our [issue tracker](https://github.com/hibiken/asynq/issues). If you can't find an issue (open or closed)\ndescribing your problem (or a very similar one) there, please open a new issue with\nthe following details:\n\n- Which versions of Go and Redis are you using?\n- What are you trying to accomplish?\n- What is the full error you are seeing?\n- How can we reproduce this?\n - Please quote as much of your code as needed to reproduce (best link to a\n public repository or Gist)\n\n## Getting Help\n\nWe run a [Gitter\nchannel](https://gitter.im/go-asynq/community) where you can ask questions and\nget help. Feel free to ask there before opening a GitHub issue.\n\n## Submitting Feature Requests\n\nIf you can't find an issue (open or closed) describing your idea on our [issue\ntracker](https://github.com/hibiken/asynq/issues), open an issue. Adding answers to the following\nquestions in your description is +1:\n\n- What do you want to do, and how do you expect Asynq to support you with that?\n- How might this be added to Asynq?\n- What are possible alternatives?\n- Are there any disadvantages?\n\nThank you! We'll try to respond as quickly as possible.\n\n## Contributing Code\n\n1. Fork this repo\n2. Download your fork `git clone git@github.com:your-username/asynq.git && cd asynq`\n3. Create your branch `git checkout -b your-branch-name`\n4. Make and commit your changes\n5. Push the branch `git push origin your-branch-name`\n6. Create a new pull request\n\nPlease try to keep your pull request focused in scope and avoid including unrelated commits.\nPlease run tests against redis cluster locally with `--redis_cluster` flag to ensure that code works for Redis cluster. TODO: Run tests using Redis cluster on CI.\n\nAfter you have submitted your pull request, we'll try to get back to you as soon as possible. We may suggest some changes or improvements.\n\nThank you for contributing!\n" }, { "path": "LICENSE", "content": "MIT License\n\nCopyright (c) 2019 Kentaro Hibino\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "Makefile", "content": "ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))\n\nproto: internal/proto/asynq.proto\n\tprotoc -I=$(ROOT_DIR)/internal/proto \\\n\t\t\t\t --go_out=$(ROOT_DIR)/internal/proto \\\n\t\t\t\t --go_opt=module=github.com/hibiken/asynq/internal/proto \\\n\t\t\t\t $(ROOT_DIR)/internal/proto/asynq.proto\n\n.PHONY: lint\nlint:\n\tgolangci-lint run\n" }, { "path": "README.md", "content": "\"Asynq\n\n# Simple, reliable & efficient distributed task queue in Go\n\n[![GoDoc](https://godoc.org/github.com/hibiken/asynq?status.svg)](https://godoc.org/github.com/hibiken/asynq)\n[![Go Report Card](https://goreportcard.com/badge/github.com/hibiken/asynq)](https://goreportcard.com/report/github.com/hibiken/asynq)\n![Build Status](https://github.com/hibiken/asynq/workflows/build/badge.svg)\n[![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](https://opensource.org/licenses/MIT)\n[![Gitter chat](https://badges.gitter.im/go-asynq/gitter.svg)](https://gitter.im/go-asynq/community)\n\nAsynq is a Go library for queueing tasks and processing them asynchronously with workers. It's backed by [Redis](https://redis.io/) and is designed to be scalable yet easy to get started.\n\nHighlevel overview of how Asynq works:\n\n- Client puts tasks on a queue\n- Server pulls tasks off queues and starts a worker goroutine for each task\n- Tasks are processed concurrently by multiple workers\n\nTask queues are used as a mechanism to distribute work across multiple machines. A system can consist of multiple worker servers and brokers, giving way to high availability and horizontal scaling.\n\n**Example use case**\n\n![Task Queue Diagram](https://user-images.githubusercontent.com/11155743/116358505-656f5f80-a806-11eb-9c16-94e49dab0f99.jpg)\n\n## Features\n\n- Guaranteed [at least one execution](https://www.cloudcomputingpatterns.org/at_least_once_delivery/) of a task\n- Scheduling of tasks\n- [Retries](https://github.com/hibiken/asynq/wiki/Task-Retry) of failed tasks\n- Automatic recovery of tasks in the event of a worker crash\n- [Weighted priority queues](https://github.com/hibiken/asynq/wiki/Queue-Priority#weighted-priority)\n- [Strict priority queues](https://github.com/hibiken/asynq/wiki/Queue-Priority#strict-priority)\n- Low latency to add a task since writes are fast in Redis\n- De-duplication of tasks using [unique option](https://github.com/hibiken/asynq/wiki/Unique-Tasks)\n- Allow [timeout and deadline per task](https://github.com/hibiken/asynq/wiki/Task-Timeout-and-Cancelation)\n- Allow [aggregating group of tasks](https://github.com/hibiken/asynq/wiki/Task-aggregation) to batch multiple successive operations\n- [Flexible handler interface with support for middlewares](https://github.com/hibiken/asynq/wiki/Handler-Deep-Dive)\n- [Ability to pause queue](/tools/asynq/README.md#pause) to stop processing tasks from the queue\n- [Periodic Tasks](https://github.com/hibiken/asynq/wiki/Periodic-Tasks)\n- [Support Redis Sentinels](https://github.com/hibiken/asynq/wiki/Automatic-Failover) for high availability\n- Integration with [Prometheus](https://prometheus.io/) to collect and visualize queue metrics\n- [Web UI](#web-ui) to inspect and remote-control queues and tasks\n- [CLI](#command-line-tool) to inspect and remote-control queues and tasks\n\n## Stability and Compatibility\n\n**Status**: The library relatively stable and is currently undergoing **moderate development** with less frequent breaking API changes.\n\n> ☝️ **Important Note**: Current major version is zero (`v0.x.x`) to accommodate rapid development and fast iteration while getting early feedback from users (_feedback on APIs are appreciated!_). The public API could change without a major version update before `v1.0.0` release.\n\n### Redis Cluster Compatibility\n\nSome of the lua scripts in this library may not be compatible with Redis Cluster.\n\n## Sponsoring\nIf you are using this package in production, **please consider sponsoring the project to show your support!**\n\n## Quickstart\nMake sure you have Go installed ([download](https://golang.org/dl/)). The **last two** Go versions are supported (See https://go.dev/dl).\n\nInitialize your project by creating a folder and then running `go mod init github.com/your/repo` ([learn more](https://blog.golang.org/using-go-modules)) inside the folder. Then install Asynq library with the [`go get`](https://golang.org/cmd/go/#hdr-Add_dependencies_to_current_module_and_install_them) command:\n\n```sh\ngo get -u github.com/hibiken/asynq\n```\n\nMake sure you're running a Redis server locally or from a [Docker](https://hub.docker.com/_/redis) container. Version `4.0` or higher is required.\n\nNext, write a package that encapsulates task creation and task handling.\n\n```go\npackage tasks\n\nimport (\n \"context\"\n \"encoding/json\"\n \"fmt\"\n \"log\"\n \"time\"\n \"github.com/hibiken/asynq\"\n)\n\n// A list of task types.\nconst (\n TypeEmailDelivery = \"email:deliver\"\n TypeImageResize = \"image:resize\"\n)\n\ntype EmailDeliveryPayload struct {\n UserID int\n TemplateID string\n}\n\ntype ImageResizePayload struct {\n SourceURL string\n}\n\n//----------------------------------------------\n// Write a function NewXXXTask to create a task.\n// A task consists of a type and a payload.\n//----------------------------------------------\n\nfunc NewEmailDeliveryTask(userID int, tmplID string) (*asynq.Task, error) {\n payload, err := json.Marshal(EmailDeliveryPayload{UserID: userID, TemplateID: tmplID})\n if err != nil {\n return nil, err\n }\n return asynq.NewTask(TypeEmailDelivery, payload), nil\n}\n\nfunc NewImageResizeTask(src string) (*asynq.Task, error) {\n payload, err := json.Marshal(ImageResizePayload{SourceURL: src})\n if err != nil {\n return nil, err\n }\n // task options can be passed to NewTask, which can be overridden at enqueue time.\n return asynq.NewTask(TypeImageResize, payload, asynq.MaxRetry(5), asynq.Timeout(20 * time.Minute)), nil\n}\n\n//---------------------------------------------------------------\n// Write a function HandleXXXTask to handle the input task.\n// Note that it satisfies the asynq.HandlerFunc interface.\n//\n// Handler doesn't need to be a function. You can define a type\n// that satisfies asynq.Handler interface. See examples below.\n//---------------------------------------------------------------\n\nfunc HandleEmailDeliveryTask(ctx context.Context, t *asynq.Task) error {\n var p EmailDeliveryPayload\n if err := json.Unmarshal(t.Payload(), &p); err != nil {\n return fmt.Errorf(\"json.Unmarshal failed: %v: %w\", err, asynq.SkipRetry)\n }\n log.Printf(\"Sending Email to User: user_id=%d, template_id=%s\", p.UserID, p.TemplateID)\n // Email delivery code ...\n return nil\n}\n\n// ImageProcessor implements asynq.Handler interface.\ntype ImageProcessor struct {\n // ... fields for struct\n}\n\nfunc (processor *ImageProcessor) ProcessTask(ctx context.Context, t *asynq.Task) error {\n var p ImageResizePayload\n if err := json.Unmarshal(t.Payload(), &p); err != nil {\n return fmt.Errorf(\"json.Unmarshal failed: %v: %w\", err, asynq.SkipRetry)\n }\n log.Printf(\"Resizing image: src=%s\", p.SourceURL)\n // Image resizing code ...\n return nil\n}\n\nfunc NewImageProcessor() *ImageProcessor {\n\treturn &ImageProcessor{}\n}\n```\n\nIn your application code, import the above package and use [`Client`](https://pkg.go.dev/github.com/hibiken/asynq?tab=doc#Client) to put tasks on queues.\n\n```go\npackage main\n\nimport (\n \"log\"\n \"time\"\n\n \"github.com/hibiken/asynq\"\n \"your/app/package/tasks\"\n)\n\nconst redisAddr = \"127.0.0.1:6379\"\n\nfunc main() {\n client := asynq.NewClient(asynq.RedisClientOpt{Addr: redisAddr})\n defer client.Close()\n\n // ------------------------------------------------------\n // Example 1: Enqueue task to be processed immediately.\n // Use (*Client).Enqueue method.\n // ------------------------------------------------------\n\n task, err := tasks.NewEmailDeliveryTask(42, \"some:template:id\")\n if err != nil {\n log.Fatalf(\"could not create task: %v\", err)\n }\n info, err := client.Enqueue(task)\n if err != nil {\n log.Fatalf(\"could not enqueue task: %v\", err)\n }\n log.Printf(\"enqueued task: id=%s queue=%s\", info.ID, info.Queue)\n\n\n // ------------------------------------------------------------\n // Example 2: Schedule task to be processed in the future.\n // Use ProcessIn or ProcessAt option.\n // ------------------------------------------------------------\n\n info, err = client.Enqueue(task, asynq.ProcessIn(24*time.Hour))\n if err != nil {\n log.Fatalf(\"could not schedule task: %v\", err)\n }\n log.Printf(\"enqueued task: id=%s queue=%s\", info.ID, info.Queue)\n\n\n // ----------------------------------------------------------------------------\n // Example 3: Set other options to tune task processing behavior.\n // Options include MaxRetry, Queue, Timeout, Deadline, Unique etc.\n // ----------------------------------------------------------------------------\n\n task, err = tasks.NewImageResizeTask(\"https://example.com/myassets/image.jpg\")\n if err != nil {\n log.Fatalf(\"could not create task: %v\", err)\n }\n info, err = client.Enqueue(task, asynq.MaxRetry(10), asynq.Timeout(3 * time.Minute))\n if err != nil {\n log.Fatalf(\"could not enqueue task: %v\", err)\n }\n log.Printf(\"enqueued task: id=%s queue=%s\", info.ID, info.Queue)\n}\n```\n\nNext, start a worker server to process these tasks in the background. To start the background workers, use [`Server`](https://pkg.go.dev/github.com/hibiken/asynq?tab=doc#Server) and provide your [`Handler`](https://pkg.go.dev/github.com/hibiken/asynq?tab=doc#Handler) to process the tasks.\n\nYou can optionally use [`ServeMux`](https://pkg.go.dev/github.com/hibiken/asynq?tab=doc#ServeMux) to create a handler, just as you would with [`net/http`](https://golang.org/pkg/net/http/) Handler.\n\n```go\npackage main\n\nimport (\n \"log\"\n\n \"github.com/hibiken/asynq\"\n \"your/app/package/tasks\"\n)\n\nconst redisAddr = \"127.0.0.1:6379\"\n\nfunc main() {\n srv := asynq.NewServer(\n asynq.RedisClientOpt{Addr: redisAddr},\n asynq.Config{\n // Specify how many concurrent workers to use\n Concurrency: 10,\n // Optionally specify multiple queues with different priority.\n Queues: map[string]int{\n \"critical\": 6,\n \"default\": 3,\n \"low\": 1,\n },\n // See the godoc for other configuration options\n },\n )\n\n // mux maps a type to a handler\n mux := asynq.NewServeMux()\n mux.HandleFunc(tasks.TypeEmailDelivery, tasks.HandleEmailDeliveryTask)\n mux.Handle(tasks.TypeImageResize, tasks.NewImageProcessor())\n // ...register other handlers...\n\n if err := srv.Run(mux); err != nil {\n log.Fatalf(\"could not run server: %v\", err)\n }\n}\n```\n\nFor a more detailed walk-through of the library, see our [Getting Started](https://github.com/hibiken/asynq/wiki/Getting-Started) guide.\n\nTo learn more about `asynq` features and APIs, see the package [godoc](https://godoc.org/github.com/hibiken/asynq).\n\n## Web UI\n\n[Asynqmon](https://github.com/hibiken/asynqmon) is a web based tool for monitoring and administrating Asynq queues and tasks.\n\nHere's a few screenshots of the Web UI:\n\n**Queues view**\n\n![Web UI Queues View](https://user-images.githubusercontent.com/11155743/114697016-07327f00-9d26-11eb-808c-0ac841dc888e.png)\n\n**Tasks view**\n\n![Web UI TasksView](https://user-images.githubusercontent.com/11155743/114697070-1f0a0300-9d26-11eb-855c-d3ec263865b7.png)\n\n**Metrics view**\n\"Screen\n\n**Settings and adaptive dark mode**\n\n![Web UI Settings and adaptive dark mode](https://user-images.githubusercontent.com/11155743/114697149-3517c380-9d26-11eb-9f7a-ae2dd00aad5b.png)\n\nFor details on how to use the tool, refer to the tool's [README](https://github.com/hibiken/asynqmon#readme).\n\n## Command Line Tool\n\nAsynq ships with a command line tool to inspect the state of queues and tasks.\n\nTo install the CLI tool, run the following command:\n\n```sh\ngo install github.com/hibiken/asynq/tools/asynq@latest\n```\n\nHere's an example of running the `asynq dash` command:\n\n![Gif](/docs/assets/dash.gif)\n\nFor details on how to use the tool, refer to the tool's [README](/tools/asynq/README.md).\n\n## Contributing\n\nWe are open to, and grateful for, any contributions (GitHub issues/PRs, feedback on [Gitter channel](https://gitter.im/go-asynq/community), etc) made by the community.\n\nPlease see the [Contribution Guide](/CONTRIBUTING.md) before contributing.\n\n## License\n\nCopyright (c) 2019-present [Ken Hibino](https://github.com/hibiken) and [Contributors](https://github.com/hibiken/asynq/graphs/contributors). `Asynq` is free and open-source software licensed under the [MIT License](https://github.com/hibiken/asynq/blob/master/LICENSE). Official logo was created by [Vic Shóstak](https://github.com/koddr) and distributed under [Creative Commons](https://creativecommons.org/publicdomain/zero/1.0/) license (CC0 1.0 Universal).\n" }, { "path": "aggregator.go", "content": "// Copyright 2022 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"context\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/log\"\n)\n\n// An aggregator is responsible for checking groups and aggregate into one task\n// if any of the grouping condition is met.\ntype aggregator struct {\n\tlogger *log.Logger\n\tbroker base.Broker\n\tclient *Client\n\n\t// channel to communicate back to the long running \"aggregator\" goroutine.\n\tdone chan struct{}\n\n\t// list of queue names to check and aggregate.\n\tqueues []string\n\n\t// Group configurations\n\tgracePeriod time.Duration\n\tmaxDelay time.Duration\n\tmaxSize int\n\n\t// User provided group aggregator.\n\tga GroupAggregator\n\n\t// interval used to check for aggregation\n\tinterval time.Duration\n\n\t// sema is a counting semaphore to ensure the number of active aggregating function\n\t// does not exceed the limit.\n\tsema chan struct{}\n}\n\ntype aggregatorParams struct {\n\tlogger *log.Logger\n\tbroker base.Broker\n\tqueues []string\n\tgracePeriod time.Duration\n\tmaxDelay time.Duration\n\tmaxSize int\n\tgroupAggregator GroupAggregator\n}\n\nconst (\n\t// Maximum number of aggregation checks in flight concurrently.\n\tmaxConcurrentAggregationChecks = 3\n\n\t// Default interval used for aggregation checks. If the provided gracePeriod is less than\n\t// the default, use the gracePeriod.\n\tdefaultAggregationCheckInterval = 7 * time.Second\n)\n\nfunc newAggregator(params aggregatorParams) *aggregator {\n\tinterval := defaultAggregationCheckInterval\n\tif params.gracePeriod < interval {\n\t\tinterval = params.gracePeriod\n\t}\n\treturn &aggregator{\n\t\tlogger: params.logger,\n\t\tbroker: params.broker,\n\t\tclient: &Client{broker: params.broker},\n\t\tdone: make(chan struct{}),\n\t\tqueues: params.queues,\n\t\tgracePeriod: params.gracePeriod,\n\t\tmaxDelay: params.maxDelay,\n\t\tmaxSize: params.maxSize,\n\t\tga: params.groupAggregator,\n\t\tsema: make(chan struct{}, maxConcurrentAggregationChecks),\n\t\tinterval: interval,\n\t}\n}\n\nfunc (a *aggregator) shutdown() {\n\tif a.ga == nil {\n\t\treturn\n\t}\n\ta.logger.Debug(\"Aggregator shutting down...\")\n\t// Signal the aggregator goroutine to stop.\n\ta.done <- struct{}{}\n}\n\nfunc (a *aggregator) start(wg *sync.WaitGroup) {\n\tif a.ga == nil {\n\t\treturn\n\t}\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tticker := time.NewTicker(a.interval)\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-a.done:\n\t\t\t\ta.logger.Debug(\"Waiting for all aggregation checks to finish...\")\n\t\t\t\t// block until all aggregation checks released the token\n\t\t\t\tfor i := 0; i < cap(a.sema); i++ {\n\t\t\t\t\ta.sema <- struct{}{}\n\t\t\t\t}\n\t\t\t\ta.logger.Debug(\"Aggregator done\")\n\t\t\t\tticker.Stop()\n\t\t\t\treturn\n\t\t\tcase t := <-ticker.C:\n\t\t\t\ta.exec(t)\n\t\t\t}\n\t\t}\n\t}()\n}\n\nfunc (a *aggregator) exec(t time.Time) {\n\tselect {\n\tcase a.sema <- struct{}{}: // acquire token\n\t\tgo a.aggregate(t)\n\tdefault:\n\t\t// If the semaphore blocks, then we are currently running max number of\n\t\t// aggregation checks. Skip this round and log warning.\n\t\ta.logger.Warnf(\"Max number of aggregation checks in flight. Skipping\")\n\t}\n}\n\nfunc (a *aggregator) aggregate(t time.Time) {\n\tdefer func() { <-a.sema /* release token */ }()\n\tfor _, qname := range a.queues {\n\t\tgroups, err := a.broker.ListGroups(qname)\n\t\tif err != nil {\n\t\t\ta.logger.Errorf(\"Failed to list groups in queue: %q\", qname)\n\t\t\tcontinue\n\t\t}\n\t\tfor _, gname := range groups {\n\t\t\taggregationSetID, err := a.broker.AggregationCheck(\n\t\t\t\tqname, gname, t, a.gracePeriod, a.maxDelay, a.maxSize)\n\t\t\tif err != nil {\n\t\t\t\ta.logger.Errorf(\"Failed to run aggregation check: queue=%q group=%q\", qname, gname)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif aggregationSetID == \"\" {\n\t\t\t\ta.logger.Debugf(\"No aggregation needed at this time: queue=%q group=%q\", qname, gname)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\t// Aggregate and enqueue.\n\t\t\tmsgs, deadline, err := a.broker.ReadAggregationSet(qname, gname, aggregationSetID)\n\t\t\tif err != nil {\n\t\t\t\ta.logger.Errorf(\"Failed to read aggregation set: queue=%q, group=%q, setID=%q\",\n\t\t\t\t\tqname, gname, aggregationSetID)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\ttasks := make([]*Task, len(msgs))\n\t\t\tfor i, m := range msgs {\n\t\t\t\ttasks[i] = NewTaskWithHeaders(m.Type, m.Payload, m.Headers)\n\t\t\t}\n\t\t\taggregatedTask := a.ga.Aggregate(gname, tasks)\n\t\t\tctx, cancel := context.WithDeadline(context.Background(), deadline)\n\t\t\tif _, err := a.client.EnqueueContext(ctx, aggregatedTask, Queue(qname)); err != nil {\n\t\t\t\ta.logger.Errorf(\"Failed to enqueue aggregated task (queue=%q, group=%q, setID=%q): %v\",\n\t\t\t\t\tqname, gname, aggregationSetID, err)\n\t\t\t\tcancel()\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif err := a.broker.DeleteAggregationSet(ctx, qname, gname, aggregationSetID); err != nil {\n\t\t\t\ta.logger.Warnf(\"Failed to delete aggregation set: queue=%q, group=%q, setID=%q\",\n\t\t\t\t\tqname, gname, aggregationSetID)\n\t\t\t}\n\t\t\tcancel()\n\t\t}\n\t}\n}\n" }, { "path": "aggregator_test.go", "content": "// Copyright 2022 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/google/go-cmp/cmp\"\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/rdb\"\n\th \"github.com/hibiken/asynq/internal/testutil\"\n)\n\nfunc TestAggregator(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\trdbClient := rdb.NewRDB(r)\n\tclient := Client{broker: rdbClient}\n\n\ttests := []struct {\n\t\tdesc string\n\t\tgracePeriod time.Duration\n\t\tmaxDelay time.Duration\n\t\tmaxSize int\n\t\taggregateFunc func(gname string, tasks []*Task) *Task\n\t\ttasks []*Task // tasks to enqueue\n\t\tenqueueFrequency time.Duration // time between one enqueue event to another\n\t\twaitTime time.Duration // time to wait\n\t\twantGroups map[string]map[string][]base.Z\n\t\twantPending map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tdesc: \"group older than the grace period should be aggregated\",\n\t\t\tgracePeriod: 1 * time.Second,\n\t\t\tmaxDelay: 0, // no maxdelay limit\n\t\t\tmaxSize: 0, // no maxsize limit\n\t\t\taggregateFunc: func(gname string, tasks []*Task) *Task {\n\t\t\t\treturn NewTask(gname, nil, MaxRetry(len(tasks))) // use max retry to see how many tasks were aggregated\n\t\t\t},\n\t\t\ttasks: []*Task{\n\t\t\t\tNewTask(\"task1\", nil, Group(\"mygroup\")),\n\t\t\t\tNewTask(\"task2\", nil, Group(\"mygroup\")),\n\t\t\t\tNewTask(\"task3\", nil, Group(\"mygroup\")),\n\t\t\t},\n\t\t\tenqueueFrequency: 300 * time.Millisecond,\n\t\t\twaitTime: 3 * time.Second,\n\t\t\twantGroups: map[string]map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t\"mygroup\": {},\n\t\t\t\t},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {\n\t\t\t\t\th.NewTaskMessageBuilder().SetType(\"mygroup\").SetRetry(3).Build(),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"group older than the max-delay should be aggregated\",\n\t\t\tgracePeriod: 2 * time.Second,\n\t\t\tmaxDelay: 4 * time.Second,\n\t\t\tmaxSize: 0, // no maxsize limit\n\t\t\taggregateFunc: func(gname string, tasks []*Task) *Task {\n\t\t\t\treturn NewTask(gname, nil, MaxRetry(len(tasks))) // use max retry to see how many tasks were aggregated\n\t\t\t},\n\t\t\ttasks: []*Task{\n\t\t\t\tNewTask(\"task1\", nil, Group(\"mygroup\")), // time 0\n\t\t\t\tNewTask(\"task2\", nil, Group(\"mygroup\")), // time 1s\n\t\t\t\tNewTask(\"task3\", nil, Group(\"mygroup\")), // time 2s\n\t\t\t\tNewTask(\"task4\", nil, Group(\"mygroup\")), // time 3s\n\t\t\t},\n\t\t\tenqueueFrequency: 1 * time.Second,\n\t\t\twaitTime: 4 * time.Second,\n\t\t\twantGroups: map[string]map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t\"mygroup\": {},\n\t\t\t\t},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {\n\t\t\t\t\th.NewTaskMessageBuilder().SetType(\"mygroup\").SetRetry(4).Build(),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"group reached the max-size should be aggregated\",\n\t\t\tgracePeriod: 1 * time.Minute,\n\t\t\tmaxDelay: 0, // no maxdelay limit\n\t\t\tmaxSize: 5,\n\t\t\taggregateFunc: func(gname string, tasks []*Task) *Task {\n\t\t\t\treturn NewTask(gname, nil, MaxRetry(len(tasks))) // use max retry to see how many tasks were aggregated\n\t\t\t},\n\t\t\ttasks: []*Task{\n\t\t\t\tNewTask(\"task1\", nil, Group(\"mygroup\")),\n\t\t\t\tNewTask(\"task2\", nil, Group(\"mygroup\")),\n\t\t\t\tNewTask(\"task3\", nil, Group(\"mygroup\")),\n\t\t\t\tNewTask(\"task4\", nil, Group(\"mygroup\")),\n\t\t\t\tNewTask(\"task5\", nil, Group(\"mygroup\")),\n\t\t\t},\n\t\t\tenqueueFrequency: 300 * time.Millisecond,\n\t\t\twaitTime: defaultAggregationCheckInterval * 2,\n\t\t\twantGroups: map[string]map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t\"mygroup\": {},\n\t\t\t\t},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {\n\t\t\t\t\th.NewTaskMessageBuilder().SetType(\"mygroup\").SetRetry(5).Build(),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\n\t\taggregator := newAggregator(aggregatorParams{\n\t\t\tlogger: testLogger,\n\t\t\tbroker: rdbClient,\n\t\t\tqueues: []string{\"default\"},\n\t\t\tgracePeriod: tc.gracePeriod,\n\t\t\tmaxDelay: tc.maxDelay,\n\t\t\tmaxSize: tc.maxSize,\n\t\t\tgroupAggregator: GroupAggregatorFunc(tc.aggregateFunc),\n\t\t})\n\n\t\tvar wg sync.WaitGroup\n\t\taggregator.start(&wg)\n\n\t\tfor _, task := range tc.tasks {\n\t\t\tif _, err := client.Enqueue(task); err != nil {\n\t\t\t\tt.Errorf(\"%s: Client Enqueue failed: %v\", tc.desc, err)\n\t\t\t\taggregator.shutdown()\n\t\t\t\twg.Wait()\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\ttime.Sleep(tc.enqueueFrequency)\n\t\t}\n\n\t\ttime.Sleep(tc.waitTime)\n\n\t\tfor qname, groups := range tc.wantGroups {\n\t\t\tfor gname, want := range groups {\n\t\t\t\tgotGroup := h.GetGroupEntries(t, r, qname, gname)\n\t\t\t\tif diff := cmp.Diff(want, gotGroup, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\t\tt.Errorf(\"%s: mismatch found in %q; (-want,+got)\\n%s\", tc.desc, base.GroupKey(qname, gname), diff)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt, h.IgnoreIDOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s: mismatch found in %q; (-want,+got)\\n%s\", tc.desc, base.PendingKey(qname), diff)\n\t\t\t}\n\t\t}\n\t\taggregator.shutdown()\n\t\twg.Wait()\n\t}\n}\n" }, { "path": "asynq.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\t\"fmt\"\n\t\"maps\"\n\t\"net\"\n\t\"net/url\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/redis/go-redis/v9\"\n)\n\n// Task represents a unit of work to be performed.\ntype Task struct {\n\t// typename indicates the type of task to be performed.\n\ttypename string\n\n\t// payload holds data needed to perform the task.\n\tpayload []byte\n\n\t// headers holds additional metadata for the task.\n\theaders map[string]string\n\n\t// opts holds options for the task.\n\topts []Option\n\n\t// w is the ResultWriter for the task.\n\tw *ResultWriter\n}\n\nfunc (t *Task) Type() string { return t.typename }\nfunc (t *Task) Payload() []byte { return t.payload }\nfunc (t *Task) Headers() map[string]string { return t.headers }\n\n// ResultWriter returns a pointer to the ResultWriter associated with the task.\n//\n// Nil pointer is returned if called on a newly created task (i.e. task created by calling NewTask).\n// Only the tasks passed to Handler.ProcessTask have a valid ResultWriter pointer.\nfunc (t *Task) ResultWriter() *ResultWriter { return t.w }\n\n// NewTask returns a new Task given a type name and payload data.\n// Options can be passed to configure task processing behavior.\nfunc NewTask(typename string, payload []byte, opts ...Option) *Task {\n\treturn &Task{\n\t\ttypename: typename,\n\t\tpayload: payload,\n\t\theaders: nil,\n\t\topts: opts,\n\t}\n}\n\n// NewTaskWithHeaders returns a new Task given a type name, payload data, and headers.\n// Options can be passed to configure task processing behavior.\n// TODO: In the next major (breaking) release, fold this functionality into NewTask\n//\n//\tso that headers are supported directly. After that, remove this method.\nfunc NewTaskWithHeaders(typename string, payload []byte, headers map[string]string, opts ...Option) *Task {\n\treturn &Task{\n\t\ttypename: typename,\n\t\tpayload: payload,\n\t\theaders: maps.Clone(headers),\n\t\topts: opts,\n\t}\n}\n\n// newTask creates a task with the given typename, payload and ResultWriter.\nfunc newTask(typename string, payload []byte, w *ResultWriter) *Task {\n\treturn &Task{\n\t\ttypename: typename,\n\t\tpayload: payload,\n\t\theaders: make(map[string]string),\n\t\tw: w,\n\t}\n}\n\n// A TaskInfo describes a task and its metadata.\ntype TaskInfo struct {\n\t// ID is the identifier of the task.\n\tID string\n\n\t// Queue is the name of the queue in which the task belongs.\n\tQueue string\n\n\t// Type is the type name of the task.\n\tType string\n\n\t// Payload is the payload data of the task.\n\tPayload []byte\n\n\t// Headers holds additional metadata for the task.\n\tHeaders map[string]string\n\n\t// State indicates the task state.\n\tState TaskState\n\n\t// MaxRetry is the maximum number of times the task can be retried.\n\tMaxRetry int\n\n\t// Retried is the number of times the task has retried so far.\n\tRetried int\n\n\t// LastErr is the error message from the last failure.\n\tLastErr string\n\n\t// LastFailedAt is the time time of the last failure if any.\n\t// If the task has no failures, LastFailedAt is zero time (i.e. time.Time{}).\n\tLastFailedAt time.Time\n\n\t// Timeout is the duration the task can be processed by Handler before being retried,\n\t// zero if not specified\n\tTimeout time.Duration\n\n\t// Deadline is the deadline for the task, zero value if not specified.\n\tDeadline time.Time\n\n\t// Group is the name of the group in which the task belongs.\n\t//\n\t// Tasks in the same queue can be grouped together by Group name and will be aggregated into one task\n\t// by a Server processing the queue.\n\t//\n\t// Empty string (default) indicates task does not belong to any groups, and no aggregation will be applied to the task.\n\tGroup string\n\n\t// NextProcessAt is the time the task is scheduled to be processed,\n\t// zero if not applicable.\n\tNextProcessAt time.Time\n\n\t// IsOrphaned describes whether the task is left in active state with no worker processing it.\n\t// An orphaned task indicates that the worker has crashed or experienced network failures and was not able to\n\t// extend its lease on the task.\n\t//\n\t// This task will be recovered by running a server against the queue the task is in.\n\t// This field is only applicable to tasks with TaskStateActive.\n\tIsOrphaned bool\n\n\t// Retention is duration of the retention period after the task is successfully processed.\n\tRetention time.Duration\n\n\t// CompletedAt is the time when the task is processed successfully.\n\t// Zero value (i.e. time.Time{}) indicates no value.\n\tCompletedAt time.Time\n\n\t// Result holds the result data associated with the task.\n\t// Use ResultWriter to write result data from the Handler.\n\tResult []byte\n}\n\n// If t is non-zero, returns time converted from t as unix time in seconds.\n// If t is zero, returns zero value of time.Time.\nfunc fromUnixTimeOrZero(t int64) time.Time {\n\tif t == 0 {\n\t\treturn time.Time{}\n\t}\n\treturn time.Unix(t, 0)\n}\n\nfunc newTaskInfo(msg *base.TaskMessage, state base.TaskState, nextProcessAt time.Time, result []byte) *TaskInfo {\n\tinfo := TaskInfo{\n\t\tID: msg.ID,\n\t\tQueue: msg.Queue,\n\t\tType: msg.Type,\n\t\tPayload: msg.Payload, // Do we need to make a copy?\n\t\tHeaders: msg.Headers,\n\t\tMaxRetry: msg.Retry,\n\t\tRetried: msg.Retried,\n\t\tLastErr: msg.ErrorMsg,\n\t\tGroup: msg.GroupKey,\n\t\tTimeout: time.Duration(msg.Timeout) * time.Second,\n\t\tDeadline: fromUnixTimeOrZero(msg.Deadline),\n\t\tRetention: time.Duration(msg.Retention) * time.Second,\n\t\tNextProcessAt: nextProcessAt,\n\t\tLastFailedAt: fromUnixTimeOrZero(msg.LastFailedAt),\n\t\tCompletedAt: fromUnixTimeOrZero(msg.CompletedAt),\n\t\tResult: result,\n\t}\n\n\tswitch state {\n\tcase base.TaskStateActive:\n\t\tinfo.State = TaskStateActive\n\tcase base.TaskStatePending:\n\t\tinfo.State = TaskStatePending\n\tcase base.TaskStateScheduled:\n\t\tinfo.State = TaskStateScheduled\n\tcase base.TaskStateRetry:\n\t\tinfo.State = TaskStateRetry\n\tcase base.TaskStateArchived:\n\t\tinfo.State = TaskStateArchived\n\tcase base.TaskStateCompleted:\n\t\tinfo.State = TaskStateCompleted\n\tcase base.TaskStateAggregating:\n\t\tinfo.State = TaskStateAggregating\n\tdefault:\n\t\tpanic(fmt.Sprintf(\"internal error: unknown state: %d\", state))\n\t}\n\treturn &info\n}\n\n// TaskState denotes the state of a task.\ntype TaskState int\n\nconst (\n\t// Indicates that the task is currently being processed by Handler.\n\tTaskStateActive TaskState = iota + 1\n\n\t// Indicates that the task is ready to be processed by Handler.\n\tTaskStatePending\n\n\t// Indicates that the task is scheduled to be processed some time in the future.\n\tTaskStateScheduled\n\n\t// Indicates that the task has previously failed and scheduled to be processed some time in the future.\n\tTaskStateRetry\n\n\t// Indicates that the task is archived and stored for inspection purposes.\n\tTaskStateArchived\n\n\t// Indicates that the task is processed successfully and retained until the retention TTL expires.\n\tTaskStateCompleted\n\n\t// Indicates that the task is waiting in a group to be aggregated into one task.\n\tTaskStateAggregating\n)\n\nfunc (s TaskState) String() string {\n\tswitch s {\n\tcase TaskStateActive:\n\t\treturn \"active\"\n\tcase TaskStatePending:\n\t\treturn \"pending\"\n\tcase TaskStateScheduled:\n\t\treturn \"scheduled\"\n\tcase TaskStateRetry:\n\t\treturn \"retry\"\n\tcase TaskStateArchived:\n\t\treturn \"archived\"\n\tcase TaskStateCompleted:\n\t\treturn \"completed\"\n\tcase TaskStateAggregating:\n\t\treturn \"aggregating\"\n\t}\n\tpanic(\"asynq: unknown task state\")\n}\n\n// RedisConnOpt is a discriminated union of types that represent Redis connection configuration option.\n//\n// RedisConnOpt represents a sum of following types:\n//\n// - RedisClientOpt\n// - RedisFailoverClientOpt\n// - RedisClusterClientOpt\ntype RedisConnOpt interface {\n\t// MakeRedisClient returns a new redis client instance.\n\t// Return value is intentionally opaque to hide the implementation detail of redis client.\n\tMakeRedisClient() interface{}\n}\n\n// RedisClientOpt is used to create a redis client that connects\n// to a redis server directly.\ntype RedisClientOpt struct {\n\t// Network type to use, either tcp or unix.\n\t// Default is tcp.\n\tNetwork string\n\n\t// Redis server address in \"host:port\" format.\n\tAddr string\n\n\t// Username to authenticate the current connection when Redis ACLs are used.\n\t// See: https://redis.io/commands/auth.\n\tUsername string\n\n\t// Password to authenticate the current connection.\n\t// See: https://redis.io/commands/auth.\n\tPassword string\n\n\t// Redis DB to select after connecting to a server.\n\t// See: https://redis.io/commands/select.\n\tDB int\n\n\t// Dial timeout for establishing new connections.\n\t// Default is 5 seconds.\n\tDialTimeout time.Duration\n\n\t// Timeout for socket reads.\n\t// If timeout is reached, read commands will fail with a timeout error\n\t// instead of blocking.\n\t//\n\t// Use value -1 for no timeout and 0 for default.\n\t// Default is 3 seconds.\n\tReadTimeout time.Duration\n\n\t// Timeout for socket writes.\n\t// If timeout is reached, write commands will fail with a timeout error\n\t// instead of blocking.\n\t//\n\t// Use value -1 for no timeout and 0 for default.\n\t// Default is ReadTimout.\n\tWriteTimeout time.Duration\n\n\t// Maximum number of socket connections.\n\t// Default is 10 connections per every CPU as reported by runtime.NumCPU.\n\tPoolSize int\n\n\t// TLS Config used to connect to a server.\n\t// TLS will be negotiated only if this field is set.\n\tTLSConfig *tls.Config\n}\n\nfunc (opt RedisClientOpt) MakeRedisClient() interface{} {\n\treturn redis.NewClient(&redis.Options{\n\t\tNetwork: opt.Network,\n\t\tAddr: opt.Addr,\n\t\tUsername: opt.Username,\n\t\tPassword: opt.Password,\n\t\tDB: opt.DB,\n\t\tDialTimeout: opt.DialTimeout,\n\t\tReadTimeout: opt.ReadTimeout,\n\t\tWriteTimeout: opt.WriteTimeout,\n\t\tPoolSize: opt.PoolSize,\n\t\tTLSConfig: opt.TLSConfig,\n\t})\n}\n\n// RedisFailoverClientOpt is used to creates a redis client that talks\n// to redis sentinels for service discovery and has an automatic failover\n// capability.\ntype RedisFailoverClientOpt struct {\n\t// Redis master name that monitored by sentinels.\n\tMasterName string\n\n\t// Addresses of sentinels in \"host:port\" format.\n\t// Use at least three sentinels to avoid problems described in\n\t// https://redis.io/topics/sentinel.\n\tSentinelAddrs []string\n\n\t// Redis sentinel username.\n\tSentinelUsername string\n\n\t// Redis sentinel password.\n\tSentinelPassword string\n\n\t// Username to authenticate the current connection when Redis ACLs are used.\n\t// See: https://redis.io/commands/auth.\n\tUsername string\n\n\t// Password to authenticate the current connection.\n\t// See: https://redis.io/commands/auth.\n\tPassword string\n\n\t// Redis DB to select after connecting to a server.\n\t// See: https://redis.io/commands/select.\n\tDB int\n\n\t// Dial timeout for establishing new connections.\n\t// Default is 5 seconds.\n\tDialTimeout time.Duration\n\n\t// Timeout for socket reads.\n\t// If timeout is reached, read commands will fail with a timeout error\n\t// instead of blocking.\n\t//\n\t// Use value -1 for no timeout and 0 for default.\n\t// Default is 3 seconds.\n\tReadTimeout time.Duration\n\n\t// Timeout for socket writes.\n\t// If timeout is reached, write commands will fail with a timeout error\n\t// instead of blocking.\n\t//\n\t// Use value -1 for no timeout and 0 for default.\n\t// Default is ReadTimeout\n\tWriteTimeout time.Duration\n\n\t// Maximum number of socket connections.\n\t// Default is 10 connections per every CPU as reported by runtime.NumCPU.\n\tPoolSize int\n\n\t// TLS Config used to connect to a server.\n\t// TLS will be negotiated only if this field is set.\n\tTLSConfig *tls.Config\n}\n\nfunc (opt RedisFailoverClientOpt) MakeRedisClient() interface{} {\n\treturn redis.NewFailoverClient(&redis.FailoverOptions{\n\t\tMasterName: opt.MasterName,\n\t\tSentinelAddrs: opt.SentinelAddrs,\n\t\tSentinelUsername: opt.SentinelUsername,\n\t\tSentinelPassword: opt.SentinelPassword,\n\t\tUsername: opt.Username,\n\t\tPassword: opt.Password,\n\t\tDB: opt.DB,\n\t\tDialTimeout: opt.DialTimeout,\n\t\tReadTimeout: opt.ReadTimeout,\n\t\tWriteTimeout: opt.WriteTimeout,\n\t\tPoolSize: opt.PoolSize,\n\t\tTLSConfig: opt.TLSConfig,\n\t})\n}\n\n// RedisClusterClientOpt is used to creates a redis client that connects to\n// redis cluster.\ntype RedisClusterClientOpt struct {\n\t// A seed list of host:port addresses of cluster nodes.\n\tAddrs []string\n\n\t// The maximum number of retries before giving up.\n\t// Command is retried on network errors and MOVED/ASK redirects.\n\t// Default is 8 retries.\n\tMaxRedirects int\n\n\t// Username to authenticate the current connection when Redis ACLs are used.\n\t// See: https://redis.io/commands/auth.\n\tUsername string\n\n\t// Password to authenticate the current connection.\n\t// See: https://redis.io/commands/auth.\n\tPassword string\n\n\t// Dial timeout for establishing new connections.\n\t// Default is 5 seconds.\n\tDialTimeout time.Duration\n\n\t// Timeout for socket reads.\n\t// If timeout is reached, read commands will fail with a timeout error\n\t// instead of blocking.\n\t//\n\t// Use value -1 for no timeout and 0 for default.\n\t// Default is 3 seconds.\n\tReadTimeout time.Duration\n\n\t// Timeout for socket writes.\n\t// If timeout is reached, write commands will fail with a timeout error\n\t// instead of blocking.\n\t//\n\t// Use value -1 for no timeout and 0 for default.\n\t// Default is ReadTimeout.\n\tWriteTimeout time.Duration\n\n\t// TLS Config used to connect to a server.\n\t// TLS will be negotiated only if this field is set.\n\tTLSConfig *tls.Config\n}\n\nfunc (opt RedisClusterClientOpt) MakeRedisClient() interface{} {\n\treturn redis.NewClusterClient(&redis.ClusterOptions{\n\t\tAddrs: opt.Addrs,\n\t\tMaxRedirects: opt.MaxRedirects,\n\t\tUsername: opt.Username,\n\t\tPassword: opt.Password,\n\t\tDialTimeout: opt.DialTimeout,\n\t\tReadTimeout: opt.ReadTimeout,\n\t\tWriteTimeout: opt.WriteTimeout,\n\t\tTLSConfig: opt.TLSConfig,\n\t})\n}\n\n// ParseRedisURI parses redis uri string and returns RedisConnOpt if uri is valid.\n// It returns a non-nil error if uri cannot be parsed.\n//\n// Three URI schemes are supported, which are redis:, rediss:, redis-socket:, and redis-sentinel:.\n// Supported formats are:\n//\n//\tredis://[:password@]host[:port][/dbnumber]\n//\trediss://[:password@]host[:port][/dbnumber]\n//\tredis-socket://[:password@]path[?db=dbnumber]\n//\tredis-sentinel://[:password@]host1[:port][,host2:[:port]][,hostN:[:port]][?master=masterName]\nfunc ParseRedisURI(uri string) (RedisConnOpt, error) {\n\tu, err := url.Parse(uri)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"asynq: could not parse redis uri: %w\", err)\n\t}\n\tswitch u.Scheme {\n\tcase \"redis\", \"rediss\":\n\t\treturn parseRedisURI(u)\n\tcase \"redis-socket\":\n\t\treturn parseRedisSocketURI(u)\n\tcase \"redis-sentinel\":\n\t\treturn parseRedisSentinelURI(u)\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"asynq: unsupported uri scheme: %q\", u.Scheme)\n\t}\n}\n\nfunc parseRedisURI(u *url.URL) (RedisConnOpt, error) {\n\tvar db int\n\tvar err error\n\tvar redisConnOpt RedisClientOpt\n\n\tif len(u.Path) > 0 {\n\t\txs := strings.Split(strings.Trim(u.Path, \"/\"), \"/\")\n\t\tdb, err = strconv.Atoi(xs[0])\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"asynq: could not parse redis uri: database number should be the first segment of the path\")\n\t\t}\n\t}\n\tvar password string\n\tif v, ok := u.User.Password(); ok {\n\t\tpassword = v\n\t}\n\n\tif u.Scheme == \"rediss\" {\n\t\th, _, err := net.SplitHostPort(u.Host)\n\t\tif err != nil {\n\t\t\th = u.Host\n\t\t}\n\t\tredisConnOpt.TLSConfig = &tls.Config{ServerName: h}\n\t}\n\n\tredisConnOpt.Addr = u.Host\n\tredisConnOpt.Password = password\n\tredisConnOpt.DB = db\n\n\treturn redisConnOpt, nil\n}\n\nfunc parseRedisSocketURI(u *url.URL) (RedisConnOpt, error) {\n\tconst errPrefix = \"asynq: could not parse redis socket uri\"\n\tif len(u.Path) == 0 {\n\t\treturn nil, fmt.Errorf(\"%s: path does not exist\", errPrefix)\n\t}\n\tq := u.Query()\n\tvar db int\n\tvar err error\n\tif n := q.Get(\"db\"); n != \"\" {\n\t\tdb, err = strconv.Atoi(n)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"%s: query param `db` should be a number\", errPrefix)\n\t\t}\n\t}\n\tvar password string\n\tif v, ok := u.User.Password(); ok {\n\t\tpassword = v\n\t}\n\treturn RedisClientOpt{Network: \"unix\", Addr: u.Path, DB: db, Password: password}, nil\n}\n\nfunc parseRedisSentinelURI(u *url.URL) (RedisConnOpt, error) {\n\taddrs := strings.Split(u.Host, \",\")\n\tmaster := u.Query().Get(\"master\")\n\tvar password string\n\tif v, ok := u.User.Password(); ok {\n\t\tpassword = v\n\t}\n\treturn RedisFailoverClientOpt{MasterName: master, SentinelAddrs: addrs, SentinelPassword: password}, nil\n}\n\n// ResultWriter is a client interface to write result data for a task.\n// It writes the data to the redis instance the server is connected to.\ntype ResultWriter struct {\n\tid string // task ID this writer is responsible for\n\tqname string // queue name the task belongs to\n\tbroker base.Broker\n\tctx context.Context // context associated with the task\n}\n\n// Write writes the given data as a result of the task the ResultWriter is associated with.\nfunc (w *ResultWriter) Write(data []byte) (n int, err error) {\n\tselect {\n\tcase <-w.ctx.Done():\n\t\treturn 0, fmt.Errorf(\"failed to write task result: %w\", w.ctx.Err())\n\tdefault:\n\t}\n\treturn w.broker.WriteResult(w.qname, w.id, data)\n}\n\n// TaskID returns the ID of the task the ResultWriter is associated with.\nfunc (w *ResultWriter) TaskID() string {\n\treturn w.id\n}\n" }, { "path": "asynq_test.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"crypto/tls\"\n\t\"flag\"\n\t\"sort\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/google/go-cmp/cmp\"\n\t\"github.com/google/go-cmp/cmp/cmpopts\"\n\t\"github.com/hibiken/asynq/internal/log\"\n\th \"github.com/hibiken/asynq/internal/testutil\"\n\t\"github.com/redis/go-redis/v9\"\n)\n\n//============================================================================\n// This file defines helper functions and variables used in other test files.\n//============================================================================\n\n// variables used for package testing.\nvar (\n\tredisAddr string\n\tredisDB int\n\n\tuseRedisCluster bool\n\tredisClusterAddrs string // comma-separated list of host:port\n\n\ttestLogLevel = FatalLevel\n)\n\nvar testLogger *log.Logger\n\nfunc init() {\n\tflag.StringVar(&redisAddr, \"redis_addr\", \"localhost:6379\", \"redis address to use in testing\")\n\tflag.IntVar(&redisDB, \"redis_db\", 14, \"redis db number to use in testing\")\n\tflag.BoolVar(&useRedisCluster, \"redis_cluster\", false, \"use redis cluster as a broker in testing\")\n\tflag.StringVar(&redisClusterAddrs, \"redis_cluster_addrs\", \"localhost:7000,localhost:7001,localhost:7002\", \"comma separated list of redis server addresses\")\n\tflag.Var(&testLogLevel, \"loglevel\", \"log level to use in testing\")\n\n\ttestLogger = log.NewLogger(nil)\n\ttestLogger.SetLevel(toInternalLogLevel(testLogLevel))\n}\n\nfunc setup(tb testing.TB) (r redis.UniversalClient) {\n\ttb.Helper()\n\tif useRedisCluster {\n\t\taddrs := strings.Split(redisClusterAddrs, \",\")\n\t\tif len(addrs) == 0 {\n\t\t\ttb.Fatal(\"No redis cluster addresses provided. Please set addresses using --redis_cluster_addrs flag.\")\n\t\t}\n\t\tr = redis.NewClusterClient(&redis.ClusterOptions{\n\t\t\tAddrs: addrs,\n\t\t})\n\t} else {\n\t\tr = redis.NewClient(&redis.Options{\n\t\t\tAddr: redisAddr,\n\t\t\tDB: redisDB,\n\t\t})\n\t}\n\t// Start each test with a clean slate.\n\th.FlushDB(tb, r)\n\treturn r\n}\n\nfunc getRedisConnOpt(tb testing.TB) RedisConnOpt {\n\ttb.Helper()\n\tif useRedisCluster {\n\t\taddrs := strings.Split(redisClusterAddrs, \",\")\n\t\tif len(addrs) == 0 {\n\t\t\ttb.Fatal(\"No redis cluster addresses provided. Please set addresses using --redis_cluster_addrs flag.\")\n\t\t}\n\t\treturn RedisClusterClientOpt{\n\t\t\tAddrs: addrs,\n\t\t}\n\t}\n\treturn RedisClientOpt{\n\t\tAddr: redisAddr,\n\t\tDB: redisDB,\n\t}\n}\n\nvar sortTaskOpt = cmp.Transformer(\"SortMsg\", func(in []*Task) []*Task {\n\tout := append([]*Task(nil), in...) // Copy input to avoid mutating it\n\tsort.Slice(out, func(i, j int) bool {\n\t\treturn out[i].Type() < out[j].Type()\n\t})\n\treturn out\n})\n\nfunc TestParseRedisURI(t *testing.T) {\n\ttests := []struct {\n\t\turi string\n\t\twant RedisConnOpt\n\t}{\n\t\t{\n\t\t\t\"redis://localhost:6379\",\n\t\t\tRedisClientOpt{Addr: \"localhost:6379\"},\n\t\t},\n\t\t{\n\t\t\t\"rediss://localhost:6379\",\n\t\t\tRedisClientOpt{Addr: \"localhost:6379\", TLSConfig: &tls.Config{ServerName: \"localhost\"}},\n\t\t},\n\t\t{\n\t\t\t\"redis://localhost:6379/3\",\n\t\t\tRedisClientOpt{Addr: \"localhost:6379\", DB: 3},\n\t\t},\n\t\t{\n\t\t\t\"redis://:mypassword@localhost:6379\",\n\t\t\tRedisClientOpt{Addr: \"localhost:6379\", Password: \"mypassword\"},\n\t\t},\n\t\t{\n\t\t\t\"redis://:mypassword@127.0.0.1:6379/11\",\n\t\t\tRedisClientOpt{Addr: \"127.0.0.1:6379\", Password: \"mypassword\", DB: 11},\n\t\t},\n\t\t{\n\t\t\t\"redis-socket:///var/run/redis/redis.sock\",\n\t\t\tRedisClientOpt{Network: \"unix\", Addr: \"/var/run/redis/redis.sock\"},\n\t\t},\n\t\t{\n\t\t\t\"redis-socket://:mypassword@/var/run/redis/redis.sock\",\n\t\t\tRedisClientOpt{Network: \"unix\", Addr: \"/var/run/redis/redis.sock\", Password: \"mypassword\"},\n\t\t},\n\t\t{\n\t\t\t\"redis-socket:///var/run/redis/redis.sock?db=7\",\n\t\t\tRedisClientOpt{Network: \"unix\", Addr: \"/var/run/redis/redis.sock\", DB: 7},\n\t\t},\n\t\t{\n\t\t\t\"redis-socket://:mypassword@/var/run/redis/redis.sock?db=12\",\n\t\t\tRedisClientOpt{Network: \"unix\", Addr: \"/var/run/redis/redis.sock\", Password: \"mypassword\", DB: 12},\n\t\t},\n\t\t{\n\t\t\t\"redis-sentinel://localhost:5000,localhost:5001,localhost:5002?master=mymaster\",\n\t\t\tRedisFailoverClientOpt{\n\t\t\t\tMasterName: \"mymaster\",\n\t\t\t\tSentinelAddrs: []string{\"localhost:5000\", \"localhost:5001\", \"localhost:5002\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t\"redis-sentinel://:mypassword@localhost:5000,localhost:5001,localhost:5002?master=mymaster\",\n\t\t\tRedisFailoverClientOpt{\n\t\t\t\tMasterName: \"mymaster\",\n\t\t\t\tSentinelAddrs: []string{\"localhost:5000\", \"localhost:5001\", \"localhost:5002\"},\n\t\t\t\tSentinelPassword: \"mypassword\",\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot, err := ParseRedisURI(tc.uri)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"ParseRedisURI(%q) returned an error: %v\", tc.uri, err)\n\t\t\tcontinue\n\t\t}\n\n\t\tif diff := cmp.Diff(tc.want, got, cmpopts.IgnoreUnexported(tls.Config{})); diff != \"\" {\n\t\t\tt.Errorf(\"ParseRedisURI(%q) = %+v, want %+v\\n(-want,+got)\\n%s\", tc.uri, got, tc.want, diff)\n\t\t}\n\t}\n}\n\nfunc TestParseRedisURIErrors(t *testing.T) {\n\ttests := []struct {\n\t\tdesc string\n\t\turi string\n\t}{\n\t\t{\n\t\t\t\"unsupported scheme\",\n\t\t\t\"rdb://localhost:6379\",\n\t\t},\n\t\t{\n\t\t\t\"missing scheme\",\n\t\t\t\"localhost:6379\",\n\t\t},\n\t\t{\n\t\t\t\"multiple db numbers\",\n\t\t\t\"redis://localhost:6379/1,2,3\",\n\t\t},\n\t\t{\n\t\t\t\"missing path for socket connection\",\n\t\t\t\"redis-socket://?db=one\",\n\t\t},\n\t\t{\n\t\t\t\"non integer for db numbers for socket\",\n\t\t\t\"redis-socket:///some/path/to/redis?db=one\",\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\t_, err := ParseRedisURI(tc.uri)\n\t\tif err == nil {\n\t\t\tt.Errorf(\"%s: ParseRedisURI(%q) succeeded for malformed input, want error\",\n\t\t\t\ttc.desc, tc.uri)\n\t\t}\n\t}\n}\n" }, { "path": "benchmark_test.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\th \"github.com/hibiken/asynq/internal/testutil\"\n)\n\n// Creates a new task of type \"task\" with payload {\"data\": n}.\nfunc makeTask(n int) *Task {\n\tb, err := json.Marshal(map[string]int{\"data\": n})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\treturn NewTask(fmt.Sprintf(\"task%d\", n), b)\n}\n\n// Simple E2E Benchmark testing with no scheduled tasks and retries.\nfunc BenchmarkEndToEndSimple(b *testing.B) {\n\tconst count = 100000\n\tfor n := 0; n < b.N; n++ {\n\t\tb.StopTimer() // begin setup\n\t\tsetup(b)\n\t\tredis := getRedisConnOpt(b)\n\t\tclient := NewClient(redis)\n\t\tsrv := NewServer(redis, Config{\n\t\t\tConcurrency: 10,\n\t\t\tRetryDelayFunc: func(n int, err error, t *Task) time.Duration {\n\t\t\t\treturn time.Second\n\t\t\t},\n\t\t\tLogLevel: testLogLevel,\n\t\t})\n\t\t// Create a bunch of tasks\n\t\tfor i := 0; i < count; i++ {\n\t\t\tif _, err := client.Enqueue(makeTask(i)); err != nil {\n\t\t\t\tb.Fatalf(\"could not enqueue a task: %v\", err)\n\t\t\t}\n\t\t}\n\t\tclient.Close()\n\n\t\tvar wg sync.WaitGroup\n\t\twg.Add(count)\n\t\thandler := func(ctx context.Context, t *Task) error {\n\t\t\twg.Done()\n\t\t\treturn nil\n\t\t}\n\t\tb.StartTimer() // end setup\n\n\t\t_ = srv.Start(HandlerFunc(handler))\n\t\twg.Wait()\n\n\t\tb.StopTimer() // begin teardown\n\t\tsrv.Stop()\n\t\tb.StartTimer() // end teardown\n\t}\n}\n\n// E2E benchmark with scheduled tasks and retries.\nfunc BenchmarkEndToEnd(b *testing.B) {\n\tconst count = 100000\n\tfor n := 0; n < b.N; n++ {\n\t\tb.StopTimer() // begin setup\n\t\tsetup(b)\n\t\tredis := getRedisConnOpt(b)\n\t\tclient := NewClient(redis)\n\t\tsrv := NewServer(redis, Config{\n\t\t\tConcurrency: 10,\n\t\t\tRetryDelayFunc: func(n int, err error, t *Task) time.Duration {\n\t\t\t\treturn time.Second\n\t\t\t},\n\t\t\tLogLevel: testLogLevel,\n\t\t})\n\t\t// Create a bunch of tasks\n\t\tfor i := 0; i < count; i++ {\n\t\t\tif _, err := client.Enqueue(makeTask(i)); err != nil {\n\t\t\t\tb.Fatalf(\"could not enqueue a task: %v\", err)\n\t\t\t}\n\t\t}\n\t\tfor i := 0; i < count; i++ {\n\t\t\tif _, err := client.Enqueue(makeTask(i), ProcessIn(1*time.Second)); err != nil {\n\t\t\t\tb.Fatalf(\"could not enqueue a task: %v\", err)\n\t\t\t}\n\t\t}\n\t\tclient.Close()\n\n\t\tvar wg sync.WaitGroup\n\t\twg.Add(count * 2)\n\t\thandler := func(ctx context.Context, t *Task) error {\n\t\t\tvar p map[string]int\n\t\t\tif err := json.Unmarshal(t.Payload(), &p); err != nil {\n\t\t\t\tb.Logf(\"internal error: %v\", err)\n\t\t\t}\n\t\t\tn, ok := p[\"data\"]\n\t\t\tif !ok {\n\t\t\t\tn = 1\n\t\t\t\tb.Logf(\"internal error: could not get data from payload\")\n\t\t\t}\n\t\t\tretried, ok := GetRetryCount(ctx)\n\t\t\tif !ok {\n\t\t\t\tb.Logf(\"internal error: could not get retry count from context\")\n\t\t\t}\n\t\t\t// Fail 1% of tasks for the first attempt.\n\t\t\tif retried == 0 && n%100 == 0 {\n\t\t\t\treturn fmt.Errorf(\":(\")\n\t\t\t}\n\t\t\twg.Done()\n\t\t\treturn nil\n\t\t}\n\t\tb.StartTimer() // end setup\n\n\t\t_ = srv.Start(HandlerFunc(handler))\n\t\twg.Wait()\n\n\t\tb.StopTimer() // begin teardown\n\t\tsrv.Stop()\n\t\tb.StartTimer() // end teardown\n\t}\n}\n\n// Simple E2E Benchmark testing with no scheduled tasks and retries with multiple queues.\nfunc BenchmarkEndToEndMultipleQueues(b *testing.B) {\n\t// number of tasks to create for each queue\n\tconst (\n\t\thighCount = 20000\n\t\tdefaultCount = 20000\n\t\tlowCount = 20000\n\t)\n\tfor n := 0; n < b.N; n++ {\n\t\tb.StopTimer() // begin setup\n\t\tsetup(b)\n\t\tredis := getRedisConnOpt(b)\n\t\tclient := NewClient(redis)\n\t\tsrv := NewServer(redis, Config{\n\t\t\tConcurrency: 10,\n\t\t\tQueues: map[string]int{\n\t\t\t\t\"high\": 6,\n\t\t\t\t\"default\": 3,\n\t\t\t\t\"low\": 1,\n\t\t\t},\n\t\t\tLogLevel: testLogLevel,\n\t\t})\n\t\t// Create a bunch of tasks\n\t\tfor i := 0; i < highCount; i++ {\n\t\t\tif _, err := client.Enqueue(makeTask(i), Queue(\"high\")); err != nil {\n\t\t\t\tb.Fatalf(\"could not enqueue a task: %v\", err)\n\t\t\t}\n\t\t}\n\t\tfor i := 0; i < defaultCount; i++ {\n\t\t\tif _, err := client.Enqueue(makeTask(i)); err != nil {\n\t\t\t\tb.Fatalf(\"could not enqueue a task: %v\", err)\n\t\t\t}\n\t\t}\n\t\tfor i := 0; i < lowCount; i++ {\n\t\t\tif _, err := client.Enqueue(makeTask(i), Queue(\"low\")); err != nil {\n\t\t\t\tb.Fatalf(\"could not enqueue a task: %v\", err)\n\t\t\t}\n\t\t}\n\t\tclient.Close()\n\n\t\tvar wg sync.WaitGroup\n\t\twg.Add(highCount + defaultCount + lowCount)\n\t\thandler := func(ctx context.Context, t *Task) error {\n\t\t\twg.Done()\n\t\t\treturn nil\n\t\t}\n\t\tb.StartTimer() // end setup\n\n\t\t_ = srv.Start(HandlerFunc(handler))\n\t\twg.Wait()\n\n\t\tb.StopTimer() // begin teardown\n\t\tsrv.Stop()\n\t\tb.StartTimer() // end teardown\n\t}\n}\n\n// E2E benchmark to check client enqueue operation performs correctly,\n// while server is busy processing tasks.\nfunc BenchmarkClientWhileServerRunning(b *testing.B) {\n\tconst count = 10000\n\tfor n := 0; n < b.N; n++ {\n\t\tb.StopTimer() // begin setup\n\t\tsetup(b)\n\t\tredis := getRedisConnOpt(b)\n\t\tclient := NewClient(redis)\n\t\tsrv := NewServer(redis, Config{\n\t\t\tConcurrency: 10,\n\t\t\tRetryDelayFunc: func(n int, err error, t *Task) time.Duration {\n\t\t\t\treturn time.Second\n\t\t\t},\n\t\t\tLogLevel: testLogLevel,\n\t\t})\n\t\t// Enqueue 10,000 tasks.\n\t\tfor i := 0; i < count; i++ {\n\t\t\tif _, err := client.Enqueue(makeTask(i)); err != nil {\n\t\t\t\tb.Fatalf(\"could not enqueue a task: %v\", err)\n\t\t\t}\n\t\t}\n\t\t// Schedule 10,000 tasks.\n\t\tfor i := 0; i < count; i++ {\n\t\t\tif _, err := client.Enqueue(makeTask(i), ProcessIn(1*time.Second)); err != nil {\n\t\t\t\tb.Fatalf(\"could not enqueue a task: %v\", err)\n\t\t\t}\n\t\t}\n\n\t\thandler := func(ctx context.Context, t *Task) error {\n\t\t\treturn nil\n\t\t}\n\t\t_ = srv.Start(HandlerFunc(handler))\n\n\t\tb.StartTimer() // end setup\n\n\t\tb.Log(\"Starting enqueueing\")\n\t\tenqueued := 0\n\t\tfor enqueued < 100000 {\n\t\t\tt := NewTask(fmt.Sprintf(\"enqueued%d\", enqueued), h.JSON(map[string]interface{}{\"data\": enqueued}))\n\t\t\tif _, err := client.Enqueue(t); err != nil {\n\t\t\t\tb.Logf(\"could not enqueue task %d: %v\", enqueued, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tenqueued++\n\t\t}\n\t\tb.Logf(\"Finished enqueueing %d tasks\", enqueued)\n\n\t\tb.StopTimer() // begin teardown\n\t\tsrv.Stop()\n\t\tclient.Close()\n\t\tb.StartTimer() // end teardown\n\t}\n}\n" }, { "path": "client.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/google/uuid\"\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/errors\"\n\t\"github.com/hibiken/asynq/internal/rdb\"\n\t\"github.com/redis/go-redis/v9\"\n)\n\n// A Client is responsible for scheduling tasks.\n//\n// A Client is used to register tasks that should be processed\n// immediately or some time in the future.\n//\n// Clients are safe for concurrent use by multiple goroutines.\ntype Client struct {\n\tbroker base.Broker\n\t// When a Client has been created with an existing Redis connection, we do\n\t// not want to close it.\n\tsharedConnection bool\n}\n\n// NewClient returns a new Client instance given a redis connection option.\nfunc NewClient(r RedisConnOpt) *Client {\n\tredisClient, ok := r.MakeRedisClient().(redis.UniversalClient)\n\tif !ok {\n\t\tpanic(fmt.Sprintf(\"asynq: unsupported RedisConnOpt type %T\", r))\n\t}\n\tclient := NewClientFromRedisClient(redisClient)\n\tclient.sharedConnection = false\n\treturn client\n}\n\n// NewClientFromRedisClient returns a new instance of Client given a redis.UniversalClient\n// Warning: The underlying redis connection pool will not be closed by Asynq, you are responsible for closing it.\nfunc NewClientFromRedisClient(c redis.UniversalClient) *Client {\n\treturn &Client{broker: rdb.NewRDB(c), sharedConnection: true}\n}\n\ntype OptionType int\n\nconst (\n\tMaxRetryOpt OptionType = iota\n\tQueueOpt\n\tTimeoutOpt\n\tDeadlineOpt\n\tUniqueOpt\n\tProcessAtOpt\n\tProcessInOpt\n\tTaskIDOpt\n\tRetentionOpt\n\tGroupOpt\n)\n\n// Option specifies the task processing behavior.\ntype Option interface {\n\t// String returns a string representation of the option.\n\tString() string\n\n\t// Type describes the type of the option.\n\tType() OptionType\n\n\t// Value returns a value used to create this option.\n\tValue() interface{}\n}\n\n// Internal option representations.\ntype (\n\tretryOption int\n\tqueueOption string\n\ttaskIDOption string\n\ttimeoutOption time.Duration\n\tdeadlineOption time.Time\n\tuniqueOption time.Duration\n\tprocessAtOption time.Time\n\tprocessInOption time.Duration\n\tretentionOption time.Duration\n\tgroupOption string\n)\n\n// MaxRetry returns an option to specify the max number of times\n// the task will be retried.\n//\n// Negative retry count is treated as zero retry.\nfunc MaxRetry(n int) Option {\n\tif n < 0 {\n\t\tn = 0\n\t}\n\treturn retryOption(n)\n}\n\nfunc (n retryOption) String() string { return fmt.Sprintf(\"MaxRetry(%d)\", int(n)) }\nfunc (n retryOption) Type() OptionType { return MaxRetryOpt }\nfunc (n retryOption) Value() interface{} { return int(n) }\n\n// Queue returns an option to specify the queue to enqueue the task into.\nfunc Queue(name string) Option {\n\treturn queueOption(name)\n}\n\nfunc (name queueOption) String() string { return fmt.Sprintf(\"Queue(%q)\", string(name)) }\nfunc (name queueOption) Type() OptionType { return QueueOpt }\nfunc (name queueOption) Value() interface{} { return string(name) }\n\n// TaskID returns an option to specify the task ID.\nfunc TaskID(id string) Option {\n\treturn taskIDOption(id)\n}\n\nfunc (id taskIDOption) String() string { return fmt.Sprintf(\"TaskID(%q)\", string(id)) }\nfunc (id taskIDOption) Type() OptionType { return TaskIDOpt }\nfunc (id taskIDOption) Value() interface{} { return string(id) }\n\n// Timeout returns an option to specify how long a task may run.\n// If the timeout elapses before the Handler returns, then the task\n// will be retried.\n//\n// Zero duration means no limit.\n//\n// If there's a conflicting Deadline option, whichever comes earliest\n// will be used.\nfunc Timeout(d time.Duration) Option {\n\treturn timeoutOption(d)\n}\n\nfunc (d timeoutOption) String() string { return fmt.Sprintf(\"Timeout(%v)\", time.Duration(d)) }\nfunc (d timeoutOption) Type() OptionType { return TimeoutOpt }\nfunc (d timeoutOption) Value() interface{} { return time.Duration(d) }\n\n// Deadline returns an option to specify the deadline for the given task.\n// If it reaches the deadline before the Handler returns, then the task\n// will be retried.\n//\n// If there's a conflicting Timeout option, whichever comes earliest\n// will be used.\nfunc Deadline(t time.Time) Option {\n\treturn deadlineOption(t)\n}\n\nfunc (t deadlineOption) String() string {\n\treturn fmt.Sprintf(\"Deadline(%v)\", time.Time(t).Format(time.UnixDate))\n}\nfunc (t deadlineOption) Type() OptionType { return DeadlineOpt }\nfunc (t deadlineOption) Value() interface{} { return time.Time(t) }\n\n// Unique returns an option to enqueue a task only if the given task is unique.\n// Task enqueued with this option is guaranteed to be unique within the given ttl.\n// Once the task gets processed successfully or once the TTL has expired,\n// another task with the same uniqueness may be enqueued.\n// ErrDuplicateTask error is returned when enqueueing a duplicate task.\n// TTL duration must be greater than or equal to 1 second.\n//\n// Uniqueness of a task is based on the following properties:\n// - Task Type\n// - Task Payload\n// - Queue Name\nfunc Unique(ttl time.Duration) Option {\n\treturn uniqueOption(ttl)\n}\n\nfunc (ttl uniqueOption) String() string { return fmt.Sprintf(\"Unique(%v)\", time.Duration(ttl)) }\nfunc (ttl uniqueOption) Type() OptionType { return UniqueOpt }\nfunc (ttl uniqueOption) Value() interface{} { return time.Duration(ttl) }\n\n// ProcessAt returns an option to specify when to process the given task.\n//\n// If there's a conflicting ProcessIn option, the last option passed to Enqueue overrides the others.\nfunc ProcessAt(t time.Time) Option {\n\treturn processAtOption(t)\n}\n\nfunc (t processAtOption) String() string {\n\treturn fmt.Sprintf(\"ProcessAt(%v)\", time.Time(t).Format(time.UnixDate))\n}\nfunc (t processAtOption) Type() OptionType { return ProcessAtOpt }\nfunc (t processAtOption) Value() interface{} { return time.Time(t) }\n\n// ProcessIn returns an option to specify when to process the given task relative to the current time.\n//\n// If there's a conflicting ProcessAt option, the last option passed to Enqueue overrides the others.\nfunc ProcessIn(d time.Duration) Option {\n\treturn processInOption(d)\n}\n\nfunc (d processInOption) String() string { return fmt.Sprintf(\"ProcessIn(%v)\", time.Duration(d)) }\nfunc (d processInOption) Type() OptionType { return ProcessInOpt }\nfunc (d processInOption) Value() interface{} { return time.Duration(d) }\n\n// Retention returns an option to specify the duration of retention period for the task.\n// If this option is provided, the task will be stored as a completed task after successful processing.\n// A completed task will be deleted after the specified duration elapses.\nfunc Retention(d time.Duration) Option {\n\treturn retentionOption(d)\n}\n\nfunc (ttl retentionOption) String() string { return fmt.Sprintf(\"Retention(%v)\", time.Duration(ttl)) }\nfunc (ttl retentionOption) Type() OptionType { return RetentionOpt }\nfunc (ttl retentionOption) Value() interface{} { return time.Duration(ttl) }\n\n// Group returns an option to specify the group used for the task.\n// Tasks in a given queue with the same group will be aggregated into one task before passed to Handler.\nfunc Group(name string) Option {\n\treturn groupOption(name)\n}\n\nfunc (name groupOption) String() string { return fmt.Sprintf(\"Group(%q)\", string(name)) }\nfunc (name groupOption) Type() OptionType { return GroupOpt }\nfunc (name groupOption) Value() interface{} { return string(name) }\n\n// ErrDuplicateTask indicates that the given task could not be enqueued since it's a duplicate of another task.\n//\n// ErrDuplicateTask error only applies to tasks enqueued with a Unique option.\nvar ErrDuplicateTask = errors.New(\"task already exists\")\n\n// ErrTaskIDConflict indicates that the given task could not be enqueued since its task ID already exists.\n//\n// ErrTaskIDConflict error only applies to tasks enqueued with a TaskID option.\nvar ErrTaskIDConflict = errors.New(\"task ID conflicts with another task\")\n\ntype option struct {\n\tretry int\n\tqueue string\n\ttaskID string\n\ttimeout time.Duration\n\tdeadline time.Time\n\tuniqueTTL time.Duration\n\tprocessAt time.Time\n\tretention time.Duration\n\tgroup string\n}\n\n// composeOptions merges user provided options into the default options\n// and returns the composed option.\n// It also validates the user provided options and returns an error if any of\n// the user provided options fail the validations.\nfunc composeOptions(opts ...Option) (option, error) {\n\tres := option{\n\t\tretry: defaultMaxRetry,\n\t\tqueue: base.DefaultQueueName,\n\t\ttaskID: uuid.NewString(),\n\t\ttimeout: 0, // do not set to defaultTimeout here\n\t\tdeadline: time.Time{},\n\t\tprocessAt: time.Now(),\n\t}\n\tfor _, opt := range opts {\n\t\tswitch opt := opt.(type) {\n\t\tcase retryOption:\n\t\t\tres.retry = int(opt)\n\t\tcase queueOption:\n\t\t\tqname := string(opt)\n\t\t\tif err := base.ValidateQueueName(qname); err != nil {\n\t\t\t\treturn option{}, err\n\t\t\t}\n\t\t\tres.queue = qname\n\t\tcase taskIDOption:\n\t\t\tid := string(opt)\n\t\t\tif isBlank(id) {\n\t\t\t\treturn option{}, errors.New(\"task ID cannot be empty\")\n\t\t\t}\n\t\t\tres.taskID = id\n\t\tcase timeoutOption:\n\t\t\tres.timeout = time.Duration(opt)\n\t\tcase deadlineOption:\n\t\t\tres.deadline = time.Time(opt)\n\t\tcase uniqueOption:\n\t\t\tttl := time.Duration(opt)\n\t\t\tif ttl < 1*time.Second {\n\t\t\t\treturn option{}, errors.New(\"Unique TTL cannot be less than 1s\")\n\t\t\t}\n\t\t\tres.uniqueTTL = ttl\n\t\tcase processAtOption:\n\t\t\tres.processAt = time.Time(opt)\n\t\tcase processInOption:\n\t\t\tres.processAt = time.Now().Add(time.Duration(opt))\n\t\tcase retentionOption:\n\t\t\tres.retention = time.Duration(opt)\n\t\tcase groupOption:\n\t\t\tkey := string(opt)\n\t\t\tif isBlank(key) {\n\t\t\t\treturn option{}, errors.New(\"group key cannot be empty\")\n\t\t\t}\n\t\t\tres.group = key\n\t\tdefault:\n\t\t\t// ignore unexpected option\n\t\t}\n\t}\n\treturn res, nil\n}\n\n// isBlank returns true if the given s is empty or consist of all whitespaces.\nfunc isBlank(s string) bool {\n\treturn strings.TrimSpace(s) == \"\"\n}\n\nconst (\n\t// Default max retry count used if nothing is specified.\n\tdefaultMaxRetry = 25\n\n\t// Default timeout used if both timeout and deadline are not specified.\n\tdefaultTimeout = 30 * time.Minute\n)\n\n// Value zero indicates no timeout and no deadline.\nvar (\n\tnoTimeout time.Duration = 0\n\tnoDeadline time.Time = time.Unix(0, 0)\n)\n\n// Close closes the connection with redis.\nfunc (c *Client) Close() error {\n\tif c.sharedConnection {\n\t\treturn fmt.Errorf(\"redis connection is shared so the Client can't be closed through asynq\")\n\t}\n\treturn c.broker.Close()\n}\n\n// Enqueue enqueues the given task to a queue.\n//\n// Enqueue returns TaskInfo and nil error if the task is enqueued successfully, otherwise returns a non-nil error.\n//\n// The argument opts specifies the behavior of task processing.\n// If there are conflicting Option values the last one overrides others.\n// Any options provided to NewTask can be overridden by options passed to Enqueue.\n// By default, max retry is set to 25 and timeout is set to 30 minutes.\n//\n// If no ProcessAt or ProcessIn options are provided, the task will be pending immediately.\n//\n// Enqueue uses context.Background internally; to specify the context, use EnqueueContext.\nfunc (c *Client) Enqueue(task *Task, opts ...Option) (*TaskInfo, error) {\n\treturn c.EnqueueContext(context.Background(), task, opts...)\n}\n\n// EnqueueContext enqueues the given task to a queue.\n//\n// EnqueueContext returns TaskInfo and nil error if the task is enqueued successfully, otherwise returns a non-nil error.\n//\n// The argument opts specifies the behavior of task processing.\n// If there are conflicting Option values the last one overrides others.\n// Any options provided to NewTask can be overridden by options passed to Enqueue.\n// By default, max retry is set to 25 and timeout is set to 30 minutes.\n//\n// If no ProcessAt or ProcessIn options are provided, the task will be pending immediately.\n//\n// The first argument context applies to the enqueue operation. To specify task timeout and deadline, use Timeout and Deadline option instead.\nfunc (c *Client) EnqueueContext(ctx context.Context, task *Task, opts ...Option) (*TaskInfo, error) {\n\tif task == nil {\n\t\treturn nil, fmt.Errorf(\"task cannot be nil\")\n\t}\n\tif strings.TrimSpace(task.Type()) == \"\" {\n\t\treturn nil, fmt.Errorf(\"task typename cannot be empty\")\n\t}\n\t// merge task options with the options provided at enqueue time.\n\topts = append(task.opts, opts...)\n\topt, err := composeOptions(opts...)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdeadline := noDeadline\n\tif !opt.deadline.IsZero() {\n\t\tdeadline = opt.deadline\n\t}\n\ttimeout := noTimeout\n\tif opt.timeout != 0 {\n\t\ttimeout = opt.timeout\n\t}\n\tif deadline.Equal(noDeadline) && timeout == noTimeout {\n\t\t// If neither deadline nor timeout are set, use default timeout.\n\t\ttimeout = defaultTimeout\n\t}\n\tvar uniqueKey string\n\tif opt.uniqueTTL > 0 {\n\t\tuniqueKey = base.UniqueKey(opt.queue, task.Type(), task.Payload())\n\t}\n\tmsg := &base.TaskMessage{\n\t\tID: opt.taskID,\n\t\tType: task.Type(),\n\t\tPayload: task.Payload(),\n\t\tHeaders: task.Headers(),\n\t\tQueue: opt.queue,\n\t\tRetry: opt.retry,\n\t\tDeadline: deadline.Unix(),\n\t\tTimeout: int64(timeout.Seconds()),\n\t\tUniqueKey: uniqueKey,\n\t\tGroupKey: opt.group,\n\t\tRetention: int64(opt.retention.Seconds()),\n\t}\n\tnow := time.Now()\n\tvar state base.TaskState\n\tif opt.processAt.After(now) {\n\t\terr = c.schedule(ctx, msg, opt.processAt, opt.uniqueTTL)\n\t\tstate = base.TaskStateScheduled\n\t} else if opt.group != \"\" {\n\t\t// Use zero value for processAt since we don't know when the task will be aggregated and processed.\n\t\topt.processAt = time.Time{}\n\t\terr = c.addToGroup(ctx, msg, opt.group, opt.uniqueTTL)\n\t\tstate = base.TaskStateAggregating\n\t} else {\n\t\topt.processAt = now\n\t\terr = c.enqueue(ctx, msg, opt.uniqueTTL)\n\t\tstate = base.TaskStatePending\n\t}\n\tswitch {\n\tcase errors.Is(err, errors.ErrDuplicateTask):\n\t\treturn nil, fmt.Errorf(\"%w\", ErrDuplicateTask)\n\tcase errors.Is(err, errors.ErrTaskIdConflict):\n\t\treturn nil, fmt.Errorf(\"%w\", ErrTaskIDConflict)\n\tcase err != nil:\n\t\treturn nil, err\n\t}\n\treturn newTaskInfo(msg, state, opt.processAt, nil), nil\n}\n\n// Ping performs a ping against the redis connection.\nfunc (c *Client) Ping() error {\n\treturn c.broker.Ping()\n}\n\nfunc (c *Client) enqueue(ctx context.Context, msg *base.TaskMessage, uniqueTTL time.Duration) error {\n\tif uniqueTTL > 0 {\n\t\treturn c.broker.EnqueueUnique(ctx, msg, uniqueTTL)\n\t}\n\treturn c.broker.Enqueue(ctx, msg)\n}\n\nfunc (c *Client) schedule(ctx context.Context, msg *base.TaskMessage, t time.Time, uniqueTTL time.Duration) error {\n\tif uniqueTTL > 0 {\n\t\tttl := time.Until(t.Add(uniqueTTL))\n\t\treturn c.broker.ScheduleUnique(ctx, msg, t, ttl)\n\t}\n\treturn c.broker.Schedule(ctx, msg, t)\n}\n\nfunc (c *Client) addToGroup(ctx context.Context, msg *base.TaskMessage, group string, uniqueTTL time.Duration) error {\n\tif uniqueTTL > 0 {\n\t\treturn c.broker.AddToGroupUnique(ctx, msg, group, uniqueTTL)\n\t}\n\treturn c.broker.AddToGroup(ctx, msg, group)\n}\n" }, { "path": "client_test.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/google/go-cmp/cmp\"\n\t\"github.com/google/go-cmp/cmp/cmpopts\"\n\t\"github.com/hibiken/asynq/internal/base\"\n\th \"github.com/hibiken/asynq/internal/testutil\"\n\t\"github.com/redis/go-redis/v9\"\n)\n\nfunc TestClientEnqueueWithProcessAtOption(t *testing.T) {\n\tr := setup(t)\n\tclient := NewClient(getRedisConnOpt(t))\n\tdefer client.Close()\n\n\ttask := NewTask(\"send_email\", h.JSON(map[string]interface{}{\"to\": \"customer@gmail.com\", \"from\": \"merchant@example.com\"}))\n\n\tvar (\n\t\tnow = time.Now()\n\t\toneHourLater = now.Add(time.Hour)\n\t)\n\n\ttests := []struct {\n\t\tdesc string\n\t\ttask *Task\n\t\tprocessAt time.Time // value for ProcessAt option\n\t\topts []Option // other options\n\t\twantInfo *TaskInfo\n\t\twantPending map[string][]*base.TaskMessage\n\t\twantScheduled map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tdesc: \"Process task immediately\",\n\t\t\ttask: task,\n\t\t\tprocessAt: now,\n\t\t\topts: []Option{},\n\t\t\twantInfo: &TaskInfo{\n\t\t\t\tQueue: \"default\",\n\t\t\t\tType: task.Type(),\n\t\t\t\tPayload: task.Payload(),\n\t\t\t\tState: TaskStatePending,\n\t\t\t\tMaxRetry: defaultMaxRetry,\n\t\t\t\tRetried: 0,\n\t\t\t\tLastErr: \"\",\n\t\t\t\tLastFailedAt: time.Time{},\n\t\t\t\tTimeout: defaultTimeout,\n\t\t\t\tDeadline: time.Time{},\n\t\t\t\tNextProcessAt: now,\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{\n\t\t\t\t\t\tType: task.Type(),\n\t\t\t\t\t\tPayload: task.Payload(),\n\t\t\t\t\t\tRetry: defaultMaxRetry,\n\t\t\t\t\t\tQueue: \"default\",\n\t\t\t\t\t\tTimeout: int64(defaultTimeout.Seconds()),\n\t\t\t\t\t\tDeadline: noDeadline.Unix(),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"Schedule task to be processed in the future\",\n\t\t\ttask: task,\n\t\t\tprocessAt: oneHourLater,\n\t\t\topts: []Option{},\n\t\t\twantInfo: &TaskInfo{\n\t\t\t\tQueue: \"default\",\n\t\t\t\tType: task.Type(),\n\t\t\t\tPayload: task.Payload(),\n\t\t\t\tState: TaskStateScheduled,\n\t\t\t\tMaxRetry: defaultMaxRetry,\n\t\t\t\tRetried: 0,\n\t\t\t\tLastErr: \"\",\n\t\t\t\tLastFailedAt: time.Time{},\n\t\t\t\tTimeout: defaultTimeout,\n\t\t\t\tDeadline: time.Time{},\n\t\t\t\tNextProcessAt: oneHourLater,\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{\n\t\t\t\t\t\tMessage: &base.TaskMessage{\n\t\t\t\t\t\t\tType: task.Type(),\n\t\t\t\t\t\t\tPayload: task.Payload(),\n\t\t\t\t\t\t\tRetry: defaultMaxRetry,\n\t\t\t\t\t\t\tQueue: \"default\",\n\t\t\t\t\t\t\tTimeout: int64(defaultTimeout.Seconds()),\n\t\t\t\t\t\t\tDeadline: noDeadline.Unix(),\n\t\t\t\t\t\t},\n\t\t\t\t\t\tScore: oneHourLater.Unix(),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r) // clean up db before each test case.\n\n\t\topts := append(tc.opts, ProcessAt(tc.processAt))\n\t\tgotInfo, err := client.Enqueue(tc.task, opts...)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t\tcontinue\n\t\t}\n\t\tcmpOptions := []cmp.Option{\n\t\t\tcmpopts.IgnoreFields(TaskInfo{}, \"ID\"),\n\t\t\tcmpopts.EquateApproxTime(500 * time.Millisecond),\n\t\t}\n\t\tif diff := cmp.Diff(tc.wantInfo, gotInfo, cmpOptions...); diff != \"\" {\n\t\t\tt.Errorf(\"%s;\\nEnqueue(task, ProcessAt(%v)) returned %v, want %v; (-want,+got)\\n%s\",\n\t\t\t\ttc.desc, tc.processAt, gotInfo, tc.wantInfo, diff)\n\t\t}\n\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.IgnoreIDOpt, cmpopts.EquateEmpty()); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s;\\nmismatch found in %q; (-want,+got)\\n%s\", tc.desc, base.PendingKey(qname), diff)\n\t\t\t}\n\t\t}\n\t\tfor qname, want := range tc.wantScheduled {\n\t\t\tgotScheduled := h.GetScheduledEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotScheduled, h.IgnoreIDOpt, cmpopts.EquateEmpty()); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s;\\nmismatch found in %q; (-want,+got)\\n%s\", tc.desc, base.ScheduledKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc testClientEnqueue(t *testing.T, client *Client, r redis.UniversalClient) {\n\ttask := NewTask(\"send_email\", h.JSON(map[string]interface{}{\"to\": \"customer@gmail.com\", \"from\": \"merchant@example.com\"}))\n\tnow := time.Now()\n\n\ttests := []struct {\n\t\tdesc string\n\t\ttask *Task\n\t\topts []Option\n\t\twantInfo *TaskInfo\n\t\twantPending map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tdesc: \"Process task immediately with a custom retry count\",\n\t\t\ttask: task,\n\t\t\topts: []Option{\n\t\t\t\tMaxRetry(3),\n\t\t\t},\n\t\t\twantInfo: &TaskInfo{\n\t\t\t\tQueue: \"default\",\n\t\t\t\tType: task.Type(),\n\t\t\t\tPayload: task.Payload(),\n\t\t\t\tState: TaskStatePending,\n\t\t\t\tMaxRetry: 3,\n\t\t\t\tRetried: 0,\n\t\t\t\tLastErr: \"\",\n\t\t\t\tLastFailedAt: time.Time{},\n\t\t\t\tTimeout: defaultTimeout,\n\t\t\t\tDeadline: time.Time{},\n\t\t\t\tNextProcessAt: now,\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{\n\t\t\t\t\t\tType: task.Type(),\n\t\t\t\t\t\tPayload: task.Payload(),\n\t\t\t\t\t\tRetry: 3,\n\t\t\t\t\t\tQueue: \"default\",\n\t\t\t\t\t\tTimeout: int64(defaultTimeout.Seconds()),\n\t\t\t\t\t\tDeadline: noDeadline.Unix(),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"Negative retry count\",\n\t\t\ttask: task,\n\t\t\topts: []Option{\n\t\t\t\tMaxRetry(-2),\n\t\t\t},\n\t\t\twantInfo: &TaskInfo{\n\t\t\t\tQueue: \"default\",\n\t\t\t\tType: task.Type(),\n\t\t\t\tPayload: task.Payload(),\n\t\t\t\tState: TaskStatePending,\n\t\t\t\tMaxRetry: 0, // Retry count should be set to zero\n\t\t\t\tRetried: 0,\n\t\t\t\tLastErr: \"\",\n\t\t\t\tLastFailedAt: time.Time{},\n\t\t\t\tTimeout: defaultTimeout,\n\t\t\t\tDeadline: time.Time{},\n\t\t\t\tNextProcessAt: now,\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{\n\t\t\t\t\t\tType: task.Type(),\n\t\t\t\t\t\tPayload: task.Payload(),\n\t\t\t\t\t\tRetry: 0, // Retry count should be set to zero\n\t\t\t\t\t\tQueue: \"default\",\n\t\t\t\t\t\tTimeout: int64(defaultTimeout.Seconds()),\n\t\t\t\t\t\tDeadline: noDeadline.Unix(),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"Conflicting options\",\n\t\t\ttask: task,\n\t\t\topts: []Option{\n\t\t\t\tMaxRetry(2),\n\t\t\t\tMaxRetry(10),\n\t\t\t},\n\t\t\twantInfo: &TaskInfo{\n\t\t\t\tQueue: \"default\",\n\t\t\t\tType: task.Type(),\n\t\t\t\tPayload: task.Payload(),\n\t\t\t\tState: TaskStatePending,\n\t\t\t\tMaxRetry: 10, // Last option takes precedence\n\t\t\t\tRetried: 0,\n\t\t\t\tLastErr: \"\",\n\t\t\t\tLastFailedAt: time.Time{},\n\t\t\t\tTimeout: defaultTimeout,\n\t\t\t\tDeadline: time.Time{},\n\t\t\t\tNextProcessAt: now,\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{\n\t\t\t\t\t\tType: task.Type(),\n\t\t\t\t\t\tPayload: task.Payload(),\n\t\t\t\t\t\tRetry: 10, // Last option takes precedence\n\t\t\t\t\t\tQueue: \"default\",\n\t\t\t\t\t\tTimeout: int64(defaultTimeout.Seconds()),\n\t\t\t\t\t\tDeadline: noDeadline.Unix(),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"With queue option\",\n\t\t\ttask: task,\n\t\t\topts: []Option{\n\t\t\t\tQueue(\"custom\"),\n\t\t\t},\n\t\t\twantInfo: &TaskInfo{\n\t\t\t\tQueue: \"custom\",\n\t\t\t\tType: task.Type(),\n\t\t\t\tPayload: task.Payload(),\n\t\t\t\tState: TaskStatePending,\n\t\t\t\tMaxRetry: defaultMaxRetry,\n\t\t\t\tRetried: 0,\n\t\t\t\tLastErr: \"\",\n\t\t\t\tLastFailedAt: time.Time{},\n\t\t\t\tTimeout: defaultTimeout,\n\t\t\t\tDeadline: time.Time{},\n\t\t\t\tNextProcessAt: now,\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{\n\t\t\t\t\t\tType: task.Type(),\n\t\t\t\t\t\tPayload: task.Payload(),\n\t\t\t\t\t\tRetry: defaultMaxRetry,\n\t\t\t\t\t\tQueue: \"custom\",\n\t\t\t\t\t\tTimeout: int64(defaultTimeout.Seconds()),\n\t\t\t\t\t\tDeadline: noDeadline.Unix(),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"Queue option should be case sensitive\",\n\t\t\ttask: task,\n\t\t\topts: []Option{\n\t\t\t\tQueue(\"MyQueue\"),\n\t\t\t},\n\t\t\twantInfo: &TaskInfo{\n\t\t\t\tQueue: \"MyQueue\",\n\t\t\t\tType: task.Type(),\n\t\t\t\tPayload: task.Payload(),\n\t\t\t\tState: TaskStatePending,\n\t\t\t\tMaxRetry: defaultMaxRetry,\n\t\t\t\tRetried: 0,\n\t\t\t\tLastErr: \"\",\n\t\t\t\tLastFailedAt: time.Time{},\n\t\t\t\tTimeout: defaultTimeout,\n\t\t\t\tDeadline: time.Time{},\n\t\t\t\tNextProcessAt: now,\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"MyQueue\": {\n\t\t\t\t\t{\n\t\t\t\t\t\tType: task.Type(),\n\t\t\t\t\t\tPayload: task.Payload(),\n\t\t\t\t\t\tRetry: defaultMaxRetry,\n\t\t\t\t\t\tQueue: \"MyQueue\",\n\t\t\t\t\t\tTimeout: int64(defaultTimeout.Seconds()),\n\t\t\t\t\t\tDeadline: noDeadline.Unix(),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"With timeout option\",\n\t\t\ttask: task,\n\t\t\topts: []Option{\n\t\t\t\tTimeout(20 * time.Second),\n\t\t\t},\n\t\t\twantInfo: &TaskInfo{\n\t\t\t\tQueue: \"default\",\n\t\t\t\tType: task.Type(),\n\t\t\t\tPayload: task.Payload(),\n\t\t\t\tState: TaskStatePending,\n\t\t\t\tMaxRetry: defaultMaxRetry,\n\t\t\t\tRetried: 0,\n\t\t\t\tLastErr: \"\",\n\t\t\t\tLastFailedAt: time.Time{},\n\t\t\t\tTimeout: 20 * time.Second,\n\t\t\t\tDeadline: time.Time{},\n\t\t\t\tNextProcessAt: now,\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{\n\t\t\t\t\t\tType: task.Type(),\n\t\t\t\t\t\tPayload: task.Payload(),\n\t\t\t\t\t\tRetry: defaultMaxRetry,\n\t\t\t\t\t\tQueue: \"default\",\n\t\t\t\t\t\tTimeout: 20,\n\t\t\t\t\t\tDeadline: noDeadline.Unix(),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"With deadline option\",\n\t\t\ttask: task,\n\t\t\topts: []Option{\n\t\t\t\tDeadline(time.Date(2020, time.June, 24, 0, 0, 0, 0, time.UTC)),\n\t\t\t},\n\t\t\twantInfo: &TaskInfo{\n\t\t\t\tQueue: \"default\",\n\t\t\t\tType: task.Type(),\n\t\t\t\tPayload: task.Payload(),\n\t\t\t\tState: TaskStatePending,\n\t\t\t\tMaxRetry: defaultMaxRetry,\n\t\t\t\tRetried: 0,\n\t\t\t\tLastErr: \"\",\n\t\t\t\tLastFailedAt: time.Time{},\n\t\t\t\tTimeout: noTimeout,\n\t\t\t\tDeadline: time.Date(2020, time.June, 24, 0, 0, 0, 0, time.UTC),\n\t\t\t\tNextProcessAt: now,\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{\n\t\t\t\t\t\tType: task.Type(),\n\t\t\t\t\t\tPayload: task.Payload(),\n\t\t\t\t\t\tRetry: defaultMaxRetry,\n\t\t\t\t\t\tQueue: \"default\",\n\t\t\t\t\t\tTimeout: int64(noTimeout.Seconds()),\n\t\t\t\t\t\tDeadline: time.Date(2020, time.June, 24, 0, 0, 0, 0, time.UTC).Unix(),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"With both deadline and timeout options\",\n\t\t\ttask: task,\n\t\t\topts: []Option{\n\t\t\t\tTimeout(20 * time.Second),\n\t\t\t\tDeadline(time.Date(2020, time.June, 24, 0, 0, 0, 0, time.UTC)),\n\t\t\t},\n\t\t\twantInfo: &TaskInfo{\n\t\t\t\tQueue: \"default\",\n\t\t\t\tType: task.Type(),\n\t\t\t\tPayload: task.Payload(),\n\t\t\t\tState: TaskStatePending,\n\t\t\t\tMaxRetry: defaultMaxRetry,\n\t\t\t\tRetried: 0,\n\t\t\t\tLastErr: \"\",\n\t\t\t\tLastFailedAt: time.Time{},\n\t\t\t\tTimeout: 20 * time.Second,\n\t\t\t\tDeadline: time.Date(2020, time.June, 24, 0, 0, 0, 0, time.UTC),\n\t\t\t\tNextProcessAt: now,\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{\n\t\t\t\t\t\tType: task.Type(),\n\t\t\t\t\t\tPayload: task.Payload(),\n\t\t\t\t\t\tRetry: defaultMaxRetry,\n\t\t\t\t\t\tQueue: \"default\",\n\t\t\t\t\t\tTimeout: 20,\n\t\t\t\t\t\tDeadline: time.Date(2020, time.June, 24, 0, 0, 0, 0, time.UTC).Unix(),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"With Retention option\",\n\t\t\ttask: task,\n\t\t\topts: []Option{\n\t\t\t\tRetention(24 * time.Hour),\n\t\t\t},\n\t\t\twantInfo: &TaskInfo{\n\t\t\t\tQueue: \"default\",\n\t\t\t\tType: task.Type(),\n\t\t\t\tPayload: task.Payload(),\n\t\t\t\tState: TaskStatePending,\n\t\t\t\tMaxRetry: defaultMaxRetry,\n\t\t\t\tRetried: 0,\n\t\t\t\tLastErr: \"\",\n\t\t\t\tLastFailedAt: time.Time{},\n\t\t\t\tTimeout: defaultTimeout,\n\t\t\t\tDeadline: time.Time{},\n\t\t\t\tNextProcessAt: now,\n\t\t\t\tRetention: 24 * time.Hour,\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{\n\t\t\t\t\t\tType: task.Type(),\n\t\t\t\t\t\tPayload: task.Payload(),\n\t\t\t\t\t\tRetry: defaultMaxRetry,\n\t\t\t\t\t\tQueue: \"default\",\n\t\t\t\t\t\tTimeout: int64(defaultTimeout.Seconds()),\n\t\t\t\t\t\tDeadline: noDeadline.Unix(),\n\t\t\t\t\t\tRetention: int64((24 * time.Hour).Seconds()),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r) // clean up db before each test case.\n\n\t\tgotInfo, err := client.Enqueue(tc.task, tc.opts...)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t\tcontinue\n\t\t}\n\t\tcmpOptions := []cmp.Option{\n\t\t\tcmpopts.IgnoreFields(TaskInfo{}, \"ID\"),\n\t\t\tcmpopts.EquateApproxTime(500 * time.Millisecond),\n\t\t}\n\t\tif diff := cmp.Diff(tc.wantInfo, gotInfo, cmpOptions...); diff != \"\" {\n\t\t\tt.Errorf(\"%s;\\nEnqueue(task) returned %v, want %v; (-want,+got)\\n%s\",\n\t\t\t\ttc.desc, gotInfo, tc.wantInfo, diff)\n\t\t}\n\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgot := h.GetPendingMessages(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, got, h.IgnoreIDOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s;\\nmismatch found in %q; (-want,+got)\\n%s\", tc.desc, base.PendingKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestClientEnqueue(t *testing.T) {\n\tr := setup(t)\n\tclient := NewClient(getRedisConnOpt(t))\n\tdefer client.Close()\n\ttestClientEnqueue(t, client, r)\n}\n\nfunc TestClientFromRedisClientEnqueue(t *testing.T) {\n\tr := setup(t)\n\tredisClient := getRedisConnOpt(t).MakeRedisClient().(redis.UniversalClient)\n\tclient := NewClientFromRedisClient(redisClient)\n\ttestClientEnqueue(t, client, r)\n\terr := client.Close()\n\tif err == nil {\n\t\tt.Error(\"client.Close() should have failed because of a shared client but it didn't\")\n\t}\n}\n\nfunc TestClientEnqueueWithGroupOption(t *testing.T) {\n\tr := setup(t)\n\tclient := NewClient(getRedisConnOpt(t))\n\tdefer client.Close()\n\n\ttask := NewTask(\"mytask\", []byte(\"foo\"))\n\tnow := time.Now()\n\n\ttests := []struct {\n\t\tdesc string\n\t\ttask *Task\n\t\topts []Option\n\t\twantInfo *TaskInfo\n\t\twantPending map[string][]*base.TaskMessage\n\t\twantGroups map[string]map[string][]base.Z // map queue name to a set of groups\n\t\twantScheduled map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tdesc: \"With only Group option\",\n\t\t\ttask: task,\n\t\t\topts: []Option{\n\t\t\t\tGroup(\"mygroup\"),\n\t\t\t},\n\t\t\twantInfo: &TaskInfo{\n\t\t\t\tQueue: \"default\",\n\t\t\t\tGroup: \"mygroup\",\n\t\t\t\tType: task.Type(),\n\t\t\t\tPayload: task.Payload(),\n\t\t\t\tState: TaskStateAggregating,\n\t\t\t\tMaxRetry: defaultMaxRetry,\n\t\t\t\tRetried: 0,\n\t\t\t\tLastErr: \"\",\n\t\t\t\tLastFailedAt: time.Time{},\n\t\t\t\tTimeout: defaultTimeout,\n\t\t\t\tDeadline: time.Time{},\n\t\t\t\tNextProcessAt: time.Time{},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {}, // should not be pending\n\t\t\t},\n\t\t\twantGroups: map[string]map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t\"mygroup\": {\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tMessage: &base.TaskMessage{\n\t\t\t\t\t\t\t\tType: task.Type(),\n\t\t\t\t\t\t\t\tPayload: task.Payload(),\n\t\t\t\t\t\t\t\tRetry: defaultMaxRetry,\n\t\t\t\t\t\t\t\tQueue: \"default\",\n\t\t\t\t\t\t\t\tTimeout: int64(defaultTimeout.Seconds()),\n\t\t\t\t\t\t\t\tDeadline: noDeadline.Unix(),\n\t\t\t\t\t\t\t\tGroupKey: \"mygroup\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\tScore: now.Unix(),\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"With Group and ProcessAt options\",\n\t\t\ttask: task,\n\t\t\topts: []Option{\n\t\t\t\tGroup(\"mygroup\"),\n\t\t\t\tProcessAt(now.Add(30 * time.Minute)),\n\t\t\t},\n\t\t\twantInfo: &TaskInfo{\n\t\t\t\tQueue: \"default\",\n\t\t\t\tGroup: \"mygroup\",\n\t\t\t\tType: task.Type(),\n\t\t\t\tPayload: task.Payload(),\n\t\t\t\tState: TaskStateScheduled,\n\t\t\t\tMaxRetry: defaultMaxRetry,\n\t\t\t\tRetried: 0,\n\t\t\t\tLastErr: \"\",\n\t\t\t\tLastFailedAt: time.Time{},\n\t\t\t\tTimeout: defaultTimeout,\n\t\t\t\tDeadline: time.Time{},\n\t\t\t\tNextProcessAt: now.Add(30 * time.Minute),\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {}, // should not be pending\n\t\t\t},\n\t\t\twantGroups: map[string]map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t\"mygroup\": {}, // should not be added to the group yet\n\t\t\t\t},\n\t\t\t},\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{\n\t\t\t\t\t\tMessage: &base.TaskMessage{\n\t\t\t\t\t\t\tType: task.Type(),\n\t\t\t\t\t\t\tPayload: task.Payload(),\n\t\t\t\t\t\t\tRetry: defaultMaxRetry,\n\t\t\t\t\t\t\tQueue: \"default\",\n\t\t\t\t\t\t\tTimeout: int64(defaultTimeout.Seconds()),\n\t\t\t\t\t\t\tDeadline: noDeadline.Unix(),\n\t\t\t\t\t\t\tGroupKey: \"mygroup\",\n\t\t\t\t\t\t},\n\t\t\t\t\t\tScore: now.Add(30 * time.Minute).Unix(),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r) // clean up db before each test case.\n\n\t\tgotInfo, err := client.Enqueue(tc.task, tc.opts...)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t\tcontinue\n\t\t}\n\t\tcmpOptions := []cmp.Option{\n\t\t\tcmpopts.IgnoreFields(TaskInfo{}, \"ID\"),\n\t\t\tcmpopts.EquateApproxTime(500 * time.Millisecond),\n\t\t}\n\t\tif diff := cmp.Diff(tc.wantInfo, gotInfo, cmpOptions...); diff != \"\" {\n\t\t\tt.Errorf(\"%s;\\nEnqueue(task) returned %v, want %v; (-want,+got)\\n%s\",\n\t\t\t\ttc.desc, gotInfo, tc.wantInfo, diff)\n\t\t}\n\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgot := h.GetPendingMessages(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, got, h.IgnoreIDOpt, cmpopts.EquateEmpty()); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s;\\nmismatch found in %q; (-want,+got)\\n%s\", tc.desc, base.PendingKey(qname), diff)\n\t\t\t}\n\t\t}\n\n\t\tfor qname, groups := range tc.wantGroups {\n\t\t\tfor groupKey, want := range groups {\n\t\t\t\tgot := h.GetGroupEntries(t, r, qname, groupKey)\n\t\t\t\tif diff := cmp.Diff(want, got, h.IgnoreIDOpt, cmpopts.EquateEmpty()); diff != \"\" {\n\t\t\t\t\tt.Errorf(\"%s;\\nmismatch found in %q; (-want,+got)\\n%s\", tc.desc, base.GroupKey(qname, groupKey), diff)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tfor qname, want := range tc.wantScheduled {\n\t\t\tgotScheduled := h.GetScheduledEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotScheduled, h.IgnoreIDOpt, cmpopts.EquateEmpty()); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s;\\nmismatch found in %q; (-want,+got)\\n%s\", tc.desc, base.ScheduledKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestClientEnqueueWithTaskIDOption(t *testing.T) {\n\tr := setup(t)\n\tclient := NewClient(getRedisConnOpt(t))\n\tdefer client.Close()\n\n\ttask := NewTask(\"send_email\", nil)\n\tnow := time.Now()\n\n\ttests := []struct {\n\t\tdesc string\n\t\ttask *Task\n\t\topts []Option\n\t\twantInfo *TaskInfo\n\t\twantPending map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tdesc: \"With a valid TaskID option\",\n\t\t\ttask: task,\n\t\t\topts: []Option{\n\t\t\t\tTaskID(\"custom_id\"),\n\t\t\t},\n\t\t\twantInfo: &TaskInfo{\n\t\t\t\tID: \"custom_id\",\n\t\t\t\tQueue: \"default\",\n\t\t\t\tType: task.Type(),\n\t\t\t\tPayload: task.Payload(),\n\t\t\t\tState: TaskStatePending,\n\t\t\t\tMaxRetry: defaultMaxRetry,\n\t\t\t\tRetried: 0,\n\t\t\t\tLastErr: \"\",\n\t\t\t\tLastFailedAt: time.Time{},\n\t\t\t\tTimeout: defaultTimeout,\n\t\t\t\tDeadline: time.Time{},\n\t\t\t\tNextProcessAt: now,\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{\n\t\t\t\t\t\tID: \"custom_id\",\n\t\t\t\t\t\tType: task.Type(),\n\t\t\t\t\t\tPayload: task.Payload(),\n\t\t\t\t\t\tRetry: defaultMaxRetry,\n\t\t\t\t\t\tQueue: \"default\",\n\t\t\t\t\t\tTimeout: int64(defaultTimeout.Seconds()),\n\t\t\t\t\t\tDeadline: noDeadline.Unix(),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r) // clean up db before each test case.\n\n\t\tgotInfo, err := client.Enqueue(tc.task, tc.opts...)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"got non-nil error %v, want nil\", err)\n\t\t\tcontinue\n\t\t}\n\n\t\tcmpOptions := []cmp.Option{\n\t\t\tcmpopts.EquateApproxTime(500 * time.Millisecond),\n\t\t}\n\t\tif diff := cmp.Diff(tc.wantInfo, gotInfo, cmpOptions...); diff != \"\" {\n\t\t\tt.Errorf(\"%s;\\nEnqueue(task) returned %v, want %v; (-want,+got)\\n%s\",\n\t\t\t\ttc.desc, gotInfo, tc.wantInfo, diff)\n\t\t}\n\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgot := h.GetPendingMessages(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, got); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s;\\nmismatch found in %q; (-want,+got)\\n%s\", tc.desc, base.PendingKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestClientEnqueueWithConflictingTaskID(t *testing.T) {\n\tsetup(t)\n\tclient := NewClient(getRedisConnOpt(t))\n\tdefer client.Close()\n\n\tconst taskID = \"custom_id\"\n\ttask := NewTask(\"foo\", nil)\n\n\tif _, err := client.Enqueue(task, TaskID(taskID)); err != nil {\n\t\tt.Fatalf(\"First task: Enqueue failed: %v\", err)\n\t}\n\t_, err := client.Enqueue(task, TaskID(taskID))\n\tif !errors.Is(err, ErrTaskIDConflict) {\n\t\tt.Errorf(\"Second task: Enqueue returned %v, want %v\", err, ErrTaskIDConflict)\n\t}\n}\n\nfunc TestClientEnqueueWithProcessInOption(t *testing.T) {\n\tr := setup(t)\n\tclient := NewClient(getRedisConnOpt(t))\n\tdefer client.Close()\n\n\ttask := NewTask(\"send_email\", h.JSON(map[string]interface{}{\"to\": \"customer@gmail.com\", \"from\": \"merchant@example.com\"}))\n\tnow := time.Now()\n\n\ttests := []struct {\n\t\tdesc string\n\t\ttask *Task\n\t\tdelay time.Duration // value for ProcessIn option\n\t\topts []Option // other options\n\t\twantInfo *TaskInfo\n\t\twantPending map[string][]*base.TaskMessage\n\t\twantScheduled map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tdesc: \"schedule a task to be processed in one hour\",\n\t\t\ttask: task,\n\t\t\tdelay: 1 * time.Hour,\n\t\t\topts: []Option{},\n\t\t\twantInfo: &TaskInfo{\n\t\t\t\tQueue: \"default\",\n\t\t\t\tType: task.Type(),\n\t\t\t\tPayload: task.Payload(),\n\t\t\t\tState: TaskStateScheduled,\n\t\t\t\tMaxRetry: defaultMaxRetry,\n\t\t\t\tRetried: 0,\n\t\t\t\tLastErr: \"\",\n\t\t\t\tLastFailedAt: time.Time{},\n\t\t\t\tTimeout: defaultTimeout,\n\t\t\t\tDeadline: time.Time{},\n\t\t\t\tNextProcessAt: time.Now().Add(1 * time.Hour),\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{\n\t\t\t\t\t\tMessage: &base.TaskMessage{\n\t\t\t\t\t\t\tType: task.Type(),\n\t\t\t\t\t\t\tPayload: task.Payload(),\n\t\t\t\t\t\t\tRetry: defaultMaxRetry,\n\t\t\t\t\t\t\tQueue: \"default\",\n\t\t\t\t\t\t\tTimeout: int64(defaultTimeout.Seconds()),\n\t\t\t\t\t\t\tDeadline: noDeadline.Unix(),\n\t\t\t\t\t\t},\n\t\t\t\t\t\tScore: time.Now().Add(time.Hour).Unix(),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"Zero delay\",\n\t\t\ttask: task,\n\t\t\tdelay: 0,\n\t\t\topts: []Option{},\n\t\t\twantInfo: &TaskInfo{\n\t\t\t\tQueue: \"default\",\n\t\t\t\tType: task.Type(),\n\t\t\t\tPayload: task.Payload(),\n\t\t\t\tState: TaskStatePending,\n\t\t\t\tMaxRetry: defaultMaxRetry,\n\t\t\t\tRetried: 0,\n\t\t\t\tLastErr: \"\",\n\t\t\t\tLastFailedAt: time.Time{},\n\t\t\t\tTimeout: defaultTimeout,\n\t\t\t\tDeadline: time.Time{},\n\t\t\t\tNextProcessAt: now,\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{\n\t\t\t\t\t\tType: task.Type(),\n\t\t\t\t\t\tPayload: task.Payload(),\n\t\t\t\t\t\tRetry: defaultMaxRetry,\n\t\t\t\t\t\tQueue: \"default\",\n\t\t\t\t\t\tTimeout: int64(defaultTimeout.Seconds()),\n\t\t\t\t\t\tDeadline: noDeadline.Unix(),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r) // clean up db before each test case.\n\n\t\topts := append(tc.opts, ProcessIn(tc.delay))\n\t\tgotInfo, err := client.Enqueue(tc.task, opts...)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t\tcontinue\n\t\t}\n\t\tcmpOptions := []cmp.Option{\n\t\t\tcmpopts.IgnoreFields(TaskInfo{}, \"ID\"),\n\t\t\tcmpopts.EquateApproxTime(500 * time.Millisecond),\n\t\t}\n\t\tif diff := cmp.Diff(tc.wantInfo, gotInfo, cmpOptions...); diff != \"\" {\n\t\t\tt.Errorf(\"%s;\\nEnqueue(task, ProcessIn(%v)) returned %v, want %v; (-want,+got)\\n%s\",\n\t\t\t\ttc.desc, tc.delay, gotInfo, tc.wantInfo, diff)\n\t\t}\n\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.IgnoreIDOpt, cmpopts.EquateEmpty()); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s;\\nmismatch found in %q; (-want,+got)\\n%s\", tc.desc, base.PendingKey(qname), diff)\n\t\t\t}\n\t\t}\n\t\tfor qname, want := range tc.wantScheduled {\n\t\t\tgotScheduled := h.GetScheduledEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotScheduled, h.IgnoreIDOpt, cmpopts.EquateEmpty()); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s;\\nmismatch found in %q; (-want,+got)\\n%s\", tc.desc, base.ScheduledKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestClientEnqueueError(t *testing.T) {\n\tr := setup(t)\n\tclient := NewClient(getRedisConnOpt(t))\n\tdefer client.Close()\n\n\ttask := NewTask(\"send_email\", h.JSON(map[string]interface{}{\"to\": \"customer@gmail.com\", \"from\": \"merchant@example.com\"}))\n\n\ttests := []struct {\n\t\tdesc string\n\t\ttask *Task\n\t\topts []Option\n\t}{\n\t\t{\n\t\t\tdesc: \"With nil task\",\n\t\t\ttask: nil,\n\t\t\topts: []Option{},\n\t\t},\n\t\t{\n\t\t\tdesc: \"With empty queue name\",\n\t\t\ttask: task,\n\t\t\topts: []Option{\n\t\t\t\tQueue(\"\"),\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"With empty task typename\",\n\t\t\ttask: NewTask(\"\", h.JSON(map[string]interface{}{})),\n\t\t\topts: []Option{},\n\t\t},\n\t\t{\n\t\t\tdesc: \"With blank task typename\",\n\t\t\ttask: NewTask(\" \", h.JSON(map[string]interface{}{})),\n\t\t\topts: []Option{},\n\t\t},\n\t\t{\n\t\t\tdesc: \"With empty task ID\",\n\t\t\ttask: NewTask(\"foo\", nil),\n\t\t\topts: []Option{TaskID(\"\")},\n\t\t},\n\t\t{\n\t\t\tdesc: \"With blank task ID\",\n\t\t\ttask: NewTask(\"foo\", nil),\n\t\t\topts: []Option{TaskID(\" \")},\n\t\t},\n\t\t{\n\t\t\tdesc: \"With unique option less than 1s\",\n\t\t\ttask: NewTask(\"foo\", nil),\n\t\t\topts: []Option{Unique(300 * time.Millisecond)},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\n\t\t_, err := client.Enqueue(tc.task, tc.opts...)\n\t\tif err == nil {\n\t\t\tt.Errorf(\"%s; client.Enqueue(task, opts...) did not return non-nil error\", tc.desc)\n\t\t}\n\t}\n}\n\nfunc TestClientWithDefaultOptions(t *testing.T) {\n\tr := setup(t)\n\n\tnow := time.Now()\n\n\ttests := []struct {\n\t\tdesc string\n\t\tdefaultOpts []Option // options set at task initialization time\n\t\topts []Option // options used at enqueue time.\n\t\ttasktype string\n\t\tpayload []byte\n\t\twantInfo *TaskInfo\n\t\tqueue string // queue that the message should go into.\n\t\twant *base.TaskMessage\n\t}{\n\t\t{\n\t\t\tdesc: \"With queue routing option\",\n\t\t\tdefaultOpts: []Option{Queue(\"feed\")},\n\t\t\topts: []Option{},\n\t\t\ttasktype: \"feed:import\",\n\t\t\tpayload: nil,\n\t\t\twantInfo: &TaskInfo{\n\t\t\t\tQueue: \"feed\",\n\t\t\t\tType: \"feed:import\",\n\t\t\t\tPayload: nil,\n\t\t\t\tState: TaskStatePending,\n\t\t\t\tMaxRetry: defaultMaxRetry,\n\t\t\t\tRetried: 0,\n\t\t\t\tLastErr: \"\",\n\t\t\t\tLastFailedAt: time.Time{},\n\t\t\t\tTimeout: defaultTimeout,\n\t\t\t\tDeadline: time.Time{},\n\t\t\t\tNextProcessAt: now,\n\t\t\t},\n\t\t\tqueue: \"feed\",\n\t\t\twant: &base.TaskMessage{\n\t\t\t\tType: \"feed:import\",\n\t\t\t\tPayload: nil,\n\t\t\t\tRetry: defaultMaxRetry,\n\t\t\t\tQueue: \"feed\",\n\t\t\t\tTimeout: int64(defaultTimeout.Seconds()),\n\t\t\t\tDeadline: noDeadline.Unix(),\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"With multiple options\",\n\t\t\tdefaultOpts: []Option{Queue(\"feed\"), MaxRetry(5)},\n\t\t\topts: []Option{},\n\t\t\ttasktype: \"feed:import\",\n\t\t\tpayload: nil,\n\t\t\twantInfo: &TaskInfo{\n\t\t\t\tQueue: \"feed\",\n\t\t\t\tType: \"feed:import\",\n\t\t\t\tPayload: nil,\n\t\t\t\tState: TaskStatePending,\n\t\t\t\tMaxRetry: 5,\n\t\t\t\tRetried: 0,\n\t\t\t\tLastErr: \"\",\n\t\t\t\tLastFailedAt: time.Time{},\n\t\t\t\tTimeout: defaultTimeout,\n\t\t\t\tDeadline: time.Time{},\n\t\t\t\tNextProcessAt: now,\n\t\t\t},\n\t\t\tqueue: \"feed\",\n\t\t\twant: &base.TaskMessage{\n\t\t\t\tType: \"feed:import\",\n\t\t\t\tPayload: nil,\n\t\t\t\tRetry: 5,\n\t\t\t\tQueue: \"feed\",\n\t\t\t\tTimeout: int64(defaultTimeout.Seconds()),\n\t\t\t\tDeadline: noDeadline.Unix(),\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"With overriding options at enqueue time\",\n\t\t\tdefaultOpts: []Option{Queue(\"feed\"), MaxRetry(5)},\n\t\t\topts: []Option{Queue(\"critical\")},\n\t\t\ttasktype: \"feed:import\",\n\t\t\tpayload: nil,\n\t\t\twantInfo: &TaskInfo{\n\t\t\t\tQueue: \"critical\",\n\t\t\t\tType: \"feed:import\",\n\t\t\t\tPayload: nil,\n\t\t\t\tState: TaskStatePending,\n\t\t\t\tMaxRetry: 5,\n\t\t\t\tLastErr: \"\",\n\t\t\t\tLastFailedAt: time.Time{},\n\t\t\t\tTimeout: defaultTimeout,\n\t\t\t\tDeadline: time.Time{},\n\t\t\t\tNextProcessAt: now,\n\t\t\t},\n\t\t\tqueue: \"critical\",\n\t\t\twant: &base.TaskMessage{\n\t\t\t\tType: \"feed:import\",\n\t\t\t\tPayload: nil,\n\t\t\t\tRetry: 5,\n\t\t\t\tQueue: \"critical\",\n\t\t\t\tTimeout: int64(defaultTimeout.Seconds()),\n\t\t\t\tDeadline: noDeadline.Unix(),\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\tc := NewClient(getRedisConnOpt(t))\n\t\tdefer c.Close()\n\t\ttask := NewTask(tc.tasktype, tc.payload, tc.defaultOpts...)\n\t\tgotInfo, err := c.Enqueue(task, tc.opts...)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tcmpOptions := []cmp.Option{\n\t\t\tcmpopts.IgnoreFields(TaskInfo{}, \"ID\"),\n\t\t\tcmpopts.EquateApproxTime(500 * time.Millisecond),\n\t\t}\n\t\tif diff := cmp.Diff(tc.wantInfo, gotInfo, cmpOptions...); diff != \"\" {\n\t\t\tt.Errorf(\"%s;\\nEnqueue(task, opts...) returned %v, want %v; (-want,+got)\\n%s\",\n\t\t\t\ttc.desc, gotInfo, tc.wantInfo, diff)\n\t\t}\n\t\tpending := h.GetPendingMessages(t, r, tc.queue)\n\t\tif len(pending) != 1 {\n\t\t\tt.Errorf(\"%s;\\nexpected queue %q to have one message; got %d messages in the queue.\",\n\t\t\t\ttc.desc, tc.queue, len(pending))\n\t\t\tcontinue\n\t\t}\n\t\tgot := pending[0]\n\t\tif diff := cmp.Diff(tc.want, got, h.IgnoreIDOpt); diff != \"\" {\n\t\t\tt.Errorf(\"%s;\\nmismatch found in pending task message; (-want,+got)\\n%s\",\n\t\t\t\ttc.desc, diff)\n\t\t}\n\t}\n}\n\nfunc TestClientEnqueueUnique(t *testing.T) {\n\tr := setup(t)\n\tc := NewClient(getRedisConnOpt(t))\n\tdefer c.Close()\n\n\ttests := []struct {\n\t\ttask *Task\n\t\tttl time.Duration\n\t}{\n\t\t{\n\t\t\tNewTask(\"email\", h.JSON(map[string]interface{}{\"user_id\": 123})),\n\t\t\ttime.Hour,\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r) // clean up db before each test case.\n\n\t\t// Enqueue the task first. It should succeed.\n\t\t_, err := c.Enqueue(tc.task, Unique(tc.ttl))\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tgotTTL := r.TTL(context.Background(), base.UniqueKey(base.DefaultQueueName, tc.task.Type(), tc.task.Payload())).Val()\n\t\tif !cmp.Equal(tc.ttl.Seconds(), gotTTL.Seconds(), cmpopts.EquateApprox(0, 1)) {\n\t\t\tt.Errorf(\"TTL = %v, want %v\", gotTTL, tc.ttl)\n\t\t\tcontinue\n\t\t}\n\n\t\t// Enqueue the task again. It should fail.\n\t\t_, err = c.Enqueue(tc.task, Unique(tc.ttl))\n\t\tif err == nil {\n\t\t\tt.Errorf(\"Enqueueing %+v did not return an error\", tc.task)\n\t\t\tcontinue\n\t\t}\n\t\tif !errors.Is(err, ErrDuplicateTask) {\n\t\t\tt.Errorf(\"Enqueueing %+v returned an error that is not ErrDuplicateTask\", tc.task)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\nfunc TestClientEnqueueUniqueWithProcessInOption(t *testing.T) {\n\tr := setup(t)\n\tc := NewClient(getRedisConnOpt(t))\n\tdefer c.Close()\n\n\ttests := []struct {\n\t\ttask *Task\n\t\td time.Duration\n\t\tttl time.Duration\n\t}{\n\t\t{\n\t\t\tNewTask(\"reindex\", nil),\n\t\t\ttime.Hour,\n\t\t\t10 * time.Minute,\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r) // clean up db before each test case.\n\n\t\t// Enqueue the task first. It should succeed.\n\t\t_, err := c.Enqueue(tc.task, ProcessIn(tc.d), Unique(tc.ttl))\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tgotTTL := r.TTL(context.Background(), base.UniqueKey(base.DefaultQueueName, tc.task.Type(), tc.task.Payload())).Val()\n\t\twantTTL := time.Duration(tc.ttl.Seconds()+tc.d.Seconds()) * time.Second\n\t\tif !cmp.Equal(wantTTL.Seconds(), gotTTL.Seconds(), cmpopts.EquateApprox(0, 1)) {\n\t\t\tt.Errorf(\"TTL = %v, want %v\", gotTTL, wantTTL)\n\t\t\tcontinue\n\t\t}\n\n\t\t// Enqueue the task again. It should fail.\n\t\t_, err = c.Enqueue(tc.task, ProcessIn(tc.d), Unique(tc.ttl))\n\t\tif err == nil {\n\t\t\tt.Errorf(\"Enqueueing %+v did not return an error\", tc.task)\n\t\t\tcontinue\n\t\t}\n\t\tif !errors.Is(err, ErrDuplicateTask) {\n\t\t\tt.Errorf(\"Enqueueing %+v returned an error that is not ErrDuplicateTask\", tc.task)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\nfunc TestClientEnqueueUniqueWithProcessAtOption(t *testing.T) {\n\tr := setup(t)\n\tc := NewClient(getRedisConnOpt(t))\n\tdefer c.Close()\n\n\ttests := []struct {\n\t\ttask *Task\n\t\tat time.Time\n\t\tttl time.Duration\n\t}{\n\t\t{\n\t\t\tNewTask(\"reindex\", nil),\n\t\t\ttime.Now().Add(time.Hour),\n\t\t\t10 * time.Minute,\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r) // clean up db before each test case.\n\n\t\t// Enqueue the task first. It should succeed.\n\t\t_, err := c.Enqueue(tc.task, ProcessAt(tc.at), Unique(tc.ttl))\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tgotTTL := r.TTL(context.Background(), base.UniqueKey(base.DefaultQueueName, tc.task.Type(), tc.task.Payload())).Val()\n\t\twantTTL := time.Until(tc.at.Add(tc.ttl))\n\t\tif !cmp.Equal(wantTTL.Seconds(), gotTTL.Seconds(), cmpopts.EquateApprox(0, 1)) {\n\t\t\tt.Errorf(\"TTL = %v, want %v\", gotTTL, wantTTL)\n\t\t\tcontinue\n\t\t}\n\n\t\t// Enqueue the task again. It should fail.\n\t\t_, err = c.Enqueue(tc.task, ProcessAt(tc.at), Unique(tc.ttl))\n\t\tif err == nil {\n\t\t\tt.Errorf(\"Enqueueing %+v did not return an error\", tc.task)\n\t\t\tcontinue\n\t\t}\n\t\tif !errors.Is(err, ErrDuplicateTask) {\n\t\t\tt.Errorf(\"Enqueueing %+v returned an error that is not ErrDuplicateTask\", tc.task)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\nfunc TestClientEnqueueWithHeaders(t *testing.T) {\n\tr := setup(t)\n\tclient := NewClient(getRedisConnOpt(t))\n\tdefer client.Close()\n\n\tnow := time.Now()\n\theaders := map[string]string{\n\t\t\"user-id\": \"123\",\n\t\t\"request-id\": \"abc-def-ghi\",\n\t\t\"priority\": \"high\",\n\t}\n\n\ttests := []struct {\n\t\tdesc string\n\t\ttask *Task\n\t\topts []Option\n\t\twantInfo *TaskInfo\n\t\twantPending map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tdesc: \"Task with headers\",\n\t\t\ttask: NewTaskWithHeaders(\"send_email\", h.JSON(map[string]interface{}{\"to\": \"user@example.com\"}), headers),\n\t\t\topts: []Option{},\n\t\t\twantInfo: &TaskInfo{\n\t\t\t\tQueue: \"default\",\n\t\t\t\tType: \"send_email\",\n\t\t\t\tPayload: h.JSON(map[string]interface{}{\"to\": \"user@example.com\"}),\n\t\t\t\tHeaders: headers,\n\t\t\t\tState: TaskStatePending,\n\t\t\t\tMaxRetry: defaultMaxRetry,\n\t\t\t\tRetried: 0,\n\t\t\t\tLastErr: \"\",\n\t\t\t\tLastFailedAt: time.Time{},\n\t\t\t\tTimeout: defaultTimeout,\n\t\t\t\tDeadline: time.Time{},\n\t\t\t\tNextProcessAt: now,\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{\n\t\t\t\t\t\tType: \"send_email\",\n\t\t\t\t\t\tPayload: h.JSON(map[string]interface{}{\"to\": \"user@example.com\"}),\n\t\t\t\t\t\tHeaders: headers,\n\t\t\t\t\t\tRetry: defaultMaxRetry,\n\t\t\t\t\t\tQueue: \"default\",\n\t\t\t\t\t\tTimeout: int64(defaultTimeout.Seconds()),\n\t\t\t\t\t\tDeadline: noDeadline.Unix(),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"Task with empty headers\",\n\t\t\ttask: NewTaskWithHeaders(\"process_data\", []byte(\"data\"), map[string]string{}),\n\t\t\topts: []Option{},\n\t\t\twantInfo: &TaskInfo{\n\t\t\t\tQueue: \"default\",\n\t\t\t\tType: \"process_data\",\n\t\t\t\tPayload: []byte(\"data\"),\n\t\t\t\tHeaders: map[string]string{},\n\t\t\t\tState: TaskStatePending,\n\t\t\t\tMaxRetry: defaultMaxRetry,\n\t\t\t\tRetried: 0,\n\t\t\t\tLastErr: \"\",\n\t\t\t\tLastFailedAt: time.Time{},\n\t\t\t\tTimeout: defaultTimeout,\n\t\t\t\tDeadline: time.Time{},\n\t\t\t\tNextProcessAt: now,\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{\n\t\t\t\t\t\tType: \"process_data\",\n\t\t\t\t\t\tPayload: []byte(\"data\"),\n\t\t\t\t\t\tHeaders: nil,\n\t\t\t\t\t\tRetry: defaultMaxRetry,\n\t\t\t\t\t\tQueue: \"default\",\n\t\t\t\t\t\tTimeout: int64(defaultTimeout.Seconds()),\n\t\t\t\t\t\tDeadline: noDeadline.Unix(),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"Task with nil headers\",\n\t\t\ttask: NewTaskWithHeaders(\"cleanup\", nil, nil),\n\t\t\topts: []Option{},\n\t\t\twantInfo: &TaskInfo{\n\t\t\t\tQueue: \"default\",\n\t\t\t\tType: \"cleanup\",\n\t\t\t\tPayload: nil,\n\t\t\t\tHeaders: nil,\n\t\t\t\tState: TaskStatePending,\n\t\t\t\tMaxRetry: defaultMaxRetry,\n\t\t\t\tRetried: 0,\n\t\t\t\tLastErr: \"\",\n\t\t\t\tLastFailedAt: time.Time{},\n\t\t\t\tTimeout: defaultTimeout,\n\t\t\t\tDeadline: time.Time{},\n\t\t\t\tNextProcessAt: now,\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{\n\t\t\t\t\t\tType: \"cleanup\",\n\t\t\t\t\t\tPayload: nil,\n\t\t\t\t\t\tHeaders: nil,\n\t\t\t\t\t\tRetry: defaultMaxRetry,\n\t\t\t\t\t\tQueue: \"default\",\n\t\t\t\t\t\tTimeout: int64(defaultTimeout.Seconds()),\n\t\t\t\t\t\tDeadline: noDeadline.Unix(),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"Task with headers and custom options\",\n\t\t\ttask: NewTaskWithHeaders(\"notify\", []byte(\"notification\"), map[string]string{\"channel\": \"email\"}),\n\t\t\topts: []Option{MaxRetry(5), Queue(\"notifications\")},\n\t\t\twantInfo: &TaskInfo{\n\t\t\t\tQueue: \"notifications\",\n\t\t\t\tType: \"notify\",\n\t\t\t\tPayload: []byte(\"notification\"),\n\t\t\t\tHeaders: map[string]string{\"channel\": \"email\"},\n\t\t\t\tState: TaskStatePending,\n\t\t\t\tMaxRetry: 5,\n\t\t\t\tRetried: 0,\n\t\t\t\tLastErr: \"\",\n\t\t\t\tLastFailedAt: time.Time{},\n\t\t\t\tTimeout: defaultTimeout,\n\t\t\t\tDeadline: time.Time{},\n\t\t\t\tNextProcessAt: now,\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"notifications\": {\n\t\t\t\t\t{\n\t\t\t\t\t\tType: \"notify\",\n\t\t\t\t\t\tPayload: []byte(\"notification\"),\n\t\t\t\t\t\tHeaders: map[string]string{\"channel\": \"email\"},\n\t\t\t\t\t\tRetry: 5,\n\t\t\t\t\t\tQueue: \"notifications\",\n\t\t\t\t\t\tTimeout: int64(defaultTimeout.Seconds()),\n\t\t\t\t\t\tDeadline: noDeadline.Unix(),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\n\t\tgotInfo, err := client.Enqueue(tc.task, tc.opts...)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s: Enqueue failed: %v\", tc.desc, err)\n\t\t\tcontinue\n\t\t}\n\n\t\tcmpOptions := []cmp.Option{\n\t\t\tcmpopts.IgnoreFields(TaskInfo{}, \"ID\"),\n\t\t\tcmpopts.EquateApproxTime(500 * time.Millisecond),\n\t\t}\n\t\tif diff := cmp.Diff(tc.wantInfo, gotInfo, cmpOptions...); diff != \"\" {\n\t\t\tt.Errorf(\"%s;\\nEnqueue(task) returned %v, want %v; (-want,+got)\\n%s\",\n\t\t\t\ttc.desc, gotInfo, tc.wantInfo, diff)\n\t\t}\n\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgot := h.GetPendingMessages(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, got, h.IgnoreIDOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s;\\nmismatch found in %q; (-want,+got)\\n%s\", tc.desc, base.PendingKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestClientEnqueueWithHeadersScheduled(t *testing.T) {\n\tr := setup(t)\n\tclient := NewClient(getRedisConnOpt(t))\n\tdefer client.Close()\n\n\tnow := time.Now()\n\toneHourLater := now.Add(time.Hour)\n\theaders := map[string]string{\n\t\t\"correlation-id\": \"xyz-123\",\n\t\t\"source\": \"api\",\n\t}\n\n\ttests := []struct {\n\t\tdesc string\n\t\ttask *Task\n\t\tprocessAt time.Time\n\t\topts []Option\n\t\twantInfo *TaskInfo\n\t\twantScheduled map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tdesc: \"Schedule task with headers\",\n\t\t\ttask: NewTaskWithHeaders(\"scheduled_task\", []byte(\"payload\"), headers),\n\t\t\tprocessAt: oneHourLater,\n\t\t\topts: []Option{},\n\t\t\twantInfo: &TaskInfo{\n\t\t\t\tQueue: \"default\",\n\t\t\t\tType: \"scheduled_task\",\n\t\t\t\tPayload: []byte(\"payload\"),\n\t\t\t\tHeaders: headers,\n\t\t\t\tState: TaskStateScheduled,\n\t\t\t\tMaxRetry: defaultMaxRetry,\n\t\t\t\tRetried: 0,\n\t\t\t\tLastErr: \"\",\n\t\t\t\tLastFailedAt: time.Time{},\n\t\t\t\tTimeout: defaultTimeout,\n\t\t\t\tDeadline: time.Time{},\n\t\t\t\tNextProcessAt: oneHourLater,\n\t\t\t},\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{\n\t\t\t\t\t\tMessage: &base.TaskMessage{\n\t\t\t\t\t\t\tType: \"scheduled_task\",\n\t\t\t\t\t\t\tPayload: []byte(\"payload\"),\n\t\t\t\t\t\t\tHeaders: headers,\n\t\t\t\t\t\t\tRetry: defaultMaxRetry,\n\t\t\t\t\t\t\tQueue: \"default\",\n\t\t\t\t\t\t\tTimeout: int64(defaultTimeout.Seconds()),\n\t\t\t\t\t\t\tDeadline: noDeadline.Unix(),\n\t\t\t\t\t\t},\n\t\t\t\t\t\tScore: oneHourLater.Unix(),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\n\t\topts := append(tc.opts, ProcessAt(tc.processAt))\n\t\tgotInfo, err := client.Enqueue(tc.task, opts...)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s: Enqueue failed: %v\", tc.desc, err)\n\t\t\tcontinue\n\t\t}\n\n\t\tcmpOptions := []cmp.Option{\n\t\t\tcmpopts.IgnoreFields(TaskInfo{}, \"ID\"),\n\t\t\tcmpopts.EquateApproxTime(500 * time.Millisecond),\n\t\t}\n\t\tif diff := cmp.Diff(tc.wantInfo, gotInfo, cmpOptions...); diff != \"\" {\n\t\t\tt.Errorf(\"%s;\\nEnqueue(task, ProcessAt(%v)) returned %v, want %v; (-want,+got)\\n%s\",\n\t\t\t\ttc.desc, tc.processAt, gotInfo, tc.wantInfo, diff)\n\t\t}\n\n\t\tfor qname, want := range tc.wantScheduled {\n\t\t\tgotScheduled := h.GetScheduledEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotScheduled, h.IgnoreIDOpt, cmpopts.EquateEmpty()); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s;\\nmismatch found in %q; (-want,+got)\\n%s\", tc.desc, base.ScheduledKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestNewTaskWithHeaders(t *testing.T) {\n\ttests := []struct {\n\t\tdesc string\n\t\ttypename string\n\t\tpayload []byte\n\t\theaders map[string]string\n\t\topts []Option\n\t\twant *Task\n\t}{\n\t\t{\n\t\t\tdesc: \"Task with headers\",\n\t\t\ttypename: \"test_task\",\n\t\t\tpayload: []byte(\"test payload\"),\n\t\t\theaders: map[string]string{\"key1\": \"value1\", \"key2\": \"value2\"},\n\t\t\topts: []Option{MaxRetry(3)},\n\t\t\twant: &Task{\n\t\t\t\ttypename: \"test_task\",\n\t\t\t\tpayload: []byte(\"test payload\"),\n\t\t\t\theaders: map[string]string{\"key1\": \"value1\", \"key2\": \"value2\"},\n\t\t\t\topts: []Option{MaxRetry(3)},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"Task with empty headers\",\n\t\t\ttypename: \"empty_headers\",\n\t\t\tpayload: nil,\n\t\t\theaders: map[string]string{},\n\t\t\topts: nil,\n\t\t\twant: &Task{\n\t\t\t\ttypename: \"empty_headers\",\n\t\t\t\tpayload: nil,\n\t\t\t\theaders: map[string]string{},\n\t\t\t\topts: nil,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"Task with nil headers\",\n\t\t\ttypename: \"nil_headers\",\n\t\t\tpayload: []byte(\"data\"),\n\t\t\theaders: nil,\n\t\t\topts: []Option{Queue(\"test\")},\n\t\t\twant: &Task{\n\t\t\t\ttypename: \"nil_headers\",\n\t\t\t\tpayload: []byte(\"data\"),\n\t\t\t\theaders: nil,\n\t\t\t\topts: []Option{Queue(\"test\")},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := NewTaskWithHeaders(tc.typename, tc.payload, tc.headers, tc.opts...)\n\n\t\tif got.Type() != tc.want.typename {\n\t\t\tt.Errorf(\"%s: Type() = %q, want %q\", tc.desc, got.Type(), tc.want.typename)\n\t\t}\n\n\t\tif diff := cmp.Diff(tc.want.payload, got.Payload()); diff != \"\" {\n\t\t\tt.Errorf(\"%s: Payload() mismatch (-want,+got)\\n%s\", tc.desc, diff)\n\t\t}\n\n\t\tif diff := cmp.Diff(tc.want.headers, got.Headers()); diff != \"\" {\n\t\t\tt.Errorf(\"%s: Headers() mismatch (-want,+got)\\n%s\", tc.desc, diff)\n\t\t}\n\n\t\tif tc.headers != nil && got.Headers() != nil {\n\t\t\ttc.headers[\"modified\"] = \"test\"\n\t\t\tif _, exists := got.Headers()[\"modified\"]; exists {\n\t\t\t\tt.Errorf(\"%s: Headers should be cloned, but modification affected task headers\", tc.desc)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestTaskHeadersMethod(t *testing.T) {\n\ttests := []struct {\n\t\tdesc string\n\t\ttask *Task\n\t\twant map[string]string\n\t\twantNil bool\n\t}{\n\t\t{\n\t\t\tdesc: \"Task created with NewTask has nil headers\",\n\t\t\ttask: NewTask(\"test\", []byte(\"data\")),\n\t\t\twant: nil,\n\t\t\twantNil: true,\n\t\t},\n\t\t{\n\t\t\tdesc: \"Task created with NewTaskWithHeaders has headers\",\n\t\t\ttask: NewTaskWithHeaders(\"test\", []byte(\"data\"), map[string]string{\"key\": \"value\"}),\n\t\t\twant: map[string]string{\"key\": \"value\"},\n\t\t},\n\t\t{\n\t\t\tdesc: \"Task created with empty headers\",\n\t\t\ttask: NewTaskWithHeaders(\"test\", []byte(\"data\"), map[string]string{}),\n\t\t\twant: map[string]string{},\n\t\t},\n\t\t{\n\t\t\tdesc: \"Task created with nil headers\",\n\t\t\ttask: NewTaskWithHeaders(\"test\", []byte(\"data\"), nil),\n\t\t\twant: nil,\n\t\t\twantNil: true,\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := tc.task.Headers()\n\n\t\tif tc.wantNil {\n\t\t\tif got != nil {\n\t\t\t\tt.Errorf(\"%s: Headers() = %v, want nil\", tc.desc, got)\n\t\t\t}\n\t\t} else {\n\t\t\tif diff := cmp.Diff(tc.want, got); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s: Headers() mismatch (-want,+got)\\n%s\", tc.desc, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestClientEnqueueWithHeadersAndGroup(t *testing.T) {\n\tr := setup(t)\n\tclient := NewClient(getRedisConnOpt(t))\n\tdefer client.Close()\n\n\tnow := time.Now()\n\theaders := map[string]string{\n\t\t\"batch-id\": \"batch-123\",\n\t\t\"priority\": \"high\",\n\t}\n\n\ttests := []struct {\n\t\tdesc string\n\t\ttask *Task\n\t\topts []Option\n\t\twantInfo *TaskInfo\n\t\twantGroups map[string]map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tdesc: \"Task with headers and group\",\n\t\t\ttask: NewTaskWithHeaders(\"batch_process\", []byte(\"item1\"), headers),\n\t\t\topts: []Option{Group(\"batch-123\")},\n\t\t\twantInfo: &TaskInfo{\n\t\t\t\tQueue: \"default\",\n\t\t\t\tGroup: \"batch-123\",\n\t\t\t\tType: \"batch_process\",\n\t\t\t\tPayload: []byte(\"item1\"),\n\t\t\t\tHeaders: headers,\n\t\t\t\tState: TaskStateAggregating,\n\t\t\t\tMaxRetry: defaultMaxRetry,\n\t\t\t\tRetried: 0,\n\t\t\t\tLastErr: \"\",\n\t\t\t\tLastFailedAt: time.Time{},\n\t\t\t\tTimeout: defaultTimeout,\n\t\t\t\tDeadline: time.Time{},\n\t\t\t\tNextProcessAt: time.Time{},\n\t\t\t},\n\t\t\twantGroups: map[string]map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t\"batch-123\": {\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tMessage: &base.TaskMessage{\n\t\t\t\t\t\t\t\tType: \"batch_process\",\n\t\t\t\t\t\t\t\tPayload: []byte(\"item1\"),\n\t\t\t\t\t\t\t\tHeaders: headers,\n\t\t\t\t\t\t\t\tRetry: defaultMaxRetry,\n\t\t\t\t\t\t\t\tQueue: \"default\",\n\t\t\t\t\t\t\t\tTimeout: int64(defaultTimeout.Seconds()),\n\t\t\t\t\t\t\t\tDeadline: noDeadline.Unix(),\n\t\t\t\t\t\t\t\tGroupKey: \"batch-123\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\tScore: now.Unix(),\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\n\t\tgotInfo, err := client.Enqueue(tc.task, tc.opts...)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s: Enqueue failed: %v\", tc.desc, err)\n\t\t\tcontinue\n\t\t}\n\n\t\tcmpOptions := []cmp.Option{\n\t\t\tcmpopts.IgnoreFields(TaskInfo{}, \"ID\"),\n\t\t\tcmpopts.EquateApproxTime(500 * time.Millisecond),\n\t\t}\n\t\tif diff := cmp.Diff(tc.wantInfo, gotInfo, cmpOptions...); diff != \"\" {\n\t\t\tt.Errorf(\"%s;\\nEnqueue(task) returned %v, want %v; (-want,+got)\\n%s\",\n\t\t\t\ttc.desc, gotInfo, tc.wantInfo, diff)\n\t\t}\n\n\t\tfor qname, groups := range tc.wantGroups {\n\t\t\tfor groupKey, want := range groups {\n\t\t\t\tgot := h.GetGroupEntries(t, r, qname, groupKey)\n\t\t\t\tif diff := cmp.Diff(want, got, h.IgnoreIDOpt, cmpopts.EquateEmpty()); diff != \"\" {\n\t\t\t\t\tt.Errorf(\"%s;\\nmismatch found in %q; (-want,+got)\\n%s\", tc.desc, base.GroupKey(qname, groupKey), diff)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n" }, { "path": "context.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"context\"\n\n\tasynqcontext \"github.com/hibiken/asynq/internal/context\"\n)\n\n// GetTaskID extracts a task ID from a context, if any.\n//\n// ID of a task is guaranteed to be unique.\n// ID of a task doesn't change if the task is being retried.\nfunc GetTaskID(ctx context.Context) (id string, ok bool) {\n\treturn asynqcontext.GetTaskID(ctx)\n}\n\n// GetRetryCount extracts retry count from a context, if any.\n//\n// Return value n indicates the number of times associated task has been\n// retried so far.\nfunc GetRetryCount(ctx context.Context) (n int, ok bool) {\n\treturn asynqcontext.GetRetryCount(ctx)\n}\n\n// GetMaxRetry extracts maximum retry from a context, if any.\n//\n// Return value n indicates the maximum number of times the associated task\n// can be retried if ProcessTask returns a non-nil error.\nfunc GetMaxRetry(ctx context.Context) (n int, ok bool) {\n\treturn asynqcontext.GetMaxRetry(ctx)\n}\n\n// GetQueueName extracts queue name from a context, if any.\n//\n// Return value queue indicates which queue the task was pulled from.\nfunc GetQueueName(ctx context.Context) (queue string, ok bool) {\n\treturn asynqcontext.GetQueueName(ctx)\n}\n" }, { "path": "doc.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\n/*\nPackage asynq provides a framework for Redis based distrubted task queue.\n\nAsynq uses Redis as a message broker. To connect to redis,\nspecify the connection using one of RedisConnOpt types.\n\n\tredisConnOpt = asynq.RedisClientOpt{\n\t Addr: \"127.0.0.1:6379\",\n\t Password: \"xxxxx\",\n\t DB: 2,\n\t}\n\nThe Client is used to enqueue a task.\n\n\tclient := asynq.NewClient(redisConnOpt)\n\n\t// Task is created with two parameters: its type and payload.\n\t// Payload data is simply an array of bytes. It can be encoded in JSON, Protocol Buffer, Gob, etc.\n\tb, err := json.Marshal(ExamplePayload{UserID: 42})\n\tif err != nil {\n\t log.Fatal(err)\n\t}\n\n\ttask := asynq.NewTask(\"example\", b)\n\n\t// Enqueue the task to be processed immediately.\n\tinfo, err := client.Enqueue(task)\n\n\t// Schedule the task to be processed after one minute.\n\tinfo, err = client.Enqueue(t, asynq.ProcessIn(1*time.Minute))\n\nThe Server is used to run the task processing workers with a given\nhandler.\n\n\tsrv := asynq.NewServer(redisConnOpt, asynq.Config{\n\t Concurrency: 10,\n\t})\n\n\tif err := srv.Run(handler); err != nil {\n\t log.Fatal(err)\n\t}\n\nHandler is an interface type with a method which\ntakes a task and returns an error. Handler should return nil if\nthe processing is successful, otherwise return a non-nil error.\nIf handler panics or returns a non-nil error, the task will be retried in the future.\n\nExample of a type that implements the Handler interface.\n\n\ttype TaskHandler struct {\n\t // ...\n\t}\n\n\tfunc (h *TaskHandler) ProcessTask(ctx context.Context, task *asynq.Task) error {\n\t switch task.Type {\n\t case \"example\":\n\t var data ExamplePayload\n\t if err := json.Unmarshal(task.Payload(), &data); err != nil {\n\t return err\n\t }\n\t // perform task with the data\n\n\t default:\n\t return fmt.Errorf(\"unexpected task type %q\", task.Type)\n\t }\n\t return nil\n\t}\n*/\npackage asynq\n" }, { "path": "example_test.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq_test\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\t\"os/signal\"\n\t\"time\"\n\n\t\"github.com/hibiken/asynq\"\n\t\"golang.org/x/sys/unix\"\n)\n\nfunc ExampleServer_Run() {\n\tsrv := asynq.NewServer(\n\t\tasynq.RedisClientOpt{Addr: \":6379\"},\n\t\tasynq.Config{Concurrency: 20},\n\t)\n\n\th := asynq.NewServeMux()\n\t// ... Register handlers\n\n\t// Run blocks and waits for os signal to terminate the program.\n\tif err := srv.Run(h); err != nil {\n\t\tlog.Fatal(err)\n\t}\n}\n\nfunc ExampleServer_Shutdown() {\n\tsrv := asynq.NewServer(\n\t\tasynq.RedisClientOpt{Addr: \":6379\"},\n\t\tasynq.Config{Concurrency: 20},\n\t)\n\n\th := asynq.NewServeMux()\n\t// ... Register handlers\n\n\tif err := srv.Start(h); err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tsigs := make(chan os.Signal, 1)\n\tsignal.Notify(sigs, unix.SIGTERM, unix.SIGINT)\n\t<-sigs // wait for termination signal\n\n\tsrv.Shutdown()\n}\n\nfunc ExampleServer_Stop() {\n\tsrv := asynq.NewServer(\n\t\tasynq.RedisClientOpt{Addr: \":6379\"},\n\t\tasynq.Config{Concurrency: 20},\n\t)\n\n\th := asynq.NewServeMux()\n\t// ... Register handlers\n\n\tif err := srv.Start(h); err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tsigs := make(chan os.Signal, 1)\n\tsignal.Notify(sigs, unix.SIGTERM, unix.SIGINT, unix.SIGTSTP)\n\t// Handle SIGTERM, SIGINT to exit the program.\n\t// Handle SIGTSTP to stop processing new tasks.\n\tfor {\n\t\ts := <-sigs\n\t\tif s == unix.SIGTSTP {\n\t\t\tsrv.Stop() // stop processing new tasks\n\t\t\tcontinue\n\t\t}\n\t\tbreak // received SIGTERM or SIGINT signal\n\t}\n\n\tsrv.Shutdown()\n}\n\nfunc ExampleScheduler() {\n\tscheduler := asynq.NewScheduler(\n\t\tasynq.RedisClientOpt{Addr: \":6379\"},\n\t\t&asynq.SchedulerOpts{Location: time.Local},\n\t)\n\n\tif _, err := scheduler.Register(\"* * * * *\", asynq.NewTask(\"task1\", nil)); err != nil {\n\t\tlog.Fatal(err)\n\t}\n\tif _, err := scheduler.Register(\"@every 30s\", asynq.NewTask(\"task2\", nil)); err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\t// Run blocks and waits for os signal to terminate the program.\n\tif err := scheduler.Run(); err != nil {\n\t\tlog.Fatal(err)\n\t}\n}\n\nfunc ExampleParseRedisURI() {\n\trconn, err := asynq.ParseRedisURI(\"redis://localhost:6379/10\")\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\tr, ok := rconn.(asynq.RedisClientOpt)\n\tif !ok {\n\t\tlog.Fatal(\"unexpected type\")\n\t}\n\tfmt.Println(r.Addr)\n\tfmt.Println(r.DB)\n\t// Output:\n\t// localhost:6379\n\t// 10\n}\n\nfunc ExampleResultWriter() {\n\t// ResultWriter is only accessible in Handler.\n\th := func(ctx context.Context, task *asynq.Task) error {\n\t\t// .. do task processing work\n\n\t\tres := []byte(\"task result data\")\n\t\tn, err := task.ResultWriter().Write(res) // implements io.Writer\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"failed to write task result: %w\", err)\n\t\t}\n\t\tlog.Printf(\" %d bytes written\", n)\n\t\treturn nil\n\t}\n\n\t_ = h\n}\n" }, { "path": "forwarder.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/log\"\n)\n\n// A forwarder is responsible for moving scheduled and retry tasks to pending state\n// so that the tasks get processed by the workers.\ntype forwarder struct {\n\tlogger *log.Logger\n\tbroker base.Broker\n\n\t// channel to communicate back to the long running \"forwarder\" goroutine.\n\tdone chan struct{}\n\n\t// list of queue names to check and enqueue.\n\tqueues []string\n\n\t// poll interval on average\n\tavgInterval time.Duration\n}\n\ntype forwarderParams struct {\n\tlogger *log.Logger\n\tbroker base.Broker\n\tqueues []string\n\tinterval time.Duration\n}\n\nfunc newForwarder(params forwarderParams) *forwarder {\n\treturn &forwarder{\n\t\tlogger: params.logger,\n\t\tbroker: params.broker,\n\t\tdone: make(chan struct{}),\n\t\tqueues: params.queues,\n\t\tavgInterval: params.interval,\n\t}\n}\n\nfunc (f *forwarder) shutdown() {\n\tf.logger.Debug(\"Forwarder shutting down...\")\n\t// Signal the forwarder goroutine to stop polling.\n\tf.done <- struct{}{}\n}\n\n// start starts the \"forwarder\" goroutine.\nfunc (f *forwarder) start(wg *sync.WaitGroup) {\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\ttimer := time.NewTimer(f.avgInterval)\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-f.done:\n\t\t\t\tf.logger.Debug(\"Forwarder done\")\n\t\t\t\treturn\n\t\t\tcase <-timer.C:\n\t\t\t\tf.exec()\n\t\t\t\ttimer.Reset(f.avgInterval)\n\t\t\t}\n\t\t}\n\t}()\n}\n\nfunc (f *forwarder) exec() {\n\tif err := f.broker.ForwardIfReady(f.queues...); err != nil {\n\t\tf.logger.Errorf(\"Failed to forward scheduled tasks: %v\", err)\n\t}\n}\n" }, { "path": "forwarder_test.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/google/go-cmp/cmp\"\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/rdb\"\n\th \"github.com/hibiken/asynq/internal/testutil\"\n)\n\nfunc TestForwarder(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\trdbClient := rdb.NewRDB(r)\n\tconst pollInterval = time.Second\n\ts := newForwarder(forwarderParams{\n\t\tlogger: testLogger,\n\t\tbroker: rdbClient,\n\t\tqueues: []string{\"default\", \"critical\"},\n\t\tinterval: pollInterval,\n\t})\n\tt1 := h.NewTaskMessageWithQueue(\"gen_thumbnail\", nil, \"default\")\n\tt2 := h.NewTaskMessageWithQueue(\"send_email\", nil, \"critical\")\n\tt3 := h.NewTaskMessageWithQueue(\"reindex\", nil, \"default\")\n\tt4 := h.NewTaskMessageWithQueue(\"sync\", nil, \"critical\")\n\tnow := time.Now()\n\n\ttests := []struct {\n\t\tinitScheduled map[string][]base.Z // scheduled queue initial state\n\t\tinitRetry map[string][]base.Z // retry queue initial state\n\t\tinitPending map[string][]*base.TaskMessage // default queue initial state\n\t\twait time.Duration // wait duration before checking for final state\n\t\twantScheduled map[string][]*base.TaskMessage // schedule queue final state\n\t\twantRetry map[string][]*base.TaskMessage // retry queue final state\n\t\twantPending map[string][]*base.TaskMessage // default queue final state\n\t}{\n\t\t{\n\t\t\tinitScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t1, Score: now.Add(time.Hour).Unix()}},\n\t\t\t\t\"critical\": {{Message: t2, Score: now.Add(-2 * time.Second).Unix()}},\n\t\t\t},\n\t\t\tinitRetry: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t3, Score: time.Now().Add(-500 * time.Millisecond).Unix()}},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\tinitPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {t4},\n\t\t\t},\n\t\t\twait: pollInterval * 2,\n\t\t\twantScheduled: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\twantRetry: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t3},\n\t\t\t\t\"critical\": {t2, t4},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinitScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: now.Unix()},\n\t\t\t\t\t{Message: t3, Score: now.Add(-500 * time.Millisecond).Unix()},\n\t\t\t\t},\n\t\t\t\t\"critical\": {\n\t\t\t\t\t{Message: t2, Score: now.Add(-2 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tinitRetry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\tinitPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {t4},\n\t\t\t},\n\t\t\twait: pollInterval * 2,\n\t\t\twantScheduled: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\twantRetry: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t3},\n\t\t\t\t\"critical\": {t2, t4},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r) // clean up db before each test case.\n\t\th.SeedAllScheduledQueues(t, r, tc.initScheduled) // initialize scheduled queue\n\t\th.SeedAllRetryQueues(t, r, tc.initRetry) // initialize retry queue\n\t\th.SeedAllPendingQueues(t, r, tc.initPending) // initialize default queue\n\n\t\tvar wg sync.WaitGroup\n\t\ts.start(&wg)\n\t\ttime.Sleep(tc.wait)\n\t\ts.shutdown()\n\n\t\tfor qname, want := range tc.wantScheduled {\n\t\t\tgotScheduled := h.GetScheduledMessages(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotScheduled, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q after running forwarder: (-want, +got)\\n%s\", base.ScheduledKey(qname), diff)\n\t\t\t}\n\t\t}\n\n\t\tfor qname, want := range tc.wantRetry {\n\t\t\tgotRetry := h.GetRetryMessages(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotRetry, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q after running forwarder: (-want, +got)\\n%s\", base.RetryKey(qname), diff)\n\t\t\t}\n\t\t}\n\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q after running forwarder: (-want, +got)\\n%s\", base.PendingKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n" }, { "path": "go.mod", "content": "module github.com/hibiken/asynq\n\ngo 1.24.0\n\nrequire (\n\tgithub.com/google/go-cmp v0.7.0\n\tgithub.com/google/uuid v1.6.0\n\tgithub.com/redis/go-redis/v9 v9.14.1\n\tgithub.com/robfig/cron/v3 v3.0.1\n\tgithub.com/spf13/cast v1.10.0\n\tgo.uber.org/goleak v1.3.0\n\tgolang.org/x/sys v0.37.0\n\tgolang.org/x/time v0.14.0\n\tgoogle.golang.org/protobuf v1.36.10\n)\n\nrequire (\n\tgithub.com/cespare/xxhash/v2 v2.3.0 // indirect\n\tgithub.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect\n)\n" }, { "path": "go.sum", "content": "github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=\ngithub.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=\ngithub.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=\ngithub.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=\ngithub.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=\ngithub.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=\ngithub.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=\ngithub.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=\ngithub.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=\ngithub.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=\ngithub.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=\ngithub.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=\ngithub.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=\ngithub.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=\ngithub.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=\ngithub.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=\ngithub.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=\ngithub.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=\ngithub.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=\ngithub.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=\ngithub.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/redis/go-redis/v9 v9.14.1 h1:nDCrEiJmfOWhD76xlaw+HXT0c9hfNWeXgl0vIRYSDvQ=\ngithub.com/redis/go-redis/v9 v9.14.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=\ngithub.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=\ngithub.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=\ngithub.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=\ngithub.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=\ngithub.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=\ngithub.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=\ngithub.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=\ngithub.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=\ngo.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=\ngo.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=\ngolang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=\ngolang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=\ngolang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=\ngolang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=\ngoogle.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=\ngoogle.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=\ngopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=\ngopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\n" }, { "path": "healthcheck.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/log\"\n)\n\n// healthchecker is responsible for pinging broker periodically\n// and call user provided HeathCheckFunc with the ping result.\ntype healthchecker struct {\n\tlogger *log.Logger\n\tbroker base.Broker\n\n\t// channel to communicate back to the long running \"healthchecker\" goroutine.\n\tdone chan struct{}\n\n\t// interval between healthchecks.\n\tinterval time.Duration\n\n\t// function to call periodically.\n\thealthcheckFunc func(error)\n}\n\ntype healthcheckerParams struct {\n\tlogger *log.Logger\n\tbroker base.Broker\n\tinterval time.Duration\n\thealthcheckFunc func(error)\n}\n\nfunc newHealthChecker(params healthcheckerParams) *healthchecker {\n\treturn &healthchecker{\n\t\tlogger: params.logger,\n\t\tbroker: params.broker,\n\t\tdone: make(chan struct{}),\n\t\tinterval: params.interval,\n\t\thealthcheckFunc: params.healthcheckFunc,\n\t}\n}\n\nfunc (hc *healthchecker) shutdown() {\n\tif hc.healthcheckFunc == nil {\n\t\treturn\n\t}\n\n\thc.logger.Debug(\"Healthchecker shutting down...\")\n\t// Signal the healthchecker goroutine to stop.\n\thc.done <- struct{}{}\n}\n\nfunc (hc *healthchecker) start(wg *sync.WaitGroup) {\n\tif hc.healthcheckFunc == nil {\n\t\treturn\n\t}\n\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\ttimer := time.NewTimer(hc.interval)\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-hc.done:\n\t\t\t\thc.logger.Debug(\"Healthchecker done\")\n\t\t\t\ttimer.Stop()\n\t\t\t\treturn\n\t\t\tcase <-timer.C:\n\t\t\t\terr := hc.broker.Ping()\n\t\t\t\thc.healthcheckFunc(err)\n\t\t\t\ttimer.Reset(hc.interval)\n\t\t\t}\n\t\t}\n\t}()\n}\n" }, { "path": "healthcheck_test.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/hibiken/asynq/internal/rdb\"\n\t\"github.com/hibiken/asynq/internal/testbroker\"\n)\n\nfunc TestHealthChecker(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\trdbClient := rdb.NewRDB(r)\n\n\tvar (\n\t\t// mu guards called and e variables.\n\t\tmu sync.Mutex\n\t\tcalled int\n\t\te error\n\t)\n\tcheckFn := func(err error) {\n\t\tmu.Lock()\n\t\tdefer mu.Unlock()\n\t\tcalled++\n\t\te = err\n\t}\n\n\thc := newHealthChecker(healthcheckerParams{\n\t\tlogger: testLogger,\n\t\tbroker: rdbClient,\n\t\tinterval: 1 * time.Second,\n\t\thealthcheckFunc: checkFn,\n\t})\n\n\thc.start(&sync.WaitGroup{})\n\n\ttime.Sleep(2 * time.Second)\n\n\tmu.Lock()\n\tif called == 0 {\n\t\tt.Errorf(\"Healthchecker did not call the provided HealthCheckFunc\")\n\t}\n\tif e != nil {\n\t\tt.Errorf(\"HealthCheckFunc was called with non-nil error: %v\", e)\n\t}\n\tmu.Unlock()\n\n\thc.shutdown()\n}\n\nfunc TestHealthCheckerWhenRedisDown(t *testing.T) {\n\t// Make sure that healthchecker goroutine doesn't panic\n\t// if it cannot connect to redis.\n\tdefer func() {\n\t\tif r := recover(); r != nil {\n\t\t\tt.Errorf(\"panic occurred: %v\", r)\n\t\t}\n\t}()\n\tr := rdb.NewRDB(setup(t))\n\tdefer r.Close()\n\ttestBroker := testbroker.NewTestBroker(r)\n\tvar (\n\t\t// mu guards called and e variables.\n\t\tmu sync.Mutex\n\t\tcalled int\n\t\te error\n\t)\n\tcheckFn := func(err error) {\n\t\tmu.Lock()\n\t\tdefer mu.Unlock()\n\t\tcalled++\n\t\te = err\n\t}\n\n\thc := newHealthChecker(healthcheckerParams{\n\t\tlogger: testLogger,\n\t\tbroker: testBroker,\n\t\tinterval: 1 * time.Second,\n\t\thealthcheckFunc: checkFn,\n\t})\n\n\ttestBroker.Sleep()\n\thc.start(&sync.WaitGroup{})\n\n\ttime.Sleep(2 * time.Second)\n\n\tmu.Lock()\n\tif called == 0 {\n\t\tt.Errorf(\"Healthchecker did not call the provided HealthCheckFunc\")\n\t}\n\tif e == nil {\n\t\tt.Errorf(\"HealthCheckFunc was called with nil; want non-nil error\")\n\t}\n\tmu.Unlock()\n\n\thc.shutdown()\n}\n" }, { "path": "heartbeat.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"os\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/google/uuid\"\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/log\"\n\t\"github.com/hibiken/asynq/internal/timeutil\"\n)\n\n// heartbeater is responsible for writing process info to redis periodically to\n// indicate that the background worker process is up.\ntype heartbeater struct {\n\tlogger *log.Logger\n\tbroker base.Broker\n\tclock timeutil.Clock\n\n\t// channel to communicate back to the long running \"heartbeater\" goroutine.\n\tdone chan struct{}\n\n\t// interval between heartbeats.\n\tinterval time.Duration\n\n\t// following fields are initialized at construction time and are immutable.\n\thost string\n\tpid int\n\tserverID string\n\tconcurrency int\n\tqueues map[string]int\n\tstrictPriority bool\n\n\t// following fields are mutable and should be accessed only by the\n\t// heartbeater goroutine. In other words, confine these variables\n\t// to this goroutine only.\n\tstarted time.Time\n\tworkers map[string]*workerInfo\n\n\t// state is shared with other goroutine but is concurrency safe.\n\tstate *serverState\n\n\t// channels to receive updates on active workers.\n\tstarting <-chan *workerInfo\n\tfinished <-chan *base.TaskMessage\n}\n\ntype heartbeaterParams struct {\n\tlogger *log.Logger\n\tbroker base.Broker\n\tinterval time.Duration\n\tconcurrency int\n\tqueues map[string]int\n\tstrictPriority bool\n\tstate *serverState\n\tstarting <-chan *workerInfo\n\tfinished <-chan *base.TaskMessage\n}\n\nfunc newHeartbeater(params heartbeaterParams) *heartbeater {\n\thost, err := os.Hostname()\n\tif err != nil {\n\t\thost = \"unknown-host\"\n\t}\n\n\treturn &heartbeater{\n\t\tlogger: params.logger,\n\t\tbroker: params.broker,\n\t\tclock: timeutil.NewRealClock(),\n\t\tdone: make(chan struct{}),\n\t\tinterval: params.interval,\n\n\t\thost: host,\n\t\tpid: os.Getpid(),\n\t\tserverID: uuid.New().String(),\n\t\tconcurrency: params.concurrency,\n\t\tqueues: params.queues,\n\t\tstrictPriority: params.strictPriority,\n\n\t\tstate: params.state,\n\t\tworkers: make(map[string]*workerInfo),\n\t\tstarting: params.starting,\n\t\tfinished: params.finished,\n\t}\n}\n\nfunc (h *heartbeater) shutdown() {\n\th.logger.Debug(\"Heartbeater shutting down...\")\n\t// Signal the heartbeater goroutine to stop.\n\th.done <- struct{}{}\n}\n\n// A workerInfo holds an active worker information.\ntype workerInfo struct {\n\t// the task message the worker is processing.\n\tmsg *base.TaskMessage\n\t// the time the worker has started processing the message.\n\tstarted time.Time\n\t// deadline the worker has to finish processing the task by.\n\tdeadline time.Time\n\t// lease the worker holds for the task.\n\tlease *base.Lease\n}\n\nfunc (h *heartbeater) start(wg *sync.WaitGroup) {\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\n\t\th.started = h.clock.Now()\n\n\t\th.beat()\n\n\t\ttimer := time.NewTimer(h.interval)\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-h.done:\n\t\t\t\tif err := h.broker.ClearServerState(h.host, h.pid, h.serverID); err != nil {\n\t\t\t\t\th.logger.Errorf(\"Failed to clear server state: %v\", err)\n\t\t\t\t}\n\t\t\t\th.logger.Debug(\"Heartbeater done\")\n\t\t\t\ttimer.Stop()\n\t\t\t\treturn\n\n\t\t\tcase <-timer.C:\n\t\t\t\th.beat()\n\t\t\t\ttimer.Reset(h.interval)\n\n\t\t\tcase w := <-h.starting:\n\t\t\t\th.workers[w.msg.ID] = w\n\n\t\t\tcase msg := <-h.finished:\n\t\t\t\tdelete(h.workers, msg.ID)\n\t\t\t}\n\t\t}\n\t}()\n}\n\n// beat extends lease for workers and writes server/worker info to redis.\nfunc (h *heartbeater) beat() {\n\th.state.mu.Lock()\n\tsrvStatus := h.state.value.String()\n\th.state.mu.Unlock()\n\n\tinfo := base.ServerInfo{\n\t\tHost: h.host,\n\t\tPID: h.pid,\n\t\tServerID: h.serverID,\n\t\tConcurrency: h.concurrency,\n\t\tQueues: h.queues,\n\t\tStrictPriority: h.strictPriority,\n\t\tStatus: srvStatus,\n\t\tStarted: h.started,\n\t\tActiveWorkerCount: len(h.workers),\n\t}\n\n\tvar ws []*base.WorkerInfo\n\tidsByQueue := make(map[string][]string)\n\tfor id, w := range h.workers {\n\t\tws = append(ws, &base.WorkerInfo{\n\t\t\tHost: h.host,\n\t\t\tPID: h.pid,\n\t\t\tServerID: h.serverID,\n\t\t\tID: id,\n\t\t\tType: w.msg.Type,\n\t\t\tQueue: w.msg.Queue,\n\t\t\tPayload: w.msg.Payload,\n\t\t\tStarted: w.started,\n\t\t\tDeadline: w.deadline,\n\t\t})\n\t\t// Check lease before adding to the set to make sure not to extend the lease if the lease is already expired.\n\t\tif w.lease.IsValid() {\n\t\t\tidsByQueue[w.msg.Queue] = append(idsByQueue[w.msg.Queue], id)\n\t\t} else {\n\t\t\tw.lease.NotifyExpiration() // notify processor if the lease is expired\n\t\t}\n\t}\n\n\t// Note: Set TTL to be long enough so that it won't expire before we write again\n\t// and short enough to expire quickly once the process is shut down or killed.\n\tif err := h.broker.WriteServerState(&info, ws, h.interval*2); err != nil {\n\t\th.logger.Errorf(\"Failed to write server state data: %v\", err)\n\t}\n\n\tfor qname, ids := range idsByQueue {\n\t\texpirationTime, err := h.broker.ExtendLease(qname, ids...)\n\t\tif err != nil {\n\t\t\th.logger.Errorf(\"Failed to extend lease for tasks %v: %v\", ids, err)\n\t\t\tcontinue\n\t\t}\n\t\tfor _, id := range ids {\n\t\t\tif l := h.workers[id].lease; !l.Reset(expirationTime) {\n\t\t\t\th.logger.Warnf(\"Lease reset failed for %s; lease deadline: %v\", id, l.Deadline())\n\t\t\t}\n\t\t}\n\t}\n}\n" }, { "path": "heartbeat_test.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"context\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/google/go-cmp/cmp\"\n\t\"github.com/google/go-cmp/cmp/cmpopts\"\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/rdb\"\n\t\"github.com/hibiken/asynq/internal/testbroker\"\n\th \"github.com/hibiken/asynq/internal/testutil\"\n\t\"github.com/hibiken/asynq/internal/timeutil\"\n)\n\n// Test goes through a few phases.\n//\n// Phase1: Simulate Server startup; Simulate starting tasks listed in startedWorkers\n// Phase2: Simulate finishing tasks listed in finishedTasks\n// Phase3: Simulate Server shutdown;\nfunc TestHeartbeater(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\trdbClient := rdb.NewRDB(r)\n\n\tnow := time.Now()\n\tconst elapsedTime = 10 * time.Second // simulated time elapsed between phase1 and phase2\n\n\tclock := timeutil.NewSimulatedClock(time.Time{}) // time will be set in each test\n\n\tt1 := h.NewTaskMessageWithQueue(\"task1\", nil, \"default\")\n\tt2 := h.NewTaskMessageWithQueue(\"task2\", nil, \"default\")\n\tt3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"default\")\n\tt4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"custom\")\n\tt5 := h.NewTaskMessageWithQueue(\"task5\", nil, \"custom\")\n\tt6 := h.NewTaskMessageWithQueue(\"task6\", nil, \"default\")\n\n\t// Note: intentionally set to time less than now.Add(rdb.LeaseDuration) to test lease extension is working.\n\tlease1 := h.NewLeaseWithClock(now.Add(10*time.Second), clock)\n\tlease2 := h.NewLeaseWithClock(now.Add(10*time.Second), clock)\n\tlease3 := h.NewLeaseWithClock(now.Add(10*time.Second), clock)\n\tlease4 := h.NewLeaseWithClock(now.Add(10*time.Second), clock)\n\tlease5 := h.NewLeaseWithClock(now.Add(10*time.Second), clock)\n\tlease6 := h.NewLeaseWithClock(now.Add(10*time.Second), clock)\n\n\ttests := []struct {\n\t\tdesc string\n\n\t\t// Interval between heartbeats.\n\t\tinterval time.Duration\n\n\t\t// Server info.\n\t\thost string\n\t\tpid int\n\t\tqueues map[string]int\n\t\tconcurrency int\n\n\t\tactive map[string][]*base.TaskMessage // initial active set state\n\t\tlease map[string][]base.Z // initial lease set state\n\t\twantLease1 map[string][]base.Z // expected lease set state after starting all startedWorkers\n\t\twantLease2 map[string][]base.Z // expected lease set state after finishing all finishedTasks\n\t\tstartedWorkers []*workerInfo // workerInfo to send via the started channel\n\t\tfinishedTasks []*base.TaskMessage // tasks to send via the finished channel\n\n\t\tstartTime time.Time // simulated start time\n\t\telapsedTime time.Duration // simulated time elapsed between starting and finishing processing tasks\n\t}{\n\t\t{\n\t\t\tdesc: \"With single queue\",\n\t\t\tinterval: 2 * time.Second,\n\t\t\thost: \"localhost\",\n\t\t\tpid: 45678,\n\t\t\tqueues: map[string]int{\"default\": 1},\n\t\t\tconcurrency: 10,\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t2, t3},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t\t{Message: t2, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t\t{Message: t3, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tstartedWorkers: []*workerInfo{\n\t\t\t\t{msg: t1, started: now, deadline: now.Add(2 * time.Minute), lease: lease1},\n\t\t\t\t{msg: t2, started: now, deadline: now.Add(2 * time.Minute), lease: lease2},\n\t\t\t\t{msg: t3, started: now, deadline: now.Add(2 * time.Minute), lease: lease3},\n\t\t\t},\n\t\t\tfinishedTasks: []*base.TaskMessage{t1, t2},\n\t\t\twantLease1: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: now.Add(rdb.LeaseDuration).Unix()},\n\t\t\t\t\t{Message: t2, Score: now.Add(rdb.LeaseDuration).Unix()},\n\t\t\t\t\t{Message: t3, Score: now.Add(rdb.LeaseDuration).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\twantLease2: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t3, Score: now.Add(elapsedTime).Add(rdb.LeaseDuration).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tstartTime: now,\n\t\t\telapsedTime: elapsedTime,\n\t\t},\n\t\t{\n\t\t\tdesc: \"With multiple queue\",\n\t\t\tinterval: 2 * time.Second,\n\t\t\thost: \"localhost\",\n\t\t\tpid: 45678,\n\t\t\tqueues: map[string]int{\"default\": 1, \"custom\": 2},\n\t\t\tconcurrency: 10,\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t6},\n\t\t\t\t\"custom\": {t4, t5},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t6, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: t4, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t\t{Message: t5, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tstartedWorkers: []*workerInfo{\n\t\t\t\t{msg: t6, started: now, deadline: now.Add(2 * time.Minute), lease: lease6},\n\t\t\t\t{msg: t4, started: now, deadline: now.Add(2 * time.Minute), lease: lease4},\n\t\t\t\t{msg: t5, started: now, deadline: now.Add(2 * time.Minute), lease: lease5},\n\t\t\t},\n\t\t\tfinishedTasks: []*base.TaskMessage{t6, t5},\n\t\t\twantLease1: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t6, Score: now.Add(rdb.LeaseDuration).Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: t4, Score: now.Add(rdb.LeaseDuration).Unix()},\n\t\t\t\t\t{Message: t5, Score: now.Add(rdb.LeaseDuration).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\twantLease2: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: t4, Score: now.Add(elapsedTime).Add(rdb.LeaseDuration).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tstartTime: now,\n\t\t\telapsedTime: elapsedTime,\n\t\t},\n\t}\n\n\ttimeCmpOpt := cmpopts.EquateApproxTime(10 * time.Millisecond)\n\tignoreOpt := cmpopts.IgnoreUnexported(base.ServerInfo{})\n\tignoreFieldOpt := cmpopts.IgnoreFields(base.ServerInfo{}, \"ServerID\")\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllActiveQueues(t, r, tc.active)\n\t\th.SeedAllLease(t, r, tc.lease)\n\n\t\tclock.SetTime(tc.startTime)\n\t\trdbClient.SetClock(clock)\n\n\t\tsrvState := &serverState{}\n\t\tstartingCh := make(chan *workerInfo)\n\t\tfinishedCh := make(chan *base.TaskMessage)\n\t\thb := newHeartbeater(heartbeaterParams{\n\t\t\tlogger: testLogger,\n\t\t\tbroker: rdbClient,\n\t\t\tinterval: tc.interval,\n\t\t\tconcurrency: tc.concurrency,\n\t\t\tqueues: tc.queues,\n\t\t\tstrictPriority: false,\n\t\t\tstate: srvState,\n\t\t\tstarting: startingCh,\n\t\t\tfinished: finishedCh,\n\t\t})\n\t\thb.clock = clock\n\n\t\t// Change host and pid fields for testing purpose.\n\t\thb.host = tc.host\n\t\thb.pid = tc.pid\n\n\t\t//===================\n\t\t// Start Phase1\n\t\t//===================\n\n\t\tsrvState.mu.Lock()\n\t\tsrvState.value = srvStateActive // simulating Server.Start\n\t\tsrvState.mu.Unlock()\n\n\t\tvar wg sync.WaitGroup\n\t\thb.start(&wg)\n\n\t\t// Simulate processor starting to work on tasks.\n\t\tfor _, w := range tc.startedWorkers {\n\t\t\tstartingCh <- w\n\t\t}\n\n\t\t// Wait for heartbeater to write to redis\n\t\ttime.Sleep(tc.interval * 2)\n\n\t\tss, err := rdbClient.ListServers()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s: could not read server info from redis: %v\", tc.desc, err)\n\t\t\thb.shutdown()\n\t\t\tcontinue\n\t\t}\n\n\t\tif len(ss) != 1 {\n\t\t\tt.Errorf(\"%s: (*RDB).ListServers returned %d server info, want 1\", tc.desc, len(ss))\n\t\t\thb.shutdown()\n\t\t\tcontinue\n\t\t}\n\n\t\twantInfo := &base.ServerInfo{\n\t\t\tHost: tc.host,\n\t\t\tPID: tc.pid,\n\t\t\tQueues: tc.queues,\n\t\t\tConcurrency: tc.concurrency,\n\t\t\tStarted: now,\n\t\t\tStatus: \"active\",\n\t\t\tActiveWorkerCount: len(tc.startedWorkers),\n\t\t}\n\t\tif diff := cmp.Diff(wantInfo, ss[0], timeCmpOpt, ignoreOpt, ignoreFieldOpt); diff != \"\" {\n\t\t\tt.Errorf(\"%s: redis stored server status %+v, want %+v; (-want, +got)\\n%s\", tc.desc, ss[0], wantInfo, diff)\n\t\t\thb.shutdown()\n\t\t\tcontinue\n\t\t}\n\n\t\tfor qname, wantLease := range tc.wantLease1 {\n\t\t\tgotLease := h.GetLeaseEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(wantLease, gotLease, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s: mismatch found in %q: (-want,+got):\\n%s\", tc.desc, base.LeaseKey(qname), diff)\n\t\t\t}\n\t\t}\n\n\t\tfor _, w := range tc.startedWorkers {\n\t\t\tif want := now.Add(rdb.LeaseDuration); w.lease.Deadline() != want {\n\t\t\t\tt.Errorf(\"%s: lease deadline for %v is set to %v, want %v\", tc.desc, w.msg, w.lease.Deadline(), want)\n\t\t\t}\n\t\t}\n\n\t\t//===================\n\t\t// Start Phase2\n\t\t//===================\n\n\t\tclock.AdvanceTime(tc.elapsedTime)\n\t\t// Simulate processor finished processing tasks.\n\t\tfor _, msg := range tc.finishedTasks {\n\t\t\tif err := rdbClient.Done(context.Background(), msg); err != nil {\n\t\t\t\tt.Fatalf(\"RDB.Done failed: %v\", err)\n\t\t\t}\n\t\t\tfinishedCh <- msg\n\t\t}\n\t\t// Wait for heartbeater to write to redis\n\t\ttime.Sleep(tc.interval * 2)\n\n\t\tfor qname, wantLease := range tc.wantLease2 {\n\t\t\tgotLease := h.GetLeaseEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(wantLease, gotLease, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s: mismatch found in %q: (-want,+got):\\n%s\", tc.desc, base.LeaseKey(qname), diff)\n\t\t\t}\n\t\t}\n\n\t\t//===================\n\t\t// Start Phase3\n\t\t//===================\n\n\t\t// Server state change; simulating Server.Shutdown\n\t\tsrvState.mu.Lock()\n\t\tsrvState.value = srvStateClosed\n\t\tsrvState.mu.Unlock()\n\n\t\t// Wait for heartbeater to write to redis\n\t\ttime.Sleep(tc.interval * 2)\n\n\t\twantInfo = &base.ServerInfo{\n\t\t\tHost: tc.host,\n\t\t\tPID: tc.pid,\n\t\t\tQueues: tc.queues,\n\t\t\tConcurrency: tc.concurrency,\n\t\t\tStarted: now,\n\t\t\tStatus: \"closed\",\n\t\t\tActiveWorkerCount: len(tc.startedWorkers) - len(tc.finishedTasks),\n\t\t}\n\t\tss, err = rdbClient.ListServers()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s: could not read server status from redis: %v\", tc.desc, err)\n\t\t\thb.shutdown()\n\t\t\tcontinue\n\t\t}\n\n\t\tif len(ss) != 1 {\n\t\t\tt.Errorf(\"%s: (*RDB).ListServers returned %d server info, want 1\", tc.desc, len(ss))\n\t\t\thb.shutdown()\n\t\t\tcontinue\n\t\t}\n\n\t\tif diff := cmp.Diff(wantInfo, ss[0], timeCmpOpt, ignoreOpt, ignoreFieldOpt); diff != \"\" {\n\t\t\tt.Errorf(\"%s: redis stored process status %+v, want %+v; (-want, +got)\\n%s\", tc.desc, ss[0], wantInfo, diff)\n\t\t\thb.shutdown()\n\t\t\tcontinue\n\t\t}\n\n\t\thb.shutdown()\n\t}\n}\n\nfunc TestHeartbeaterWithRedisDown(t *testing.T) {\n\t// Make sure that heartbeater goroutine doesn't panic\n\t// if it cannot connect to redis.\n\tdefer func() {\n\t\tif r := recover(); r != nil {\n\t\t\tt.Errorf(\"panic occurred: %v\", r)\n\t\t}\n\t}()\n\tr := rdb.NewRDB(setup(t))\n\tdefer r.Close()\n\ttestBroker := testbroker.NewTestBroker(r)\n\tstate := &serverState{value: srvStateActive}\n\thb := newHeartbeater(heartbeaterParams{\n\t\tlogger: testLogger,\n\t\tbroker: testBroker,\n\t\tinterval: time.Second,\n\t\tconcurrency: 10,\n\t\tqueues: map[string]int{\"default\": 1},\n\t\tstrictPriority: false,\n\t\tstate: state,\n\t\tstarting: make(chan *workerInfo),\n\t\tfinished: make(chan *base.TaskMessage),\n\t})\n\n\ttestBroker.Sleep()\n\tvar wg sync.WaitGroup\n\thb.start(&wg)\n\n\t// wait for heartbeater to try writing data to redis\n\ttime.Sleep(2 * time.Second)\n\n\thb.shutdown()\n}\n" }, { "path": "inspector.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/errors\"\n\t\"github.com/hibiken/asynq/internal/rdb\"\n\t\"github.com/redis/go-redis/v9\"\n)\n\n// Inspector is a client interface to inspect and mutate the state of\n// queues and tasks.\ntype Inspector struct {\n\trdb *rdb.RDB\n\t// When an Inspector has been created with an existing Redis connection, we do\n\t// not want to close it.\n\tsharedConnection bool\n}\n\n// New returns a new instance of Inspector.\nfunc NewInspector(r RedisConnOpt) *Inspector {\n\tc, ok := r.MakeRedisClient().(redis.UniversalClient)\n\tif !ok {\n\t\tpanic(fmt.Sprintf(\"inspeq: unsupported RedisConnOpt type %T\", r))\n\t}\n\tinspector := NewInspectorFromRedisClient(c)\n\tinspector.sharedConnection = false\n\treturn inspector\n}\n\n// NewInspectorFromRedisClient returns a new instance of Inspector given a redis.UniversalClient\n// Warning: The underlying redis connection pool will not be closed by Asynq, you are responsible for closing it.\nfunc NewInspectorFromRedisClient(c redis.UniversalClient) *Inspector {\n\treturn &Inspector{\n\t\trdb: rdb.NewRDB(c),\n\t\tsharedConnection: true,\n\t}\n}\n\n// Close closes the connection with redis.\nfunc (i *Inspector) Close() error {\n\tif i.sharedConnection {\n\t\treturn fmt.Errorf(\"redis connection is shared so the Inspector can't be closed through asynq\")\n\t}\n\treturn i.rdb.Close()\n}\n\n// Queues returns a list of all queue names.\nfunc (i *Inspector) Queues() ([]string, error) {\n\treturn i.rdb.AllQueues()\n}\n\n// Groups returns a list of all groups within the given queue.\nfunc (i *Inspector) Groups(queue string) ([]*GroupInfo, error) {\n\tstats, err := i.rdb.GroupStats(queue)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tvar res []*GroupInfo\n\tfor _, s := range stats {\n\t\tres = append(res, &GroupInfo{\n\t\t\tGroup: s.Group,\n\t\t\tSize: s.Size,\n\t\t})\n\t}\n\treturn res, nil\n}\n\n// GroupInfo represents a state of a group at a certain time.\ntype GroupInfo struct {\n\t// Name of the group.\n\tGroup string\n\n\t// Size is the total number of tasks in the group.\n\tSize int\n}\n\n// QueueInfo represents a state of a queue at a certain time.\ntype QueueInfo struct {\n\t// Name of the queue.\n\tQueue string\n\n\t// Total number of bytes that the queue and its tasks require to be stored in redis.\n\t// It is an approximate memory usage value in bytes since the value is computed by sampling.\n\tMemoryUsage int64\n\n\t// Latency of the queue, measured by the oldest pending task in the queue.\n\tLatency time.Duration\n\n\t// Size is the total number of tasks in the queue.\n\t// The value is the sum of Pending, Active, Scheduled, Retry, Aggregating and Archived.\n\tSize int\n\n\t// Groups is the total number of groups in the queue.\n\tGroups int\n\n\t// Number of pending tasks.\n\tPending int\n\t// Number of active tasks.\n\tActive int\n\t// Number of scheduled tasks.\n\tScheduled int\n\t// Number of retry tasks.\n\tRetry int\n\t// Number of archived tasks.\n\tArchived int\n\t// Number of stored completed tasks.\n\tCompleted int\n\t// Number of aggregating tasks.\n\tAggregating int\n\n\t// Total number of tasks being processed within the given date (counter resets daily).\n\t// The number includes both succeeded and failed tasks.\n\tProcessed int\n\t// Total number of tasks failed to be processed within the given date (counter resets daily).\n\tFailed int\n\n\t// Total number of tasks processed (cumulative).\n\tProcessedTotal int\n\t// Total number of tasks failed (cumulative).\n\tFailedTotal int\n\n\t// Paused indicates whether the queue is paused.\n\t// If true, tasks in the queue will not be processed.\n\tPaused bool\n\n\t// Time when this queue info snapshot was taken.\n\tTimestamp time.Time\n}\n\n// GetQueueInfo returns current information of the given queue.\nfunc (i *Inspector) GetQueueInfo(queue string) (*QueueInfo, error) {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn nil, err\n\t}\n\tstats, err := i.rdb.CurrentStats(queue)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &QueueInfo{\n\t\tQueue: stats.Queue,\n\t\tMemoryUsage: stats.MemoryUsage,\n\t\tLatency: stats.Latency,\n\t\tSize: stats.Size,\n\t\tGroups: stats.Groups,\n\t\tPending: stats.Pending,\n\t\tActive: stats.Active,\n\t\tScheduled: stats.Scheduled,\n\t\tRetry: stats.Retry,\n\t\tArchived: stats.Archived,\n\t\tCompleted: stats.Completed,\n\t\tAggregating: stats.Aggregating,\n\t\tProcessed: stats.Processed,\n\t\tFailed: stats.Failed,\n\t\tProcessedTotal: stats.ProcessedTotal,\n\t\tFailedTotal: stats.FailedTotal,\n\t\tPaused: stats.Paused,\n\t\tTimestamp: stats.Timestamp,\n\t}, nil\n}\n\n// DailyStats holds aggregate data for a given day for a given queue.\ntype DailyStats struct {\n\t// Name of the queue.\n\tQueue string\n\t// Total number of tasks being processed during the given date.\n\t// The number includes both succeeded and failed tasks.\n\tProcessed int\n\t// Total number of tasks failed to be processed during the given date.\n\tFailed int\n\t// Date this stats was taken.\n\tDate time.Time\n}\n\n// History returns a list of stats from the last n days.\nfunc (i *Inspector) History(queue string, n int) ([]*DailyStats, error) {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn nil, err\n\t}\n\tstats, err := i.rdb.HistoricalStats(queue, n)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tvar res []*DailyStats\n\tfor _, s := range stats {\n\t\tres = append(res, &DailyStats{\n\t\t\tQueue: s.Queue,\n\t\t\tProcessed: s.Processed,\n\t\t\tFailed: s.Failed,\n\t\t\tDate: s.Time,\n\t\t})\n\t}\n\treturn res, nil\n}\n\nvar (\n\t// ErrQueueNotFound indicates that the specified queue does not exist.\n\tErrQueueNotFound = errors.New(\"queue not found\")\n\n\t// ErrQueueNotEmpty indicates that the specified queue is not empty.\n\tErrQueueNotEmpty = errors.New(\"queue is not empty\")\n\n\t// ErrTaskNotFound indicates that the specified task cannot be found in the queue.\n\tErrTaskNotFound = errors.New(\"task not found\")\n)\n\n// DeleteQueue removes the specified queue.\n//\n// If force is set to true, DeleteQueue will remove the queue regardless of\n// the queue size as long as no tasks are active in the queue.\n// If force is set to false, DeleteQueue will remove the queue only if\n// the queue is empty.\n//\n// If the specified queue does not exist, DeleteQueue returns ErrQueueNotFound.\n// If force is set to false and the specified queue is not empty, DeleteQueue\n// returns ErrQueueNotEmpty.\nfunc (i *Inspector) DeleteQueue(queue string, force bool) error {\n\terr := i.rdb.RemoveQueue(queue, force)\n\tif errors.IsQueueNotFound(err) {\n\t\treturn fmt.Errorf(\"%w: queue=%q\", ErrQueueNotFound, queue)\n\t}\n\tif errors.IsQueueNotEmpty(err) {\n\t\treturn fmt.Errorf(\"%w: queue=%q\", ErrQueueNotEmpty, queue)\n\t}\n\treturn err\n}\n\n// GetTaskInfo retrieves task information given a task id and queue name.\n//\n// Returns an error wrapping ErrQueueNotFound if a queue with the given name doesn't exist.\n// Returns an error wrapping ErrTaskNotFound if a task with the given id doesn't exist in the queue.\nfunc (i *Inspector) GetTaskInfo(queue, id string) (*TaskInfo, error) {\n\tinfo, err := i.rdb.GetTaskInfo(queue, id)\n\tswitch {\n\tcase errors.IsQueueNotFound(err):\n\t\treturn nil, fmt.Errorf(\"asynq: %w\", ErrQueueNotFound)\n\tcase errors.IsTaskNotFound(err):\n\t\treturn nil, fmt.Errorf(\"asynq: %w\", ErrTaskNotFound)\n\tcase err != nil:\n\t\treturn nil, fmt.Errorf(\"asynq: %w\", err)\n\t}\n\treturn newTaskInfo(info.Message, info.State, info.NextProcessAt, info.Result), nil\n}\n\n// ListOption specifies behavior of list operation.\ntype ListOption interface{}\n\n// Internal list option representations.\ntype (\n\tpageSizeOpt int\n\tpageNumOpt int\n)\n\ntype listOption struct {\n\tpageSize int\n\tpageNum int\n}\n\nconst (\n\t// Page size used by default in list operation.\n\tdefaultPageSize = 30\n\n\t// Page number used by default in list operation.\n\tdefaultPageNum = 1\n)\n\nfunc composeListOptions(opts ...ListOption) listOption {\n\tres := listOption{\n\t\tpageSize: defaultPageSize,\n\t\tpageNum: defaultPageNum,\n\t}\n\tfor _, opt := range opts {\n\t\tswitch opt := opt.(type) {\n\t\tcase pageSizeOpt:\n\t\t\tres.pageSize = int(opt)\n\t\tcase pageNumOpt:\n\t\t\tres.pageNum = int(opt)\n\t\tdefault:\n\t\t\t// ignore unexpected option\n\t\t}\n\t}\n\treturn res\n}\n\n// PageSize returns an option to specify the page size for list operation.\n//\n// Negative page size is treated as zero.\nfunc PageSize(n int) ListOption {\n\tif n < 0 {\n\t\tn = 0\n\t}\n\treturn pageSizeOpt(n)\n}\n\n// Page returns an option to specify the page number for list operation.\n// The value 1 fetches the first page.\n//\n// Negative page number is treated as one.\nfunc Page(n int) ListOption {\n\tif n < 0 {\n\t\tn = 1\n\t}\n\treturn pageNumOpt(n)\n}\n\n// ListPendingTasks retrieves pending tasks from the specified queue.\n//\n// By default, it retrieves the first 30 tasks.\nfunc (i *Inspector) ListPendingTasks(queue string, opts ...ListOption) ([]*TaskInfo, error) {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn nil, fmt.Errorf(\"asynq: %w\", err)\n\t}\n\topt := composeListOptions(opts...)\n\tpgn := rdb.Pagination{Size: opt.pageSize, Page: opt.pageNum - 1}\n\tinfos, err := i.rdb.ListPending(queue, pgn)\n\tswitch {\n\tcase errors.IsQueueNotFound(err):\n\t\treturn nil, fmt.Errorf(\"asynq: %w\", ErrQueueNotFound)\n\tcase err != nil:\n\t\treturn nil, fmt.Errorf(\"asynq: %w\", err)\n\t}\n\tvar tasks []*TaskInfo\n\tfor _, i := range infos {\n\t\ttasks = append(tasks, newTaskInfo(\n\t\t\ti.Message,\n\t\t\ti.State,\n\t\t\ti.NextProcessAt,\n\t\t\ti.Result,\n\t\t))\n\t}\n\treturn tasks, err\n}\n\n// ListActiveTasks retrieves active tasks from the specified queue.\n//\n// By default, it retrieves the first 30 tasks.\nfunc (i *Inspector) ListActiveTasks(queue string, opts ...ListOption) ([]*TaskInfo, error) {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn nil, fmt.Errorf(\"asynq: %w\", err)\n\t}\n\topt := composeListOptions(opts...)\n\tpgn := rdb.Pagination{Size: opt.pageSize, Page: opt.pageNum - 1}\n\tinfos, err := i.rdb.ListActive(queue, pgn)\n\tswitch {\n\tcase errors.IsQueueNotFound(err):\n\t\treturn nil, fmt.Errorf(\"asynq: %w\", ErrQueueNotFound)\n\tcase err != nil:\n\t\treturn nil, fmt.Errorf(\"asynq: %w\", err)\n\t}\n\texpired, err := i.rdb.ListLeaseExpired(time.Now(), queue)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"asynq: %w\", err)\n\t}\n\texpiredSet := make(map[string]struct{}) // set of expired message IDs\n\tfor _, msg := range expired {\n\t\texpiredSet[msg.ID] = struct{}{}\n\t}\n\tvar tasks []*TaskInfo\n\tfor _, i := range infos {\n\t\tt := newTaskInfo(\n\t\t\ti.Message,\n\t\t\ti.State,\n\t\t\ti.NextProcessAt,\n\t\t\ti.Result,\n\t\t)\n\t\tif _, ok := expiredSet[i.Message.ID]; ok {\n\t\t\tt.IsOrphaned = true\n\t\t}\n\t\ttasks = append(tasks, t)\n\t}\n\treturn tasks, nil\n}\n\n// ListAggregatingTasks retrieves scheduled tasks from the specified group.\n//\n// By default, it retrieves the first 30 tasks.\nfunc (i *Inspector) ListAggregatingTasks(queue, group string, opts ...ListOption) ([]*TaskInfo, error) {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn nil, fmt.Errorf(\"asynq: %w\", err)\n\t}\n\topt := composeListOptions(opts...)\n\tpgn := rdb.Pagination{Size: opt.pageSize, Page: opt.pageNum - 1}\n\tinfos, err := i.rdb.ListAggregating(queue, group, pgn)\n\tswitch {\n\tcase errors.IsQueueNotFound(err):\n\t\treturn nil, fmt.Errorf(\"asynq: %w\", ErrQueueNotFound)\n\tcase err != nil:\n\t\treturn nil, fmt.Errorf(\"asynq: %w\", err)\n\t}\n\tvar tasks []*TaskInfo\n\tfor _, i := range infos {\n\t\ttasks = append(tasks, newTaskInfo(\n\t\t\ti.Message,\n\t\t\ti.State,\n\t\t\ti.NextProcessAt,\n\t\t\ti.Result,\n\t\t))\n\t}\n\treturn tasks, nil\n}\n\n// ListScheduledTasks retrieves scheduled tasks from the specified queue.\n// Tasks are sorted by NextProcessAt in ascending order.\n//\n// By default, it retrieves the first 30 tasks.\nfunc (i *Inspector) ListScheduledTasks(queue string, opts ...ListOption) ([]*TaskInfo, error) {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn nil, fmt.Errorf(\"asynq: %w\", err)\n\t}\n\topt := composeListOptions(opts...)\n\tpgn := rdb.Pagination{Size: opt.pageSize, Page: opt.pageNum - 1}\n\tinfos, err := i.rdb.ListScheduled(queue, pgn)\n\tswitch {\n\tcase errors.IsQueueNotFound(err):\n\t\treturn nil, fmt.Errorf(\"asynq: %w\", ErrQueueNotFound)\n\tcase err != nil:\n\t\treturn nil, fmt.Errorf(\"asynq: %w\", err)\n\t}\n\tvar tasks []*TaskInfo\n\tfor _, i := range infos {\n\t\ttasks = append(tasks, newTaskInfo(\n\t\t\ti.Message,\n\t\t\ti.State,\n\t\t\ti.NextProcessAt,\n\t\t\ti.Result,\n\t\t))\n\t}\n\treturn tasks, nil\n}\n\n// ListRetryTasks retrieves retry tasks from the specified queue.\n// Tasks are sorted by NextProcessAt in ascending order.\n//\n// By default, it retrieves the first 30 tasks.\nfunc (i *Inspector) ListRetryTasks(queue string, opts ...ListOption) ([]*TaskInfo, error) {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn nil, fmt.Errorf(\"asynq: %w\", err)\n\t}\n\topt := composeListOptions(opts...)\n\tpgn := rdb.Pagination{Size: opt.pageSize, Page: opt.pageNum - 1}\n\tinfos, err := i.rdb.ListRetry(queue, pgn)\n\tswitch {\n\tcase errors.IsQueueNotFound(err):\n\t\treturn nil, fmt.Errorf(\"asynq: %w\", ErrQueueNotFound)\n\tcase err != nil:\n\t\treturn nil, fmt.Errorf(\"asynq: %w\", err)\n\t}\n\tvar tasks []*TaskInfo\n\tfor _, i := range infos {\n\t\ttasks = append(tasks, newTaskInfo(\n\t\t\ti.Message,\n\t\t\ti.State,\n\t\t\ti.NextProcessAt,\n\t\t\ti.Result,\n\t\t))\n\t}\n\treturn tasks, nil\n}\n\n// ListArchivedTasks retrieves archived tasks from the specified queue.\n// Tasks are sorted by LastFailedAt in descending order.\n//\n// By default, it retrieves the first 30 tasks.\nfunc (i *Inspector) ListArchivedTasks(queue string, opts ...ListOption) ([]*TaskInfo, error) {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn nil, fmt.Errorf(\"asynq: %w\", err)\n\t}\n\topt := composeListOptions(opts...)\n\tpgn := rdb.Pagination{Size: opt.pageSize, Page: opt.pageNum - 1}\n\tinfos, err := i.rdb.ListArchived(queue, pgn)\n\tswitch {\n\tcase errors.IsQueueNotFound(err):\n\t\treturn nil, fmt.Errorf(\"asynq: %w\", ErrQueueNotFound)\n\tcase err != nil:\n\t\treturn nil, fmt.Errorf(\"asynq: %w\", err)\n\t}\n\tvar tasks []*TaskInfo\n\tfor _, i := range infos {\n\t\ttasks = append(tasks, newTaskInfo(\n\t\t\ti.Message,\n\t\t\ti.State,\n\t\t\ti.NextProcessAt,\n\t\t\ti.Result,\n\t\t))\n\t}\n\treturn tasks, nil\n}\n\n// ListCompletedTasks retrieves completed tasks from the specified queue.\n// Tasks are sorted by expiration time (i.e. CompletedAt + Retention) in descending order.\n//\n// By default, it retrieves the first 30 tasks.\nfunc (i *Inspector) ListCompletedTasks(queue string, opts ...ListOption) ([]*TaskInfo, error) {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn nil, fmt.Errorf(\"asynq: %w\", err)\n\t}\n\topt := composeListOptions(opts...)\n\tpgn := rdb.Pagination{Size: opt.pageSize, Page: opt.pageNum - 1}\n\tinfos, err := i.rdb.ListCompleted(queue, pgn)\n\tswitch {\n\tcase errors.IsQueueNotFound(err):\n\t\treturn nil, fmt.Errorf(\"asynq: %w\", ErrQueueNotFound)\n\tcase err != nil:\n\t\treturn nil, fmt.Errorf(\"asynq: %w\", err)\n\t}\n\tvar tasks []*TaskInfo\n\tfor _, i := range infos {\n\t\ttasks = append(tasks, newTaskInfo(\n\t\t\ti.Message,\n\t\t\ti.State,\n\t\t\ti.NextProcessAt,\n\t\t\ti.Result,\n\t\t))\n\t}\n\treturn tasks, nil\n}\n\n// DeleteAllPendingTasks deletes all pending tasks from the specified queue,\n// and reports the number tasks deleted.\nfunc (i *Inspector) DeleteAllPendingTasks(queue string) (int, error) {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn 0, err\n\t}\n\tn, err := i.rdb.DeleteAllPendingTasks(queue)\n\treturn int(n), err\n}\n\n// DeleteAllScheduledTasks deletes all scheduled tasks from the specified queue,\n// and reports the number tasks deleted.\nfunc (i *Inspector) DeleteAllScheduledTasks(queue string) (int, error) {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn 0, err\n\t}\n\tn, err := i.rdb.DeleteAllScheduledTasks(queue)\n\treturn int(n), err\n}\n\n// DeleteAllRetryTasks deletes all retry tasks from the specified queue,\n// and reports the number tasks deleted.\nfunc (i *Inspector) DeleteAllRetryTasks(queue string) (int, error) {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn 0, err\n\t}\n\tn, err := i.rdb.DeleteAllRetryTasks(queue)\n\treturn int(n), err\n}\n\n// DeleteAllArchivedTasks deletes all archived tasks from the specified queue,\n// and reports the number tasks deleted.\nfunc (i *Inspector) DeleteAllArchivedTasks(queue string) (int, error) {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn 0, err\n\t}\n\tn, err := i.rdb.DeleteAllArchivedTasks(queue)\n\treturn int(n), err\n}\n\n// DeleteAllCompletedTasks deletes all completed tasks from the specified queue,\n// and reports the number tasks deleted.\nfunc (i *Inspector) DeleteAllCompletedTasks(queue string) (int, error) {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn 0, err\n\t}\n\tn, err := i.rdb.DeleteAllCompletedTasks(queue)\n\treturn int(n), err\n}\n\n// DeleteAllAggregatingTasks deletes all tasks from the specified group,\n// and reports the number of tasks deleted.\nfunc (i *Inspector) DeleteAllAggregatingTasks(queue, group string) (int, error) {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn 0, err\n\t}\n\tn, err := i.rdb.DeleteAllAggregatingTasks(queue, group)\n\treturn int(n), err\n}\n\n// UpdateTaskPayload updates a task with the given id from the given queue with given payload.\n// The task needs to be in scheduled state,\n// otherwise UpdateTaskPayload will return an error.\n//\n// If a queue with the given name doesn't exist, it returns an error wrapping ErrQueueNotFound.\n// If a task with the given id doesn't exist in the queue, it returns an error wrapping ErrTaskNotFound.\n// If the task is not in scheduled state, it returns a non-nil error.\nfunc (i *Inspector) UpdateTaskPayload(queue, id string, payload []byte) error {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn fmt.Errorf(\"asynq: %v\", err)\n\t}\n\terr := i.rdb.UpdateTaskPayload(queue, id, payload)\n\tswitch {\n\tcase errors.IsQueueNotFound(err):\n\t\treturn fmt.Errorf(\"asynq: %w\", ErrQueueNotFound)\n\tcase errors.IsTaskNotFound(err):\n\t\treturn fmt.Errorf(\"asynq: %w\", ErrTaskNotFound)\n\tcase err != nil:\n\t\treturn fmt.Errorf(\"asynq: %v\", err)\n\t}\n\treturn nil\n\n}\n\n// DeleteTask deletes a task with the given id from the given queue.\n// The task needs to be in pending, scheduled, retry, or archived state,\n// otherwise DeleteTask will return an error.\n//\n// If a queue with the given name doesn't exist, it returns an error wrapping ErrQueueNotFound.\n// If a task with the given id doesn't exist in the queue, it returns an error wrapping ErrTaskNotFound.\n// If the task is in active state, it returns a non-nil error.\nfunc (i *Inspector) DeleteTask(queue, id string) error {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn fmt.Errorf(\"asynq: %w\", err)\n\t}\n\terr := i.rdb.DeleteTask(queue, id)\n\tswitch {\n\tcase errors.IsQueueNotFound(err):\n\t\treturn fmt.Errorf(\"asynq: %w\", ErrQueueNotFound)\n\tcase errors.IsTaskNotFound(err):\n\t\treturn fmt.Errorf(\"asynq: %w\", ErrTaskNotFound)\n\tcase err != nil:\n\t\treturn fmt.Errorf(\"asynq: %w\", err)\n\t}\n\treturn nil\n\n}\n\n// RunAllScheduledTasks schedules all scheduled tasks from the given queue to run,\n// and reports the number of tasks scheduled to run.\nfunc (i *Inspector) RunAllScheduledTasks(queue string) (int, error) {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn 0, err\n\t}\n\tn, err := i.rdb.RunAllScheduledTasks(queue)\n\treturn int(n), err\n}\n\n// RunAllRetryTasks schedules all retry tasks from the given queue to run,\n// and reports the number of tasks scheduled to run.\nfunc (i *Inspector) RunAllRetryTasks(queue string) (int, error) {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn 0, err\n\t}\n\tn, err := i.rdb.RunAllRetryTasks(queue)\n\treturn int(n), err\n}\n\n// RunAllArchivedTasks schedules all archived tasks from the given queue to run,\n// and reports the number of tasks scheduled to run.\nfunc (i *Inspector) RunAllArchivedTasks(queue string) (int, error) {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn 0, err\n\t}\n\tn, err := i.rdb.RunAllArchivedTasks(queue)\n\treturn int(n), err\n}\n\n// RunAllAggregatingTasks schedules all tasks from the given grou to run.\n// and reports the number of tasks scheduled to run.\nfunc (i *Inspector) RunAllAggregatingTasks(queue, group string) (int, error) {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn 0, err\n\t}\n\tn, err := i.rdb.RunAllAggregatingTasks(queue, group)\n\treturn int(n), err\n}\n\n// RunTask updates the task to pending state given a queue name and task id.\n// The task needs to be in scheduled, retry, or archived state, otherwise RunTask\n// will return an error.\n//\n// If a queue with the given name doesn't exist, it returns an error wrapping ErrQueueNotFound.\n// If a task with the given id doesn't exist in the queue, it returns an error wrapping ErrTaskNotFound.\n// If the task is in pending or active state, it returns a non-nil error.\nfunc (i *Inspector) RunTask(queue, id string) error {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn fmt.Errorf(\"asynq: %w\", err)\n\t}\n\terr := i.rdb.RunTask(queue, id)\n\tswitch {\n\tcase errors.IsQueueNotFound(err):\n\t\treturn fmt.Errorf(\"asynq: %w\", ErrQueueNotFound)\n\tcase errors.IsTaskNotFound(err):\n\t\treturn fmt.Errorf(\"asynq: %w\", ErrTaskNotFound)\n\tcase err != nil:\n\t\treturn fmt.Errorf(\"asynq: %w\", err)\n\t}\n\treturn nil\n}\n\n// ArchiveAllPendingTasks archives all pending tasks from the given queue,\n// and reports the number of tasks archived.\nfunc (i *Inspector) ArchiveAllPendingTasks(queue string) (int, error) {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn 0, err\n\t}\n\tn, err := i.rdb.ArchiveAllPendingTasks(queue)\n\treturn int(n), err\n}\n\n// ArchiveAllScheduledTasks archives all scheduled tasks from the given queue,\n// and reports the number of tasks archiveed.\nfunc (i *Inspector) ArchiveAllScheduledTasks(queue string) (int, error) {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn 0, err\n\t}\n\tn, err := i.rdb.ArchiveAllScheduledTasks(queue)\n\treturn int(n), err\n}\n\n// ArchiveAllRetryTasks archives all retry tasks from the given queue,\n// and reports the number of tasks archiveed.\nfunc (i *Inspector) ArchiveAllRetryTasks(queue string) (int, error) {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn 0, err\n\t}\n\tn, err := i.rdb.ArchiveAllRetryTasks(queue)\n\treturn int(n), err\n}\n\n// ArchiveAllAggregatingTasks archives all tasks from the given group,\n// and reports the number of tasks archived.\nfunc (i *Inspector) ArchiveAllAggregatingTasks(queue, group string) (int, error) {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn 0, err\n\t}\n\tn, err := i.rdb.ArchiveAllAggregatingTasks(queue, group)\n\treturn int(n), err\n}\n\n// ArchiveTask archives a task with the given id in the given queue.\n// The task needs to be in pending, scheduled, or retry state, otherwise ArchiveTask\n// will return an error.\n//\n// If a queue with the given name doesn't exist, it returns an error wrapping ErrQueueNotFound.\n// If a task with the given id doesn't exist in the queue, it returns an error wrapping ErrTaskNotFound.\n// If the task is in already archived, it returns a non-nil error.\nfunc (i *Inspector) ArchiveTask(queue, id string) error {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn fmt.Errorf(\"asynq: err\")\n\t}\n\terr := i.rdb.ArchiveTask(queue, id)\n\tswitch {\n\tcase errors.IsQueueNotFound(err):\n\t\treturn fmt.Errorf(\"asynq: %w\", ErrQueueNotFound)\n\tcase errors.IsTaskNotFound(err):\n\t\treturn fmt.Errorf(\"asynq: %w\", ErrTaskNotFound)\n\tcase err != nil:\n\t\treturn fmt.Errorf(\"asynq: %w\", err)\n\t}\n\treturn nil\n}\n\n// CancelProcessing sends a signal to cancel processing of the task\n// given a task id. CancelProcessing is best-effort, which means that it does not\n// guarantee that the task with the given id will be canceled. The return\n// value only indicates whether the cancelation signal has been sent.\nfunc (i *Inspector) CancelProcessing(id string) error {\n\treturn i.rdb.PublishCancelation(id)\n}\n\n// PauseQueue pauses task processing on the specified queue.\n// If the queue is already paused, it will return a non-nil error.\nfunc (i *Inspector) PauseQueue(queue string) error {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn err\n\t}\n\treturn i.rdb.Pause(queue)\n}\n\n// UnpauseQueue resumes task processing on the specified queue.\n// If the queue is not paused, it will return a non-nil error.\nfunc (i *Inspector) UnpauseQueue(queue string) error {\n\tif err := base.ValidateQueueName(queue); err != nil {\n\t\treturn err\n\t}\n\treturn i.rdb.Unpause(queue)\n}\n\n// Servers return a list of running servers' information.\nfunc (i *Inspector) Servers() ([]*ServerInfo, error) {\n\tservers, err := i.rdb.ListServers()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tworkers, err := i.rdb.ListWorkers()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tm := make(map[string]*ServerInfo) // ServerInfo keyed by serverID\n\tfor _, s := range servers {\n\t\tm[s.ServerID] = &ServerInfo{\n\t\t\tID: s.ServerID,\n\t\t\tHost: s.Host,\n\t\t\tPID: s.PID,\n\t\t\tConcurrency: s.Concurrency,\n\t\t\tQueues: s.Queues,\n\t\t\tStrictPriority: s.StrictPriority,\n\t\t\tStarted: s.Started,\n\t\t\tStatus: s.Status,\n\t\t\tActiveWorkers: make([]*WorkerInfo, 0),\n\t\t}\n\t}\n\tfor _, w := range workers {\n\t\tsrvInfo, ok := m[w.ServerID]\n\t\tif !ok {\n\t\t\tcontinue\n\t\t}\n\t\twrkInfo := &WorkerInfo{\n\t\t\tTaskID: w.ID,\n\t\t\tTaskType: w.Type,\n\t\t\tTaskPayload: w.Payload,\n\t\t\tQueue: w.Queue,\n\t\t\tStarted: w.Started,\n\t\t\tDeadline: w.Deadline,\n\t\t}\n\t\tsrvInfo.ActiveWorkers = append(srvInfo.ActiveWorkers, wrkInfo)\n\t}\n\tvar out []*ServerInfo\n\tfor _, srvInfo := range m {\n\t\tout = append(out, srvInfo)\n\t}\n\treturn out, nil\n}\n\n// ServerInfo describes a running Server instance.\ntype ServerInfo struct {\n\t// Unique Identifier for the server.\n\tID string\n\t// Host machine on which the server is running.\n\tHost string\n\t// PID of the process in which the server is running.\n\tPID int\n\n\t// Server configuration details.\n\t// See Config doc for field descriptions.\n\tConcurrency int\n\tQueues map[string]int\n\tStrictPriority bool\n\n\t// Time the server started.\n\tStarted time.Time\n\t// Status indicates the status of the server.\n\t// TODO: Update comment with more details.\n\tStatus string\n\t// A List of active workers currently processing tasks.\n\tActiveWorkers []*WorkerInfo\n}\n\n// WorkerInfo describes a running worker processing a task.\ntype WorkerInfo struct {\n\t// ID of the task the worker is processing.\n\tTaskID string\n\t// Type of the task the worker is processing.\n\tTaskType string\n\t// Payload of the task the worker is processing.\n\tTaskPayload []byte\n\t// Queue from which the worker got its task.\n\tQueue string\n\t// Time the worker started processing the task.\n\tStarted time.Time\n\t// Time the worker needs to finish processing the task by.\n\tDeadline time.Time\n}\n\n// ClusterKeySlot returns an integer identifying the hash slot the given queue hashes to.\nfunc (i *Inspector) ClusterKeySlot(queue string) (int64, error) {\n\treturn i.rdb.ClusterKeySlot(queue)\n}\n\n// ClusterNode describes a node in redis cluster.\ntype ClusterNode struct {\n\t// Node ID in the cluster.\n\tID string\n\n\t// Address of the node.\n\tAddr string\n}\n\n// ClusterNodes returns a list of nodes the given queue belongs to.\n//\n// Only relevant if task queues are stored in redis cluster.\nfunc (i *Inspector) ClusterNodes(queue string) ([]*ClusterNode, error) {\n\tnodes, err := i.rdb.ClusterNodes(queue)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tvar res []*ClusterNode\n\tfor _, node := range nodes {\n\t\tres = append(res, &ClusterNode{ID: node.ID, Addr: node.Addr})\n\t}\n\treturn res, nil\n}\n\n// SchedulerEntry holds information about a periodic task registered with a scheduler.\ntype SchedulerEntry struct {\n\t// Identifier of this entry.\n\tID string\n\n\t// Spec describes the schedule of this entry.\n\tSpec string\n\n\t// Periodic Task registered for this entry.\n\tTask *Task\n\n\t// Opts is the options for the periodic task.\n\tOpts []Option\n\n\t// Next shows the next time the task will be enqueued.\n\tNext time.Time\n\n\t// Prev shows the last time the task was enqueued.\n\t// Zero time if task was never enqueued.\n\tPrev time.Time\n}\n\n// SchedulerEntries returns a list of all entries registered with\n// currently running schedulers.\nfunc (i *Inspector) SchedulerEntries() ([]*SchedulerEntry, error) {\n\tvar entries []*SchedulerEntry\n\tres, err := i.rdb.ListSchedulerEntries()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tfor _, e := range res {\n\t\ttask := NewTask(e.Type, e.Payload)\n\t\tvar opts []Option\n\t\tfor _, s := range e.Opts {\n\t\t\tif o, err := parseOption(s); err == nil {\n\t\t\t\t// ignore bad data\n\t\t\t\topts = append(opts, o)\n\t\t\t}\n\t\t}\n\t\tentries = append(entries, &SchedulerEntry{\n\t\t\tID: e.ID,\n\t\t\tSpec: e.Spec,\n\t\t\tTask: task,\n\t\t\tOpts: opts,\n\t\t\tNext: e.Next,\n\t\t\tPrev: e.Prev,\n\t\t})\n\t}\n\treturn entries, nil\n}\n\n// parseOption interprets a string s as an Option and returns the Option if parsing is successful,\n// otherwise returns non-nil error.\nfunc parseOption(s string) (Option, error) {\n\tfn, arg := parseOptionFunc(s), parseOptionArg(s)\n\tswitch fn {\n\tcase \"Queue\":\n\t\tqueue, err := strconv.Unquote(arg)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn Queue(queue), nil\n\tcase \"MaxRetry\":\n\t\tn, err := strconv.Atoi(arg)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn MaxRetry(n), nil\n\tcase \"Timeout\":\n\t\td, err := time.ParseDuration(arg)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn Timeout(d), nil\n\tcase \"Deadline\":\n\t\tt, err := time.Parse(time.UnixDate, arg)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn Deadline(t), nil\n\tcase \"Unique\":\n\t\td, err := time.ParseDuration(arg)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn Unique(d), nil\n\tcase \"ProcessAt\":\n\t\tt, err := time.Parse(time.UnixDate, arg)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn ProcessAt(t), nil\n\tcase \"ProcessIn\":\n\t\td, err := time.ParseDuration(arg)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn ProcessIn(d), nil\n\tcase \"Retention\":\n\t\td, err := time.ParseDuration(arg)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn Retention(d), nil\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"cannot not parse option string %q\", s)\n\t}\n}\n\nfunc parseOptionFunc(s string) string {\n\ti := strings.Index(s, \"(\")\n\treturn s[:i]\n}\n\nfunc parseOptionArg(s string) string {\n\ti := strings.Index(s, \"(\")\n\tif i >= 0 {\n\t\tj := strings.Index(s, \")\")\n\t\tif j > i {\n\t\t\treturn s[i+1 : j]\n\t\t}\n\t}\n\treturn \"\"\n}\n\n// SchedulerEnqueueEvent holds information about an enqueue event by a scheduler.\ntype SchedulerEnqueueEvent struct {\n\t// ID of the task that was enqueued.\n\tTaskID string\n\n\t// Time the task was enqueued.\n\tEnqueuedAt time.Time\n}\n\n// ListSchedulerEnqueueEvents retrieves a list of enqueue events from the specified scheduler entry.\n//\n// By default, it retrieves the first 30 tasks.\nfunc (i *Inspector) ListSchedulerEnqueueEvents(entryID string, opts ...ListOption) ([]*SchedulerEnqueueEvent, error) {\n\topt := composeListOptions(opts...)\n\tpgn := rdb.Pagination{Size: opt.pageSize, Page: opt.pageNum - 1}\n\tdata, err := i.rdb.ListSchedulerEnqueueEvents(entryID, pgn)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tvar events []*SchedulerEnqueueEvent\n\tfor _, e := range data {\n\t\tevents = append(events, &SchedulerEnqueueEvent{TaskID: e.TaskID, EnqueuedAt: e.EnqueuedAt})\n\t}\n\treturn events, nil\n}\n" }, { "path": "inspector_test.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"sort\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/google/go-cmp/cmp\"\n\t\"github.com/google/go-cmp/cmp/cmpopts\"\n\t\"github.com/google/uuid\"\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/rdb\"\n\th \"github.com/hibiken/asynq/internal/testutil\"\n\t\"github.com/hibiken/asynq/internal/timeutil\"\n\t\"github.com/redis/go-redis/v9\"\n)\n\nfunc testInspectorQueues(t *testing.T, inspector *Inspector, r redis.UniversalClient) {\n\ttests := []struct {\n\t\tqueues []string\n\t}{\n\t\t{queues: []string{\"default\"}},\n\t\t{queues: []string{\"custom1\", \"custom2\"}},\n\t\t{queues: []string{\"default\", \"custom1\", \"custom2\"}},\n\t\t{queues: []string{}},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\tfor _, qname := range tc.queues {\n\t\t\tif err := r.SAdd(context.Background(), base.AllQueues, qname).Err(); err != nil {\n\t\t\t\tt.Fatalf(\"could not initialize all queue set: %v\", err)\n\t\t\t}\n\t\t}\n\t\tgot, err := inspector.Queues()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"Queues() returned an error: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tif diff := cmp.Diff(tc.queues, got, h.SortStringSliceOpt); diff != \"\" {\n\t\t\tt.Errorf(\"Queues() = %v, want %v; (-want, +got)\\n%s\", got, tc.queues, diff)\n\t\t}\n\t}\n}\n\nfunc TestInspectorQueues(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tinspector := NewInspector(getRedisConnOpt(t))\n\ttestInspectorQueues(t, inspector, r)\n}\n\nfunc TestInspectorFromRedisClientQueues(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tredisClient := getRedisConnOpt(t).MakeRedisClient().(redis.UniversalClient)\n\tinspector := NewInspectorFromRedisClient(redisClient)\n\ttestInspectorQueues(t, inspector, r)\n}\n\nfunc TestInspectorDeleteQueue(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tinspector := NewInspector(getRedisConnOpt(t))\n\tdefer inspector.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tm4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"custom\")\n\n\ttests := []struct {\n\t\tpending map[string][]*base.TaskMessage\n\t\tactive map[string][]*base.TaskMessage\n\t\tscheduled map[string][]base.Z\n\t\tretry map[string][]base.Z\n\t\tarchived map[string][]base.Z\n\t\tqname string // queue to remove\n\t\tforce bool\n\t}{\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\tforce: false,\n\t\t},\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t\t\"custom\": {m3},\n\t\t\t},\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {{Message: m4, Score: time.Now().Unix()}},\n\t\t\t},\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\tforce: true, // allow removing non-empty queue\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllPendingQueues(t, r, tc.pending)\n\t\th.SeedAllActiveQueues(t, r, tc.active)\n\t\th.SeedAllScheduledQueues(t, r, tc.scheduled)\n\t\th.SeedAllRetryQueues(t, r, tc.retry)\n\t\th.SeedAllArchivedQueues(t, r, tc.archived)\n\n\t\terr := inspector.DeleteQueue(tc.qname, tc.force)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"DeleteQueue(%q, %t) = %v, want nil\",\n\t\t\t\ttc.qname, tc.force, err)\n\t\t\tcontinue\n\t\t}\n\t\tif r.SIsMember(context.Background(), base.AllQueues, tc.qname).Val() {\n\t\t\tt.Errorf(\"%q is a member of %q\", tc.qname, base.AllQueues)\n\t\t}\n\t}\n}\n\nfunc TestInspectorDeleteQueueErrorQueueNotEmpty(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tinspector := NewInspector(getRedisConnOpt(t))\n\tdefer inspector.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tm4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"custom\")\n\n\ttests := []struct {\n\t\tpending map[string][]*base.TaskMessage\n\t\tactive map[string][]*base.TaskMessage\n\t\tscheduled map[string][]base.Z\n\t\tretry map[string][]base.Z\n\t\tarchived map[string][]base.Z\n\t\tqname string // queue to remove\n\t\tforce bool\n\t}{\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t},\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m3, m4},\n\t\t\t},\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tforce: false,\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllPendingQueues(t, r, tc.pending)\n\t\th.SeedAllActiveQueues(t, r, tc.active)\n\t\th.SeedAllScheduledQueues(t, r, tc.scheduled)\n\t\th.SeedAllRetryQueues(t, r, tc.retry)\n\t\th.SeedAllArchivedQueues(t, r, tc.archived)\n\n\t\terr := inspector.DeleteQueue(tc.qname, tc.force)\n\t\tif !errors.Is(err, ErrQueueNotEmpty) {\n\t\t\tt.Errorf(\"DeleteQueue(%v, %t) did not return ErrQueueNotEmpty\",\n\t\t\t\ttc.qname, tc.force)\n\t\t}\n\t}\n}\n\nfunc TestInspectorDeleteQueueErrorQueueNotFound(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tinspector := NewInspector(getRedisConnOpt(t))\n\tdefer inspector.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tm4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"custom\")\n\n\ttests := []struct {\n\t\tpending map[string][]*base.TaskMessage\n\t\tactive map[string][]*base.TaskMessage\n\t\tscheduled map[string][]base.Z\n\t\tretry map[string][]base.Z\n\t\tarchived map[string][]base.Z\n\t\tqname string // queue to remove\n\t\tforce bool\n\t}{\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t},\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m3, m4},\n\t\t\t},\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"nonexistent\",\n\t\t\tforce: false,\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllPendingQueues(t, r, tc.pending)\n\t\th.SeedAllActiveQueues(t, r, tc.active)\n\t\th.SeedAllScheduledQueues(t, r, tc.scheduled)\n\t\th.SeedAllRetryQueues(t, r, tc.retry)\n\t\th.SeedAllArchivedQueues(t, r, tc.archived)\n\n\t\terr := inspector.DeleteQueue(tc.qname, tc.force)\n\t\tif !errors.Is(err, ErrQueueNotFound) {\n\t\t\tt.Errorf(\"DeleteQueue(%v, %t) did not return ErrQueueNotFound\",\n\t\t\t\ttc.qname, tc.force)\n\t\t}\n\t}\n}\n\nfunc TestInspectorGetQueueInfo(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessage(\"task3\", nil)\n\tm4 := h.NewTaskMessage(\"task4\", nil)\n\tm5 := h.NewTaskMessageWithQueue(\"task5\", nil, \"critical\")\n\tm6 := h.NewTaskMessageWithQueue(\"task6\", nil, \"low\")\n\tnow := time.Now()\n\ttimeCmpOpt := cmpopts.EquateApproxTime(time.Second)\n\tignoreMemUsg := cmpopts.IgnoreFields(QueueInfo{}, \"MemoryUsage\")\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\tinspector.rdb.SetClock(timeutil.NewSimulatedClock(now))\n\n\ttests := []struct {\n\t\tpending map[string][]*base.TaskMessage\n\t\tactive map[string][]*base.TaskMessage\n\t\tscheduled map[string][]base.Z\n\t\tretry map[string][]base.Z\n\t\tarchived map[string][]base.Z\n\t\tcompleted map[string][]base.Z\n\t\tprocessed map[string]int\n\t\tfailed map[string]int\n\t\tprocessedTotal map[string]int\n\t\tfailedTotal map[string]int\n\t\toldestPendingMessageEnqueueTime map[string]time.Time\n\t\tqname string\n\t\twant *QueueInfo\n\t}{\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1},\n\t\t\t\t\"critical\": {m5},\n\t\t\t\t\"low\": {m6},\n\t\t\t},\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m2},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m3, Score: now.Add(time.Hour).Unix()},\n\t\t\t\t\t{Message: m4, Score: now.Unix()},\n\t\t\t\t},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t\tcompleted: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t\tprocessed: map[string]int{\n\t\t\t\t\"default\": 120,\n\t\t\t\t\"critical\": 100,\n\t\t\t\t\"low\": 42,\n\t\t\t},\n\t\t\tfailed: map[string]int{\n\t\t\t\t\"default\": 2,\n\t\t\t\t\"critical\": 0,\n\t\t\t\t\"low\": 5,\n\t\t\t},\n\t\t\tprocessedTotal: map[string]int{\n\t\t\t\t\"default\": 11111,\n\t\t\t\t\"critical\": 22222,\n\t\t\t\t\"low\": 33333,\n\t\t\t},\n\t\t\tfailedTotal: map[string]int{\n\t\t\t\t\"default\": 111,\n\t\t\t\t\"critical\": 222,\n\t\t\t\t\"low\": 333,\n\t\t\t},\n\t\t\toldestPendingMessageEnqueueTime: map[string]time.Time{\n\t\t\t\t\"default\": now.Add(-15 * time.Second),\n\t\t\t\t\"critical\": now.Add(-200 * time.Millisecond),\n\t\t\t\t\"low\": now.Add(-30 * time.Second),\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: &QueueInfo{\n\t\t\t\tQueue: \"default\",\n\t\t\t\tLatency: 15 * time.Second,\n\t\t\t\tSize: 4,\n\t\t\t\tPending: 1,\n\t\t\t\tActive: 1,\n\t\t\t\tScheduled: 2,\n\t\t\t\tRetry: 0,\n\t\t\t\tArchived: 0,\n\t\t\t\tCompleted: 0,\n\t\t\t\tProcessed: 120,\n\t\t\t\tFailed: 2,\n\t\t\t\tProcessedTotal: 11111,\n\t\t\t\tFailedTotal: 111,\n\t\t\t\tPaused: false,\n\t\t\t\tTimestamp: now,\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllPendingQueues(t, r, tc.pending)\n\t\th.SeedAllActiveQueues(t, r, tc.active)\n\t\th.SeedAllScheduledQueues(t, r, tc.scheduled)\n\t\th.SeedAllRetryQueues(t, r, tc.retry)\n\t\th.SeedAllArchivedQueues(t, r, tc.archived)\n\t\th.SeedAllCompletedQueues(t, r, tc.completed)\n\t\tctx := context.Background()\n\t\tfor qname, n := range tc.processed {\n\t\t\tr.Set(ctx, base.ProcessedKey(qname, now), n, 0)\n\t\t}\n\t\tfor qname, n := range tc.failed {\n\t\t\tr.Set(ctx, base.FailedKey(qname, now), n, 0)\n\t\t}\n\t\tfor qname, n := range tc.processedTotal {\n\t\t\tr.Set(ctx, base.ProcessedTotalKey(qname), n, 0)\n\t\t}\n\t\tfor qname, n := range tc.failedTotal {\n\t\t\tr.Set(ctx, base.FailedTotalKey(qname), n, 0)\n\t\t}\n\t\tfor qname, enqueueTime := range tc.oldestPendingMessageEnqueueTime {\n\t\t\tif enqueueTime.IsZero() {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\toldestPendingMessageID := r.LRange(ctx, base.PendingKey(qname), -1, -1).Val()[0] // get the right most msg in the list\n\t\t\tr.HSet(ctx, base.TaskKey(qname, oldestPendingMessageID), \"pending_since\", enqueueTime.UnixNano())\n\t\t}\n\n\t\tgot, err := inspector.GetQueueInfo(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"r.GetQueueInfo(%q) = %v, %v, want %v, nil\",\n\t\t\t\ttc.qname, got, err, tc.want)\n\t\t\tcontinue\n\t\t}\n\t\tif diff := cmp.Diff(tc.want, got, timeCmpOpt, ignoreMemUsg); diff != \"\" {\n\t\t\tt.Errorf(\"r.GetQueueInfo(%q) = %v, %v, want %v, nil; (-want, +got)\\n%s\",\n\t\t\t\ttc.qname, got, err, tc.want, diff)\n\t\t\tcontinue\n\t\t}\n\t}\n\n}\n\nfunc TestInspectorHistory(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tnow := time.Now().UTC()\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\tqname string // queue of interest\n\t\tn int // number of days\n\t}{\n\t\t{\"default\", 90},\n\t\t{\"custom\", 7},\n\t\t{\"default\", 1},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\n\t\tr.SAdd(context.Background(), base.AllQueues, tc.qname)\n\t\t// populate last n days data\n\t\tfor i := 0; i < tc.n; i++ {\n\t\t\tts := now.Add(-time.Duration(i) * 24 * time.Hour)\n\t\t\tprocessedKey := base.ProcessedKey(tc.qname, ts)\n\t\t\tfailedKey := base.FailedKey(tc.qname, ts)\n\t\t\tr.Set(context.Background(), processedKey, (i+1)*1000, 0)\n\t\t\tr.Set(context.Background(), failedKey, (i+1)*10, 0)\n\t\t}\n\n\t\tgot, err := inspector.History(tc.qname, tc.n)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"Inspector.History(%q, %d) returned error: %v\", tc.qname, tc.n, err)\n\t\t\tcontinue\n\t\t}\n\t\tif len(got) != tc.n {\n\t\t\tt.Errorf(\"Inspector.History(%q, %d) returned %d daily stats, want %d\",\n\t\t\t\ttc.qname, tc.n, len(got), tc.n)\n\t\t\tcontinue\n\t\t}\n\t\tfor i := 0; i < tc.n; i++ {\n\t\t\twant := &DailyStats{\n\t\t\t\tQueue: tc.qname,\n\t\t\t\tProcessed: (i + 1) * 1000,\n\t\t\t\tFailed: (i + 1) * 10,\n\t\t\t\tDate: now.Add(-time.Duration(i) * 24 * time.Hour),\n\t\t\t}\n\t\t\t// Allow 2 seconds difference in timestamp.\n\t\t\ttimeCmpOpt := cmpopts.EquateApproxTime(2 * time.Second)\n\t\t\tif diff := cmp.Diff(want, got[i], timeCmpOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"Inspector.History %d days ago data; got %+v, want %+v; (-want,+got):\\n%s\",\n\t\t\t\t\ti, got[i], want, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc createPendingTask(msg *base.TaskMessage) *TaskInfo {\n\treturn newTaskInfo(msg, base.TaskStatePending, time.Now(), nil)\n}\n\nfunc TestInspectorGetTaskInfo(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tm1 := h.NewTaskMessageWithQueue(\"task1\", nil, \"default\")\n\tm2 := h.NewTaskMessageWithQueue(\"task2\", nil, \"default\")\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tm4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"custom\")\n\tm5 := h.NewTaskMessageWithQueue(\"task5\", nil, \"custom\")\n\n\tnow := time.Now()\n\tfiveMinsFromNow := now.Add(5 * time.Minute)\n\toneHourFromNow := now.Add(1 * time.Hour)\n\ttwoHoursAgo := now.Add(-2 * time.Hour)\n\n\tfixtures := struct {\n\t\tactive map[string][]*base.TaskMessage\n\t\tpending map[string][]*base.TaskMessage\n\t\tscheduled map[string][]base.Z\n\t\tretry map[string][]base.Z\n\t\tarchived map[string][]base.Z\n\t}{\n\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\"default\": {m1},\n\t\t\t\"custom\": {},\n\t\t},\n\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\"default\": {},\n\t\t\t\"custom\": {m5},\n\t\t},\n\t\tscheduled: map[string][]base.Z{\n\t\t\t\"default\": {{Message: m2, Score: fiveMinsFromNow.Unix()}},\n\t\t\t\"custom\": {},\n\t\t},\n\t\tretry: map[string][]base.Z{\n\t\t\t\"default\": {},\n\t\t\t\"custom\": {{Message: m3, Score: oneHourFromNow.Unix()}},\n\t\t},\n\t\tarchived: map[string][]base.Z{\n\t\t\t\"default\": {},\n\t\t\t\"custom\": {{Message: m4, Score: twoHoursAgo.Unix()}},\n\t\t},\n\t}\n\n\th.SeedAllActiveQueues(t, r, fixtures.active)\n\th.SeedAllPendingQueues(t, r, fixtures.pending)\n\th.SeedAllScheduledQueues(t, r, fixtures.scheduled)\n\th.SeedAllRetryQueues(t, r, fixtures.retry)\n\th.SeedAllArchivedQueues(t, r, fixtures.archived)\n\n\ttests := []struct {\n\t\tqname string\n\t\tid string\n\t\twant *TaskInfo\n\t}{\n\t\t{\n\t\t\tqname: \"default\",\n\t\t\tid: m1.ID,\n\t\t\twant: newTaskInfo(\n\t\t\t\tm1,\n\t\t\t\tbase.TaskStateActive,\n\t\t\t\ttime.Time{}, // zero value for n/a\n\t\t\t\tnil,\n\t\t\t),\n\t\t},\n\t\t{\n\t\t\tqname: \"default\",\n\t\t\tid: m2.ID,\n\t\t\twant: newTaskInfo(\n\t\t\t\tm2,\n\t\t\t\tbase.TaskStateScheduled,\n\t\t\t\tfiveMinsFromNow,\n\t\t\t\tnil,\n\t\t\t),\n\t\t},\n\t\t{\n\t\t\tqname: \"custom\",\n\t\t\tid: m3.ID,\n\t\t\twant: newTaskInfo(\n\t\t\t\tm3,\n\t\t\t\tbase.TaskStateRetry,\n\t\t\t\toneHourFromNow,\n\t\t\t\tnil,\n\t\t\t),\n\t\t},\n\t\t{\n\t\t\tqname: \"custom\",\n\t\t\tid: m4.ID,\n\t\t\twant: newTaskInfo(\n\t\t\t\tm4,\n\t\t\t\tbase.TaskStateArchived,\n\t\t\t\ttime.Time{}, // zero value for n/a\n\t\t\t\tnil,\n\t\t\t),\n\t\t},\n\t\t{\n\t\t\tqname: \"custom\",\n\t\t\tid: m5.ID,\n\t\t\twant: newTaskInfo(\n\t\t\t\tm5,\n\t\t\t\tbase.TaskStatePending,\n\t\t\t\tnow,\n\t\t\t\tnil,\n\t\t\t),\n\t\t},\n\t}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\tfor _, tc := range tests {\n\t\tgot, err := inspector.GetTaskInfo(tc.qname, tc.id)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"GetTaskInfo(%q, %q) returned error: %v\", tc.qname, tc.id, err)\n\t\t\tcontinue\n\t\t}\n\t\tcmpOpts := []cmp.Option{\n\t\t\tcmp.AllowUnexported(TaskInfo{}),\n\t\t\tcmpopts.EquateApproxTime(2 * time.Second),\n\t\t}\n\t\tif diff := cmp.Diff(tc.want, got, cmpOpts...); diff != \"\" {\n\t\t\tt.Errorf(\"GetTaskInfo(%q, %q) = %v, want %v; (-want, +got)\\n%s\", tc.qname, tc.id, got, tc.want, diff)\n\t\t}\n\t}\n}\n\nfunc TestInspectorGetTaskInfoError(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tm1 := h.NewTaskMessageWithQueue(\"task1\", nil, \"default\")\n\tm2 := h.NewTaskMessageWithQueue(\"task2\", nil, \"default\")\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tm4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"custom\")\n\tm5 := h.NewTaskMessageWithQueue(\"task5\", nil, \"custom\")\n\n\tnow := time.Now()\n\tfiveMinsFromNow := now.Add(5 * time.Minute)\n\toneHourFromNow := now.Add(1 * time.Hour)\n\ttwoHoursAgo := now.Add(-2 * time.Hour)\n\n\tfixtures := struct {\n\t\tactive map[string][]*base.TaskMessage\n\t\tpending map[string][]*base.TaskMessage\n\t\tscheduled map[string][]base.Z\n\t\tretry map[string][]base.Z\n\t\tarchived map[string][]base.Z\n\t}{\n\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\"default\": {m1},\n\t\t\t\"custom\": {},\n\t\t},\n\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\"default\": {},\n\t\t\t\"custom\": {m5},\n\t\t},\n\t\tscheduled: map[string][]base.Z{\n\t\t\t\"default\": {{Message: m2, Score: fiveMinsFromNow.Unix()}},\n\t\t\t\"custom\": {},\n\t\t},\n\t\tretry: map[string][]base.Z{\n\t\t\t\"default\": {},\n\t\t\t\"custom\": {{Message: m3, Score: oneHourFromNow.Unix()}},\n\t\t},\n\t\tarchived: map[string][]base.Z{\n\t\t\t\"default\": {},\n\t\t\t\"custom\": {{Message: m4, Score: twoHoursAgo.Unix()}},\n\t\t},\n\t}\n\n\th.SeedAllActiveQueues(t, r, fixtures.active)\n\th.SeedAllPendingQueues(t, r, fixtures.pending)\n\th.SeedAllScheduledQueues(t, r, fixtures.scheduled)\n\th.SeedAllRetryQueues(t, r, fixtures.retry)\n\th.SeedAllArchivedQueues(t, r, fixtures.archived)\n\n\ttests := []struct {\n\t\tqname string\n\t\tid string\n\t\twantErr error\n\t}{\n\t\t{\n\t\t\tqname: \"nonexistent\",\n\t\t\tid: m1.ID,\n\t\t\twantErr: ErrQueueNotFound,\n\t\t},\n\t\t{\n\t\t\tqname: \"default\",\n\t\t\tid: uuid.NewString(),\n\t\t\twantErr: ErrTaskNotFound,\n\t\t},\n\t}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\tfor _, tc := range tests {\n\t\tinfo, err := inspector.GetTaskInfo(tc.qname, tc.id)\n\t\tif info != nil {\n\t\t\tt.Errorf(\"GetTaskInfo(%q, %q) returned info: %v\", tc.qname, tc.id, info)\n\t\t}\n\t\tif !errors.Is(err, tc.wantErr) {\n\t\t\tt.Errorf(\"GetTaskInfo(%q, %q) returned unexpected error: %v, want %v\", tc.qname, tc.id, err, tc.wantErr)\n\t\t}\n\t}\n}\n\nfunc TestInspectorListPendingTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"critical\")\n\tm4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"low\")\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\tdesc string\n\t\tpending map[string][]*base.TaskMessage\n\t\tqname string\n\t\twant []*TaskInfo\n\t}{\n\t\t{\n\t\t\tdesc: \"with default queue\",\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: []*TaskInfo{\n\t\t\t\tcreatePendingTask(m1),\n\t\t\t\tcreatePendingTask(m2),\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with named queue\",\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t\t\"critical\": {m3},\n\t\t\t\t\"low\": {m4},\n\t\t\t},\n\t\t\tqname: \"critical\",\n\t\t\twant: []*TaskInfo{\n\t\t\t\tcreatePendingTask(m3),\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with empty queue\",\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: []*TaskInfo(nil),\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\tfor q, msgs := range tc.pending {\n\t\t\th.SeedPendingQueue(t, r, msgs, q)\n\t\t}\n\n\t\tgot, err := inspector.ListPendingTasks(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s; ListPendingTasks(%q) returned error: %v\",\n\t\t\t\ttc.desc, tc.qname, err)\n\t\t\tcontinue\n\t\t}\n\t\tcmpOpts := []cmp.Option{\n\t\t\tcmpopts.EquateApproxTime(2 * time.Second),\n\t\t\tcmp.AllowUnexported(TaskInfo{}),\n\t\t}\n\t\tif diff := cmp.Diff(tc.want, got, cmpOpts...); diff != \"\" {\n\t\t\tt.Errorf(\"%s; ListPendingTasks(%q) = %v, want %v; (-want,+got)\\n%s\",\n\t\t\t\ttc.desc, tc.qname, got, tc.want, diff)\n\t\t}\n\t}\n}\n\nfunc newOrphanedTaskInfo(msg *base.TaskMessage) *TaskInfo {\n\tinfo := newTaskInfo(msg, base.TaskStateActive, time.Time{}, nil)\n\tinfo.IsOrphaned = true\n\treturn info\n}\n\nfunc TestInspectorListActiveTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tm4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"custom\")\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\tnow := time.Now()\n\n\ttests := []struct {\n\t\tdesc string\n\t\tactive map[string][]*base.TaskMessage\n\t\tlease map[string][]base.Z\n\t\tqname string\n\t\twant []*TaskInfo\n\t}{\n\t\t{\n\t\t\tdesc: \"with a few active tasks\",\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t\t\"custom\": {m3, m4},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: now.Add(20 * time.Second).Unix()},\n\t\t\t\t\t{Message: m2, Score: now.Add(20 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: m3, Score: now.Add(20 * time.Second).Unix()},\n\t\t\t\t\t{Message: m4, Score: now.Add(20 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\twant: []*TaskInfo{\n\t\t\t\tnewTaskInfo(m3, base.TaskStateActive, time.Time{}, nil),\n\t\t\t\tnewTaskInfo(m4, base.TaskStateActive, time.Time{}, nil),\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with an orphaned task\",\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t\t\"custom\": {m3, m4},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: now.Add(20 * time.Second).Unix()},\n\t\t\t\t\t{Message: m2, Score: now.Add(-10 * time.Second).Unix()}, // orphaned task\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: m3, Score: now.Add(20 * time.Second).Unix()},\n\t\t\t\t\t{Message: m4, Score: now.Add(20 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: []*TaskInfo{\n\t\t\t\tnewTaskInfo(m1, base.TaskStateActive, time.Time{}, nil),\n\t\t\t\tnewOrphanedTaskInfo(m2),\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllActiveQueues(t, r, tc.active)\n\t\th.SeedAllLease(t, r, tc.lease)\n\n\t\tgot, err := inspector.ListActiveTasks(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s; ListActiveTasks(%q) returned error: %v\", tc.qname, tc.desc, err)\n\t\t\tcontinue\n\t\t}\n\t\tif diff := cmp.Diff(tc.want, got, cmp.AllowUnexported(TaskInfo{})); diff != \"\" {\n\t\t\tt.Errorf(\"%s; ListActiveTask(%q) = %v, want %v; (-want,+got)\\n%s\",\n\t\t\t\ttc.desc, tc.qname, got, tc.want, diff)\n\t\t}\n\t}\n}\n\nfunc createScheduledTask(z base.Z) *TaskInfo {\n\treturn newTaskInfo(\n\t\tz.Message,\n\t\tbase.TaskStateScheduled,\n\t\ttime.Unix(z.Score, 0),\n\t\tnil,\n\t)\n}\n\nfunc TestInspectorListScheduledTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessage(\"task3\", nil)\n\tm4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"custom\")\n\tnow := time.Now()\n\tz1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()}\n\tz2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()}\n\tz3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()}\n\tz4 := base.Z{Message: m4, Score: now.Add(2 * time.Minute).Unix()}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\tdesc string\n\t\tscheduled map[string][]base.Z\n\t\tqname string\n\t\twant []*TaskInfo\n\t}{\n\t\t{\n\t\t\tdesc: \"with a few scheduled tasks\",\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2, z3},\n\t\t\t\t\"custom\": {z4},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\t// Should be sorted by NextProcessAt.\n\t\t\twant: []*TaskInfo{\n\t\t\t\tcreateScheduledTask(z3),\n\t\t\t\tcreateScheduledTask(z1),\n\t\t\t\tcreateScheduledTask(z2),\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with empty scheduled queue\",\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: []*TaskInfo(nil),\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllScheduledQueues(t, r, tc.scheduled)\n\n\t\tgot, err := inspector.ListScheduledTasks(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s; ListScheduledTasks(%q) returned error: %v\", tc.desc, tc.qname, err)\n\t\t\tcontinue\n\t\t}\n\t\tif diff := cmp.Diff(tc.want, got, cmp.AllowUnexported(TaskInfo{})); diff != \"\" {\n\t\t\tt.Errorf(\"%s; ListScheduledTask(%q) = %v, want %v; (-want,+got)\\n%s\",\n\t\t\t\ttc.desc, tc.qname, got, tc.want, diff)\n\t\t}\n\t}\n}\n\nfunc createRetryTask(z base.Z) *TaskInfo {\n\treturn newTaskInfo(\n\t\tz.Message,\n\t\tbase.TaskStateRetry,\n\t\ttime.Unix(z.Score, 0),\n\t\tnil,\n\t)\n}\n\nfunc TestInspectorListRetryTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessage(\"task3\", nil)\n\tm4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"custom\")\n\tnow := time.Now()\n\tz1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()}\n\tz2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()}\n\tz3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()}\n\tz4 := base.Z{Message: m4, Score: now.Add(2 * time.Minute).Unix()}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\tdesc string\n\t\tretry map[string][]base.Z\n\t\tqname string\n\t\twant []*TaskInfo\n\t}{\n\t\t{\n\t\t\tdesc: \"with a few retry tasks\",\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2, z3},\n\t\t\t\t\"custom\": {z4},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\t// Should be sorted by NextProcessAt.\n\t\t\twant: []*TaskInfo{\n\t\t\t\tcreateRetryTask(z3),\n\t\t\t\tcreateRetryTask(z1),\n\t\t\t\tcreateRetryTask(z2),\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with empty retry queue\",\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: []*TaskInfo(nil),\n\t\t},\n\t\t// TODO(hibiken): ErrQueueNotFound when queue doesn't exist\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllRetryQueues(t, r, tc.retry)\n\n\t\tgot, err := inspector.ListRetryTasks(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s; ListRetryTasks(%q) returned error: %v\", tc.desc, tc.qname, err)\n\t\t\tcontinue\n\t\t}\n\t\tif diff := cmp.Diff(tc.want, got, cmp.AllowUnexported(TaskInfo{})); diff != \"\" {\n\t\t\tt.Errorf(\"%s; ListRetryTask(%q) = %v, want %v; (-want,+got)\\n%s\",\n\t\t\t\ttc.desc, tc.qname, got, tc.want, diff)\n\t\t}\n\t}\n}\n\nfunc createArchivedTask(z base.Z) *TaskInfo {\n\treturn newTaskInfo(\n\t\tz.Message,\n\t\tbase.TaskStateArchived,\n\t\ttime.Time{}, // zero value for n/a\n\t\tnil,\n\t)\n}\n\nfunc TestInspectorListArchivedTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessage(\"task3\", nil)\n\tm4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"custom\")\n\tnow := time.Now()\n\tz1 := base.Z{Message: m1, Score: now.Add(-5 * time.Minute).Unix()}\n\tz2 := base.Z{Message: m2, Score: now.Add(-15 * time.Minute).Unix()}\n\tz3 := base.Z{Message: m3, Score: now.Add(-2 * time.Minute).Unix()}\n\tz4 := base.Z{Message: m4, Score: now.Add(-2 * time.Minute).Unix()}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\tdesc string\n\t\tarchived map[string][]base.Z\n\t\tqname string\n\t\twant []*TaskInfo\n\t}{\n\t\t{\n\t\t\tdesc: \"with a few archived tasks\",\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2, z3},\n\t\t\t\t\"custom\": {z4},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\t// Should be sorted by LastFailedAt.\n\t\t\twant: []*TaskInfo{\n\t\t\t\tcreateArchivedTask(z2),\n\t\t\t\tcreateArchivedTask(z1),\n\t\t\t\tcreateArchivedTask(z3),\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with empty archived queue\",\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: []*TaskInfo(nil),\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllArchivedQueues(t, r, tc.archived)\n\n\t\tgot, err := inspector.ListArchivedTasks(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s; ListArchivedTasks(%q) returned error: %v\", tc.desc, tc.qname, err)\n\t\t\tcontinue\n\t\t}\n\t\tif diff := cmp.Diff(tc.want, got, cmp.AllowUnexported(TaskInfo{})); diff != \"\" {\n\t\t\tt.Errorf(\"%s; ListArchivedTask(%q) = %v, want %v; (-want,+got)\\n%s\",\n\t\t\t\ttc.desc, tc.qname, got, tc.want, diff)\n\t\t}\n\t}\n}\n\nfunc newCompletedTaskMessage(typename, qname string, retention time.Duration, completedAt time.Time) *base.TaskMessage {\n\tmsg := h.NewTaskMessageWithQueue(typename, nil, qname)\n\tmsg.Retention = int64(retention.Seconds())\n\tmsg.CompletedAt = completedAt.Unix()\n\treturn msg\n}\n\nfunc createCompletedTask(z base.Z) *TaskInfo {\n\treturn newTaskInfo(\n\t\tz.Message,\n\t\tbase.TaskStateCompleted,\n\t\ttime.Time{}, // zero value for n/a\n\t\tnil, // TODO: Test with result data\n\t)\n}\n\nfunc TestInspectorListCompletedTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tnow := time.Now()\n\tm1 := newCompletedTaskMessage(\"task1\", \"default\", 1*time.Hour, now.Add(-3*time.Minute)) // Expires in 57 mins\n\tm2 := newCompletedTaskMessage(\"task2\", \"default\", 30*time.Minute, now.Add(-10*time.Minute)) // Expires in 20 mins\n\tm3 := newCompletedTaskMessage(\"task3\", \"default\", 2*time.Hour, now.Add(-30*time.Minute)) // Expires in 90 mins\n\tm4 := newCompletedTaskMessage(\"task4\", \"custom\", 15*time.Minute, now.Add(-2*time.Minute)) // Expires in 13 mins\n\tz1 := base.Z{Message: m1, Score: m1.CompletedAt + m1.Retention}\n\tz2 := base.Z{Message: m2, Score: m2.CompletedAt + m2.Retention}\n\tz3 := base.Z{Message: m3, Score: m3.CompletedAt + m3.Retention}\n\tz4 := base.Z{Message: m4, Score: m4.CompletedAt + m4.Retention}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\tdesc string\n\t\tcompleted map[string][]base.Z\n\t\tqname string\n\t\twant []*TaskInfo\n\t}{\n\t\t{\n\t\t\tdesc: \"with a few completed tasks\",\n\t\t\tcompleted: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2, z3},\n\t\t\t\t\"custom\": {z4},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\t// Should be sorted by expiration time (CompletedAt + Retention).\n\t\t\twant: []*TaskInfo{\n\t\t\t\tcreateCompletedTask(z2),\n\t\t\t\tcreateCompletedTask(z1),\n\t\t\t\tcreateCompletedTask(z3),\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with empty completed queue\",\n\t\t\tcompleted: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: []*TaskInfo(nil),\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllCompletedQueues(t, r, tc.completed)\n\n\t\tgot, err := inspector.ListCompletedTasks(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s; ListCompletedTasks(%q) returned error: %v\", tc.desc, tc.qname, err)\n\t\t\tcontinue\n\t\t}\n\t\tif diff := cmp.Diff(tc.want, got, cmp.AllowUnexported(TaskInfo{})); diff != \"\" {\n\t\t\tt.Errorf(\"%s; ListCompletedTasks(%q) = %v, want %v; (-want,+got)\\n%s\",\n\t\t\t\ttc.desc, tc.qname, got, tc.want, diff)\n\t\t}\n\t}\n}\n\nfunc TestInspectorListAggregatingTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tnow := time.Now()\n\n\tm1 := h.NewTaskMessageBuilder().SetType(\"task1\").SetQueue(\"default\").SetGroup(\"group1\").Build()\n\tm2 := h.NewTaskMessageBuilder().SetType(\"task2\").SetQueue(\"default\").SetGroup(\"group1\").Build()\n\tm3 := h.NewTaskMessageBuilder().SetType(\"task3\").SetQueue(\"default\").SetGroup(\"group1\").Build()\n\tm4 := h.NewTaskMessageBuilder().SetType(\"task4\").SetQueue(\"default\").SetGroup(\"group2\").Build()\n\tm5 := h.NewTaskMessageBuilder().SetType(\"task5\").SetQueue(\"custom\").SetGroup(\"group1\").Build()\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\tfxt := struct {\n\t\ttasks []*h.TaskSeedData\n\t\tallQueues []string\n\t\tallGroups map[string][]string\n\t\tgroups map[string][]redis.Z\n\t}{\n\t\ttasks: []*h.TaskSeedData{\n\t\t\t{Msg: m1, State: base.TaskStateAggregating},\n\t\t\t{Msg: m2, State: base.TaskStateAggregating},\n\t\t\t{Msg: m3, State: base.TaskStateAggregating},\n\t\t\t{Msg: m4, State: base.TaskStateAggregating},\n\t\t\t{Msg: m5, State: base.TaskStateAggregating},\n\t\t},\n\t\tallQueues: []string{\"default\", \"custom\"},\n\t\tallGroups: map[string][]string{\n\t\t\tbase.AllGroups(\"default\"): {\"group1\", \"group2\"},\n\t\t\tbase.AllGroups(\"custom\"): {\"group1\"},\n\t\t},\n\t\tgroups: map[string][]redis.Z{\n\t\t\tbase.GroupKey(\"default\", \"group1\"): {\n\t\t\t\t{Member: m1.ID, Score: float64(now.Add(-30 * time.Second).Unix())},\n\t\t\t\t{Member: m2.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t\t{Member: m3.ID, Score: float64(now.Add(-10 * time.Second).Unix())},\n\t\t\t},\n\t\t\tbase.GroupKey(\"default\", \"group2\"): {\n\t\t\t\t{Member: m4.ID, Score: float64(now.Add(-30 * time.Second).Unix())},\n\t\t\t},\n\t\t\tbase.GroupKey(\"custom\", \"group1\"): {\n\t\t\t\t{Member: m5.ID, Score: float64(now.Add(-30 * time.Second).Unix())},\n\t\t\t},\n\t\t},\n\t}\n\n\ttests := []struct {\n\t\tdesc string\n\t\tqname string\n\t\tgname string\n\t\twant []*TaskInfo\n\t}{\n\t\t{\n\t\t\tdesc: \"default queue group1\",\n\t\t\tqname: \"default\",\n\t\t\tgname: \"group1\",\n\t\t\twant: []*TaskInfo{\n\t\t\t\tcreateAggregatingTaskInfo(m1),\n\t\t\t\tcreateAggregatingTaskInfo(m2),\n\t\t\t\tcreateAggregatingTaskInfo(m3),\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"custom queue group1\",\n\t\t\tqname: \"custom\",\n\t\t\tgname: \"group1\",\n\t\t\twant: []*TaskInfo{\n\t\t\t\tcreateAggregatingTaskInfo(m5),\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedTasks(t, r, fxt.tasks)\n\t\th.SeedRedisSet(t, r, base.AllQueues, fxt.allQueues)\n\t\th.SeedRedisSets(t, r, fxt.allGroups)\n\t\th.SeedRedisZSets(t, r, fxt.groups)\n\n\t\tt.Run(tc.desc, func(t *testing.T) {\n\t\t\tgot, err := inspector.ListAggregatingTasks(tc.qname, tc.gname)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"ListAggregatingTasks returned error: %v\", err)\n\t\t\t}\n\n\t\t\tcmpOpts := []cmp.Option{\n\t\t\t\tcmpopts.EquateApproxTime(2 * time.Second),\n\t\t\t\tcmp.AllowUnexported(TaskInfo{}),\n\t\t\t}\n\t\t\tif diff := cmp.Diff(tc.want, got, cmpOpts...); diff != \"\" {\n\t\t\t\tt.Errorf(\"ListAggregatingTasks = %v, want = %v; (-want,+got)\\n%s\", got, tc.want, diff)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc createAggregatingTaskInfo(msg *base.TaskMessage) *TaskInfo {\n\treturn newTaskInfo(msg, base.TaskStateAggregating, time.Time{}, nil)\n}\n\nfunc TestInspectorListPagination(t *testing.T) {\n\t// Create 100 tasks.\n\tvar msgs []*base.TaskMessage\n\tfor i := 0; i <= 99; i++ {\n\t\tmsgs = append(msgs,\n\t\t\th.NewTaskMessage(fmt.Sprintf(\"task%d\", i), nil))\n\t}\n\tr := setup(t)\n\tdefer r.Close()\n\th.SeedPendingQueue(t, r, msgs, base.DefaultQueueName)\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\tpage int\n\t\tpageSize int\n\t\twant []*TaskInfo\n\t}{\n\t\t{\n\t\t\tpage: 1,\n\t\t\tpageSize: 5,\n\t\t\twant: []*TaskInfo{\n\t\t\t\tcreatePendingTask(msgs[0]),\n\t\t\t\tcreatePendingTask(msgs[1]),\n\t\t\t\tcreatePendingTask(msgs[2]),\n\t\t\t\tcreatePendingTask(msgs[3]),\n\t\t\t\tcreatePendingTask(msgs[4]),\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tpage: 3,\n\t\t\tpageSize: 10,\n\t\t\twant: []*TaskInfo{\n\t\t\t\tcreatePendingTask(msgs[20]),\n\t\t\t\tcreatePendingTask(msgs[21]),\n\t\t\t\tcreatePendingTask(msgs[22]),\n\t\t\t\tcreatePendingTask(msgs[23]),\n\t\t\t\tcreatePendingTask(msgs[24]),\n\t\t\t\tcreatePendingTask(msgs[25]),\n\t\t\t\tcreatePendingTask(msgs[26]),\n\t\t\t\tcreatePendingTask(msgs[27]),\n\t\t\t\tcreatePendingTask(msgs[28]),\n\t\t\t\tcreatePendingTask(msgs[29]),\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot, err := inspector.ListPendingTasks(\"default\", Page(tc.page), PageSize(tc.pageSize))\n\t\tif err != nil {\n\t\t\tt.Errorf(\"ListPendingTask('default') returned error: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tcmpOpts := []cmp.Option{\n\t\t\tcmpopts.EquateApproxTime(2 * time.Second),\n\t\t\tcmp.AllowUnexported(TaskInfo{}),\n\t\t}\n\t\tif diff := cmp.Diff(tc.want, got, cmpOpts...); diff != \"\" {\n\t\t\tt.Errorf(\"ListPendingTask('default') = %v, want %v; (-want,+got)\\n%s\",\n\t\t\t\tgot, tc.want, diff)\n\t\t}\n\t}\n}\n\nfunc TestInspectorListTasksQueueNotFoundError(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\tqname string\n\t\twantErr error\n\t}{\n\t\t{\n\t\t\tqname: \"nonexistent\",\n\t\t\twantErr: ErrQueueNotFound,\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\n\t\tif _, err := inspector.ListActiveTasks(tc.qname); !errors.Is(err, tc.wantErr) {\n\t\t\tt.Errorf(\"ListActiveTasks(%q) returned error %v, want %v\", tc.qname, err, tc.wantErr)\n\t\t}\n\t\tif _, err := inspector.ListPendingTasks(tc.qname); !errors.Is(err, tc.wantErr) {\n\t\t\tt.Errorf(\"ListPendingTasks(%q) returned error %v, want %v\", tc.qname, err, tc.wantErr)\n\t\t}\n\t\tif _, err := inspector.ListScheduledTasks(tc.qname); !errors.Is(err, tc.wantErr) {\n\t\t\tt.Errorf(\"ListScheduledTasks(%q) returned error %v, want %v\", tc.qname, err, tc.wantErr)\n\t\t}\n\t\tif _, err := inspector.ListRetryTasks(tc.qname); !errors.Is(err, tc.wantErr) {\n\t\t\tt.Errorf(\"ListRetryTasks(%q) returned error %v, want %v\", tc.qname, err, tc.wantErr)\n\t\t}\n\t\tif _, err := inspector.ListArchivedTasks(tc.qname); !errors.Is(err, tc.wantErr) {\n\t\t\tt.Errorf(\"ListArchivedTasks(%q) returned error %v, want %v\", tc.qname, err, tc.wantErr)\n\t\t}\n\t\tif _, err := inspector.ListCompletedTasks(tc.qname); !errors.Is(err, tc.wantErr) {\n\t\t\tt.Errorf(\"ListCompletedTasks(%q) returned error %v, want %v\", tc.qname, err, tc.wantErr)\n\t\t}\n\t\tif _, err := inspector.ListAggregatingTasks(tc.qname, \"mygroup\"); !errors.Is(err, tc.wantErr) {\n\t\t\tt.Errorf(\"ListAggregatingTasks(%q, \\\"mygroup\\\") returned error %v, want %v\", tc.qname, err, tc.wantErr)\n\t\t}\n\t}\n}\n\nfunc TestInspectorDeleteAllPendingTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessage(\"task3\", nil)\n\tm4 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\tpending map[string][]*base.TaskMessage\n\t\tqname string\n\t\twant int\n\t\twantPending map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2, m3},\n\t\t\t\t\"custom\": {m4},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 3,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {m4},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2, m3},\n\t\t\t\t\"custom\": {m4},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\twant: 1,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2, m3},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllPendingQueues(t, r, tc.pending)\n\n\t\tgot, err := inspector.DeleteAllPendingTasks(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"DeleteAllPendingTasks(%q) returned error: %v\", tc.qname, err)\n\t\t\tcontinue\n\t\t}\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"DeleteAllPendingTasks(%q) = %d, want %d\", tc.qname, got, tc.want)\n\t\t}\n\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected pending tasks in queue %q: (-want, +got)\\n%s\", qname, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestInspectorDeleteAllScheduledTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessage(\"task3\", nil)\n\tm4 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tnow := time.Now()\n\tz1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()}\n\tz2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()}\n\tz3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()}\n\tz4 := base.Z{Message: m4, Score: now.Add(2 * time.Minute).Unix()}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\tscheduled map[string][]base.Z\n\t\tqname string\n\t\twant int\n\t\twantScheduled map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2, z3},\n\t\t\t\t\"custom\": {z4},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 3,\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {z4},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 0,\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllScheduledQueues(t, r, tc.scheduled)\n\n\t\tgot, err := inspector.DeleteAllScheduledTasks(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"DeleteAllScheduledTasks(%q) returned error: %v\", tc.qname, err)\n\t\t\tcontinue\n\t\t}\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"DeleteAllScheduledTasks(%q) = %d, want %d\", tc.qname, got, tc.want)\n\t\t}\n\t\tfor qname, want := range tc.wantScheduled {\n\t\t\tgotScheduled := h.GetScheduledEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotScheduled, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected scheduled tasks in queue %q: (-want, +got)\\n%s\", qname, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestInspectorDeleteAllRetryTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessage(\"task3\", nil)\n\tm4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"custom\")\n\tnow := time.Now()\n\tz1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()}\n\tz2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()}\n\tz3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()}\n\tz4 := base.Z{Message: m4, Score: now.Add(2 * time.Minute).Unix()}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\tretry map[string][]base.Z\n\t\tqname string\n\t\twant int\n\t\twantRetry map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2, z3},\n\t\t\t\t\"custom\": {z4},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 3,\n\t\t\twantRetry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {z4},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 0,\n\t\t\twantRetry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllRetryQueues(t, r, tc.retry)\n\n\t\tgot, err := inspector.DeleteAllRetryTasks(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"DeleteAllRetryTasks(%q) returned error: %v\", tc.qname, err)\n\t\t\tcontinue\n\t\t}\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"DeleteAllRetryTasks(%q) = %d, want %d\", tc.qname, got, tc.want)\n\t\t}\n\t\tfor qname, want := range tc.wantRetry {\n\t\t\tgotRetry := h.GetRetryEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotRetry, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected retry tasks in queue %q: (-want, +got)\\n%s\", qname, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestInspectorDeleteAllArchivedTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessage(\"task3\", nil)\n\tm4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"custom\")\n\tnow := time.Now()\n\tz1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()}\n\tz2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()}\n\tz3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()}\n\tz4 := base.Z{Message: m4, Score: now.Add(2 * time.Minute).Unix()}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\tarchived map[string][]base.Z\n\t\tqname string\n\t\twant int\n\t\twantArchived map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2, z3},\n\t\t\t\t\"custom\": {z4},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 3,\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {z4},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 0,\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllArchivedQueues(t, r, tc.archived)\n\n\t\tgot, err := inspector.DeleteAllArchivedTasks(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"DeleteAllArchivedTasks(%q) returned error: %v\", tc.qname, err)\n\t\t\tcontinue\n\t\t}\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"DeleteAllArchivedTasks(%q) = %d, want %d\", tc.qname, got, tc.want)\n\t\t}\n\t\tfor qname, want := range tc.wantArchived {\n\t\t\tgotArchived := h.GetArchivedEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected archived tasks in queue %q: (-want, +got)\\n%s\", qname, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestInspectorDeleteAllCompletedTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tnow := time.Now()\n\tm1 := newCompletedTaskMessage(\"task1\", \"default\", 30*time.Minute, now.Add(-2*time.Minute))\n\tm2 := newCompletedTaskMessage(\"task2\", \"default\", 30*time.Minute, now.Add(-5*time.Minute))\n\tm3 := newCompletedTaskMessage(\"task3\", \"default\", 30*time.Minute, now.Add(-10*time.Minute))\n\tm4 := newCompletedTaskMessage(\"task4\", \"custom\", 30*time.Minute, now.Add(-3*time.Minute))\n\tz1 := base.Z{Message: m1, Score: m1.CompletedAt + m1.Retention}\n\tz2 := base.Z{Message: m2, Score: m2.CompletedAt + m2.Retention}\n\tz3 := base.Z{Message: m3, Score: m3.CompletedAt + m3.Retention}\n\tz4 := base.Z{Message: m4, Score: m4.CompletedAt + m4.Retention}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\tcompleted map[string][]base.Z\n\t\tqname string\n\t\twant int\n\t\twantCompleted map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tcompleted: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2, z3},\n\t\t\t\t\"custom\": {z4},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 3,\n\t\t\twantCompleted: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {z4},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tcompleted: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 0,\n\t\t\twantCompleted: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllCompletedQueues(t, r, tc.completed)\n\n\t\tgot, err := inspector.DeleteAllCompletedTasks(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"DeleteAllCompletedTasks(%q) returned error: %v\", tc.qname, err)\n\t\t\tcontinue\n\t\t}\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"DeleteAllCompletedTasks(%q) = %d, want %d\", tc.qname, got, tc.want)\n\t\t}\n\t\tfor qname, want := range tc.wantCompleted {\n\t\t\tgotCompleted := h.GetCompletedEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotCompleted, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected completed tasks in queue %q: (-want, +got)\\n%s\", qname, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestInspectorArchiveAllPendingTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessage(\"task3\", nil)\n\tm4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"custom\")\n\tnow := time.Now()\n\tz1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()}\n\tz2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()}\n\tinspector := NewInspector(getRedisConnOpt(t))\n\tinspector.rdb.SetClock(timeutil.NewSimulatedClock(now))\n\n\ttests := []struct {\n\t\tpending map[string][]*base.TaskMessage\n\t\tarchived map[string][]base.Z\n\t\tqname string\n\t\twant int\n\t\twantPending map[string][]*base.TaskMessage\n\t\twantArchived map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2, m3},\n\t\t\t\t\"custom\": {m4},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 3,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {m4},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\tbase.Z{Message: m1, Score: now.Unix()},\n\t\t\t\t\tbase.Z{Message: m2, Score: now.Unix()},\n\t\t\t\t\tbase.Z{Message: m3, Score: now.Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 0,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m3},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 1,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\tz1,\n\t\t\t\t\tz2,\n\t\t\t\t\tbase.Z{Message: m3, Score: now.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllPendingQueues(t, r, tc.pending)\n\t\th.SeedAllArchivedQueues(t, r, tc.archived)\n\n\t\tgot, err := inspector.ArchiveAllPendingTasks(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"ArchiveAllPendingTasks(%q) returned error: %v\", tc.qname, err)\n\t\t\tcontinue\n\t\t}\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"ArchiveAllPendingTasks(%q) = %d, want %d\", tc.qname, got, tc.want)\n\t\t}\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected pending tasks in queue %q: (-want, +got)\\n%s\", qname, diff)\n\t\t\t}\n\t\t}\n\t\tfor qname, want := range tc.wantArchived {\n\t\t\tgotArchived := h.GetArchivedEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected archived tasks in queue %q: (-want, +got)\\n%s\", qname, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestInspectorArchiveAllScheduledTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessage(\"task3\", nil)\n\tm4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"custom\")\n\tnow := time.Now()\n\tz1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()}\n\tz2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()}\n\tz3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()}\n\tz4 := base.Z{Message: m4, Score: now.Add(2 * time.Minute).Unix()}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\tinspector.rdb.SetClock(timeutil.NewSimulatedClock(now))\n\n\ttests := []struct {\n\t\tscheduled map[string][]base.Z\n\t\tarchived map[string][]base.Z\n\t\tqname string\n\t\twant int\n\t\twantScheduled map[string][]base.Z\n\t\twantArchived map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2, z3},\n\t\t\t\t\"custom\": {z4},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 3,\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {z4},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\tbase.Z{Message: m1, Score: now.Unix()},\n\t\t\t\t\tbase.Z{Message: m2, Score: now.Unix()},\n\t\t\t\t\tbase.Z{Message: m3, Score: now.Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {z3},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 2,\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\tz3,\n\t\t\t\t\tbase.Z{Message: m1, Score: now.Unix()},\n\t\t\t\t\tbase.Z{Message: m2, Score: now.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 0,\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 0,\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllScheduledQueues(t, r, tc.scheduled)\n\t\th.SeedAllArchivedQueues(t, r, tc.archived)\n\n\t\tgot, err := inspector.ArchiveAllScheduledTasks(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"ArchiveAllScheduledTasks(%q) returned error: %v\", tc.qname, err)\n\t\t\tcontinue\n\t\t}\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"ArchiveAllScheduledTasks(%q) = %d, want %d\", tc.qname, got, tc.want)\n\t\t}\n\t\tfor qname, want := range tc.wantScheduled {\n\t\t\tgotScheduled := h.GetScheduledEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotScheduled, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected scheduled tasks in queue %q: (-want, +got)\\n%s\", qname, diff)\n\t\t\t}\n\t\t}\n\t\tfor qname, want := range tc.wantArchived {\n\t\t\tgotArchived := h.GetArchivedEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected archived tasks in queue %q: (-want, +got)\\n%s\", qname, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestInspectorArchiveAllRetryTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessage(\"task3\", nil)\n\tm4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"custom\")\n\tnow := time.Now()\n\tz1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()}\n\tz2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()}\n\tz3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()}\n\tz4 := base.Z{Message: m4, Score: now.Add(2 * time.Minute).Unix()}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\tinspector.rdb.SetClock(timeutil.NewSimulatedClock(now))\n\n\ttests := []struct {\n\t\tretry map[string][]base.Z\n\t\tarchived map[string][]base.Z\n\t\tqname string\n\t\twant int\n\t\twantRetry map[string][]base.Z\n\t\twantArchived map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2, z3},\n\t\t\t\t\"custom\": {z4},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 3,\n\t\t\twantRetry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {z4},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\tbase.Z{Message: m1, Score: now.Unix()},\n\t\t\t\t\tbase.Z{Message: m2, Score: now.Unix()},\n\t\t\t\t\tbase.Z{Message: m3, Score: now.Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {z3},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 2,\n\t\t\twantRetry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\tz3,\n\t\t\t\t\tbase.Z{Message: m1, Score: now.Unix()},\n\t\t\t\t\tbase.Z{Message: m2, Score: now.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 0,\n\t\t\twantRetry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllRetryQueues(t, r, tc.retry)\n\t\th.SeedAllArchivedQueues(t, r, tc.archived)\n\n\t\tgot, err := inspector.ArchiveAllRetryTasks(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"ArchiveAllRetryTasks(%q) returned error: %v\", tc.qname, err)\n\t\t\tcontinue\n\t\t}\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"ArchiveAllRetryTasks(%q) = %d, want %d\", tc.qname, got, tc.want)\n\t\t}\n\t\tfor qname, want := range tc.wantRetry {\n\t\t\tgotRetry := h.GetRetryEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotRetry, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected retry tasks in queue %q: (-want, +got)\\n%s\", qname, diff)\n\t\t\t}\n\t\t}\n\t\tfor qname, want := range tc.wantArchived {\n\t\t\twantArchived := h.GetArchivedEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, wantArchived, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected archived tasks in queue %q: (-want, +got)\\n%s\", qname, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestInspectorRunAllScheduledTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessageWithQueue(\"task2\", nil, \"critical\")\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"low\")\n\tm4 := h.NewTaskMessage(\"task4\", nil)\n\tnow := time.Now()\n\tz1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()}\n\tz2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()}\n\tz3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()}\n\tz4 := base.Z{Message: m4, Score: now.Add(2 * time.Minute).Unix()}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\tscheduled map[string][]base.Z\n\t\tpending map[string][]*base.TaskMessage\n\t\tqname string\n\t\twant int\n\t\twantScheduled map[string][]base.Z\n\t\twantPending map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z4},\n\t\t\t\t\"critical\": {z2},\n\t\t\t\t\"low\": {z3},\n\t\t\t},\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 2,\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {z2},\n\t\t\t\t\"low\": {z3},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m4},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {z1},\n\t\t\t\t\"critical\": {z2},\n\t\t\t\t\"low\": {z3},\n\t\t\t},\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m4},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 1,\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {z2},\n\t\t\t\t\"low\": {z3},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m4, m1},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m4},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 0,\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m4},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllScheduledQueues(t, r, tc.scheduled)\n\t\th.SeedAllPendingQueues(t, r, tc.pending)\n\n\t\tgot, err := inspector.RunAllScheduledTasks(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"RunAllScheduledTasks(%q) returned error: %v\", tc.qname, err)\n\t\t\tcontinue\n\t\t}\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"RunAllScheduledTasks(%q) = %d, want %d\", tc.qname, got, tc.want)\n\t\t}\n\t\tfor qname, want := range tc.wantScheduled {\n\t\t\tgotScheduled := h.GetScheduledEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotScheduled, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected scheduled tasks in queue %q: (-want, +got)\\n%s\", qname, diff)\n\t\t\t}\n\t\t}\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected pending tasks in queue %q: (-want, +got)\\n%s\", qname, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestInspectorRunAllRetryTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessageWithQueue(\"task2\", nil, \"critical\")\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"low\")\n\tm4 := h.NewTaskMessage(\"task2\", nil)\n\tnow := time.Now()\n\tz1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()}\n\tz2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()}\n\tz3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()}\n\tz4 := base.Z{Message: m4, Score: now.Add(2 * time.Minute).Unix()}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\tretry map[string][]base.Z\n\t\tpending map[string][]*base.TaskMessage\n\t\tqname string\n\t\twant int\n\t\twantRetry map[string][]base.Z\n\t\twantPending map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z4},\n\t\t\t\t\"critical\": {z2},\n\t\t\t\t\"low\": {z3},\n\t\t\t},\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 2,\n\t\t\twantRetry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {z2},\n\t\t\t\t\"low\": {z3},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m4},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {z1},\n\t\t\t\t\"critical\": {z2},\n\t\t\t\t\"low\": {z3},\n\t\t\t},\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m4},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 1,\n\t\t\twantRetry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {z2},\n\t\t\t\t\"low\": {z3},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m4, m1},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m4},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 0,\n\t\t\twantRetry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m4},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllRetryQueues(t, r, tc.retry)\n\t\th.SeedAllPendingQueues(t, r, tc.pending)\n\n\t\tgot, err := inspector.RunAllRetryTasks(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"RunAllRetryTasks(%q) returned error: %v\", tc.qname, err)\n\t\t\tcontinue\n\t\t}\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"RunAllRetryTasks(%q) = %d, want %d\", tc.qname, got, tc.want)\n\t\t}\n\t\tfor qname, want := range tc.wantRetry {\n\t\t\tgotRetry := h.GetRetryEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotRetry, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected retry tasks in queue %q: (-want, +got)\\n%s\", qname, diff)\n\t\t\t}\n\t\t}\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected pending tasks in queue %q: (-want, +got)\\n%s\", qname, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestInspectorRunAllArchivedTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessageWithQueue(\"task2\", nil, \"critical\")\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"low\")\n\tm4 := h.NewTaskMessage(\"task2\", nil)\n\tnow := time.Now()\n\tz1 := base.Z{Message: m1, Score: now.Add(-5 * time.Minute).Unix()}\n\tz2 := base.Z{Message: m2, Score: now.Add(-15 * time.Minute).Unix()}\n\tz3 := base.Z{Message: m3, Score: now.Add(-2 * time.Minute).Unix()}\n\tz4 := base.Z{Message: m4, Score: now.Add(-2 * time.Minute).Unix()}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\tarchived map[string][]base.Z\n\t\tpending map[string][]*base.TaskMessage\n\t\tqname string\n\t\twant int\n\t\twantArchived map[string][]base.Z\n\t\twantPending map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z4},\n\t\t\t\t\"critical\": {z2},\n\t\t\t\t\"low\": {z3},\n\t\t\t},\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 2,\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {z2},\n\t\t\t\t\"low\": {z3},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m4},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {z1},\n\t\t\t\t\"critical\": {z2},\n\t\t\t},\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m4},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 1,\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {z2},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m4, m1},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m4},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 0,\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m4},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllArchivedQueues(t, r, tc.archived)\n\t\th.SeedAllPendingQueues(t, r, tc.pending)\n\n\t\tgot, err := inspector.RunAllArchivedTasks(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"RunAllArchivedTasks(%q) returned error: %v\", tc.qname, err)\n\t\t\tcontinue\n\t\t}\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"RunAllArchivedTasks(%q) = %d, want %d\", tc.qname, got, tc.want)\n\t\t}\n\t\tfor qname, want := range tc.wantArchived {\n\t\t\twantArchived := h.GetArchivedEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, wantArchived, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected archived tasks in queue %q: (-want, +got)\\n%s\", qname, diff)\n\t\t\t}\n\n\t\t}\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected pending tasks in queue %q: (-want, +got)\\n%s\", qname, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestInspectorUpdateTaskPayloadUpdatesScheduledTaskPayload(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1_old := h.NewTaskMessage(\"task1\", []byte(\"m1_old\"))\n\tm1_new := h.NewTaskMessage(\"task1\", nil)\n\tm1_new.ID = m1_old.ID\n\tm2_old := h.NewTaskMessage(\"task2\", nil)\n\tm2_new := h.NewTaskMessage(\"task2\", []byte(\"m2_new\"))\n\tm2_new.ID = m2_old.ID\n\tm3_old := h.NewTaskMessageWithQueue(\"task3\", []byte(\"m3_old\"), \"custom\")\n\tm3_new := h.NewTaskMessageWithQueue(\"task3\", []byte(\"m3_new\"), \"custom\")\n\tm3_new.ID = m3_old.ID\n\n\tnow := time.Now()\n\tz1_old := base.Z{Message: m1_old, Score: now.Add(5 * time.Minute).Unix()}\n\tz1_new := base.Z{Message: m1_new, Score: now.Add(5 * time.Minute).Unix()}\n\tz2_old := base.Z{Message: m2_old, Score: now.Add(15 * time.Minute).Unix()}\n\tz2_new := base.Z{Message: m2_new, Score: now.Add(15 * time.Minute).Unix()}\n\tz3_old := base.Z{Message: m3_old, Score: now.Add(2 * time.Minute).Unix()}\n\tz3_new := base.Z{Message: m3_new, Score: now.Add(2 * time.Minute).Unix()}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\tscheduled map[string][]base.Z\n\t\tqname string\n\t\tid string\n\t\tnewPayload []byte\n\t\twantScheduled map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {z1_old, z2_old},\n\t\t\t\t\"custom\": {z3_old},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: createScheduledTask(z2_old).ID,\n\t\t\tnewPayload: m2_new.Payload,\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {z1_old, z2_new},\n\t\t\t\t\"custom\": {z3_old},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {z1_old, z2_old},\n\t\t\t\t\"custom\": {z3_old},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: createScheduledTask(z1_old).ID,\n\t\t\tnewPayload: m1_new.Payload,\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {z1_new, z2_old},\n\t\t\t\t\"custom\": {z3_old},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {z1_old, z2_old},\n\t\t\t\t\"custom\": {z3_old},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\tid: createScheduledTask(z3_old).ID,\n\t\t\tnewPayload: m3_new.Payload,\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {z1_old, z2_old},\n\t\t\t\t\"custom\": {z3_new},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllScheduledQueues(t, r, tc.scheduled)\n\n\t\tif err := inspector.UpdateTaskPayload(tc.qname, tc.id, tc.newPayload); err != nil {\n\t\t\tt.Errorf(\"UpdateTask(%q, %q) returned error: %v\", tc.qname, tc.id, err)\n\t\t}\n\t\tfor qname, want := range tc.wantScheduled {\n\t\t\tgotScheduled := h.GetScheduledEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotScheduled, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected scheduled tasks in queue %q: (-want, +got)\\n%s\", qname, diff)\n\t\t\t}\n\n\t\t}\n\t}\n}\n\nfunc TestInspectorUpdateTaskPayloadError(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\n\tnow := time.Now()\n\tz1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()}\n\tz2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()}\n\tz3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\ttasks map[string][]base.Z\n\t\tqname string\n\t\tid string\n\t\tnewPayload []byte\n\t\twantErr error\n\t}{\n\t\t{\n\t\t\ttasks: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2},\n\t\t\t\t\"custom\": {z3},\n\t\t\t},\n\t\t\tqname: \"nonexistent\",\n\t\t\tid: createScheduledTask(z2).ID,\n\t\t\tnewPayload: nil,\n\t\t\twantErr: ErrQueueNotFound,\n\t\t},\n\t\t{\n\t\t\ttasks: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2},\n\t\t\t\t\"custom\": {z3},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: uuid.NewString(),\n\t\t\tnewPayload: nil,\n\t\t\twantErr: ErrTaskNotFound,\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllScheduledQueues(t, r, tc.tasks)\n\n\t\tif err := inspector.UpdateTaskPayload(tc.qname, tc.id, tc.newPayload); !errors.Is(err, tc.wantErr) {\n\t\t\tt.Errorf(\"UpdateTask(%q, %q) = %v, want %v\", tc.qname, tc.id, err, tc.wantErr)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\nfunc TestInspectorDeleteTaskDeletesPendingTask(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\tpending map[string][]*base.TaskMessage\n\t\tqname string\n\t\tid string\n\t\twantPending map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t\t\"custom\": {m3},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: createPendingTask(m2).ID,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1},\n\t\t\t\t\"custom\": {m3},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t\t\"custom\": {m3},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\tid: createPendingTask(m3).ID,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllPendingQueues(t, r, tc.pending)\n\n\t\tif err := inspector.DeleteTask(tc.qname, tc.id); err != nil {\n\t\t\tt.Errorf(\"DeleteTask(%q, %q) returned error: %v\", tc.qname, tc.id, err)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgot := h.GetPendingMessages(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, got, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unspected pending tasks in queue %q: (-want,+got):\\n%s\",\n\t\t\t\t\tqname, diff)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestInspectorDeleteTaskDeletesScheduledTask(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tnow := time.Now()\n\tz1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()}\n\tz2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()}\n\tz3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\tscheduled map[string][]base.Z\n\t\tqname string\n\t\tid string\n\t\twantScheduled map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2},\n\t\t\t\t\"custom\": {z3},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: createScheduledTask(z2).ID,\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {z1},\n\t\t\t\t\"custom\": {z3},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllScheduledQueues(t, r, tc.scheduled)\n\n\t\tif err := inspector.DeleteTask(tc.qname, tc.id); err != nil {\n\t\t\tt.Errorf(\"DeleteTask(%q, %q) returned error: %v\", tc.qname, tc.id, err)\n\t\t}\n\t\tfor qname, want := range tc.wantScheduled {\n\t\t\tgotScheduled := h.GetScheduledEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotScheduled, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected scheduled tasks in queue %q: (-want, +got)\\n%s\", qname, diff)\n\t\t\t}\n\n\t\t}\n\t}\n}\n\nfunc TestInspectorDeleteTaskDeletesRetryTask(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tnow := time.Now()\n\tz1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()}\n\tz2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()}\n\tz3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\tretry map[string][]base.Z\n\t\tqname string\n\t\tid string\n\t\twantRetry map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2},\n\t\t\t\t\"custom\": {z3},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: createRetryTask(z2).ID,\n\t\t\twantRetry: map[string][]base.Z{\n\t\t\t\t\"default\": {z1},\n\t\t\t\t\"custom\": {z3},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllRetryQueues(t, r, tc.retry)\n\n\t\tif err := inspector.DeleteTask(tc.qname, tc.id); err != nil {\n\t\t\tt.Errorf(\"DeleteTask(%q, %q) returned error: %v\", tc.qname, tc.id, err)\n\t\t\tcontinue\n\t\t}\n\t\tfor qname, want := range tc.wantRetry {\n\t\t\tgotRetry := h.GetRetryEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotRetry, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected retry tasks in queue %q: (-want, +got)\\n%s\", qname, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestInspectorDeleteTaskDeletesArchivedTask(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tnow := time.Now()\n\tz1 := base.Z{Message: m1, Score: now.Add(-5 * time.Minute).Unix()}\n\tz2 := base.Z{Message: m2, Score: now.Add(-15 * time.Minute).Unix()}\n\tz3 := base.Z{Message: m3, Score: now.Add(-2 * time.Minute).Unix()}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\tarchived map[string][]base.Z\n\t\tqname string\n\t\tid string\n\t\twantArchived map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2},\n\t\t\t\t\"custom\": {z3},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: createArchivedTask(z2).ID,\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {z1},\n\t\t\t\t\"custom\": {z3},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllArchivedQueues(t, r, tc.archived)\n\n\t\tif err := inspector.DeleteTask(tc.qname, tc.id); err != nil {\n\t\t\tt.Errorf(\"DeleteTask(%q, %q) returned error: %v\", tc.qname, tc.id, err)\n\t\t\tcontinue\n\t\t}\n\t\tfor qname, want := range tc.wantArchived {\n\t\t\twantArchived := h.GetArchivedEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, wantArchived, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected archived tasks in queue %q: (-want, +got)\\n%s\", qname, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestInspectorDeleteTaskError(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tnow := time.Now()\n\tz1 := base.Z{Message: m1, Score: now.Add(-5 * time.Minute).Unix()}\n\tz2 := base.Z{Message: m2, Score: now.Add(-15 * time.Minute).Unix()}\n\tz3 := base.Z{Message: m3, Score: now.Add(-2 * time.Minute).Unix()}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\tarchived map[string][]base.Z\n\t\tqname string\n\t\tid string\n\t\twantErr error\n\t\twantArchived map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2},\n\t\t\t\t\"custom\": {z3},\n\t\t\t},\n\t\t\tqname: \"nonexistent\",\n\t\t\tid: createArchivedTask(z2).ID,\n\t\t\twantErr: ErrQueueNotFound,\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2},\n\t\t\t\t\"custom\": {z3},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2},\n\t\t\t\t\"custom\": {z3},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: uuid.NewString(),\n\t\t\twantErr: ErrTaskNotFound,\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2},\n\t\t\t\t\"custom\": {z3},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllArchivedQueues(t, r, tc.archived)\n\n\t\tif err := inspector.DeleteTask(tc.qname, tc.id); !errors.Is(err, tc.wantErr) {\n\t\t\tt.Errorf(\"DeleteTask(%q, %q) = %v, want %v\", tc.qname, tc.id, err, tc.wantErr)\n\t\t\tcontinue\n\t\t}\n\t\tfor qname, want := range tc.wantArchived {\n\t\t\twantArchived := h.GetArchivedEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, wantArchived, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected archived tasks in queue %q: (-want, +got)\\n%s\", qname, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestInspectorRunTaskRunsScheduledTask(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tnow := time.Now()\n\tz1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()}\n\tz2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()}\n\tz3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\tscheduled map[string][]base.Z\n\t\tpending map[string][]*base.TaskMessage\n\t\tqname string\n\t\tid string\n\t\twantScheduled map[string][]base.Z\n\t\twantPending map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {z1, z2},\n\t\t\t\t\"custom\": {z3},\n\t\t\t},\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: createScheduledTask(z2).ID,\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {z1},\n\t\t\t\t\"custom\": {z3},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m2},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllScheduledQueues(t, r, tc.scheduled)\n\t\th.SeedAllPendingQueues(t, r, tc.pending)\n\n\t\tif err := inspector.RunTask(tc.qname, tc.id); err != nil {\n\t\t\tt.Errorf(\"RunTask(%q, %q) returned error: %v\", tc.qname, tc.id, err)\n\t\t\tcontinue\n\t\t}\n\t\tfor qname, want := range tc.wantScheduled {\n\t\t\tgotScheduled := h.GetScheduledEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotScheduled, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected scheduled tasks in queue %q: (-want, +got)\\n%s\",\n\t\t\t\t\tqname, diff)\n\t\t\t}\n\n\t\t}\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected pending tasks in queue %q: (-want, +got)\\n%s\",\n\t\t\t\t\tqname, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestInspectorRunTaskRunsRetryTask(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessageWithQueue(\"task2\", nil, \"custom\")\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tnow := time.Now()\n\tz1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()}\n\tz2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()}\n\tz3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\tretry map[string][]base.Z\n\t\tpending map[string][]*base.TaskMessage\n\t\tqname string\n\t\tid string\n\t\twantRetry map[string][]base.Z\n\t\twantPending map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {z1},\n\t\t\t\t\"custom\": {z2, z3},\n\t\t\t},\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\tid: createRetryTask(z2).ID,\n\t\t\twantRetry: map[string][]base.Z{\n\t\t\t\t\"default\": {z1},\n\t\t\t\t\"custom\": {z3},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {m2},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllRetryQueues(t, r, tc.retry)\n\t\th.SeedAllPendingQueues(t, r, tc.pending)\n\n\t\tif err := inspector.RunTask(tc.qname, tc.id); err != nil {\n\t\t\tt.Errorf(\"RunTaskBy(%q, %q) returned error: %v\", tc.qname, tc.id, err)\n\t\t\tcontinue\n\t\t}\n\t\tfor qname, want := range tc.wantRetry {\n\t\t\tgotRetry := h.GetRetryEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotRetry, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected retry tasks in queue %q: (-want, +got)\\n%s\",\n\t\t\t\t\tqname, diff)\n\t\t\t}\n\t\t}\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected pending tasks in queue %q: (-want, +got)\\n%s\",\n\t\t\t\t\tqname, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestInspectorRunTaskRunsArchivedTask(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessageWithQueue(\"task2\", nil, \"critical\")\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"low\")\n\tnow := time.Now()\n\tz1 := base.Z{Message: m1, Score: now.Add(-5 * time.Minute).Unix()}\n\tz2 := base.Z{Message: m2, Score: now.Add(-15 * time.Minute).Unix()}\n\tz3 := base.Z{Message: m3, Score: now.Add(-2 * time.Minute).Unix()}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\tarchived map[string][]base.Z\n\t\tpending map[string][]*base.TaskMessage\n\t\tqname string\n\t\tid string\n\t\twantArchived map[string][]base.Z\n\t\twantPending map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {z1},\n\t\t\t\t\"critical\": {z2},\n\t\t\t\t\"low\": {z3},\n\t\t\t},\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t\tqname: \"critical\",\n\t\t\tid: createArchivedTask(z2).ID,\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {z1},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {z3},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {m2},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllArchivedQueues(t, r, tc.archived)\n\t\th.SeedAllPendingQueues(t, r, tc.pending)\n\n\t\tif err := inspector.RunTask(tc.qname, tc.id); err != nil {\n\t\t\tt.Errorf(\"RunTask(%q, %q) returned error: %v\", tc.qname, tc.id, err)\n\t\t\tcontinue\n\t\t}\n\t\tfor qname, want := range tc.wantArchived {\n\t\t\twantArchived := h.GetArchivedEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, wantArchived, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected archived tasks in queue %q: (-want, +got)\\n%s\",\n\t\t\t\t\tqname, diff)\n\t\t\t}\n\t\t}\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected pending tasks in queue %q: (-want, +got)\\n%s\",\n\t\t\t\t\tqname, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestInspectorRunTaskError(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessageWithQueue(\"task2\", nil, \"critical\")\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"low\")\n\tnow := time.Now()\n\tz1 := base.Z{Message: m1, Score: now.Add(-5 * time.Minute).Unix()}\n\tz2 := base.Z{Message: m2, Score: now.Add(-15 * time.Minute).Unix()}\n\tz3 := base.Z{Message: m3, Score: now.Add(-2 * time.Minute).Unix()}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\ttests := []struct {\n\t\tarchived map[string][]base.Z\n\t\tpending map[string][]*base.TaskMessage\n\t\tqname string\n\t\tid string\n\t\twantErr error\n\t\twantArchived map[string][]base.Z\n\t\twantPending map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {z1},\n\t\t\t\t\"critical\": {z2},\n\t\t\t\t\"low\": {z3},\n\t\t\t},\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t\tqname: \"nonexistent\",\n\t\t\tid: createArchivedTask(z2).ID,\n\t\t\twantErr: ErrQueueNotFound,\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {z1},\n\t\t\t\t\"critical\": {z2},\n\t\t\t\t\"low\": {z3},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {z1},\n\t\t\t\t\"critical\": {z2},\n\t\t\t\t\"low\": {z3},\n\t\t\t},\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: uuid.NewString(),\n\t\t\twantErr: ErrTaskNotFound,\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {z1},\n\t\t\t\t\"critical\": {z2},\n\t\t\t\t\"low\": {z3},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllArchivedQueues(t, r, tc.archived)\n\t\th.SeedAllPendingQueues(t, r, tc.pending)\n\n\t\tif err := inspector.RunTask(tc.qname, tc.id); !errors.Is(err, tc.wantErr) {\n\t\t\tt.Errorf(\"RunTask(%q, %q) = %v, want %v\", tc.qname, tc.id, err, tc.wantErr)\n\t\t\tcontinue\n\t\t}\n\t\tfor qname, want := range tc.wantArchived {\n\t\t\twantArchived := h.GetArchivedEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, wantArchived, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected archived tasks in queue %q: (-want, +got)\\n%s\",\n\t\t\t\t\tqname, diff)\n\t\t\t}\n\t\t}\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected pending tasks in queue %q: (-want, +got)\\n%s\",\n\t\t\t\t\tqname, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestInspectorArchiveTaskArchivesPendingTask(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessageWithQueue(\"task2\", nil, \"custom\")\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tnow := time.Now()\n\tinspector := NewInspector(getRedisConnOpt(t))\n\tinspector.rdb.SetClock(timeutil.NewSimulatedClock(now))\n\n\ttests := []struct {\n\t\tpending map[string][]*base.TaskMessage\n\t\tarchived map[string][]base.Z\n\t\tqname string\n\t\tid string\n\t\twantPending map[string][]*base.TaskMessage\n\t\twantArchived map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1},\n\t\t\t\t\"custom\": {m2, m3},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: createPendingTask(m1).ID,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {m2, m3},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: now.Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1},\n\t\t\t\t\"custom\": {m2, m3},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\tid: createPendingTask(m2).ID,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1},\n\t\t\t\t\"custom\": {m3},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: m2, Score: now.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllPendingQueues(t, r, tc.pending)\n\t\th.SeedAllArchivedQueues(t, r, tc.archived)\n\n\t\tif err := inspector.ArchiveTask(tc.qname, tc.id); err != nil {\n\t\t\tt.Errorf(\"ArchiveTask(%q, %q) returned error: %v\", tc.qname, tc.id, err)\n\t\t\tcontinue\n\t\t}\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected pending tasks in queue %q: (-want,+got)\\n%s\",\n\t\t\t\t\tqname, diff)\n\t\t\t}\n\n\t\t}\n\t\tfor qname, want := range tc.wantArchived {\n\t\t\twantArchived := h.GetArchivedEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, wantArchived, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected archived tasks in queue %q: (-want,+got)\\n%s\",\n\t\t\t\t\tqname, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestInspectorArchiveTaskArchivesScheduledTask(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessageWithQueue(\"task2\", nil, \"custom\")\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tnow := time.Now()\n\tz1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()}\n\tz2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()}\n\tz3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\tinspector.rdb.SetClock(timeutil.NewSimulatedClock(now))\n\n\ttests := []struct {\n\t\tscheduled map[string][]base.Z\n\t\tarchived map[string][]base.Z\n\t\tqname string\n\t\tid string\n\t\twant string\n\t\twantScheduled map[string][]base.Z\n\t\twantArchived map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {z1},\n\t\t\t\t\"custom\": {z2, z3},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\tid: createScheduledTask(z2).ID,\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {z1},\n\t\t\t\t\"custom\": {z3},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{\n\t\t\t\t\t\tMessage: m2,\n\t\t\t\t\t\tScore: now.Unix(),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllScheduledQueues(t, r, tc.scheduled)\n\t\th.SeedAllArchivedQueues(t, r, tc.archived)\n\n\t\tif err := inspector.ArchiveTask(tc.qname, tc.id); err != nil {\n\t\t\tt.Errorf(\"ArchiveTask(%q, %q) returned error: %v\", tc.qname, tc.id, err)\n\t\t\tcontinue\n\t\t}\n\t\tfor qname, want := range tc.wantScheduled {\n\t\t\tgotScheduled := h.GetScheduledEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotScheduled, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected scheduled tasks in queue %q: (-want, +got)\\n%s\",\n\t\t\t\t\tqname, diff)\n\t\t\t}\n\n\t\t}\n\t\tfor qname, want := range tc.wantArchived {\n\t\t\twantArchived := h.GetArchivedEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, wantArchived, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected archived tasks in queue %q: (-want, +got)\\n%s\",\n\t\t\t\t\tqname, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestInspectorArchiveTaskArchivesRetryTask(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessageWithQueue(\"task2\", nil, \"custom\")\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tnow := time.Now()\n\tz1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()}\n\tz2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()}\n\tz3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\tinspector.rdb.SetClock(timeutil.NewSimulatedClock(now))\n\n\ttests := []struct {\n\t\tretry map[string][]base.Z\n\t\tarchived map[string][]base.Z\n\t\tqname string\n\t\tid string\n\t\twantRetry map[string][]base.Z\n\t\twantArchived map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {z1},\n\t\t\t\t\"custom\": {z2, z3},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\tid: createRetryTask(z2).ID,\n\t\t\twantRetry: map[string][]base.Z{\n\t\t\t\t\"default\": {z1},\n\t\t\t\t\"custom\": {z3},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{\n\t\t\t\t\t\tMessage: m2,\n\t\t\t\t\t\tScore: now.Unix(),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllRetryQueues(t, r, tc.retry)\n\t\th.SeedAllArchivedQueues(t, r, tc.archived)\n\n\t\tif err := inspector.ArchiveTask(tc.qname, tc.id); err != nil {\n\t\t\tt.Errorf(\"ArchiveTask(%q, %q) returned error: %v\", tc.qname, tc.id, err)\n\t\t\tcontinue\n\t\t}\n\t\tfor qname, want := range tc.wantRetry {\n\t\t\tgotRetry := h.GetRetryEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotRetry, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected retry tasks in queue %q: (-want, +got)\\n%s\",\n\t\t\t\t\tqname, diff)\n\t\t\t}\n\t\t}\n\t\tfor qname, want := range tc.wantArchived {\n\t\t\twantArchived := h.GetArchivedEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, wantArchived, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected archived tasks in queue %q: (-want, +got)\\n%s\",\n\t\t\t\t\tqname, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestInspectorArchiveTaskError(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessageWithQueue(\"task2\", nil, \"custom\")\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tnow := time.Now()\n\tz1 := base.Z{Message: m1, Score: now.Add(5 * time.Minute).Unix()}\n\tz2 := base.Z{Message: m2, Score: now.Add(15 * time.Minute).Unix()}\n\tz3 := base.Z{Message: m3, Score: now.Add(2 * time.Minute).Unix()}\n\n\tinspector := NewInspector(getRedisConnOpt(t))\n\tinspector.rdb.SetClock(timeutil.NewSimulatedClock(now))\n\n\ttests := []struct {\n\t\tretry map[string][]base.Z\n\t\tarchived map[string][]base.Z\n\t\tqname string\n\t\tid string\n\t\twantErr error\n\t\twantRetry map[string][]base.Z\n\t\twantArchived map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {z1},\n\t\t\t\t\"custom\": {z2, z3},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqname: \"nonexistent\",\n\t\t\tid: createRetryTask(z2).ID,\n\t\t\twantErr: ErrQueueNotFound,\n\t\t\twantRetry: map[string][]base.Z{\n\t\t\t\t\"default\": {z1},\n\t\t\t\t\"custom\": {z2, z3},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {z1},\n\t\t\t\t\"custom\": {z2, z3},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\tid: uuid.NewString(),\n\t\t\twantErr: ErrTaskNotFound,\n\t\t\twantRetry: map[string][]base.Z{\n\t\t\t\t\"default\": {z1},\n\t\t\t\t\"custom\": {z2, z3},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllRetryQueues(t, r, tc.retry)\n\t\th.SeedAllArchivedQueues(t, r, tc.archived)\n\n\t\tif err := inspector.ArchiveTask(tc.qname, tc.id); !errors.Is(err, tc.wantErr) {\n\t\t\tt.Errorf(\"ArchiveTask(%q, %q) = %v, want %v\", tc.qname, tc.id, err, tc.wantErr)\n\t\t\tcontinue\n\t\t}\n\t\tfor qname, want := range tc.wantRetry {\n\t\t\tgotRetry := h.GetRetryEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotRetry, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected retry tasks in queue %q: (-want, +got)\\n%s\",\n\t\t\t\t\tqname, diff)\n\t\t\t}\n\t\t}\n\t\tfor qname, want := range tc.wantArchived {\n\t\t\twantArchived := h.GetArchivedEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, wantArchived, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"unexpected archived tasks in queue %q: (-want, +got)\\n%s\",\n\t\t\t\t\tqname, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nvar sortSchedulerEntry = cmp.Transformer(\"SortSchedulerEntry\", func(in []*SchedulerEntry) []*SchedulerEntry {\n\tout := append([]*SchedulerEntry(nil), in...)\n\tsort.Slice(out, func(i, j int) bool {\n\t\treturn out[i].Spec < out[j].Spec\n\t})\n\treturn out\n})\n\nfunc TestInspectorSchedulerEntries(t *testing.T) {\n\tr := setup(t)\n\trdbClient := rdb.NewRDB(r)\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\tnow := time.Now().UTC()\n\tschedulerID := \"127.0.0.1:9876:abc123\"\n\n\ttests := []struct {\n\t\tdata []*base.SchedulerEntry // data to seed redis\n\t\twant []*SchedulerEntry\n\t}{\n\t\t{\n\t\t\tdata: []*base.SchedulerEntry{\n\t\t\t\t{\n\t\t\t\t\tSpec: \"* * * * *\",\n\t\t\t\t\tType: \"foo\",\n\t\t\t\t\tPayload: nil,\n\t\t\t\t\tOpts: nil,\n\t\t\t\t\tNext: now.Add(5 * time.Hour),\n\t\t\t\t\tPrev: now.Add(-2 * time.Hour),\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tSpec: \"@every 20m\",\n\t\t\t\t\tType: \"bar\",\n\t\t\t\t\tPayload: h.JSON(map[string]interface{}{\"fiz\": \"baz\"}),\n\t\t\t\t\tOpts: []string{`Queue(\"bar\")`, `MaxRetry(20)`},\n\t\t\t\t\tNext: now.Add(1 * time.Minute),\n\t\t\t\t\tPrev: now.Add(-19 * time.Minute),\n\t\t\t\t},\n\t\t\t},\n\t\t\twant: []*SchedulerEntry{\n\t\t\t\t{\n\t\t\t\t\tSpec: \"* * * * *\",\n\t\t\t\t\tTask: NewTask(\"foo\", nil),\n\t\t\t\t\tOpts: nil,\n\t\t\t\t\tNext: now.Add(5 * time.Hour),\n\t\t\t\t\tPrev: now.Add(-2 * time.Hour),\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tSpec: \"@every 20m\",\n\t\t\t\t\tTask: NewTask(\"bar\", h.JSON(map[string]interface{}{\"fiz\": \"baz\"})),\n\t\t\t\t\tOpts: []Option{Queue(\"bar\"), MaxRetry(20)},\n\t\t\t\t\tNext: now.Add(1 * time.Minute),\n\t\t\t\t\tPrev: now.Add(-19 * time.Minute),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\terr := rdbClient.WriteSchedulerEntries(schedulerID, tc.data, time.Minute)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"could not write data: %v\", err)\n\t\t}\n\t\tgot, err := inspector.SchedulerEntries()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"SchedulerEntries() returned error: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tignoreOpt := cmpopts.IgnoreUnexported(Task{})\n\t\tif diff := cmp.Diff(tc.want, got, sortSchedulerEntry, ignoreOpt); diff != \"\" {\n\t\t\tt.Errorf(\"SchedulerEntries() = %v, want %v; (-want,+got)\\n%s\",\n\t\t\t\tgot, tc.want, diff)\n\t\t}\n\t}\n}\n\nfunc TestParseOption(t *testing.T) {\n\toneHourFromNow := time.Now().Add(1 * time.Hour)\n\ttests := []struct {\n\t\ts string\n\t\twantType OptionType\n\t\twantVal interface{}\n\t}{\n\t\t{`MaxRetry(10)`, MaxRetryOpt, 10},\n\t\t{`Queue(\"email\")`, QueueOpt, \"email\"},\n\t\t{`Timeout(3m)`, TimeoutOpt, 3 * time.Minute},\n\t\t{Deadline(oneHourFromNow).String(), DeadlineOpt, oneHourFromNow},\n\t\t{`Unique(1h)`, UniqueOpt, 1 * time.Hour},\n\t\t{ProcessAt(oneHourFromNow).String(), ProcessAtOpt, oneHourFromNow},\n\t\t{`ProcessIn(10m)`, ProcessInOpt, 10 * time.Minute},\n\t\t{`Retention(24h)`, RetentionOpt, 24 * time.Hour},\n\t}\n\n\tfor _, tc := range tests {\n\t\tt.Run(tc.s, func(t *testing.T) {\n\t\t\tgot, err := parseOption(tc.s)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"returned error: %v\", err)\n\t\t\t}\n\t\t\tif got == nil {\n\t\t\t\tt.Fatal(\"returned nil\")\n\t\t\t}\n\t\t\tif got.Type() != tc.wantType {\n\t\t\t\tt.Fatalf(\"got type %v, want type %v \", got.Type(), tc.wantType)\n\t\t\t}\n\t\t\tswitch tc.wantType {\n\t\t\tcase QueueOpt:\n\t\t\t\tgotVal, ok := got.Value().(string)\n\t\t\t\tif !ok {\n\t\t\t\t\tt.Fatal(\"returned Option with non-string value\")\n\t\t\t\t}\n\t\t\t\tif gotVal != tc.wantVal.(string) {\n\t\t\t\t\tt.Fatalf(\"got value %v, want %v\", gotVal, tc.wantVal)\n\t\t\t\t}\n\t\t\tcase MaxRetryOpt:\n\t\t\t\tgotVal, ok := got.Value().(int)\n\t\t\t\tif !ok {\n\t\t\t\t\tt.Fatal(\"returned Option with non-int value\")\n\t\t\t\t}\n\t\t\t\tif gotVal != tc.wantVal.(int) {\n\t\t\t\t\tt.Fatalf(\"got value %v, want %v\", gotVal, tc.wantVal)\n\t\t\t\t}\n\t\t\tcase TimeoutOpt, UniqueOpt, ProcessInOpt, RetentionOpt:\n\t\t\t\tgotVal, ok := got.Value().(time.Duration)\n\t\t\t\tif !ok {\n\t\t\t\t\tt.Fatal(\"returned Option with non duration value\")\n\t\t\t\t}\n\t\t\t\tif gotVal != tc.wantVal.(time.Duration) {\n\t\t\t\t\tt.Fatalf(\"got value %v, want %v\", gotVal, tc.wantVal)\n\t\t\t\t}\n\t\t\tcase DeadlineOpt, ProcessAtOpt:\n\t\t\t\tgotVal, ok := got.Value().(time.Time)\n\t\t\t\tif !ok {\n\t\t\t\t\tt.Fatal(\"returned Option with non time value\")\n\t\t\t\t}\n\t\t\t\tif cmp.Equal(gotVal, tc.wantVal.(time.Time)) {\n\t\t\t\t\tt.Fatalf(\"got value %v, want %v\", gotVal, tc.wantVal)\n\t\t\t\t}\n\t\t\tdefault:\n\t\t\t\tt.Fatalf(\"returned Option with unexpected type: %v\", got.Type())\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestInspectorGroups(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tinspector := NewInspector(getRedisConnOpt(t))\n\n\tm1 := h.NewTaskMessageBuilder().SetGroup(\"group1\").Build()\n\tm2 := h.NewTaskMessageBuilder().SetGroup(\"group1\").Build()\n\tm3 := h.NewTaskMessageBuilder().SetGroup(\"group1\").Build()\n\tm4 := h.NewTaskMessageBuilder().SetGroup(\"group2\").Build()\n\tm5 := h.NewTaskMessageBuilder().SetQueue(\"custom\").SetGroup(\"group1\").Build()\n\tm6 := h.NewTaskMessageBuilder().SetQueue(\"custom\").SetGroup(\"group1\").Build()\n\n\tnow := time.Now()\n\n\tfixtures := struct {\n\t\ttasks []*h.TaskSeedData\n\t\tallGroups map[string][]string\n\t\tgroups map[string][]redis.Z\n\t}{\n\t\ttasks: []*h.TaskSeedData{\n\t\t\t{Msg: m1, State: base.TaskStateAggregating},\n\t\t\t{Msg: m2, State: base.TaskStateAggregating},\n\t\t\t{Msg: m3, State: base.TaskStateAggregating},\n\t\t\t{Msg: m4, State: base.TaskStateAggregating},\n\t\t\t{Msg: m5, State: base.TaskStateAggregating},\n\t\t},\n\t\tallGroups: map[string][]string{\n\t\t\tbase.AllGroups(\"default\"): {\"group1\", \"group2\"},\n\t\t\tbase.AllGroups(\"custom\"): {\"group1\"},\n\t\t},\n\t\tgroups: map[string][]redis.Z{\n\t\t\tbase.GroupKey(\"default\", \"group1\"): {\n\t\t\t\t{Member: m1.ID, Score: float64(now.Add(-10 * time.Second).Unix())},\n\t\t\t\t{Member: m2.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t\t{Member: m3.ID, Score: float64(now.Add(-30 * time.Second).Unix())},\n\t\t\t},\n\t\t\tbase.GroupKey(\"default\", \"group2\"): {\n\t\t\t\t{Member: m4.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t},\n\t\t\tbase.GroupKey(\"custom\", \"group1\"): {\n\t\t\t\t{Member: m5.ID, Score: float64(now.Add(-10 * time.Second).Unix())},\n\t\t\t\t{Member: m6.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t},\n\t\t},\n\t}\n\n\ttests := []struct {\n\t\tdesc string\n\t\tqname string\n\t\twant []*GroupInfo\n\t}{\n\t\t{\n\t\t\tdesc: \"default queue groups\",\n\t\t\tqname: \"default\",\n\t\t\twant: []*GroupInfo{\n\t\t\t\t{Group: \"group1\", Size: 3},\n\t\t\t\t{Group: \"group2\", Size: 1},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"custom queue groups\",\n\t\t\tqname: \"custom\",\n\t\t\twant: []*GroupInfo{\n\t\t\t\t{Group: \"group1\", Size: 2},\n\t\t\t},\n\t\t},\n\t}\n\n\tvar sortGroupInfosOpt = cmp.Transformer(\n\t\t\"SortGroupInfos\",\n\t\tfunc(in []*GroupInfo) []*GroupInfo {\n\t\t\tout := append([]*GroupInfo(nil), in...)\n\t\t\tsort.Slice(out, func(i, j int) bool {\n\t\t\t\treturn out[i].Group < out[j].Group\n\t\t\t})\n\t\t\treturn out\n\t\t})\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedTasks(t, r, fixtures.tasks)\n\t\th.SeedRedisSets(t, r, fixtures.allGroups)\n\t\th.SeedRedisZSets(t, r, fixtures.groups)\n\n\t\tt.Run(tc.desc, func(t *testing.T) {\n\t\t\tgot, err := inspector.Groups(tc.qname)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Groups returned error: %v\", err)\n\t\t\t}\n\t\t\tif diff := cmp.Diff(tc.want, got, sortGroupInfosOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"Groups = %v, want %v; (-want,+got)\\n%s\", got, tc.want, diff)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "internal/base/base.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\n// Package base defines foundational types and constants used in asynq package.\npackage base\n\nimport (\n\t\"context\"\n\t\"crypto/md5\"\n\t\"encoding/hex\"\n\t\"fmt\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/hibiken/asynq/internal/errors\"\n\tpb \"github.com/hibiken/asynq/internal/proto\"\n\t\"github.com/hibiken/asynq/internal/timeutil\"\n\t\"github.com/redis/go-redis/v9\"\n\t\"google.golang.org/protobuf/proto\"\n\t\"google.golang.org/protobuf/types/known/timestamppb\"\n)\n\n// Version of asynq library and CLI.\nconst Version = \"0.26.0\"\n\n// DefaultQueueName is the queue name used if none are specified by user.\nconst DefaultQueueName = \"default\"\n\n// DefaultQueue is the redis key for the default queue.\nvar DefaultQueue = PendingKey(DefaultQueueName)\n\n// Global Redis keys.\nconst (\n\tAllServers = \"asynq:servers\" // ZSET\n\tAllWorkers = \"asynq:workers\" // ZSET\n\tAllSchedulers = \"asynq:schedulers\" // ZSET\n\tAllQueues = \"asynq:queues\" // SET\n\tCancelChannel = \"asynq:cancel\" // PubSub channel\n)\n\n// TaskState denotes the state of a task.\ntype TaskState int\n\nconst (\n\tTaskStateActive TaskState = iota + 1\n\tTaskStatePending\n\tTaskStateScheduled\n\tTaskStateRetry\n\tTaskStateArchived\n\tTaskStateCompleted\n\tTaskStateAggregating // describes a state where task is waiting in a group to be aggregated\n)\n\nfunc (s TaskState) String() string {\n\tswitch s {\n\tcase TaskStateActive:\n\t\treturn \"active\"\n\tcase TaskStatePending:\n\t\treturn \"pending\"\n\tcase TaskStateScheduled:\n\t\treturn \"scheduled\"\n\tcase TaskStateRetry:\n\t\treturn \"retry\"\n\tcase TaskStateArchived:\n\t\treturn \"archived\"\n\tcase TaskStateCompleted:\n\t\treturn \"completed\"\n\tcase TaskStateAggregating:\n\t\treturn \"aggregating\"\n\t}\n\tpanic(fmt.Sprintf(\"internal error: unknown task state %d\", s))\n}\n\nfunc TaskStateFromString(s string) (TaskState, error) {\n\tswitch s {\n\tcase \"active\":\n\t\treturn TaskStateActive, nil\n\tcase \"pending\":\n\t\treturn TaskStatePending, nil\n\tcase \"scheduled\":\n\t\treturn TaskStateScheduled, nil\n\tcase \"retry\":\n\t\treturn TaskStateRetry, nil\n\tcase \"archived\":\n\t\treturn TaskStateArchived, nil\n\tcase \"completed\":\n\t\treturn TaskStateCompleted, nil\n\tcase \"aggregating\":\n\t\treturn TaskStateAggregating, nil\n\t}\n\treturn 0, errors.E(errors.FailedPrecondition, fmt.Sprintf(\"%q is not supported task state\", s))\n}\n\n// ValidateQueueName validates a given qname to be used as a queue name.\n// Returns nil if valid, otherwise returns non-nil error.\nfunc ValidateQueueName(qname string) error {\n\tif len(strings.TrimSpace(qname)) == 0 {\n\t\treturn fmt.Errorf(\"queue name must contain one or more characters\")\n\t}\n\treturn nil\n}\n\n// QueueKeyPrefix returns a prefix for all keys in the given queue.\nfunc QueueKeyPrefix(qname string) string {\n\treturn \"asynq:{\" + qname + \"}:\"\n}\n\n// TaskKeyPrefix returns a prefix for task key.\nfunc TaskKeyPrefix(qname string) string {\n\treturn QueueKeyPrefix(qname) + \"t:\"\n}\n\n// TaskKey returns a redis key for the given task message.\nfunc TaskKey(qname, id string) string {\n\treturn TaskKeyPrefix(qname) + id\n}\n\n// PendingKey returns a redis key for the given queue name.\nfunc PendingKey(qname string) string {\n\treturn QueueKeyPrefix(qname) + \"pending\"\n}\n\n// ActiveKey returns a redis key for the active tasks.\nfunc ActiveKey(qname string) string {\n\treturn QueueKeyPrefix(qname) + \"active\"\n}\n\n// ScheduledKey returns a redis key for the scheduled tasks.\nfunc ScheduledKey(qname string) string {\n\treturn QueueKeyPrefix(qname) + \"scheduled\"\n}\n\n// RetryKey returns a redis key for the retry tasks.\nfunc RetryKey(qname string) string {\n\treturn QueueKeyPrefix(qname) + \"retry\"\n}\n\n// ArchivedKey returns a redis key for the archived tasks.\nfunc ArchivedKey(qname string) string {\n\treturn QueueKeyPrefix(qname) + \"archived\"\n}\n\n// LeaseKey returns a redis key for the lease.\nfunc LeaseKey(qname string) string {\n\treturn QueueKeyPrefix(qname) + \"lease\"\n}\n\nfunc CompletedKey(qname string) string {\n\treturn QueueKeyPrefix(qname) + \"completed\"\n}\n\n// PausedKey returns a redis key to indicate that the given queue is paused.\nfunc PausedKey(qname string) string {\n\treturn QueueKeyPrefix(qname) + \"paused\"\n}\n\n// ProcessedTotalKey returns a redis key for total processed count for the given queue.\nfunc ProcessedTotalKey(qname string) string {\n\treturn QueueKeyPrefix(qname) + \"processed\"\n}\n\n// FailedTotalKey returns a redis key for total failure count for the given queue.\nfunc FailedTotalKey(qname string) string {\n\treturn QueueKeyPrefix(qname) + \"failed\"\n}\n\n// ProcessedKey returns a redis key for processed count for the given day for the queue.\nfunc ProcessedKey(qname string, t time.Time) string {\n\treturn QueueKeyPrefix(qname) + \"processed:\" + t.UTC().Format(\"2006-01-02\")\n}\n\n// FailedKey returns a redis key for failure count for the given day for the queue.\nfunc FailedKey(qname string, t time.Time) string {\n\treturn QueueKeyPrefix(qname) + \"failed:\" + t.UTC().Format(\"2006-01-02\")\n}\n\n// ServerInfoKey returns a redis key for process info.\nfunc ServerInfoKey(hostname string, pid int, serverID string) string {\n\treturn fmt.Sprintf(\"asynq:servers:{%s:%d:%s}\", hostname, pid, serverID)\n}\n\n// WorkersKey returns a redis key for the workers given hostname, pid, and server ID.\nfunc WorkersKey(hostname string, pid int, serverID string) string {\n\treturn fmt.Sprintf(\"asynq:workers:{%s:%d:%s}\", hostname, pid, serverID)\n}\n\n// SchedulerEntriesKey returns a redis key for the scheduler entries given scheduler ID.\nfunc SchedulerEntriesKey(schedulerID string) string {\n\treturn \"asynq:schedulers:{\" + schedulerID + \"}\"\n}\n\n// SchedulerHistoryKey returns a redis key for the scheduler's history for the given entry.\nfunc SchedulerHistoryKey(entryID string) string {\n\treturn \"asynq:scheduler_history:\" + entryID\n}\n\n// UniqueKey returns a redis key with the given type, payload, and queue name.\nfunc UniqueKey(qname, tasktype string, payload []byte) string {\n\tif payload == nil {\n\t\treturn QueueKeyPrefix(qname) + \"unique:\" + tasktype + \":\"\n\t}\n\tchecksum := md5.Sum(payload)\n\treturn QueueKeyPrefix(qname) + \"unique:\" + tasktype + \":\" + hex.EncodeToString(checksum[:])\n}\n\n// GroupKeyPrefix returns a prefix for group key.\nfunc GroupKeyPrefix(qname string) string {\n\treturn QueueKeyPrefix(qname) + \"g:\"\n}\n\n// GroupKey returns a redis key used to group tasks belong in the same group.\nfunc GroupKey(qname, gkey string) string {\n\treturn GroupKeyPrefix(qname) + gkey\n}\n\n// AggregationSetKey returns a redis key used for an aggregation set.\nfunc AggregationSetKey(qname, gname, setID string) string {\n\treturn GroupKey(qname, gname) + \":\" + setID\n}\n\n// AllGroups return a redis key used to store all group keys used in a given queue.\nfunc AllGroups(qname string) string {\n\treturn QueueKeyPrefix(qname) + \"groups\"\n}\n\n// AllAggregationSets returns a redis key used to store all aggregation sets (set of tasks staged to be aggregated)\n// in a given queue.\nfunc AllAggregationSets(qname string) string {\n\treturn QueueKeyPrefix(qname) + \"aggregation_sets\"\n}\n\n// TaskMessage is the internal representation of a task with additional metadata fields.\n// Serialized data of this type gets written to redis.\ntype TaskMessage struct {\n\t// Type indicates the kind of the task to be performed.\n\tType string\n\n\t// Payload holds data needed to process the task.\n\tPayload []byte\n\n\t// Headers holds additional metadata for the task.\n\tHeaders map[string]string\n\n\t// ID is a unique identifier for each task.\n\tID string\n\n\t// Queue is a name this message should be enqueued to.\n\tQueue string\n\n\t// Retry is the max number of retry for this task.\n\tRetry int\n\n\t// Retried is the number of times we've retried this task so far.\n\tRetried int\n\n\t// ErrorMsg holds the error message from the last failure.\n\tErrorMsg string\n\n\t// Time of last failure in Unix time,\n\t// the number of seconds elapsed since January 1, 1970 UTC.\n\t//\n\t// Use zero to indicate no last failure\n\tLastFailedAt int64\n\n\t// Timeout specifies timeout in seconds.\n\t// If task processing doesn't complete within the timeout, the task will be retried\n\t// if retry count is remaining. Otherwise it will be moved to the archive.\n\t//\n\t// Use zero to indicate no timeout.\n\tTimeout int64\n\n\t// Deadline specifies the deadline for the task in Unix time,\n\t// the number of seconds elapsed since January 1, 1970 UTC.\n\t// If task processing doesn't complete before the deadline, the task will be retried\n\t// if retry count is remaining. Otherwise it will be moved to the archive.\n\t//\n\t// Use zero to indicate no deadline.\n\tDeadline int64\n\n\t// UniqueKey holds the redis key used for uniqueness lock for this task.\n\t//\n\t// Empty string indicates that no uniqueness lock was used.\n\tUniqueKey string\n\n\t// GroupKey holds the group key used for task aggregation.\n\t//\n\t// Empty string indicates no aggregation is used for this task.\n\tGroupKey string\n\n\t// Retention specifies the number of seconds the task should be retained after completion.\n\tRetention int64\n\n\t// CompletedAt is the time the task was processed successfully in Unix time,\n\t// the number of seconds elapsed since January 1, 1970 UTC.\n\t//\n\t// Use zero to indicate no value.\n\tCompletedAt int64\n}\n\n// EncodeMessage marshals the given task message and returns an encoded bytes.\nfunc EncodeMessage(msg *TaskMessage) ([]byte, error) {\n\tif msg == nil {\n\t\treturn nil, fmt.Errorf(\"cannot encode nil message\")\n\t}\n\treturn proto.Marshal(&pb.TaskMessage{\n\t\tType: msg.Type,\n\t\tPayload: msg.Payload,\n\t\tHeaders: msg.Headers,\n\t\tId: msg.ID,\n\t\tQueue: msg.Queue,\n\t\tRetry: int32(msg.Retry),\n\t\tRetried: int32(msg.Retried),\n\t\tErrorMsg: msg.ErrorMsg,\n\t\tLastFailedAt: msg.LastFailedAt,\n\t\tTimeout: msg.Timeout,\n\t\tDeadline: msg.Deadline,\n\t\tUniqueKey: msg.UniqueKey,\n\t\tGroupKey: msg.GroupKey,\n\t\tRetention: msg.Retention,\n\t\tCompletedAt: msg.CompletedAt,\n\t})\n}\n\n// DecodeMessage unmarshals the given bytes and returns a decoded task message.\nfunc DecodeMessage(data []byte) (*TaskMessage, error) {\n\tvar pbmsg pb.TaskMessage\n\tif err := proto.Unmarshal(data, &pbmsg); err != nil {\n\t\treturn nil, err\n\t}\n\treturn &TaskMessage{\n\t\tType: pbmsg.GetType(),\n\t\tPayload: pbmsg.GetPayload(),\n\t\tHeaders: pbmsg.GetHeaders(),\n\t\tID: pbmsg.GetId(),\n\t\tQueue: pbmsg.GetQueue(),\n\t\tRetry: int(pbmsg.GetRetry()),\n\t\tRetried: int(pbmsg.GetRetried()),\n\t\tErrorMsg: pbmsg.GetErrorMsg(),\n\t\tLastFailedAt: pbmsg.GetLastFailedAt(),\n\t\tTimeout: pbmsg.GetTimeout(),\n\t\tDeadline: pbmsg.GetDeadline(),\n\t\tUniqueKey: pbmsg.GetUniqueKey(),\n\t\tGroupKey: pbmsg.GetGroupKey(),\n\t\tRetention: pbmsg.GetRetention(),\n\t\tCompletedAt: pbmsg.GetCompletedAt(),\n\t}, nil\n}\n\n// TaskInfo describes a task message and its metadata.\ntype TaskInfo struct {\n\tMessage *TaskMessage\n\tState TaskState\n\tNextProcessAt time.Time\n\tResult []byte\n}\n\n// Z represents sorted set member.\ntype Z struct {\n\tMessage *TaskMessage\n\tScore int64\n}\n\n// ServerInfo holds information about a running server.\ntype ServerInfo struct {\n\tHost string\n\tPID int\n\tServerID string\n\tConcurrency int\n\tQueues map[string]int\n\tStrictPriority bool\n\tStatus string\n\tStarted time.Time\n\tActiveWorkerCount int\n}\n\n// EncodeServerInfo marshals the given ServerInfo and returns the encoded bytes.\nfunc EncodeServerInfo(info *ServerInfo) ([]byte, error) {\n\tif info == nil {\n\t\treturn nil, fmt.Errorf(\"cannot encode nil server info\")\n\t}\n\tqueues := make(map[string]int32, len(info.Queues))\n\tfor q, p := range info.Queues {\n\t\tqueues[q] = int32(p)\n\t}\n\tstarted := timestamppb.New(info.Started)\n\n\treturn proto.Marshal(&pb.ServerInfo{\n\t\tHost: info.Host,\n\t\tPid: int32(info.PID),\n\t\tServerId: info.ServerID,\n\t\tConcurrency: int32(info.Concurrency),\n\t\tQueues: queues,\n\t\tStrictPriority: info.StrictPriority,\n\t\tStatus: info.Status,\n\t\tStartTime: started,\n\t\tActiveWorkerCount: int32(info.ActiveWorkerCount),\n\t})\n}\n\n// DecodeServerInfo decodes the given bytes into ServerInfo.\nfunc DecodeServerInfo(b []byte) (*ServerInfo, error) {\n\tvar pbmsg pb.ServerInfo\n\tif err := proto.Unmarshal(b, &pbmsg); err != nil {\n\t\treturn nil, err\n\t}\n\tqueues := make(map[string]int, len(pbmsg.GetQueues()))\n\tfor q, p := range pbmsg.GetQueues() {\n\t\tqueues[q] = int(p)\n\t}\n\tstartTime := pbmsg.GetStartTime()\n\n\treturn &ServerInfo{\n\t\tHost: pbmsg.GetHost(),\n\t\tPID: int(pbmsg.GetPid()),\n\t\tServerID: pbmsg.GetServerId(),\n\t\tConcurrency: int(pbmsg.GetConcurrency()),\n\t\tQueues: queues,\n\t\tStrictPriority: pbmsg.GetStrictPriority(),\n\t\tStatus: pbmsg.GetStatus(),\n\t\tStarted: startTime.AsTime(),\n\t\tActiveWorkerCount: int(pbmsg.GetActiveWorkerCount()),\n\t}, nil\n}\n\n// WorkerInfo holds information about a running worker.\ntype WorkerInfo struct {\n\tHost string\n\tPID int\n\tServerID string\n\tID string\n\tType string\n\tPayload []byte\n\tQueue string\n\tStarted time.Time\n\tDeadline time.Time\n}\n\n// EncodeWorkerInfo marshals the given WorkerInfo and returns the encoded bytes.\nfunc EncodeWorkerInfo(info *WorkerInfo) ([]byte, error) {\n\tif info == nil {\n\t\treturn nil, fmt.Errorf(\"cannot encode nil worker info\")\n\t}\n\tstartTime := timestamppb.New(info.Started)\n\tdeadline := timestamppb.New(info.Deadline)\n\n\treturn proto.Marshal(&pb.WorkerInfo{\n\t\tHost: info.Host,\n\t\tPid: int32(info.PID),\n\t\tServerId: info.ServerID,\n\t\tTaskId: info.ID,\n\t\tTaskType: info.Type,\n\t\tTaskPayload: info.Payload,\n\t\tQueue: info.Queue,\n\t\tStartTime: startTime,\n\t\tDeadline: deadline,\n\t})\n}\n\n// DecodeWorkerInfo decodes the given bytes into WorkerInfo.\nfunc DecodeWorkerInfo(b []byte) (*WorkerInfo, error) {\n\tvar pbmsg pb.WorkerInfo\n\tif err := proto.Unmarshal(b, &pbmsg); err != nil {\n\t\treturn nil, err\n\t}\n\tstartTime := pbmsg.GetStartTime()\n\tdeadline := pbmsg.GetDeadline()\n\n\treturn &WorkerInfo{\n\t\tHost: pbmsg.GetHost(),\n\t\tPID: int(pbmsg.GetPid()),\n\t\tServerID: pbmsg.GetServerId(),\n\t\tID: pbmsg.GetTaskId(),\n\t\tType: pbmsg.GetTaskType(),\n\t\tPayload: pbmsg.GetTaskPayload(),\n\t\tQueue: pbmsg.GetQueue(),\n\t\tStarted: startTime.AsTime(),\n\t\tDeadline: deadline.AsTime(),\n\t}, nil\n}\n\n// SchedulerEntry holds information about a periodic task registered with a scheduler.\ntype SchedulerEntry struct {\n\t// Identifier of this entry.\n\tID string\n\n\t// Spec describes the schedule of this entry.\n\tSpec string\n\n\t// Type is the task type of the periodic task.\n\tType string\n\n\t// Payload is the payload of the periodic task.\n\tPayload []byte\n\n\t// Opts is the options for the periodic task.\n\tOpts []string\n\n\t// Next shows the next time the task will be enqueued.\n\tNext time.Time\n\n\t// Prev shows the last time the task was enqueued.\n\t// Zero time if task was never enqueued.\n\tPrev time.Time\n}\n\n// EncodeSchedulerEntry marshals the given entry and returns an encoded bytes.\nfunc EncodeSchedulerEntry(entry *SchedulerEntry) ([]byte, error) {\n\tif entry == nil {\n\t\treturn nil, fmt.Errorf(\"cannot encode nil scheduler entry\")\n\t}\n\tnext := timestamppb.New(entry.Next)\n\tprev := timestamppb.New(entry.Prev)\n\n\treturn proto.Marshal(&pb.SchedulerEntry{\n\t\tId: entry.ID,\n\t\tSpec: entry.Spec,\n\t\tTaskType: entry.Type,\n\t\tTaskPayload: entry.Payload,\n\t\tEnqueueOptions: entry.Opts,\n\t\tNextEnqueueTime: next,\n\t\tPrevEnqueueTime: prev,\n\t})\n}\n\n// DecodeSchedulerEntry unmarshals the given bytes and returns a decoded SchedulerEntry.\nfunc DecodeSchedulerEntry(b []byte) (*SchedulerEntry, error) {\n\tvar pbmsg pb.SchedulerEntry\n\tif err := proto.Unmarshal(b, &pbmsg); err != nil {\n\t\treturn nil, err\n\t}\n\tnext := pbmsg.GetNextEnqueueTime()\n\tprev := pbmsg.GetPrevEnqueueTime()\n\n\treturn &SchedulerEntry{\n\t\tID: pbmsg.GetId(),\n\t\tSpec: pbmsg.GetSpec(),\n\t\tType: pbmsg.GetTaskType(),\n\t\tPayload: pbmsg.GetTaskPayload(),\n\t\tOpts: pbmsg.GetEnqueueOptions(),\n\t\tNext: next.AsTime(),\n\t\tPrev: prev.AsTime(),\n\t}, nil\n}\n\n// SchedulerEnqueueEvent holds information about an enqueue event by a scheduler.\ntype SchedulerEnqueueEvent struct {\n\t// ID of the task that was enqueued.\n\tTaskID string\n\n\t// Time the task was enqueued.\n\tEnqueuedAt time.Time\n}\n\n// EncodeSchedulerEnqueueEvent marshals the given event\n// and returns an encoded bytes.\nfunc EncodeSchedulerEnqueueEvent(event *SchedulerEnqueueEvent) ([]byte, error) {\n\tif event == nil {\n\t\treturn nil, fmt.Errorf(\"cannot encode nil enqueue event\")\n\t}\n\tenqueuedAt := timestamppb.New(event.EnqueuedAt)\n\treturn proto.Marshal(&pb.SchedulerEnqueueEvent{\n\t\tTaskId: event.TaskID,\n\t\tEnqueueTime: enqueuedAt,\n\t})\n}\n\n// DecodeSchedulerEnqueueEvent unmarshals the given bytes\n// and returns a decoded SchedulerEnqueueEvent.\nfunc DecodeSchedulerEnqueueEvent(b []byte) (*SchedulerEnqueueEvent, error) {\n\tvar pbmsg pb.SchedulerEnqueueEvent\n\tif err := proto.Unmarshal(b, &pbmsg); err != nil {\n\t\treturn nil, err\n\t}\n\tenqueuedAt := pbmsg.GetEnqueueTime()\n\treturn &SchedulerEnqueueEvent{\n\t\tTaskID: pbmsg.GetTaskId(),\n\t\tEnqueuedAt: enqueuedAt.AsTime(),\n\t}, nil\n}\n\n// Cancelations is a collection that holds cancel functions for all active tasks.\n//\n// Cancelations are safe for concurrent use by multiple goroutines.\ntype Cancelations struct {\n\tmu sync.Mutex\n\tcancelFuncs map[string]context.CancelFunc\n}\n\n// NewCancelations returns a Cancelations instance.\nfunc NewCancelations() *Cancelations {\n\treturn &Cancelations{\n\t\tcancelFuncs: make(map[string]context.CancelFunc),\n\t}\n}\n\n// Add adds a new cancel func to the collection.\nfunc (c *Cancelations) Add(id string, fn context.CancelFunc) {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\tc.cancelFuncs[id] = fn\n}\n\n// Delete deletes a cancel func from the collection given an id.\nfunc (c *Cancelations) Delete(id string) {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\tdelete(c.cancelFuncs, id)\n}\n\n// Get returns a cancel func given an id.\nfunc (c *Cancelations) Get(id string) (fn context.CancelFunc, ok bool) {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\tfn, ok = c.cancelFuncs[id]\n\treturn fn, ok\n}\n\n// Lease is a time bound lease for worker to process task.\n// It provides a communication channel between lessor and lessee about lease expiration.\ntype Lease struct {\n\tonce sync.Once\n\tch chan struct{}\n\n\tClock timeutil.Clock\n\n\tmu sync.Mutex\n\texpireAt time.Time // guarded by mu\n}\n\nfunc NewLease(expirationTime time.Time) *Lease {\n\treturn &Lease{\n\t\tch: make(chan struct{}),\n\t\texpireAt: expirationTime,\n\t\tClock: timeutil.NewRealClock(),\n\t}\n}\n\n// Reset changes the lease to expire at the given time.\n// It returns true if the lease is still valid and reset operation was successful, false if the lease had been expired.\nfunc (l *Lease) Reset(expirationTime time.Time) bool {\n\tif !l.IsValid() {\n\t\treturn false\n\t}\n\tl.mu.Lock()\n\tdefer l.mu.Unlock()\n\tl.expireAt = expirationTime\n\treturn true\n}\n\n// Sends a notification to lessee about expired lease\n// Returns true if notification was sent, returns false if the lease is still valid and notification was not sent.\nfunc (l *Lease) NotifyExpiration() bool {\n\tif l.IsValid() {\n\t\treturn false\n\t}\n\tl.once.Do(l.closeCh)\n\treturn true\n}\n\nfunc (l *Lease) closeCh() {\n\tclose(l.ch)\n}\n\n// Done returns a communication channel from which the lessee can read to get notified when lessor notifies about lease expiration.\nfunc (l *Lease) Done() <-chan struct{} {\n\treturn l.ch\n}\n\n// Deadline returns the expiration time of the lease.\nfunc (l *Lease) Deadline() time.Time {\n\tl.mu.Lock()\n\tdefer l.mu.Unlock()\n\treturn l.expireAt\n}\n\n// IsValid returns true if the lease's expiration time is in the future or equals to the current time,\n// returns false otherwise.\nfunc (l *Lease) IsValid() bool {\n\tnow := l.Clock.Now()\n\tl.mu.Lock()\n\tdefer l.mu.Unlock()\n\treturn l.expireAt.After(now) || l.expireAt.Equal(now)\n}\n\n// Broker is a message broker that supports operations to manage task queues.\n//\n// See rdb.RDB as a reference implementation.\ntype Broker interface {\n\tPing() error\n\tClose() error\n\tEnqueue(ctx context.Context, msg *TaskMessage) error\n\tEnqueueUnique(ctx context.Context, msg *TaskMessage, ttl time.Duration) error\n\tDequeue(qnames ...string) (*TaskMessage, time.Time, error)\n\tDone(ctx context.Context, msg *TaskMessage) error\n\tMarkAsComplete(ctx context.Context, msg *TaskMessage) error\n\tRequeue(ctx context.Context, msg *TaskMessage) error\n\tSchedule(ctx context.Context, msg *TaskMessage, processAt time.Time) error\n\tScheduleUnique(ctx context.Context, msg *TaskMessage, processAt time.Time, ttl time.Duration) error\n\tRetry(ctx context.Context, msg *TaskMessage, processAt time.Time, errMsg string, isFailure bool) error\n\tArchive(ctx context.Context, msg *TaskMessage, errMsg string) error\n\tForwardIfReady(qnames ...string) error\n\n\t// Group aggregation related methods\n\tAddToGroup(ctx context.Context, msg *TaskMessage, gname string) error\n\tAddToGroupUnique(ctx context.Context, msg *TaskMessage, groupKey string, ttl time.Duration) error\n\tListGroups(qname string) ([]string, error)\n\tAggregationCheck(qname, gname string, t time.Time, gracePeriod, maxDelay time.Duration, maxSize int) (aggregationSetID string, err error)\n\tReadAggregationSet(qname, gname, aggregationSetID string) ([]*TaskMessage, time.Time, error)\n\tDeleteAggregationSet(ctx context.Context, qname, gname, aggregationSetID string) error\n\tReclaimStaleAggregationSets(qname string) error\n\n\t// Task retention related method\n\tDeleteExpiredCompletedTasks(qname string, batchSize int) error\n\n\t// Lease related methods\n\tListLeaseExpired(cutoff time.Time, qnames ...string) ([]*TaskMessage, error)\n\tExtendLease(qname string, ids ...string) (time.Time, error)\n\n\t// State snapshot related methods\n\tWriteServerState(info *ServerInfo, workers []*WorkerInfo, ttl time.Duration) error\n\tClearServerState(host string, pid int, serverID string) error\n\n\t// Cancelation related methods\n\tCancelationPubSub() (*redis.PubSub, error) // TODO: Need to decouple from redis to support other brokers\n\tPublishCancelation(id string) error\n\n\tWriteResult(qname, id string, data []byte) (n int, err error)\n}\n" }, { "path": "internal/base/base_test.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage base\n\nimport (\n\t\"context\"\n\t\"crypto/md5\"\n\t\"encoding/hex\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/google/go-cmp/cmp\"\n\t\"github.com/google/uuid\"\n\t\"github.com/hibiken/asynq/internal/timeutil\"\n)\n\nfunc TestTaskKey(t *testing.T) {\n\tid := uuid.NewString()\n\n\ttests := []struct {\n\t\tqname string\n\t\tid string\n\t\twant string\n\t}{\n\t\t{\"default\", id, fmt.Sprintf(\"asynq:{default}:t:%s\", id)},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := TaskKey(tc.qname, tc.id)\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"TaskKey(%q, %s) = %q, want %q\", tc.qname, tc.id, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestQueueKey(t *testing.T) {\n\ttests := []struct {\n\t\tqname string\n\t\twant string\n\t}{\n\t\t{\"default\", \"asynq:{default}:pending\"},\n\t\t{\"custom\", \"asynq:{custom}:pending\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := PendingKey(tc.qname)\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"QueueKey(%q) = %q, want %q\", tc.qname, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestActiveKey(t *testing.T) {\n\ttests := []struct {\n\t\tqname string\n\t\twant string\n\t}{\n\t\t{\"default\", \"asynq:{default}:active\"},\n\t\t{\"custom\", \"asynq:{custom}:active\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := ActiveKey(tc.qname)\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"ActiveKey(%q) = %q, want %q\", tc.qname, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestLeaseKey(t *testing.T) {\n\ttests := []struct {\n\t\tqname string\n\t\twant string\n\t}{\n\t\t{\"default\", \"asynq:{default}:lease\"},\n\t\t{\"custom\", \"asynq:{custom}:lease\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := LeaseKey(tc.qname)\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"LeaseKey(%q) = %q, want %q\", tc.qname, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestScheduledKey(t *testing.T) {\n\ttests := []struct {\n\t\tqname string\n\t\twant string\n\t}{\n\t\t{\"default\", \"asynq:{default}:scheduled\"},\n\t\t{\"custom\", \"asynq:{custom}:scheduled\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := ScheduledKey(tc.qname)\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"ScheduledKey(%q) = %q, want %q\", tc.qname, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestRetryKey(t *testing.T) {\n\ttests := []struct {\n\t\tqname string\n\t\twant string\n\t}{\n\t\t{\"default\", \"asynq:{default}:retry\"},\n\t\t{\"custom\", \"asynq:{custom}:retry\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := RetryKey(tc.qname)\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"RetryKey(%q) = %q, want %q\", tc.qname, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestArchivedKey(t *testing.T) {\n\ttests := []struct {\n\t\tqname string\n\t\twant string\n\t}{\n\t\t{\"default\", \"asynq:{default}:archived\"},\n\t\t{\"custom\", \"asynq:{custom}:archived\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := ArchivedKey(tc.qname)\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"ArchivedKey(%q) = %q, want %q\", tc.qname, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestCompletedKey(t *testing.T) {\n\ttests := []struct {\n\t\tqname string\n\t\twant string\n\t}{\n\t\t{\"default\", \"asynq:{default}:completed\"},\n\t\t{\"custom\", \"asynq:{custom}:completed\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := CompletedKey(tc.qname)\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"CompletedKey(%q) = %q, want %q\", tc.qname, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestPausedKey(t *testing.T) {\n\ttests := []struct {\n\t\tqname string\n\t\twant string\n\t}{\n\t\t{\"default\", \"asynq:{default}:paused\"},\n\t\t{\"custom\", \"asynq:{custom}:paused\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := PausedKey(tc.qname)\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"PausedKey(%q) = %q, want %q\", tc.qname, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestProcessedTotalKey(t *testing.T) {\n\ttests := []struct {\n\t\tqname string\n\t\twant string\n\t}{\n\t\t{\"default\", \"asynq:{default}:processed\"},\n\t\t{\"custom\", \"asynq:{custom}:processed\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := ProcessedTotalKey(tc.qname)\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"ProcessedTotalKey(%q) = %q, want %q\", tc.qname, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestFailedTotalKey(t *testing.T) {\n\ttests := []struct {\n\t\tqname string\n\t\twant string\n\t}{\n\t\t{\"default\", \"asynq:{default}:failed\"},\n\t\t{\"custom\", \"asynq:{custom}:failed\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := FailedTotalKey(tc.qname)\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"FailedTotalKey(%q) = %q, want %q\", tc.qname, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestProcessedKey(t *testing.T) {\n\ttests := []struct {\n\t\tqname string\n\t\tinput time.Time\n\t\twant string\n\t}{\n\t\t{\"default\", time.Date(2019, 11, 14, 10, 30, 1, 1, time.UTC), \"asynq:{default}:processed:2019-11-14\"},\n\t\t{\"critical\", time.Date(2020, 12, 1, 1, 0, 1, 1, time.UTC), \"asynq:{critical}:processed:2020-12-01\"},\n\t\t{\"default\", time.Date(2020, 1, 6, 15, 02, 1, 1, time.UTC), \"asynq:{default}:processed:2020-01-06\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := ProcessedKey(tc.qname, tc.input)\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"ProcessedKey(%v) = %q, want %q\", tc.input, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestFailedKey(t *testing.T) {\n\ttests := []struct {\n\t\tqname string\n\t\tinput time.Time\n\t\twant string\n\t}{\n\t\t{\"default\", time.Date(2019, 11, 14, 10, 30, 1, 1, time.UTC), \"asynq:{default}:failed:2019-11-14\"},\n\t\t{\"custom\", time.Date(2020, 12, 1, 1, 0, 1, 1, time.UTC), \"asynq:{custom}:failed:2020-12-01\"},\n\t\t{\"low\", time.Date(2020, 1, 6, 15, 02, 1, 1, time.UTC), \"asynq:{low}:failed:2020-01-06\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := FailedKey(tc.qname, tc.input)\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"FailureKey(%v) = %q, want %q\", tc.input, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestServerInfoKey(t *testing.T) {\n\ttests := []struct {\n\t\thostname string\n\t\tpid int\n\t\tsid string\n\t\twant string\n\t}{\n\t\t{\"localhost\", 9876, \"server123\", \"asynq:servers:{localhost:9876:server123}\"},\n\t\t{\"127.0.0.1\", 1234, \"server987\", \"asynq:servers:{127.0.0.1:1234:server987}\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := ServerInfoKey(tc.hostname, tc.pid, tc.sid)\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"ServerInfoKey(%q, %d, %q) = %q, want %q\",\n\t\t\t\ttc.hostname, tc.pid, tc.sid, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestWorkersKey(t *testing.T) {\n\ttests := []struct {\n\t\thostname string\n\t\tpid int\n\t\tsid string\n\t\twant string\n\t}{\n\t\t{\"localhost\", 9876, \"server1\", \"asynq:workers:{localhost:9876:server1}\"},\n\t\t{\"127.0.0.1\", 1234, \"server2\", \"asynq:workers:{127.0.0.1:1234:server2}\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := WorkersKey(tc.hostname, tc.pid, tc.sid)\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"WorkersKey(%q, %d, %q) = %q, want = %q\",\n\t\t\t\ttc.hostname, tc.pid, tc.sid, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestSchedulerEntriesKey(t *testing.T) {\n\ttests := []struct {\n\t\tschedulerID string\n\t\twant string\n\t}{\n\t\t{\"localhost:9876:scheduler123\", \"asynq:schedulers:{localhost:9876:scheduler123}\"},\n\t\t{\"127.0.0.1:1234:scheduler987\", \"asynq:schedulers:{127.0.0.1:1234:scheduler987}\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := SchedulerEntriesKey(tc.schedulerID)\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"SchedulerEntriesKey(%q) = %q, want %q\", tc.schedulerID, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestSchedulerHistoryKey(t *testing.T) {\n\ttests := []struct {\n\t\tentryID string\n\t\twant string\n\t}{\n\t\t{\"entry876\", \"asynq:scheduler_history:entry876\"},\n\t\t{\"entry345\", \"asynq:scheduler_history:entry345\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := SchedulerHistoryKey(tc.entryID)\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"SchedulerHistoryKey(%q) = %q, want %q\",\n\t\t\t\ttc.entryID, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc toBytes(m map[string]interface{}) []byte {\n\tb, err := json.Marshal(m)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\treturn b\n}\n\nfunc TestUniqueKey(t *testing.T) {\n\tpayload1 := toBytes(map[string]interface{}{\"a\": 123, \"b\": \"hello\", \"c\": true})\n\tpayload2 := toBytes(map[string]interface{}{\"b\": \"hello\", \"c\": true, \"a\": 123})\n\tpayload3 := toBytes(map[string]interface{}{\n\t\t\"address\": map[string]string{\"line\": \"123 Main St\", \"city\": \"Boston\", \"state\": \"MA\"},\n\t\t\"names\": []string{\"bob\", \"mike\", \"rob\"}})\n\tpayload4 := toBytes(map[string]interface{}{\n\t\t\"time\": time.Date(2020, time.July, 28, 0, 0, 0, 0, time.UTC),\n\t\t\"duration\": time.Hour})\n\n\tchecksum := func(data []byte) string {\n\t\tsum := md5.Sum(data)\n\t\treturn hex.EncodeToString(sum[:])\n\t}\n\ttests := []struct {\n\t\tdesc string\n\t\tqname string\n\t\ttasktype string\n\t\tpayload []byte\n\t\twant string\n\t}{\n\t\t{\n\t\t\t\"with primitive types\",\n\t\t\t\"default\",\n\t\t\t\"email:send\",\n\t\t\tpayload1,\n\t\t\tfmt.Sprintf(\"asynq:{default}:unique:email:send:%s\", checksum(payload1)),\n\t\t},\n\t\t{\n\t\t\t\"with unsorted keys\",\n\t\t\t\"default\",\n\t\t\t\"email:send\",\n\t\t\tpayload2,\n\t\t\tfmt.Sprintf(\"asynq:{default}:unique:email:send:%s\", checksum(payload2)),\n\t\t},\n\t\t{\n\t\t\t\"with composite types\",\n\t\t\t\"default\",\n\t\t\t\"email:send\",\n\t\t\tpayload3,\n\t\t\tfmt.Sprintf(\"asynq:{default}:unique:email:send:%s\", checksum(payload3)),\n\t\t},\n\t\t{\n\t\t\t\"with complex types\",\n\t\t\t\"default\",\n\t\t\t\"email:send\",\n\t\t\tpayload4,\n\t\t\tfmt.Sprintf(\"asynq:{default}:unique:email:send:%s\", checksum(payload4)),\n\t\t},\n\t\t{\n\t\t\t\"with nil payload\",\n\t\t\t\"default\",\n\t\t\t\"reindex\",\n\t\t\tnil,\n\t\t\t\"asynq:{default}:unique:reindex:\",\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := UniqueKey(tc.qname, tc.tasktype, tc.payload)\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"%s: UniqueKey(%q, %q, %v) = %q, want %q\", tc.desc, tc.qname, tc.tasktype, tc.payload, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestGroupKey(t *testing.T) {\n\ttests := []struct {\n\t\tqname string\n\t\tgkey string\n\t\twant string\n\t}{\n\t\t{\n\t\t\tqname: \"default\",\n\t\t\tgkey: \"mygroup\",\n\t\t\twant: \"asynq:{default}:g:mygroup\",\n\t\t},\n\t\t{\n\t\t\tqname: \"custom\",\n\t\t\tgkey: \"foo\",\n\t\t\twant: \"asynq:{custom}:g:foo\",\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := GroupKey(tc.qname, tc.gkey)\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"GroupKey(%q, %q) = %q, want %q\", tc.qname, tc.gkey, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestAggregationSetKey(t *testing.T) {\n\ttests := []struct {\n\t\tqname string\n\t\tgname string\n\t\tsetID string\n\t\twant string\n\t}{\n\t\t{\n\t\t\tqname: \"default\",\n\t\t\tgname: \"mygroup\",\n\t\t\tsetID: \"12345\",\n\t\t\twant: \"asynq:{default}:g:mygroup:12345\",\n\t\t},\n\t\t{\n\t\t\tqname: \"custom\",\n\t\t\tgname: \"foo\",\n\t\t\tsetID: \"98765\",\n\t\t\twant: \"asynq:{custom}:g:foo:98765\",\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := AggregationSetKey(tc.qname, tc.gname, tc.setID)\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"AggregationSetKey(%q, %q, %q) = %q, want %q\", tc.qname, tc.gname, tc.setID, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestAllGroups(t *testing.T) {\n\ttests := []struct {\n\t\tqname string\n\t\twant string\n\t}{\n\t\t{\n\t\t\tqname: \"default\",\n\t\t\twant: \"asynq:{default}:groups\",\n\t\t},\n\t\t{\n\t\t\tqname: \"custom\",\n\t\t\twant: \"asynq:{custom}:groups\",\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := AllGroups(tc.qname)\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"AllGroups(%q) = %q, want %q\", tc.qname, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestAllAggregationSets(t *testing.T) {\n\ttests := []struct {\n\t\tqname string\n\t\twant string\n\t}{\n\t\t{\n\t\t\tqname: \"default\",\n\t\t\twant: \"asynq:{default}:aggregation_sets\",\n\t\t},\n\t\t{\n\t\t\tqname: \"custom\",\n\t\t\twant: \"asynq:{custom}:aggregation_sets\",\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := AllAggregationSets(tc.qname)\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"AllAggregationSets(%q) = %q, want %q\", tc.qname, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestMessageEncoding(t *testing.T) {\n\tid := uuid.NewString()\n\ttests := []struct {\n\t\tin *TaskMessage\n\t\tout *TaskMessage\n\t}{\n\t\t{\n\t\t\tin: &TaskMessage{\n\t\t\t\tType: \"task1\",\n\t\t\t\tPayload: toBytes(map[string]interface{}{\"a\": 1, \"b\": \"hello!\", \"c\": true}),\n\t\t\t\tID: id,\n\t\t\t\tQueue: \"default\",\n\t\t\t\tGroupKey: \"mygroup\",\n\t\t\t\tRetry: 10,\n\t\t\t\tRetried: 0,\n\t\t\t\tTimeout: 1800,\n\t\t\t\tDeadline: 1692311100,\n\t\t\t\tRetention: 3600,\n\t\t\t},\n\t\t\tout: &TaskMessage{\n\t\t\t\tType: \"task1\",\n\t\t\t\tPayload: toBytes(map[string]interface{}{\"a\": json.Number(\"1\"), \"b\": \"hello!\", \"c\": true}),\n\t\t\t\tID: id,\n\t\t\t\tQueue: \"default\",\n\t\t\t\tGroupKey: \"mygroup\",\n\t\t\t\tRetry: 10,\n\t\t\t\tRetried: 0,\n\t\t\t\tTimeout: 1800,\n\t\t\t\tDeadline: 1692311100,\n\t\t\t\tRetention: 3600,\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tencoded, err := EncodeMessage(tc.in)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"EncodeMessage(msg) returned error: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tdecoded, err := DecodeMessage(encoded)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"DecodeMessage(encoded) returned error: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tif diff := cmp.Diff(tc.out, decoded); diff != \"\" {\n\t\t\tt.Errorf(\"Decoded message == %+v, want %+v;(-want,+got)\\n%s\",\n\t\t\t\tdecoded, tc.out, diff)\n\t\t}\n\t}\n}\n\nfunc TestServerInfoEncoding(t *testing.T) {\n\ttests := []struct {\n\t\tinfo ServerInfo\n\t}{\n\t\t{\n\t\t\tinfo: ServerInfo{\n\t\t\t\tHost: \"127.0.0.1\",\n\t\t\t\tPID: 9876,\n\t\t\t\tServerID: \"abc123\",\n\t\t\t\tConcurrency: 10,\n\t\t\t\tQueues: map[string]int{\"default\": 1, \"critical\": 2},\n\t\t\t\tStrictPriority: false,\n\t\t\t\tStatus: \"active\",\n\t\t\t\tStarted: time.Now().Add(-3 * time.Hour),\n\t\t\t\tActiveWorkerCount: 8,\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tencoded, err := EncodeServerInfo(&tc.info)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"EncodeServerInfo(info) returned error: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tdecoded, err := DecodeServerInfo(encoded)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"DecodeServerInfo(encoded) returned error: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tif diff := cmp.Diff(&tc.info, decoded); diff != \"\" {\n\t\t\tt.Errorf(\"Decoded ServerInfo == %+v, want %+v;(-want,+got)\\n%s\",\n\t\t\t\tdecoded, tc.info, diff)\n\t\t}\n\t}\n}\n\nfunc TestWorkerInfoEncoding(t *testing.T) {\n\ttests := []struct {\n\t\tinfo WorkerInfo\n\t}{\n\t\t{\n\t\t\tinfo: WorkerInfo{\n\t\t\t\tHost: \"127.0.0.1\",\n\t\t\t\tPID: 9876,\n\t\t\t\tServerID: \"abc123\",\n\t\t\t\tID: uuid.NewString(),\n\t\t\t\tType: \"taskA\",\n\t\t\t\tPayload: toBytes(map[string]interface{}{\"foo\": \"bar\"}),\n\t\t\t\tQueue: \"default\",\n\t\t\t\tStarted: time.Now().Add(-3 * time.Hour),\n\t\t\t\tDeadline: time.Now().Add(30 * time.Second),\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tencoded, err := EncodeWorkerInfo(&tc.info)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"EncodeWorkerInfo(info) returned error: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tdecoded, err := DecodeWorkerInfo(encoded)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"DecodeWorkerInfo(encoded) returned error: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tif diff := cmp.Diff(&tc.info, decoded); diff != \"\" {\n\t\t\tt.Errorf(\"Decoded WorkerInfo == %+v, want %+v;(-want,+got)\\n%s\",\n\t\t\t\tdecoded, tc.info, diff)\n\t\t}\n\t}\n}\n\nfunc TestSchedulerEntryEncoding(t *testing.T) {\n\ttests := []struct {\n\t\tentry SchedulerEntry\n\t}{\n\t\t{\n\t\t\tentry: SchedulerEntry{\n\t\t\t\tID: uuid.NewString(),\n\t\t\t\tSpec: \"* * * * *\",\n\t\t\t\tType: \"task_A\",\n\t\t\t\tPayload: toBytes(map[string]interface{}{\"foo\": \"bar\"}),\n\t\t\t\tOpts: []string{\"Queue('email')\"},\n\t\t\t\tNext: time.Now().Add(30 * time.Second).UTC(),\n\t\t\t\tPrev: time.Now().Add(-2 * time.Minute).UTC(),\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tencoded, err := EncodeSchedulerEntry(&tc.entry)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"EncodeSchedulerEntry(entry) returned error: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tdecoded, err := DecodeSchedulerEntry(encoded)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"DecodeSchedulerEntry(encoded) returned error: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tif diff := cmp.Diff(&tc.entry, decoded); diff != \"\" {\n\t\t\tt.Errorf(\"Decoded SchedulerEntry == %+v, want %+v;(-want,+got)\\n%s\",\n\t\t\t\tdecoded, tc.entry, diff)\n\t\t}\n\t}\n}\n\nfunc TestSchedulerEnqueueEventEncoding(t *testing.T) {\n\ttests := []struct {\n\t\tevent SchedulerEnqueueEvent\n\t}{\n\t\t{\n\t\t\tevent: SchedulerEnqueueEvent{\n\t\t\t\tTaskID: uuid.NewString(),\n\t\t\t\tEnqueuedAt: time.Now().Add(-30 * time.Second).UTC(),\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tencoded, err := EncodeSchedulerEnqueueEvent(&tc.event)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"EncodeSchedulerEnqueueEvent(event) returned error: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tdecoded, err := DecodeSchedulerEnqueueEvent(encoded)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"DecodeSchedulerEnqueueEvent(encoded) returned error: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tif diff := cmp.Diff(&tc.event, decoded); diff != \"\" {\n\t\t\tt.Errorf(\"Decoded SchedulerEnqueueEvent == %+v, want %+v;(-want,+got)\\n%s\",\n\t\t\t\tdecoded, tc.event, diff)\n\t\t}\n\t}\n}\n\n// Test for cancelations being accessed by multiple goroutines.\n// Run with -race flag to check for data race.\nfunc TestCancelationsConcurrentAccess(t *testing.T) {\n\tc := NewCancelations()\n\n\t_, cancel1 := context.WithCancel(context.Background())\n\t_, cancel2 := context.WithCancel(context.Background())\n\t_, cancel3 := context.WithCancel(context.Background())\n\tvar key1, key2, key3 = \"key1\", \"key2\", \"key3\"\n\n\tvar wg sync.WaitGroup\n\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tc.Add(key1, cancel1)\n\t}()\n\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tc.Add(key2, cancel2)\n\t\ttime.Sleep(200 * time.Millisecond)\n\t\tc.Delete(key2)\n\t}()\n\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tc.Add(key3, cancel3)\n\t}()\n\n\twg.Wait()\n\n\t_, ok := c.Get(key1)\n\tif !ok {\n\t\tt.Errorf(\"(*Cancelations).Get(%q) = _, false, want , true\", key1)\n\t}\n\n\t_, ok = c.Get(key2)\n\tif ok {\n\t\tt.Errorf(\"(*Cancelations).Get(%q) = _, true, want , false\", key2)\n\t}\n}\n\nfunc TestLeaseReset(t *testing.T) {\n\tnow := time.Now()\n\tclock := timeutil.NewSimulatedClock(now)\n\n\tl := NewLease(now.Add(30 * time.Second))\n\tl.Clock = clock\n\n\t// Check initial state\n\tif !l.IsValid() {\n\t\tt.Errorf(\"lease should be valid when expiration is set to a future time\")\n\t}\n\tif want := now.Add(30 * time.Second); l.Deadline() != want {\n\t\tt.Errorf(\"Lease.Deadline() = %v, want %v\", l.Deadline(), want)\n\t}\n\n\t// Test Reset\n\tif !l.Reset(now.Add(45 * time.Second)) {\n\t\tt.Fatalf(\"Lease.Reset returned false when extending\")\n\t}\n\tif want := now.Add(45 * time.Second); l.Deadline() != want {\n\t\tt.Errorf(\"After Reset: Lease.Deadline() = %v, want %v\", l.Deadline(), want)\n\t}\n\n\tclock.AdvanceTime(1 * time.Minute) // simulate lease expiration\n\n\tif l.IsValid() {\n\t\tt.Errorf(\"lease should be invalid after expiration\")\n\t}\n\n\t// Reset should return false if lease is expired.\n\tif l.Reset(time.Now().Add(20 * time.Second)) {\n\t\tt.Errorf(\"Lease.Reset should return false after expiration\")\n\t}\n}\n\nfunc TestLeaseNotifyExpiration(t *testing.T) {\n\tnow := time.Now()\n\tclock := timeutil.NewSimulatedClock(now)\n\n\tl := NewLease(now.Add(30 * time.Second))\n\tl.Clock = clock\n\n\tselect {\n\tcase <-l.Done():\n\t\tt.Fatalf(\"Lease.Done() did not block\")\n\tdefault:\n\t}\n\n\tif l.NotifyExpiration() {\n\t\tt.Fatalf(\"Lease.NotifyExpiration() should return false when lease is still valid\")\n\t}\n\n\tclock.AdvanceTime(1 * time.Minute) // simulate lease expiration\n\n\tif l.IsValid() {\n\t\tt.Errorf(\"Lease should be invalid after expiration\")\n\t}\n\tif !l.NotifyExpiration() {\n\t\tt.Errorf(\"Lease.NotifyExpiration() return return true after expiration\")\n\t}\n\tif !l.NotifyExpiration() {\n\t\tt.Errorf(\"It should be leagal to call Lease.NotifyExpiration multiple times\")\n\t}\n\n\tselect {\n\tcase <-l.Done():\n\t\t// expected\n\tdefault:\n\t\tt.Errorf(\"Lease.Done() blocked after call to Lease.NotifyExpiration()\")\n\t}\n}\n" }, { "path": "internal/context/context.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage context\n\nimport (\n\t\"context\"\n\t\"time\"\n\n\t\"github.com/hibiken/asynq/internal/base\"\n)\n\n// A taskMetadata holds task scoped data to put in context.\ntype taskMetadata struct {\n\tid string\n\tmaxRetry int\n\tretryCount int\n\tqname string\n}\n\n// ctxKey type is unexported to prevent collisions with context keys defined in\n// other packages.\ntype ctxKey int\n\n// metadataCtxKey is the context key for the task metadata.\n// Its value of zero is arbitrary.\nconst metadataCtxKey ctxKey = 0\n\n// New returns a context and cancel function for a given task message.\nfunc New(base context.Context, msg *base.TaskMessage, deadline time.Time) (context.Context, context.CancelFunc) {\n\tmetadata := taskMetadata{\n\t\tid: msg.ID,\n\t\tmaxRetry: msg.Retry,\n\t\tretryCount: msg.Retried,\n\t\tqname: msg.Queue,\n\t}\n\tctx := context.WithValue(base, metadataCtxKey, metadata)\n\treturn context.WithDeadline(ctx, deadline)\n}\n\n// GetTaskID extracts a task ID from a context, if any.\n//\n// ID of a task is guaranteed to be unique.\n// ID of a task doesn't change if the task is being retried.\nfunc GetTaskID(ctx context.Context) (id string, ok bool) {\n\tmetadata, ok := ctx.Value(metadataCtxKey).(taskMetadata)\n\tif !ok {\n\t\treturn \"\", false\n\t}\n\treturn metadata.id, true\n}\n\n// GetRetryCount extracts retry count from a context, if any.\n//\n// Return value n indicates the number of times associated task has been\n// retried so far.\nfunc GetRetryCount(ctx context.Context) (n int, ok bool) {\n\tmetadata, ok := ctx.Value(metadataCtxKey).(taskMetadata)\n\tif !ok {\n\t\treturn 0, false\n\t}\n\treturn metadata.retryCount, true\n}\n\n// GetMaxRetry extracts maximum retry from a context, if any.\n//\n// Return value n indicates the maximum number of times the associated task\n// can be retried if ProcessTask returns a non-nil error.\nfunc GetMaxRetry(ctx context.Context) (n int, ok bool) {\n\tmetadata, ok := ctx.Value(metadataCtxKey).(taskMetadata)\n\tif !ok {\n\t\treturn 0, false\n\t}\n\treturn metadata.maxRetry, true\n}\n\n// GetQueueName extracts queue name from a context, if any.\n//\n// Return value qname indicates which queue the task was pulled from.\nfunc GetQueueName(ctx context.Context) (qname string, ok bool) {\n\tmetadata, ok := ctx.Value(metadataCtxKey).(taskMetadata)\n\tif !ok {\n\t\treturn \"\", false\n\t}\n\treturn metadata.qname, true\n}\n" }, { "path": "internal/context/context_test.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage context\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/google/go-cmp/cmp\"\n\t\"github.com/google/uuid\"\n\t\"github.com/hibiken/asynq/internal/base\"\n)\n\nfunc TestCreateContextWithFutureDeadline(t *testing.T) {\n\ttests := []struct {\n\t\tdeadline time.Time\n\t}{\n\t\t{time.Now().Add(time.Hour)},\n\t}\n\n\tfor _, tc := range tests {\n\t\tmsg := &base.TaskMessage{\n\t\t\tType: \"something\",\n\t\t\tID: uuid.NewString(),\n\t\t\tPayload: nil,\n\t\t}\n\n\t\tctx, cancel := New(context.Background(), msg, tc.deadline)\n\t\tselect {\n\t\tcase x := <-ctx.Done():\n\t\t\tt.Errorf(\"<-ctx.Done() == %v, want nothing (it should block)\", x)\n\t\tdefault:\n\t\t}\n\n\t\tgot, ok := ctx.Deadline()\n\t\tif !ok {\n\t\t\tt.Errorf(\"ctx.Deadline() returned false, want deadline to be set\")\n\t\t}\n\t\tif !cmp.Equal(tc.deadline, got) {\n\t\t\tt.Errorf(\"ctx.Deadline() returned %v, want %v\", got, tc.deadline)\n\t\t}\n\n\t\tcancel()\n\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\tdefault:\n\t\t\tt.Errorf(\"ctx.Done() blocked, want it to be non-blocking\")\n\t\t}\n\t}\n}\n\nfunc TestCreateContextWithBaseContext(t *testing.T) {\n\ttype ctxKey string\n\ttype ctxValue string\n\tvar key ctxKey = \"key\"\n\tvar value ctxValue = \"value\"\n\n\ttests := []struct {\n\t\tbaseCtx context.Context\n\t\tvalidate func(ctx context.Context, t *testing.T) error\n\t}{\n\t\t{\n\t\t\tbaseCtx: context.WithValue(context.Background(), key, value),\n\t\t\tvalidate: func(ctx context.Context, t *testing.T) error {\n\t\t\t\tgot, ok := ctx.Value(key).(ctxValue)\n\t\t\t\tif !ok {\n\t\t\t\t\treturn fmt.Errorf(\"ctx.Value().(ctxValue) returned false, expected to be true\")\n\t\t\t\t}\n\t\t\t\tif want := value; got != want {\n\t\t\t\t\treturn fmt.Errorf(\"ctx.Value().(ctxValue) returned unknown value (%v), expected to be %s\", got, value)\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tmsg := &base.TaskMessage{\n\t\t\tType: \"something\",\n\t\t\tID: uuid.NewString(),\n\t\t\tPayload: nil,\n\t\t}\n\n\t\tctx, cancel := New(tc.baseCtx, msg, time.Now().Add(30*time.Minute))\n\t\tdefer cancel()\n\n\t\tselect {\n\t\tcase x := <-ctx.Done():\n\t\t\tt.Errorf(\"<-ctx.Done() == %v, want nothing (it should block)\", x)\n\t\tdefault:\n\t\t}\n\n\t\tif err := tc.validate(ctx, t); err != nil {\n\t\t\tt.Errorf(\"%v\", err)\n\t\t}\n\t}\n}\n\nfunc TestCreateContextWithPastDeadline(t *testing.T) {\n\ttests := []struct {\n\t\tdeadline time.Time\n\t}{\n\t\t{time.Now().Add(-2 * time.Hour)},\n\t}\n\n\tfor _, tc := range tests {\n\t\tmsg := &base.TaskMessage{\n\t\t\tType: \"something\",\n\t\t\tID: uuid.NewString(),\n\t\t\tPayload: nil,\n\t\t}\n\n\t\tctx, cancel := New(context.Background(), msg, tc.deadline)\n\t\tdefer cancel()\n\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\tdefault:\n\t\t\tt.Errorf(\"ctx.Done() blocked, want it to be non-blocking\")\n\t\t}\n\n\t\tgot, ok := ctx.Deadline()\n\t\tif !ok {\n\t\t\tt.Errorf(\"ctx.Deadline() returned false, want deadline to be set\")\n\t\t}\n\t\tif !cmp.Equal(tc.deadline, got) {\n\t\t\tt.Errorf(\"ctx.Deadline() returned %v, want %v\", got, tc.deadline)\n\t\t}\n\t}\n}\n\nfunc TestGetTaskMetadataFromContext(t *testing.T) {\n\ttests := []struct {\n\t\tdesc string\n\t\tmsg *base.TaskMessage\n\t}{\n\t\t{\"with zero retried message\", &base.TaskMessage{Type: \"something\", ID: uuid.NewString(), Retry: 25, Retried: 0, Timeout: 1800, Queue: \"default\"}},\n\t\t{\"with non-zero retried message\", &base.TaskMessage{Type: \"something\", ID: uuid.NewString(), Retry: 10, Retried: 5, Timeout: 1800, Queue: \"default\"}},\n\t\t{\"with custom queue name\", &base.TaskMessage{Type: \"something\", ID: uuid.NewString(), Retry: 25, Retried: 0, Timeout: 1800, Queue: \"custom\"}},\n\t}\n\n\tfor _, tc := range tests {\n\t\tctx, cancel := New(context.Background(), tc.msg, time.Now().Add(30*time.Minute))\n\t\tdefer cancel()\n\n\t\tid, ok := GetTaskID(ctx)\n\t\tif !ok {\n\t\t\tt.Errorf(\"%s: GetTaskID(ctx) returned ok == false\", tc.desc)\n\t\t}\n\t\tif ok && id != tc.msg.ID {\n\t\t\tt.Errorf(\"%s: GetTaskID(ctx) returned id == %q, want %q\", tc.desc, id, tc.msg.ID)\n\t\t}\n\n\t\tretried, ok := GetRetryCount(ctx)\n\t\tif !ok {\n\t\t\tt.Errorf(\"%s: GetRetryCount(ctx) returned ok == false\", tc.desc)\n\t\t}\n\t\tif ok && retried != tc.msg.Retried {\n\t\t\tt.Errorf(\"%s: GetRetryCount(ctx) returned n == %d want %d\", tc.desc, retried, tc.msg.Retried)\n\t\t}\n\n\t\tmaxRetry, ok := GetMaxRetry(ctx)\n\t\tif !ok {\n\t\t\tt.Errorf(\"%s: GetMaxRetry(ctx) returned ok == false\", tc.desc)\n\t\t}\n\t\tif ok && maxRetry != tc.msg.Retry {\n\t\t\tt.Errorf(\"%s: GetMaxRetry(ctx) returned n == %d want %d\", tc.desc, maxRetry, tc.msg.Retry)\n\t\t}\n\n\t\tqname, ok := GetQueueName(ctx)\n\t\tif !ok {\n\t\t\tt.Errorf(\"%s: GetQueueName(ctx) returned ok == false\", tc.desc)\n\t\t}\n\t\tif ok && qname != tc.msg.Queue {\n\t\t\tt.Errorf(\"%s: GetQueueName(ctx) returned qname == %q, want %q\", tc.desc, qname, tc.msg.Queue)\n\t\t}\n\t}\n}\n\nfunc TestGetTaskMetadataFromContextError(t *testing.T) {\n\ttests := []struct {\n\t\tdesc string\n\t\tctx context.Context\n\t}{\n\t\t{\"with background context\", context.Background()},\n\t}\n\n\tfor _, tc := range tests {\n\t\tif _, ok := GetTaskID(tc.ctx); ok {\n\t\t\tt.Errorf(\"%s: GetTaskID(ctx) returned ok == true\", tc.desc)\n\t\t}\n\t\tif _, ok := GetRetryCount(tc.ctx); ok {\n\t\t\tt.Errorf(\"%s: GetRetryCount(ctx) returned ok == true\", tc.desc)\n\t\t}\n\t\tif _, ok := GetMaxRetry(tc.ctx); ok {\n\t\t\tt.Errorf(\"%s: GetMaxRetry(ctx) returned ok == true\", tc.desc)\n\t\t}\n\t\tif _, ok := GetQueueName(tc.ctx); ok {\n\t\t\tt.Errorf(\"%s: GetQueueName(ctx) returned ok == true\", tc.desc)\n\t\t}\n\t}\n}\n" }, { "path": "internal/errors/errors.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\n// Package errors defines the error type and functions used by\n// asynq and its internal packages.\npackage errors\n\n// Note: This package is inspired by a blog post about error handling in project Upspin\n// https://commandcenter.blogspot.com/2017/12/error-handling-in-upspin.html.\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"log\"\n\t\"runtime\"\n\t\"strings\"\n)\n\n// Error is the type that implements the error interface.\n// It contains a number of fields, each of different type.\n// An Error value may leave some values unset.\ntype Error struct {\n\tCode Code\n\tOp Op\n\tErr error\n}\n\nfunc (e *Error) DebugString() string {\n\tvar b strings.Builder\n\tif e.Op != \"\" {\n\t\tb.WriteString(string(e.Op))\n\t}\n\tif e.Code != Unspecified {\n\t\tif b.Len() > 0 {\n\t\t\tb.WriteString(\": \")\n\t\t}\n\t\tb.WriteString(e.Code.String())\n\t}\n\tif e.Err != nil {\n\t\tif b.Len() > 0 {\n\t\t\tb.WriteString(\": \")\n\t\t}\n\t\tb.WriteString(e.Err.Error())\n\t}\n\treturn b.String()\n}\n\nfunc (e *Error) Error() string {\n\tvar b strings.Builder\n\tif e.Code != Unspecified {\n\t\tb.WriteString(e.Code.String())\n\t}\n\tif e.Err != nil {\n\t\tif b.Len() > 0 {\n\t\t\tb.WriteString(\": \")\n\t\t}\n\t\tb.WriteString(e.Err.Error())\n\t}\n\treturn b.String()\n}\n\nfunc (e *Error) Unwrap() error {\n\treturn e.Err\n}\n\n// Code defines the canonical error code.\ntype Code uint8\n\n// List of canonical error codes.\nconst (\n\tUnspecified Code = iota\n\tNotFound\n\tFailedPrecondition\n\tInternal\n\tAlreadyExists\n\tUnknown\n\t// Note: If you add a new value here, make sure to update String method.\n)\n\nfunc (c Code) String() string {\n\tswitch c {\n\tcase Unspecified:\n\t\treturn \"ERROR_CODE_UNSPECIFIED\"\n\tcase NotFound:\n\t\treturn \"NOT_FOUND\"\n\tcase FailedPrecondition:\n\t\treturn \"FAILED_PRECONDITION\"\n\tcase Internal:\n\t\treturn \"INTERNAL_ERROR\"\n\tcase AlreadyExists:\n\t\treturn \"ALREADY_EXISTS\"\n\tcase Unknown:\n\t\treturn \"UNKNOWN\"\n\t}\n\tpanic(fmt.Sprintf(\"unknown error code %d\", c))\n}\n\n// Op describes an operation, usually as the package and method,\n// such as \"rdb.Enqueue\".\ntype Op string\n\n// E builds an error value from its arguments.\n// There must be at least one argument or E panics.\n// The type of each argument determines its meaning.\n// If more than one argument of a given type is presented,\n// only the last one is recorded.\n//\n// The types are:\n//\n//\terrors.Op\n//\t\tThe operation being performed, usually the method\n//\t\tbeing invoked (Get, Put, etc.).\n//\terrors.Code\n//\t\tThe canonical error code, such as NOT_FOUND.\n//\tstring\n//\t\tTreated as an error message and assigned to the\n//\t\tErr field after a call to errors.New.\n//\terror\n//\t\tThe underlying error that triggered this one.\n//\n// If the error is printed, only those items that have been\n// set to non-zero values will appear in the result.\nfunc E(args ...interface{}) error {\n\tif len(args) == 0 {\n\t\tpanic(\"call to errors.E with no arguments\")\n\t}\n\te := &Error{}\n\tfor _, arg := range args {\n\t\tswitch arg := arg.(type) {\n\t\tcase Op:\n\t\t\te.Op = arg\n\t\tcase Code:\n\t\t\te.Code = arg\n\t\tcase error:\n\t\t\te.Err = arg\n\t\tcase string:\n\t\t\te.Err = errors.New(arg)\n\t\tdefault:\n\t\t\t_, file, line, _ := runtime.Caller(1)\n\t\t\tlog.Printf(\"errors.E: bad call from %s:%d: %v\", file, line, args)\n\t\t\treturn fmt.Errorf(\"unknown type %T, value %v in error call\", arg, arg)\n\t\t}\n\t}\n\treturn e\n}\n\n// CanonicalCode returns the canonical code of the given error if one is present.\n// Otherwise it returns Unspecified.\nfunc CanonicalCode(err error) Code {\n\tif err == nil {\n\t\treturn Unspecified\n\t}\n\te, ok := err.(*Error)\n\tif !ok {\n\t\treturn Unspecified\n\t}\n\tif e.Code == Unspecified {\n\t\treturn CanonicalCode(e.Err)\n\t}\n\treturn e.Code\n}\n\n/******************************************\n Domain Specific Error Types & Values\n*******************************************/\n\nvar (\n\t// ErrNoProcessableTask indicates that there are no tasks ready to be processed.\n\tErrNoProcessableTask = errors.New(\"no tasks are ready for processing\")\n\n\t// ErrDuplicateTask indicates that another task with the same unique key holds the uniqueness lock.\n\tErrDuplicateTask = errors.New(\"task already exists\")\n\n\t// ErrTaskIdConflict indicates that another task with the same task ID already exist\n\tErrTaskIdConflict = errors.New(\"task id conflicts with another task\")\n)\n\n// TaskNotFoundError indicates that a task with the given ID does not exist\n// in the given queue.\ntype TaskNotFoundError struct {\n\tQueue string // queue name\n\tID string // task id\n}\n\nfunc (e *TaskNotFoundError) Error() string {\n\treturn fmt.Sprintf(\"cannot find task with id=%s in queue %q\", e.ID, e.Queue)\n}\n\n// IsTaskNotFound reports whether any error in err's chain is of type TaskNotFoundError.\nfunc IsTaskNotFound(err error) bool {\n\tvar target *TaskNotFoundError\n\treturn As(err, &target)\n}\n\n// QueueNotFoundError indicates that a queue with the given name does not exist.\ntype QueueNotFoundError struct {\n\tQueue string // queue name\n}\n\nfunc (e *QueueNotFoundError) Error() string {\n\treturn fmt.Sprintf(\"queue %q does not exist\", e.Queue)\n}\n\n// IsQueueNotFound reports whether any error in err's chain is of type QueueNotFoundError.\nfunc IsQueueNotFound(err error) bool {\n\tvar target *QueueNotFoundError\n\treturn As(err, &target)\n}\n\n// QueueNotEmptyError indicates that the given queue is not empty.\ntype QueueNotEmptyError struct {\n\tQueue string // queue name\n}\n\nfunc (e *QueueNotEmptyError) Error() string {\n\treturn fmt.Sprintf(\"queue %q is not empty\", e.Queue)\n}\n\n// IsQueueNotEmpty reports whether any error in err's chain is of type QueueNotEmptyError.\nfunc IsQueueNotEmpty(err error) bool {\n\tvar target *QueueNotEmptyError\n\treturn As(err, &target)\n}\n\n// TaskAlreadyArchivedError indicates that the task in question is already archived.\ntype TaskAlreadyArchivedError struct {\n\tQueue string // queue name\n\tID string // task id\n}\n\nfunc (e *TaskAlreadyArchivedError) Error() string {\n\treturn fmt.Sprintf(\"task is already archived: id=%s, queue=%s\", e.ID, e.Queue)\n}\n\n// IsTaskAlreadyArchived reports whether any error in err's chain is of type TaskAlreadyArchivedError.\nfunc IsTaskAlreadyArchived(err error) bool {\n\tvar target *TaskAlreadyArchivedError\n\treturn As(err, &target)\n}\n\n// RedisCommandError indicates that the given redis command returned error.\ntype RedisCommandError struct {\n\tCommand string // redis command (e.g. LRANGE, ZADD, etc)\n\tErr error // underlying error\n}\n\nfunc (e *RedisCommandError) Error() string {\n\treturn fmt.Sprintf(\"redis command error: %s failed: %v\", strings.ToUpper(e.Command), e.Err)\n}\n\nfunc (e *RedisCommandError) Unwrap() error { return e.Err }\n\n// IsRedisCommandError reports whether any error in err's chain is of type RedisCommandError.\nfunc IsRedisCommandError(err error) bool {\n\tvar target *RedisCommandError\n\treturn As(err, &target)\n}\n\n// PanicError defines an error when occurred a panic error.\ntype PanicError struct {\n\tErrMsg string\n}\n\nfunc (e *PanicError) Error() string {\n\treturn fmt.Sprintf(\"panic error cause by: %s\", e.ErrMsg)\n}\n\n// IsPanicError reports whether any error in err's chain is of type PanicError.\nfunc IsPanicError(err error) bool {\n\tvar target *PanicError\n\treturn As(err, &target)\n}\n\n/*************************************************\n Standard Library errors package functions\n*************************************************/\n\n// New returns an error that formats as the given text.\n// Each call to New returns a distinct error value even if the text is identical.\n//\n// This function is the errors.New function from the standard library (https://golang.org/pkg/errors/#New).\n// It is exported from this package for import convenience.\nfunc New(text string) error { return errors.New(text) }\n\n// Is reports whether any error in err's chain matches target.\n//\n// This function is the errors.Is function from the standard library (https://golang.org/pkg/errors/#Is).\n// It is exported from this package for import convenience.\nfunc Is(err, target error) bool { return errors.Is(err, target) }\n\n// As finds the first error in err's chain that matches target, and if so, sets target to that error value and returns true.\n// Otherwise, it returns false.\n//\n// This function is the errors.As function from the standard library (https://golang.org/pkg/errors/#As).\n// It is exported from this package for import convenience.\nfunc As(err error, target interface{}) bool { return errors.As(err, target) }\n\n// Unwrap returns the result of calling the Unwrap method on err, if err's type contains an Unwrap method returning error.\n// Otherwise, Unwrap returns nil.\n//\n// This function is the errors.Unwrap function from the standard library (https://golang.org/pkg/errors/#Unwrap).\n// It is exported from this package for import convenience.\nfunc Unwrap(err error) error { return errors.Unwrap(err) }\n" }, { "path": "internal/errors/errors_test.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage errors\n\nimport \"testing\"\n\nfunc TestErrorDebugString(t *testing.T) {\n\t// DebugString should include Op since its meant to be used by\n\t// maintainers/contributors of the asynq package.\n\ttests := []struct {\n\t\tdesc string\n\t\terr error\n\t\twant string\n\t}{\n\t\t{\n\t\t\tdesc: \"With Op, Code, and string\",\n\t\t\terr: E(Op(\"rdb.DeleteTask\"), NotFound, \"cannot find task with id=123\"),\n\t\t\twant: \"rdb.DeleteTask: NOT_FOUND: cannot find task with id=123\",\n\t\t},\n\t\t{\n\t\t\tdesc: \"With Op, Code and error\",\n\t\t\terr: E(Op(\"rdb.DeleteTask\"), NotFound, &TaskNotFoundError{Queue: \"default\", ID: \"123\"}),\n\t\t\twant: `rdb.DeleteTask: NOT_FOUND: cannot find task with id=123 in queue \"default\"`,\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tif got := tc.err.(*Error).DebugString(); got != tc.want {\n\t\t\tt.Errorf(\"%s: got=%q, want=%q\", tc.desc, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestErrorString(t *testing.T) {\n\t// String method should omit Op since op is an internal detail\n\t// and we don't want to provide it to users of the package.\n\ttests := []struct {\n\t\tdesc string\n\t\terr error\n\t\twant string\n\t}{\n\t\t{\n\t\t\tdesc: \"With Op, Code, and string\",\n\t\t\terr: E(Op(\"rdb.DeleteTask\"), NotFound, \"cannot find task with id=123\"),\n\t\t\twant: \"NOT_FOUND: cannot find task with id=123\",\n\t\t},\n\t\t{\n\t\t\tdesc: \"With Op, Code and error\",\n\t\t\terr: E(Op(\"rdb.DeleteTask\"), NotFound, &TaskNotFoundError{Queue: \"default\", ID: \"123\"}),\n\t\t\twant: `NOT_FOUND: cannot find task with id=123 in queue \"default\"`,\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tif got := tc.err.Error(); got != tc.want {\n\t\t\tt.Errorf(\"%s: got=%q, want=%q\", tc.desc, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestErrorIs(t *testing.T) {\n\tvar ErrCustom = New(\"custom sentinel error\")\n\n\ttests := []struct {\n\t\tdesc string\n\t\terr error\n\t\ttarget error\n\t\twant bool\n\t}{\n\t\t{\n\t\t\tdesc: \"should unwrap one level\",\n\t\t\terr: E(Op(\"rdb.DeleteTask\"), ErrCustom),\n\t\t\ttarget: ErrCustom,\n\t\t\twant: true,\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tif got := Is(tc.err, tc.target); got != tc.want {\n\t\t\tt.Errorf(\"%s: got=%t, want=%t\", tc.desc, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestErrorAs(t *testing.T) {\n\ttests := []struct {\n\t\tdesc string\n\t\terr error\n\t\ttarget interface{}\n\t\twant bool\n\t}{\n\t\t{\n\t\t\tdesc: \"should unwrap one level\",\n\t\t\terr: E(Op(\"rdb.DeleteTask\"), NotFound, &QueueNotFoundError{Queue: \"email\"}),\n\t\t\ttarget: &QueueNotFoundError{},\n\t\t\twant: true,\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tif got := As(tc.err, &tc.target); got != tc.want {\n\t\t\tt.Errorf(\"%s: got=%t, want=%t\", tc.desc, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestErrorPredicates(t *testing.T) {\n\ttests := []struct {\n\t\tdesc string\n\t\tfn func(err error) bool\n\t\terr error\n\t\twant bool\n\t}{\n\t\t{\n\t\t\tdesc: \"IsTaskNotFound should detect presence of TaskNotFoundError in err's chain\",\n\t\t\tfn: IsTaskNotFound,\n\t\t\terr: E(Op(\"rdb.ArchiveTask\"), NotFound, &TaskNotFoundError{Queue: \"default\", ID: \"9876\"}),\n\t\t\twant: true,\n\t\t},\n\t\t{\n\t\t\tdesc: \"IsTaskNotFound should detect absence of TaskNotFoundError in err's chain\",\n\t\t\tfn: IsTaskNotFound,\n\t\t\terr: E(Op(\"rdb.ArchiveTask\"), NotFound, &QueueNotFoundError{Queue: \"default\"}),\n\t\t\twant: false,\n\t\t},\n\t\t{\n\t\t\tdesc: \"IsQueueNotFound should detect presence of QueueNotFoundError in err's chain\",\n\t\t\tfn: IsQueueNotFound,\n\t\t\terr: E(Op(\"rdb.ArchiveTask\"), NotFound, &QueueNotFoundError{Queue: \"default\"}),\n\t\t\twant: true,\n\t\t},\n\t\t{\n\t\t\tdesc: \"IsPanicError should detect presence of PanicError in err's chain\",\n\t\t\tfn: IsPanicError,\n\t\t\terr: E(Op(\"unknown\"), Unknown, &PanicError{ErrMsg: \"Something went wrong\"}),\n\t\t\twant: true,\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tif got := tc.fn(tc.err); got != tc.want {\n\t\t\tt.Errorf(\"%s: got=%t, want=%t\", tc.desc, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestCanonicalCode(t *testing.T) {\n\ttests := []struct {\n\t\tdesc string\n\t\terr error\n\t\twant Code\n\t}{\n\t\t{\n\t\t\tdesc: \"without nesting\",\n\t\t\terr: E(Op(\"rdb.DeleteTask\"), NotFound, &TaskNotFoundError{Queue: \"default\", ID: \"123\"}),\n\t\t\twant: NotFound,\n\t\t},\n\t\t{\n\t\t\tdesc: \"with nesting\",\n\t\t\terr: E(FailedPrecondition, E(NotFound)),\n\t\t\twant: FailedPrecondition,\n\t\t},\n\t\t{\n\t\t\tdesc: \"returns Unspecified if err is not *Error\",\n\t\t\terr: New(\"some other error\"),\n\t\t\twant: Unspecified,\n\t\t},\n\t\t{\n\t\t\tdesc: \"returns Unspecified if err is nil\",\n\t\t\terr: nil,\n\t\t\twant: Unspecified,\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tif got := CanonicalCode(tc.err); got != tc.want {\n\t\t\tt.Errorf(\"%s: got=%s, want=%s\", tc.desc, got, tc.want)\n\t\t}\n\t}\n}\n" }, { "path": "internal/log/log.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\n// Package log exports logging related types and functions.\npackage log\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\tstdlog \"log\"\n\t\"os\"\n\t\"sync\"\n)\n\n// Base supports logging at various log levels.\ntype Base interface {\n\t// Debug logs a message at Debug level.\n\tDebug(args ...interface{})\n\n\t// Info logs a message at Info level.\n\tInfo(args ...interface{})\n\n\t// Warn logs a message at Warning level.\n\tWarn(args ...interface{})\n\n\t// Error logs a message at Error level.\n\tError(args ...interface{})\n\n\t// Fatal logs a message at Fatal level\n\t// and process will exit with status set to 1.\n\tFatal(args ...interface{})\n}\n\n// baseLogger is a wrapper object around log.Logger from the standard library.\n// It supports logging at various log levels.\ntype baseLogger struct {\n\t*stdlog.Logger\n}\n\n// Debug logs a message at Debug level.\nfunc (l *baseLogger) Debug(args ...interface{}) {\n\tl.prefixPrint(\"DEBUG: \", args...)\n}\n\n// Info logs a message at Info level.\nfunc (l *baseLogger) Info(args ...interface{}) {\n\tl.prefixPrint(\"INFO: \", args...)\n}\n\n// Warn logs a message at Warning level.\nfunc (l *baseLogger) Warn(args ...interface{}) {\n\tl.prefixPrint(\"WARN: \", args...)\n}\n\n// Error logs a message at Error level.\nfunc (l *baseLogger) Error(args ...interface{}) {\n\tl.prefixPrint(\"ERROR: \", args...)\n}\n\n// Fatal logs a message at Fatal level\n// and process will exit with status set to 1.\nfunc (l *baseLogger) Fatal(args ...interface{}) {\n\tl.prefixPrint(\"FATAL: \", args...)\n\tos.Exit(1)\n}\n\nfunc (l *baseLogger) prefixPrint(prefix string, args ...interface{}) {\n\targs = append([]interface{}{prefix}, args...)\n\tl.Print(args...)\n}\n\n// newBase creates and returns a new instance of baseLogger.\nfunc newBase(out io.Writer) *baseLogger {\n\tprefix := fmt.Sprintf(\"asynq: pid=%d \", os.Getpid())\n\treturn &baseLogger{\n\t\tstdlog.New(out, prefix, stdlog.Ldate|stdlog.Ltime|stdlog.Lmicroseconds|stdlog.LUTC),\n\t}\n}\n\n// NewLogger creates and returns a new instance of Logger.\n// Log level is set to DebugLevel by default.\nfunc NewLogger(base Base) *Logger {\n\tif base == nil {\n\t\tbase = newBase(os.Stderr)\n\t}\n\treturn &Logger{base: base, level: DebugLevel}\n}\n\n// Logger logs message to io.Writer at various log levels.\ntype Logger struct {\n\tbase Base\n\n\tmu sync.Mutex\n\t// Minimum log level for this logger.\n\t// Message with level lower than this level won't be outputted.\n\tlevel Level\n}\n\n// Level represents a log level.\ntype Level int32\n\nconst (\n\t// DebugLevel is the lowest level of logging.\n\t// Debug logs are intended for debugging and development purposes.\n\tDebugLevel Level = iota\n\n\t// InfoLevel is used for general informational log messages.\n\tInfoLevel\n\n\t// WarnLevel is used for undesired but relatively expected events,\n\t// which may indicate a problem.\n\tWarnLevel\n\n\t// ErrorLevel is used for undesired and unexpected events that\n\t// the program can recover from.\n\tErrorLevel\n\n\t// FatalLevel is used for undesired and unexpected events that\n\t// the program cannot recover from.\n\tFatalLevel\n)\n\n// String is part of the fmt.Stringer interface.\n//\n// Used for testing and debugging purposes.\nfunc (l Level) String() string {\n\tswitch l {\n\tcase DebugLevel:\n\t\treturn \"debug\"\n\tcase InfoLevel:\n\t\treturn \"info\"\n\tcase WarnLevel:\n\t\treturn \"warning\"\n\tcase ErrorLevel:\n\t\treturn \"error\"\n\tcase FatalLevel:\n\t\treturn \"fatal\"\n\tdefault:\n\t\treturn \"unknown\"\n\t}\n}\n\n// canLogAt reports whether logger can log at level v.\nfunc (l *Logger) canLogAt(v Level) bool {\n\tl.mu.Lock()\n\tdefer l.mu.Unlock()\n\treturn v >= l.level\n}\n\nfunc (l *Logger) Debug(args ...interface{}) {\n\tif !l.canLogAt(DebugLevel) {\n\t\treturn\n\t}\n\tl.base.Debug(args...)\n}\n\nfunc (l *Logger) Info(args ...interface{}) {\n\tif !l.canLogAt(InfoLevel) {\n\t\treturn\n\t}\n\tl.base.Info(args...)\n}\n\nfunc (l *Logger) Warn(args ...interface{}) {\n\tif !l.canLogAt(WarnLevel) {\n\t\treturn\n\t}\n\tl.base.Warn(args...)\n}\n\nfunc (l *Logger) Error(args ...interface{}) {\n\tif !l.canLogAt(ErrorLevel) {\n\t\treturn\n\t}\n\tl.base.Error(args...)\n}\n\nfunc (l *Logger) Fatal(args ...interface{}) {\n\tif !l.canLogAt(FatalLevel) {\n\t\treturn\n\t}\n\tl.base.Fatal(args...)\n}\n\nfunc (l *Logger) Debugf(format string, args ...interface{}) {\n\tl.Debug(fmt.Sprintf(format, args...))\n}\n\nfunc (l *Logger) Infof(format string, args ...interface{}) {\n\tl.Info(fmt.Sprintf(format, args...))\n}\n\nfunc (l *Logger) Warnf(format string, args ...interface{}) {\n\tl.Warn(fmt.Sprintf(format, args...))\n}\n\nfunc (l *Logger) Errorf(format string, args ...interface{}) {\n\tl.Error(fmt.Sprintf(format, args...))\n}\n\nfunc (l *Logger) Fatalf(format string, args ...interface{}) {\n\tl.Fatal(fmt.Sprintf(format, args...))\n}\n\n// SetLevel sets the logger level.\n// It panics if v is less than DebugLevel or greater than FatalLevel.\nfunc (l *Logger) SetLevel(v Level) {\n\tl.mu.Lock()\n\tdefer l.mu.Unlock()\n\tif v < DebugLevel || v > FatalLevel {\n\t\tpanic(\"log: invalid log level\")\n\t}\n\tl.level = v\n}\n" }, { "path": "internal/log/log_test.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage log\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"regexp\"\n\t\"testing\"\n)\n\n// regexp for timestamps\nconst (\n\trgxPID = `[0-9]+`\n\trgxdate = `[0-9][0-9][0-9][0-9]/[0-9][0-9]/[0-9][0-9]`\n\trgxtime = `[0-9][0-9]:[0-9][0-9]:[0-9][0-9]`\n\trgxmicroseconds = `\\.[0-9][0-9][0-9][0-9][0-9][0-9]`\n)\n\ntype tester struct {\n\tdesc string\n\tmessage string\n\twantPattern string // regexp that log output must match\n}\n\nfunc TestLoggerDebug(t *testing.T) {\n\ttests := []tester{\n\t\t{\n\t\t\tdesc: \"without trailing newline, logger adds newline\",\n\t\t\tmessage: \"hello, world!\",\n\t\t\twantPattern: fmt.Sprintf(\"^asynq: pid=%s %s %s%s DEBUG: hello, world!\\n$\",\n\t\t\t\trgxPID, rgxdate, rgxtime, rgxmicroseconds),\n\t\t},\n\t\t{\n\t\t\tdesc: \"with trailing newline, logger preserves newline\",\n\t\t\tmessage: \"hello, world!\\n\",\n\t\t\twantPattern: fmt.Sprintf(\"^asynq: pid=%s %s %s%s DEBUG: hello, world!\\n$\",\n\t\t\t\trgxPID, rgxdate, rgxtime, rgxmicroseconds),\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tvar buf bytes.Buffer\n\t\tlogger := NewLogger(newBase(&buf))\n\n\t\tlogger.Debug(tc.message)\n\n\t\tgot := buf.String()\n\t\tmatched, err := regexp.MatchString(tc.wantPattern, got)\n\t\tif err != nil {\n\t\t\tt.Fatal(\"pattern did not compile:\", err)\n\t\t}\n\t\tif !matched {\n\t\t\tt.Errorf(\"logger.Debug(%q) outputted %q, should match pattern %q\",\n\t\t\t\ttc.message, got, tc.wantPattern)\n\t\t}\n\t}\n}\n\nfunc TestLoggerInfo(t *testing.T) {\n\ttests := []tester{\n\t\t{\n\t\t\tdesc: \"without trailing newline, logger adds newline\",\n\t\t\tmessage: \"hello, world!\",\n\t\t\twantPattern: fmt.Sprintf(\"^asynq: pid=%s %s %s%s INFO: hello, world!\\n$\",\n\t\t\t\trgxPID, rgxdate, rgxtime, rgxmicroseconds),\n\t\t},\n\t\t{\n\t\t\tdesc: \"with trailing newline, logger preserves newline\",\n\t\t\tmessage: \"hello, world!\\n\",\n\t\t\twantPattern: fmt.Sprintf(\"^asynq: pid=%s %s %s%s INFO: hello, world!\\n$\",\n\t\t\t\trgxPID, rgxdate, rgxtime, rgxmicroseconds),\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tvar buf bytes.Buffer\n\t\tlogger := NewLogger(newBase(&buf))\n\n\t\tlogger.Info(tc.message)\n\n\t\tgot := buf.String()\n\t\tmatched, err := regexp.MatchString(tc.wantPattern, got)\n\t\tif err != nil {\n\t\t\tt.Fatal(\"pattern did not compile:\", err)\n\t\t}\n\t\tif !matched {\n\t\t\tt.Errorf(\"logger.Info(%q) outputted %q, should match pattern %q\",\n\t\t\t\ttc.message, got, tc.wantPattern)\n\t\t}\n\t}\n}\n\nfunc TestLoggerWarn(t *testing.T) {\n\ttests := []tester{\n\t\t{\n\t\t\tdesc: \"without trailing newline, logger adds newline\",\n\t\t\tmessage: \"hello, world!\",\n\t\t\twantPattern: fmt.Sprintf(\"^asynq: pid=%s %s %s%s WARN: hello, world!\\n$\",\n\t\t\t\trgxPID, rgxdate, rgxtime, rgxmicroseconds),\n\t\t},\n\t\t{\n\t\t\tdesc: \"with trailing newline, logger preserves newline\",\n\t\t\tmessage: \"hello, world!\\n\",\n\t\t\twantPattern: fmt.Sprintf(\"^asynq: pid=%s %s %s%s WARN: hello, world!\\n$\",\n\t\t\t\trgxPID, rgxdate, rgxtime, rgxmicroseconds),\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tvar buf bytes.Buffer\n\t\tlogger := NewLogger(newBase(&buf))\n\n\t\tlogger.Warn(tc.message)\n\n\t\tgot := buf.String()\n\t\tmatched, err := regexp.MatchString(tc.wantPattern, got)\n\t\tif err != nil {\n\t\t\tt.Fatal(\"pattern did not compile:\", err)\n\t\t}\n\t\tif !matched {\n\t\t\tt.Errorf(\"logger.Warn(%q) outputted %q, should match pattern %q\",\n\t\t\t\ttc.message, got, tc.wantPattern)\n\t\t}\n\t}\n}\n\nfunc TestLoggerError(t *testing.T) {\n\ttests := []tester{\n\t\t{\n\t\t\tdesc: \"without trailing newline, logger adds newline\",\n\t\t\tmessage: \"hello, world!\",\n\t\t\twantPattern: fmt.Sprintf(\"^asynq: pid=%s %s %s%s ERROR: hello, world!\\n$\",\n\t\t\t\trgxPID, rgxdate, rgxtime, rgxmicroseconds),\n\t\t},\n\t\t{\n\t\t\tdesc: \"with trailing newline, logger preserves newline\",\n\t\t\tmessage: \"hello, world!\\n\",\n\t\t\twantPattern: fmt.Sprintf(\"^asynq: pid=%s %s %s%s ERROR: hello, world!\\n$\",\n\t\t\t\trgxPID, rgxdate, rgxtime, rgxmicroseconds),\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tvar buf bytes.Buffer\n\t\tlogger := NewLogger(newBase(&buf))\n\n\t\tlogger.Error(tc.message)\n\n\t\tgot := buf.String()\n\t\tmatched, err := regexp.MatchString(tc.wantPattern, got)\n\t\tif err != nil {\n\t\t\tt.Fatal(\"pattern did not compile:\", err)\n\t\t}\n\t\tif !matched {\n\t\t\tt.Errorf(\"logger.Error(%q) outputted %q, should match pattern %q\",\n\t\t\t\ttc.message, got, tc.wantPattern)\n\t\t}\n\t}\n}\n\ntype formatTester struct {\n\tdesc string\n\tformat string\n\targs []interface{}\n\twantPattern string // regexp that log output must match\n}\n\nfunc TestLoggerDebugf(t *testing.T) {\n\ttests := []formatTester{\n\t\t{\n\t\t\tdesc: \"Formats message with DEBUG prefix\",\n\t\t\tformat: \"hello, %s!\",\n\t\t\targs: []interface{}{\"Gopher\"},\n\t\t\twantPattern: fmt.Sprintf(\"^asynq: pid=%s %s %s%s DEBUG: hello, Gopher!\\n$\",\n\t\t\t\trgxPID, rgxdate, rgxtime, rgxmicroseconds),\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tvar buf bytes.Buffer\n\t\tlogger := NewLogger(newBase(&buf))\n\n\t\tlogger.Debugf(tc.format, tc.args...)\n\n\t\tgot := buf.String()\n\t\tmatched, err := regexp.MatchString(tc.wantPattern, got)\n\t\tif err != nil {\n\t\t\tt.Fatal(\"pattern did not compile:\", err)\n\t\t}\n\t\tif !matched {\n\t\t\tt.Errorf(\"logger.Debugf(%q, %v) outputted %q, should match pattern %q\",\n\t\t\t\ttc.format, tc.args, got, tc.wantPattern)\n\t\t}\n\t}\n}\n\nfunc TestLoggerInfof(t *testing.T) {\n\ttests := []formatTester{\n\t\t{\n\t\t\tdesc: \"Formats message with INFO prefix\",\n\t\t\tformat: \"%d,%d,%d\",\n\t\t\targs: []interface{}{1, 2, 3},\n\t\t\twantPattern: fmt.Sprintf(\"^asynq: pid=%s %s %s%s INFO: 1,2,3\\n$\",\n\t\t\t\trgxPID, rgxdate, rgxtime, rgxmicroseconds),\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tvar buf bytes.Buffer\n\t\tlogger := NewLogger(newBase(&buf))\n\n\t\tlogger.Infof(tc.format, tc.args...)\n\n\t\tgot := buf.String()\n\t\tmatched, err := regexp.MatchString(tc.wantPattern, got)\n\t\tif err != nil {\n\t\t\tt.Fatal(\"pattern did not compile:\", err)\n\t\t}\n\t\tif !matched {\n\t\t\tt.Errorf(\"logger.Infof(%q, %v) outputted %q, should match pattern %q\",\n\t\t\t\ttc.format, tc.args, got, tc.wantPattern)\n\t\t}\n\t}\n}\n\nfunc TestLoggerWarnf(t *testing.T) {\n\ttests := []formatTester{\n\t\t{\n\t\t\tdesc: \"Formats message with WARN prefix\",\n\t\t\tformat: \"hello, %s\",\n\t\t\targs: []interface{}{\"Gophers\"},\n\t\t\twantPattern: fmt.Sprintf(\"^asynq: pid=%s %s %s%s WARN: hello, Gophers\\n$\",\n\t\t\t\trgxPID, rgxdate, rgxtime, rgxmicroseconds),\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tvar buf bytes.Buffer\n\t\tlogger := NewLogger(newBase(&buf))\n\n\t\tlogger.Warnf(tc.format, tc.args...)\n\n\t\tgot := buf.String()\n\t\tmatched, err := regexp.MatchString(tc.wantPattern, got)\n\t\tif err != nil {\n\t\t\tt.Fatal(\"pattern did not compile:\", err)\n\t\t}\n\t\tif !matched {\n\t\t\tt.Errorf(\"logger.Warnf(%q, %v) outputted %q, should match pattern %q\",\n\t\t\t\ttc.format, tc.args, got, tc.wantPattern)\n\t\t}\n\t}\n}\n\nfunc TestLoggerErrorf(t *testing.T) {\n\ttests := []formatTester{\n\t\t{\n\t\t\tdesc: \"Formats message with ERROR prefix\",\n\t\t\tformat: \"hello, %s\",\n\t\t\targs: []interface{}{\"Gophers\"},\n\t\t\twantPattern: fmt.Sprintf(\"^asynq: pid=%s %s %s%s ERROR: hello, Gophers\\n$\",\n\t\t\t\trgxPID, rgxdate, rgxtime, rgxmicroseconds),\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tvar buf bytes.Buffer\n\t\tlogger := NewLogger(newBase(&buf))\n\n\t\tlogger.Errorf(tc.format, tc.args...)\n\n\t\tgot := buf.String()\n\t\tmatched, err := regexp.MatchString(tc.wantPattern, got)\n\t\tif err != nil {\n\t\t\tt.Fatal(\"pattern did not compile:\", err)\n\t\t}\n\t\tif !matched {\n\t\t\tt.Errorf(\"logger.Errorf(%q, %v) outputted %q, should match pattern %q\",\n\t\t\t\ttc.format, tc.args, got, tc.wantPattern)\n\t\t}\n\t}\n}\n\nfunc TestLoggerWithLowerLevels(t *testing.T) {\n\t// Logger should not log messages at a level\n\t// lower than the specified level.\n\ttests := []struct {\n\t\tlevel Level\n\t\top string\n\t}{\n\t\t// with level one above\n\t\t{InfoLevel, \"Debug\"},\n\t\t{InfoLevel, \"Debugf\"},\n\t\t{WarnLevel, \"Info\"},\n\t\t{WarnLevel, \"Infof\"},\n\t\t{ErrorLevel, \"Warn\"},\n\t\t{ErrorLevel, \"Warnf\"},\n\t\t{FatalLevel, \"Error\"},\n\t\t{FatalLevel, \"Errorf\"},\n\t\t// with skip level\n\t\t{WarnLevel, \"Debug\"},\n\t\t{ErrorLevel, \"Infof\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\tvar buf bytes.Buffer\n\t\tlogger := NewLogger(newBase(&buf))\n\t\tlogger.SetLevel(tc.level)\n\n\t\tswitch tc.op {\n\t\tcase \"Debug\":\n\t\t\tlogger.Debug(\"hello\")\n\t\tcase \"Debugf\":\n\t\t\tlogger.Debugf(\"hello, %s\", \"world\")\n\t\tcase \"Info\":\n\t\t\tlogger.Info(\"hello\")\n\t\tcase \"Infof\":\n\t\t\tlogger.Infof(\"hello, %s\", \"world\")\n\t\tcase \"Warn\":\n\t\t\tlogger.Warn(\"hello\")\n\t\tcase \"Warnf\":\n\t\t\tlogger.Warnf(\"hello, %s\", \"world\")\n\t\tcase \"Error\":\n\t\t\tlogger.Error(\"hello\")\n\t\tcase \"Errorf\":\n\t\t\tlogger.Errorf(\"hello, %s\", \"world\")\n\t\tdefault:\n\t\t\tt.Fatalf(\"unexpected op: %q\", tc.op)\n\t\t}\n\n\t\tif buf.String() != \"\" {\n\t\t\tt.Errorf(\"logger.%s outputted log message when level is set to %v\", tc.op, tc.level)\n\t\t}\n\t}\n}\n\nfunc TestLoggerWithSameOrHigherLevels(t *testing.T) {\n\t// Logger should log messages at a level\n\t// same as or higher than the specified level.\n\ttests := []struct {\n\t\tlevel Level\n\t\top string\n\t}{\n\t\t// same level\n\t\t{DebugLevel, \"Debug\"},\n\t\t{InfoLevel, \"Infof\"},\n\t\t{WarnLevel, \"Warn\"},\n\t\t{ErrorLevel, \"Errorf\"},\n\t\t// higher level\n\t\t{DebugLevel, \"Info\"},\n\t\t{InfoLevel, \"Warnf\"},\n\t\t{WarnLevel, \"Error\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\tvar buf bytes.Buffer\n\t\tlogger := NewLogger(newBase(&buf))\n\t\tlogger.SetLevel(tc.level)\n\n\t\tswitch tc.op {\n\t\tcase \"Debug\":\n\t\t\tlogger.Debug(\"hello\")\n\t\tcase \"Debugf\":\n\t\t\tlogger.Debugf(\"hello, %s\", \"world\")\n\t\tcase \"Info\":\n\t\t\tlogger.Info(\"hello\")\n\t\tcase \"Infof\":\n\t\t\tlogger.Infof(\"hello, %s\", \"world\")\n\t\tcase \"Warn\":\n\t\t\tlogger.Warn(\"hello\")\n\t\tcase \"Warnf\":\n\t\t\tlogger.Warnf(\"hello, %s\", \"world\")\n\t\tcase \"Error\":\n\t\t\tlogger.Error(\"hello\")\n\t\tcase \"Errorf\":\n\t\t\tlogger.Errorf(\"hello, %s\", \"world\")\n\t\tdefault:\n\t\t\tt.Fatalf(\"unexpected op: %q\", tc.op)\n\t\t}\n\n\t\tif buf.String() == \"\" {\n\t\t\tt.Errorf(\"logger.%s did not output log message when level is set to %v\", tc.op, tc.level)\n\t\t}\n\t}\n}\n" }, { "path": "internal/proto/asynq.pb.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\n// Code generated by protoc-gen-go. DO NOT EDIT.\n// versions:\n// \tprotoc-gen-go v1.36.6\n// \tprotoc v5.29.3\n// source: asynq.proto\n\npackage proto\n\nimport (\n\tprotoreflect \"google.golang.org/protobuf/reflect/protoreflect\"\n\tprotoimpl \"google.golang.org/protobuf/runtime/protoimpl\"\n\ttimestamppb \"google.golang.org/protobuf/types/known/timestamppb\"\n\treflect \"reflect\"\n\tsync \"sync\"\n\tunsafe \"unsafe\"\n)\n\nconst (\n\t// Verify that this generated code is sufficiently up-to-date.\n\t_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)\n\t// Verify that runtime/protoimpl is sufficiently up-to-date.\n\t_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)\n)\n\n// TaskMessage is the internal representation of a task with additional\n// metadata fields.\n// Next ID: 16\ntype TaskMessage struct {\n\tstate protoimpl.MessageState `protogen:\"open.v1\"`\n\t// Type indicates the kind of the task to be performed.\n\tType string `protobuf:\"bytes,1,opt,name=type,proto3\" json:\"type,omitempty\"`\n\t// Payload holds data needed to process the task.\n\tPayload []byte `protobuf:\"bytes,2,opt,name=payload,proto3\" json:\"payload,omitempty\"`\n\t// Headers holds additional metadata for the task.\n\tHeaders map[string]string `protobuf:\"bytes,15,rep,name=headers,proto3\" json:\"headers,omitempty\" protobuf_key:\"bytes,1,opt,name=key\" protobuf_val:\"bytes,2,opt,name=value\"`\n\t// Unique identifier for the task.\n\tId string `protobuf:\"bytes,3,opt,name=id,proto3\" json:\"id,omitempty\"`\n\t// Name of the queue to which this task belongs.\n\tQueue string `protobuf:\"bytes,4,opt,name=queue,proto3\" json:\"queue,omitempty\"`\n\t// Max number of retries for this task.\n\tRetry int32 `protobuf:\"varint,5,opt,name=retry,proto3\" json:\"retry,omitempty\"`\n\t// Number of times this task has been retried so far.\n\tRetried int32 `protobuf:\"varint,6,opt,name=retried,proto3\" json:\"retried,omitempty\"`\n\t// Error message from the last failure.\n\tErrorMsg string `protobuf:\"bytes,7,opt,name=error_msg,json=errorMsg,proto3\" json:\"error_msg,omitempty\"`\n\t// Time of last failure in Unix time,\n\t// the number of seconds elapsed since January 1, 1970 UTC.\n\t// Use zero to indicate no last failure.\n\tLastFailedAt int64 `protobuf:\"varint,11,opt,name=last_failed_at,json=lastFailedAt,proto3\" json:\"last_failed_at,omitempty\"`\n\t// Timeout specifies timeout in seconds.\n\t// Use zero to indicate no timeout.\n\tTimeout int64 `protobuf:\"varint,8,opt,name=timeout,proto3\" json:\"timeout,omitempty\"`\n\t// Deadline specifies the deadline for the task in Unix time,\n\t// the number of seconds elapsed since January 1, 1970 UTC.\n\t// Use zero to indicate no deadline.\n\tDeadline int64 `protobuf:\"varint,9,opt,name=deadline,proto3\" json:\"deadline,omitempty\"`\n\t// UniqueKey holds the redis key used for uniqueness lock for this task.\n\t// Empty string indicates that no uniqueness lock was used.\n\tUniqueKey string `protobuf:\"bytes,10,opt,name=unique_key,json=uniqueKey,proto3\" json:\"unique_key,omitempty\"`\n\t// GroupKey is a name of the group used for task aggregation.\n\t// This field is optional and empty value means no aggregation for the task.\n\tGroupKey string `protobuf:\"bytes,14,opt,name=group_key,json=groupKey,proto3\" json:\"group_key,omitempty\"`\n\t// Retention period specified in a number of seconds.\n\t// The task will be stored in redis as a completed task until the TTL\n\t// expires.\n\tRetention int64 `protobuf:\"varint,12,opt,name=retention,proto3\" json:\"retention,omitempty\"`\n\t// Time when the task completed in success in Unix time,\n\t// the number of seconds elapsed since January 1, 1970 UTC.\n\t// This field is populated if result_ttl > 0 upon completion.\n\tCompletedAt int64 `protobuf:\"varint,13,opt,name=completed_at,json=completedAt,proto3\" json:\"completed_at,omitempty\"`\n\tunknownFields protoimpl.UnknownFields\n\tsizeCache protoimpl.SizeCache\n}\n\nfunc (x *TaskMessage) Reset() {\n\t*x = TaskMessage{}\n\tmi := &file_asynq_proto_msgTypes[0]\n\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\tms.StoreMessageInfo(mi)\n}\n\nfunc (x *TaskMessage) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*TaskMessage) ProtoMessage() {}\n\nfunc (x *TaskMessage) ProtoReflect() protoreflect.Message {\n\tmi := &file_asynq_proto_msgTypes[0]\n\tif x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use TaskMessage.ProtoReflect.Descriptor instead.\nfunc (*TaskMessage) Descriptor() ([]byte, []int) {\n\treturn file_asynq_proto_rawDescGZIP(), []int{0}\n}\n\nfunc (x *TaskMessage) GetType() string {\n\tif x != nil {\n\t\treturn x.Type\n\t}\n\treturn \"\"\n}\n\nfunc (x *TaskMessage) GetPayload() []byte {\n\tif x != nil {\n\t\treturn x.Payload\n\t}\n\treturn nil\n}\n\nfunc (x *TaskMessage) GetHeaders() map[string]string {\n\tif x != nil {\n\t\treturn x.Headers\n\t}\n\treturn nil\n}\n\nfunc (x *TaskMessage) GetId() string {\n\tif x != nil {\n\t\treturn x.Id\n\t}\n\treturn \"\"\n}\n\nfunc (x *TaskMessage) GetQueue() string {\n\tif x != nil {\n\t\treturn x.Queue\n\t}\n\treturn \"\"\n}\n\nfunc (x *TaskMessage) GetRetry() int32 {\n\tif x != nil {\n\t\treturn x.Retry\n\t}\n\treturn 0\n}\n\nfunc (x *TaskMessage) GetRetried() int32 {\n\tif x != nil {\n\t\treturn x.Retried\n\t}\n\treturn 0\n}\n\nfunc (x *TaskMessage) GetErrorMsg() string {\n\tif x != nil {\n\t\treturn x.ErrorMsg\n\t}\n\treturn \"\"\n}\n\nfunc (x *TaskMessage) GetLastFailedAt() int64 {\n\tif x != nil {\n\t\treturn x.LastFailedAt\n\t}\n\treturn 0\n}\n\nfunc (x *TaskMessage) GetTimeout() int64 {\n\tif x != nil {\n\t\treturn x.Timeout\n\t}\n\treturn 0\n}\n\nfunc (x *TaskMessage) GetDeadline() int64 {\n\tif x != nil {\n\t\treturn x.Deadline\n\t}\n\treturn 0\n}\n\nfunc (x *TaskMessage) GetUniqueKey() string {\n\tif x != nil {\n\t\treturn x.UniqueKey\n\t}\n\treturn \"\"\n}\n\nfunc (x *TaskMessage) GetGroupKey() string {\n\tif x != nil {\n\t\treturn x.GroupKey\n\t}\n\treturn \"\"\n}\n\nfunc (x *TaskMessage) GetRetention() int64 {\n\tif x != nil {\n\t\treturn x.Retention\n\t}\n\treturn 0\n}\n\nfunc (x *TaskMessage) GetCompletedAt() int64 {\n\tif x != nil {\n\t\treturn x.CompletedAt\n\t}\n\treturn 0\n}\n\n// ServerInfo holds information about a running server.\ntype ServerInfo struct {\n\tstate protoimpl.MessageState `protogen:\"open.v1\"`\n\t// Host machine the server is running on.\n\tHost string `protobuf:\"bytes,1,opt,name=host,proto3\" json:\"host,omitempty\"`\n\t// PID of the server process.\n\tPid int32 `protobuf:\"varint,2,opt,name=pid,proto3\" json:\"pid,omitempty\"`\n\t// Unique identifier for this server.\n\tServerId string `protobuf:\"bytes,3,opt,name=server_id,json=serverId,proto3\" json:\"server_id,omitempty\"`\n\t// Maximum number of concurrency this server will use.\n\tConcurrency int32 `protobuf:\"varint,4,opt,name=concurrency,proto3\" json:\"concurrency,omitempty\"`\n\t// List of queue names with their priorities.\n\t// The server will consume tasks from the queues and prioritize\n\t// queues with higher priority numbers.\n\tQueues map[string]int32 `protobuf:\"bytes,5,rep,name=queues,proto3\" json:\"queues,omitempty\" protobuf_key:\"bytes,1,opt,name=key\" protobuf_val:\"varint,2,opt,name=value\"`\n\t// If set, the server will always consume tasks from a queue with higher\n\t// priority.\n\tStrictPriority bool `protobuf:\"varint,6,opt,name=strict_priority,json=strictPriority,proto3\" json:\"strict_priority,omitempty\"`\n\t// Status indicates the status of the server.\n\tStatus string `protobuf:\"bytes,7,opt,name=status,proto3\" json:\"status,omitempty\"`\n\t// Time this server was started.\n\tStartTime *timestamppb.Timestamp `protobuf:\"bytes,8,opt,name=start_time,json=startTime,proto3\" json:\"start_time,omitempty\"`\n\t// Number of workers currently processing tasks.\n\tActiveWorkerCount int32 `protobuf:\"varint,9,opt,name=active_worker_count,json=activeWorkerCount,proto3\" json:\"active_worker_count,omitempty\"`\n\tunknownFields protoimpl.UnknownFields\n\tsizeCache protoimpl.SizeCache\n}\n\nfunc (x *ServerInfo) Reset() {\n\t*x = ServerInfo{}\n\tmi := &file_asynq_proto_msgTypes[1]\n\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\tms.StoreMessageInfo(mi)\n}\n\nfunc (x *ServerInfo) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*ServerInfo) ProtoMessage() {}\n\nfunc (x *ServerInfo) ProtoReflect() protoreflect.Message {\n\tmi := &file_asynq_proto_msgTypes[1]\n\tif x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use ServerInfo.ProtoReflect.Descriptor instead.\nfunc (*ServerInfo) Descriptor() ([]byte, []int) {\n\treturn file_asynq_proto_rawDescGZIP(), []int{1}\n}\n\nfunc (x *ServerInfo) GetHost() string {\n\tif x != nil {\n\t\treturn x.Host\n\t}\n\treturn \"\"\n}\n\nfunc (x *ServerInfo) GetPid() int32 {\n\tif x != nil {\n\t\treturn x.Pid\n\t}\n\treturn 0\n}\n\nfunc (x *ServerInfo) GetServerId() string {\n\tif x != nil {\n\t\treturn x.ServerId\n\t}\n\treturn \"\"\n}\n\nfunc (x *ServerInfo) GetConcurrency() int32 {\n\tif x != nil {\n\t\treturn x.Concurrency\n\t}\n\treturn 0\n}\n\nfunc (x *ServerInfo) GetQueues() map[string]int32 {\n\tif x != nil {\n\t\treturn x.Queues\n\t}\n\treturn nil\n}\n\nfunc (x *ServerInfo) GetStrictPriority() bool {\n\tif x != nil {\n\t\treturn x.StrictPriority\n\t}\n\treturn false\n}\n\nfunc (x *ServerInfo) GetStatus() string {\n\tif x != nil {\n\t\treturn x.Status\n\t}\n\treturn \"\"\n}\n\nfunc (x *ServerInfo) GetStartTime() *timestamppb.Timestamp {\n\tif x != nil {\n\t\treturn x.StartTime\n\t}\n\treturn nil\n}\n\nfunc (x *ServerInfo) GetActiveWorkerCount() int32 {\n\tif x != nil {\n\t\treturn x.ActiveWorkerCount\n\t}\n\treturn 0\n}\n\n// WorkerInfo holds information about a running worker.\ntype WorkerInfo struct {\n\tstate protoimpl.MessageState `protogen:\"open.v1\"`\n\t// Host matchine this worker is running on.\n\tHost string `protobuf:\"bytes,1,opt,name=host,proto3\" json:\"host,omitempty\"`\n\t// PID of the process in which this worker is running.\n\tPid int32 `protobuf:\"varint,2,opt,name=pid,proto3\" json:\"pid,omitempty\"`\n\t// ID of the server in which this worker is running.\n\tServerId string `protobuf:\"bytes,3,opt,name=server_id,json=serverId,proto3\" json:\"server_id,omitempty\"`\n\t// ID of the task this worker is processing.\n\tTaskId string `protobuf:\"bytes,4,opt,name=task_id,json=taskId,proto3\" json:\"task_id,omitempty\"`\n\t// Type of the task this worker is processing.\n\tTaskType string `protobuf:\"bytes,5,opt,name=task_type,json=taskType,proto3\" json:\"task_type,omitempty\"`\n\t// Payload of the task this worker is processing.\n\tTaskPayload []byte `protobuf:\"bytes,6,opt,name=task_payload,json=taskPayload,proto3\" json:\"task_payload,omitempty\"`\n\t// Name of the queue the task the worker is processing belongs.\n\tQueue string `protobuf:\"bytes,7,opt,name=queue,proto3\" json:\"queue,omitempty\"`\n\t// Time this worker started processing the task.\n\tStartTime *timestamppb.Timestamp `protobuf:\"bytes,8,opt,name=start_time,json=startTime,proto3\" json:\"start_time,omitempty\"`\n\t// Deadline by which the worker needs to complete processing\n\t// the task. If worker exceeds the deadline, the task will fail.\n\tDeadline *timestamppb.Timestamp `protobuf:\"bytes,9,opt,name=deadline,proto3\" json:\"deadline,omitempty\"`\n\tunknownFields protoimpl.UnknownFields\n\tsizeCache protoimpl.SizeCache\n}\n\nfunc (x *WorkerInfo) Reset() {\n\t*x = WorkerInfo{}\n\tmi := &file_asynq_proto_msgTypes[2]\n\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\tms.StoreMessageInfo(mi)\n}\n\nfunc (x *WorkerInfo) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*WorkerInfo) ProtoMessage() {}\n\nfunc (x *WorkerInfo) ProtoReflect() protoreflect.Message {\n\tmi := &file_asynq_proto_msgTypes[2]\n\tif x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use WorkerInfo.ProtoReflect.Descriptor instead.\nfunc (*WorkerInfo) Descriptor() ([]byte, []int) {\n\treturn file_asynq_proto_rawDescGZIP(), []int{2}\n}\n\nfunc (x *WorkerInfo) GetHost() string {\n\tif x != nil {\n\t\treturn x.Host\n\t}\n\treturn \"\"\n}\n\nfunc (x *WorkerInfo) GetPid() int32 {\n\tif x != nil {\n\t\treturn x.Pid\n\t}\n\treturn 0\n}\n\nfunc (x *WorkerInfo) GetServerId() string {\n\tif x != nil {\n\t\treturn x.ServerId\n\t}\n\treturn \"\"\n}\n\nfunc (x *WorkerInfo) GetTaskId() string {\n\tif x != nil {\n\t\treturn x.TaskId\n\t}\n\treturn \"\"\n}\n\nfunc (x *WorkerInfo) GetTaskType() string {\n\tif x != nil {\n\t\treturn x.TaskType\n\t}\n\treturn \"\"\n}\n\nfunc (x *WorkerInfo) GetTaskPayload() []byte {\n\tif x != nil {\n\t\treturn x.TaskPayload\n\t}\n\treturn nil\n}\n\nfunc (x *WorkerInfo) GetQueue() string {\n\tif x != nil {\n\t\treturn x.Queue\n\t}\n\treturn \"\"\n}\n\nfunc (x *WorkerInfo) GetStartTime() *timestamppb.Timestamp {\n\tif x != nil {\n\t\treturn x.StartTime\n\t}\n\treturn nil\n}\n\nfunc (x *WorkerInfo) GetDeadline() *timestamppb.Timestamp {\n\tif x != nil {\n\t\treturn x.Deadline\n\t}\n\treturn nil\n}\n\n// SchedulerEntry holds information about a periodic task registered\n// with a scheduler.\ntype SchedulerEntry struct {\n\tstate protoimpl.MessageState `protogen:\"open.v1\"`\n\t// Identifier of the scheduler entry.\n\tId string `protobuf:\"bytes,1,opt,name=id,proto3\" json:\"id,omitempty\"`\n\t// Periodic schedule spec of the entry.\n\tSpec string `protobuf:\"bytes,2,opt,name=spec,proto3\" json:\"spec,omitempty\"`\n\t// Task type of the periodic task.\n\tTaskType string `protobuf:\"bytes,3,opt,name=task_type,json=taskType,proto3\" json:\"task_type,omitempty\"`\n\t// Task payload of the periodic task.\n\tTaskPayload []byte `protobuf:\"bytes,4,opt,name=task_payload,json=taskPayload,proto3\" json:\"task_payload,omitempty\"`\n\t// Options used to enqueue the periodic task.\n\tEnqueueOptions []string `protobuf:\"bytes,5,rep,name=enqueue_options,json=enqueueOptions,proto3\" json:\"enqueue_options,omitempty\"`\n\t// Next time the task will be enqueued.\n\tNextEnqueueTime *timestamppb.Timestamp `protobuf:\"bytes,6,opt,name=next_enqueue_time,json=nextEnqueueTime,proto3\" json:\"next_enqueue_time,omitempty\"`\n\t// Last time the task was enqueued.\n\t// Zero time if task was never enqueued.\n\tPrevEnqueueTime *timestamppb.Timestamp `protobuf:\"bytes,7,opt,name=prev_enqueue_time,json=prevEnqueueTime,proto3\" json:\"prev_enqueue_time,omitempty\"`\n\tunknownFields protoimpl.UnknownFields\n\tsizeCache protoimpl.SizeCache\n}\n\nfunc (x *SchedulerEntry) Reset() {\n\t*x = SchedulerEntry{}\n\tmi := &file_asynq_proto_msgTypes[3]\n\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\tms.StoreMessageInfo(mi)\n}\n\nfunc (x *SchedulerEntry) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*SchedulerEntry) ProtoMessage() {}\n\nfunc (x *SchedulerEntry) ProtoReflect() protoreflect.Message {\n\tmi := &file_asynq_proto_msgTypes[3]\n\tif x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use SchedulerEntry.ProtoReflect.Descriptor instead.\nfunc (*SchedulerEntry) Descriptor() ([]byte, []int) {\n\treturn file_asynq_proto_rawDescGZIP(), []int{3}\n}\n\nfunc (x *SchedulerEntry) GetId() string {\n\tif x != nil {\n\t\treturn x.Id\n\t}\n\treturn \"\"\n}\n\nfunc (x *SchedulerEntry) GetSpec() string {\n\tif x != nil {\n\t\treturn x.Spec\n\t}\n\treturn \"\"\n}\n\nfunc (x *SchedulerEntry) GetTaskType() string {\n\tif x != nil {\n\t\treturn x.TaskType\n\t}\n\treturn \"\"\n}\n\nfunc (x *SchedulerEntry) GetTaskPayload() []byte {\n\tif x != nil {\n\t\treturn x.TaskPayload\n\t}\n\treturn nil\n}\n\nfunc (x *SchedulerEntry) GetEnqueueOptions() []string {\n\tif x != nil {\n\t\treturn x.EnqueueOptions\n\t}\n\treturn nil\n}\n\nfunc (x *SchedulerEntry) GetNextEnqueueTime() *timestamppb.Timestamp {\n\tif x != nil {\n\t\treturn x.NextEnqueueTime\n\t}\n\treturn nil\n}\n\nfunc (x *SchedulerEntry) GetPrevEnqueueTime() *timestamppb.Timestamp {\n\tif x != nil {\n\t\treturn x.PrevEnqueueTime\n\t}\n\treturn nil\n}\n\n// SchedulerEnqueueEvent holds information about an enqueue event\n// by a scheduler.\ntype SchedulerEnqueueEvent struct {\n\tstate protoimpl.MessageState `protogen:\"open.v1\"`\n\t// ID of the task that was enqueued.\n\tTaskId string `protobuf:\"bytes,1,opt,name=task_id,json=taskId,proto3\" json:\"task_id,omitempty\"`\n\t// Time the task was enqueued.\n\tEnqueueTime *timestamppb.Timestamp `protobuf:\"bytes,2,opt,name=enqueue_time,json=enqueueTime,proto3\" json:\"enqueue_time,omitempty\"`\n\tunknownFields protoimpl.UnknownFields\n\tsizeCache protoimpl.SizeCache\n}\n\nfunc (x *SchedulerEnqueueEvent) Reset() {\n\t*x = SchedulerEnqueueEvent{}\n\tmi := &file_asynq_proto_msgTypes[4]\n\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\tms.StoreMessageInfo(mi)\n}\n\nfunc (x *SchedulerEnqueueEvent) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*SchedulerEnqueueEvent) ProtoMessage() {}\n\nfunc (x *SchedulerEnqueueEvent) ProtoReflect() protoreflect.Message {\n\tmi := &file_asynq_proto_msgTypes[4]\n\tif x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use SchedulerEnqueueEvent.ProtoReflect.Descriptor instead.\nfunc (*SchedulerEnqueueEvent) Descriptor() ([]byte, []int) {\n\treturn file_asynq_proto_rawDescGZIP(), []int{4}\n}\n\nfunc (x *SchedulerEnqueueEvent) GetTaskId() string {\n\tif x != nil {\n\t\treturn x.TaskId\n\t}\n\treturn \"\"\n}\n\nfunc (x *SchedulerEnqueueEvent) GetEnqueueTime() *timestamppb.Timestamp {\n\tif x != nil {\n\t\treturn x.EnqueueTime\n\t}\n\treturn nil\n}\n\nvar File_asynq_proto protoreflect.FileDescriptor\n\nconst file_asynq_proto_rawDesc = \"\" +\n\t\"\\n\" +\n\t\"\\vasynq.proto\\x12\\x05asynq\\x1a\\x1fgoogle/protobuf/timestamp.proto\\\"\\xfe\\x03\\n\" +\n\t\"\\vTaskMessage\\x12\\x12\\n\" +\n\t\"\\x04type\\x18\\x01 \\x01(\\tR\\x04type\\x12\\x18\\n\" +\n\t\"\\apayload\\x18\\x02 \\x01(\\fR\\apayload\\x129\\n\" +\n\t\"\\aheaders\\x18\\x0f \\x03(\\v2\\x1f.asynq.TaskMessage.HeadersEntryR\\aheaders\\x12\\x0e\\n\" +\n\t\"\\x02id\\x18\\x03 \\x01(\\tR\\x02id\\x12\\x14\\n\" +\n\t\"\\x05queue\\x18\\x04 \\x01(\\tR\\x05queue\\x12\\x14\\n\" +\n\t\"\\x05retry\\x18\\x05 \\x01(\\x05R\\x05retry\\x12\\x18\\n\" +\n\t\"\\aretried\\x18\\x06 \\x01(\\x05R\\aretried\\x12\\x1b\\n\" +\n\t\"\\terror_msg\\x18\\a \\x01(\\tR\\berrorMsg\\x12$\\n\" +\n\t\"\\x0elast_failed_at\\x18\\v \\x01(\\x03R\\flastFailedAt\\x12\\x18\\n\" +\n\t\"\\atimeout\\x18\\b \\x01(\\x03R\\atimeout\\x12\\x1a\\n\" +\n\t\"\\bdeadline\\x18\\t \\x01(\\x03R\\bdeadline\\x12\\x1d\\n\" +\n\t\"\\n\" +\n\t\"unique_key\\x18\\n\" +\n\t\" \\x01(\\tR\\tuniqueKey\\x12\\x1b\\n\" +\n\t\"\\tgroup_key\\x18\\x0e \\x01(\\tR\\bgroupKey\\x12\\x1c\\n\" +\n\t\"\\tretention\\x18\\f \\x01(\\x03R\\tretention\\x12!\\n\" +\n\t\"\\fcompleted_at\\x18\\r \\x01(\\x03R\\vcompletedAt\\x1a:\\n\" +\n\t\"\\fHeadersEntry\\x12\\x10\\n\" +\n\t\"\\x03key\\x18\\x01 \\x01(\\tR\\x03key\\x12\\x14\\n\" +\n\t\"\\x05value\\x18\\x02 \\x01(\\tR\\x05value:\\x028\\x01\\\"\\x8f\\x03\\n\" +\n\t\"\\n\" +\n\t\"ServerInfo\\x12\\x12\\n\" +\n\t\"\\x04host\\x18\\x01 \\x01(\\tR\\x04host\\x12\\x10\\n\" +\n\t\"\\x03pid\\x18\\x02 \\x01(\\x05R\\x03pid\\x12\\x1b\\n\" +\n\t\"\\tserver_id\\x18\\x03 \\x01(\\tR\\bserverId\\x12 \\n\" +\n\t\"\\vconcurrency\\x18\\x04 \\x01(\\x05R\\vconcurrency\\x125\\n\" +\n\t\"\\x06queues\\x18\\x05 \\x03(\\v2\\x1d.asynq.ServerInfo.QueuesEntryR\\x06queues\\x12'\\n\" +\n\t\"\\x0fstrict_priority\\x18\\x06 \\x01(\\bR\\x0estrictPriority\\x12\\x16\\n\" +\n\t\"\\x06status\\x18\\a \\x01(\\tR\\x06status\\x129\\n\" +\n\t\"\\n\" +\n\t\"start_time\\x18\\b \\x01(\\v2\\x1a.google.protobuf.TimestampR\\tstartTime\\x12.\\n\" +\n\t\"\\x13active_worker_count\\x18\\t \\x01(\\x05R\\x11activeWorkerCount\\x1a9\\n\" +\n\t\"\\vQueuesEntry\\x12\\x10\\n\" +\n\t\"\\x03key\\x18\\x01 \\x01(\\tR\\x03key\\x12\\x14\\n\" +\n\t\"\\x05value\\x18\\x02 \\x01(\\x05R\\x05value:\\x028\\x01\\\"\\xb1\\x02\\n\" +\n\t\"\\n\" +\n\t\"WorkerInfo\\x12\\x12\\n\" +\n\t\"\\x04host\\x18\\x01 \\x01(\\tR\\x04host\\x12\\x10\\n\" +\n\t\"\\x03pid\\x18\\x02 \\x01(\\x05R\\x03pid\\x12\\x1b\\n\" +\n\t\"\\tserver_id\\x18\\x03 \\x01(\\tR\\bserverId\\x12\\x17\\n\" +\n\t\"\\atask_id\\x18\\x04 \\x01(\\tR\\x06taskId\\x12\\x1b\\n\" +\n\t\"\\ttask_type\\x18\\x05 \\x01(\\tR\\btaskType\\x12!\\n\" +\n\t\"\\ftask_payload\\x18\\x06 \\x01(\\fR\\vtaskPayload\\x12\\x14\\n\" +\n\t\"\\x05queue\\x18\\a \\x01(\\tR\\x05queue\\x129\\n\" +\n\t\"\\n\" +\n\t\"start_time\\x18\\b \\x01(\\v2\\x1a.google.protobuf.TimestampR\\tstartTime\\x126\\n\" +\n\t\"\\bdeadline\\x18\\t \\x01(\\v2\\x1a.google.protobuf.TimestampR\\bdeadline\\\"\\xad\\x02\\n\" +\n\t\"\\x0eSchedulerEntry\\x12\\x0e\\n\" +\n\t\"\\x02id\\x18\\x01 \\x01(\\tR\\x02id\\x12\\x12\\n\" +\n\t\"\\x04spec\\x18\\x02 \\x01(\\tR\\x04spec\\x12\\x1b\\n\" +\n\t\"\\ttask_type\\x18\\x03 \\x01(\\tR\\btaskType\\x12!\\n\" +\n\t\"\\ftask_payload\\x18\\x04 \\x01(\\fR\\vtaskPayload\\x12'\\n\" +\n\t\"\\x0fenqueue_options\\x18\\x05 \\x03(\\tR\\x0eenqueueOptions\\x12F\\n\" +\n\t\"\\x11next_enqueue_time\\x18\\x06 \\x01(\\v2\\x1a.google.protobuf.TimestampR\\x0fnextEnqueueTime\\x12F\\n\" +\n\t\"\\x11prev_enqueue_time\\x18\\a \\x01(\\v2\\x1a.google.protobuf.TimestampR\\x0fprevEnqueueTime\\\"o\\n\" +\n\t\"\\x15SchedulerEnqueueEvent\\x12\\x17\\n\" +\n\t\"\\atask_id\\x18\\x01 \\x01(\\tR\\x06taskId\\x12=\\n\" +\n\t\"\\fenqueue_time\\x18\\x02 \\x01(\\v2\\x1a.google.protobuf.TimestampR\\venqueueTimeB)Z'github.com/hibiken/asynq/internal/protob\\x06proto3\"\n\nvar (\n\tfile_asynq_proto_rawDescOnce sync.Once\n\tfile_asynq_proto_rawDescData []byte\n)\n\nfunc file_asynq_proto_rawDescGZIP() []byte {\n\tfile_asynq_proto_rawDescOnce.Do(func() {\n\t\tfile_asynq_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_asynq_proto_rawDesc), len(file_asynq_proto_rawDesc)))\n\t})\n\treturn file_asynq_proto_rawDescData\n}\n\nvar file_asynq_proto_msgTypes = make([]protoimpl.MessageInfo, 7)\nvar file_asynq_proto_goTypes = []any{\n\t(*TaskMessage)(nil), // 0: asynq.TaskMessage\n\t(*ServerInfo)(nil), // 1: asynq.ServerInfo\n\t(*WorkerInfo)(nil), // 2: asynq.WorkerInfo\n\t(*SchedulerEntry)(nil), // 3: asynq.SchedulerEntry\n\t(*SchedulerEnqueueEvent)(nil), // 4: asynq.SchedulerEnqueueEvent\n\tnil, // 5: asynq.TaskMessage.HeadersEntry\n\tnil, // 6: asynq.ServerInfo.QueuesEntry\n\t(*timestamppb.Timestamp)(nil), // 7: google.protobuf.Timestamp\n}\nvar file_asynq_proto_depIdxs = []int32{\n\t5, // 0: asynq.TaskMessage.headers:type_name -> asynq.TaskMessage.HeadersEntry\n\t6, // 1: asynq.ServerInfo.queues:type_name -> asynq.ServerInfo.QueuesEntry\n\t7, // 2: asynq.ServerInfo.start_time:type_name -> google.protobuf.Timestamp\n\t7, // 3: asynq.WorkerInfo.start_time:type_name -> google.protobuf.Timestamp\n\t7, // 4: asynq.WorkerInfo.deadline:type_name -> google.protobuf.Timestamp\n\t7, // 5: asynq.SchedulerEntry.next_enqueue_time:type_name -> google.protobuf.Timestamp\n\t7, // 6: asynq.SchedulerEntry.prev_enqueue_time:type_name -> google.protobuf.Timestamp\n\t7, // 7: asynq.SchedulerEnqueueEvent.enqueue_time:type_name -> google.protobuf.Timestamp\n\t8, // [8:8] is the sub-list for method output_type\n\t8, // [8:8] is the sub-list for method input_type\n\t8, // [8:8] is the sub-list for extension type_name\n\t8, // [8:8] is the sub-list for extension extendee\n\t0, // [0:8] is the sub-list for field type_name\n}\n\nfunc init() { file_asynq_proto_init() }\nfunc file_asynq_proto_init() {\n\tif File_asynq_proto != nil {\n\t\treturn\n\t}\n\ttype x struct{}\n\tout := protoimpl.TypeBuilder{\n\t\tFile: protoimpl.DescBuilder{\n\t\t\tGoPackagePath: reflect.TypeOf(x{}).PkgPath(),\n\t\t\tRawDescriptor: unsafe.Slice(unsafe.StringData(file_asynq_proto_rawDesc), len(file_asynq_proto_rawDesc)),\n\t\t\tNumEnums: 0,\n\t\t\tNumMessages: 7,\n\t\t\tNumExtensions: 0,\n\t\t\tNumServices: 0,\n\t\t},\n\t\tGoTypes: file_asynq_proto_goTypes,\n\t\tDependencyIndexes: file_asynq_proto_depIdxs,\n\t\tMessageInfos: file_asynq_proto_msgTypes,\n\t}.Build()\n\tFile_asynq_proto = out.File\n\tfile_asynq_proto_goTypes = nil\n\tfile_asynq_proto_depIdxs = nil\n}\n" }, { "path": "internal/proto/asynq.proto", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\nsyntax = \"proto3\";\npackage asynq;\n\nimport \"google/protobuf/timestamp.proto\";\n\noption go_package = \"github.com/hibiken/asynq/internal/proto\";\n\n// TaskMessage is the internal representation of a task with additional\n// metadata fields.\n// Next ID: 16\nmessage TaskMessage {\n\t// Type indicates the kind of the task to be performed.\n string type = 1;\n\n\t// Payload holds data needed to process the task.\n bytes payload = 2;\n\n\t// Headers holds additional metadata for the task.\n map headers = 15;\n\n\t// Unique identifier for the task.\n string id = 3;\n\n\t// Name of the queue to which this task belongs.\n string queue = 4;\n\n\t// Max number of retries for this task.\n int32 retry = 5;\n\n\t// Number of times this task has been retried so far.\n int32 retried = 6;\n\n\t// Error message from the last failure.\n string error_msg = 7;\n\n // Time of last failure in Unix time,\n // the number of seconds elapsed since January 1, 1970 UTC.\n // Use zero to indicate no last failure.\n int64 last_failed_at = 11;\n\n\t// Timeout specifies timeout in seconds.\n\t// Use zero to indicate no timeout.\n int64 timeout = 8;\n\n\t// Deadline specifies the deadline for the task in Unix time,\n\t// the number of seconds elapsed since January 1, 1970 UTC.\n\t// Use zero to indicate no deadline.\n int64 deadline = 9;\n\n\t// UniqueKey holds the redis key used for uniqueness lock for this task.\n\t// Empty string indicates that no uniqueness lock was used.\n string unique_key = 10;\n\n // GroupKey is a name of the group used for task aggregation.\n // This field is optional and empty value means no aggregation for the task.\n string group_key = 14;\n\n // Retention period specified in a number of seconds.\n // The task will be stored in redis as a completed task until the TTL\n // expires.\n int64 retention = 12;\n\n // Time when the task completed in success in Unix time,\n // the number of seconds elapsed since January 1, 1970 UTC.\n // This field is populated if result_ttl > 0 upon completion.\n int64 completed_at = 13;\n};\n\n// ServerInfo holds information about a running server.\nmessage ServerInfo {\n // Host machine the server is running on.\n string host = 1;\n\n // PID of the server process.\n int32 pid = 2;\n\n // Unique identifier for this server.\n string server_id = 3;\n\n // Maximum number of concurrency this server will use.\n int32 concurrency = 4;\n\n // List of queue names with their priorities.\n // The server will consume tasks from the queues and prioritize\n // queues with higher priority numbers.\n map queues = 5;\n\n // If set, the server will always consume tasks from a queue with higher\n // priority.\n bool strict_priority = 6;\n\n // Status indicates the status of the server.\n string status = 7;\n\n // Time this server was started.\n google.protobuf.Timestamp start_time = 8;\n\n // Number of workers currently processing tasks.\n int32 active_worker_count = 9;\n};\n\n// WorkerInfo holds information about a running worker.\nmessage WorkerInfo {\n // Host matchine this worker is running on.\n string host = 1;\n\n // PID of the process in which this worker is running.\n int32 pid = 2;\n\n // ID of the server in which this worker is running.\n string server_id = 3;\n\n // ID of the task this worker is processing.\n string task_id = 4;\n\n // Type of the task this worker is processing.\n string task_type = 5;\n\n // Payload of the task this worker is processing.\n bytes task_payload = 6;\n\n // Name of the queue the task the worker is processing belongs.\n string queue = 7;\n\n // Time this worker started processing the task.\n google.protobuf.Timestamp start_time = 8;\n\n // Deadline by which the worker needs to complete processing \n // the task. If worker exceeds the deadline, the task will fail.\n google.protobuf.Timestamp deadline = 9;\n};\n\n// SchedulerEntry holds information about a periodic task registered \n// with a scheduler.\nmessage SchedulerEntry {\n\t// Identifier of the scheduler entry.\n\tstring id = 1;\n\n\t// Periodic schedule spec of the entry.\n\tstring spec = 2;\n\n\t// Task type of the periodic task.\n\tstring task_type = 3;\n\n\t// Task payload of the periodic task.\n\tbytes task_payload = 4;\n\n\t// Options used to enqueue the periodic task.\n\trepeated string enqueue_options = 5;\n\n\t// Next time the task will be enqueued.\n google.protobuf.Timestamp next_enqueue_time = 6;\n\n\t// Last time the task was enqueued.\n\t// Zero time if task was never enqueued.\n google.protobuf.Timestamp prev_enqueue_time = 7;\n};\n\n// SchedulerEnqueueEvent holds information about an enqueue event\n// by a scheduler.\nmessage SchedulerEnqueueEvent {\n\t// ID of the task that was enqueued.\n string task_id = 1;\n\n\t// Time the task was enqueued.\n google.protobuf.Timestamp enqueue_time = 2;\n};\n" }, { "path": "internal/rdb/benchmark_test.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage rdb\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/testutil\"\n)\n\nfunc BenchmarkEnqueue(b *testing.B) {\n\tr := setup(b)\n\tctx := context.Background()\n\tmsg := testutil.NewTaskMessage(\"task1\", nil)\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tb.StopTimer()\n\t\ttestutil.FlushDB(b, r.client)\n\t\tb.StartTimer()\n\n\t\tif err := r.Enqueue(ctx, msg); err != nil {\n\t\t\tb.Fatalf(\"Enqueue failed: %v\", err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkEnqueueUnique(b *testing.B) {\n\tr := setup(b)\n\tctx := context.Background()\n\tmsg := &base.TaskMessage{\n\t\tType: \"task1\",\n\t\tPayload: nil,\n\t\tQueue: base.DefaultQueueName,\n\t\tUniqueKey: base.UniqueKey(\"default\", \"task1\", nil),\n\t}\n\tuniqueTTL := 5 * time.Minute\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tb.StopTimer()\n\t\ttestutil.FlushDB(b, r.client)\n\t\tb.StartTimer()\n\n\t\tif err := r.EnqueueUnique(ctx, msg, uniqueTTL); err != nil {\n\t\t\tb.Fatalf(\"EnqueueUnique failed: %v\", err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkSchedule(b *testing.B) {\n\tr := setup(b)\n\tctx := context.Background()\n\tmsg := testutil.NewTaskMessage(\"task1\", nil)\n\tprocessAt := time.Now().Add(3 * time.Minute)\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tb.StopTimer()\n\t\ttestutil.FlushDB(b, r.client)\n\t\tb.StartTimer()\n\n\t\tif err := r.Schedule(ctx, msg, processAt); err != nil {\n\t\t\tb.Fatalf(\"Schedule failed: %v\", err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkScheduleUnique(b *testing.B) {\n\tr := setup(b)\n\tctx := context.Background()\n\tmsg := &base.TaskMessage{\n\t\tType: \"task1\",\n\t\tPayload: nil,\n\t\tQueue: base.DefaultQueueName,\n\t\tUniqueKey: base.UniqueKey(\"default\", \"task1\", nil),\n\t}\n\tprocessAt := time.Now().Add(3 * time.Minute)\n\tuniqueTTL := 5 * time.Minute\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tb.StopTimer()\n\t\ttestutil.FlushDB(b, r.client)\n\t\tb.StartTimer()\n\n\t\tif err := r.ScheduleUnique(ctx, msg, processAt, uniqueTTL); err != nil {\n\t\t\tb.Fatalf(\"EnqueueUnique failed: %v\", err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkDequeueSingleQueue(b *testing.B) {\n\tr := setup(b)\n\tctx := context.Background()\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tb.StopTimer()\n\t\ttestutil.FlushDB(b, r.client)\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tm := testutil.NewTaskMessageWithQueue(\n\t\t\t\tfmt.Sprintf(\"task%d\", i), nil, base.DefaultQueueName)\n\t\t\tif err := r.Enqueue(ctx, m); err != nil {\n\t\t\t\tb.Fatalf(\"Enqueue failed: %v\", err)\n\t\t\t}\n\t\t}\n\t\tb.StartTimer()\n\n\t\tif _, _, err := r.Dequeue(base.DefaultQueueName); err != nil {\n\t\t\tb.Fatalf(\"Dequeue failed: %v\", err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkDequeueMultipleQueues(b *testing.B) {\n\tqnames := []string{\"critical\", \"default\", \"low\"}\n\tr := setup(b)\n\tctx := context.Background()\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tb.StopTimer()\n\t\ttestutil.FlushDB(b, r.client)\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tfor _, qname := range qnames {\n\t\t\t\tm := testutil.NewTaskMessageWithQueue(\n\t\t\t\t\tfmt.Sprintf(\"%s_task%d\", qname, i), nil, qname)\n\t\t\t\tif err := r.Enqueue(ctx, m); err != nil {\n\t\t\t\t\tb.Fatalf(\"Enqueue failed: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tb.StartTimer()\n\n\t\tif _, _, err := r.Dequeue(qnames...); err != nil {\n\t\t\tb.Fatalf(\"Dequeue failed: %v\", err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkDone(b *testing.B) {\n\tr := setup(b)\n\tm1 := testutil.NewTaskMessage(\"task1\", nil)\n\tm2 := testutil.NewTaskMessage(\"task2\", nil)\n\tm3 := testutil.NewTaskMessage(\"task3\", nil)\n\tmsgs := []*base.TaskMessage{m1, m2, m3}\n\tzs := []base.Z{\n\t\t{Message: m1, Score: time.Now().Add(10 * time.Second).Unix()},\n\t\t{Message: m2, Score: time.Now().Add(20 * time.Second).Unix()},\n\t\t{Message: m3, Score: time.Now().Add(30 * time.Second).Unix()},\n\t}\n\tctx := context.Background()\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tb.StopTimer()\n\t\ttestutil.FlushDB(b, r.client)\n\t\ttestutil.SeedActiveQueue(b, r.client, msgs, base.DefaultQueueName)\n\t\ttestutil.SeedLease(b, r.client, zs, base.DefaultQueueName)\n\t\tb.StartTimer()\n\n\t\tif err := r.Done(ctx, msgs[0]); err != nil {\n\t\t\tb.Fatalf(\"Done failed: %v\", err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkRetry(b *testing.B) {\n\tr := setup(b)\n\tm1 := testutil.NewTaskMessage(\"task1\", nil)\n\tm2 := testutil.NewTaskMessage(\"task2\", nil)\n\tm3 := testutil.NewTaskMessage(\"task3\", nil)\n\tmsgs := []*base.TaskMessage{m1, m2, m3}\n\tzs := []base.Z{\n\t\t{Message: m1, Score: time.Now().Add(10 * time.Second).Unix()},\n\t\t{Message: m2, Score: time.Now().Add(20 * time.Second).Unix()},\n\t\t{Message: m3, Score: time.Now().Add(30 * time.Second).Unix()},\n\t}\n\tctx := context.Background()\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tb.StopTimer()\n\t\ttestutil.FlushDB(b, r.client)\n\t\ttestutil.SeedActiveQueue(b, r.client, msgs, base.DefaultQueueName)\n\t\ttestutil.SeedLease(b, r.client, zs, base.DefaultQueueName)\n\t\tb.StartTimer()\n\n\t\tif err := r.Retry(ctx, msgs[0], time.Now().Add(1*time.Minute), \"error\", true /*isFailure*/); err != nil {\n\t\t\tb.Fatalf(\"Retry failed: %v\", err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkArchive(b *testing.B) {\n\tr := setup(b)\n\tm1 := testutil.NewTaskMessage(\"task1\", nil)\n\tm2 := testutil.NewTaskMessage(\"task2\", nil)\n\tm3 := testutil.NewTaskMessage(\"task3\", nil)\n\tmsgs := []*base.TaskMessage{m1, m2, m3}\n\tzs := []base.Z{\n\t\t{Message: m1, Score: time.Now().Add(10 * time.Second).Unix()},\n\t\t{Message: m2, Score: time.Now().Add(20 * time.Second).Unix()},\n\t\t{Message: m3, Score: time.Now().Add(30 * time.Second).Unix()},\n\t}\n\tctx := context.Background()\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tb.StopTimer()\n\t\ttestutil.FlushDB(b, r.client)\n\t\ttestutil.SeedActiveQueue(b, r.client, msgs, base.DefaultQueueName)\n\t\ttestutil.SeedLease(b, r.client, zs, base.DefaultQueueName)\n\t\tb.StartTimer()\n\n\t\tif err := r.Archive(ctx, msgs[0], \"error\"); err != nil {\n\t\t\tb.Fatalf(\"Archive failed: %v\", err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkRequeue(b *testing.B) {\n\tr := setup(b)\n\tm1 := testutil.NewTaskMessage(\"task1\", nil)\n\tm2 := testutil.NewTaskMessage(\"task2\", nil)\n\tm3 := testutil.NewTaskMessage(\"task3\", nil)\n\tmsgs := []*base.TaskMessage{m1, m2, m3}\n\tzs := []base.Z{\n\t\t{Message: m1, Score: time.Now().Add(10 * time.Second).Unix()},\n\t\t{Message: m2, Score: time.Now().Add(20 * time.Second).Unix()},\n\t\t{Message: m3, Score: time.Now().Add(30 * time.Second).Unix()},\n\t}\n\tctx := context.Background()\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tb.StopTimer()\n\t\ttestutil.FlushDB(b, r.client)\n\t\ttestutil.SeedActiveQueue(b, r.client, msgs, base.DefaultQueueName)\n\t\ttestutil.SeedLease(b, r.client, zs, base.DefaultQueueName)\n\t\tb.StartTimer()\n\n\t\tif err := r.Requeue(ctx, msgs[0]); err != nil {\n\t\t\tb.Fatalf(\"Requeue failed: %v\", err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkCheckAndEnqueue(b *testing.B) {\n\tr := setup(b)\n\tnow := time.Now()\n\tvar zs []base.Z\n\tfor i := -100; i < 100; i++ {\n\t\tmsg := testutil.NewTaskMessage(fmt.Sprintf(\"task%d\", i), nil)\n\t\tscore := now.Add(time.Duration(i) * time.Second).Unix()\n\t\tzs = append(zs, base.Z{Message: msg, Score: score})\n\t}\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tb.StopTimer()\n\t\ttestutil.FlushDB(b, r.client)\n\t\ttestutil.SeedScheduledQueue(b, r.client, zs, base.DefaultQueueName)\n\t\tb.StartTimer()\n\n\t\tif err := r.ForwardIfReady(base.DefaultQueueName); err != nil {\n\t\t\tb.Fatalf(\"ForwardIfReady failed: %v\", err)\n\t\t}\n\t}\n}\n" }, { "path": "internal/rdb/inspect.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage rdb\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/errors\"\n\t\"github.com/redis/go-redis/v9\"\n\t\"github.com/spf13/cast\"\n)\n\n// AllQueues returns a list of all queue names.\nfunc (r *RDB) AllQueues() ([]string, error) {\n\treturn r.client.SMembers(context.Background(), base.AllQueues).Result()\n}\n\n// Stats represents a state of queues at a certain time.\ntype Stats struct {\n\t// Name of the queue (e.g. \"default\", \"critical\").\n\tQueue string\n\t// MemoryUsage is the total number of bytes the queue and its tasks require\n\t// to be stored in redis. It is an approximate memory usage value in bytes\n\t// since the value is computed by sampling.\n\tMemoryUsage int64\n\t// Paused indicates whether the queue is paused.\n\t// If true, tasks in the queue should not be processed.\n\tPaused bool\n\t// Size is the total number of tasks in the queue.\n\tSize int\n\n\t// Groups is the total number of groups in the queue.\n\tGroups int\n\n\t// Number of tasks in each state.\n\tPending int\n\tActive int\n\tScheduled int\n\tRetry int\n\tArchived int\n\tCompleted int\n\tAggregating int\n\n\t// Number of tasks processed within the current date.\n\t// The number includes both succeeded and failed tasks.\n\tProcessed int\n\t// Number of tasks failed within the current date.\n\tFailed int\n\n\t// Total number of tasks processed (both succeeded and failed) from this queue.\n\tProcessedTotal int\n\t// Total number of tasks failed.\n\tFailedTotal int\n\n\t// Latency of the queue, measured by the oldest pending task in the queue.\n\tLatency time.Duration\n\t// Time this stats was taken.\n\tTimestamp time.Time\n}\n\n// DailyStats holds aggregate data for a given day.\ntype DailyStats struct {\n\t// Name of the queue (e.g. \"default\", \"critical\").\n\tQueue string\n\t// Total number of tasks processed during the given day.\n\t// The number includes both succeeded and failed tasks.\n\tProcessed int\n\t// Total number of tasks failed during the given day.\n\tFailed int\n\t// Date this stats was taken.\n\tTime time.Time\n}\n\n// KEYS[1] -> asynq::pending\n// KEYS[2] -> asynq::active\n// KEYS[3] -> asynq::scheduled\n// KEYS[4] -> asynq::retry\n// KEYS[5] -> asynq::archived\n// KEYS[6] -> asynq::completed\n// KEYS[7] -> asynq::processed:\n// KEYS[8] -> asynq::failed:\n// KEYS[9] -> asynq::processed\n// KEYS[10] -> asynq::failed\n// KEYS[11] -> asynq::paused\n// KEYS[12] -> asynq::groups\n// --------\n// ARGV[1] -> task key prefix\n// ARGV[2] -> group key prefix\nvar currentStatsCmd = redis.NewScript(`\nlocal res = {}\nlocal pendingTaskCount = redis.call(\"LLEN\", KEYS[1])\ntable.insert(res, KEYS[1])\ntable.insert(res, pendingTaskCount)\ntable.insert(res, KEYS[2])\ntable.insert(res, redis.call(\"LLEN\", KEYS[2]))\ntable.insert(res, KEYS[3])\ntable.insert(res, redis.call(\"ZCARD\", KEYS[3]))\ntable.insert(res, KEYS[4])\ntable.insert(res, redis.call(\"ZCARD\", KEYS[4]))\ntable.insert(res, KEYS[5])\ntable.insert(res, redis.call(\"ZCARD\", KEYS[5]))\ntable.insert(res, KEYS[6])\ntable.insert(res, redis.call(\"ZCARD\", KEYS[6]))\nfor i=7,10 do\n local count = 0\n\tlocal n = redis.call(\"GET\", KEYS[i])\n\tif n then\n\t count = tonumber(n)\n\tend\n\ttable.insert(res, KEYS[i])\n\ttable.insert(res, count)\nend\ntable.insert(res, KEYS[11])\ntable.insert(res, redis.call(\"EXISTS\", KEYS[11]))\ntable.insert(res, \"oldest_pending_since\")\nif pendingTaskCount > 0 then\n\tlocal id = redis.call(\"LRANGE\", KEYS[1], -1, -1)[1]\n\ttable.insert(res, redis.call(\"HGET\", ARGV[1] .. id, \"pending_since\"))\nelse\n\ttable.insert(res, 0)\nend\nlocal group_names = redis.call(\"SMEMBERS\", KEYS[12])\ntable.insert(res, \"group_size\")\ntable.insert(res, table.getn(group_names))\nlocal aggregating_count = 0\nfor _, gname in ipairs(group_names) do\n\taggregating_count = aggregating_count + redis.call(\"ZCARD\", ARGV[2] .. gname)\nend\ntable.insert(res, \"aggregating_count\")\ntable.insert(res, aggregating_count)\nreturn res`)\n\n// CurrentStats returns a current state of the queues.\nfunc (r *RDB) CurrentStats(qname string) (*Stats, error) {\n\tvar op errors.Op = \"rdb.CurrentStats\"\n\texists, err := r.queueExists(qname)\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.Unknown, err)\n\t}\n\tif !exists {\n\t\treturn nil, errors.E(op, errors.NotFound, &errors.QueueNotFoundError{Queue: qname})\n\t}\n\tnow := r.clock.Now()\n\tkeys := []string{\n\t\tbase.PendingKey(qname),\n\t\tbase.ActiveKey(qname),\n\t\tbase.ScheduledKey(qname),\n\t\tbase.RetryKey(qname),\n\t\tbase.ArchivedKey(qname),\n\t\tbase.CompletedKey(qname),\n\t\tbase.ProcessedKey(qname, now),\n\t\tbase.FailedKey(qname, now),\n\t\tbase.ProcessedTotalKey(qname),\n\t\tbase.FailedTotalKey(qname),\n\t\tbase.PausedKey(qname),\n\t\tbase.AllGroups(qname),\n\t}\n\targv := []interface{}{\n\t\tbase.TaskKeyPrefix(qname),\n\t\tbase.GroupKeyPrefix(qname),\n\t}\n\tres, err := currentStatsCmd.Run(context.Background(), r.client, keys, argv...).Result()\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.Unknown, err)\n\t}\n\tdata, err := cast.ToSliceE(res)\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.Internal, \"cast error: unexpected return value from Lua script\")\n\t}\n\tstats := &Stats{\n\t\tQueue: qname,\n\t\tTimestamp: now,\n\t}\n\tsize := 0\n\tfor i := 0; i < len(data); i += 2 {\n\t\tkey := cast.ToString(data[i])\n\t\tval := cast.ToInt(data[i+1])\n\t\tswitch key {\n\t\tcase base.PendingKey(qname):\n\t\t\tstats.Pending = val\n\t\t\tsize += val\n\t\tcase base.ActiveKey(qname):\n\t\t\tstats.Active = val\n\t\t\tsize += val\n\t\tcase base.ScheduledKey(qname):\n\t\t\tstats.Scheduled = val\n\t\t\tsize += val\n\t\tcase base.RetryKey(qname):\n\t\t\tstats.Retry = val\n\t\t\tsize += val\n\t\tcase base.ArchivedKey(qname):\n\t\t\tstats.Archived = val\n\t\t\tsize += val\n\t\tcase base.CompletedKey(qname):\n\t\t\tstats.Completed = val\n\t\t\tsize += val\n\t\tcase base.ProcessedKey(qname, now):\n\t\t\tstats.Processed = val\n\t\tcase base.FailedKey(qname, now):\n\t\t\tstats.Failed = val\n\t\tcase base.ProcessedTotalKey(qname):\n\t\t\tstats.ProcessedTotal = val\n\t\tcase base.FailedTotalKey(qname):\n\t\t\tstats.FailedTotal = val\n\t\tcase base.PausedKey(qname):\n\t\t\tif val == 0 {\n\t\t\t\tstats.Paused = false\n\t\t\t} else {\n\t\t\t\tstats.Paused = true\n\t\t\t}\n\t\tcase \"oldest_pending_since\":\n\t\t\tif val == 0 {\n\t\t\t\tstats.Latency = 0\n\t\t\t} else {\n\t\t\t\tstats.Latency = r.clock.Now().Sub(time.Unix(0, int64(val)))\n\t\t\t}\n\t\tcase \"group_size\":\n\t\t\tstats.Groups = val\n\t\tcase \"aggregating_count\":\n\t\t\tstats.Aggregating = val\n\t\t\tsize += val\n\t\t}\n\t}\n\tstats.Size = size\n\tmemusg, err := r.memoryUsage(qname)\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.CanonicalCode(err), err)\n\t}\n\tstats.MemoryUsage = memusg\n\treturn stats, nil\n}\n\n// Computes memory usage for the given queue by sampling tasks\n// from each redis list/zset. Returns approximate memory usage value\n// in bytes.\n//\n// KEYS[1] -> asynq:{qname}:active\n// KEYS[2] -> asynq:{qname}:pending\n// KEYS[3] -> asynq:{qname}:scheduled\n// KEYS[4] -> asynq:{qname}:retry\n// KEYS[5] -> asynq:{qname}:archived\n// KEYS[6] -> asynq:{qname}:completed\n// KEYS[7] -> asynq:{qname}:groups\n// -------\n// ARGV[1] -> asynq:{qname}:t: (task key prefix)\n// ARGV[2] -> task sample size per redis list/zset (e.g 20)\n// ARGV[3] -> group sample size\n// ARGV[4] -> asynq:{qname}:g: (group key prefix)\nvar memoryUsageCmd = redis.NewScript(`\nlocal sample_size = tonumber(ARGV[2])\nif sample_size <= 0 then\n return redis.error_reply(\"sample size must be a positive number\")\nend\nlocal memusg = 0\nfor i=1,2 do\n local ids = redis.call(\"LRANGE\", KEYS[i], 0, sample_size - 1)\n local sample_total = 0\n if (table.getn(ids) > 0) then\n for _, id in ipairs(ids) do\n local bytes = redis.call(\"MEMORY\", \"USAGE\", ARGV[1] .. id)\n sample_total = sample_total + bytes\n end\n local n = redis.call(\"LLEN\", KEYS[i])\n local avg = sample_total / table.getn(ids)\n memusg = memusg + (avg * n)\n end\n local m = redis.call(\"MEMORY\", \"USAGE\", KEYS[i])\n if (m) then\n memusg = memusg + m\n end\nend\nfor i=3,6 do\n local ids = redis.call(\"ZRANGE\", KEYS[i], 0, sample_size - 1)\n local sample_total = 0\n if (table.getn(ids) > 0) then\n for _, id in ipairs(ids) do\n local bytes = redis.call(\"MEMORY\", \"USAGE\", ARGV[1] .. id)\n sample_total = sample_total + bytes\n end\n local n = redis.call(\"ZCARD\", KEYS[i])\n local avg = sample_total / table.getn(ids)\n memusg = memusg + (avg * n)\n end\n local m = redis.call(\"MEMORY\", \"USAGE\", KEYS[i])\n if (m) then\n memusg = memusg + m\n end\nend\nlocal groups = redis.call(\"SMEMBERS\", KEYS[7])\nif table.getn(groups) > 0 then\n\tlocal agg_task_count = 0\n\tlocal agg_task_sample_total = 0\n\tlocal agg_task_sample_size = 0\n\tfor i, gname in ipairs(groups) do\n\t\tlocal group_key = ARGV[4] .. gname\n\t\tagg_task_count = agg_task_count + redis.call(\"ZCARD\", group_key)\n\t\tif i <= tonumber(ARGV[3]) then\n\t\t\tlocal ids = redis.call(\"ZRANGE\", group_key, 0, sample_size - 1)\n\t\t\tfor _, id in ipairs(ids) do\n\t\t\t\tlocal bytes = redis.call(\"MEMORY\", \"USAGE\", ARGV[1] .. id)\n\t\t\t\tagg_task_sample_total = agg_task_sample_total + bytes\n\t\t\t\tagg_task_sample_size = agg_task_sample_size + 1\n\t\t\tend\n\t\tend\n\tend\n\tlocal avg = agg_task_sample_total / agg_task_sample_size\n\tmemusg = memusg + (avg * agg_task_count)\nend\nreturn memusg\n`)\n\nfunc (r *RDB) memoryUsage(qname string) (int64, error) {\n\tvar op errors.Op = \"rdb.memoryUsage\"\n\tconst (\n\t\ttaskSampleSize = 20\n\t\tgroupSampleSize = 5\n\t)\n\n\tkeys := []string{\n\t\tbase.ActiveKey(qname),\n\t\tbase.PendingKey(qname),\n\t\tbase.ScheduledKey(qname),\n\t\tbase.RetryKey(qname),\n\t\tbase.ArchivedKey(qname),\n\t\tbase.CompletedKey(qname),\n\t\tbase.AllGroups(qname),\n\t}\n\targv := []interface{}{\n\t\tbase.TaskKeyPrefix(qname),\n\t\ttaskSampleSize,\n\t\tgroupSampleSize,\n\t\tbase.GroupKeyPrefix(qname),\n\t}\n\tres, err := memoryUsageCmd.Run(context.Background(), r.client, keys, argv...).Result()\n\tif err != nil {\n\t\treturn 0, errors.E(op, errors.Unknown, fmt.Sprintf(\"redis eval error: %v\", err))\n\t}\n\tusg, err := cast.ToInt64E(res)\n\tif err != nil {\n\t\treturn 0, errors.E(op, errors.Internal, \"could not cast script return value to int64\")\n\t}\n\treturn usg, nil\n}\n\nvar historicalStatsCmd = redis.NewScript(`\nlocal res = {}\nfor _, key in ipairs(KEYS) do\n\tlocal n = redis.call(\"GET\", key)\n\tif not n then\n\t\tn = 0\n\tend\n\ttable.insert(res, tonumber(n))\nend\nreturn res`)\n\n// HistoricalStats returns a list of stats from the last n days for the given queue.\nfunc (r *RDB) HistoricalStats(qname string, n int) ([]*DailyStats, error) {\n\tvar op errors.Op = \"rdb.HistoricalStats\"\n\tif n < 1 {\n\t\treturn nil, errors.E(op, errors.FailedPrecondition, \"the number of days must be positive\")\n\t}\n\texists, err := r.queueExists(qname)\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: \"sismember\", Err: err})\n\t}\n\tif !exists {\n\t\treturn nil, errors.E(op, errors.NotFound, &errors.QueueNotFoundError{Queue: qname})\n\t}\n\tconst day = 24 * time.Hour\n\tnow := r.clock.Now().UTC()\n\tvar days []time.Time\n\tvar keys []string\n\tfor i := 0; i < n; i++ {\n\t\tts := now.Add(-time.Duration(i) * day)\n\t\tdays = append(days, ts)\n\t\tkeys = append(keys, base.ProcessedKey(qname, ts))\n\t\tkeys = append(keys, base.FailedKey(qname, ts))\n\t}\n\tres, err := historicalStatsCmd.Run(context.Background(), r.client, keys).Result()\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.Unknown, fmt.Sprintf(\"redis eval error: %v\", err))\n\t}\n\tdata, err := cast.ToIntSliceE(res)\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.Internal, fmt.Sprintf(\"cast error: unexpected return value from Lua script: %v\", res))\n\t}\n\tvar stats []*DailyStats\n\tfor i := 0; i < len(data); i += 2 {\n\t\tstats = append(stats, &DailyStats{\n\t\t\tQueue: qname,\n\t\t\tProcessed: data[i],\n\t\t\tFailed: data[i+1],\n\t\t\tTime: days[i/2],\n\t\t})\n\t}\n\treturn stats, nil\n}\n\n// RedisInfo returns a map of redis info.\nfunc (r *RDB) RedisInfo() (map[string]string, error) {\n\tres, err := r.client.Info(context.Background()).Result()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn parseInfo(res)\n}\n\n// RedisClusterInfo returns a map of redis cluster info.\nfunc (r *RDB) RedisClusterInfo() (map[string]string, error) {\n\tres, err := r.client.ClusterInfo(context.Background()).Result()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn parseInfo(res)\n}\n\nfunc parseInfo(infoStr string) (map[string]string, error) {\n\tinfo := make(map[string]string)\n\tlines := strings.Split(infoStr, \"\\r\\n\")\n\tfor _, l := range lines {\n\t\tkv := strings.Split(l, \":\")\n\t\tif len(kv) == 2 {\n\t\t\tinfo[kv[0]] = kv[1]\n\t\t}\n\t}\n\treturn info, nil\n}\n\n// TODO: Use generics once available.\nfunc reverse(x []*base.TaskInfo) {\n\tfor i := len(x)/2 - 1; i >= 0; i-- {\n\t\topp := len(x) - 1 - i\n\t\tx[i], x[opp] = x[opp], x[i]\n\t}\n}\n\n// checkQueueExists verifies whether the queue exists.\n// It returns QueueNotFoundError if queue doesn't exist.\nfunc (r *RDB) checkQueueExists(qname string) error {\n\texists, err := r.queueExists(qname)\n\tif err != nil {\n\t\treturn errors.E(errors.Unknown, &errors.RedisCommandError{Command: \"sismember\", Err: err})\n\t}\n\tif !exists {\n\t\treturn errors.E(errors.Internal, &errors.QueueNotFoundError{Queue: qname})\n\t}\n\treturn nil\n}\n\n// Input:\n// KEYS[1] -> task key (asynq:{}:t:)\n// ARGV[1] -> task id\n// ARGV[2] -> current time in Unix time (seconds)\n// ARGV[3] -> queue key prefix (asynq:{}:)\n//\n// Output:\n// Tuple of {msg, state, nextProcessAt, result}\n// msg: encoded task message\n// state: string describing the state of the task\n// nextProcessAt: unix time in seconds, zero if not applicable.\n// result: result data associated with the task\n//\n// If the task key doesn't exist, it returns error with a message \"NOT FOUND\"\nvar getTaskInfoCmd = redis.NewScript(`\n\tif redis.call(\"EXISTS\", KEYS[1]) == 0 then\n\t\treturn redis.error_reply(\"NOT FOUND\")\n\tend\n\tlocal msg, state, result = unpack(redis.call(\"HMGET\", KEYS[1], \"msg\", \"state\", \"result\"))\n\tif state == \"scheduled\" or state == \"retry\" then\n\t\treturn {msg, state, redis.call(\"ZSCORE\", ARGV[3] .. state, ARGV[1]), result}\n\tend\n\tif state == \"pending\" then\n\t\treturn {msg, state, ARGV[2], result}\n\tend\n\treturn {msg, state, 0, result}\n`)\n\n// GetTaskInfo returns a TaskInfo describing the task from the given queue.\nfunc (r *RDB) GetTaskInfo(qname, id string) (*base.TaskInfo, error) {\n\tvar op errors.Op = \"rdb.GetTaskInfo\"\n\tif err := r.checkQueueExists(qname); err != nil {\n\t\treturn nil, errors.E(op, errors.CanonicalCode(err), err)\n\t}\n\tkeys := []string{base.TaskKey(qname, id)}\n\targv := []interface{}{\n\t\tid,\n\t\tr.clock.Now().Unix(),\n\t\tbase.QueueKeyPrefix(qname),\n\t}\n\tres, err := getTaskInfoCmd.Run(context.Background(), r.client, keys, argv...).Result()\n\tif err != nil {\n\t\tif err.Error() == \"NOT FOUND\" {\n\t\t\treturn nil, errors.E(op, errors.NotFound, &errors.TaskNotFoundError{Queue: qname, ID: id})\n\t\t}\n\t\treturn nil, errors.E(op, errors.Unknown, err)\n\t}\n\tvals, err := cast.ToSliceE(res)\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.Internal, \"unexpected value returned from Lua script\")\n\t}\n\tif len(vals) != 4 {\n\t\treturn nil, errors.E(op, errors.Internal, \"unepxected number of values returned from Lua script\")\n\t}\n\tencoded, err := cast.ToStringE(vals[0])\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.Internal, \"unexpected value returned from Lua script\")\n\t}\n\tstateStr, err := cast.ToStringE(vals[1])\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.Internal, \"unexpected value returned from Lua script\")\n\t}\n\tprocessAtUnix, err := cast.ToInt64E(vals[2])\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.Internal, \"unexpected value returned from Lua script\")\n\t}\n\tresultStr, err := cast.ToStringE(vals[3])\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.Internal, \"unexpected value returned from Lua script\")\n\t}\n\tmsg, err := base.DecodeMessage([]byte(encoded))\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.Internal, \"could not decode task message\")\n\t}\n\tstate, err := base.TaskStateFromString(stateStr)\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.CanonicalCode(err), err)\n\t}\n\tvar nextProcessAt time.Time\n\tif processAtUnix != 0 {\n\t\tnextProcessAt = time.Unix(processAtUnix, 0)\n\t}\n\tvar result []byte\n\tif len(resultStr) > 0 {\n\t\tresult = []byte(resultStr)\n\t}\n\treturn &base.TaskInfo{\n\t\tMessage: msg,\n\t\tState: state,\n\t\tNextProcessAt: nextProcessAt,\n\t\tResult: result,\n\t}, nil\n}\n\ntype GroupStat struct {\n\t// Name of the group.\n\tGroup string\n\n\t// Size of the group.\n\tSize int\n}\n\n// KEYS[1] -> asynq:{}:groups\n// -------\n// ARGV[1] -> group key prefix\n//\n// Output:\n// list of group name and size (e.g. group1 size1 group2 size2 ...)\n//\n// Time Complexity:\n// O(N) where N being the number of groups in the given queue.\nvar groupStatsCmd = redis.NewScript(`\nlocal res = {}\nlocal group_names = redis.call(\"SMEMBERS\", KEYS[1])\nfor _, gname in ipairs(group_names) do\n\tlocal size = redis.call(\"ZCARD\", ARGV[1] .. gname)\n\ttable.insert(res, gname)\n\ttable.insert(res, size)\nend\nreturn res\n`)\n\nfunc (r *RDB) GroupStats(qname string) ([]*GroupStat, error) {\n\tvar op errors.Op = \"RDB.GroupStats\"\n\tkeys := []string{base.AllGroups(qname)}\n\targv := []interface{}{base.GroupKeyPrefix(qname)}\n\tres, err := groupStatsCmd.Run(context.Background(), r.client, keys, argv...).Result()\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.Unknown, err)\n\t}\n\tdata, err := cast.ToSliceE(res)\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.Internal, \"cast error: unexpected return value from Lua script\")\n\t}\n\tvar stats []*GroupStat\n\tfor i := 0; i < len(data); i += 2 {\n\t\tstats = append(stats, &GroupStat{\n\t\t\tGroup: data[i].(string),\n\t\t\tSize: int(data[i+1].(int64)),\n\t\t})\n\t}\n\treturn stats, nil\n}\n\n// Pagination specifies the page size and page number\n// for the list operation.\ntype Pagination struct {\n\t// Number of items in the page.\n\tSize int\n\n\t// Page number starting from zero.\n\tPage int\n}\n\nfunc (p Pagination) start() int64 {\n\treturn int64(p.Size * p.Page)\n}\n\nfunc (p Pagination) stop() int64 {\n\treturn int64(p.Size*p.Page + p.Size - 1)\n}\n\n// ListPending returns pending tasks that are ready to be processed.\nfunc (r *RDB) ListPending(qname string, pgn Pagination) ([]*base.TaskInfo, error) {\n\tvar op errors.Op = \"rdb.ListPending\"\n\texists, err := r.queueExists(qname)\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: \"sismember\", Err: err})\n\t}\n\tif !exists {\n\t\treturn nil, errors.E(op, errors.NotFound, &errors.QueueNotFoundError{Queue: qname})\n\t}\n\tres, err := r.listMessages(qname, base.TaskStatePending, pgn)\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.CanonicalCode(err), err)\n\t}\n\treturn res, nil\n}\n\n// ListActive returns all tasks that are currently being processed for the given queue.\nfunc (r *RDB) ListActive(qname string, pgn Pagination) ([]*base.TaskInfo, error) {\n\tvar op errors.Op = \"rdb.ListActive\"\n\texists, err := r.queueExists(qname)\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: \"sismember\", Err: err})\n\t}\n\tif !exists {\n\t\treturn nil, errors.E(op, errors.NotFound, &errors.QueueNotFoundError{Queue: qname})\n\t}\n\tres, err := r.listMessages(qname, base.TaskStateActive, pgn)\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.CanonicalCode(err), err)\n\t}\n\treturn res, nil\n}\n\n// KEYS[1] -> key for id list (e.g. asynq:{}:pending)\n// ARGV[1] -> start offset\n// ARGV[2] -> stop offset\n// ARGV[3] -> task key prefix\nvar listMessagesCmd = redis.NewScript(`\nlocal ids = redis.call(\"LRange\", KEYS[1], ARGV[1], ARGV[2])\nlocal data = {}\nfor _, id in ipairs(ids) do\n\tlocal key = ARGV[3] .. id\n\tlocal msg, result = unpack(redis.call(\"HMGET\", key, \"msg\",\"result\"))\n\ttable.insert(data, msg)\n\ttable.insert(data, result)\nend\nreturn data\n`)\n\n// listMessages returns a list of TaskInfo in Redis list with the given key.\nfunc (r *RDB) listMessages(qname string, state base.TaskState, pgn Pagination) ([]*base.TaskInfo, error) {\n\tvar key string\n\tswitch state {\n\tcase base.TaskStateActive:\n\t\tkey = base.ActiveKey(qname)\n\tcase base.TaskStatePending:\n\t\tkey = base.PendingKey(qname)\n\tdefault:\n\t\tpanic(fmt.Sprintf(\"unsupported task state: %v\", state))\n\t}\n\t// Note: Because we use LPUSH to redis list, we need to calculate the\n\t// correct range and reverse the list to get the tasks with pagination.\n\tstop := -pgn.start() - 1\n\tstart := -pgn.stop() - 1\n\tres, err := listMessagesCmd.Run(context.Background(), r.client,\n\t\t[]string{key}, start, stop, base.TaskKeyPrefix(qname)).Result()\n\tif err != nil {\n\t\treturn nil, errors.E(errors.Unknown, err)\n\t}\n\tdata, err := cast.ToStringSliceE(res)\n\tif err != nil {\n\t\treturn nil, errors.E(errors.Internal, fmt.Errorf(\"cast error: Lua script returned unexpected value: %v\", res))\n\t}\n\tvar infos []*base.TaskInfo\n\tfor i := 0; i < len(data); i += 2 {\n\t\tm, err := base.DecodeMessage([]byte(data[i]))\n\t\tif err != nil {\n\t\t\tcontinue // bad data, ignore and continue\n\t\t}\n\t\tvar res []byte\n\t\tif len(data[i+1]) > 0 {\n\t\t\tres = []byte(data[i+1])\n\t\t}\n\t\tvar nextProcessAt time.Time\n\t\tif state == base.TaskStatePending {\n\t\t\tnextProcessAt = r.clock.Now()\n\t\t}\n\t\tinfos = append(infos, &base.TaskInfo{\n\t\t\tMessage: m,\n\t\t\tState: state,\n\t\t\tNextProcessAt: nextProcessAt,\n\t\t\tResult: res,\n\t\t})\n\t}\n\treverse(infos)\n\treturn infos, nil\n\n}\n\n// ListScheduled returns all tasks from the given queue that are scheduled\n// to be processed in the future.\nfunc (r *RDB) ListScheduled(qname string, pgn Pagination) ([]*base.TaskInfo, error) {\n\tvar op errors.Op = \"rdb.ListScheduled\"\n\texists, err := r.queueExists(qname)\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: \"sismember\", Err: err})\n\t}\n\tif !exists {\n\t\treturn nil, errors.E(op, errors.NotFound, &errors.QueueNotFoundError{Queue: qname})\n\t}\n\tres, err := r.listZSetEntries(qname, base.TaskStateScheduled, base.ScheduledKey(qname), pgn)\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.CanonicalCode(err), err)\n\t}\n\treturn res, nil\n}\n\n// ListRetry returns all tasks from the given queue that have failed before\n// and willl be retried in the future.\nfunc (r *RDB) ListRetry(qname string, pgn Pagination) ([]*base.TaskInfo, error) {\n\tvar op errors.Op = \"rdb.ListRetry\"\n\texists, err := r.queueExists(qname)\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: \"sismember\", Err: err})\n\t}\n\tif !exists {\n\t\treturn nil, errors.E(op, errors.NotFound, &errors.QueueNotFoundError{Queue: qname})\n\t}\n\tres, err := r.listZSetEntries(qname, base.TaskStateRetry, base.RetryKey(qname), pgn)\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.CanonicalCode(err), err)\n\t}\n\treturn res, nil\n}\n\n// ListArchived returns all tasks from the given queue that have exhausted its retry limit.\nfunc (r *RDB) ListArchived(qname string, pgn Pagination) ([]*base.TaskInfo, error) {\n\tvar op errors.Op = \"rdb.ListArchived\"\n\texists, err := r.queueExists(qname)\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: \"sismember\", Err: err})\n\t}\n\tif !exists {\n\t\treturn nil, errors.E(op, errors.NotFound, &errors.QueueNotFoundError{Queue: qname})\n\t}\n\tzs, err := r.listZSetEntries(qname, base.TaskStateArchived, base.ArchivedKey(qname), pgn)\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.CanonicalCode(err), err)\n\t}\n\treturn zs, nil\n}\n\n// ListCompleted returns all tasks from the given queue that have completed successfully.\nfunc (r *RDB) ListCompleted(qname string, pgn Pagination) ([]*base.TaskInfo, error) {\n\tvar op errors.Op = \"rdb.ListCompleted\"\n\texists, err := r.queueExists(qname)\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: \"sismember\", Err: err})\n\t}\n\tif !exists {\n\t\treturn nil, errors.E(op, errors.NotFound, &errors.QueueNotFoundError{Queue: qname})\n\t}\n\tzs, err := r.listZSetEntries(qname, base.TaskStateCompleted, base.CompletedKey(qname), pgn)\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.CanonicalCode(err), err)\n\t}\n\treturn zs, nil\n}\n\n// ListAggregating returns all tasks from the given group.\nfunc (r *RDB) ListAggregating(qname, gname string, pgn Pagination) ([]*base.TaskInfo, error) {\n\tvar op errors.Op = \"rdb.ListAggregating\"\n\texists, err := r.queueExists(qname)\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: \"sismember\", Err: err})\n\t}\n\tif !exists {\n\t\treturn nil, errors.E(op, errors.NotFound, &errors.QueueNotFoundError{Queue: qname})\n\t}\n\tzs, err := r.listZSetEntries(qname, base.TaskStateAggregating, base.GroupKey(qname, gname), pgn)\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.CanonicalCode(err), err)\n\t}\n\treturn zs, nil\n}\n\n// Reports whether a queue with the given name exists.\nfunc (r *RDB) queueExists(qname string) (bool, error) {\n\treturn r.client.SIsMember(context.Background(), base.AllQueues, qname).Result()\n}\n\n// KEYS[1] -> key for ids set (e.g. asynq:{}:scheduled)\n// ARGV[1] -> min\n// ARGV[2] -> max\n// ARGV[3] -> task key prefix\n//\n// Returns an array populated with\n// [msg1, score1, result1, msg2, score2, result2, ..., msgN, scoreN, resultN]\nvar listZSetEntriesCmd = redis.NewScript(`\nlocal data = {}\nlocal id_score_pairs = redis.call(\"ZRANGE\", KEYS[1], ARGV[1], ARGV[2], \"WITHSCORES\")\nfor i = 1, table.getn(id_score_pairs), 2 do\n\tlocal id = id_score_pairs[i]\n\tlocal score = id_score_pairs[i+1]\n\tlocal key = ARGV[3] .. id\n\tlocal msg, res = unpack(redis.call(\"HMGET\", key, \"msg\", \"result\"))\n\ttable.insert(data, msg)\n\ttable.insert(data, score)\n\ttable.insert(data, res)\nend\nreturn data\n`)\n\n// listZSetEntries returns a list of message and score pairs in Redis sorted-set\n// with the given key.\nfunc (r *RDB) listZSetEntries(qname string, state base.TaskState, key string, pgn Pagination) ([]*base.TaskInfo, error) {\n\tres, err := listZSetEntriesCmd.Run(context.Background(), r.client, []string{key},\n\t\tpgn.start(), pgn.stop(), base.TaskKeyPrefix(qname)).Result()\n\tif err != nil {\n\t\treturn nil, errors.E(errors.Unknown, err)\n\t}\n\tdata, err := cast.ToSliceE(res)\n\tif err != nil {\n\t\treturn nil, errors.E(errors.Internal, fmt.Errorf(\"cast error: Lua script returned unexpected value: %v\", res))\n\t}\n\tvar infos []*base.TaskInfo\n\tfor i := 0; i < len(data); i += 3 {\n\t\ts, err := cast.ToStringE(data[i])\n\t\tif err != nil {\n\t\t\treturn nil, errors.E(errors.Internal, fmt.Errorf(\"cast error: Lua script returned unexpected value: %v\", res))\n\t\t}\n\t\tscore, err := cast.ToInt64E(data[i+1])\n\t\tif err != nil {\n\t\t\treturn nil, errors.E(errors.Internal, fmt.Errorf(\"cast error: Lua script returned unexpected value: %v\", res))\n\t\t}\n\t\tresStr, err := cast.ToStringE(data[i+2])\n\t\tif err != nil {\n\t\t\treturn nil, errors.E(errors.Internal, fmt.Errorf(\"cast error: Lua script returned unexpected value: %v\", res))\n\t\t}\n\t\tmsg, err := base.DecodeMessage([]byte(s))\n\t\tif err != nil {\n\t\t\tcontinue // bad data, ignore and continue\n\t\t}\n\t\tvar nextProcessAt time.Time\n\t\tif state == base.TaskStateScheduled || state == base.TaskStateRetry {\n\t\t\tnextProcessAt = time.Unix(score, 0)\n\t\t}\n\t\tvar resBytes []byte\n\t\tif len(resStr) > 0 {\n\t\t\tresBytes = []byte(resStr)\n\t\t}\n\t\tinfos = append(infos, &base.TaskInfo{\n\t\t\tMessage: msg,\n\t\t\tState: state,\n\t\t\tNextProcessAt: nextProcessAt,\n\t\t\tResult: resBytes,\n\t\t})\n\t}\n\treturn infos, nil\n}\n\n// RunAllScheduledTasks enqueues all scheduled tasks from the given queue\n// and returns the number of tasks enqueued.\n// If a queue with the given name doesn't exist, it returns QueueNotFoundError.\nfunc (r *RDB) RunAllScheduledTasks(qname string) (int64, error) {\n\tvar op errors.Op = \"rdb.RunAllScheduledTasks\"\n\tn, err := r.runAll(base.ScheduledKey(qname), qname)\n\tif errors.IsQueueNotFound(err) {\n\t\treturn 0, errors.E(op, errors.NotFound, err)\n\t}\n\tif err != nil {\n\t\treturn 0, errors.E(op, errors.Unknown, err)\n\t}\n\treturn n, nil\n}\n\n// RunAllRetryTasks enqueues all retry tasks from the given queue\n// and returns the number of tasks enqueued.\n// If a queue with the given name doesn't exist, it returns QueueNotFoundError.\nfunc (r *RDB) RunAllRetryTasks(qname string) (int64, error) {\n\tvar op errors.Op = \"rdb.RunAllRetryTasks\"\n\tn, err := r.runAll(base.RetryKey(qname), qname)\n\tif errors.IsQueueNotFound(err) {\n\t\treturn 0, errors.E(op, errors.NotFound, err)\n\t}\n\tif err != nil {\n\t\treturn 0, errors.E(op, errors.Unknown, err)\n\t}\n\treturn n, nil\n}\n\n// RunAllArchivedTasks enqueues all archived tasks from the given queue\n// and returns the number of tasks enqueued.\n// If a queue with the given name doesn't exist, it returns QueueNotFoundError.\nfunc (r *RDB) RunAllArchivedTasks(qname string) (int64, error) {\n\tvar op errors.Op = \"rdb.RunAllArchivedTasks\"\n\tn, err := r.runAll(base.ArchivedKey(qname), qname)\n\tif errors.IsQueueNotFound(err) {\n\t\treturn 0, errors.E(op, errors.NotFound, err)\n\t}\n\tif err != nil {\n\t\treturn 0, errors.E(op, errors.Unknown, err)\n\t}\n\treturn n, nil\n}\n\n// runAllAggregatingCmd schedules all tasks in the group to run individually.\n//\n// Input:\n// KEYS[1] -> asynq:{}:g:\n// KEYS[2] -> asynq:{}:pending\n// KEYS[3] -> asynq:{}:groups\n// -------\n// ARGV[1] -> task key prefix\n// ARGV[2] -> group name\n//\n// Output:\n// integer: number of tasks scheduled to run\nvar runAllAggregatingCmd = redis.NewScript(`\nlocal ids = redis.call(\"ZRANGE\", KEYS[1], 0, -1)\nfor _, id in ipairs(ids) do\n\tredis.call(\"LPUSH\", KEYS[2], id)\n\tredis.call(\"HSET\", ARGV[1] .. id, \"state\", \"pending\")\nend\nredis.call(\"DEL\", KEYS[1])\nredis.call(\"SREM\", KEYS[3], ARGV[2])\nreturn table.getn(ids)\n`)\n\n// RunAllAggregatingTasks schedules all tasks from the given queue to run\n// and returns the number of tasks scheduled to run.\n// If a queue with the given name doesn't exist, it returns QueueNotFoundError.\nfunc (r *RDB) RunAllAggregatingTasks(qname, gname string) (int64, error) {\n\tvar op errors.Op = \"rdb.RunAllAggregatingTasks\"\n\tif err := r.checkQueueExists(qname); err != nil {\n\t\treturn 0, errors.E(op, errors.CanonicalCode(err), err)\n\t}\n\tkeys := []string{\n\t\tbase.GroupKey(qname, gname),\n\t\tbase.PendingKey(qname),\n\t\tbase.AllGroups(qname),\n\t}\n\targv := []interface{}{\n\t\tbase.TaskKeyPrefix(qname),\n\t\tgname,\n\t}\n\tres, err := runAllAggregatingCmd.Run(context.Background(), r.client, keys, argv...).Result()\n\tif err != nil {\n\t\treturn 0, errors.E(op, errors.Internal, err)\n\t}\n\tn, ok := res.(int64)\n\tif !ok {\n\t\treturn 0, errors.E(op, errors.Internal, fmt.Sprintf(\"unexpected return value from script %v\", res))\n\t}\n\treturn n, nil\n}\n\n// runTaskCmd is a Lua script that updates the given task to pending state.\n//\n// Input:\n// KEYS[1] -> asynq:{}:t:\n// KEYS[2] -> asynq:{}:pending\n// KEYS[3] -> asynq:{}:groups\n// --\n// ARGV[1] -> task ID\n// ARGV[2] -> queue key prefix; asynq:{}:\n// ARGV[3] -> group key prefix\n//\n// Output:\n// Numeric code indicating the status:\n// Returns 1 if task is successfully updated.\n// Returns 0 if task is not found.\n// Returns -1 if task is in active state.\n// Returns -2 if task is in pending state.\n// Returns error reply if unexpected error occurs.\nvar runTaskCmd = redis.NewScript(`\nif redis.call(\"EXISTS\", KEYS[1]) == 0 then\n\treturn 0\nend\nlocal state, group = unpack(redis.call(\"HMGET\", KEYS[1], \"state\", \"group\"))\nif state == \"active\" then\n\treturn -1\nelseif state == \"pending\" then\n\treturn -2\nelseif state == \"aggregating\" then\n\tlocal n = redis.call(\"ZREM\", ARGV[3] .. group, ARGV[1])\n\tif n == 0 then\n\t\treturn redis.error_reply(\"internal error: task id not found in zset \" .. tostring(ARGV[3] .. group))\n\tend\n\tif redis.call(\"ZCARD\", ARGV[3] .. group) == 0 then\n\t\tredis.call(\"SREM\", KEYS[3], group)\n\tend\nelse\n\tlocal n = redis.call(\"ZREM\", ARGV[2] .. state, ARGV[1])\n\tif n == 0 then\n\t\treturn redis.error_reply(\"internal error: task id not found in zset \" .. tostring(ARGV[2] .. state))\n\tend\nend\nredis.call(\"LPUSH\", KEYS[2], ARGV[1])\nredis.call(\"HSET\", KEYS[1], \"state\", \"pending\")\nreturn 1\n`)\n\n// RunTask finds a task that matches the id from the given queue and updates it to pending state.\n// It returns nil if it successfully updated the task.\n//\n// If a queue with the given name doesn't exist, it returns QueueNotFoundError.\n// If a task with the given id doesn't exist in the queue, it returns TaskNotFoundError\n// If a task is in active or pending state it returns non-nil error with Code FailedPrecondition.\nfunc (r *RDB) RunTask(qname, id string) error {\n\tvar op errors.Op = \"rdb.RunTask\"\n\tif err := r.checkQueueExists(qname); err != nil {\n\t\treturn errors.E(op, errors.CanonicalCode(err), err)\n\t}\n\tkeys := []string{\n\t\tbase.TaskKey(qname, id),\n\t\tbase.PendingKey(qname),\n\t\tbase.AllGroups(qname),\n\t}\n\targv := []interface{}{\n\t\tid,\n\t\tbase.QueueKeyPrefix(qname),\n\t\tbase.GroupKeyPrefix(qname),\n\t}\n\tres, err := runTaskCmd.Run(context.Background(), r.client, keys, argv...).Result()\n\tif err != nil {\n\t\treturn errors.E(op, errors.Unknown, err)\n\t}\n\tn, ok := res.(int64)\n\tif !ok {\n\t\treturn errors.E(op, errors.Internal, fmt.Sprintf(\"cast error: unexpected return value from Lua script: %v\", res))\n\t}\n\tswitch n {\n\tcase 1:\n\t\treturn nil\n\tcase 0:\n\t\treturn errors.E(op, errors.NotFound, &errors.TaskNotFoundError{Queue: qname, ID: id})\n\tcase -1:\n\t\treturn errors.E(op, errors.FailedPrecondition, \"task is already running\")\n\tcase -2:\n\t\treturn errors.E(op, errors.FailedPrecondition, \"task is already in pending state\")\n\tdefault:\n\t\treturn errors.E(op, errors.Internal, fmt.Sprintf(\"unexpected return value from Lua script %d\", n))\n\t}\n}\n\n// runAllCmd is a Lua script that moves all tasks in the given state\n// (one of: scheduled, retry, archived) to pending state.\n//\n// Input:\n// KEYS[1] -> zset which holds task ids (e.g. asynq:{}:scheduled)\n// KEYS[2] -> asynq:{}:pending\n// --\n// ARGV[1] -> task key prefix\n//\n// Output:\n// integer: number of tasks updated to pending state.\nvar runAllCmd = redis.NewScript(`\nlocal ids = redis.call(\"ZRANGE\", KEYS[1], 0, -1)\nfor _, id in ipairs(ids) do\n\tredis.call(\"LPUSH\", KEYS[2], id)\n\tredis.call(\"HSET\", ARGV[1] .. id, \"state\", \"pending\")\nend\nredis.call(\"DEL\", KEYS[1])\nreturn table.getn(ids)`)\n\nfunc (r *RDB) runAll(zset, qname string) (int64, error) {\n\tif err := r.checkQueueExists(qname); err != nil {\n\t\treturn 0, err\n\t}\n\tkeys := []string{\n\t\tzset,\n\t\tbase.PendingKey(qname),\n\t}\n\targv := []interface{}{\n\t\tbase.TaskKeyPrefix(qname),\n\t}\n\tres, err := runAllCmd.Run(context.Background(), r.client, keys, argv...).Result()\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tn, ok := res.(int64)\n\tif !ok {\n\t\treturn 0, fmt.Errorf(\"could not cast %v to int64\", res)\n\t}\n\tif n == -1 {\n\t\treturn 0, &errors.QueueNotFoundError{Queue: qname}\n\t}\n\treturn n, nil\n}\n\n// ArchiveAllRetryTasks archives all retry tasks from the given queue and\n// returns the number of tasks that were moved.\n// If a queue with the given name doesn't exist, it returns QueueNotFoundError.\nfunc (r *RDB) ArchiveAllRetryTasks(qname string) (int64, error) {\n\tvar op errors.Op = \"rdb.ArchiveAllRetryTasks\"\n\tn, err := r.archiveAll(base.RetryKey(qname), base.ArchivedKey(qname), qname)\n\tif errors.IsQueueNotFound(err) {\n\t\treturn 0, errors.E(op, errors.NotFound, err)\n\t}\n\tif err != nil {\n\t\treturn 0, errors.E(op, errors.Internal, err)\n\t}\n\treturn n, nil\n}\n\n// ArchiveAllScheduledTasks archives all scheduled tasks from the given queue and\n// returns the number of tasks that were moved.\n// If a queue with the given name doesn't exist, it returns QueueNotFoundError.\nfunc (r *RDB) ArchiveAllScheduledTasks(qname string) (int64, error) {\n\tvar op errors.Op = \"rdb.ArchiveAllScheduledTasks\"\n\tn, err := r.archiveAll(base.ScheduledKey(qname), base.ArchivedKey(qname), qname)\n\tif errors.IsQueueNotFound(err) {\n\t\treturn 0, errors.E(op, errors.NotFound, err)\n\t}\n\tif err != nil {\n\t\treturn 0, errors.E(op, errors.Internal, err)\n\t}\n\treturn n, nil\n}\n\n// archiveAllAggregatingCmd archives all tasks in the given group.\n//\n// Input:\n// KEYS[1] -> asynq:{}:g:\n// KEYS[2] -> asynq:{}:archived\n// KEYS[3] -> asynq:{}:groups\n// -------\n// ARGV[1] -> current timestamp\n// ARGV[2] -> cutoff timestamp (e.g., 90 days ago)\n// ARGV[3] -> max number of tasks in archive (e.g., 100)\n// ARGV[4] -> task key prefix (asynq:{}:t:)\n// ARGV[5] -> group name\n//\n// Output:\n// integer: Number of tasks archived\nvar archiveAllAggregatingCmd = redis.NewScript(`\nlocal ids = redis.call(\"ZRANGE\", KEYS[1], 0, -1)\nfor _, id in ipairs(ids) do\n\tredis.call(\"ZADD\", KEYS[2], ARGV[1], id)\n\tredis.call(\"HSET\", ARGV[4] .. id, \"state\", \"archived\")\nend\nredis.call(\"ZREMRANGEBYSCORE\", KEYS[2], \"-inf\", ARGV[2])\nredis.call(\"ZREMRANGEBYRANK\", KEYS[2], 0, -ARGV[3])\nredis.call(\"DEL\", KEYS[1])\nredis.call(\"SREM\", KEYS[3], ARGV[5])\nreturn table.getn(ids)\n`)\n\n// ArchiveAllAggregatingTasks archives all aggregating tasks from the given group\n// and returns the number of tasks archived.\n// If a queue with the given name doesn't exist, it returns QueueNotFoundError.\nfunc (r *RDB) ArchiveAllAggregatingTasks(qname, gname string) (int64, error) {\n\tvar op errors.Op = \"rdb.ArchiveAllAggregatingTasks\"\n\tif err := r.checkQueueExists(qname); err != nil {\n\t\treturn 0, errors.E(op, errors.CanonicalCode(err), err)\n\t}\n\tkeys := []string{\n\t\tbase.GroupKey(qname, gname),\n\t\tbase.ArchivedKey(qname),\n\t\tbase.AllGroups(qname),\n\t}\n\tnow := r.clock.Now()\n\targv := []interface{}{\n\t\tnow.Unix(),\n\t\tnow.AddDate(0, 0, -archivedExpirationInDays).Unix(),\n\t\tmaxArchiveSize,\n\t\tbase.TaskKeyPrefix(qname),\n\t\tgname,\n\t}\n\tres, err := archiveAllAggregatingCmd.Run(context.Background(), r.client, keys, argv...).Result()\n\tif err != nil {\n\t\treturn 0, errors.E(op, errors.Internal, err)\n\t}\n\tn, ok := res.(int64)\n\tif !ok {\n\t\treturn 0, errors.E(op, errors.Internal, fmt.Sprintf(\"unexpected return value from script %v\", res))\n\t}\n\treturn n, nil\n}\n\n// archiveAllPendingCmd is a Lua script that moves all pending tasks from\n// the given queue to archived state.\n//\n// Input:\n// KEYS[1] -> asynq:{}:pending\n// KEYS[2] -> asynq:{}:archived\n// --\n// ARGV[1] -> current timestamp\n// ARGV[2] -> cutoff timestamp (e.g., 90 days ago)\n// ARGV[3] -> max number of tasks in archive (e.g., 100)\n// ARGV[4] -> task key prefix (asynq:{}:t:)\n//\n// Output:\n// integer: Number of tasks archived\nvar archiveAllPendingCmd = redis.NewScript(`\nlocal ids = redis.call(\"LRANGE\", KEYS[1], 0, -1)\nfor _, id in ipairs(ids) do\n\tredis.call(\"ZADD\", KEYS[2], ARGV[1], id)\n\tredis.call(\"HSET\", ARGV[4] .. id, \"state\", \"archived\")\nend\nredis.call(\"ZREMRANGEBYSCORE\", KEYS[2], \"-inf\", ARGV[2])\nredis.call(\"ZREMRANGEBYRANK\", KEYS[2], 0, -ARGV[3])\nredis.call(\"DEL\", KEYS[1])\nreturn table.getn(ids)`)\n\n// ArchiveAllPendingTasks archives all pending tasks from the given queue and\n// returns the number of tasks moved.\n// If a queue with the given name doesn't exist, it returns QueueNotFoundError.\nfunc (r *RDB) ArchiveAllPendingTasks(qname string) (int64, error) {\n\tvar op errors.Op = \"rdb.ArchiveAllPendingTasks\"\n\tif err := r.checkQueueExists(qname); err != nil {\n\t\treturn 0, errors.E(op, errors.CanonicalCode(err), err)\n\t}\n\tkeys := []string{\n\t\tbase.PendingKey(qname),\n\t\tbase.ArchivedKey(qname),\n\t}\n\tnow := r.clock.Now()\n\targv := []interface{}{\n\t\tnow.Unix(),\n\t\tnow.AddDate(0, 0, -archivedExpirationInDays).Unix(),\n\t\tmaxArchiveSize,\n\t\tbase.TaskKeyPrefix(qname),\n\t}\n\tres, err := archiveAllPendingCmd.Run(context.Background(), r.client, keys, argv...).Result()\n\tif err != nil {\n\t\treturn 0, errors.E(op, errors.Internal, err)\n\t}\n\tn, ok := res.(int64)\n\tif !ok {\n\t\treturn 0, errors.E(op, errors.Internal, fmt.Sprintf(\"unexpected return value from script %v\", res))\n\t}\n\treturn n, nil\n}\n\n// archiveTaskCmd is a Lua script that archives a task given a task id.\n//\n// Input:\n// KEYS[1] -> task key (asynq:{}:t:)\n// KEYS[2] -> archived key (asynq:{}:archived)\n// KEYS[3] -> all groups key (asynq:{}:groups)\n// --\n// ARGV[1] -> id of the task to archive\n// ARGV[2] -> current timestamp\n// ARGV[3] -> cutoff timestamp (e.g., 90 days ago)\n// ARGV[4] -> max number of tasks in archived state (e.g., 100)\n// ARGV[5] -> queue key prefix (asynq:{}:)\n// ARGV[6] -> group key prefix (asynq:{}:g:)\n//\n// Output:\n// Numeric code indicating the status:\n// Returns 1 if task is successfully archived.\n// Returns 0 if task is not found.\n// Returns -1 if task is already archived.\n// Returns -2 if task is in active state.\n// Returns error reply if unexpected error occurs.\nvar archiveTaskCmd = redis.NewScript(`\nif redis.call(\"EXISTS\", KEYS[1]) == 0 then\n\treturn 0\nend\nlocal state, group = unpack(redis.call(\"HMGET\", KEYS[1], \"state\", \"group\"))\nif state == \"active\" then\n\treturn -2\nend\nif state == \"archived\" then\n\treturn -1\nend\nif state == \"pending\" then\n\tif redis.call(\"LREM\", ARGV[5] .. state, 1, ARGV[1]) == 0 then\n\t\treturn redis.error_reply(\"task id not found in list \" .. tostring(ARGV[5] .. state))\n\tend\nelseif state == \"aggregating\" then\n\tif redis.call(\"ZREM\", ARGV[6] .. group, ARGV[1]) == 0 then\n\t\treturn redis.error_reply(\"task id not found in zset \" .. tostring(ARGV[6] .. group))\n\tend\n\tif redis.call(\"ZCARD\", ARGV[6] .. group) == 0 then\n\t\tredis.call(\"SREM\", KEYS[3], group)\n\tend\nelse\n\tif redis.call(\"ZREM\", ARGV[5] .. state, ARGV[1]) == 0 then\n\t\treturn redis.error_reply(\"task id not found in zset \" .. tostring(ARGV[5] .. state))\n\tend\nend\nredis.call(\"ZADD\", KEYS[2], ARGV[2], ARGV[1])\nredis.call(\"HSET\", KEYS[1], \"state\", \"archived\")\nredis.call(\"ZREMRANGEBYSCORE\", KEYS[2], \"-inf\", ARGV[3])\nredis.call(\"ZREMRANGEBYRANK\", KEYS[2], 0, -ARGV[4])\nreturn 1\n`)\n\n// ArchiveTask finds a task that matches the id from the given queue and archives it.\n// It returns nil if it successfully archived the task.\n//\n// If a queue with the given name doesn't exist, it returns QueueNotFoundError.\n// If a task with the given id doesn't exist in the queue, it returns TaskNotFoundError\n// If a task is already archived, it returns TaskAlreadyArchivedError.\n// If a task is in active state it returns non-nil error with FailedPrecondition code.\nfunc (r *RDB) ArchiveTask(qname, id string) error {\n\tvar op errors.Op = \"rdb.ArchiveTask\"\n\tif err := r.checkQueueExists(qname); err != nil {\n\t\treturn errors.E(op, errors.CanonicalCode(err), err)\n\t}\n\tkeys := []string{\n\t\tbase.TaskKey(qname, id),\n\t\tbase.ArchivedKey(qname),\n\t\tbase.AllGroups(qname),\n\t}\n\tnow := r.clock.Now()\n\targv := []interface{}{\n\t\tid,\n\t\tnow.Unix(),\n\t\tnow.AddDate(0, 0, -archivedExpirationInDays).Unix(),\n\t\tmaxArchiveSize,\n\t\tbase.QueueKeyPrefix(qname),\n\t\tbase.GroupKeyPrefix(qname),\n\t}\n\tres, err := archiveTaskCmd.Run(context.Background(), r.client, keys, argv...).Result()\n\tif err != nil {\n\t\treturn errors.E(op, errors.Unknown, err)\n\t}\n\tn, ok := res.(int64)\n\tif !ok {\n\t\treturn errors.E(op, errors.Internal, fmt.Sprintf(\"could not cast the return value %v from archiveTaskCmd to int64.\", res))\n\t}\n\tswitch n {\n\tcase 1:\n\t\treturn nil\n\tcase 0:\n\t\treturn errors.E(op, errors.NotFound, &errors.TaskNotFoundError{Queue: qname, ID: id})\n\tcase -1:\n\t\treturn errors.E(op, errors.FailedPrecondition, &errors.TaskAlreadyArchivedError{Queue: qname, ID: id})\n\tcase -2:\n\t\treturn errors.E(op, errors.FailedPrecondition, \"cannot archive task in active state. use CancelProcessing instead.\")\n\tcase -3:\n\t\treturn errors.E(op, errors.NotFound, &errors.QueueNotFoundError{Queue: qname})\n\tdefault:\n\t\treturn errors.E(op, errors.Internal, fmt.Sprintf(\"unexpected return value from archiveTaskCmd script: %d\", n))\n\t}\n}\n\n// archiveAllCmd is a Lua script that archives all tasks in either scheduled\n// or retry state from the given queue.\n//\n// Input:\n// KEYS[1] -> ZSET to move task from (e.g., asynq:{}:retry)\n// KEYS[2] -> asynq:{}:archived\n// --\n// ARGV[1] -> current timestamp\n// ARGV[2] -> cutoff timestamp (e.g., 90 days ago)\n// ARGV[3] -> max number of tasks in archive (e.g., 100)\n// ARGV[4] -> task key prefix (asynq:{}:t:)\n//\n// Output:\n// integer: number of tasks archived\nvar archiveAllCmd = redis.NewScript(`\nlocal ids = redis.call(\"ZRANGE\", KEYS[1], 0, -1)\nfor _, id in ipairs(ids) do\n\tredis.call(\"ZADD\", KEYS[2], ARGV[1], id)\n\tredis.call(\"HSET\", ARGV[4] .. id, \"state\", \"archived\")\nend\nredis.call(\"ZREMRANGEBYSCORE\", KEYS[2], \"-inf\", ARGV[2])\nredis.call(\"ZREMRANGEBYRANK\", KEYS[2], 0, -ARGV[3])\nredis.call(\"DEL\", KEYS[1])\nreturn table.getn(ids)`)\n\nfunc (r *RDB) archiveAll(src, dst, qname string) (int64, error) {\n\tif err := r.checkQueueExists(qname); err != nil {\n\t\treturn 0, err\n\t}\n\tkeys := []string{\n\t\tsrc,\n\t\tdst,\n\t}\n\tnow := r.clock.Now()\n\targv := []interface{}{\n\t\tnow.Unix(),\n\t\tnow.AddDate(0, 0, -archivedExpirationInDays).Unix(),\n\t\tmaxArchiveSize,\n\t\tbase.TaskKeyPrefix(qname),\n\t\tqname,\n\t}\n\tres, err := archiveAllCmd.Run(context.Background(), r.client, keys, argv...).Result()\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tn, ok := res.(int64)\n\tif !ok {\n\t\treturn 0, fmt.Errorf(\"unexpected return value from script: %v\", res)\n\t}\n\tif n == -1 {\n\t\treturn 0, &errors.QueueNotFoundError{Queue: qname}\n\t}\n\treturn n, nil\n}\n\n// Input:\n// KEYS[1] -> asynq:{}:t:\n// --\n// ARGV[1] -> task message data\n//\n// Output:\n// Numeric code indicating the status:\n// Returns 1 if task is successfully updated.\n// Returns 0 if task is not found.\n// Returns -1 if task is not in scheduled state.\nvar updateTaskPayloadCmd = redis.NewScript(`\n-- Check if given taks exists\nif redis.call(\"EXISTS\", KEYS[1]) == 0 then\n\treturn 0\nend\nlocal state, pending_since, group, unique_key = unpack(redis.call(\"HMGET\", KEYS[1], \"state\", \"pending_since\", \"group\", \"unique_key\"))\nif state ~= \"scheduled\" then\n return -1\nend\nlocal redis_call_args = {\"state\", state}\n\nif pending_since then\n table.insert(redis_call_args, \"pending_since\")\n table.insert(redis_call_args, pending_since)\nend\nif group then\n table.insert(redis_call_args, \"group\")\n table.insert(redis_call_args, group)\nend\nif unique_key then\n table.insert(redis_call_args, \"unique_key\")\n table.insert(redis_call_args, unique_key)\nend\nredis.call(\"HSET\", KEYS[1], \"msg\", ARGV[1], unpack(redis_call_args))\nreturn 1\n`)\n\n// UpdateTaskPayload finds a task that matches the id from the given queue and updates it's payload.\n// It returns nil if it successfully updated the task payload.\n//\n// If a queue with the given name doesn't exist, it returns QueueNotFoundError.\n// If a task with the given id doesn't exist in the queue, it returns TaskNotFoundError\n// If a task is in active state it returns non-nil error with Code FailedPrecondition.\nfunc (r *RDB) UpdateTaskPayload(qname, id string, payload []byte) error {\n\tvar op errors.Op = \"rdb.UpdateTask\"\n\tif err := r.checkQueueExists(qname); err != nil {\n\t\treturn errors.E(op, errors.CanonicalCode(err), err)\n\t}\n\n\ttaskInfo, err := r.GetTaskInfo(qname, id)\n\tif err != nil {\n\t\treturn errors.E(op, errors.Unknown, err)\n\t}\n\n\ttaskInfo.Message.Payload = payload\n\n\tencoded, err := base.EncodeMessage(taskInfo.Message)\n\tif err != nil {\n\t\treturn errors.E(op, errors.Unknown, fmt.Sprintf(\"cannot encode message: %v\", err))\n\t}\n\tkeys := []string{\n\t\tbase.TaskKey(qname, id),\n\t}\n\targv := []interface{}{\n\t\tencoded,\n\t}\n\n\tres, err := updateTaskPayloadCmd.Run(context.Background(), r.client, keys, argv...).Result()\n\tif err != nil {\n\t\treturn errors.E(op, errors.Unknown, err)\n\t}\n\tn, ok := res.(int64)\n\tif !ok {\n\t\treturn errors.E(op, errors.Internal, fmt.Sprintf(\"cast error: updateTaskCmd script returned unexported value %v\", res))\n\t}\n\tswitch n {\n\tcase 1:\n\t\treturn nil\n\tcase 0:\n\t\treturn errors.E(op, errors.NotFound, &errors.TaskNotFoundError{Queue: qname, ID: id})\n\tcase -1:\n\t\treturn errors.E(op, errors.FailedPrecondition, \"cannot update task that is not in scheduled state.\")\n\tdefault:\n\t\treturn errors.E(op, errors.Internal, fmt.Sprintf(\"unexpected return value from updateTaskCmd script: %d\", n))\n\t}\n}\n\n// Input:\n// KEYS[1] -> asynq:{}:t:\n// KEYS[2] -> asynq:{}:groups\n// --\n// ARGV[1] -> task ID\n// ARGV[2] -> queue key prefix\n// ARGV[3] -> group key prefix\n//\n// Output:\n// Numeric code indicating the status:\n// Returns 1 if task is successfully deleted.\n// Returns 0 if task is not found.\n// Returns -1 if task is in active state.\nvar deleteTaskCmd = redis.NewScript(`\nif redis.call(\"EXISTS\", KEYS[1]) == 0 then\n\treturn 0\nend\nlocal state, group = unpack(redis.call(\"HMGET\", KEYS[1], \"state\", \"group\"))\nif state == \"active\" then\n\treturn -1\nend\nif state == \"pending\" then\n\tif redis.call(\"LREM\", ARGV[2] .. state, 0, ARGV[1]) == 0 then\n\t\treturn redis.error_reply(\"task is not found in list: \" .. tostring(ARGV[2] .. state))\n\tend\nelseif state == \"aggregating\" then\n\tif redis.call(\"ZREM\", ARGV[3] .. group, ARGV[1]) == 0 then\n\t\treturn redis.error_reply(\"task is not found in zset: \" .. tostring(ARGV[3] .. group))\n\tend\n\tif redis.call(\"ZCARD\", ARGV[3] .. group) == 0 then\n\t\tredis.call(\"SREM\", KEYS[2], group)\n\tend\nelse\n\tif redis.call(\"ZREM\", ARGV[2] .. state, ARGV[1]) == 0 then\n\t\treturn redis.error_reply(\"task is not found in zset: \" .. tostring(ARGV[2] .. state))\n\tend\nend\nlocal unique_key = redis.call(\"HGET\", KEYS[1], \"unique_key\")\nif unique_key and unique_key ~= \"\" and redis.call(\"GET\", unique_key) == ARGV[1] then\n\tredis.call(\"DEL\", unique_key)\nend\nreturn redis.call(\"DEL\", KEYS[1])\n`)\n\n// DeleteTask finds a task that matches the id from the given queue and deletes it.\n// It returns nil if it successfully archived the task.\n//\n// If a queue with the given name doesn't exist, it returns QueueNotFoundError.\n// If a task with the given id doesn't exist in the queue, it returns TaskNotFoundError\n// If a task is in active state it returns non-nil error with Code FailedPrecondition.\nfunc (r *RDB) DeleteTask(qname, id string) error {\n\tvar op errors.Op = \"rdb.DeleteTask\"\n\tif err := r.checkQueueExists(qname); err != nil {\n\t\treturn errors.E(op, errors.CanonicalCode(err), err)\n\t}\n\tkeys := []string{\n\t\tbase.TaskKey(qname, id),\n\t\tbase.AllGroups(qname),\n\t}\n\targv := []interface{}{\n\t\tid,\n\t\tbase.QueueKeyPrefix(qname),\n\t\tbase.GroupKeyPrefix(qname),\n\t}\n\tres, err := deleteTaskCmd.Run(context.Background(), r.client, keys, argv...).Result()\n\tif err != nil {\n\t\treturn errors.E(op, errors.Unknown, err)\n\t}\n\tn, ok := res.(int64)\n\tif !ok {\n\t\treturn errors.E(op, errors.Internal, fmt.Sprintf(\"cast error: deleteTaskCmd script returned unexported value %v\", res))\n\t}\n\tswitch n {\n\tcase 1:\n\t\treturn nil\n\tcase 0:\n\t\treturn errors.E(op, errors.NotFound, &errors.TaskNotFoundError{Queue: qname, ID: id})\n\tcase -1:\n\t\treturn errors.E(op, errors.FailedPrecondition, \"cannot delete task in active state. use CancelProcessing instead.\")\n\tdefault:\n\t\treturn errors.E(op, errors.Internal, fmt.Sprintf(\"unexpected return value from deleteTaskCmd script: %d\", n))\n\t}\n}\n\n// DeleteAllArchivedTasks deletes all archived tasks from the given queue\n// and returns the number of tasks deleted.\nfunc (r *RDB) DeleteAllArchivedTasks(qname string) (int64, error) {\n\tvar op errors.Op = \"rdb.DeleteAllArchivedTasks\"\n\tn, err := r.deleteAll(base.ArchivedKey(qname), qname)\n\tif errors.IsQueueNotFound(err) {\n\t\treturn 0, errors.E(op, errors.NotFound, err)\n\t}\n\tif err != nil {\n\t\treturn 0, errors.E(op, errors.Unknown, err)\n\t}\n\treturn n, nil\n}\n\n// DeleteAllRetryTasks deletes all retry tasks from the given queue\n// and returns the number of tasks deleted.\nfunc (r *RDB) DeleteAllRetryTasks(qname string) (int64, error) {\n\tvar op errors.Op = \"rdb.DeleteAllRetryTasks\"\n\tn, err := r.deleteAll(base.RetryKey(qname), qname)\n\tif errors.IsQueueNotFound(err) {\n\t\treturn 0, errors.E(op, errors.NotFound, err)\n\t}\n\tif err != nil {\n\t\treturn 0, errors.E(op, errors.Unknown, err)\n\t}\n\treturn n, nil\n}\n\n// DeleteAllScheduledTasks deletes all scheduled tasks from the given queue\n// and returns the number of tasks deleted.\nfunc (r *RDB) DeleteAllScheduledTasks(qname string) (int64, error) {\n\tvar op errors.Op = \"rdb.DeleteAllScheduledTasks\"\n\tn, err := r.deleteAll(base.ScheduledKey(qname), qname)\n\tif errors.IsQueueNotFound(err) {\n\t\treturn 0, errors.E(op, errors.NotFound, err)\n\t}\n\tif err != nil {\n\t\treturn 0, errors.E(op, errors.Unknown, err)\n\t}\n\treturn n, nil\n}\n\n// DeleteAllCompletedTasks deletes all completed tasks from the given queue\n// and returns the number of tasks deleted.\nfunc (r *RDB) DeleteAllCompletedTasks(qname string) (int64, error) {\n\tvar op errors.Op = \"rdb.DeleteAllCompletedTasks\"\n\tn, err := r.deleteAll(base.CompletedKey(qname), qname)\n\tif errors.IsQueueNotFound(err) {\n\t\treturn 0, errors.E(op, errors.NotFound, err)\n\t}\n\tif err != nil {\n\t\treturn 0, errors.E(op, errors.Unknown, err)\n\t}\n\treturn n, nil\n}\n\n// deleteAllCmd deletes tasks from the given zset.\n//\n// Input:\n// KEYS[1] -> zset holding the task ids.\n// --\n// ARGV[1] -> task key prefix\n//\n// Output:\n// integer: number of tasks deleted\nvar deleteAllCmd = redis.NewScript(`\nlocal ids = redis.call(\"ZRANGE\", KEYS[1], 0, -1)\nfor _, id in ipairs(ids) do\n\tlocal task_key = ARGV[1] .. id\n\tlocal unique_key = redis.call(\"HGET\", task_key, \"unique_key\")\n\tif unique_key and unique_key ~= \"\" and redis.call(\"GET\", unique_key) == id then\n\t\tredis.call(\"DEL\", unique_key)\n\tend\n\tredis.call(\"DEL\", task_key)\nend\nredis.call(\"DEL\", KEYS[1])\nreturn table.getn(ids)`)\n\nfunc (r *RDB) deleteAll(key, qname string) (int64, error) {\n\tif err := r.checkQueueExists(qname); err != nil {\n\t\treturn 0, err\n\t}\n\targv := []interface{}{\n\t\tbase.TaskKeyPrefix(qname),\n\t\tqname,\n\t}\n\tres, err := deleteAllCmd.Run(context.Background(), r.client, []string{key}, argv...).Result()\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tn, ok := res.(int64)\n\tif !ok {\n\t\treturn 0, fmt.Errorf(\"unexpected return value from Lua script: %v\", res)\n\t}\n\treturn n, nil\n}\n\n// deleteAllAggregatingCmd deletes all tasks from the given group.\n//\n// Input:\n// KEYS[1] -> asynq:{}:g:\n// KEYS[2] -> asynq:{}:groups\n// -------\n// ARGV[1] -> task key prefix\n// ARGV[2] -> group name\nvar deleteAllAggregatingCmd = redis.NewScript(`\nlocal ids = redis.call(\"ZRANGE\", KEYS[1], 0, -1)\nfor _, id in ipairs(ids) do\n\tredis.call(\"DEL\", ARGV[1] .. id)\nend\nredis.call(\"SREM\", KEYS[2], ARGV[2])\nredis.call(\"DEL\", KEYS[1])\nreturn table.getn(ids)\n`)\n\n// DeleteAllAggregatingTasks deletes all aggregating tasks from the given group\n// and returns the number of tasks deleted.\nfunc (r *RDB) DeleteAllAggregatingTasks(qname, gname string) (int64, error) {\n\tvar op errors.Op = \"rdb.DeleteAllAggregatingTasks\"\n\tif err := r.checkQueueExists(qname); err != nil {\n\t\treturn 0, errors.E(op, errors.CanonicalCode(err), err)\n\t}\n\tkeys := []string{\n\t\tbase.GroupKey(qname, gname),\n\t\tbase.AllGroups(qname),\n\t}\n\targv := []interface{}{\n\t\tbase.TaskKeyPrefix(qname),\n\t\tgname,\n\t}\n\tres, err := deleteAllAggregatingCmd.Run(context.Background(), r.client, keys, argv...).Result()\n\tif err != nil {\n\t\treturn 0, errors.E(op, errors.Unknown, err)\n\t}\n\tn, ok := res.(int64)\n\tif !ok {\n\t\treturn 0, errors.E(op, errors.Internal, \"command error: unexpected return value %v\", res)\n\t}\n\treturn n, nil\n}\n\n// deleteAllPendingCmd deletes all pending tasks from the given queue.\n//\n// Input:\n// KEYS[1] -> asynq:{}:pending\n// --\n// ARGV[1] -> task key prefix\n//\n// Output:\n// integer: number of tasks deleted\nvar deleteAllPendingCmd = redis.NewScript(`\nlocal ids = redis.call(\"LRANGE\", KEYS[1], 0, -1)\nfor _, id in ipairs(ids) do\n\tredis.call(\"DEL\", ARGV[1] .. id)\nend\nredis.call(\"DEL\", KEYS[1])\nreturn table.getn(ids)`)\n\n// DeleteAllPendingTasks deletes all pending tasks from the given queue\n// and returns the number of tasks deleted.\nfunc (r *RDB) DeleteAllPendingTasks(qname string) (int64, error) {\n\tvar op errors.Op = \"rdb.DeleteAllPendingTasks\"\n\tif err := r.checkQueueExists(qname); err != nil {\n\t\treturn 0, errors.E(op, errors.CanonicalCode(err), err)\n\t}\n\tkeys := []string{\n\t\tbase.PendingKey(qname),\n\t}\n\targv := []interface{}{\n\t\tbase.TaskKeyPrefix(qname),\n\t}\n\tres, err := deleteAllPendingCmd.Run(context.Background(), r.client, keys, argv...).Result()\n\tif err != nil {\n\t\treturn 0, errors.E(op, errors.Unknown, err)\n\t}\n\tn, ok := res.(int64)\n\tif !ok {\n\t\treturn 0, errors.E(op, errors.Internal, \"command error: unexpected return value %v\", res)\n\t}\n\treturn n, nil\n}\n\n// removeQueueForceCmd removes the given queue regardless of\n// whether the queue is empty.\n// It only check whether active queue is empty before removing.\n//\n// Input:\n// KEYS[1] -> asynq:{}\n// KEYS[2] -> asynq:{}:active\n// KEYS[3] -> asynq:{}:scheduled\n// KEYS[4] -> asynq:{}:retry\n// KEYS[5] -> asynq:{}:archived\n// KEYS[6] -> asynq:{}:lease\n// --\n// ARGV[1] -> task key prefix\n//\n// Output:\n// Numeric code to indicate the status.\n// Returns 1 if successfully removed.\n// Returns -2 if the queue has active tasks.\nvar removeQueueForceCmd = redis.NewScript(`\nlocal active = redis.call(\"LLEN\", KEYS[2])\nif active > 0 then\n return -2\nend\nfor _, id in ipairs(redis.call(\"LRANGE\", KEYS[1], 0, -1)) do\n\tredis.call(\"DEL\", ARGV[1] .. id)\nend\nfor _, id in ipairs(redis.call(\"LRANGE\", KEYS[2], 0, -1)) do\n\tredis.call(\"DEL\", ARGV[1] .. id)\nend\nfor _, id in ipairs(redis.call(\"ZRANGE\", KEYS[3], 0, -1)) do\n\tredis.call(\"DEL\", ARGV[1] .. id)\nend\nfor _, id in ipairs(redis.call(\"ZRANGE\", KEYS[4], 0, -1)) do\n\tredis.call(\"DEL\", ARGV[1] .. id)\nend\nfor _, id in ipairs(redis.call(\"ZRANGE\", KEYS[5], 0, -1)) do\n\tredis.call(\"DEL\", ARGV[1] .. id)\nend\nfor _, id in ipairs(redis.call(\"LRANGE\", KEYS[1], 0, -1)) do\n\tredis.call(\"DEL\", ARGV[1] .. id)\nend\nfor _, id in ipairs(redis.call(\"LRANGE\", KEYS[2], 0, -1)) do\n\tredis.call(\"DEL\", ARGV[1] .. id)\nend\nfor _, id in ipairs(redis.call(\"ZRANGE\", KEYS[3], 0, -1)) do\n\tredis.call(\"DEL\", ARGV[1] .. id)\nend\nfor _, id in ipairs(redis.call(\"ZRANGE\", KEYS[4], 0, -1)) do\n\tredis.call(\"DEL\", ARGV[1] .. id)\nend\nfor _, id in ipairs(redis.call(\"ZRANGE\", KEYS[5], 0, -1)) do\n\tredis.call(\"DEL\", ARGV[1] .. id)\nend\nredis.call(\"DEL\", KEYS[1])\nredis.call(\"DEL\", KEYS[2])\nredis.call(\"DEL\", KEYS[3])\nredis.call(\"DEL\", KEYS[4])\nredis.call(\"DEL\", KEYS[5])\nredis.call(\"DEL\", KEYS[6])\nreturn 1`)\n\n// removeQueueCmd removes the given queue.\n// It checks whether queue is empty before removing.\n//\n// Input:\n// KEYS[1] -> asynq:{}:pending\n// KEYS[2] -> asynq:{}:active\n// KEYS[3] -> asynq:{}:scheduled\n// KEYS[4] -> asynq:{}:retry\n// KEYS[5] -> asynq:{}:archived\n// KEYS[6] -> asynq:{}:lease\n// --\n// ARGV[1] -> task key prefix\n//\n// Output:\n// Numeric code to indicate the status\n// Returns 1 if successfully removed.\n// Returns -1 if queue is not empty\nvar removeQueueCmd = redis.NewScript(`\nlocal ids = {}\nfor _, id in ipairs(redis.call(\"LRANGE\", KEYS[1], 0, -1)) do\n\ttable.insert(ids, id)\nend\nfor _, id in ipairs(redis.call(\"LRANGE\", KEYS[2], 0, -1)) do\n\ttable.insert(ids, id)\nend\nfor _, id in ipairs(redis.call(\"ZRANGE\", KEYS[3], 0, -1)) do\n\ttable.insert(ids, id)\nend\nfor _, id in ipairs(redis.call(\"ZRANGE\", KEYS[4], 0, -1)) do\n\ttable.insert(ids, id)\nend\nfor _, id in ipairs(redis.call(\"ZRANGE\", KEYS[5], 0, -1)) do\n\ttable.insert(ids, id)\nend\nif table.getn(ids) > 0 then\n\treturn -1\nend\nfor _, id in ipairs(ids) do\n\tredis.call(\"DEL\", ARGV[1] .. id)\nend\nfor _, id in ipairs(ids) do\n\tredis.call(\"DEL\", ARGV[1] .. id)\nend\nredis.call(\"DEL\", KEYS[1])\nredis.call(\"DEL\", KEYS[2])\nredis.call(\"DEL\", KEYS[3])\nredis.call(\"DEL\", KEYS[4])\nredis.call(\"DEL\", KEYS[5])\nredis.call(\"DEL\", KEYS[6])\nreturn 1`)\n\n// RemoveQueue removes the specified queue.\n//\n// If force is set to true, it will remove the queue regardless\n// as long as no tasks are active for the queue.\n// If force is set to false, it will only remove the queue if\n// the queue is empty.\nfunc (r *RDB) RemoveQueue(qname string, force bool) error {\n\tvar op errors.Op = \"rdb.RemoveQueue\"\n\texists, err := r.queueExists(qname)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif !exists {\n\t\treturn errors.E(op, errors.NotFound, &errors.QueueNotFoundError{Queue: qname})\n\t}\n\tvar script *redis.Script\n\tif force {\n\t\tscript = removeQueueForceCmd\n\t} else {\n\t\tscript = removeQueueCmd\n\t}\n\tkeys := []string{\n\t\tbase.PendingKey(qname),\n\t\tbase.ActiveKey(qname),\n\t\tbase.ScheduledKey(qname),\n\t\tbase.RetryKey(qname),\n\t\tbase.ArchivedKey(qname),\n\t\tbase.LeaseKey(qname),\n\t}\n\tres, err := script.Run(context.Background(), r.client, keys, base.TaskKeyPrefix(qname)).Result()\n\tif err != nil {\n\t\treturn errors.E(op, errors.Unknown, err)\n\t}\n\tn, ok := res.(int64)\n\tif !ok {\n\t\treturn errors.E(op, errors.Internal, fmt.Sprintf(\"unexpeced return value from Lua script: %v\", res))\n\t}\n\tswitch n {\n\tcase 1:\n\t\tif err := r.client.SRem(context.Background(), base.AllQueues, qname).Err(); err != nil {\n\t\t\treturn errors.E(op, errors.Unknown, err)\n\t\t}\n\t\tr.queuesPublished.Delete(qname)\n\t\treturn nil\n\tcase -1:\n\t\treturn errors.E(op, errors.NotFound, &errors.QueueNotEmptyError{Queue: qname})\n\tcase -2:\n\t\treturn errors.E(op, errors.FailedPrecondition, \"cannot remove queue with active tasks\")\n\tdefault:\n\t\treturn errors.E(op, errors.Unknown, fmt.Sprintf(\"unexpected return value from Lua script: %d\", n))\n\t}\n}\n\n// Note: Script also removes stale keys.\nvar listServerKeysCmd = redis.NewScript(`\nlocal now = tonumber(ARGV[1])\nlocal keys = redis.call(\"ZRANGEBYSCORE\", KEYS[1], now, \"+inf\")\nredis.call(\"ZREMRANGEBYSCORE\", KEYS[1], \"-inf\", now-1)\nreturn keys`)\n\n// ListServers returns the list of server info.\nfunc (r *RDB) ListServers() ([]*base.ServerInfo, error) {\n\tnow := r.clock.Now()\n\tres, err := listServerKeysCmd.Run(context.Background(), r.client, []string{base.AllServers}, now.Unix()).Result()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tkeys, err := cast.ToStringSliceE(res)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tvar servers []*base.ServerInfo\n\tfor _, key := range keys {\n\t\tdata, err := r.client.Get(context.Background(), key).Result()\n\t\tif err != nil {\n\t\t\tcontinue // skip bad data\n\t\t}\n\t\tinfo, err := base.DecodeServerInfo([]byte(data))\n\t\tif err != nil {\n\t\t\tcontinue // skip bad data\n\t\t}\n\t\tservers = append(servers, info)\n\t}\n\treturn servers, nil\n}\n\n// Note: Script also removes stale keys.\nvar listWorkersCmd = redis.NewScript(`\nlocal now = tonumber(ARGV[1])\nlocal keys = redis.call(\"ZRANGEBYSCORE\", KEYS[1], now, \"+inf\")\nredis.call(\"ZREMRANGEBYSCORE\", KEYS[1], \"-inf\", now-1)\nreturn keys`)\n\n// ListWorkers returns the list of worker stats.\nfunc (r *RDB) ListWorkers() ([]*base.WorkerInfo, error) {\n\tvar op errors.Op = \"rdb.ListWorkers\"\n\tnow := r.clock.Now()\n\tres, err := listWorkersCmd.Run(context.Background(), r.client, []string{base.AllWorkers}, now.Unix()).Result()\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.Unknown, err)\n\t}\n\tkeys, err := cast.ToStringSliceE(res)\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.Internal, fmt.Sprintf(\"unexpeced return value from Lua script: %v\", res))\n\t}\n\tvar workers []*base.WorkerInfo\n\tfor _, key := range keys {\n\t\tdata, err := r.client.HVals(context.Background(), key).Result()\n\t\tif err != nil {\n\t\t\tcontinue // skip bad data\n\t\t}\n\t\tfor _, s := range data {\n\t\t\tw, err := base.DecodeWorkerInfo([]byte(s))\n\t\t\tif err != nil {\n\t\t\t\tcontinue // skip bad data\n\t\t\t}\n\t\t\tworkers = append(workers, w)\n\t\t}\n\t}\n\treturn workers, nil\n}\n\n// Note: Script also removes stale keys.\nvar listSchedulerKeysCmd = redis.NewScript(`\nlocal now = tonumber(ARGV[1])\nlocal keys = redis.call(\"ZRANGEBYSCORE\", KEYS[1], now, \"+inf\")\nredis.call(\"ZREMRANGEBYSCORE\", KEYS[1], \"-inf\", now-1)\nreturn keys`)\n\n// ListSchedulerEntries returns the list of scheduler entries.\nfunc (r *RDB) ListSchedulerEntries() ([]*base.SchedulerEntry, error) {\n\tnow := r.clock.Now()\n\tres, err := listSchedulerKeysCmd.Run(context.Background(), r.client, []string{base.AllSchedulers}, now.Unix()).Result()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tkeys, err := cast.ToStringSliceE(res)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tvar entries []*base.SchedulerEntry\n\tfor _, key := range keys {\n\t\tdata, err := r.client.LRange(context.Background(), key, 0, -1).Result()\n\t\tif err != nil {\n\t\t\tcontinue // skip bad data\n\t\t}\n\t\tfor _, s := range data {\n\t\t\te, err := base.DecodeSchedulerEntry([]byte(s))\n\t\t\tif err != nil {\n\t\t\t\tcontinue // skip bad data\n\t\t\t}\n\t\t\tentries = append(entries, e)\n\t\t}\n\t}\n\treturn entries, nil\n}\n\n// ListSchedulerEnqueueEvents returns the list of scheduler enqueue events.\nfunc (r *RDB) ListSchedulerEnqueueEvents(entryID string, pgn Pagination) ([]*base.SchedulerEnqueueEvent, error) {\n\tkey := base.SchedulerHistoryKey(entryID)\n\tzs, err := r.client.ZRevRangeWithScores(context.Background(), key, pgn.start(), pgn.stop()).Result()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tvar events []*base.SchedulerEnqueueEvent\n\tfor _, z := range zs {\n\t\tdata, err := cast.ToStringE(z.Member)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\te, err := base.DecodeSchedulerEnqueueEvent([]byte(data))\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tevents = append(events, e)\n\t}\n\treturn events, nil\n}\n\n// Pause pauses processing of tasks from the given queue.\nfunc (r *RDB) Pause(qname string) error {\n\tkey := base.PausedKey(qname)\n\tok, err := r.client.SetNX(context.Background(), key, r.clock.Now().Unix(), 0).Result()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif !ok {\n\t\treturn fmt.Errorf(\"queue %q is already paused\", qname)\n\t}\n\treturn nil\n}\n\n// Unpause resumes processing of tasks from the given queue.\nfunc (r *RDB) Unpause(qname string) error {\n\tkey := base.PausedKey(qname)\n\tdeleted, err := r.client.Del(context.Background(), key).Result()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif deleted == 0 {\n\t\treturn fmt.Errorf(\"queue %q is not paused\", qname)\n\t}\n\treturn nil\n}\n\n// ClusterKeySlot returns an integer identifying the hash slot the given queue hashes to.\nfunc (r *RDB) ClusterKeySlot(qname string) (int64, error) {\n\tkey := base.PendingKey(qname)\n\treturn r.client.ClusterKeySlot(context.Background(), key).Result()\n}\n\n// ClusterNodes returns a list of nodes the given queue belongs to.\nfunc (r *RDB) ClusterNodes(qname string) ([]redis.ClusterNode, error) {\n\tkeyslot, err := r.ClusterKeySlot(qname)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tclusterSlots, err := r.client.ClusterSlots(context.Background()).Result()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tfor _, slotRange := range clusterSlots {\n\t\tif int64(slotRange.Start) <= keyslot && keyslot <= int64(slotRange.End) {\n\t\t\treturn slotRange.Nodes, nil\n\t\t}\n\t}\n\treturn nil, fmt.Errorf(\"nodes not found\")\n}\n" }, { "path": "internal/rdb/inspect_test.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage rdb\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"sort\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/google/go-cmp/cmp\"\n\t\"github.com/google/go-cmp/cmp/cmpopts\"\n\t\"github.com/google/uuid\"\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/errors\"\n\th \"github.com/hibiken/asynq/internal/testutil\"\n\t\"github.com/hibiken/asynq/internal/timeutil\"\n\t\"github.com/redis/go-redis/v9\"\n)\n\nfunc TestAllQueues(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\ttests := []struct {\n\t\tqueues []string\n\t}{\n\t\t{queues: []string{\"default\"}},\n\t\t{queues: []string{\"custom1\", \"custom2\"}},\n\t\t{queues: []string{\"default\", \"custom1\", \"custom2\"}},\n\t\t{queues: []string{}},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\tfor _, qname := range tc.queues {\n\t\t\tif err := r.client.SAdd(context.Background(), base.AllQueues, qname).Err(); err != nil {\n\t\t\t\tt.Fatalf(\"could not initialize all queue set: %v\", err)\n\t\t\t}\n\t\t}\n\t\tgot, err := r.AllQueues()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"AllQueues() returned an error: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tif diff := cmp.Diff(tc.queues, got, h.SortStringSliceOpt); diff != \"\" {\n\t\t\tt.Errorf(\"AllQueues() = %v, want %v; (-want, +got)\\n%s\", got, tc.queues, diff)\n\t\t}\n\t}\n}\n\nfunc TestCurrentStats(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tm1 := h.NewTaskMessageBuilder().SetType(\"send_email\").Build()\n\tm2 := h.NewTaskMessageBuilder().SetType(\"reindex\").Build()\n\tm3 := h.NewTaskMessageBuilder().SetType(\"gen_thumbnail\").Build()\n\tm4 := h.NewTaskMessageBuilder().SetType(\"sync\").Build()\n\tm5 := h.NewTaskMessageBuilder().SetType(\"important_notification\").SetQueue(\"critical\").Build()\n\tm6 := h.NewTaskMessageBuilder().SetType(\"minor_notification\").SetQueue(\"low\").Build()\n\tm7 := h.NewTaskMessageBuilder().SetType(\"send_sms\").Build()\n\tnow := time.Now()\n\tr.SetClock(timeutil.NewSimulatedClock(now))\n\n\ttests := []struct {\n\t\ttasks []*h.TaskSeedData\n\t\tallQueues []string\n\t\tallGroups map[string][]string\n\t\tpending map[string][]string\n\t\tactive map[string][]string\n\t\tscheduled map[string][]redis.Z\n\t\tretry map[string][]redis.Z\n\t\tarchived map[string][]redis.Z\n\t\tcompleted map[string][]redis.Z\n\t\tgroups map[string][]redis.Z\n\t\tprocessed map[string]int\n\t\tfailed map[string]int\n\t\tprocessedTotal map[string]int\n\t\tfailedTotal map[string]int\n\t\tpaused []string\n\t\toldestPendingMessageEnqueueTime map[string]time.Time\n\t\tqname string\n\t\twant *Stats\n\t}{\n\t\t{\n\t\t\ttasks: []*h.TaskSeedData{\n\t\t\t\t{Msg: m1, State: base.TaskStatePending},\n\t\t\t\t{Msg: m2, State: base.TaskStateActive},\n\t\t\t\t{Msg: m3, State: base.TaskStateScheduled},\n\t\t\t\t{Msg: m4, State: base.TaskStateScheduled},\n\t\t\t\t{Msg: m5, State: base.TaskStatePending},\n\t\t\t\t{Msg: m6, State: base.TaskStatePending},\n\t\t\t\t{Msg: m7, State: base.TaskStateAggregating},\n\t\t\t},\n\t\t\tallQueues: []string{\"default\", \"critical\", \"low\"},\n\t\t\tallGroups: map[string][]string{\n\t\t\t\tbase.AllGroups(\"default\"): {\"sms:user1\"},\n\t\t\t},\n\t\t\tpending: map[string][]string{\n\t\t\t\tbase.PendingKey(\"default\"): {m1.ID},\n\t\t\t\tbase.PendingKey(\"critical\"): {m5.ID},\n\t\t\t\tbase.PendingKey(\"low\"): {m6.ID},\n\t\t\t},\n\t\t\tactive: map[string][]string{\n\t\t\t\tbase.ActiveKey(\"default\"): {m2.ID},\n\t\t\t\tbase.ActiveKey(\"critical\"): {},\n\t\t\t\tbase.ActiveKey(\"low\"): {},\n\t\t\t},\n\t\t\tscheduled: map[string][]redis.Z{\n\t\t\t\tbase.ScheduledKey(\"default\"): {\n\t\t\t\t\t{Member: m3.ID, Score: float64(now.Add(time.Hour).Unix())},\n\t\t\t\t\t{Member: m4.ID, Score: float64(now.Unix())},\n\t\t\t\t},\n\t\t\t\tbase.ScheduledKey(\"critical\"): {},\n\t\t\t\tbase.ScheduledKey(\"low\"): {},\n\t\t\t},\n\t\t\tretry: map[string][]redis.Z{\n\t\t\t\tbase.RetryKey(\"default\"): {},\n\t\t\t\tbase.RetryKey(\"critical\"): {},\n\t\t\t\tbase.RetryKey(\"low\"): {},\n\t\t\t},\n\t\t\tarchived: map[string][]redis.Z{\n\t\t\t\tbase.ArchivedKey(\"default\"): {},\n\t\t\t\tbase.ArchivedKey(\"critical\"): {},\n\t\t\t\tbase.ArchivedKey(\"low\"): {},\n\t\t\t},\n\t\t\tcompleted: map[string][]redis.Z{\n\t\t\t\tbase.CompletedKey(\"default\"): {},\n\t\t\t\tbase.CompletedKey(\"critical\"): {},\n\t\t\t\tbase.CompletedKey(\"low\"): {},\n\t\t\t},\n\t\t\tgroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"sms:user1\"): {\n\t\t\t\t\t{Member: m7.ID, Score: float64(now.Add(-3 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\tprocessed: map[string]int{\n\t\t\t\t\"default\": 120,\n\t\t\t\t\"critical\": 100,\n\t\t\t\t\"low\": 50,\n\t\t\t},\n\t\t\tfailed: map[string]int{\n\t\t\t\t\"default\": 2,\n\t\t\t\t\"critical\": 0,\n\t\t\t\t\"low\": 1,\n\t\t\t},\n\t\t\tprocessedTotal: map[string]int{\n\t\t\t\t\"default\": 11111,\n\t\t\t\t\"critical\": 22222,\n\t\t\t\t\"low\": 33333,\n\t\t\t},\n\t\t\tfailedTotal: map[string]int{\n\t\t\t\t\"default\": 111,\n\t\t\t\t\"critical\": 222,\n\t\t\t\t\"low\": 333,\n\t\t\t},\n\t\t\toldestPendingMessageEnqueueTime: map[string]time.Time{\n\t\t\t\t\"default\": now.Add(-15 * time.Second),\n\t\t\t\t\"critical\": now.Add(-200 * time.Millisecond),\n\t\t\t\t\"low\": now.Add(-30 * time.Second),\n\t\t\t},\n\t\t\tpaused: []string{},\n\t\t\tqname: \"default\",\n\t\t\twant: &Stats{\n\t\t\t\tQueue: \"default\",\n\t\t\t\tPaused: false,\n\t\t\t\tSize: 5,\n\t\t\t\tGroups: 1,\n\t\t\t\tPending: 1,\n\t\t\t\tActive: 1,\n\t\t\t\tScheduled: 2,\n\t\t\t\tRetry: 0,\n\t\t\t\tArchived: 0,\n\t\t\t\tCompleted: 0,\n\t\t\t\tAggregating: 1,\n\t\t\t\tProcessed: 120,\n\t\t\t\tFailed: 2,\n\t\t\t\tProcessedTotal: 11111,\n\t\t\t\tFailedTotal: 111,\n\t\t\t\tLatency: 15 * time.Second,\n\t\t\t\tTimestamp: now,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\ttasks: []*h.TaskSeedData{\n\t\t\t\t{Msg: m1, State: base.TaskStatePending},\n\t\t\t\t{Msg: m2, State: base.TaskStateActive},\n\t\t\t\t{Msg: m3, State: base.TaskStateScheduled},\n\t\t\t\t{Msg: m4, State: base.TaskStateScheduled},\n\t\t\t\t{Msg: m6, State: base.TaskStatePending},\n\t\t\t},\n\t\t\tallQueues: []string{\"default\", \"critical\", \"low\"},\n\t\t\tpending: map[string][]string{\n\t\t\t\tbase.PendingKey(\"default\"): {m1.ID},\n\t\t\t\tbase.PendingKey(\"critical\"): {},\n\t\t\t\tbase.PendingKey(\"low\"): {m6.ID},\n\t\t\t},\n\t\t\tactive: map[string][]string{\n\t\t\t\tbase.ActiveKey(\"default\"): {m2.ID},\n\t\t\t\tbase.ActiveKey(\"critical\"): {},\n\t\t\t\tbase.ActiveKey(\"low\"): {},\n\t\t\t},\n\t\t\tscheduled: map[string][]redis.Z{\n\t\t\t\tbase.ScheduledKey(\"default\"): {\n\t\t\t\t\t{Member: m3.ID, Score: float64(now.Add(time.Hour).Unix())},\n\t\t\t\t\t{Member: m4.ID, Score: float64(now.Unix())},\n\t\t\t\t},\n\t\t\t\tbase.ScheduledKey(\"critical\"): {},\n\t\t\t\tbase.ScheduledKey(\"low\"): {},\n\t\t\t},\n\t\t\tretry: map[string][]redis.Z{\n\t\t\t\tbase.RetryKey(\"default\"): {},\n\t\t\t\tbase.RetryKey(\"critical\"): {},\n\t\t\t\tbase.RetryKey(\"low\"): {},\n\t\t\t},\n\t\t\tarchived: map[string][]redis.Z{\n\t\t\t\tbase.ArchivedKey(\"default\"): {},\n\t\t\t\tbase.ArchivedKey(\"critical\"): {},\n\t\t\t\tbase.ArchivedKey(\"low\"): {},\n\t\t\t},\n\t\t\tcompleted: map[string][]redis.Z{\n\t\t\t\tbase.CompletedKey(\"default\"): {},\n\t\t\t\tbase.CompletedKey(\"critical\"): {},\n\t\t\t\tbase.CompletedKey(\"low\"): {},\n\t\t\t},\n\t\t\tprocessed: map[string]int{\n\t\t\t\t\"default\": 120,\n\t\t\t\t\"critical\": 100,\n\t\t\t\t\"low\": 50,\n\t\t\t},\n\t\t\tfailed: map[string]int{\n\t\t\t\t\"default\": 2,\n\t\t\t\t\"critical\": 0,\n\t\t\t\t\"low\": 1,\n\t\t\t},\n\t\t\tprocessedTotal: map[string]int{\n\t\t\t\t\"default\": 11111,\n\t\t\t\t\"critical\": 22222,\n\t\t\t\t\"low\": 33333,\n\t\t\t},\n\t\t\tfailedTotal: map[string]int{\n\t\t\t\t\"default\": 111,\n\t\t\t\t\"critical\": 222,\n\t\t\t\t\"low\": 333,\n\t\t\t},\n\t\t\toldestPendingMessageEnqueueTime: map[string]time.Time{\n\t\t\t\t\"default\": now.Add(-15 * time.Second),\n\t\t\t\t\"critical\": {}, // zero value since there's no pending task in this queue\n\t\t\t\t\"low\": now.Add(-30 * time.Second),\n\t\t\t},\n\t\t\tpaused: []string{\"critical\", \"low\"},\n\t\t\tqname: \"critical\",\n\t\t\twant: &Stats{\n\t\t\t\tQueue: \"critical\",\n\t\t\t\tPaused: true,\n\t\t\t\tSize: 0,\n\t\t\t\tGroups: 0,\n\t\t\t\tPending: 0,\n\t\t\t\tActive: 0,\n\t\t\t\tScheduled: 0,\n\t\t\t\tRetry: 0,\n\t\t\t\tArchived: 0,\n\t\t\t\tCompleted: 0,\n\t\t\t\tAggregating: 0,\n\t\t\t\tProcessed: 100,\n\t\t\t\tFailed: 0,\n\t\t\t\tProcessedTotal: 22222,\n\t\t\t\tFailedTotal: 222,\n\t\t\t\tLatency: 0,\n\t\t\t\tTimestamp: now,\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\tfor _, qname := range tc.paused {\n\t\t\tif err := r.Pause(qname); err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t}\n\t\th.SeedRedisSet(t, r.client, base.AllQueues, tc.allQueues)\n\t\th.SeedRedisSets(t, r.client, tc.allGroups)\n\t\th.SeedTasks(t, r.client, tc.tasks)\n\t\th.SeedRedisLists(t, r.client, tc.pending)\n\t\th.SeedRedisLists(t, r.client, tc.active)\n\t\th.SeedRedisZSets(t, r.client, tc.scheduled)\n\t\th.SeedRedisZSets(t, r.client, tc.retry)\n\t\th.SeedRedisZSets(t, r.client, tc.archived)\n\t\th.SeedRedisZSets(t, r.client, tc.completed)\n\t\th.SeedRedisZSets(t, r.client, tc.groups)\n\t\tctx := context.Background()\n\t\tfor qname, n := range tc.processed {\n\t\t\tr.client.Set(ctx, base.ProcessedKey(qname, now), n, 0)\n\t\t}\n\t\tfor qname, n := range tc.failed {\n\t\t\tr.client.Set(ctx, base.FailedKey(qname, now), n, 0)\n\t\t}\n\t\tfor qname, n := range tc.processedTotal {\n\t\t\tr.client.Set(ctx, base.ProcessedTotalKey(qname), n, 0)\n\t\t}\n\t\tfor qname, n := range tc.failedTotal {\n\t\t\tr.client.Set(ctx, base.FailedTotalKey(qname), n, 0)\n\t\t}\n\t\tfor qname, enqueueTime := range tc.oldestPendingMessageEnqueueTime {\n\t\t\tif enqueueTime.IsZero() {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\toldestPendingMessageID := r.client.LRange(ctx, base.PendingKey(qname), -1, -1).Val()[0] // get the right most msg in the list\n\t\t\tr.client.HSet(ctx, base.TaskKey(qname, oldestPendingMessageID), \"pending_since\", enqueueTime.UnixNano())\n\t\t}\n\n\t\tgot, err := r.CurrentStats(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"r.CurrentStats(%q) = %v, %v, want %v, nil\", tc.qname, got, err, tc.want)\n\t\t\tcontinue\n\t\t}\n\n\t\tignoreMemUsg := cmpopts.IgnoreFields(Stats{}, \"MemoryUsage\")\n\t\tif diff := cmp.Diff(tc.want, got, timeCmpOpt, ignoreMemUsg); diff != \"\" {\n\t\t\tt.Errorf(\"r.CurrentStats(%q) = %v, %v, want %v, nil; (-want, +got)\\n%s\", tc.qname, got, err, tc.want, diff)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\nfunc TestCurrentStatsWithNonExistentQueue(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tqname := \"non-existent\"\n\tgot, err := r.CurrentStats(qname)\n\tif !errors.IsQueueNotFound(err) {\n\t\tt.Fatalf(\"r.CurrentStats(%q) = %v, %v, want nil, %v\", qname, got, err, &errors.QueueNotFoundError{Queue: qname})\n\t}\n}\n\nfunc TestHistoricalStats(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tnow := time.Now().UTC()\n\n\ttests := []struct {\n\t\tqname string // queue of interest\n\t\tn int // number of days\n\t}{\n\t\t{\"default\", 90},\n\t\t{\"custom\", 7},\n\t\t{\"default\", 1},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\n\t\tr.client.SAdd(context.Background(), base.AllQueues, tc.qname)\n\t\t// populate last n days data\n\t\tfor i := 0; i < tc.n; i++ {\n\t\t\tts := now.Add(-time.Duration(i) * 24 * time.Hour)\n\t\t\tprocessedKey := base.ProcessedKey(tc.qname, ts)\n\t\t\tfailedKey := base.FailedKey(tc.qname, ts)\n\t\t\tr.client.Set(context.Background(), processedKey, (i+1)*1000, 0)\n\t\t\tr.client.Set(context.Background(), failedKey, (i+1)*10, 0)\n\t\t}\n\n\t\tgot, err := r.HistoricalStats(tc.qname, tc.n)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"RDB.HistoricalStats(%q, %d) returned error: %v\", tc.qname, tc.n, err)\n\t\t\tcontinue\n\t\t}\n\n\t\tif len(got) != tc.n {\n\t\t\tt.Errorf(\"RDB.HistorycalStats(%q, %d) returned %d daily stats, want %d\", tc.qname, tc.n, len(got), tc.n)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor i := 0; i < tc.n; i++ {\n\t\t\twant := &DailyStats{\n\t\t\t\tQueue: tc.qname,\n\t\t\t\tProcessed: (i + 1) * 1000,\n\t\t\t\tFailed: (i + 1) * 10,\n\t\t\t\tTime: now.Add(-time.Duration(i) * 24 * time.Hour),\n\t\t\t}\n\t\t\t// Allow 2 seconds difference in timestamp.\n\t\t\tcmpOpt := cmpopts.EquateApproxTime(2 * time.Second)\n\t\t\tif diff := cmp.Diff(want, got[i], cmpOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"RDB.HistoricalStats for the last %d days; got %+v, want %+v; (-want,+got):\\n%s\", i, got[i], want, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestRedisInfo(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tinfo, err := r.RedisInfo()\n\tif err != nil {\n\t\tt.Fatalf(\"RDB.RedisInfo() returned error: %v\", err)\n\t}\n\n\twantKeys := []string{\n\t\t\"redis_version\",\n\t\t\"uptime_in_days\",\n\t\t\"connected_clients\",\n\t\t\"used_memory_human\",\n\t\t\"used_memory_peak_human\",\n\t\t\"used_memory_peak_perc\",\n\t}\n\n\tfor _, key := range wantKeys {\n\t\tif _, ok := info[key]; !ok {\n\t\t\tt.Errorf(\"RDB.RedisInfo() = %v is missing entry for %q\", info, key)\n\t\t}\n\t}\n}\n\nfunc TestGroupStats(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tm1 := h.NewTaskMessageBuilder().SetGroup(\"group1\").Build()\n\tm2 := h.NewTaskMessageBuilder().SetGroup(\"group1\").Build()\n\tm3 := h.NewTaskMessageBuilder().SetGroup(\"group1\").Build()\n\tm4 := h.NewTaskMessageBuilder().SetGroup(\"group2\").Build()\n\tm5 := h.NewTaskMessageBuilder().SetQueue(\"custom\").SetGroup(\"group1\").Build()\n\tm6 := h.NewTaskMessageBuilder().SetQueue(\"custom\").SetGroup(\"group1\").Build()\n\n\tnow := time.Now()\n\n\tfixtures := struct {\n\t\ttasks []*h.TaskSeedData\n\t\tallGroups map[string][]string\n\t\tgroups map[string][]redis.Z\n\t}{\n\t\ttasks: []*h.TaskSeedData{\n\t\t\t{Msg: m1, State: base.TaskStateAggregating},\n\t\t\t{Msg: m2, State: base.TaskStateAggregating},\n\t\t\t{Msg: m3, State: base.TaskStateAggregating},\n\t\t\t{Msg: m4, State: base.TaskStateAggregating},\n\t\t\t{Msg: m5, State: base.TaskStateAggregating},\n\t\t},\n\t\tallGroups: map[string][]string{\n\t\t\tbase.AllGroups(\"default\"): {\"group1\", \"group2\"},\n\t\t\tbase.AllGroups(\"custom\"): {\"group1\"},\n\t\t},\n\t\tgroups: map[string][]redis.Z{\n\t\t\tbase.GroupKey(\"default\", \"group1\"): {\n\t\t\t\t{Member: m1.ID, Score: float64(now.Add(-10 * time.Second).Unix())},\n\t\t\t\t{Member: m2.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t\t{Member: m3.ID, Score: float64(now.Add(-30 * time.Second).Unix())},\n\t\t\t},\n\t\t\tbase.GroupKey(\"default\", \"group2\"): {\n\t\t\t\t{Member: m4.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t},\n\t\t\tbase.GroupKey(\"custom\", \"group1\"): {\n\t\t\t\t{Member: m5.ID, Score: float64(now.Add(-10 * time.Second).Unix())},\n\t\t\t\t{Member: m6.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t},\n\t\t},\n\t}\n\n\ttests := []struct {\n\t\tdesc string\n\t\tqname string\n\t\twant []*GroupStat\n\t}{\n\t\t{\n\t\t\tdesc: \"default queue groups\",\n\t\t\tqname: \"default\",\n\t\t\twant: []*GroupStat{\n\t\t\t\t{Group: \"group1\", Size: 3},\n\t\t\t\t{Group: \"group2\", Size: 1},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"custom queue groups\",\n\t\t\tqname: \"custom\",\n\t\t\twant: []*GroupStat{\n\t\t\t\t{Group: \"group1\", Size: 2},\n\t\t\t},\n\t\t},\n\t}\n\n\tsortGroupStatsOpt := cmp.Transformer(\n\t\t\"SortGroupStats\",\n\t\tfunc(in []*GroupStat) []*GroupStat {\n\t\t\tout := append([]*GroupStat(nil), in...)\n\t\t\tsort.Slice(out, func(i, j int) bool {\n\t\t\t\treturn out[i].Group < out[j].Group\n\t\t\t})\n\t\t\treturn out\n\t\t})\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\th.SeedTasks(t, r.client, fixtures.tasks)\n\t\th.SeedRedisSets(t, r.client, fixtures.allGroups)\n\t\th.SeedRedisZSets(t, r.client, fixtures.groups)\n\n\t\tt.Run(tc.desc, func(t *testing.T) {\n\t\t\tgot, err := r.GroupStats(tc.qname)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"GroupStats returned error: %v\", err)\n\t\t\t}\n\t\t\tif diff := cmp.Diff(tc.want, got, sortGroupStatsOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"GroupStats = %v, want %v; (-want,+got)\\n%s\", got, tc.want, diff)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestGetTaskInfo(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tnow := time.Now()\n\tfiveMinsFromNow := now.Add(5 * time.Minute)\n\toneHourFromNow := now.Add(1 * time.Hour)\n\ttwoHoursAgo := now.Add(-2 * time.Hour)\n\n\tm1 := h.NewTaskMessageWithQueue(\"task1\", nil, \"default\")\n\tm2 := h.NewTaskMessageWithQueue(\"task2\", nil, \"default\")\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tm4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"custom\")\n\tm5 := h.NewTaskMessageWithQueue(\"task5\", nil, \"custom\")\n\tm6 := h.NewTaskMessageWithQueue(\"task5\", nil, \"custom\")\n\tm6.CompletedAt = twoHoursAgo.Unix()\n\tm6.Retention = int64((24 * time.Hour).Seconds())\n\n\tfixtures := struct {\n\t\tactive map[string][]*base.TaskMessage\n\t\tpending map[string][]*base.TaskMessage\n\t\tscheduled map[string][]base.Z\n\t\tretry map[string][]base.Z\n\t\tarchived map[string][]base.Z\n\t\tcompleted map[string][]base.Z\n\t}{\n\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\"default\": {m1},\n\t\t\t\"custom\": {},\n\t\t},\n\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\"default\": {},\n\t\t\t\"custom\": {m5},\n\t\t},\n\t\tscheduled: map[string][]base.Z{\n\t\t\t\"default\": {{Message: m2, Score: fiveMinsFromNow.Unix()}},\n\t\t\t\"custom\": {},\n\t\t},\n\t\tretry: map[string][]base.Z{\n\t\t\t\"default\": {},\n\t\t\t\"custom\": {{Message: m3, Score: oneHourFromNow.Unix()}},\n\t\t},\n\t\tarchived: map[string][]base.Z{\n\t\t\t\"default\": {},\n\t\t\t\"custom\": {{Message: m4, Score: twoHoursAgo.Unix()}},\n\t\t},\n\t\tcompleted: map[string][]base.Z{\n\t\t\t\"default\": {},\n\t\t\t\"custom\": {{Message: m6, Score: m6.CompletedAt + m6.Retention}},\n\t\t},\n\t}\n\n\th.SeedAllActiveQueues(t, r.client, fixtures.active)\n\th.SeedAllPendingQueues(t, r.client, fixtures.pending)\n\th.SeedAllScheduledQueues(t, r.client, fixtures.scheduled)\n\th.SeedAllRetryQueues(t, r.client, fixtures.retry)\n\th.SeedAllArchivedQueues(t, r.client, fixtures.archived)\n\th.SeedAllCompletedQueues(t, r.client, fixtures.completed)\n\t// Write result data for the completed task.\n\tif err := r.client.HSet(context.Background(), base.TaskKey(m6.Queue, m6.ID), \"result\", \"foobar\").Err(); err != nil {\n\t\tt.Fatalf(\"Failed to write result data under task key: %v\", err)\n\t}\n\n\ttests := []struct {\n\t\tqname string\n\t\tid string\n\t\twant *base.TaskInfo\n\t}{\n\t\t{\n\t\t\tqname: \"default\",\n\t\t\tid: m1.ID,\n\t\t\twant: &base.TaskInfo{\n\t\t\t\tMessage: m1,\n\t\t\t\tState: base.TaskStateActive,\n\t\t\t\tNextProcessAt: time.Time{}, // zero value for N/A\n\t\t\t\tResult: nil,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tqname: \"default\",\n\t\t\tid: m2.ID,\n\t\t\twant: &base.TaskInfo{\n\t\t\t\tMessage: m2,\n\t\t\t\tState: base.TaskStateScheduled,\n\t\t\t\tNextProcessAt: fiveMinsFromNow,\n\t\t\t\tResult: nil,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tqname: \"custom\",\n\t\t\tid: m3.ID,\n\t\t\twant: &base.TaskInfo{\n\t\t\t\tMessage: m3,\n\t\t\t\tState: base.TaskStateRetry,\n\t\t\t\tNextProcessAt: oneHourFromNow,\n\t\t\t\tResult: nil,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tqname: \"custom\",\n\t\t\tid: m4.ID,\n\t\t\twant: &base.TaskInfo{\n\t\t\t\tMessage: m4,\n\t\t\t\tState: base.TaskStateArchived,\n\t\t\t\tNextProcessAt: time.Time{}, // zero value for N/A\n\t\t\t\tResult: nil,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tqname: \"custom\",\n\t\t\tid: m5.ID,\n\t\t\twant: &base.TaskInfo{\n\t\t\t\tMessage: m5,\n\t\t\t\tState: base.TaskStatePending,\n\t\t\t\tNextProcessAt: now,\n\t\t\t\tResult: nil,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tqname: \"custom\",\n\t\t\tid: m6.ID,\n\t\t\twant: &base.TaskInfo{\n\t\t\t\tMessage: m6,\n\t\t\t\tState: base.TaskStateCompleted,\n\t\t\t\tNextProcessAt: time.Time{}, // zero value for N/A\n\t\t\t\tResult: []byte(\"foobar\"),\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot, err := r.GetTaskInfo(tc.qname, tc.id)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"GetTaskInfo(%q, %v) returned error: %v\", tc.qname, tc.id, err)\n\t\t\tcontinue\n\t\t}\n\t\tif diff := cmp.Diff(tc.want, got, cmpopts.EquateApproxTime(2*time.Second)); diff != \"\" {\n\t\t\tt.Errorf(\"GetTaskInfo(%q, %v) = %v, want %v; (-want,+got)\\n%s\",\n\t\t\t\ttc.qname, tc.id, got, tc.want, diff)\n\t\t}\n\t}\n}\n\nfunc TestGetTaskInfoError(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tm1 := h.NewTaskMessageWithQueue(\"task1\", nil, \"default\")\n\tm2 := h.NewTaskMessageWithQueue(\"task2\", nil, \"default\")\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tm4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"custom\")\n\tm5 := h.NewTaskMessageWithQueue(\"task5\", nil, \"custom\")\n\n\tnow := time.Now()\n\tfiveMinsFromNow := now.Add(5 * time.Minute)\n\toneHourFromNow := now.Add(1 * time.Hour)\n\ttwoHoursAgo := now.Add(-2 * time.Hour)\n\n\tfixtures := struct {\n\t\tactive map[string][]*base.TaskMessage\n\t\tpending map[string][]*base.TaskMessage\n\t\tscheduled map[string][]base.Z\n\t\tretry map[string][]base.Z\n\t\tarchived map[string][]base.Z\n\t}{\n\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\"default\": {m1},\n\t\t\t\"custom\": {},\n\t\t},\n\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\"default\": {},\n\t\t\t\"custom\": {m5},\n\t\t},\n\t\tscheduled: map[string][]base.Z{\n\t\t\t\"default\": {{Message: m2, Score: fiveMinsFromNow.Unix()}},\n\t\t\t\"custom\": {},\n\t\t},\n\t\tretry: map[string][]base.Z{\n\t\t\t\"default\": {},\n\t\t\t\"custom\": {{Message: m3, Score: oneHourFromNow.Unix()}},\n\t\t},\n\t\tarchived: map[string][]base.Z{\n\t\t\t\"default\": {},\n\t\t\t\"custom\": {{Message: m4, Score: twoHoursAgo.Unix()}},\n\t\t},\n\t}\n\n\th.SeedAllActiveQueues(t, r.client, fixtures.active)\n\th.SeedAllPendingQueues(t, r.client, fixtures.pending)\n\th.SeedAllScheduledQueues(t, r.client, fixtures.scheduled)\n\th.SeedAllRetryQueues(t, r.client, fixtures.retry)\n\th.SeedAllArchivedQueues(t, r.client, fixtures.archived)\n\n\ttests := []struct {\n\t\tqname string\n\t\tid string\n\t\tmatch func(err error) bool\n\t}{\n\t\t{\n\t\t\tqname: \"nonexistent\",\n\t\t\tid: m1.ID,\n\t\t\tmatch: errors.IsQueueNotFound,\n\t\t},\n\t\t{\n\t\t\tqname: \"default\",\n\t\t\tid: uuid.NewString(),\n\t\t\tmatch: errors.IsTaskNotFound,\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tinfo, err := r.GetTaskInfo(tc.qname, tc.id)\n\t\tif info != nil {\n\t\t\tt.Errorf(\"GetTaskInfo(%q, %v) returned info: %v\", tc.qname, tc.id, info)\n\t\t}\n\t\tif !tc.match(err) {\n\t\t\tt.Errorf(\"GetTaskInfo(%q, %v) returned unexpected error: %v\", tc.qname, tc.id, err)\n\t\t}\n\t}\n}\n\nfunc TestListPending(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tm1 := h.NewTaskMessage(\"send_email\", h.JSON(map[string]interface{}{\"subject\": \"hello\"}))\n\tm2 := h.NewTaskMessage(\"reindex\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"important_notification\", nil, \"critical\")\n\tm4 := h.NewTaskMessageWithQueue(\"minor_notification\", nil, \"low\")\n\n\ttests := []struct {\n\t\tpending map[string][]*base.TaskMessage\n\t\tqname string\n\t\twant []*base.TaskInfo\n\t}{\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\tbase.DefaultQueueName: {m1, m2},\n\t\t\t},\n\t\t\tqname: base.DefaultQueueName,\n\t\t\twant: []*base.TaskInfo{\n\t\t\t\t{Message: m1, State: base.TaskStatePending, NextProcessAt: time.Now(), Result: nil},\n\t\t\t\t{Message: m2, State: base.TaskStatePending, NextProcessAt: time.Now(), Result: nil},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\tbase.DefaultQueueName: nil,\n\t\t\t},\n\t\t\tqname: base.DefaultQueueName,\n\t\t\twant: []*base.TaskInfo(nil),\n\t\t},\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\tbase.DefaultQueueName: {m1, m2},\n\t\t\t\t\"critical\": {m3},\n\t\t\t\t\"low\": {m4},\n\t\t\t},\n\t\t\tqname: base.DefaultQueueName,\n\t\t\twant: []*base.TaskInfo{\n\t\t\t\t{Message: m1, State: base.TaskStatePending, NextProcessAt: time.Now(), Result: nil},\n\t\t\t\t{Message: m2, State: base.TaskStatePending, NextProcessAt: time.Now(), Result: nil},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\tbase.DefaultQueueName: {m1, m2},\n\t\t\t\t\"critical\": {m3},\n\t\t\t\t\"low\": {m4},\n\t\t\t},\n\t\t\tqname: \"critical\",\n\t\t\twant: []*base.TaskInfo{\n\t\t\t\t{Message: m3, State: base.TaskStatePending, NextProcessAt: time.Now(), Result: nil},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllPendingQueues(t, r.client, tc.pending)\n\n\t\tgot, err := r.ListPending(tc.qname, Pagination{Size: 20, Page: 0})\n\t\top := fmt.Sprintf(\"r.ListPending(%q, Pagination{Size: 20, Page: 0})\", tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s = %v, %v, want %v, nil\", op, got, err, tc.want)\n\t\t\tcontinue\n\t\t}\n\t\tif diff := cmp.Diff(tc.want, got, cmpopts.EquateApproxTime(2*time.Second)); diff != \"\" {\n\t\t\tt.Errorf(\"%s = %v, %v, want %v, nil; (-want, +got)\\n%s\", op, got, err, tc.want, diff)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\nfunc TestListPendingPagination(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tvar msgs []*base.TaskMessage\n\tfor i := 0; i < 100; i++ {\n\t\tmsg := h.NewTaskMessage(fmt.Sprintf(\"task %d\", i), nil)\n\t\tmsgs = append(msgs, msg)\n\t}\n\t// create 100 tasks in default queue\n\th.SeedPendingQueue(t, r.client, msgs, \"default\")\n\n\tmsgs = []*base.TaskMessage(nil) // empty list\n\tfor i := 0; i < 100; i++ {\n\t\tmsg := h.NewTaskMessageWithQueue(fmt.Sprintf(\"custom %d\", i), nil, \"custom\")\n\t\tmsgs = append(msgs, msg)\n\t}\n\t// create 100 tasks in custom queue\n\th.SeedPendingQueue(t, r.client, msgs, \"custom\")\n\n\ttests := []struct {\n\t\tdesc string\n\t\tqname string\n\t\tpage int\n\t\tsize int\n\t\twantSize int\n\t\twantFirst string\n\t\twantLast string\n\t}{\n\t\t{\"first page\", \"default\", 0, 20, 20, \"task 0\", \"task 19\"},\n\t\t{\"second page\", \"default\", 1, 20, 20, \"task 20\", \"task 39\"},\n\t\t{\"different page size\", \"default\", 2, 30, 30, \"task 60\", \"task 89\"},\n\t\t{\"last page\", \"default\", 3, 30, 10, \"task 90\", \"task 99\"},\n\t\t{\"out of range\", \"default\", 4, 30, 0, \"\", \"\"},\n\t\t{\"second page with custom queue\", \"custom\", 1, 20, 20, \"custom 20\", \"custom 39\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot, err := r.ListPending(tc.qname, Pagination{Size: tc.size, Page: tc.page})\n\t\top := fmt.Sprintf(\"r.ListPending(%q, Pagination{Size: %d, Page: %d})\", tc.qname, tc.size, tc.page)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s; %s returned error %v\", tc.desc, op, err)\n\t\t\tcontinue\n\t\t}\n\n\t\tif len(got) != tc.wantSize {\n\t\t\tt.Errorf(\"%s; %s returned a list of size %d, want %d\", tc.desc, op, len(got), tc.wantSize)\n\t\t\tcontinue\n\t\t}\n\n\t\tif tc.wantSize == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tfirst := got[0].Message\n\t\tif first.Type != tc.wantFirst {\n\t\t\tt.Errorf(\"%s; %s returned a list with first message %q, want %q\",\n\t\t\t\ttc.desc, op, first.Type, tc.wantFirst)\n\t\t}\n\n\t\tlast := got[len(got)-1].Message\n\t\tif last.Type != tc.wantLast {\n\t\t\tt.Errorf(\"%s; %s returned a list with the last message %q, want %q\",\n\t\t\t\ttc.desc, op, last.Type, tc.wantLast)\n\t\t}\n\t}\n}\n\nfunc TestListActive(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task2\", nil, \"critical\")\n\tm4 := h.NewTaskMessageWithQueue(\"task2\", nil, \"low\")\n\n\ttests := []struct {\n\t\tinProgress map[string][]*base.TaskMessage\n\t\tqname string\n\t\twant []*base.TaskInfo\n\t}{\n\t\t{\n\t\t\tinProgress: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t\t\"critical\": {m3},\n\t\t\t\t\"low\": {m4},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: []*base.TaskInfo{\n\t\t\t\t{Message: m1, State: base.TaskStateActive, NextProcessAt: time.Time{}, Result: nil},\n\t\t\t\t{Message: m2, State: base.TaskStateActive, NextProcessAt: time.Time{}, Result: nil},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinProgress: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: []*base.TaskInfo(nil),\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllActiveQueues(t, r.client, tc.inProgress)\n\n\t\tgot, err := r.ListActive(tc.qname, Pagination{Size: 20, Page: 0})\n\t\top := fmt.Sprintf(\"r.ListActive(%q, Pagination{Size: 20, Page: 0})\", tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s = %v, %v, want %v, nil\", op, got, err, tc.inProgress)\n\t\t\tcontinue\n\t\t}\n\t\tif diff := cmp.Diff(tc.want, got, cmpopts.EquateApproxTime(1*time.Second)); diff != \"\" {\n\t\t\tt.Errorf(\"%s = %v, %v, want %v, nil; (-want, +got)\\n%s\", op, got, err, tc.want, diff)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\nfunc TestListActivePagination(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tvar msgs []*base.TaskMessage\n\tfor i := 0; i < 100; i++ {\n\t\tmsg := h.NewTaskMessage(fmt.Sprintf(\"task %d\", i), nil)\n\t\tmsgs = append(msgs, msg)\n\t}\n\th.SeedActiveQueue(t, r.client, msgs, \"default\")\n\n\ttests := []struct {\n\t\tdesc string\n\t\tqname string\n\t\tpage int\n\t\tsize int\n\t\twantSize int\n\t\twantFirst string\n\t\twantLast string\n\t}{\n\t\t{\"first page\", \"default\", 0, 20, 20, \"task 0\", \"task 19\"},\n\t\t{\"second page\", \"default\", 1, 20, 20, \"task 20\", \"task 39\"},\n\t\t{\"different page size\", \"default\", 2, 30, 30, \"task 60\", \"task 89\"},\n\t\t{\"last page\", \"default\", 3, 30, 10, \"task 90\", \"task 99\"},\n\t\t{\"out of range\", \"default\", 4, 30, 0, \"\", \"\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot, err := r.ListActive(tc.qname, Pagination{Size: tc.size, Page: tc.page})\n\t\top := fmt.Sprintf(\"r.ListActive(%q, Pagination{Size: %d, Page: %d})\", tc.qname, tc.size, tc.page)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s; %s returned error %v\", tc.desc, op, err)\n\t\t\tcontinue\n\t\t}\n\n\t\tif len(got) != tc.wantSize {\n\t\t\tt.Errorf(\"%s; %s returned list of size %d, want %d\", tc.desc, op, len(got), tc.wantSize)\n\t\t\tcontinue\n\t\t}\n\n\t\tif tc.wantSize == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tfirst := got[0].Message\n\t\tif first.Type != tc.wantFirst {\n\t\t\tt.Errorf(\"%s; %s returned a list with first message %q, want %q\",\n\t\t\t\ttc.desc, op, first.Type, tc.wantFirst)\n\t\t}\n\n\t\tlast := got[len(got)-1].Message\n\t\tif last.Type != tc.wantLast {\n\t\t\tt.Errorf(\"%s; %s returned a list with the last message %q, want %q\",\n\t\t\t\ttc.desc, op, last.Type, tc.wantLast)\n\t\t}\n\t}\n}\n\nfunc TestListScheduled(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessage(\"task3\", nil)\n\tm4 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tp1 := time.Now().Add(30 * time.Minute)\n\tp2 := time.Now().Add(24 * time.Hour)\n\tp3 := time.Now().Add(5 * time.Minute)\n\tp4 := time.Now().Add(2 * time.Minute)\n\n\ttests := []struct {\n\t\tscheduled map[string][]base.Z\n\t\tqname string\n\t\twant []*base.TaskInfo\n\t}{\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: p1.Unix()},\n\t\t\t\t\t{Message: m2, Score: p2.Unix()},\n\t\t\t\t\t{Message: m3, Score: p3.Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: m4, Score: p4.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\t// should be sorted by score in ascending order\n\t\t\twant: []*base.TaskInfo{\n\t\t\t\t{Message: m3, NextProcessAt: p3, State: base.TaskStateScheduled, Result: nil},\n\t\t\t\t{Message: m1, NextProcessAt: p1, State: base.TaskStateScheduled, Result: nil},\n\t\t\t\t{Message: m2, NextProcessAt: p2, State: base.TaskStateScheduled, Result: nil},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: p1.Unix()},\n\t\t\t\t\t{Message: m2, Score: p2.Unix()},\n\t\t\t\t\t{Message: m3, Score: p3.Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: m4, Score: p4.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\twant: []*base.TaskInfo{\n\t\t\t\t{Message: m4, NextProcessAt: p4, State: base.TaskStateScheduled, Result: nil},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: []*base.TaskInfo(nil),\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllScheduledQueues(t, r.client, tc.scheduled)\n\n\t\tgot, err := r.ListScheduled(tc.qname, Pagination{Size: 20, Page: 0})\n\t\top := fmt.Sprintf(\"r.ListScheduled(%q, Pagination{Size: 20, Page: 0})\", tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s = %v, %v, want %v, nil\", op, got, err, tc.want)\n\t\t\tcontinue\n\t\t}\n\t\tif diff := cmp.Diff(tc.want, got, cmpopts.EquateApproxTime(1*time.Second)); diff != \"\" {\n\t\t\tt.Errorf(\"%s = %v, %v, want %v, nil; (-want, +got)\\n%s\", op, got, err, tc.want, diff)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\nfunc TestListScheduledPagination(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\t// create 100 tasks with an increasing number of wait time.\n\tfor i := 0; i < 100; i++ {\n\t\tmsg := h.NewTaskMessage(fmt.Sprintf(\"task %d\", i), nil)\n\t\tif err := r.Schedule(context.Background(), msg, time.Now().Add(time.Duration(i)*time.Second)); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\ttests := []struct {\n\t\tdesc string\n\t\tqname string\n\t\tpage int\n\t\tsize int\n\t\twantSize int\n\t\twantFirst string\n\t\twantLast string\n\t}{\n\t\t{\"first page\", \"default\", 0, 20, 20, \"task 0\", \"task 19\"},\n\t\t{\"second page\", \"default\", 1, 20, 20, \"task 20\", \"task 39\"},\n\t\t{\"different page size\", \"default\", 2, 30, 30, \"task 60\", \"task 89\"},\n\t\t{\"last page\", \"default\", 3, 30, 10, \"task 90\", \"task 99\"},\n\t\t{\"out of range\", \"default\", 4, 30, 0, \"\", \"\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot, err := r.ListScheduled(tc.qname, Pagination{Size: tc.size, Page: tc.page})\n\t\top := fmt.Sprintf(\"r.ListScheduled(%q, Pagination{Size: %d, Page: %d})\", tc.qname, tc.size, tc.page)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s; %s returned error %v\", tc.desc, op, err)\n\t\t\tcontinue\n\t\t}\n\n\t\tif len(got) != tc.wantSize {\n\t\t\tt.Errorf(\"%s; %s returned list of size %d, want %d\", tc.desc, op, len(got), tc.wantSize)\n\t\t\tcontinue\n\t\t}\n\n\t\tif tc.wantSize == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tfirst := got[0].Message\n\t\tif first.Type != tc.wantFirst {\n\t\t\tt.Errorf(\"%s; %s returned a list with first message %q, want %q\",\n\t\t\t\ttc.desc, op, first.Type, tc.wantFirst)\n\t\t}\n\n\t\tlast := got[len(got)-1].Message\n\t\tif last.Type != tc.wantLast {\n\t\t\tt.Errorf(\"%s; %s returned a list with the last message %q, want %q\",\n\t\t\t\ttc.desc, op, last.Type, tc.wantLast)\n\t\t}\n\t}\n}\n\nfunc TestListRetry(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"task1\",\n\t\tQueue: \"default\",\n\t\tPayload: nil,\n\t\tErrorMsg: \"some error occurred\",\n\t\tRetry: 25,\n\t\tRetried: 10,\n\t}\n\tm2 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"task2\",\n\t\tQueue: \"default\",\n\t\tPayload: nil,\n\t\tErrorMsg: \"some error occurred\",\n\t\tRetry: 25,\n\t\tRetried: 2,\n\t}\n\tm3 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"task3\",\n\t\tQueue: \"custom\",\n\t\tPayload: nil,\n\t\tErrorMsg: \"some error occurred\",\n\t\tRetry: 25,\n\t\tRetried: 3,\n\t}\n\tp1 := time.Now().Add(5 * time.Minute)\n\tp2 := time.Now().Add(24 * time.Hour)\n\tp3 := time.Now().Add(24 * time.Hour)\n\n\ttests := []struct {\n\t\tretry map[string][]base.Z\n\t\tqname string\n\t\twant []*base.TaskInfo\n\t}{\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: p1.Unix()},\n\t\t\t\t\t{Message: m2, Score: p2.Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: m3, Score: p3.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: []*base.TaskInfo{\n\t\t\t\t{Message: m1, NextProcessAt: p1, State: base.TaskStateRetry, Result: nil},\n\t\t\t\t{Message: m2, NextProcessAt: p2, State: base.TaskStateRetry, Result: nil},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: p1.Unix()},\n\t\t\t\t\t{Message: m2, Score: p2.Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: m3, Score: p3.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\twant: []*base.TaskInfo{\n\t\t\t\t{Message: m3, NextProcessAt: p3, State: base.TaskStateRetry, Result: nil},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: []*base.TaskInfo(nil),\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllRetryQueues(t, r.client, tc.retry)\n\n\t\tgot, err := r.ListRetry(tc.qname, Pagination{Size: 20, Page: 0})\n\t\top := fmt.Sprintf(\"r.ListRetry(%q, Pagination{Size: 20, Page: 0})\", tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s = %v, %v, want %v, nil\", op, got, err, tc.want)\n\t\t\tcontinue\n\t\t}\n\t\tif diff := cmp.Diff(tc.want, got, cmpopts.EquateApproxTime(1*time.Second)); diff != \"\" {\n\t\t\tt.Errorf(\"%s = %v, %v, want %v, nil; (-want, +got)\\n%s\",\n\t\t\t\top, got, err, tc.want, diff)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\nfunc TestListRetryPagination(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\t// create 100 tasks with an increasing number of wait time.\n\tnow := time.Now()\n\tvar seed []base.Z\n\tfor i := 0; i < 100; i++ {\n\t\tmsg := h.NewTaskMessage(fmt.Sprintf(\"task %d\", i), nil)\n\t\tprocessAt := now.Add(time.Duration(i) * time.Second)\n\t\tseed = append(seed, base.Z{Message: msg, Score: processAt.Unix()})\n\t}\n\th.SeedRetryQueue(t, r.client, seed, \"default\")\n\n\ttests := []struct {\n\t\tdesc string\n\t\tqname string\n\t\tpage int\n\t\tsize int\n\t\twantSize int\n\t\twantFirst string\n\t\twantLast string\n\t}{\n\t\t{\"first page\", \"default\", 0, 20, 20, \"task 0\", \"task 19\"},\n\t\t{\"second page\", \"default\", 1, 20, 20, \"task 20\", \"task 39\"},\n\t\t{\"different page size\", \"default\", 2, 30, 30, \"task 60\", \"task 89\"},\n\t\t{\"last page\", \"default\", 3, 30, 10, \"task 90\", \"task 99\"},\n\t\t{\"out of range\", \"default\", 4, 30, 0, \"\", \"\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot, err := r.ListRetry(tc.qname, Pagination{Size: tc.size, Page: tc.page})\n\t\top := fmt.Sprintf(\"r.ListRetry(%q, Pagination{Size: %d, Page: %d})\",\n\t\t\ttc.qname, tc.size, tc.page)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s; %s returned error %v\", tc.desc, op, err)\n\t\t\tcontinue\n\t\t}\n\n\t\tif len(got) != tc.wantSize {\n\t\t\tt.Errorf(\"%s; %s returned list of size %d, want %d\",\n\t\t\t\ttc.desc, op, len(got), tc.wantSize)\n\t\t\tcontinue\n\t\t}\n\n\t\tif tc.wantSize == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tfirst := got[0].Message\n\t\tif first.Type != tc.wantFirst {\n\t\t\tt.Errorf(\"%s; %s returned a list with first message %q, want %q\",\n\t\t\t\ttc.desc, op, first.Type, tc.wantFirst)\n\t\t}\n\n\t\tlast := got[len(got)-1].Message\n\t\tif last.Type != tc.wantLast {\n\t\t\tt.Errorf(\"%s; %s returned a list with the last message %q, want %q\",\n\t\t\t\ttc.desc, op, last.Type, tc.wantLast)\n\t\t}\n\t}\n}\n\nfunc TestListArchived(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"task1\",\n\t\tQueue: \"default\",\n\t\tPayload: nil,\n\t\tErrorMsg: \"some error occurred\",\n\t}\n\tm2 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"task2\",\n\t\tQueue: \"default\",\n\t\tPayload: nil,\n\t\tErrorMsg: \"some error occurred\",\n\t}\n\tm3 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"task3\",\n\t\tQueue: \"custom\",\n\t\tPayload: nil,\n\t\tErrorMsg: \"some error occurred\",\n\t}\n\tf1 := time.Now().Add(-5 * time.Minute)\n\tf2 := time.Now().Add(-24 * time.Hour)\n\tf3 := time.Now().Add(-4 * time.Hour)\n\n\ttests := []struct {\n\t\tarchived map[string][]base.Z\n\t\tqname string\n\t\twant []*base.TaskInfo\n\t}{\n\t\t{\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: f1.Unix()},\n\t\t\t\t\t{Message: m2, Score: f2.Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: m3, Score: f3.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: []*base.TaskInfo{\n\t\t\t\t{Message: m2, NextProcessAt: time.Time{}, State: base.TaskStateArchived, Result: nil}, // FIXME: shouldn't be sorted in the other order?\n\t\t\t\t{Message: m1, NextProcessAt: time.Time{}, State: base.TaskStateArchived, Result: nil},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: f1.Unix()},\n\t\t\t\t\t{Message: m2, Score: f2.Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: m3, Score: f3.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\twant: []*base.TaskInfo{\n\t\t\t\t{Message: m3, NextProcessAt: time.Time{}, State: base.TaskStateArchived, Result: nil},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: []*base.TaskInfo(nil),\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllArchivedQueues(t, r.client, tc.archived)\n\n\t\tgot, err := r.ListArchived(tc.qname, Pagination{Size: 20, Page: 0})\n\t\top := fmt.Sprintf(\"r.ListArchived(%q, Pagination{Size: 20, Page: 0})\", tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s = %v, %v, want %v, nil\", op, got, err, tc.want)\n\t\t\tcontinue\n\t\t}\n\t\tif diff := cmp.Diff(tc.want, got); diff != \"\" {\n\t\t\tt.Errorf(\"%s = %v, %v, want %v, nil; (-want, +got)\\n%s\",\n\t\t\t\top, got, err, tc.want, diff)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\nfunc TestListArchivedPagination(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tvar entries []base.Z\n\tfor i := 0; i < 100; i++ {\n\t\tmsg := h.NewTaskMessage(fmt.Sprintf(\"task %d\", i), nil)\n\t\tentries = append(entries, base.Z{Message: msg, Score: int64(i)})\n\t}\n\th.SeedArchivedQueue(t, r.client, entries, \"default\")\n\n\ttests := []struct {\n\t\tdesc string\n\t\tqname string\n\t\tpage int\n\t\tsize int\n\t\twantSize int\n\t\twantFirst string\n\t\twantLast string\n\t}{\n\t\t{\"first page\", \"default\", 0, 20, 20, \"task 0\", \"task 19\"},\n\t\t{\"second page\", \"default\", 1, 20, 20, \"task 20\", \"task 39\"},\n\t\t{\"different page size\", \"default\", 2, 30, 30, \"task 60\", \"task 89\"},\n\t\t{\"last page\", \"default\", 3, 30, 10, \"task 90\", \"task 99\"},\n\t\t{\"out of range\", \"default\", 4, 30, 0, \"\", \"\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot, err := r.ListArchived(tc.qname, Pagination{Size: tc.size, Page: tc.page})\n\t\top := fmt.Sprintf(\"r.ListArchived(Pagination{Size: %d, Page: %d})\",\n\t\t\ttc.size, tc.page)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s; %s returned error %v\", tc.desc, op, err)\n\t\t\tcontinue\n\t\t}\n\n\t\tif len(got) != tc.wantSize {\n\t\t\tt.Errorf(\"%s; %s returned list of size %d, want %d\",\n\t\t\t\ttc.desc, op, len(got), tc.wantSize)\n\t\t\tcontinue\n\t\t}\n\n\t\tif tc.wantSize == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tfirst := got[0].Message\n\t\tif first.Type != tc.wantFirst {\n\t\t\tt.Errorf(\"%s; %s returned a list with first message %q, want %q\",\n\t\t\t\ttc.desc, op, first.Type, tc.wantFirst)\n\t\t}\n\n\t\tlast := got[len(got)-1].Message\n\t\tif last.Type != tc.wantLast {\n\t\t\tt.Errorf(\"%s; %s returned a list with the last message %q, want %q\",\n\t\t\t\ttc.desc, op, last.Type, tc.wantLast)\n\t\t}\n\t}\n}\n\nfunc TestListCompleted(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tmsg1 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"foo\",\n\t\tQueue: \"default\",\n\t\tCompletedAt: time.Now().Add(-2 * time.Hour).Unix(),\n\t}\n\tmsg2 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"foo\",\n\t\tQueue: \"default\",\n\t\tCompletedAt: time.Now().Add(-5 * time.Hour).Unix(),\n\t}\n\tmsg3 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"foo\",\n\t\tQueue: \"custom\",\n\t\tCompletedAt: time.Now().Add(-5 * time.Hour).Unix(),\n\t}\n\texpireAt1 := time.Now().Add(3 * time.Hour)\n\texpireAt2 := time.Now().Add(4 * time.Hour)\n\texpireAt3 := time.Now().Add(5 * time.Hour)\n\n\ttests := []struct {\n\t\tcompleted map[string][]base.Z\n\t\tqname string\n\t\twant []*base.TaskInfo\n\t}{\n\t\t{\n\t\t\tcompleted: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: msg1, Score: expireAt1.Unix()},\n\t\t\t\t\t{Message: msg2, Score: expireAt2.Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: msg3, Score: expireAt3.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: []*base.TaskInfo{\n\t\t\t\t{Message: msg1, NextProcessAt: time.Time{}, State: base.TaskStateCompleted, Result: nil},\n\t\t\t\t{Message: msg2, NextProcessAt: time.Time{}, State: base.TaskStateCompleted, Result: nil},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tcompleted: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: msg1, Score: expireAt1.Unix()},\n\t\t\t\t\t{Message: msg2, Score: expireAt2.Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: msg3, Score: expireAt3.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\twant: []*base.TaskInfo{\n\t\t\t\t{Message: msg3, NextProcessAt: time.Time{}, State: base.TaskStateCompleted, Result: nil},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllCompletedQueues(t, r.client, tc.completed)\n\n\t\tgot, err := r.ListCompleted(tc.qname, Pagination{Size: 20, Page: 0})\n\t\top := fmt.Sprintf(\"r.ListCompleted(%q, Pagination{Size: 20, Page: 0})\", tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s = %v, %v, want %v, nil\", op, got, err, tc.want)\n\t\t\tcontinue\n\t\t}\n\t\tif diff := cmp.Diff(tc.want, got); diff != \"\" {\n\t\t\tt.Errorf(\"%s = %v, %v, want %v, nil; (-want, +got)\\n%s\",\n\t\t\t\top, got, err, tc.want, diff)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\nfunc TestListCompletedPagination(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tvar entries []base.Z\n\tfor i := 0; i < 100; i++ {\n\t\tmsg := h.NewTaskMessage(fmt.Sprintf(\"task %d\", i), nil)\n\t\tentries = append(entries, base.Z{Message: msg, Score: int64(i)})\n\t}\n\th.SeedCompletedQueue(t, r.client, entries, \"default\")\n\n\ttests := []struct {\n\t\tdesc string\n\t\tqname string\n\t\tpage int\n\t\tsize int\n\t\twantSize int\n\t\twantFirst string\n\t\twantLast string\n\t}{\n\t\t{\"first page\", \"default\", 0, 20, 20, \"task 0\", \"task 19\"},\n\t\t{\"second page\", \"default\", 1, 20, 20, \"task 20\", \"task 39\"},\n\t\t{\"different page size\", \"default\", 2, 30, 30, \"task 60\", \"task 89\"},\n\t\t{\"last page\", \"default\", 3, 30, 10, \"task 90\", \"task 99\"},\n\t\t{\"out of range\", \"default\", 4, 30, 0, \"\", \"\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot, err := r.ListCompleted(tc.qname, Pagination{Size: tc.size, Page: tc.page})\n\t\top := fmt.Sprintf(\"r.ListCompleted(Pagination{Size: %d, Page: %d})\",\n\t\t\ttc.size, tc.page)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s; %s returned error %v\", tc.desc, op, err)\n\t\t\tcontinue\n\t\t}\n\n\t\tif len(got) != tc.wantSize {\n\t\t\tt.Errorf(\"%s; %s returned list of size %d, want %d\",\n\t\t\t\ttc.desc, op, len(got), tc.wantSize)\n\t\t\tcontinue\n\t\t}\n\n\t\tif tc.wantSize == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tfirst := got[0].Message\n\t\tif first.Type != tc.wantFirst {\n\t\t\tt.Errorf(\"%s; %s returned a list with first message %q, want %q\",\n\t\t\t\ttc.desc, op, first.Type, tc.wantFirst)\n\t\t}\n\n\t\tlast := got[len(got)-1].Message\n\t\tif last.Type != tc.wantLast {\n\t\t\tt.Errorf(\"%s; %s returned a list with the last message %q, want %q\",\n\t\t\t\ttc.desc, op, last.Type, tc.wantLast)\n\t\t}\n\t}\n}\n\nfunc TestListAggregating(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tnow := time.Now()\n\tm1 := h.NewTaskMessageBuilder().SetType(\"task1\").SetQueue(\"default\").SetGroup(\"group1\").Build()\n\tm2 := h.NewTaskMessageBuilder().SetType(\"task2\").SetQueue(\"default\").SetGroup(\"group1\").Build()\n\tm3 := h.NewTaskMessageBuilder().SetType(\"task3\").SetQueue(\"default\").SetGroup(\"group2\").Build()\n\tm4 := h.NewTaskMessageBuilder().SetType(\"task4\").SetQueue(\"custom\").SetGroup(\"group3\").Build()\n\n\tfxt := struct {\n\t\ttasks []*h.TaskSeedData\n\t\tallQueues []string\n\t\tallGroups map[string][]string\n\t\tgroups map[string][]redis.Z\n\t}{\n\t\ttasks: []*h.TaskSeedData{\n\t\t\t{Msg: m1, State: base.TaskStateAggregating},\n\t\t\t{Msg: m2, State: base.TaskStateAggregating},\n\t\t\t{Msg: m3, State: base.TaskStateAggregating},\n\t\t\t{Msg: m4, State: base.TaskStateAggregating},\n\t\t},\n\t\tallQueues: []string{\"default\", \"custom\"},\n\t\tallGroups: map[string][]string{\n\t\t\tbase.AllGroups(\"default\"): {\"group1\", \"group2\"},\n\t\t\tbase.AllGroups(\"custom\"): {\"group3\"},\n\t\t},\n\t\tgroups: map[string][]redis.Z{\n\t\t\tbase.GroupKey(\"default\", \"group1\"): {\n\t\t\t\t{Member: m1.ID, Score: float64(now.Add(-30 * time.Second).Unix())},\n\t\t\t\t{Member: m2.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t},\n\t\t\tbase.GroupKey(\"default\", \"group2\"): {\n\t\t\t\t{Member: m3.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t},\n\t\t\tbase.GroupKey(\"custom\", \"group3\"): {\n\t\t\t\t{Member: m4.ID, Score: float64(now.Add(-40 * time.Second).Unix())},\n\t\t\t},\n\t\t},\n\t}\n\n\ttests := []struct {\n\t\tdesc string\n\t\tqname string\n\t\tgname string\n\t\twant []*base.TaskInfo\n\t}{\n\t\t{\n\t\t\tdesc: \"with group1 in default queue\",\n\t\t\tqname: \"default\",\n\t\t\tgname: \"group1\",\n\t\t\twant: []*base.TaskInfo{\n\t\t\t\t{Message: m1, State: base.TaskStateAggregating, NextProcessAt: time.Time{}, Result: nil},\n\t\t\t\t{Message: m2, State: base.TaskStateAggregating, NextProcessAt: time.Time{}, Result: nil},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with group3 in custom queue\",\n\t\t\tqname: \"custom\",\n\t\t\tgname: \"group3\",\n\t\t\twant: []*base.TaskInfo{\n\t\t\t\t{Message: m4, State: base.TaskStateAggregating, NextProcessAt: time.Time{}, Result: nil},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\th.SeedRedisSet(t, r.client, base.AllQueues, fxt.allQueues)\n\t\th.SeedRedisSets(t, r.client, fxt.allGroups)\n\t\th.SeedTasks(t, r.client, fxt.tasks)\n\t\th.SeedRedisZSets(t, r.client, fxt.groups)\n\n\t\tt.Run(tc.desc, func(t *testing.T) {\n\t\t\tgot, err := r.ListAggregating(tc.qname, tc.gname, Pagination{})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"ListAggregating returned error: %v\", err)\n\t\t\t}\n\t\t\tif diff := cmp.Diff(tc.want, got); diff != \"\" {\n\t\t\t\tt.Errorf(\"ListAggregating = %v, want %v; (-want,+got)\\n%s\", got, tc.want, diff)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestListAggregatingPagination(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tgroupkey := base.GroupKey(\"default\", \"mygroup\")\n\tfxt := struct {\n\t\ttasks []*h.TaskSeedData\n\t\tallQueues []string\n\t\tallGroups map[string][]string\n\t\tgroups map[string][]redis.Z\n\t}{\n\t\ttasks: []*h.TaskSeedData{}, // will be populated below\n\t\tallQueues: []string{\"default\"},\n\t\tallGroups: map[string][]string{\n\t\t\tbase.AllGroups(\"default\"): {\"mygroup\"},\n\t\t},\n\t\tgroups: map[string][]redis.Z{\n\t\t\tgroupkey: {}, // will be populated below\n\t\t},\n\t}\n\n\tnow := time.Now()\n\tfor i := 0; i < 100; i++ {\n\t\tmsg := h.NewTaskMessageBuilder().SetType(fmt.Sprintf(\"task%d\", i)).SetGroup(\"mygroup\").Build()\n\t\tfxt.tasks = append(fxt.tasks, &h.TaskSeedData{\n\t\t\tMsg: msg, State: base.TaskStateAggregating,\n\t\t})\n\t\tfxt.groups[groupkey] = append(fxt.groups[groupkey], redis.Z{\n\t\t\tMember: msg.ID,\n\t\t\tScore: float64(now.Add(-time.Duration(100-i) * time.Second).Unix()),\n\t\t})\n\t}\n\n\ttests := []struct {\n\t\tdesc string\n\t\tqname string\n\t\tgname string\n\t\tpage int\n\t\tsize int\n\t\twantSize int\n\t\twantFirst string\n\t\twantLast string\n\t}{\n\t\t{\n\t\t\tdesc: \"first page\",\n\t\t\tqname: \"default\",\n\t\t\tgname: \"mygroup\",\n\t\t\tpage: 0,\n\t\t\tsize: 20,\n\t\t\twantSize: 20,\n\t\t\twantFirst: \"task0\",\n\t\t\twantLast: \"task19\",\n\t\t},\n\t\t{\n\t\t\tdesc: \"second page\",\n\t\t\tqname: \"default\",\n\t\t\tgname: \"mygroup\",\n\t\t\tpage: 1,\n\t\t\tsize: 20,\n\t\t\twantSize: 20,\n\t\t\twantFirst: \"task20\",\n\t\t\twantLast: \"task39\",\n\t\t},\n\t\t{\n\t\t\tdesc: \"with different page size\",\n\t\t\tqname: \"default\",\n\t\t\tgname: \"mygroup\",\n\t\t\tpage: 2,\n\t\t\tsize: 30,\n\t\t\twantSize: 30,\n\t\t\twantFirst: \"task60\",\n\t\t\twantLast: \"task89\",\n\t\t},\n\t\t{\n\t\t\tdesc: \"last page\",\n\t\t\tqname: \"default\",\n\t\t\tgname: \"mygroup\",\n\t\t\tpage: 3,\n\t\t\tsize: 30,\n\t\t\twantSize: 10,\n\t\t\twantFirst: \"task90\",\n\t\t\twantLast: \"task99\",\n\t\t},\n\t\t{\n\t\t\tdesc: \"out of range\",\n\t\t\tqname: \"default\",\n\t\t\tgname: \"mygroup\",\n\t\t\tpage: 4,\n\t\t\tsize: 30,\n\t\t\twantSize: 0,\n\t\t\twantFirst: \"\",\n\t\t\twantLast: \"\",\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\th.SeedRedisSet(t, r.client, base.AllQueues, fxt.allQueues)\n\t\th.SeedRedisSets(t, r.client, fxt.allGroups)\n\t\th.SeedTasks(t, r.client, fxt.tasks)\n\t\th.SeedRedisZSets(t, r.client, fxt.groups)\n\n\t\tt.Run(tc.desc, func(t *testing.T) {\n\t\t\tgot, err := r.ListAggregating(tc.qname, tc.gname, Pagination{Page: tc.page, Size: tc.size})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"ListAggregating returned error: %v\", err)\n\t\t\t}\n\n\t\t\tif len(got) != tc.wantSize {\n\t\t\t\tt.Errorf(\"got %d results, want %d\", len(got), tc.wantSize)\n\t\t\t}\n\n\t\t\tif len(got) == 0 {\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tfirst := got[0].Message\n\t\t\tif first.Type != tc.wantFirst {\n\t\t\t\tt.Errorf(\"First message %q, want %q\", first.Type, tc.wantFirst)\n\t\t\t}\n\n\t\t\tlast := got[len(got)-1].Message\n\t\t\tif last.Type != tc.wantLast {\n\t\t\t\tt.Errorf(\"Last message %q, want %q\", last.Type, tc.wantLast)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestListTasksError(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\ttests := []struct {\n\t\tdesc string\n\t\tqname string\n\t\tmatch func(err error) bool\n\t}{\n\t\t{\n\t\t\tdesc: \"It returns QueueNotFoundError if queue doesn't exist\",\n\t\t\tqname: \"nonexistent\",\n\t\t\tmatch: errors.IsQueueNotFound,\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tpgn := Pagination{Page: 0, Size: 20}\n\t\tif _, got := r.ListActive(tc.qname, pgn); !tc.match(got) {\n\t\t\tt.Errorf(\"%s: ListActive returned %v\", tc.desc, got)\n\t\t}\n\t\tif _, got := r.ListPending(tc.qname, pgn); !tc.match(got) {\n\t\t\tt.Errorf(\"%s: ListPending returned %v\", tc.desc, got)\n\t\t}\n\t\tif _, got := r.ListScheduled(tc.qname, pgn); !tc.match(got) {\n\t\t\tt.Errorf(\"%s: ListScheduled returned %v\", tc.desc, got)\n\t\t}\n\t\tif _, got := r.ListRetry(tc.qname, pgn); !tc.match(got) {\n\t\t\tt.Errorf(\"%s: ListRetry returned %v\", tc.desc, got)\n\t\t}\n\t\tif _, got := r.ListArchived(tc.qname, pgn); !tc.match(got) {\n\t\t\tt.Errorf(\"%s: ListArchived returned %v\", tc.desc, got)\n\t\t}\n\t}\n}\n\nvar (\n\ttimeCmpOpt = cmpopts.EquateApproxTime(2 * time.Second) // allow for 2 seconds margin in time.Time\n\tzScoreCmpOpt = h.EquateInt64Approx(2) // allow for 2 seconds margin in Z.Score\n)\n\nfunc TestRunArchivedTask(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tt1 := h.NewTaskMessage(\"send_email\", nil)\n\tt2 := h.NewTaskMessage(\"gen_thumbnail\", nil)\n\tt3 := h.NewTaskMessageWithQueue(\"send_notification\", nil, \"critical\")\n\ts1 := time.Now().Add(-5 * time.Minute).Unix()\n\ts2 := time.Now().Add(-time.Hour).Unix()\n\n\ttests := []struct {\n\t\tarchived map[string][]base.Z\n\t\tqname string\n\t\tid string\n\t\twantArchived map[string][]*base.TaskMessage\n\t\twantPending map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: s1},\n\t\t\t\t\t{Message: t2, Score: s2},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: t2.ID,\n\t\t\twantArchived: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t2},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: s1},\n\t\t\t\t\t{Message: t2, Score: s2},\n\t\t\t\t},\n\t\t\t\t\"critical\": {\n\t\t\t\t\t{Message: t3, Score: s1},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"critical\",\n\t\t\tid: t3.ID,\n\t\t\twantArchived: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t2},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {t3},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllArchivedQueues(t, r.client, tc.archived)\n\n\t\tif got := r.RunTask(tc.qname, tc.id); got != nil {\n\t\t\tt.Errorf(\"r.RunTask(%q, %s) returned error: %v\", tc.qname, tc.id, got)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.PendingKey(qname), diff)\n\t\t\t}\n\t\t}\n\n\t\tfor qname, want := range tc.wantArchived {\n\t\t\tgotArchived := h.GetArchivedMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotArchived, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q, (-want, +got)\\n%s\", base.ArchivedKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestRunRetryTask(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tt1 := h.NewTaskMessage(\"send_email\", nil)\n\tt2 := h.NewTaskMessage(\"gen_thumbnail\", nil)\n\tt3 := h.NewTaskMessageWithQueue(\"send_notification\", nil, \"low\")\n\ts1 := time.Now().Add(-5 * time.Minute).Unix()\n\ts2 := time.Now().Add(-time.Hour).Unix()\n\ttests := []struct {\n\t\tretry map[string][]base.Z\n\t\tqname string\n\t\tid string\n\t\twantRetry map[string][]*base.TaskMessage\n\t\twantPending map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: s1},\n\t\t\t\t\t{Message: t2, Score: s2},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: t2.ID,\n\t\t\twantRetry: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t2},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: s1},\n\t\t\t\t\t{Message: t2, Score: s2},\n\t\t\t\t},\n\t\t\t\t\"low\": {\n\t\t\t\t\t{Message: t3, Score: s2},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"low\",\n\t\t\tid: t3.ID,\n\t\t\twantRetry: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t2},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"low\": {t3},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllRetryQueues(t, r.client, tc.retry) // initialize retry queue\n\n\t\tif got := r.RunTask(tc.qname, tc.id); got != nil {\n\t\t\tt.Errorf(\"r.RunTask(%q, %s) returned error: %v\", tc.qname, tc.id, got)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.PendingKey(qname), diff)\n\t\t\t}\n\t\t}\n\n\t\tfor qname, want := range tc.wantRetry {\n\t\t\tgotRetry := h.GetRetryMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotRetry, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q, (-want, +got)\\n%s\", base.RetryKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestRunAggregatingTask(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tnow := time.Now()\n\tr.SetClock(timeutil.NewSimulatedClock(now))\n\tm1 := h.NewTaskMessageBuilder().SetQueue(\"default\").SetType(\"task1\").SetGroup(\"group1\").Build()\n\tm2 := h.NewTaskMessageBuilder().SetQueue(\"default\").SetType(\"task2\").SetGroup(\"group1\").Build()\n\tm3 := h.NewTaskMessageBuilder().SetQueue(\"custom\").SetType(\"task3\").SetGroup(\"group1\").Build()\n\n\tfxt := struct {\n\t\ttasks []*h.TaskSeedData\n\t\tallQueues []string\n\t\tallGroups map[string][]string\n\t\tgroups map[string][]redis.Z\n\t}{\n\t\ttasks: []*h.TaskSeedData{\n\t\t\t{Msg: m1, State: base.TaskStateAggregating},\n\t\t\t{Msg: m2, State: base.TaskStateAggregating},\n\t\t\t{Msg: m3, State: base.TaskStateAggregating},\n\t\t},\n\t\tallQueues: []string{\"default\", \"custom\"},\n\t\tallGroups: map[string][]string{\n\t\t\tbase.AllGroups(\"default\"): {\"group1\"},\n\t\t\tbase.AllGroups(\"custom\"): {\"group1\"},\n\t\t},\n\t\tgroups: map[string][]redis.Z{\n\t\t\tbase.GroupKey(\"default\", \"group1\"): {\n\t\t\t\t{Member: m1.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t\t{Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())},\n\t\t\t},\n\t\t\tbase.GroupKey(\"custom\", \"group1\"): {\n\t\t\t\t{Member: m3.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t},\n\t\t},\n\t}\n\n\ttests := []struct {\n\t\tdesc string\n\t\tqname string\n\t\tid string\n\t\twantPending map[string][]string\n\t\twantAllGroups map[string][]string\n\t\twantGroups map[string][]redis.Z\n\t}{\n\t\t{\n\t\t\tdesc: \"schedules task from a group with multiple tasks\",\n\t\t\tqname: \"default\",\n\t\t\tid: m1.ID,\n\t\t\twantPending: map[string][]string{\n\t\t\t\tbase.PendingKey(\"default\"): {m1.ID},\n\t\t\t},\n\t\t\twantAllGroups: map[string][]string{\n\t\t\t\tbase.AllGroups(\"default\"): {\"group1\"},\n\t\t\t\tbase.AllGroups(\"custom\"): {\"group1\"},\n\t\t\t},\n\t\t\twantGroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"group1\"): {\n\t\t\t\t\t{Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t\tbase.GroupKey(\"custom\", \"group1\"): {\n\t\t\t\t\t{Member: m3.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"schedules task from a group with a single task\",\n\t\t\tqname: \"custom\",\n\t\t\tid: m3.ID,\n\t\t\twantPending: map[string][]string{\n\t\t\t\tbase.PendingKey(\"custom\"): {m3.ID},\n\t\t\t},\n\t\t\twantAllGroups: map[string][]string{\n\t\t\t\tbase.AllGroups(\"default\"): {\"group1\"},\n\t\t\t\tbase.AllGroups(\"custom\"): {},\n\t\t\t},\n\t\t\twantGroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"group1\"): {\n\t\t\t\t\t{Member: m1.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t\t\t{Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t\tbase.GroupKey(\"custom\", \"group1\"): {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\th.SeedTasks(t, r.client, fxt.tasks)\n\t\th.SeedRedisSet(t, r.client, base.AllQueues, fxt.allQueues)\n\t\th.SeedRedisSets(t, r.client, fxt.allGroups)\n\t\th.SeedRedisZSets(t, r.client, fxt.groups)\n\n\t\tt.Run(tc.desc, func(t *testing.T) {\n\t\t\terr := r.RunTask(tc.qname, tc.id)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"RunTask returned error: %v\", err)\n\t\t\t}\n\n\t\t\th.AssertRedisLists(t, r.client, tc.wantPending)\n\t\t\th.AssertRedisZSets(t, r.client, tc.wantGroups)\n\t\t\th.AssertRedisSets(t, r.client, tc.wantAllGroups)\n\t\t})\n\t}\n}\n\nfunc TestRunScheduledTask(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tt1 := h.NewTaskMessage(\"send_email\", nil)\n\tt2 := h.NewTaskMessage(\"gen_thumbnail\", nil)\n\tt3 := h.NewTaskMessageWithQueue(\"send_notification\", nil, \"notifications\")\n\ts1 := time.Now().Add(-5 * time.Minute).Unix()\n\ts2 := time.Now().Add(-time.Hour).Unix()\n\n\ttests := []struct {\n\t\tscheduled map[string][]base.Z\n\t\tqname string\n\t\tid string\n\t\twantScheduled map[string][]*base.TaskMessage\n\t\twantPending map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: s1},\n\t\t\t\t\t{Message: t2, Score: s2},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: t2.ID,\n\t\t\twantScheduled: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t2},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: s1},\n\t\t\t\t\t{Message: t2, Score: s2},\n\t\t\t\t},\n\t\t\t\t\"notifications\": {\n\t\t\t\t\t{Message: t3, Score: s1},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"notifications\",\n\t\t\tid: t3.ID,\n\t\t\twantScheduled: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t2},\n\t\t\t\t\"notifications\": {},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"notifications\": {t3},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllScheduledQueues(t, r.client, tc.scheduled)\n\n\t\tif got := r.RunTask(tc.qname, tc.id); got != nil {\n\t\t\tt.Errorf(\"r.RunTask(%q, %s) returned error: %v\", tc.qname, tc.id, got)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.PendingKey(qname), diff)\n\t\t\t}\n\t\t}\n\n\t\tfor qname, want := range tc.wantScheduled {\n\t\t\tgotScheduled := h.GetScheduledMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotScheduled, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q, (-want, +got)\\n%s\", base.ScheduledKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestRunTaskError(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tt1 := h.NewTaskMessage(\"send_email\", nil)\n\ts1 := time.Now().Add(-5 * time.Minute).Unix()\n\n\ttests := []struct {\n\t\tdesc string\n\t\tactive map[string][]*base.TaskMessage\n\t\tpending map[string][]*base.TaskMessage\n\t\tscheduled map[string][]base.Z\n\t\tqname string\n\t\tid string\n\t\tmatch func(err error) bool\n\t\twantActive map[string][]*base.TaskMessage\n\t\twantPending map[string][]*base.TaskMessage\n\t\twantScheduled map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tdesc: \"It should return QueueNotFoundError if the queue doesn't exist\",\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: s1},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"nonexistent\",\n\t\t\tid: t1.ID,\n\t\t\tmatch: errors.IsQueueNotFound,\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantScheduled: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"It should return TaskNotFound if the task is not found in the queue\",\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: s1},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: uuid.NewString(),\n\t\t\tmatch: errors.IsTaskNotFound,\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantScheduled: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"It should return FailedPrecondition error if task is already active\",\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t},\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: t1.ID,\n\t\t\tmatch: func(err error) bool { return errors.CanonicalCode(err) == errors.FailedPrecondition },\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantScheduled: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"It should return FailedPrecondition error if task is already pending\",\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t},\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: t1.ID,\n\t\t\tmatch: func(err error) bool { return errors.CanonicalCode(err) == errors.FailedPrecondition },\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t},\n\t\t\twantScheduled: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllActiveQueues(t, r.client, tc.active)\n\t\th.SeedAllPendingQueues(t, r.client, tc.pending)\n\t\th.SeedAllScheduledQueues(t, r.client, tc.scheduled)\n\n\t\tgot := r.RunTask(tc.qname, tc.id)\n\t\tif !tc.match(got) {\n\t\t\tt.Errorf(\"%s: unexpected return value %v\", tc.desc, got)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor qname, want := range tc.wantActive {\n\t\t\tgotActive := h.GetActiveMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q: (-want, +got)\\n%s\", base.ActiveKey(qname), diff)\n\t\t\t}\n\t\t}\n\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.PendingKey(qname), diff)\n\t\t\t}\n\t\t}\n\n\t\tfor qname, want := range tc.wantScheduled {\n\t\t\tgotScheduled := h.GetScheduledMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotScheduled, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q, (-want, +got)\\n%s\", base.ScheduledKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestRunAllScheduledTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tt1 := h.NewTaskMessage(\"send_email\", nil)\n\tt2 := h.NewTaskMessage(\"gen_thumbnail\", nil)\n\tt3 := h.NewTaskMessage(\"reindex\", nil)\n\tt4 := h.NewTaskMessageWithQueue(\"important_notification\", nil, \"custom\")\n\tt5 := h.NewTaskMessageWithQueue(\"minor_notification\", nil, \"custom\")\n\n\ttests := []struct {\n\t\tdesc string\n\t\tscheduled map[string][]base.Z\n\t\tqname string\n\t\twant int64\n\t\twantPending map[string][]*base.TaskMessage\n\t\twantScheduled map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tdesc: \"with tasks in scheduled queue\",\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: time.Now().Add(time.Hour).Unix()},\n\t\t\t\t\t{Message: t2, Score: time.Now().Add(time.Hour).Unix()},\n\t\t\t\t\t{Message: t3, Score: time.Now().Add(time.Hour).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 3,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t2, t3},\n\t\t\t},\n\t\t\twantScheduled: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with empty scheduled queue\",\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 0,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantScheduled: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with custom queues\",\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: time.Now().Add(time.Hour).Unix()},\n\t\t\t\t\t{Message: t2, Score: time.Now().Add(time.Hour).Unix()},\n\t\t\t\t\t{Message: t3, Score: time.Now().Add(time.Hour).Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: t4, Score: time.Now().Add(time.Hour).Unix()},\n\t\t\t\t\t{Message: t5, Score: time.Now().Add(time.Hour).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\twant: 2,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {t4, t5},\n\t\t\t},\n\t\t\twantScheduled: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t2, t3},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllScheduledQueues(t, r.client, tc.scheduled)\n\n\t\tgot, err := r.RunAllScheduledTasks(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s; r.RunAllScheduledTasks(%q) = %v, %v; want %v, nil\",\n\t\t\t\ttc.desc, tc.qname, got, err, tc.want)\n\t\t\tcontinue\n\t\t}\n\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"%s; r.RunAllScheduledTasks(%q) = %v, %v; want %v, nil\",\n\t\t\t\ttc.desc, tc.qname, got, err, tc.want)\n\t\t}\n\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s; mismatch found in %q; (-want, +got)\\n%s\", tc.desc, base.PendingKey(qname), diff)\n\t\t\t}\n\t\t}\n\t\tfor qname, want := range tc.wantScheduled {\n\t\t\tgotScheduled := h.GetScheduledMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotScheduled, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s; mismatch found in %q; (-want, +got)\\n%s\", tc.desc, base.ScheduledKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestRunAllRetryTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tt1 := h.NewTaskMessage(\"send_email\", nil)\n\tt2 := h.NewTaskMessage(\"gen_thumbnail\", nil)\n\tt3 := h.NewTaskMessage(\"reindex\", nil)\n\tt4 := h.NewTaskMessageWithQueue(\"important_notification\", nil, \"custom\")\n\tt5 := h.NewTaskMessageWithQueue(\"minor_notification\", nil, \"custom\")\n\n\ttests := []struct {\n\t\tdesc string\n\t\tretry map[string][]base.Z\n\t\tqname string\n\t\twant int64\n\t\twantPending map[string][]*base.TaskMessage\n\t\twantRetry map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tdesc: \"with tasks in retry queue\",\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: time.Now().Add(time.Hour).Unix()},\n\t\t\t\t\t{Message: t2, Score: time.Now().Add(time.Hour).Unix()},\n\t\t\t\t\t{Message: t3, Score: time.Now().Add(time.Hour).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 3,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t2, t3},\n\t\t\t},\n\t\t\twantRetry: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with empty retry queue\",\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 0,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantRetry: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with custom queues\",\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: time.Now().Add(time.Hour).Unix()},\n\t\t\t\t\t{Message: t2, Score: time.Now().Add(time.Hour).Unix()},\n\t\t\t\t\t{Message: t3, Score: time.Now().Add(time.Hour).Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: t4, Score: time.Now().Add(time.Hour).Unix()},\n\t\t\t\t\t{Message: t5, Score: time.Now().Add(time.Hour).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\twant: 2,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {t4, t5},\n\t\t\t},\n\t\t\twantRetry: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t2, t3},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllRetryQueues(t, r.client, tc.retry)\n\n\t\tgot, err := r.RunAllRetryTasks(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s; r.RunAllRetryTasks(%q) = %v, %v; want %v, nil\",\n\t\t\t\ttc.desc, tc.qname, got, err, tc.want)\n\t\t\tcontinue\n\t\t}\n\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"%s; r.RunAllRetryTasks(%q) = %v, %v; want %v, nil\",\n\t\t\t\ttc.desc, tc.qname, got, err, tc.want)\n\t\t}\n\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s; mismatch found in %q; (-want, +got)\\n%s\", tc.desc, base.PendingKey(qname), diff)\n\t\t\t}\n\t\t}\n\t\tfor qname, want := range tc.wantRetry {\n\t\t\tgotRetry := h.GetRetryMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotRetry, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s; mismatch found in %q; (-want, +got)\\n%s\", tc.desc, base.RetryKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestRunAllArchivedTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tt1 := h.NewTaskMessage(\"send_email\", nil)\n\tt2 := h.NewTaskMessage(\"gen_thumbnail\", nil)\n\tt3 := h.NewTaskMessage(\"reindex\", nil)\n\tt4 := h.NewTaskMessageWithQueue(\"important_notification\", nil, \"custom\")\n\tt5 := h.NewTaskMessageWithQueue(\"minor_notification\", nil, \"custom\")\n\n\ttests := []struct {\n\t\tdesc string\n\t\tarchived map[string][]base.Z\n\t\tqname string\n\t\twant int64\n\t\twantPending map[string][]*base.TaskMessage\n\t\twantArchived map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tdesc: \"with tasks in archived queue\",\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: time.Now().Add(-time.Minute).Unix()},\n\t\t\t\t\t{Message: t2, Score: time.Now().Add(-time.Minute).Unix()},\n\t\t\t\t\t{Message: t3, Score: time.Now().Add(-time.Minute).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 3,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t2, t3},\n\t\t\t},\n\t\t\twantArchived: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with empty archived queue\",\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 0,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantArchived: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with custom queues\",\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: time.Now().Add(-time.Minute).Unix()},\n\t\t\t\t\t{Message: t2, Score: time.Now().Add(-time.Minute).Unix()},\n\t\t\t\t\t{Message: t3, Score: time.Now().Add(-time.Minute).Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: t4, Score: time.Now().Add(-time.Minute).Unix()},\n\t\t\t\t\t{Message: t5, Score: time.Now().Add(-time.Minute).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\twant: 2,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {t4, t5},\n\t\t\t},\n\t\t\twantArchived: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t2, t3},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllArchivedQueues(t, r.client, tc.archived)\n\n\t\tgot, err := r.RunAllArchivedTasks(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s; r.RunAllArchivedTasks(%q) = %v, %v; want %v, nil\",\n\t\t\t\ttc.desc, tc.qname, got, err, tc.want)\n\t\t\tcontinue\n\t\t}\n\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"%s; r.RunAllArchivedTasks(%q) = %v, %v; want %v, nil\",\n\t\t\t\ttc.desc, tc.qname, got, err, tc.want)\n\t\t}\n\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s; mismatch found in %q; (-want, +got)\\n%s\", tc.desc, base.PendingKey(qname), diff)\n\t\t\t}\n\t\t}\n\t\tfor qname, want := range tc.wantArchived {\n\t\t\tgotArchived := h.GetArchivedMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotArchived, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s; mismatch found in %q; (-want, +got)\\n%s\", tc.desc, base.ArchivedKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestRunAllTasksError(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\ttests := []struct {\n\t\tdesc string\n\t\tqname string\n\t\tmatch func(err error) bool\n\t}{\n\t\t{\n\t\t\tdesc: \"It returns QueueNotFoundError if queue doesn't exist\",\n\t\t\tqname: \"nonexistent\",\n\t\t\tmatch: errors.IsQueueNotFound,\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tif _, got := r.RunAllScheduledTasks(tc.qname); !tc.match(got) {\n\t\t\tt.Errorf(\"%s: RunAllScheduledTasks returned %v\", tc.desc, got)\n\t\t}\n\t\tif _, got := r.RunAllRetryTasks(tc.qname); !tc.match(got) {\n\t\t\tt.Errorf(\"%s: RunAllRetryTasks returned %v\", tc.desc, got)\n\t\t}\n\t\tif _, got := r.RunAllArchivedTasks(tc.qname); !tc.match(got) {\n\t\t\tt.Errorf(\"%s: RunAllArchivedTasks returned %v\", tc.desc, got)\n\t\t}\n\t\tif _, got := r.RunAllAggregatingTasks(tc.qname, \"mygroup\"); !tc.match(got) {\n\t\t\tt.Errorf(\"%s: RunAllAggregatingTasks returned %v\", tc.desc, got)\n\t\t}\n\t}\n}\n\nfunc TestRunAllAggregatingTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tnow := time.Now()\n\tr.SetClock(timeutil.NewSimulatedClock(now))\n\n\tm1 := h.NewTaskMessageBuilder().SetQueue(\"default\").SetType(\"task1\").SetGroup(\"group1\").Build()\n\tm2 := h.NewTaskMessageBuilder().SetQueue(\"default\").SetType(\"task2\").SetGroup(\"group1\").Build()\n\tm3 := h.NewTaskMessageBuilder().SetQueue(\"custom\").SetType(\"task3\").SetGroup(\"group2\").Build()\n\n\tfxt := struct {\n\t\ttasks []*h.TaskSeedData\n\t\tallQueues []string\n\t\tallGroups map[string][]string\n\t\tgroups map[string][]redis.Z\n\t}{\n\t\ttasks: []*h.TaskSeedData{\n\t\t\t{Msg: m1, State: base.TaskStateAggregating},\n\t\t\t{Msg: m2, State: base.TaskStateAggregating},\n\t\t\t{Msg: m3, State: base.TaskStateAggregating},\n\t\t},\n\t\tallQueues: []string{\"default\", \"custom\"},\n\t\tallGroups: map[string][]string{\n\t\t\tbase.AllGroups(\"default\"): {\"group1\"},\n\t\t\tbase.AllGroups(\"custom\"): {\"group2\"},\n\t\t},\n\t\tgroups: map[string][]redis.Z{\n\t\t\tbase.GroupKey(\"default\", \"group1\"): {\n\t\t\t\t{Member: m1.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t\t{Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())},\n\t\t\t},\n\t\t\tbase.GroupKey(\"custom\", \"group2\"): {\n\t\t\t\t{Member: m3.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t},\n\t\t},\n\t}\n\n\ttests := []struct {\n\t\tdesc string\n\t\tqname string\n\t\tgname string\n\t\twant int64\n\t\twantPending map[string][]string\n\t\twantGroups map[string][]redis.Z\n\t\twantAllGroups map[string][]string\n\t}{\n\t\t{\n\t\t\tdesc: \"schedules tasks in a group with multiple tasks\",\n\t\t\tqname: \"default\",\n\t\t\tgname: \"group1\",\n\t\t\twant: 2,\n\t\t\twantPending: map[string][]string{\n\t\t\t\tbase.PendingKey(\"default\"): {m1.ID, m2.ID},\n\t\t\t},\n\t\t\twantGroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"group1\"): {},\n\t\t\t\tbase.GroupKey(\"custom\", \"group2\"): {\n\t\t\t\t\t{Member: m3.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\twantAllGroups: map[string][]string{\n\t\t\t\tbase.AllGroups(\"default\"): {},\n\t\t\t\tbase.AllGroups(\"custom\"): {\"group2\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"schedules tasks in a group with a single task\",\n\t\t\tqname: \"custom\",\n\t\t\tgname: \"group2\",\n\t\t\twant: 1,\n\t\t\twantPending: map[string][]string{\n\t\t\t\tbase.PendingKey(\"custom\"): {m3.ID},\n\t\t\t},\n\t\t\twantGroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"group1\"): {\n\t\t\t\t\t{Member: m1.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t\t\t{Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t\tbase.GroupKey(\"custom\", \"group2\"): {},\n\t\t\t},\n\t\t\twantAllGroups: map[string][]string{\n\t\t\t\tbase.AllGroups(\"default\"): {\"group1\"},\n\t\t\t\tbase.AllGroups(\"custom\"): {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\th.SeedTasks(t, r.client, fxt.tasks)\n\t\th.SeedRedisSet(t, r.client, base.AllQueues, fxt.allQueues)\n\t\th.SeedRedisSets(t, r.client, fxt.allGroups)\n\t\th.SeedRedisZSets(t, r.client, fxt.groups)\n\n\t\tt.Run(tc.desc, func(t *testing.T) {\n\t\t\tgot, err := r.RunAllAggregatingTasks(tc.qname, tc.gname)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"RunAllAggregatingTasks returned error: %v\", err)\n\t\t\t}\n\t\t\tif got != tc.want {\n\t\t\t\tt.Errorf(\"RunAllAggregatingTasks = %d, want %d\", got, tc.want)\n\t\t\t}\n\t\t\th.AssertRedisLists(t, r.client, tc.wantPending)\n\t\t\th.AssertRedisZSets(t, r.client, tc.wantGroups)\n\t\t\th.AssertRedisSets(t, r.client, tc.wantAllGroups)\n\t\t})\n\t}\n}\n\nfunc TestArchiveRetryTask(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tm4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"custom\")\n\tt1 := time.Now().Add(1 * time.Minute)\n\tt2 := time.Now().Add(1 * time.Hour)\n\tt3 := time.Now().Add(2 * time.Hour)\n\tt4 := time.Now().Add(3 * time.Hour)\n\n\ttests := []struct {\n\t\tretry map[string][]base.Z\n\t\tarchived map[string][]base.Z\n\t\tqname string\n\t\tid string\n\t\twantRetry map[string][]base.Z\n\t\twantArchived map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: t1.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: m1.ID,\n\t\t\twantRetry: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: m2, Score: t2.Unix()}},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: m1, Score: time.Now().Unix()}},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: t1.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: m3, Score: t3.Unix()},\n\t\t\t\t\t{Message: m4, Score: t4.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\tid: m3.ID,\n\t\t\twantRetry: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: t1.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: m4, Score: t4.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {{Message: m3, Score: time.Now().Unix()}},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\th.SeedAllRetryQueues(t, r.client, tc.retry)\n\t\th.SeedAllArchivedQueues(t, r.client, tc.archived)\n\n\t\tif got := r.ArchiveTask(tc.qname, tc.id); got != nil {\n\t\t\tt.Errorf(\"(*RDB).ArchiveTask(%q, %v) returned error: %v\",\n\t\t\t\ttc.qname, tc.id, got)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor qname, want := range tc.wantRetry {\n\t\t\tgotRetry := h.GetRetryEntries(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotRetry, h.SortZSetEntryOpt, zScoreCmpOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want,+got)\\n%s\",\n\t\t\t\t\tbase.RetryKey(qname), diff)\n\t\t\t}\n\t\t}\n\n\t\tfor qname, want := range tc.wantArchived {\n\t\t\tgotArchived := h.GetArchivedEntries(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt, zScoreCmpOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want,+got)\\n%s\",\n\t\t\t\t\tbase.ArchivedKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestArchiveScheduledTask(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tm4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"custom\")\n\tt1 := time.Now().Add(1 * time.Minute)\n\tt2 := time.Now().Add(1 * time.Hour)\n\tt3 := time.Now().Add(2 * time.Hour)\n\tt4 := time.Now().Add(3 * time.Hour)\n\n\ttests := []struct {\n\t\tscheduled map[string][]base.Z\n\t\tarchived map[string][]base.Z\n\t\tqname string\n\t\tid string\n\t\twantScheduled map[string][]base.Z\n\t\twantArchived map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: t1.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: m1.ID,\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: m2, Score: t2.Unix()}},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: m1, Score: time.Now().Unix()}},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: t1.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: m3, Score: t3.Unix()},\n\t\t\t\t\t{Message: m4, Score: t4.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\tid: m3.ID,\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: t1.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: m4, Score: t4.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {{Message: m3, Score: time.Now().Unix()}},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\th.SeedAllScheduledQueues(t, r.client, tc.scheduled)\n\t\th.SeedAllArchivedQueues(t, r.client, tc.archived)\n\n\t\tif got := r.ArchiveTask(tc.qname, tc.id); got != nil {\n\t\t\tt.Errorf(\"(*RDB).ArchiveTask(%q, %v) returned error: %v\",\n\t\t\t\ttc.qname, tc.id, got)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor qname, want := range tc.wantScheduled {\n\t\t\tgotScheduled := h.GetScheduledEntries(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotScheduled, h.SortZSetEntryOpt, zScoreCmpOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want,+got)\\n%s\",\n\t\t\t\t\tbase.ScheduledKey(qname), diff)\n\t\t\t}\n\t\t}\n\n\t\tfor qname, want := range tc.wantArchived {\n\t\t\tgotArchived := h.GetArchivedEntries(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt, zScoreCmpOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want,+got)\\n%s\",\n\t\t\t\t\tbase.ArchivedKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestArchiveAggregatingTask(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tnow := time.Now()\n\tr.SetClock(timeutil.NewSimulatedClock(now))\n\tm1 := h.NewTaskMessageBuilder().SetQueue(\"default\").SetType(\"task1\").SetGroup(\"group1\").Build()\n\tm2 := h.NewTaskMessageBuilder().SetQueue(\"default\").SetType(\"task2\").SetGroup(\"group1\").Build()\n\tm3 := h.NewTaskMessageBuilder().SetQueue(\"custom\").SetType(\"task3\").SetGroup(\"group1\").Build()\n\n\tfxt := struct {\n\t\ttasks []*h.TaskSeedData\n\t\tallQueues []string\n\t\tallGroups map[string][]string\n\t\tgroups map[string][]redis.Z\n\t}{\n\t\ttasks: []*h.TaskSeedData{\n\t\t\t{Msg: m1, State: base.TaskStateAggregating},\n\t\t\t{Msg: m2, State: base.TaskStateAggregating},\n\t\t\t{Msg: m3, State: base.TaskStateAggregating},\n\t\t},\n\t\tallQueues: []string{\"default\", \"custom\"},\n\t\tallGroups: map[string][]string{\n\t\t\tbase.AllGroups(\"default\"): {\"group1\"},\n\t\t\tbase.AllGroups(\"custom\"): {\"group1\"},\n\t\t},\n\t\tgroups: map[string][]redis.Z{\n\t\t\tbase.GroupKey(\"default\", \"group1\"): {\n\t\t\t\t{Member: m1.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t\t{Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())},\n\t\t\t},\n\t\t\tbase.GroupKey(\"custom\", \"group1\"): {\n\t\t\t\t{Member: m3.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t},\n\t\t},\n\t}\n\n\ttests := []struct {\n\t\tdesc string\n\t\tqname string\n\t\tid string\n\t\twantArchived map[string][]redis.Z\n\t\twantAllGroups map[string][]string\n\t\twantGroups map[string][]redis.Z\n\t}{\n\t\t{\n\t\t\tdesc: \"archive task from a group with multiple tasks\",\n\t\t\tqname: \"default\",\n\t\t\tid: m1.ID,\n\t\t\twantArchived: map[string][]redis.Z{\n\t\t\t\tbase.ArchivedKey(\"default\"): {\n\t\t\t\t\t{Member: m1.ID, Score: float64(now.Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\twantAllGroups: map[string][]string{\n\t\t\t\tbase.AllGroups(\"default\"): {\"group1\"},\n\t\t\t\tbase.AllGroups(\"custom\"): {\"group1\"},\n\t\t\t},\n\t\t\twantGroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"group1\"): {\n\t\t\t\t\t{Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t\tbase.GroupKey(\"custom\", \"group1\"): {\n\t\t\t\t\t{Member: m3.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"archive task from a group with a single task\",\n\t\t\tqname: \"custom\",\n\t\t\tid: m3.ID,\n\t\t\twantArchived: map[string][]redis.Z{\n\t\t\t\tbase.ArchivedKey(\"custom\"): {\n\t\t\t\t\t{Member: m3.ID, Score: float64(now.Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\twantAllGroups: map[string][]string{\n\t\t\t\tbase.AllGroups(\"default\"): {\"group1\"},\n\t\t\t\tbase.AllGroups(\"custom\"): {},\n\t\t\t},\n\t\t\twantGroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"group1\"): {\n\t\t\t\t\t{Member: m1.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t\t\t{Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t\tbase.GroupKey(\"custom\", \"group1\"): {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\th.SeedTasks(t, r.client, fxt.tasks)\n\t\th.SeedRedisSet(t, r.client, base.AllQueues, fxt.allQueues)\n\t\th.SeedRedisSets(t, r.client, fxt.allGroups)\n\t\th.SeedRedisZSets(t, r.client, fxt.groups)\n\n\t\tt.Run(tc.desc, func(t *testing.T) {\n\t\t\terr := r.ArchiveTask(tc.qname, tc.id)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"ArchiveTask returned error: %v\", err)\n\t\t\t}\n\n\t\t\th.AssertRedisZSets(t, r.client, tc.wantArchived)\n\t\t\th.AssertRedisZSets(t, r.client, tc.wantGroups)\n\t\t\th.AssertRedisSets(t, r.client, tc.wantAllGroups)\n\t\t})\n\t}\n}\n\nfunc TestArchivePendingTask(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tm4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"custom\")\n\n\ttests := []struct {\n\t\tpending map[string][]*base.TaskMessage\n\t\tarchived map[string][]base.Z\n\t\tqname string\n\t\tid string\n\t\twantPending map[string][]*base.TaskMessage\n\t\twantArchived map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: m1.ID,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m2},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: m1, Score: time.Now().Unix()}},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t\t\"custom\": {m3, m4},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\tid: m3.ID,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t\t\"custom\": {m4},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {{Message: m3, Score: time.Now().Unix()}},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\th.SeedAllPendingQueues(t, r.client, tc.pending)\n\t\th.SeedAllArchivedQueues(t, r.client, tc.archived)\n\n\t\tif got := r.ArchiveTask(tc.qname, tc.id); got != nil {\n\t\t\tt.Errorf(\"(*RDB).ArchiveTask(%q, %v) returned error: %v\",\n\t\t\t\ttc.qname, tc.id, got)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want,+got)\\n%s\",\n\t\t\t\t\tbase.PendingKey(qname), diff)\n\t\t\t}\n\t\t}\n\n\t\tfor qname, want := range tc.wantArchived {\n\t\t\tgotArchived := h.GetArchivedEntries(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt, zScoreCmpOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want,+got)\\n%s\",\n\t\t\t\t\tbase.ArchivedKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestArchiveTaskError(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tt1 := time.Now().Add(1 * time.Minute)\n\tt2 := time.Now().Add(1 * time.Hour)\n\n\ttests := []struct {\n\t\tdesc string\n\t\tactive map[string][]*base.TaskMessage\n\t\tscheduled map[string][]base.Z\n\t\tarchived map[string][]base.Z\n\t\tqname string\n\t\tid string\n\t\tmatch func(err error) bool\n\t\twantActive map[string][]*base.TaskMessage\n\t\twantScheduled map[string][]base.Z\n\t\twantArchived map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tdesc: \"It should return QueueNotFoundError if provided queue name doesn't exist\",\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: m1, Score: t1.Unix()}},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: m2, Score: t2.Unix()}},\n\t\t\t},\n\t\t\tqname: \"nonexistent\",\n\t\t\tid: m2.ID,\n\t\t\tmatch: errors.IsQueueNotFound,\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: m1, Score: t1.Unix()}},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: m2, Score: t2.Unix()}},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"It should return TaskNotFoundError if provided task ID doesn't exist in the queue\",\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: m1, Score: t1.Unix()}},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: m2, Score: t2.Unix()}},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: uuid.NewString(),\n\t\t\tmatch: errors.IsTaskNotFound,\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: m1, Score: t1.Unix()}},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: m2, Score: t2.Unix()}},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"It should return TaskAlreadyArchivedError if task is already in archived state\",\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: m1, Score: t1.Unix()}},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: m2, Score: t2.Unix()}},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: m2.ID,\n\t\t\tmatch: errors.IsTaskAlreadyArchived,\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: m1, Score: t1.Unix()}},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: m2, Score: t2.Unix()}},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"It should return FailedPrecondition error if task is active\",\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1},\n\t\t\t},\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: m1.ID,\n\t\t\tmatch: func(err error) bool { return errors.CanonicalCode(err) == errors.FailedPrecondition },\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1},\n\t\t\t},\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\th.SeedAllActiveQueues(t, r.client, tc.active)\n\t\th.SeedAllScheduledQueues(t, r.client, tc.scheduled)\n\t\th.SeedAllArchivedQueues(t, r.client, tc.archived)\n\n\t\tgot := r.ArchiveTask(tc.qname, tc.id)\n\t\tif !tc.match(got) {\n\t\t\tt.Errorf(\"%s: returned error didn't match: got=%v\", tc.desc, got)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor qname, want := range tc.wantActive {\n\t\t\tgotActive := h.GetActiveMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\",\n\t\t\t\t\tbase.ActiveKey(qname), diff)\n\t\t\t}\n\t\t}\n\n\t\tfor qname, want := range tc.wantScheduled {\n\t\t\tgotScheduled := h.GetScheduledEntries(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotScheduled, h.SortZSetEntryOpt, zScoreCmpOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want,+got)\\n%s\",\n\t\t\t\t\tbase.ScheduledKey(qname), diff)\n\t\t\t}\n\t\t}\n\n\t\tfor qname, want := range tc.wantArchived {\n\t\t\tgotArchived := h.GetArchivedEntries(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt, zScoreCmpOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want,+got)\\n%s\",\n\t\t\t\t\tbase.ArchivedKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestArchiveAllPendingTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tm4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"custom\")\n\tnow := time.Now()\n\tt1 := now.Add(1 * time.Minute)\n\tt2 := now.Add(1 * time.Hour)\n\n\tr.SetClock(timeutil.NewSimulatedClock(now))\n\n\ttests := []struct {\n\t\tpending map[string][]*base.TaskMessage\n\t\tarchived map[string][]base.Z\n\t\tqname string\n\t\twant int64\n\t\twantPending map[string][]*base.TaskMessage\n\t\twantArchived map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 2,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: now.Unix()},\n\t\t\t\t\t{Message: m2, Score: now.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: m2, Score: t2.Unix()}},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 1,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: now.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: t1.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 0,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: t1.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t\t\"custom\": {m3, m4},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\twant: 2,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: m3, Score: now.Unix()},\n\t\t\t\t\t{Message: m4, Score: now.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\th.SeedAllPendingQueues(t, r.client, tc.pending)\n\t\th.SeedAllArchivedQueues(t, r.client, tc.archived)\n\n\t\tgot, err := r.ArchiveAllPendingTasks(tc.qname)\n\t\tif got != tc.want || err != nil {\n\t\t\tt.Errorf(\"(*RDB).KillAllRetryTasks(%q) = %v, %v; want %v, nil\",\n\t\t\t\ttc.qname, got, err, tc.want)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want,+got)\\n%s\",\n\t\t\t\t\tbase.PendingKey(qname), diff)\n\t\t\t}\n\t\t}\n\n\t\tfor qname, want := range tc.wantArchived {\n\t\t\tgotArchived := h.GetArchivedEntries(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want,+got)\\n%s\",\n\t\t\t\t\tbase.ArchivedKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestArchiveAllAggregatingTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tnow := time.Now()\n\tr.SetClock(timeutil.NewSimulatedClock(now))\n\n\tm1 := h.NewTaskMessageBuilder().SetQueue(\"default\").SetType(\"task1\").SetGroup(\"group1\").Build()\n\tm2 := h.NewTaskMessageBuilder().SetQueue(\"default\").SetType(\"task2\").SetGroup(\"group1\").Build()\n\tm3 := h.NewTaskMessageBuilder().SetQueue(\"custom\").SetType(\"task3\").SetGroup(\"group2\").Build()\n\n\tfxt := struct {\n\t\ttasks []*h.TaskSeedData\n\t\tallQueues []string\n\t\tallGroups map[string][]string\n\t\tgroups map[string][]redis.Z\n\t}{\n\t\ttasks: []*h.TaskSeedData{\n\t\t\t{Msg: m1, State: base.TaskStateAggregating},\n\t\t\t{Msg: m2, State: base.TaskStateAggregating},\n\t\t\t{Msg: m3, State: base.TaskStateAggregating},\n\t\t},\n\t\tallQueues: []string{\"default\", \"custom\"},\n\t\tallGroups: map[string][]string{\n\t\t\tbase.AllGroups(\"default\"): {\"group1\"},\n\t\t\tbase.AllGroups(\"custom\"): {\"group2\"},\n\t\t},\n\t\tgroups: map[string][]redis.Z{\n\t\t\tbase.GroupKey(\"default\", \"group1\"): {\n\t\t\t\t{Member: m1.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t\t{Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())},\n\t\t\t},\n\t\t\tbase.GroupKey(\"custom\", \"group2\"): {\n\t\t\t\t{Member: m3.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t},\n\t\t},\n\t}\n\n\ttests := []struct {\n\t\tdesc string\n\t\tqname string\n\t\tgname string\n\t\twant int64\n\t\twantArchived map[string][]redis.Z\n\t\twantGroups map[string][]redis.Z\n\t\twantAllGroups map[string][]string\n\t}{\n\t\t{\n\t\t\tdesc: \"archive tasks in a group with multiple tasks\",\n\t\t\tqname: \"default\",\n\t\t\tgname: \"group1\",\n\t\t\twant: 2,\n\t\t\twantArchived: map[string][]redis.Z{\n\t\t\t\tbase.ArchivedKey(\"default\"): {\n\t\t\t\t\t{Member: m1.ID, Score: float64(now.Unix())},\n\t\t\t\t\t{Member: m2.ID, Score: float64(now.Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\twantGroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"group1\"): {},\n\t\t\t\tbase.GroupKey(\"custom\", \"group2\"): {\n\t\t\t\t\t{Member: m3.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\twantAllGroups: map[string][]string{\n\t\t\t\tbase.AllGroups(\"default\"): {},\n\t\t\t\tbase.AllGroups(\"custom\"): {\"group2\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"archive tasks in a group with a single task\",\n\t\t\tqname: \"custom\",\n\t\t\tgname: \"group2\",\n\t\t\twant: 1,\n\t\t\twantArchived: map[string][]redis.Z{\n\t\t\t\tbase.ArchivedKey(\"custom\"): {\n\t\t\t\t\t{Member: m3.ID, Score: float64(now.Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\twantGroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"group1\"): {\n\t\t\t\t\t{Member: m1.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t\t\t{Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t\tbase.GroupKey(\"custom\", \"group2\"): {},\n\t\t\t},\n\t\t\twantAllGroups: map[string][]string{\n\t\t\t\tbase.AllGroups(\"default\"): {\"group1\"},\n\t\t\t\tbase.AllGroups(\"custom\"): {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\th.SeedTasks(t, r.client, fxt.tasks)\n\t\th.SeedRedisSet(t, r.client, base.AllQueues, fxt.allQueues)\n\t\th.SeedRedisSets(t, r.client, fxt.allGroups)\n\t\th.SeedRedisZSets(t, r.client, fxt.groups)\n\n\t\tt.Run(tc.desc, func(t *testing.T) {\n\t\t\tgot, err := r.ArchiveAllAggregatingTasks(tc.qname, tc.gname)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"ArchiveAllAggregatingTasks returned error: %v\", err)\n\t\t\t}\n\t\t\tif got != tc.want {\n\t\t\t\tt.Errorf(\"ArchiveAllAggregatingTasks = %d, want %d\", got, tc.want)\n\t\t\t}\n\t\t\th.AssertRedisZSets(t, r.client, tc.wantArchived)\n\t\t\th.AssertRedisZSets(t, r.client, tc.wantGroups)\n\t\t\th.AssertRedisSets(t, r.client, tc.wantAllGroups)\n\t\t})\n\t}\n}\n\nfunc TestArchiveAllRetryTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tm4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"custom\")\n\tnow := time.Now()\n\tt1 := now.Add(1 * time.Minute)\n\tt2 := now.Add(1 * time.Hour)\n\tt3 := now.Add(2 * time.Hour)\n\tt4 := now.Add(3 * time.Hour)\n\n\tr.SetClock(timeutil.NewSimulatedClock(now))\n\n\ttests := []struct {\n\t\tretry map[string][]base.Z\n\t\tarchived map[string][]base.Z\n\t\tqname string\n\t\twant int64\n\t\twantRetry map[string][]base.Z\n\t\twantArchived map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: t1.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 2,\n\t\t\twantRetry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: now.Unix()},\n\t\t\t\t\t{Message: m2, Score: now.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: m1, Score: t1.Unix()}},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: m2, Score: t2.Unix()}},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 1,\n\t\t\twantRetry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: now.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: t1.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 0,\n\t\t\twantRetry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: t1.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: t1.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: m3, Score: t3.Unix()},\n\t\t\t\t\t{Message: m4, Score: t4.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\twant: 2,\n\t\t\twantRetry: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: t1.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: m3, Score: now.Unix()},\n\t\t\t\t\t{Message: m4, Score: now.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\th.SeedAllRetryQueues(t, r.client, tc.retry)\n\t\th.SeedAllArchivedQueues(t, r.client, tc.archived)\n\n\t\tgot, err := r.ArchiveAllRetryTasks(tc.qname)\n\t\tif got != tc.want || err != nil {\n\t\t\tt.Errorf(\"(*RDB).KillAllRetryTasks(%q) = %v, %v; want %v, nil\",\n\t\t\t\ttc.qname, got, err, tc.want)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor qname, want := range tc.wantRetry {\n\t\t\tgotRetry := h.GetRetryEntries(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotRetry, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want,+got)\\n%s\",\n\t\t\t\t\tbase.RetryKey(qname), diff)\n\t\t\t}\n\t\t}\n\n\t\tfor qname, want := range tc.wantArchived {\n\t\t\tgotArchived := h.GetArchivedEntries(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want,+got)\\n%s\",\n\t\t\t\t\tbase.ArchivedKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestArchiveAllScheduledTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tm4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"custom\")\n\tnow := time.Now()\n\tt1 := now.Add(time.Minute)\n\tt2 := now.Add(time.Hour)\n\tt3 := now.Add(time.Hour)\n\tt4 := now.Add(time.Hour)\n\n\tr.SetClock(timeutil.NewSimulatedClock(now))\n\n\ttests := []struct {\n\t\tscheduled map[string][]base.Z\n\t\tarchived map[string][]base.Z\n\t\tqname string\n\t\twant int64\n\t\twantScheduled map[string][]base.Z\n\t\twantArchived map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: t1.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 2,\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: now.Unix()},\n\t\t\t\t\t{Message: m2, Score: now.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: m1, Score: t1.Unix()}},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: m2, Score: t2.Unix()}},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 1,\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: now.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: t1.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 0,\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: t1.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: t1.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: m3, Score: t3.Unix()},\n\t\t\t\t\t{Message: m4, Score: t4.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\twant: 2,\n\t\t\twantScheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: t1.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: m3, Score: now.Unix()},\n\t\t\t\t\t{Message: m4, Score: now.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\th.SeedAllScheduledQueues(t, r.client, tc.scheduled)\n\t\th.SeedAllArchivedQueues(t, r.client, tc.archived)\n\n\t\tgot, err := r.ArchiveAllScheduledTasks(tc.qname)\n\t\tif got != tc.want || err != nil {\n\t\t\tt.Errorf(\"(*RDB).KillAllScheduledTasks(%q) = %v, %v; want %v, nil\",\n\t\t\t\ttc.qname, got, err, tc.want)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor qname, want := range tc.wantScheduled {\n\t\t\tgotScheduled := h.GetScheduledEntries(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotScheduled, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want,+got)\\n%s\",\n\t\t\t\t\tbase.ScheduledKey(qname), diff)\n\t\t\t}\n\t\t}\n\n\t\tfor qname, want := range tc.wantArchived {\n\t\t\tgotArchived := h.GetArchivedEntries(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want,+got)\\n%s\",\n\t\t\t\t\tbase.ArchivedKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestArchiveAllTasksError(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\ttests := []struct {\n\t\tdesc string\n\t\tqname string\n\t\tmatch func(err error) bool\n\t}{\n\t\t{\n\t\t\tdesc: \"It returns QueueNotFoundError if queue doesn't exist\",\n\t\t\tqname: \"nonexistent\",\n\t\t\tmatch: errors.IsQueueNotFound,\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tif _, got := r.ArchiveAllPendingTasks(tc.qname); !tc.match(got) {\n\t\t\tt.Errorf(\"%s: ArchiveAllPendingTasks returned %v\", tc.desc, got)\n\t\t}\n\t\tif _, got := r.ArchiveAllScheduledTasks(tc.qname); !tc.match(got) {\n\t\t\tt.Errorf(\"%s: ArchiveAllScheduledTasks returned %v\", tc.desc, got)\n\t\t}\n\t\tif _, got := r.ArchiveAllRetryTasks(tc.qname); !tc.match(got) {\n\t\t\tt.Errorf(\"%s: ArchiveAllRetryTasks returned %v\", tc.desc, got)\n\t\t}\n\t}\n}\n\nfunc TestDeleteArchivedTask(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tt1 := time.Now().Add(-5 * time.Minute)\n\tt2 := time.Now().Add(-time.Hour)\n\tt3 := time.Now().Add(-time.Hour)\n\n\ttests := []struct {\n\t\tarchived map[string][]base.Z\n\t\tqname string\n\t\tid string\n\t\twantArchived map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: t1.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: m1.ID,\n\t\t\twantArchived: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m2},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: t1.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: m3, Score: t3.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\tid: m3.ID,\n\t\t\twantArchived: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllArchivedQueues(t, r.client, tc.archived)\n\n\t\tif got := r.DeleteTask(tc.qname, tc.id); got != nil {\n\t\t\tt.Errorf(\"r.DeleteTask(%q, %v) returned error: %v\", tc.qname, tc.id, got)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor qname, want := range tc.wantArchived {\n\t\t\tgotArchived := h.GetArchivedMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotArchived, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.ArchivedKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestDeleteRetryTask(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tt1 := time.Now().Add(5 * time.Minute)\n\tt2 := time.Now().Add(time.Hour)\n\tt3 := time.Now().Add(time.Hour)\n\n\ttests := []struct {\n\t\tretry map[string][]base.Z\n\t\tqname string\n\t\tid string\n\t\twantRetry map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: t1.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: m1.ID,\n\t\t\twantRetry: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m2},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: t1.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: m3, Score: t3.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\tid: m3.ID,\n\t\t\twantRetry: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllRetryQueues(t, r.client, tc.retry)\n\n\t\tif got := r.DeleteTask(tc.qname, tc.id); got != nil {\n\t\t\tt.Errorf(\"r.DeleteTask(%q, %v) returned error: %v\", tc.qname, tc.id, got)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor qname, want := range tc.wantRetry {\n\t\t\tgotRetry := h.GetRetryMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotRetry, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.RetryKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestDeleteScheduledTask(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tt1 := time.Now().Add(5 * time.Minute)\n\tt2 := time.Now().Add(time.Hour)\n\tt3 := time.Now().Add(time.Hour)\n\n\ttests := []struct {\n\t\tscheduled map[string][]base.Z\n\t\tqname string\n\t\tid string\n\t\twantScheduled map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: t1.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: m1.ID,\n\t\t\twantScheduled: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m2},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: t1.Unix()},\n\t\t\t\t\t{Message: m2, Score: t2.Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: m3, Score: t3.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\tid: m3.ID,\n\t\t\twantScheduled: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllScheduledQueues(t, r.client, tc.scheduled)\n\n\t\tif got := r.DeleteTask(tc.qname, tc.id); got != nil {\n\t\t\tt.Errorf(\"r.DeleteTask(%q, %v) returned error: %v\", tc.qname, tc.id, got)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor qname, want := range tc.wantScheduled {\n\t\t\tgotScheduled := h.GetScheduledMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotScheduled, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.ScheduledKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestDeleteAggregatingTask(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tnow := time.Now()\n\tm1 := h.NewTaskMessageBuilder().SetQueue(\"default\").SetType(\"task1\").SetGroup(\"group1\").Build()\n\tm2 := h.NewTaskMessageBuilder().SetQueue(\"default\").SetType(\"task2\").SetGroup(\"group1\").Build()\n\tm3 := h.NewTaskMessageBuilder().SetQueue(\"custom\").SetType(\"task3\").SetGroup(\"group1\").Build()\n\n\tfxt := struct {\n\t\ttasks []*h.TaskSeedData\n\t\tallQueues []string\n\t\tallGroups map[string][]string\n\t\tgroups map[string][]redis.Z\n\t}{\n\t\ttasks: []*h.TaskSeedData{\n\t\t\t{Msg: m1, State: base.TaskStateAggregating},\n\t\t\t{Msg: m2, State: base.TaskStateAggregating},\n\t\t\t{Msg: m3, State: base.TaskStateAggregating},\n\t\t},\n\t\tallQueues: []string{\"default\", \"custom\"},\n\t\tallGroups: map[string][]string{\n\t\t\tbase.AllGroups(\"default\"): {\"group1\"},\n\t\t\tbase.AllGroups(\"custom\"): {\"group1\"},\n\t\t},\n\t\tgroups: map[string][]redis.Z{\n\t\t\tbase.GroupKey(\"default\", \"group1\"): {\n\t\t\t\t{Member: m1.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t\t{Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())},\n\t\t\t},\n\t\t\tbase.GroupKey(\"custom\", \"group1\"): {\n\t\t\t\t{Member: m3.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t},\n\t\t},\n\t}\n\n\ttests := []struct {\n\t\tdesc string\n\t\tqname string\n\t\tid string\n\t\twantAllGroups map[string][]string\n\t\twantGroups map[string][]redis.Z\n\t}{\n\t\t{\n\t\t\tdesc: \"deletes a task from group with multiple tasks\",\n\t\t\tqname: \"default\",\n\t\t\tid: m1.ID,\n\t\t\twantAllGroups: map[string][]string{\n\t\t\t\tbase.AllGroups(\"default\"): {\"group1\"},\n\t\t\t\tbase.AllGroups(\"custom\"): {\"group1\"},\n\t\t\t},\n\t\t\twantGroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"group1\"): {\n\t\t\t\t\t{Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t\tbase.GroupKey(\"custom\", \"group1\"): {\n\t\t\t\t\t{Member: m3.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"deletes a task from group with single task\",\n\t\t\tqname: \"custom\",\n\t\t\tid: m3.ID,\n\t\t\twantAllGroups: map[string][]string{\n\t\t\t\tbase.AllGroups(\"default\"): {\"group1\"},\n\t\t\t\tbase.AllGroups(\"custom\"): {}, // should be clear out group from all-groups set\n\t\t\t},\n\t\t\twantGroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"group1\"): {\n\t\t\t\t\t{Member: m1.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t\t\t{Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t\tbase.GroupKey(\"custom\", \"group1\"): {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\th.SeedTasks(t, r.client, fxt.tasks)\n\t\th.SeedRedisSet(t, r.client, base.AllQueues, fxt.allQueues)\n\t\th.SeedRedisSets(t, r.client, fxt.allGroups)\n\t\th.SeedRedisZSets(t, r.client, fxt.groups)\n\n\t\tt.Run(tc.desc, func(t *testing.T) {\n\t\t\terr := r.DeleteTask(tc.qname, tc.id)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"DeleteTask returned error: %v\", err)\n\t\t\t}\n\t\t\th.AssertRedisSets(t, r.client, tc.wantAllGroups)\n\t\t\th.AssertRedisZSets(t, r.client, tc.wantGroups)\n\t\t})\n\t}\n}\n\nfunc TestDeletePendingTask(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\n\ttests := []struct {\n\t\tpending map[string][]*base.TaskMessage\n\t\tqname string\n\t\tid string\n\t\twantPending map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: m1.ID,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m2},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t\t\"custom\": {m3},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\tid: m3.ID,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\th.SeedAllPendingQueues(t, r.client, tc.pending)\n\n\t\tif got := r.DeleteTask(tc.qname, tc.id); got != nil {\n\t\t\tt.Errorf(\"r.DeleteTask(%q, %v) returned error: %v\", tc.qname, tc.id, got)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.PendingKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestDeleteTaskWithUniqueLock(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"email\",\n\t\tPayload: h.JSON(map[string]interface{}{\"user_id\": json.Number(\"123\")}),\n\t\tQueue: base.DefaultQueueName,\n\t\tUniqueKey: base.UniqueKey(base.DefaultQueueName, \"email\", h.JSON(map[string]interface{}{\"user_id\": 123})),\n\t}\n\tt1 := time.Now().Add(3 * time.Hour)\n\n\ttests := []struct {\n\t\tscheduled map[string][]base.Z\n\t\tqname string\n\t\tid string\n\t\tuniqueKey string\n\t\twantScheduled map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: t1.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: m1.ID,\n\t\t\tuniqueKey: m1.UniqueKey,\n\t\t\twantScheduled: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllScheduledQueues(t, r.client, tc.scheduled)\n\n\t\tif got := r.DeleteTask(tc.qname, tc.id); got != nil {\n\t\t\tt.Errorf(\"r.DeleteTask(%q, %v) returned error: %v\", tc.qname, tc.id, got)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor qname, want := range tc.wantScheduled {\n\t\t\tgotScheduled := h.GetScheduledMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotScheduled, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.ScheduledKey(qname), diff)\n\t\t\t}\n\t\t}\n\n\t\tif r.client.Exists(context.Background(), tc.uniqueKey).Val() != 0 {\n\t\t\tt.Errorf(\"Uniqueness lock %q still exists\", tc.uniqueKey)\n\t\t}\n\t}\n}\n\nfunc TestDeleteTaskError(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tt1 := time.Now().Add(5 * time.Minute)\n\n\ttests := []struct {\n\t\tdesc string\n\t\tactive map[string][]*base.TaskMessage\n\t\tscheduled map[string][]base.Z\n\t\tqname string\n\t\tid string\n\t\tmatch func(err error) bool\n\t\twantActive map[string][]*base.TaskMessage\n\t\twantScheduled map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tdesc: \"It should return TaskNotFoundError if task doesn't exist the queue\",\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: m1, Score: t1.Unix()}},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: uuid.NewString(),\n\t\t\tmatch: errors.IsTaskNotFound,\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantScheduled: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"It should return QueueNotFoundError if the queue doesn't exist\",\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: m1, Score: t1.Unix()}},\n\t\t\t},\n\t\t\tqname: \"nonexistent\",\n\t\t\tid: uuid.NewString(),\n\t\t\tmatch: errors.IsQueueNotFound,\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantScheduled: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"It should return FailedPrecondition error if task is active\",\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1},\n\t\t\t},\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tid: m1.ID,\n\t\t\tmatch: func(err error) bool { return errors.CanonicalCode(err) == errors.FailedPrecondition },\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1},\n\t\t\t},\n\t\t\twantScheduled: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllActiveQueues(t, r.client, tc.active)\n\t\th.SeedAllScheduledQueues(t, r.client, tc.scheduled)\n\n\t\tgot := r.DeleteTask(tc.qname, tc.id)\n\t\tif !tc.match(got) {\n\t\t\tt.Errorf(\"%s: r.DeleteTask(qname, id) returned %v\", tc.desc, got)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor qname, want := range tc.wantActive {\n\t\t\tgotActive := h.GetActiveMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.ActiveKey(qname), diff)\n\t\t\t}\n\t\t}\n\n\t\tfor qname, want := range tc.wantScheduled {\n\t\t\tgotScheduled := h.GetScheduledMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotScheduled, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.ScheduledKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestDeleteAllArchivedTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\n\ttests := []struct {\n\t\tarchived map[string][]base.Z\n\t\tqname string\n\t\twant int64\n\t\twantArchived map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: time.Now().Unix()},\n\t\t\t\t\t{Message: m2, Score: time.Now().Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: m3, Score: time.Now().Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 2,\n\t\t\twantArchived: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {m3},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 0,\n\t\t\twantArchived: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllArchivedQueues(t, r.client, tc.archived)\n\n\t\tgot, err := r.DeleteAllArchivedTasks(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"r.DeleteAllArchivedTasks(%q) returned error: %v\", tc.qname, err)\n\t\t}\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"r.DeleteAllArchivedTasks(%q) = %d, nil, want %d, nil\", tc.qname, got, tc.want)\n\t\t}\n\t\tfor qname, want := range tc.wantArchived {\n\t\t\tgotArchived := h.GetArchivedMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotArchived, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.ArchivedKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc newCompletedTaskMessage(qname, typename string, retention time.Duration, completedAt time.Time) *base.TaskMessage {\n\tmsg := h.NewTaskMessageWithQueue(typename, nil, qname)\n\tmsg.Retention = int64(retention.Seconds())\n\tmsg.CompletedAt = completedAt.Unix()\n\treturn msg\n}\n\nfunc TestDeleteAllCompletedTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tnow := time.Now()\n\tm1 := newCompletedTaskMessage(\"default\", \"task1\", 30*time.Minute, now.Add(-2*time.Minute))\n\tm2 := newCompletedTaskMessage(\"default\", \"task2\", 30*time.Minute, now.Add(-5*time.Minute))\n\tm3 := newCompletedTaskMessage(\"custom\", \"task3\", 30*time.Minute, now.Add(-5*time.Minute))\n\n\ttests := []struct {\n\t\tcompleted map[string][]base.Z\n\t\tqname string\n\t\twant int64\n\t\twantCompleted map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tcompleted: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: m1.CompletedAt + m1.Retention},\n\t\t\t\t\t{Message: m2, Score: m2.CompletedAt + m2.Retention},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: m3, Score: m2.CompletedAt + m3.Retention},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 2,\n\t\t\twantCompleted: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {m3},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tcompleted: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 0,\n\t\t\twantCompleted: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllCompletedQueues(t, r.client, tc.completed)\n\n\t\tgot, err := r.DeleteAllCompletedTasks(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"r.DeleteAllCompletedTasks(%q) returned error: %v\", tc.qname, err)\n\t\t}\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"r.DeleteAllCompletedTasks(%q) = %d, nil, want %d, nil\", tc.qname, got, tc.want)\n\t\t}\n\t\tfor qname, want := range tc.wantCompleted {\n\t\t\tgotCompleted := h.GetCompletedMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotCompleted, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.CompletedKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestDeleteAllArchivedTasksWithUniqueKey(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"task1\",\n\t\tPayload: nil,\n\t\tTimeout: 1800,\n\t\tDeadline: 0,\n\t\tUniqueKey: \"asynq:{default}:unique:task1:nil\",\n\t\tQueue: \"default\",\n\t}\n\tm2 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"task2\",\n\t\tPayload: nil,\n\t\tTimeout: 1800,\n\t\tDeadline: 0,\n\t\tUniqueKey: \"asynq:{default}:unique:task2:nil\",\n\t\tQueue: \"default\",\n\t}\n\tm3 := h.NewTaskMessage(\"task3\", nil)\n\n\ttests := []struct {\n\t\tarchived map[string][]base.Z\n\t\tqname string\n\t\twant int64\n\t\tuniqueKeys []string // list of unique keys that should be cleared\n\t\twantArchived map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: time.Now().Unix()},\n\t\t\t\t\t{Message: m2, Score: time.Now().Unix()},\n\t\t\t\t\t{Message: m3, Score: time.Now().Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 3,\n\t\t\tuniqueKeys: []string{m1.UniqueKey, m2.UniqueKey},\n\t\t\twantArchived: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllArchivedQueues(t, r.client, tc.archived)\n\n\t\tgot, err := r.DeleteAllArchivedTasks(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"r.DeleteAllArchivedTasks(%q) returned error: %v\", tc.qname, err)\n\t\t}\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"r.DeleteAllArchivedTasks(%q) = %d, nil, want %d, nil\", tc.qname, got, tc.want)\n\t\t}\n\t\tfor qname, want := range tc.wantArchived {\n\t\t\tgotArchived := h.GetArchivedMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotArchived, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.ArchivedKey(qname), diff)\n\t\t\t}\n\t\t}\n\n\t\tfor _, uniqueKey := range tc.uniqueKeys {\n\t\t\tif r.client.Exists(context.Background(), uniqueKey).Val() != 0 {\n\t\t\t\tt.Errorf(\"Uniqueness lock %q still exists\", uniqueKey)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestDeleteAllRetryTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\n\ttests := []struct {\n\t\tretry map[string][]base.Z\n\t\tqname string\n\t\twant int64\n\t\twantRetry map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: time.Now().Unix()},\n\t\t\t\t\t{Message: m2, Score: time.Now().Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: m3, Score: time.Now().Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\twant: 1,\n\t\t\twantRetry: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 0,\n\t\t\twantRetry: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllRetryQueues(t, r.client, tc.retry)\n\n\t\tgot, err := r.DeleteAllRetryTasks(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"r.DeleteAllRetryTasks(%q) returned error: %v\", tc.qname, err)\n\t\t}\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"r.DeleteAllRetryTasks(%q) = %d, nil, want %d, nil\", tc.qname, got, tc.want)\n\t\t}\n\t\tfor qname, want := range tc.wantRetry {\n\t\t\tgotRetry := h.GetRetryMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotRetry, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.RetryKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestDeleteAllScheduledTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\n\ttests := []struct {\n\t\tscheduled map[string][]base.Z\n\t\tqname string\n\t\twant int64\n\t\twantScheduled map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: m1, Score: time.Now().Add(time.Minute).Unix()},\n\t\t\t\t\t{Message: m2, Score: time.Now().Add(time.Minute).Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: m3, Score: time.Now().Add(time.Minute).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 2,\n\t\t\twantScheduled: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {m3},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\twant: 0,\n\t\t\twantScheduled: map[string][]*base.TaskMessage{\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllScheduledQueues(t, r.client, tc.scheduled)\n\n\t\tgot, err := r.DeleteAllScheduledTasks(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"r.DeleteAllScheduledTasks(%q) returned error: %v\", tc.qname, err)\n\t\t}\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"r.DeleteAllScheduledTasks(%q) = %d, nil, want %d, nil\", tc.qname, got, tc.want)\n\t\t}\n\t\tfor qname, want := range tc.wantScheduled {\n\t\t\tgotScheduled := h.GetScheduledMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotScheduled, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.ScheduledKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestDeleteAllAggregatingTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tnow := time.Now()\n\tm1 := h.NewTaskMessageBuilder().SetQueue(\"default\").SetType(\"task1\").SetGroup(\"group1\").Build()\n\tm2 := h.NewTaskMessageBuilder().SetQueue(\"default\").SetType(\"task2\").SetGroup(\"group1\").Build()\n\tm3 := h.NewTaskMessageBuilder().SetQueue(\"custom\").SetType(\"task3\").SetGroup(\"group1\").Build()\n\n\tfxt := struct {\n\t\ttasks []*h.TaskSeedData\n\t\tallQueues []string\n\t\tallGroups map[string][]string\n\t\tgroups map[string][]redis.Z\n\t}{\n\t\ttasks: []*h.TaskSeedData{\n\t\t\t{Msg: m1, State: base.TaskStateAggregating},\n\t\t\t{Msg: m2, State: base.TaskStateAggregating},\n\t\t\t{Msg: m3, State: base.TaskStateAggregating},\n\t\t},\n\t\tallQueues: []string{\"default\", \"custom\"},\n\t\tallGroups: map[string][]string{\n\t\t\tbase.AllGroups(\"default\"): {\"group1\"},\n\t\t\tbase.AllGroups(\"custom\"): {\"group1\"},\n\t\t},\n\t\tgroups: map[string][]redis.Z{\n\t\t\tbase.GroupKey(\"default\", \"group1\"): {\n\t\t\t\t{Member: m1.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t\t{Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())},\n\t\t\t},\n\t\t\tbase.GroupKey(\"custom\", \"group1\"): {\n\t\t\t\t{Member: m3.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t},\n\t\t},\n\t}\n\n\ttests := []struct {\n\t\tdesc string\n\t\tqname string\n\t\tgname string\n\t\twant int64\n\t\twantAllGroups map[string][]string\n\t\twantGroups map[string][]redis.Z\n\t}{\n\t\t{\n\t\t\tdesc: \"default queue group1\",\n\t\t\tqname: \"default\",\n\t\t\tgname: \"group1\",\n\t\t\twant: 2,\n\t\t\twantAllGroups: map[string][]string{\n\t\t\t\tbase.AllGroups(\"default\"): {},\n\t\t\t\tbase.AllGroups(\"custom\"): {\"group1\"},\n\t\t\t},\n\t\t\twantGroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"group1\"): nil,\n\t\t\t\tbase.GroupKey(\"custom\", \"group1\"): {\n\t\t\t\t\t{Member: m3.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"custom queue group1\",\n\t\t\tqname: \"custom\",\n\t\t\tgname: \"group1\",\n\t\t\twant: 1,\n\t\t\twantAllGroups: map[string][]string{\n\t\t\t\tbase.AllGroups(\"default\"): {\"group1\"},\n\t\t\t\tbase.AllGroups(\"custom\"): {},\n\t\t\t},\n\t\t\twantGroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"group1\"): {\n\t\t\t\t\t{Member: m1.ID, Score: float64(now.Add(-20 * time.Second).Unix())},\n\t\t\t\t\t{Member: m2.ID, Score: float64(now.Add(-25 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t\tbase.GroupKey(\"custom\", \"group1\"): nil,\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\th.SeedTasks(t, r.client, fxt.tasks)\n\t\th.SeedRedisSet(t, r.client, base.AllQueues, fxt.allQueues)\n\t\th.SeedRedisSets(t, r.client, fxt.allGroups)\n\t\th.SeedRedisZSets(t, r.client, fxt.groups)\n\n\t\tt.Run(tc.desc, func(t *testing.T) {\n\t\t\tgot, err := r.DeleteAllAggregatingTasks(tc.qname, tc.gname)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"DeleteAllAggregatingTasks returned error: %v\", err)\n\t\t\t}\n\t\t\tif got != tc.want {\n\t\t\t\tt.Errorf(\"DeleteAllAggregatingTasks = %d, want %d\", got, tc.want)\n\t\t\t}\n\t\t\th.AssertRedisSets(t, r.client, tc.wantAllGroups)\n\t\t\th.AssertRedisZSets(t, r.client, tc.wantGroups)\n\t\t})\n\t}\n}\n\nfunc TestDeleteAllPendingTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\n\ttests := []struct {\n\t\tpending map[string][]*base.TaskMessage\n\t\tqname string\n\t\twant int64\n\t\twantPending map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t\t\"custom\": {m3},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: 2,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {m3},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\twant: 0,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllPendingQueues(t, r.client, tc.pending)\n\n\t\tgot, err := r.DeleteAllPendingTasks(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"r.DeleteAllPendingTasks(%q) returned error: %v\", tc.qname, err)\n\t\t}\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"r.DeleteAllPendingTasks(%q) = %d, nil, want %d, nil\", tc.qname, got, tc.want)\n\t\t}\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.PendingKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestDeleteAllTasksError(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\ttests := []struct {\n\t\tdesc string\n\t\tqname string\n\t\tmatch func(err error) bool\n\t}{\n\t\t{\n\t\t\tdesc: \"It returns QueueNotFoundError if queue doesn't exist\",\n\t\t\tqname: \"nonexistent\",\n\t\t\tmatch: errors.IsQueueNotFound,\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tif _, got := r.DeleteAllPendingTasks(tc.qname); !tc.match(got) {\n\t\t\tt.Errorf(\"%s: DeleteAllPendingTasks returned %v\", tc.desc, got)\n\t\t}\n\t\tif _, got := r.DeleteAllScheduledTasks(tc.qname); !tc.match(got) {\n\t\t\tt.Errorf(\"%s: DeleteAllScheduledTasks returned %v\", tc.desc, got)\n\t\t}\n\t\tif _, got := r.DeleteAllRetryTasks(tc.qname); !tc.match(got) {\n\t\t\tt.Errorf(\"%s: DeleteAllRetryTasks returned %v\", tc.desc, got)\n\t\t}\n\t\tif _, got := r.DeleteAllArchivedTasks(tc.qname); !tc.match(got) {\n\t\t\tt.Errorf(\"%s: DeleteAllArchivedTasks returned %v\", tc.desc, got)\n\t\t}\n\t}\n}\n\nfunc TestRemoveQueue(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tm4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"custom\")\n\n\ttests := []struct {\n\t\tpending map[string][]*base.TaskMessage\n\t\tinProgress map[string][]*base.TaskMessage\n\t\tscheduled map[string][]base.Z\n\t\tretry map[string][]base.Z\n\t\tarchived map[string][]base.Z\n\t\tqname string // queue to remove\n\t\tforce bool\n\t}{\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tinProgress: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\tforce: false,\n\t\t},\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t\t\"custom\": {m3},\n\t\t\t},\n\t\t\tinProgress: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {{Message: m4, Score: time.Now().Unix()}},\n\t\t\t},\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\tforce: true, // allow removing non-empty queue\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\th.SeedAllPendingQueues(t, r.client, tc.pending)\n\t\th.SeedAllActiveQueues(t, r.client, tc.inProgress)\n\t\th.SeedAllScheduledQueues(t, r.client, tc.scheduled)\n\t\th.SeedAllRetryQueues(t, r.client, tc.retry)\n\t\th.SeedAllArchivedQueues(t, r.client, tc.archived)\n\n\t\terr := r.RemoveQueue(tc.qname, tc.force)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"(*RDB).RemoveQueue(%q, %t) = %v, want nil\",\n\t\t\t\ttc.qname, tc.force, err)\n\t\t\tcontinue\n\t\t}\n\t\tif r.client.SIsMember(context.Background(), base.AllQueues, tc.qname).Val() {\n\t\t\tt.Errorf(\"%q is a member of %q\", tc.qname, base.AllQueues)\n\t\t}\n\n\t\tkeys := []string{\n\t\t\tbase.PendingKey(tc.qname),\n\t\t\tbase.ActiveKey(tc.qname),\n\t\t\tbase.LeaseKey(tc.qname),\n\t\t\tbase.ScheduledKey(tc.qname),\n\t\t\tbase.RetryKey(tc.qname),\n\t\t\tbase.ArchivedKey(tc.qname),\n\t\t}\n\t\tfor _, key := range keys {\n\t\t\tif r.client.Exists(context.Background(), key).Val() != 0 {\n\t\t\t\tt.Errorf(\"key %q still exists\", key)\n\t\t\t}\n\t\t}\n\n\t\tif n := len(r.client.Keys(context.Background(), base.TaskKeyPrefix(tc.qname)+\"*\").Val()); n != 0 {\n\t\t\tt.Errorf(\"%d keys still exists for tasks\", n)\n\t\t}\n\t}\n}\n\nfunc TestRemoveQueueError(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"custom\")\n\tm4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"custom\")\n\n\ttests := []struct {\n\t\tdesc string\n\t\tpending map[string][]*base.TaskMessage\n\t\tinProgress map[string][]*base.TaskMessage\n\t\tscheduled map[string][]base.Z\n\t\tretry map[string][]base.Z\n\t\tarchived map[string][]base.Z\n\t\tqname string // queue to remove\n\t\tforce bool\n\t\tmatch func(err error) bool\n\t}{\n\t\t{\n\t\t\tdesc: \"removing non-existent queue\",\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t\t\"custom\": {m3},\n\t\t\t},\n\t\t\tinProgress: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqname: \"nonexistent\",\n\t\t\tforce: false,\n\t\t\tmatch: errors.IsQueueNotFound,\n\t\t},\n\t\t{\n\t\t\tdesc: \"removing non-empty queue\",\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t\t\"custom\": {m3},\n\t\t\t},\n\t\t\tinProgress: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {{Message: m4, Score: time.Now().Unix()}},\n\t\t\t},\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\tforce: false,\n\t\t\tmatch: errors.IsQueueNotEmpty,\n\t\t},\n\t\t{\n\t\t\tdesc: \"force removing queue with active tasks\",\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t\t\"custom\": {m3},\n\t\t\t},\n\t\t\tinProgress: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {m4},\n\t\t\t},\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\t// Even with force=true, it should error if there are active tasks.\n\t\t\tforce: true,\n\t\t\tmatch: func(err error) bool { return errors.CanonicalCode(err) == errors.FailedPrecondition },\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\th.SeedAllPendingQueues(t, r.client, tc.pending)\n\t\th.SeedAllActiveQueues(t, r.client, tc.inProgress)\n\t\th.SeedAllScheduledQueues(t, r.client, tc.scheduled)\n\t\th.SeedAllRetryQueues(t, r.client, tc.retry)\n\t\th.SeedAllArchivedQueues(t, r.client, tc.archived)\n\n\t\tgot := r.RemoveQueue(tc.qname, tc.force)\n\t\tif !tc.match(got) {\n\t\t\tt.Errorf(\"%s; returned error didn't match expected value; got=%v\", tc.desc, got)\n\t\t\tcontinue\n\t\t}\n\n\t\t// Make sure that nothing changed\n\t\tfor qname, want := range tc.pending {\n\t\t\tgotPending := h.GetPendingMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s;mismatch found in %q; (-want,+got):\\n%s\", tc.desc, base.PendingKey(qname), diff)\n\t\t\t}\n\t\t}\n\t\tfor qname, want := range tc.inProgress {\n\t\t\tgotActive := h.GetActiveMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s;mismatch found in %q; (-want,+got):\\n%s\", tc.desc, base.ActiveKey(qname), diff)\n\t\t\t}\n\t\t}\n\t\tfor qname, want := range tc.scheduled {\n\t\t\tgotScheduled := h.GetScheduledEntries(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotScheduled, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s;mismatch found in %q; (-want,+got):\\n%s\", tc.desc, base.ScheduledKey(qname), diff)\n\t\t\t}\n\t\t}\n\t\tfor qname, want := range tc.retry {\n\t\t\tgotRetry := h.GetRetryEntries(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotRetry, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s;mismatch found in %q; (-want,+got):\\n%s\", tc.desc, base.RetryKey(qname), diff)\n\t\t\t}\n\t\t}\n\t\tfor qname, want := range tc.archived {\n\t\t\tgotArchived := h.GetArchivedEntries(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s;mismatch found in %q; (-want,+got):\\n%s\", tc.desc, base.ArchivedKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestListServers(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tstarted1 := time.Now().Add(-time.Hour)\n\tinfo1 := &base.ServerInfo{\n\t\tHost: \"do.droplet1\",\n\t\tPID: 1234,\n\t\tServerID: \"server123\",\n\t\tConcurrency: 10,\n\t\tQueues: map[string]int{\"default\": 1},\n\t\tStatus: \"active\",\n\t\tStarted: started1,\n\t\tActiveWorkerCount: 0,\n\t}\n\n\tstarted2 := time.Now().Add(-2 * time.Hour)\n\tinfo2 := &base.ServerInfo{\n\t\tHost: \"do.droplet2\",\n\t\tPID: 9876,\n\t\tServerID: \"server456\",\n\t\tConcurrency: 20,\n\t\tQueues: map[string]int{\"email\": 1},\n\t\tStatus: \"stopped\",\n\t\tStarted: started2,\n\t\tActiveWorkerCount: 1,\n\t}\n\n\ttests := []struct {\n\t\tdata []*base.ServerInfo\n\t}{\n\t\t{\n\t\t\tdata: []*base.ServerInfo{},\n\t\t},\n\t\t{\n\t\t\tdata: []*base.ServerInfo{info1},\n\t\t},\n\t\t{\n\t\t\tdata: []*base.ServerInfo{info1, info2},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\n\t\tfor _, info := range tc.data {\n\t\t\tif err := r.WriteServerState(info, []*base.WorkerInfo{}, 5*time.Second); err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t}\n\n\t\tgot, err := r.ListServers()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"r.ListServers returned an error: %v\", err)\n\t\t}\n\t\tif diff := cmp.Diff(tc.data, got, h.SortServerInfoOpt); diff != \"\" {\n\t\t\tt.Errorf(\"r.ListServers returned %v, want %v; (-want,+got)\\n%s\",\n\t\t\t\tgot, tc.data, diff)\n\t\t}\n\t}\n}\n\nfunc TestListWorkers(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tvar (\n\t\thost = \"127.0.0.1\"\n\t\tpid = 4567\n\t\tserverID = \"server123\"\n\n\t\tm1 = h.NewTaskMessage(\"send_email\", h.JSON(map[string]interface{}{\"user_id\": \"abc123\"}))\n\t\tm2 = h.NewTaskMessage(\"gen_thumbnail\", h.JSON(map[string]interface{}{\"path\": \"some/path/to/image/file\"}))\n\t\tm3 = h.NewTaskMessage(\"reindex\", h.JSON(map[string]interface{}{}))\n\t)\n\n\ttests := []struct {\n\t\tdata []*base.WorkerInfo\n\t}{\n\t\t{\n\t\t\tdata: []*base.WorkerInfo{\n\t\t\t\t{\n\t\t\t\t\tHost: host,\n\t\t\t\t\tPID: pid,\n\t\t\t\t\tServerID: serverID,\n\t\t\t\t\tID: m1.ID,\n\t\t\t\t\tType: m1.Type,\n\t\t\t\t\tQueue: m1.Queue,\n\t\t\t\t\tPayload: m1.Payload,\n\t\t\t\t\tStarted: time.Now().Add(-1 * time.Second),\n\t\t\t\t\tDeadline: time.Now().Add(30 * time.Second),\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tHost: host,\n\t\t\t\t\tPID: pid,\n\t\t\t\t\tServerID: serverID,\n\t\t\t\t\tID: m2.ID,\n\t\t\t\t\tType: m2.Type,\n\t\t\t\t\tQueue: m2.Queue,\n\t\t\t\t\tPayload: m2.Payload,\n\t\t\t\t\tStarted: time.Now().Add(-5 * time.Second),\n\t\t\t\t\tDeadline: time.Now().Add(10 * time.Minute),\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tHost: host,\n\t\t\t\t\tPID: pid,\n\t\t\t\t\tServerID: serverID,\n\t\t\t\t\tID: m3.ID,\n\t\t\t\t\tType: m3.Type,\n\t\t\t\t\tQueue: m3.Queue,\n\t\t\t\t\tPayload: m3.Payload,\n\t\t\t\t\tStarted: time.Now().Add(-30 * time.Second),\n\t\t\t\t\tDeadline: time.Now().Add(30 * time.Minute),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\n\t\terr := r.WriteServerState(&base.ServerInfo{}, tc.data, time.Minute)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"could not write server state to redis: %v\", err)\n\t\t\tcontinue\n\t\t}\n\n\t\tgot, err := r.ListWorkers()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"(*RDB).ListWorkers() returned an error: %v\", err)\n\t\t\tcontinue\n\t\t}\n\n\t\tif diff := cmp.Diff(tc.data, got, h.SortWorkerInfoOpt); diff != \"\" {\n\t\t\tt.Errorf(\"(*RDB).ListWorkers() = %v, want = %v; (-want,+got)\\n%s\", got, tc.data, diff)\n\t\t}\n\t}\n}\n\nfunc TestWriteListClearSchedulerEntries(t *testing.T) {\n\tr := setup(t)\n\tnow := time.Now().UTC()\n\tschedulerID := \"127.0.0.1:9876:abc123\"\n\n\tdata := []*base.SchedulerEntry{\n\t\t{\n\t\t\tSpec: \"* * * * *\",\n\t\t\tType: \"foo\",\n\t\t\tPayload: nil,\n\t\t\tOpts: nil,\n\t\t\tNext: now.Add(5 * time.Hour),\n\t\t\tPrev: now.Add(-2 * time.Hour),\n\t\t},\n\t\t{\n\t\t\tSpec: \"@every 20m\",\n\t\t\tType: \"bar\",\n\t\t\tPayload: h.JSON(map[string]interface{}{\"fiz\": \"baz\"}),\n\t\t\tOpts: nil,\n\t\t\tNext: now.Add(1 * time.Minute),\n\t\t\tPrev: now.Add(-19 * time.Minute),\n\t\t},\n\t}\n\n\tif err := r.WriteSchedulerEntries(schedulerID, data, 30*time.Second); err != nil {\n\t\tt.Fatalf(\"WriteSchedulerEnties failed: %v\", err)\n\t}\n\tentries, err := r.ListSchedulerEntries()\n\tif err != nil {\n\t\tt.Fatalf(\"ListSchedulerEntries failed: %v\", err)\n\t}\n\tif diff := cmp.Diff(data, entries, h.SortSchedulerEntryOpt); diff != \"\" {\n\t\tt.Errorf(\"ListSchedulerEntries() = %v, want %v; (-want,+got)\\n%s\", entries, data, diff)\n\t}\n\tif err := r.ClearSchedulerEntries(schedulerID); err != nil {\n\t\tt.Fatalf(\"ClearSchedulerEntries failed: %v\", err)\n\t}\n\tentries, err = r.ListSchedulerEntries()\n\tif err != nil {\n\t\tt.Fatalf(\"ListSchedulerEntries() after clear failed: %v\", err)\n\t}\n\tif len(entries) != 0 {\n\t\tt.Errorf(\"found %d entries, want 0 after clearing\", len(entries))\n\t}\n}\n\nfunc TestSchedulerEnqueueEvents(t *testing.T) {\n\tr := setup(t)\n\n\tvar (\n\t\tnow = time.Now()\n\t\toneDayAgo = now.Add(-24 * time.Hour)\n\t\tfiveHoursAgo = now.Add(-5 * time.Hour)\n\t\toneHourAgo = now.Add(-1 * time.Hour)\n\t)\n\n\ttests := []struct {\n\t\tentryID string\n\t\tevents []*base.SchedulerEnqueueEvent\n\t\twant []*base.SchedulerEnqueueEvent\n\t}{\n\t\t{\n\t\t\tentryID: \"entry123\",\n\t\t\tevents: []*base.SchedulerEnqueueEvent{\n\t\t\t\t{TaskID: \"task123\", EnqueuedAt: oneDayAgo},\n\t\t\t\t{TaskID: \"task789\", EnqueuedAt: oneHourAgo},\n\t\t\t\t{TaskID: \"task456\", EnqueuedAt: fiveHoursAgo},\n\t\t\t},\n\t\t\t// Recent events first\n\t\t\twant: []*base.SchedulerEnqueueEvent{\n\t\t\t\t{TaskID: \"task789\", EnqueuedAt: oneHourAgo},\n\t\t\t\t{TaskID: \"task456\", EnqueuedAt: fiveHoursAgo},\n\t\t\t\t{TaskID: \"task123\", EnqueuedAt: oneDayAgo},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tentryID: \"entry456\",\n\t\t\tevents: nil,\n\t\t\twant: nil,\n\t\t},\n\t}\n\nloop:\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\n\t\tfor _, e := range tc.events {\n\t\t\tif err := r.RecordSchedulerEnqueueEvent(tc.entryID, e); err != nil {\n\t\t\t\tt.Errorf(\"RecordSchedulerEnqueueEvent(%q, %v) failed: %v\", tc.entryID, e, err)\n\t\t\t\tcontinue loop\n\t\t\t}\n\t\t}\n\t\tgot, err := r.ListSchedulerEnqueueEvents(tc.entryID, Pagination{Size: 20, Page: 0})\n\t\tif err != nil {\n\t\t\tt.Errorf(\"ListSchedulerEnqueueEvents(%q) failed: %v\", tc.entryID, err)\n\t\t\tcontinue\n\t\t}\n\t\tif diff := cmp.Diff(tc.want, got, timeCmpOpt); diff != \"\" {\n\t\t\tt.Errorf(\"ListSchedulerEnqueueEvent(%q) = %v, want %v; (-want,+got)\\n%s\",\n\t\t\t\ttc.entryID, got, tc.want, diff)\n\t\t}\n\t}\n}\n\nfunc TestRecordSchedulerEnqueueEventTrimsDataSet(t *testing.T) {\n\tr := setup(t)\n\tvar (\n\t\tentryID = \"entry123\"\n\t\tnow = time.Now()\n\t\tkey = base.SchedulerHistoryKey(entryID)\n\t)\n\n\t// Record maximum number of events.\n\tfor i := 1; i <= maxEvents; i++ {\n\t\tevent := base.SchedulerEnqueueEvent{\n\t\t\tTaskID: fmt.Sprintf(\"task%d\", i),\n\t\t\tEnqueuedAt: now.Add(-time.Duration(i) * time.Second),\n\t\t}\n\t\tif err := r.RecordSchedulerEnqueueEvent(entryID, &event); err != nil {\n\t\t\tt.Fatalf(\"RecordSchedulerEnqueueEvent failed: %v\", err)\n\t\t}\n\t}\n\n\t// Make sure the set is full.\n\tif n := r.client.ZCard(context.Background(), key).Val(); n != maxEvents {\n\t\tt.Fatalf(\"unexpected number of events; got %d, want %d\", n, maxEvents)\n\t}\n\n\t// Record one more event, should evict the oldest event.\n\tevent := base.SchedulerEnqueueEvent{\n\t\tTaskID: \"latest\",\n\t\tEnqueuedAt: now,\n\t}\n\tif err := r.RecordSchedulerEnqueueEvent(entryID, &event); err != nil {\n\t\tt.Fatalf(\"RecordSchedulerEnqueueEvent failed: %v\", err)\n\t}\n\tif n := r.client.ZCard(context.Background(), key).Val(); n != maxEvents {\n\t\tt.Fatalf(\"unexpected number of events; got %d, want %d\", n, maxEvents)\n\t}\n\tevents, err := r.ListSchedulerEnqueueEvents(entryID, Pagination{Size: maxEvents})\n\tif err != nil {\n\t\tt.Fatalf(\"ListSchedulerEnqueueEvents failed: %v\", err)\n\t}\n\tif first := events[0]; first.TaskID != \"latest\" {\n\t\tt.Errorf(\"unexpected first event; got %q, want %q\", first.TaskID, \"latest\")\n\t}\n\tif last := events[maxEvents-1]; last.TaskID != fmt.Sprintf(\"task%d\", maxEvents-1) {\n\t\tt.Errorf(\"unexpected last event; got %q, want %q\", last.TaskID, fmt.Sprintf(\"task%d\", maxEvents-1))\n\t}\n}\n\nfunc TestPause(t *testing.T) {\n\tr := setup(t)\n\n\ttests := []struct {\n\t\tqname string // name of the queue to pause\n\t}{\n\t\t{qname: \"default\"},\n\t\t{qname: \"custom\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\n\t\terr := r.Pause(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"Pause(%q) returned error: %v\", tc.qname, err)\n\t\t}\n\t\tkey := base.PausedKey(tc.qname)\n\t\tif r.client.Exists(context.Background(), key).Val() == 0 {\n\t\t\tt.Errorf(\"key %q does not exist\", key)\n\t\t}\n\t}\n}\n\nfunc TestPauseError(t *testing.T) {\n\tr := setup(t)\n\n\ttests := []struct {\n\t\tdesc string // test case description\n\t\tpaused []string // already paused queues\n\t\tqname string // name of the queue to pause\n\t}{\n\t\t{\"queue already paused\", []string{\"default\", \"custom\"}, \"default\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\tfor _, qname := range tc.paused {\n\t\t\tif err := r.Pause(qname); err != nil {\n\t\t\t\tt.Fatalf(\"could not pause %q: %v\", qname, err)\n\t\t\t}\n\t\t}\n\n\t\terr := r.Pause(tc.qname)\n\t\tif err == nil {\n\t\t\tt.Errorf(\"%s; Pause(%q) returned nil: want error\", tc.desc, tc.qname)\n\t\t}\n\t}\n}\n\nfunc TestUnpause(t *testing.T) {\n\tr := setup(t)\n\n\ttests := []struct {\n\t\tpaused []string // already paused queues\n\t\tqname string // name of the queue to unpause\n\t}{\n\t\t{[]string{\"default\", \"custom\"}, \"default\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\tfor _, qname := range tc.paused {\n\t\t\tif err := r.Pause(qname); err != nil {\n\t\t\t\tt.Fatalf(\"could not pause %q: %v\", qname, err)\n\t\t\t}\n\t\t}\n\n\t\terr := r.Unpause(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"Unpause(%q) returned error: %v\", tc.qname, err)\n\t\t}\n\t\tkey := base.PausedKey(tc.qname)\n\t\tif r.client.Exists(context.Background(), key).Val() == 1 {\n\t\t\tt.Errorf(\"key %q exists\", key)\n\t\t}\n\t}\n}\n\nfunc TestUnpauseError(t *testing.T) {\n\tr := setup(t)\n\n\ttests := []struct {\n\t\tdesc string // test case description\n\t\tpaused []string // already paused queues\n\t\tqname string // name of the queue to unpause\n\t}{\n\t\t{\"queue is not paused\", []string{\"default\"}, \"custom\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\tfor _, qname := range tc.paused {\n\t\t\tif err := r.Pause(qname); err != nil {\n\t\t\t\tt.Fatalf(\"could not pause %q: %v\", qname, err)\n\t\t\t}\n\t\t}\n\n\t\terr := r.Unpause(tc.qname)\n\t\tif err == nil {\n\t\t\tt.Errorf(\"%s; Unpause(%q) returned nil: want error\", tc.desc, tc.qname)\n\t\t}\n\t}\n}\n" }, { "path": "internal/rdb/rdb.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\n// Package rdb encapsulates the interactions with redis.\npackage rdb\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"math\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/google/uuid\"\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/errors\"\n\t\"github.com/hibiken/asynq/internal/timeutil\"\n\t\"github.com/redis/go-redis/v9\"\n\t\"github.com/spf13/cast\"\n)\n\nconst statsTTL = 90 * 24 * time.Hour // 90 days\n\n// LeaseDuration is the duration used to initially create a lease and to extend it thereafter.\nconst LeaseDuration = 30 * time.Second\n\n// RDB is a client interface to query and mutate task queues.\ntype RDB struct {\n\tclient redis.UniversalClient\n\tclock timeutil.Clock\n\tqueuesPublished sync.Map\n}\n\n// NewRDB returns a new instance of RDB.\nfunc NewRDB(client redis.UniversalClient) *RDB {\n\treturn &RDB{\n\t\tclient: client,\n\t\tclock: timeutil.NewRealClock(),\n\t}\n}\n\n// Close closes the connection with redis server.\nfunc (r *RDB) Close() error {\n\treturn r.client.Close()\n}\n\n// Client returns the reference to underlying redis client.\nfunc (r *RDB) Client() redis.UniversalClient {\n\treturn r.client\n}\n\n// SetClock sets the clock used by RDB to the given clock.\n//\n// Use this function to set the clock to SimulatedClock in tests.\nfunc (r *RDB) SetClock(c timeutil.Clock) {\n\tr.clock = c\n}\n\n// Ping checks the connection with redis server.\nfunc (r *RDB) Ping() error {\n\treturn r.client.Ping(context.Background()).Err()\n}\n\nfunc (r *RDB) runScript(ctx context.Context, op errors.Op, script *redis.Script, keys []string, args ...interface{}) error {\n\tif err := script.Run(ctx, r.client, keys, args...).Err(); err != nil {\n\t\treturn errors.E(op, errors.Internal, fmt.Sprintf(\"redis eval error: %v\", err))\n\t}\n\treturn nil\n}\n\n// Runs the given script with keys and args and returns the script's return value as int64.\nfunc (r *RDB) runScriptWithErrorCode(ctx context.Context, op errors.Op, script *redis.Script, keys []string, args ...interface{}) (int64, error) {\n\tres, err := script.Run(ctx, r.client, keys, args...).Result()\n\tif err != nil {\n\t\treturn 0, errors.E(op, errors.Unknown, fmt.Sprintf(\"redis eval error: %v\", err))\n\t}\n\tn, ok := res.(int64)\n\tif !ok {\n\t\treturn 0, errors.E(op, errors.Internal, fmt.Sprintf(\"unexpected return value from Lua script: %v\", res))\n\t}\n\treturn n, nil\n}\n\n// enqueueCmd enqueues a given task message.\n//\n// Input:\n// KEYS[1] -> asynq:{}:t:\n// KEYS[2] -> asynq:{}:pending\n// --\n// ARGV[1] -> task message data\n// ARGV[2] -> task ID\n// ARGV[3] -> current unix time in nsec\n//\n// Output:\n// Returns 1 if successfully enqueued\n// Returns 0 if task ID already exists\nvar enqueueCmd = redis.NewScript(`\nif redis.call(\"EXISTS\", KEYS[1]) == 1 then\n\treturn 0\nend\nredis.call(\"HSET\", KEYS[1],\n \"msg\", ARGV[1],\n \"state\", \"pending\",\n \"pending_since\", ARGV[3])\nredis.call(\"LPUSH\", KEYS[2], ARGV[2])\nreturn 1\n`)\n\n// Enqueue adds the given task to the pending list of the queue.\nfunc (r *RDB) Enqueue(ctx context.Context, msg *base.TaskMessage) error {\n\tvar op errors.Op = \"rdb.Enqueue\"\n\tencoded, err := base.EncodeMessage(msg)\n\tif err != nil {\n\t\treturn errors.E(op, errors.Unknown, fmt.Sprintf(\"cannot encode message: %v\", err))\n\t}\n\tif _, found := r.queuesPublished.Load(msg.Queue); !found {\n\t\tif err := r.client.SAdd(ctx, base.AllQueues, msg.Queue).Err(); err != nil {\n\t\t\treturn errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: \"sadd\", Err: err})\n\t\t}\n\t\tr.queuesPublished.Store(msg.Queue, true)\n\t}\n\tkeys := []string{\n\t\tbase.TaskKey(msg.Queue, msg.ID),\n\t\tbase.PendingKey(msg.Queue),\n\t}\n\targv := []interface{}{\n\t\tencoded,\n\t\tmsg.ID,\n\t\tr.clock.Now().UnixNano(),\n\t}\n\tn, err := r.runScriptWithErrorCode(ctx, op, enqueueCmd, keys, argv...)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif n == 0 {\n\t\treturn errors.E(op, errors.AlreadyExists, errors.ErrTaskIdConflict)\n\t}\n\treturn nil\n}\n\n// enqueueUniqueCmd enqueues the task message if the task is unique.\n//\n// KEYS[1] -> unique key\n// KEYS[2] -> asynq:{}:t:\n// KEYS[3] -> asynq:{}:pending\n// --\n// ARGV[1] -> task ID\n// ARGV[2] -> uniqueness lock TTL\n// ARGV[3] -> task message data\n// ARGV[4] -> current unix time in nsec\n//\n// Output:\n// Returns 1 if successfully enqueued\n// Returns 0 if task ID conflicts with another task\n// Returns -1 if task unique key already exists\nvar enqueueUniqueCmd = redis.NewScript(`\nlocal ok = redis.call(\"SET\", KEYS[1], ARGV[1], \"NX\", \"EX\", ARGV[2])\nif not ok then\n return -1\nend\nif redis.call(\"EXISTS\", KEYS[2]) == 1 then\n return 0\nend\nredis.call(\"HSET\", KEYS[2],\n \"msg\", ARGV[3],\n \"state\", \"pending\",\n \"pending_since\", ARGV[4],\n \"unique_key\", KEYS[1])\nredis.call(\"LPUSH\", KEYS[3], ARGV[1])\nreturn 1\n`)\n\n// EnqueueUnique inserts the given task if the task's uniqueness lock can be acquired.\n// It returns ErrDuplicateTask if the lock cannot be acquired.\nfunc (r *RDB) EnqueueUnique(ctx context.Context, msg *base.TaskMessage, ttl time.Duration) error {\n\tvar op errors.Op = \"rdb.EnqueueUnique\"\n\tencoded, err := base.EncodeMessage(msg)\n\tif err != nil {\n\t\treturn errors.E(op, errors.Internal, \"cannot encode task message: %v\", err)\n\t}\n\tif _, found := r.queuesPublished.Load(msg.Queue); !found {\n\t\tif err := r.client.SAdd(ctx, base.AllQueues, msg.Queue).Err(); err != nil {\n\t\t\treturn errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: \"sadd\", Err: err})\n\t\t}\n\t\tr.queuesPublished.Store(msg.Queue, true)\n\t}\n\tkeys := []string{\n\t\tmsg.UniqueKey,\n\t\tbase.TaskKey(msg.Queue, msg.ID),\n\t\tbase.PendingKey(msg.Queue),\n\t}\n\targv := []interface{}{\n\t\tmsg.ID,\n\t\tint(ttl.Seconds()),\n\t\tencoded,\n\t\tr.clock.Now().UnixNano(),\n\t}\n\tn, err := r.runScriptWithErrorCode(ctx, op, enqueueUniqueCmd, keys, argv...)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif n == -1 {\n\t\treturn errors.E(op, errors.AlreadyExists, errors.ErrDuplicateTask)\n\t}\n\tif n == 0 {\n\t\treturn errors.E(op, errors.AlreadyExists, errors.ErrTaskIdConflict)\n\t}\n\treturn nil\n}\n\n// Input:\n// KEYS[1] -> asynq:{}:pending\n// KEYS[2] -> asynq:{}:paused\n// KEYS[3] -> asynq:{}:active\n// KEYS[4] -> asynq:{}:lease\n// --\n// ARGV[1] -> initial lease expiration Unix time\n// ARGV[2] -> task key prefix\n//\n// Output:\n// Returns nil if no processable task is found in the given queue.\n// Returns an encoded TaskMessage.\n//\n// Note: dequeueCmd checks whether a queue is paused first, before\n// calling RPOPLPUSH to pop a task from the queue.\nvar dequeueCmd = redis.NewScript(`\nif redis.call(\"EXISTS\", KEYS[2]) == 0 then\n\tlocal id = redis.call(\"RPOPLPUSH\", KEYS[1], KEYS[3])\n\tif id then\n\t\tlocal key = ARGV[2] .. id\n\t\tredis.call(\"HSET\", key, \"state\", \"active\")\n\t\tredis.call(\"HDEL\", key, \"pending_since\")\n\t\tredis.call(\"ZADD\", KEYS[4], ARGV[1], id)\n\t\treturn redis.call(\"HGET\", key, \"msg\")\n\tend\nend\nreturn nil`)\n\n// Dequeue queries given queues in order and pops a task message\n// off a queue if one exists and returns the message and its lease expiration time.\n// Dequeue skips a queue if the queue is paused.\n// If all queues are empty, ErrNoProcessableTask error is returned.\nfunc (r *RDB) Dequeue(qnames ...string) (msg *base.TaskMessage, leaseExpirationTime time.Time, err error) {\n\tvar op errors.Op = \"rdb.Dequeue\"\n\tfor _, qname := range qnames {\n\t\tkeys := []string{\n\t\t\tbase.PendingKey(qname),\n\t\t\tbase.PausedKey(qname),\n\t\t\tbase.ActiveKey(qname),\n\t\t\tbase.LeaseKey(qname),\n\t\t}\n\t\tleaseExpirationTime = r.clock.Now().Add(LeaseDuration)\n\t\targv := []interface{}{\n\t\t\tleaseExpirationTime.Unix(),\n\t\t\tbase.TaskKeyPrefix(qname),\n\t\t}\n\t\tres, err := dequeueCmd.Run(context.Background(), r.client, keys, argv...).Result()\n\t\tif err == redis.Nil {\n\t\t\tcontinue\n\t\t} else if err != nil {\n\t\t\treturn nil, time.Time{}, errors.E(op, errors.Unknown, fmt.Sprintf(\"redis eval error: %v\", err))\n\t\t}\n\t\tencoded, err := cast.ToStringE(res)\n\t\tif err != nil {\n\t\t\treturn nil, time.Time{}, errors.E(op, errors.Internal, fmt.Sprintf(\"cast error: unexpected return value from Lua script: %v\", res))\n\t\t}\n\t\tif msg, err = base.DecodeMessage([]byte(encoded)); err != nil {\n\t\t\treturn nil, time.Time{}, errors.E(op, errors.Internal, fmt.Sprintf(\"cannot decode message: %v\", err))\n\t\t}\n\t\treturn msg, leaseExpirationTime, nil\n\t}\n\treturn nil, time.Time{}, errors.E(op, errors.NotFound, errors.ErrNoProcessableTask)\n}\n\n// KEYS[1] -> asynq:{}:active\n// KEYS[2] -> asynq:{}:lease\n// KEYS[3] -> asynq:{}:t:\n// KEYS[4] -> asynq:{}:processed:\n// KEYS[5] -> asynq:{}:processed\n// -------\n// ARGV[1] -> task ID\n// ARGV[2] -> stats expiration timestamp\n// ARGV[3] -> max int64 value\nvar doneCmd = redis.NewScript(`\nif redis.call(\"LREM\", KEYS[1], 0, ARGV[1]) == 0 then\n return redis.error_reply(\"NOT FOUND\")\nend\nif redis.call(\"ZREM\", KEYS[2], ARGV[1]) == 0 then\n return redis.error_reply(\"NOT FOUND\")\nend\nif redis.call(\"DEL\", KEYS[3]) == 0 then\n return redis.error_reply(\"NOT FOUND\")\nend\nlocal n = redis.call(\"INCR\", KEYS[4])\nif tonumber(n) == 1 then\n\tredis.call(\"EXPIREAT\", KEYS[4], ARGV[2])\nend\nlocal total = redis.call(\"GET\", KEYS[5])\nif tonumber(total) == tonumber(ARGV[3]) then\n\tredis.call(\"SET\", KEYS[5], 1)\nelse\n\tredis.call(\"INCR\", KEYS[5])\nend\nreturn redis.status_reply(\"OK\")\n`)\n\n// KEYS[1] -> asynq:{}:active\n// KEYS[2] -> asynq:{}:lease\n// KEYS[3] -> asynq:{}:t:\n// KEYS[4] -> asynq:{}:processed:\n// KEYS[5] -> asynq:{}:processed\n// KEYS[6] -> unique key\n// -------\n// ARGV[1] -> task ID\n// ARGV[2] -> stats expiration timestamp\n// ARGV[3] -> max int64 value\nvar doneUniqueCmd = redis.NewScript(`\nif redis.call(\"LREM\", KEYS[1], 0, ARGV[1]) == 0 then\n return redis.error_reply(\"NOT FOUND\")\nend\nif redis.call(\"ZREM\", KEYS[2], ARGV[1]) == 0 then\n return redis.error_reply(\"NOT FOUND\")\nend\nif redis.call(\"DEL\", KEYS[3]) == 0 then\n return redis.error_reply(\"NOT FOUND\")\nend\nlocal n = redis.call(\"INCR\", KEYS[4])\nif tonumber(n) == 1 then\n\tredis.call(\"EXPIREAT\", KEYS[4], ARGV[2])\nend\nlocal total = redis.call(\"GET\", KEYS[5])\nif tonumber(total) == tonumber(ARGV[3]) then\n\tredis.call(\"SET\", KEYS[5], 1)\nelse\n\tredis.call(\"INCR\", KEYS[5])\nend\nif redis.call(\"GET\", KEYS[6]) == ARGV[1] then\n redis.call(\"DEL\", KEYS[6])\nend\nreturn redis.status_reply(\"OK\")\n`)\n\n// Done removes the task from active queue and deletes the task.\n// It removes a uniqueness lock acquired by the task, if any.\nfunc (r *RDB) Done(ctx context.Context, msg *base.TaskMessage) error {\n\tvar op errors.Op = \"rdb.Done\"\n\tnow := r.clock.Now()\n\texpireAt := now.Add(statsTTL)\n\tkeys := []string{\n\t\tbase.ActiveKey(msg.Queue),\n\t\tbase.LeaseKey(msg.Queue),\n\t\tbase.TaskKey(msg.Queue, msg.ID),\n\t\tbase.ProcessedKey(msg.Queue, now),\n\t\tbase.ProcessedTotalKey(msg.Queue),\n\t}\n\targv := []interface{}{\n\t\tmsg.ID,\n\t\texpireAt.Unix(),\n\t\tint64(math.MaxInt64),\n\t}\n\t// Note: We cannot pass empty unique key when running this script in redis-cluster.\n\tif len(msg.UniqueKey) > 0 {\n\t\tkeys = append(keys, msg.UniqueKey)\n\t\treturn r.runScript(ctx, op, doneUniqueCmd, keys, argv...)\n\t}\n\treturn r.runScript(ctx, op, doneCmd, keys, argv...)\n}\n\n// KEYS[1] -> asynq:{}:active\n// KEYS[2] -> asynq:{}:lease\n// KEYS[3] -> asynq:{}:completed\n// KEYS[4] -> asynq:{}:t:\n// KEYS[5] -> asynq:{}:processed:\n// KEYS[6] -> asynq:{}:processed\n//\n// ARGV[1] -> task ID\n// ARGV[2] -> stats expiration timestamp\n// ARGV[3] -> task expiration time in unix time\n// ARGV[4] -> task message data\n// ARGV[5] -> max int64 value\nvar markAsCompleteCmd = redis.NewScript(`\nif redis.call(\"LREM\", KEYS[1], 0, ARGV[1]) == 0 then\n return redis.error_reply(\"NOT FOUND\")\nend\nif redis.call(\"ZREM\", KEYS[2], ARGV[1]) == 0 then\n return redis.error_reply(\"NOT FOUND\")\nend\nif redis.call(\"ZADD\", KEYS[3], ARGV[3], ARGV[1]) ~= 1 then\n return redis.error_reply(\"INTERNAL\")\nend\nredis.call(\"HSET\", KEYS[4], \"msg\", ARGV[4], \"state\", \"completed\")\nlocal n = redis.call(\"INCR\", KEYS[5])\nif tonumber(n) == 1 then\n\tredis.call(\"EXPIREAT\", KEYS[5], ARGV[2])\nend\nlocal total = redis.call(\"GET\", KEYS[6])\nif tonumber(total) == tonumber(ARGV[5]) then\n\tredis.call(\"SET\", KEYS[6], 1)\nelse\n\tredis.call(\"INCR\", KEYS[6])\nend\nreturn redis.status_reply(\"OK\")\n`)\n\n// KEYS[1] -> asynq:{}:active\n// KEYS[2] -> asynq:{}:lease\n// KEYS[3] -> asynq:{}:completed\n// KEYS[4] -> asynq:{}:t:\n// KEYS[5] -> asynq:{}:processed:\n// KEYS[6] -> asynq:{}:processed\n// KEYS[7] -> asynq:{}:unique:{}\n//\n// ARGV[1] -> task ID\n// ARGV[2] -> stats expiration timestamp\n// ARGV[3] -> task expiration time in unix time\n// ARGV[4] -> task message data\n// ARGV[5] -> max int64 value\nvar markAsCompleteUniqueCmd = redis.NewScript(`\nif redis.call(\"LREM\", KEYS[1], 0, ARGV[1]) == 0 then\n return redis.error_reply(\"NOT FOUND\")\nend\nif redis.call(\"ZREM\", KEYS[2], ARGV[1]) == 0 then\n return redis.error_reply(\"NOT FOUND\")\nend\nif redis.call(\"ZADD\", KEYS[3], ARGV[3], ARGV[1]) ~= 1 then\n return redis.error_reply(\"INTERNAL\")\nend\nredis.call(\"HSET\", KEYS[4], \"msg\", ARGV[4], \"state\", \"completed\")\nlocal n = redis.call(\"INCR\", KEYS[5])\nif tonumber(n) == 1 then\n\tredis.call(\"EXPIREAT\", KEYS[5], ARGV[2])\nend\nlocal total = redis.call(\"GET\", KEYS[6])\nif tonumber(total) == tonumber(ARGV[5]) then\n\tredis.call(\"SET\", KEYS[6], 1)\nelse\n\tredis.call(\"INCR\", KEYS[6])\nend\nif redis.call(\"GET\", KEYS[7]) == ARGV[1] then\n redis.call(\"DEL\", KEYS[7])\nend\nreturn redis.status_reply(\"OK\")\n`)\n\n// MarkAsComplete removes the task from active queue to mark the task as completed.\n// It removes a uniqueness lock acquired by the task, if any.\nfunc (r *RDB) MarkAsComplete(ctx context.Context, msg *base.TaskMessage) error {\n\tvar op errors.Op = \"rdb.MarkAsComplete\"\n\tnow := r.clock.Now()\n\tstatsExpireAt := now.Add(statsTTL)\n\tmsg.CompletedAt = now.Unix()\n\tencoded, err := base.EncodeMessage(msg)\n\tif err != nil {\n\t\treturn errors.E(op, errors.Unknown, fmt.Sprintf(\"cannot encode message: %v\", err))\n\t}\n\tkeys := []string{\n\t\tbase.ActiveKey(msg.Queue),\n\t\tbase.LeaseKey(msg.Queue),\n\t\tbase.CompletedKey(msg.Queue),\n\t\tbase.TaskKey(msg.Queue, msg.ID),\n\t\tbase.ProcessedKey(msg.Queue, now),\n\t\tbase.ProcessedTotalKey(msg.Queue),\n\t}\n\targv := []interface{}{\n\t\tmsg.ID,\n\t\tstatsExpireAt.Unix(),\n\t\tnow.Unix() + msg.Retention,\n\t\tencoded,\n\t\tint64(math.MaxInt64),\n\t}\n\t// Note: We cannot pass empty unique key when running this script in redis-cluster.\n\tif len(msg.UniqueKey) > 0 {\n\t\tkeys = append(keys, msg.UniqueKey)\n\t\treturn r.runScript(ctx, op, markAsCompleteUniqueCmd, keys, argv...)\n\t}\n\treturn r.runScript(ctx, op, markAsCompleteCmd, keys, argv...)\n}\n\n// KEYS[1] -> asynq:{}:active\n// KEYS[2] -> asynq:{}:lease\n// KEYS[3] -> asynq:{}:pending\n// KEYS[4] -> asynq:{}:t:\n// ARGV[1] -> task ID\n// Note: Use RPUSH to push to the head of the queue.\nvar requeueCmd = redis.NewScript(`\nif redis.call(\"LREM\", KEYS[1], 0, ARGV[1]) == 0 then\n return redis.error_reply(\"NOT FOUND\")\nend\nif redis.call(\"ZREM\", KEYS[2], ARGV[1]) == 0 then\n return redis.error_reply(\"NOT FOUND\")\nend\nredis.call(\"RPUSH\", KEYS[3], ARGV[1])\nredis.call(\"HSET\", KEYS[4], \"state\", \"pending\")\nreturn redis.status_reply(\"OK\")`)\n\n// Requeue moves the task from active queue to the specified queue.\nfunc (r *RDB) Requeue(ctx context.Context, msg *base.TaskMessage) error {\n\tvar op errors.Op = \"rdb.Requeue\"\n\tkeys := []string{\n\t\tbase.ActiveKey(msg.Queue),\n\t\tbase.LeaseKey(msg.Queue),\n\t\tbase.PendingKey(msg.Queue),\n\t\tbase.TaskKey(msg.Queue, msg.ID),\n\t}\n\treturn r.runScript(ctx, op, requeueCmd, keys, msg.ID)\n}\n\n// KEYS[1] -> asynq:{}:t:\n// KEYS[2] -> asynq:{}:g:\n// KEYS[3] -> asynq:{}:groups\n// -------\n// ARGV[1] -> task message data\n// ARGV[2] -> task ID\n// ARGV[3] -> current time in Unix time\n// ARGV[4] -> group key\n//\n// Output:\n// Returns 1 if successfully added\n// Returns 0 if task ID already exists\nvar addToGroupCmd = redis.NewScript(`\nif redis.call(\"EXISTS\", KEYS[1]) == 1 then\n\treturn 0\nend\nredis.call(\"HSET\", KEYS[1],\n \"msg\", ARGV[1],\n \"state\", \"aggregating\",\n\t \"group\", ARGV[4])\nredis.call(\"ZADD\", KEYS[2], ARGV[3], ARGV[2])\nredis.call(\"SADD\", KEYS[3], ARGV[4])\nreturn 1\n`)\n\nfunc (r *RDB) AddToGroup(ctx context.Context, msg *base.TaskMessage, groupKey string) error {\n\tvar op errors.Op = \"rdb.AddToGroup\"\n\tencoded, err := base.EncodeMessage(msg)\n\tif err != nil {\n\t\treturn errors.E(op, errors.Unknown, fmt.Sprintf(\"cannot encode message: %v\", err))\n\t}\n\tif _, found := r.queuesPublished.Load(msg.Queue); !found {\n\t\tif err := r.client.SAdd(ctx, base.AllQueues, msg.Queue).Err(); err != nil {\n\t\t\treturn errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: \"sadd\", Err: err})\n\t\t}\n\t\tr.queuesPublished.Store(msg.Queue, true)\n\t}\n\tkeys := []string{\n\t\tbase.TaskKey(msg.Queue, msg.ID),\n\t\tbase.GroupKey(msg.Queue, groupKey),\n\t\tbase.AllGroups(msg.Queue),\n\t}\n\targv := []interface{}{\n\t\tencoded,\n\t\tmsg.ID,\n\t\tr.clock.Now().Unix(),\n\t\tgroupKey,\n\t}\n\tn, err := r.runScriptWithErrorCode(ctx, op, addToGroupCmd, keys, argv...)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif n == 0 {\n\t\treturn errors.E(op, errors.AlreadyExists, errors.ErrTaskIdConflict)\n\t}\n\treturn nil\n}\n\n// KEYS[1] -> asynq:{}:t:\n// KEYS[2] -> asynq:{}:g:\n// KEYS[3] -> asynq:{}:groups\n// KEYS[4] -> unique key\n// -------\n// ARGV[1] -> task message data\n// ARGV[2] -> task ID\n// ARGV[3] -> current time in Unix time\n// ARGV[4] -> group key\n// ARGV[5] -> uniqueness lock TTL\n//\n// Output:\n// Returns 1 if successfully added\n// Returns 0 if task ID already exists\n// Returns -1 if task unique key already exists\nvar addToGroupUniqueCmd = redis.NewScript(`\nlocal ok = redis.call(\"SET\", KEYS[4], ARGV[2], \"NX\", \"EX\", ARGV[5])\nif not ok then\n return -1\nend\nif redis.call(\"EXISTS\", KEYS[1]) == 1 then\n\treturn 0\nend\nredis.call(\"HSET\", KEYS[1],\n \"msg\", ARGV[1],\n \"state\", \"aggregating\",\n\t \"group\", ARGV[4])\nredis.call(\"ZADD\", KEYS[2], ARGV[3], ARGV[2])\nredis.call(\"SADD\", KEYS[3], ARGV[4])\nreturn 1\n`)\n\nfunc (r *RDB) AddToGroupUnique(ctx context.Context, msg *base.TaskMessage, groupKey string, ttl time.Duration) error {\n\tvar op errors.Op = \"rdb.AddToGroupUnique\"\n\tencoded, err := base.EncodeMessage(msg)\n\tif err != nil {\n\t\treturn errors.E(op, errors.Unknown, fmt.Sprintf(\"cannot encode message: %v\", err))\n\t}\n\tif _, found := r.queuesPublished.Load(msg.Queue); !found {\n\t\tif err := r.client.SAdd(ctx, base.AllQueues, msg.Queue).Err(); err != nil {\n\t\t\treturn errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: \"sadd\", Err: err})\n\t\t}\n\t\tr.queuesPublished.Store(msg.Queue, true)\n\t}\n\tkeys := []string{\n\t\tbase.TaskKey(msg.Queue, msg.ID),\n\t\tbase.GroupKey(msg.Queue, groupKey),\n\t\tbase.AllGroups(msg.Queue),\n\t\tbase.UniqueKey(msg.Queue, msg.Type, msg.Payload),\n\t}\n\targv := []interface{}{\n\t\tencoded,\n\t\tmsg.ID,\n\t\tr.clock.Now().Unix(),\n\t\tgroupKey,\n\t\tint(ttl.Seconds()),\n\t}\n\tn, err := r.runScriptWithErrorCode(ctx, op, addToGroupUniqueCmd, keys, argv...)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif n == -1 {\n\t\treturn errors.E(op, errors.AlreadyExists, errors.ErrDuplicateTask)\n\t}\n\tif n == 0 {\n\t\treturn errors.E(op, errors.AlreadyExists, errors.ErrTaskIdConflict)\n\t}\n\treturn nil\n}\n\n// KEYS[1] -> asynq:{}:t:\n// KEYS[2] -> asynq:{}:scheduled\n// -------\n// ARGV[1] -> task message data\n// ARGV[2] -> process_at time in Unix time\n// ARGV[3] -> task ID\n//\n// Output:\n// Returns 1 if successfully enqueued\n// Returns 0 if task ID already exists\nvar scheduleCmd = redis.NewScript(`\nif redis.call(\"EXISTS\", KEYS[1]) == 1 then\n\treturn 0\nend\nredis.call(\"HSET\", KEYS[1],\n \"msg\", ARGV[1],\n \"state\", \"scheduled\")\nredis.call(\"ZADD\", KEYS[2], ARGV[2], ARGV[3])\nreturn 1\n`)\n\n// Schedule adds the task to the scheduled set to be processed in the future.\nfunc (r *RDB) Schedule(ctx context.Context, msg *base.TaskMessage, processAt time.Time) error {\n\tvar op errors.Op = \"rdb.Schedule\"\n\tencoded, err := base.EncodeMessage(msg)\n\tif err != nil {\n\t\treturn errors.E(op, errors.Unknown, fmt.Sprintf(\"cannot encode message: %v\", err))\n\t}\n\tif _, found := r.queuesPublished.Load(msg.Queue); !found {\n\t\tif err := r.client.SAdd(ctx, base.AllQueues, msg.Queue).Err(); err != nil {\n\t\t\treturn errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: \"sadd\", Err: err})\n\t\t}\n\t\tr.queuesPublished.Store(msg.Queue, true)\n\t}\n\tkeys := []string{\n\t\tbase.TaskKey(msg.Queue, msg.ID),\n\t\tbase.ScheduledKey(msg.Queue),\n\t}\n\targv := []interface{}{\n\t\tencoded,\n\t\tprocessAt.Unix(),\n\t\tmsg.ID,\n\t}\n\tn, err := r.runScriptWithErrorCode(ctx, op, scheduleCmd, keys, argv...)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif n == 0 {\n\t\treturn errors.E(op, errors.AlreadyExists, errors.ErrTaskIdConflict)\n\t}\n\treturn nil\n}\n\n// KEYS[1] -> unique key\n// KEYS[2] -> asynq:{}:t:\n// KEYS[3] -> asynq:{}:scheduled\n// -------\n// ARGV[1] -> task ID\n// ARGV[2] -> uniqueness lock TTL\n// ARGV[3] -> score (process_at timestamp)\n// ARGV[4] -> task message\n//\n// Output:\n// Returns 1 if successfully scheduled\n// Returns 0 if task ID already exists\n// Returns -1 if task unique key already exists\nvar scheduleUniqueCmd = redis.NewScript(`\nlocal ok = redis.call(\"SET\", KEYS[1], ARGV[1], \"NX\", \"EX\", ARGV[2])\nif not ok then\n return -1\nend\nif redis.call(\"EXISTS\", KEYS[2]) == 1 then\n return 0\nend\nredis.call(\"HSET\", KEYS[2],\n \"msg\", ARGV[4],\n \"state\", \"scheduled\",\n \"unique_key\", KEYS[1])\nredis.call(\"ZADD\", KEYS[3], ARGV[3], ARGV[1])\nreturn 1\n`)\n\n// ScheduleUnique adds the task to the backlog queue to be processed in the future if the uniqueness lock can be acquired.\n// It returns ErrDuplicateTask if the lock cannot be acquired.\nfunc (r *RDB) ScheduleUnique(ctx context.Context, msg *base.TaskMessage, processAt time.Time, ttl time.Duration) error {\n\tvar op errors.Op = \"rdb.ScheduleUnique\"\n\tencoded, err := base.EncodeMessage(msg)\n\tif err != nil {\n\t\treturn errors.E(op, errors.Internal, fmt.Sprintf(\"cannot encode task message: %v\", err))\n\t}\n\tif _, found := r.queuesPublished.Load(msg.Queue); !found {\n\t\tif err := r.client.SAdd(ctx, base.AllQueues, msg.Queue).Err(); err != nil {\n\t\t\treturn errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: \"sadd\", Err: err})\n\t\t}\n\t\tr.queuesPublished.Store(msg.Queue, true)\n\t}\n\tkeys := []string{\n\t\tmsg.UniqueKey,\n\t\tbase.TaskKey(msg.Queue, msg.ID),\n\t\tbase.ScheduledKey(msg.Queue),\n\t}\n\targv := []interface{}{\n\t\tmsg.ID,\n\t\tint(ttl.Seconds()),\n\t\tprocessAt.Unix(),\n\t\tencoded,\n\t}\n\tn, err := r.runScriptWithErrorCode(ctx, op, scheduleUniqueCmd, keys, argv...)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif n == -1 {\n\t\treturn errors.E(op, errors.AlreadyExists, errors.ErrDuplicateTask)\n\t}\n\tif n == 0 {\n\t\treturn errors.E(op, errors.AlreadyExists, errors.ErrTaskIdConflict)\n\t}\n\treturn nil\n}\n\n// KEYS[1] -> asynq:{}:t:\n// KEYS[2] -> asynq:{}:active\n// KEYS[3] -> asynq:{}:lease\n// KEYS[4] -> asynq:{}:retry\n// KEYS[5] -> asynq:{}:processed:\n// KEYS[6] -> asynq:{}:failed:\n// KEYS[7] -> asynq:{}:processed\n// KEYS[8] -> asynq:{}:failed\n// -------\n// ARGV[1] -> task ID\n// ARGV[2] -> updated base.TaskMessage value\n// ARGV[3] -> retry_at UNIX timestamp\n// ARGV[4] -> stats expiration timestamp\n// ARGV[5] -> is_failure (bool)\n// ARGV[6] -> max int64 value\nvar retryCmd = redis.NewScript(`\nif redis.call(\"LREM\", KEYS[2], 0, ARGV[1]) == 0 then\n return redis.error_reply(\"NOT FOUND\")\nend\nif redis.call(\"ZREM\", KEYS[3], ARGV[1]) == 0 then\n return redis.error_reply(\"NOT FOUND\")\nend\nredis.call(\"ZADD\", KEYS[4], ARGV[3], ARGV[1])\nredis.call(\"HSET\", KEYS[1], \"msg\", ARGV[2], \"state\", \"retry\")\nif tonumber(ARGV[5]) == 1 then\n\tlocal n = redis.call(\"INCR\", KEYS[5])\n\tif tonumber(n) == 1 then\n\t\tredis.call(\"EXPIREAT\", KEYS[5], ARGV[4])\n\tend\n\tlocal m = redis.call(\"INCR\", KEYS[6])\n\tif tonumber(m) == 1 then\n\t\tredis.call(\"EXPIREAT\", KEYS[6], ARGV[4])\n\tend\n local total = redis.call(\"GET\", KEYS[7])\n if tonumber(total) == tonumber(ARGV[6]) then\n \tredis.call(\"SET\", KEYS[7], 1)\n \tredis.call(\"SET\", KEYS[8], 1)\n else\n \tredis.call(\"INCR\", KEYS[7])\n \tredis.call(\"INCR\", KEYS[8])\n end\nend\nreturn redis.status_reply(\"OK\")`)\n\n// Retry moves the task from active to retry queue.\n// It also annotates the message with the given error message and\n// if isFailure is true increments the retried counter.\nfunc (r *RDB) Retry(ctx context.Context, msg *base.TaskMessage, processAt time.Time, errMsg string, isFailure bool) error {\n\tvar op errors.Op = \"rdb.Retry\"\n\tnow := r.clock.Now()\n\tmodified := *msg\n\tif isFailure {\n\t\tmodified.Retried++\n\t}\n\tmodified.ErrorMsg = errMsg\n\tmodified.LastFailedAt = now.Unix()\n\tencoded, err := base.EncodeMessage(&modified)\n\tif err != nil {\n\t\treturn errors.E(op, errors.Internal, fmt.Sprintf(\"cannot encode message: %v\", err))\n\t}\n\texpireAt := now.Add(statsTTL)\n\tkeys := []string{\n\t\tbase.TaskKey(msg.Queue, msg.ID),\n\t\tbase.ActiveKey(msg.Queue),\n\t\tbase.LeaseKey(msg.Queue),\n\t\tbase.RetryKey(msg.Queue),\n\t\tbase.ProcessedKey(msg.Queue, now),\n\t\tbase.FailedKey(msg.Queue, now),\n\t\tbase.ProcessedTotalKey(msg.Queue),\n\t\tbase.FailedTotalKey(msg.Queue),\n\t}\n\targv := []interface{}{\n\t\tmsg.ID,\n\t\tencoded,\n\t\tprocessAt.Unix(),\n\t\texpireAt.Unix(),\n\t\tisFailure,\n\t\tint64(math.MaxInt64),\n\t}\n\treturn r.runScript(ctx, op, retryCmd, keys, argv...)\n}\n\nconst (\n\tmaxArchiveSize = 10000 // maximum number of tasks in archive\n\tarchivedExpirationInDays = 90 // number of days before an archived task gets deleted permanently\n)\n\n// KEYS[1] -> asynq:{}:t:\n// KEYS[2] -> asynq:{}:active\n// KEYS[3] -> asynq:{}:lease\n// KEYS[4] -> asynq:{}:archived\n// KEYS[5] -> asynq:{}:processed:\n// KEYS[6] -> asynq:{}:failed:\n// KEYS[7] -> asynq:{}:processed\n// KEYS[8] -> asynq:{}:failed\n// KEYS[9] -> asynq:{}:t:\n// -------\n// ARGV[1] -> task ID\n// ARGV[2] -> updated base.TaskMessage value\n// ARGV[3] -> died_at UNIX timestamp\n// ARGV[4] -> cutoff timestamp (e.g., 90 days ago)\n// ARGV[5] -> max number of tasks in archive (e.g., 100)\n// ARGV[6] -> stats expiration timestamp\n// ARGV[7] -> max int64 value\nvar archiveCmd = redis.NewScript(`\nif redis.call(\"LREM\", KEYS[2], 0, ARGV[1]) == 0 then\n return redis.error_reply(\"NOT FOUND\")\nend\nif redis.call(\"ZREM\", KEYS[3], ARGV[1]) == 0 then\n return redis.error_reply(\"NOT FOUND\")\nend\nredis.call(\"ZADD\", KEYS[4], ARGV[3], ARGV[1])\nlocal old = redis.call(\"ZRANGE\", KEYS[4], \"-inf\", ARGV[4], \"BYSCORE\")\nif #old > 0 then\n\tfor _, id in ipairs(old) do\n\t\tredis.call(\"DEL\", KEYS[9] .. id)\n\tend\n\tredis.call(\"ZREM\", KEYS[4], unpack(old))\nend\n\nlocal extra = redis.call(\"ZRANGE\", KEYS[4], 0, -ARGV[5])\nif #extra > 0 then\n\tfor _, id in ipairs(extra) do\n\t\tredis.call(\"DEL\", KEYS[9] .. id)\n\tend\n\tredis.call(\"ZREM\", KEYS[4], unpack(extra))\nend\n\nredis.call(\"HSET\", KEYS[1], \"msg\", ARGV[2], \"state\", \"archived\")\nlocal n = redis.call(\"INCR\", KEYS[5])\nif tonumber(n) == 1 then\n\tredis.call(\"EXPIREAT\", KEYS[5], ARGV[6])\nend\nlocal m = redis.call(\"INCR\", KEYS[6])\nif tonumber(m) == 1 then\n\tredis.call(\"EXPIREAT\", KEYS[6], ARGV[6])\nend\nlocal total = redis.call(\"GET\", KEYS[7])\nif tonumber(total) == tonumber(ARGV[7]) then\n \tredis.call(\"SET\", KEYS[7], 1)\n \tredis.call(\"SET\", KEYS[8], 1)\nelse\n \tredis.call(\"INCR\", KEYS[7])\n \tredis.call(\"INCR\", KEYS[8])\nend\nreturn redis.status_reply(\"OK\")`)\n\n// Archive sends the given task to archive, attaching the error message to the task.\n// It also trims the archive by timestamp and set size.\nfunc (r *RDB) Archive(ctx context.Context, msg *base.TaskMessage, errMsg string) error {\n\tvar op errors.Op = \"rdb.Archive\"\n\tnow := r.clock.Now()\n\tmodified := *msg\n\tmodified.ErrorMsg = errMsg\n\tmodified.LastFailedAt = now.Unix()\n\tencoded, err := base.EncodeMessage(&modified)\n\tif err != nil {\n\t\treturn errors.E(op, errors.Internal, fmt.Sprintf(\"cannot encode message: %v\", err))\n\t}\n\tcutoff := now.AddDate(0, 0, -archivedExpirationInDays)\n\texpireAt := now.Add(statsTTL)\n\tkeys := []string{\n\t\tbase.TaskKey(msg.Queue, msg.ID),\n\t\tbase.ActiveKey(msg.Queue),\n\t\tbase.LeaseKey(msg.Queue),\n\t\tbase.ArchivedKey(msg.Queue),\n\t\tbase.ProcessedKey(msg.Queue, now),\n\t\tbase.FailedKey(msg.Queue, now),\n\t\tbase.ProcessedTotalKey(msg.Queue),\n\t\tbase.FailedTotalKey(msg.Queue),\n\t\tbase.TaskKeyPrefix(msg.Queue),\n\t}\n\targv := []interface{}{\n\t\tmsg.ID,\n\t\tencoded,\n\t\tnow.Unix(),\n\t\tcutoff.Unix(),\n\t\tmaxArchiveSize,\n\t\texpireAt.Unix(),\n\t\tint64(math.MaxInt64),\n\t}\n\treturn r.runScript(ctx, op, archiveCmd, keys, argv...)\n}\n\n// ForwardIfReady checks scheduled and retry sets of the given queues\n// and move any tasks that are ready to be processed to the pending set.\nfunc (r *RDB) ForwardIfReady(qnames ...string) error {\n\tvar op errors.Op = \"rdb.ForwardIfReady\"\n\tfor _, qname := range qnames {\n\t\tif err := r.forwardAll(qname); err != nil {\n\t\t\treturn errors.E(op, errors.CanonicalCode(err), err)\n\t\t}\n\t}\n\treturn nil\n}\n\n// KEYS[1] -> source queue (e.g. asynq:{:scheduled or asynq:{}:retry})\n// KEYS[2] -> asynq:{}:pending\n// ARGV[1] -> current unix time in seconds\n// ARGV[2] -> task key prefix\n// ARGV[3] -> current unix time in nsec\n// ARGV[4] -> group key prefix\n// Note: Script moves tasks up to 100 at a time to keep the runtime of script short.\nvar forwardCmd = redis.NewScript(`\nlocal ids = redis.call(\"ZRANGEBYSCORE\", KEYS[1], \"-inf\", ARGV[1], \"LIMIT\", 0, 100)\nfor _, id in ipairs(ids) do\n\tlocal taskKey = ARGV[2] .. id\n\tlocal group = redis.call(\"HGET\", taskKey, \"group\")\n\tif group and group ~= '' then\n\t redis.call(\"ZADD\", ARGV[4] .. group, ARGV[1], id)\n\t\tredis.call(\"ZREM\", KEYS[1], id)\n\t\tredis.call(\"HSET\", taskKey,\n\t\t\t\t \"state\", \"aggregating\")\n\telse\n\t\tredis.call(\"LPUSH\", KEYS[2], id)\n\t\tredis.call(\"ZREM\", KEYS[1], id)\n\t\tredis.call(\"HSET\", taskKey,\n\t\t\t\t \"state\", \"pending\",\n\t\t\t\t \"pending_since\", ARGV[3])\n\tend\nend\nreturn table.getn(ids)`)\n\n// forward moves tasks with a score less than the current unix time from the delayed (i.e. scheduled | retry) zset\n// to the pending list or group set.\n// It returns the number of tasks moved.\nfunc (r *RDB) forward(delayedKey, pendingKey, taskKeyPrefix, groupKeyPrefix string) (int, error) {\n\tnow := r.clock.Now()\n\tkeys := []string{delayedKey, pendingKey}\n\targv := []interface{}{\n\t\tnow.Unix(),\n\t\ttaskKeyPrefix,\n\t\tnow.UnixNano(),\n\t\tgroupKeyPrefix,\n\t}\n\tres, err := forwardCmd.Run(context.Background(), r.client, keys, argv...).Result()\n\tif err != nil {\n\t\treturn 0, errors.E(errors.Internal, fmt.Sprintf(\"redis eval error: %v\", err))\n\t}\n\tn, err := cast.ToIntE(res)\n\tif err != nil {\n\t\treturn 0, errors.E(errors.Internal, fmt.Sprintf(\"cast error: Lua script returned unexpected value: %v\", res))\n\t}\n\treturn n, nil\n}\n\n// forwardAll checks for tasks in scheduled/retry state that are ready to be run, and updates\n// their state to \"pending\" or \"aggregating\".\nfunc (r *RDB) forwardAll(qname string) (err error) {\n\tdelayedKeys := []string{base.ScheduledKey(qname), base.RetryKey(qname)}\n\tpendingKey := base.PendingKey(qname)\n\ttaskKeyPrefix := base.TaskKeyPrefix(qname)\n\tgroupKeyPrefix := base.GroupKeyPrefix(qname)\n\tfor _, delayedKey := range delayedKeys {\n\t\tn := 1\n\t\tfor n != 0 {\n\t\t\tn, err = r.forward(delayedKey, pendingKey, taskKeyPrefix, groupKeyPrefix)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\n// ListGroups returns a list of all known groups in the given queue.\nfunc (r *RDB) ListGroups(qname string) ([]string, error) {\n\tvar op errors.Op = \"RDB.ListGroups\"\n\tgroups, err := r.client.SMembers(context.Background(), base.AllGroups(qname)).Result()\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: \"smembers\", Err: err})\n\t}\n\treturn groups, nil\n}\n\n// aggregationCheckCmd checks the given group for whether to create an aggregation set.\n// An aggregation set is created if one of the aggregation criteria is met:\n// 1) group has reached or exceeded its max size\n// 2) group's oldest task has reached or exceeded its max delay\n// 3) group's latest task has reached or exceeded its grace period\n// if aggreation criteria is met, the command moves those tasks from the group\n// and put them in an aggregation set. Additionally, if the creation of aggregation set\n// empties the group, it will clear the group name from the all groups set.\n//\n// KEYS[1] -> asynq:{}:g:\n// KEYS[2] -> asynq:{}:g::\n// KEYS[3] -> asynq:{}:aggregation_sets\n// KEYS[4] -> asynq:{}:groups\n// -------\n// ARGV[1] -> max group size\n// ARGV[2] -> max group delay in unix time\n// ARGV[3] -> start time of the grace period\n// ARGV[4] -> aggregation set expire time\n// ARGV[5] -> current time in unix time\n// ARGV[6] -> group name\n//\n// Output:\n// Returns 0 if no aggregation set was created\n// Returns 1 if an aggregation set was created\n//\n// Time Complexity:\n// O(log(N) + M) with N being the number tasks in the group zset\n// and M being the max size.\nvar aggregationCheckCmd = redis.NewScript(`\nlocal size = redis.call(\"ZCARD\", KEYS[1])\nif size == 0 then\n\treturn 0\nend\nlocal maxSize = tonumber(ARGV[1])\nif maxSize ~= 0 and size >= maxSize then\n\tlocal res = redis.call(\"ZRANGE\", KEYS[1], 0, maxSize-1, \"WITHSCORES\")\n\tfor i=1, table.getn(res)-1, 2 do\n\t\tredis.call(\"ZADD\", KEYS[2], tonumber(res[i+1]), res[i])\n\tend\n\tredis.call(\"ZREMRANGEBYRANK\", KEYS[1], 0, maxSize-1)\n\tredis.call(\"ZADD\", KEYS[3], ARGV[4], KEYS[2])\n\tif size == maxSize then\n\t\tredis.call(\"SREM\", KEYS[4], ARGV[6])\n\tend\n\treturn 1\nend\nlocal maxDelay = tonumber(ARGV[2])\nlocal currentTime = tonumber(ARGV[5])\nif maxDelay ~= 0 then\n\tlocal oldestEntry = redis.call(\"ZRANGE\", KEYS[1], 0, 0, \"WITHSCORES\")\n\tlocal oldestEntryScore = tonumber(oldestEntry[2])\n\tlocal maxDelayTime = currentTime - maxDelay\n\tif oldestEntryScore <= maxDelayTime then\n\t\tlocal res = redis.call(\"ZRANGE\", KEYS[1], 0, maxSize-1, \"WITHSCORES\")\n\t\tfor i=1, table.getn(res)-1, 2 do\n\t\t\tredis.call(\"ZADD\", KEYS[2], tonumber(res[i+1]), res[i])\n\t\tend\n\t\tredis.call(\"ZREMRANGEBYRANK\", KEYS[1], 0, maxSize-1)\n\t\tredis.call(\"ZADD\", KEYS[3], ARGV[4], KEYS[2])\n\t\tif size <= maxSize or maxSize == 0 then\n\t\t\tredis.call(\"SREM\", KEYS[4], ARGV[6])\n\t\tend\n\t\treturn 1\n\tend\nend\nlocal latestEntry = redis.call(\"ZREVRANGE\", KEYS[1], 0, 0, \"WITHSCORES\")\nlocal latestEntryScore = tonumber(latestEntry[2])\nlocal gracePeriodStartTime = currentTime - tonumber(ARGV[3])\nif latestEntryScore <= gracePeriodStartTime then\n\tlocal res = redis.call(\"ZRANGE\", KEYS[1], 0, maxSize-1, \"WITHSCORES\")\n\tfor i=1, table.getn(res)-1, 2 do\n\t\tredis.call(\"ZADD\", KEYS[2], tonumber(res[i+1]), res[i])\n\tend\n\tredis.call(\"ZREMRANGEBYRANK\", KEYS[1], 0, maxSize-1)\n\tredis.call(\"ZADD\", KEYS[3], ARGV[4], KEYS[2])\n\tif size <= maxSize or maxSize == 0 then\n\t\tredis.call(\"SREM\", KEYS[4], ARGV[6])\n\tend\n\treturn 1\nend\nreturn 0\n`)\n\n// Task aggregation should finish within this timeout.\n// Otherwise an aggregation set should be reclaimed by the recoverer.\nconst aggregationTimeout = 2 * time.Minute\n\n// AggregationCheck checks the group identified by the given queue and group name to see if the tasks in the\n// group are ready to be aggregated. If so, it moves the tasks to be aggregated to a aggregation set and returns\n// the set ID. If not, it returns an empty string for the set ID.\n// The time for gracePeriod and maxDelay is computed relative to the time t.\n//\n// Note: It assumes that this function is called at frequency less than or equal to the gracePeriod. In other words,\n// the function only checks the most recently added task against the given gracePeriod.\nfunc (r *RDB) AggregationCheck(qname, gname string, t time.Time, gracePeriod, maxDelay time.Duration, maxSize int) (string, error) {\n\tvar op errors.Op = \"RDB.AggregationCheck\"\n\taggregationSetID := uuid.NewString()\n\texpireTime := r.clock.Now().Add(aggregationTimeout)\n\tkeys := []string{\n\t\tbase.GroupKey(qname, gname),\n\t\tbase.AggregationSetKey(qname, gname, aggregationSetID),\n\t\tbase.AllAggregationSets(qname),\n\t\tbase.AllGroups(qname),\n\t}\n\targv := []interface{}{\n\t\tmaxSize,\n\t\tint64(maxDelay.Seconds()),\n\t\tint64(gracePeriod.Seconds()),\n\t\texpireTime.Unix(),\n\t\tt.Unix(),\n\t\tgname,\n\t}\n\tn, err := r.runScriptWithErrorCode(context.Background(), op, aggregationCheckCmd, keys, argv...)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tswitch n {\n\tcase 0:\n\t\treturn \"\", nil\n\tcase 1:\n\t\treturn aggregationSetID, nil\n\tdefault:\n\t\treturn \"\", errors.E(op, errors.Internal, fmt.Sprintf(\"unexpected return value from lua script: %d\", n))\n\t}\n}\n\n// KEYS[1] -> asynq:{}:g::\n// ------\n// ARGV[1] -> task key prefix\n//\n// Output:\n// Array of encoded task messages\n//\n// Time Complexity:\n// O(N) with N being the number of tasks in the aggregation set.\nvar readAggregationSetCmd = redis.NewScript(`\nlocal msgs = {}\nlocal ids = redis.call(\"ZRANGE\", KEYS[1], 0, -1)\nfor _, id in ipairs(ids) do\n\tlocal key = ARGV[1] .. id\n\ttable.insert(msgs, redis.call(\"HGET\", key, \"msg\"))\nend\nreturn msgs\n`)\n\n// ReadAggregationSet retrieves members of an aggregation set and returns a list of tasks in the set and\n// the deadline for aggregating those tasks.\nfunc (r *RDB) ReadAggregationSet(qname, gname, setID string) ([]*base.TaskMessage, time.Time, error) {\n\tvar op errors.Op = \"RDB.ReadAggregationSet\"\n\tctx := context.Background()\n\taggSetKey := base.AggregationSetKey(qname, gname, setID)\n\tres, err := readAggregationSetCmd.Run(ctx, r.client,\n\t\t[]string{aggSetKey}, base.TaskKeyPrefix(qname)).Result()\n\tif err != nil {\n\t\treturn nil, time.Time{}, errors.E(op, errors.Unknown, fmt.Sprintf(\"redis eval error: %v\", err))\n\t}\n\tdata, err := cast.ToStringSliceE(res)\n\tif err != nil {\n\t\treturn nil, time.Time{}, errors.E(op, errors.Internal, fmt.Sprintf(\"cast error: Lua script returned unexpected value: %v\", res))\n\t}\n\tvar msgs []*base.TaskMessage\n\tfor _, s := range data {\n\t\tmsg, err := base.DecodeMessage([]byte(s))\n\t\tif err != nil {\n\t\t\treturn nil, time.Time{}, errors.E(op, errors.Internal, fmt.Sprintf(\"cannot decode message: %v\", err))\n\t\t}\n\t\tmsgs = append(msgs, msg)\n\t}\n\tdeadlineUnix, err := r.client.ZScore(ctx, base.AllAggregationSets(qname), aggSetKey).Result()\n\tif err != nil {\n\t\treturn nil, time.Time{}, errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: \"zscore\", Err: err})\n\t}\n\treturn msgs, time.Unix(int64(deadlineUnix), 0), nil\n}\n\n// KEYS[1] -> asynq:{}:g::\n// KEYS[2] -> asynq:{}:aggregation_sets\n// -------\n// ARGV[1] -> task key prefix\n//\n// Output:\n// Redis status reply\n//\n// Time Complexity:\n// max(O(N), O(log(M))) with N being the number of tasks in the aggregation set\n// and M being the number of elements in the all-aggregation-sets list.\nvar deleteAggregationSetCmd = redis.NewScript(`\nlocal ids = redis.call(\"ZRANGE\", KEYS[1], 0, -1)\nfor _, id in ipairs(ids) do\n\tredis.call(\"DEL\", ARGV[1] .. id)\nend\nredis.call(\"DEL\", KEYS[1])\nredis.call(\"ZREM\", KEYS[2], KEYS[1])\nreturn redis.status_reply(\"OK\")\n`)\n\n// DeleteAggregationSet deletes the aggregation set and its members identified by the parameters.\nfunc (r *RDB) DeleteAggregationSet(ctx context.Context, qname, gname, setID string) error {\n\tvar op errors.Op = \"RDB.DeleteAggregationSet\"\n\tkeys := []string{\n\t\tbase.AggregationSetKey(qname, gname, setID),\n\t\tbase.AllAggregationSets(qname),\n\t}\n\treturn r.runScript(ctx, op, deleteAggregationSetCmd, keys, base.TaskKeyPrefix(qname))\n}\n\n// KEYS[1] -> asynq:{}:aggregation_sets\n// -------\n// ARGV[1] -> current time in unix time\nvar reclaimStateAggregationSetsCmd = redis.NewScript(`\nlocal staleSetKeys = redis.call(\"ZRANGEBYSCORE\", KEYS[1], \"-inf\", ARGV[1])\nfor _, key in ipairs(staleSetKeys) do\n\tlocal idx = string.find(key, \":[^:]*$\")\n\tlocal groupKey = string.sub(key, 1, idx-1)\n\tlocal res = redis.call(\"ZRANGE\", key, 0, -1, \"WITHSCORES\")\n\tfor i=1, table.getn(res)-1, 2 do\n\t\tredis.call(\"ZADD\", groupKey, tonumber(res[i+1]), res[i])\n\tend\n\tredis.call(\"DEL\", key)\nend\nredis.call(\"ZREMRANGEBYSCORE\", KEYS[1], \"-inf\", ARGV[1])\nreturn redis.status_reply(\"OK\")\n`)\n\n// ReclaimStaleAggregationSets checks for any stale aggregation sets in the given queue, and\n// reclaim tasks in the stale aggregation set by putting them back in the group.\nfunc (r *RDB) ReclaimStaleAggregationSets(qname string) error {\n\tvar op errors.Op = \"RDB.ReclaimStaleAggregationSets\"\n\treturn r.runScript(context.Background(), op, reclaimStateAggregationSetsCmd,\n\t\t[]string{base.AllAggregationSets(qname)}, r.clock.Now().Unix())\n}\n\n// KEYS[1] -> asynq:{}:completed\n// ARGV[1] -> current time in unix time\n// ARGV[2] -> task key prefix\n// ARGV[3] -> batch size (i.e. maximum number of tasks to delete)\n//\n// Returns the number of tasks deleted.\nvar deleteExpiredCompletedTasksCmd = redis.NewScript(`\nlocal ids = redis.call(\"ZRANGEBYSCORE\", KEYS[1], \"-inf\", ARGV[1], \"LIMIT\", 0, tonumber(ARGV[3]))\nfor _, id in ipairs(ids) do\n\tredis.call(\"DEL\", ARGV[2] .. id)\n\tredis.call(\"ZREM\", KEYS[1], id)\nend\nreturn table.getn(ids)`)\n\n// DeleteExpiredCompletedTasks checks for any expired tasks in the given queue's completed set,\n// and delete all expired tasks.\nfunc (r *RDB) DeleteExpiredCompletedTasks(qname string, batchSize int) error {\n\tfor {\n\t\tn, err := r.deleteExpiredCompletedTasks(qname, batchSize)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif n == 0 {\n\t\t\treturn nil\n\t\t}\n\t}\n}\n\n// deleteExpiredCompletedTasks runs the lua script to delete expired deleted task with the specified\n// batch size. It reports the number of tasks deleted.\nfunc (r *RDB) deleteExpiredCompletedTasks(qname string, batchSize int) (int64, error) {\n\tvar op errors.Op = \"rdb.DeleteExpiredCompletedTasks\"\n\tkeys := []string{base.CompletedKey(qname)}\n\targv := []interface{}{\n\t\tr.clock.Now().Unix(),\n\t\tbase.TaskKeyPrefix(qname),\n\t\tbatchSize,\n\t}\n\tres, err := deleteExpiredCompletedTasksCmd.Run(context.Background(), r.client, keys, argv...).Result()\n\tif err != nil {\n\t\treturn 0, errors.E(op, errors.Internal, fmt.Sprintf(\"redis eval error: %v\", err))\n\t}\n\tn, ok := res.(int64)\n\tif !ok {\n\t\treturn 0, errors.E(op, errors.Internal, fmt.Sprintf(\"unexpected return value from Lua script: %v\", res))\n\t}\n\treturn n, nil\n}\n\n// KEYS[1] -> asynq:{}:lease\n// ARGV[1] -> cutoff in unix time\n// ARGV[2] -> task key prefix\nvar listLeaseExpiredCmd = redis.NewScript(`\nlocal res = {}\nlocal ids = redis.call(\"ZRANGEBYSCORE\", KEYS[1], \"-inf\", ARGV[1])\nfor _, id in ipairs(ids) do\n\tlocal key = ARGV[2] .. id\n\tlocal v = redis.call(\"HGET\", key, \"msg\")\n\tif v then\n\t\ttable.insert(res, v)\n\tend\nend\nreturn res\n`)\n\n// ListLeaseExpired returns a list of task messages with an expired lease from the given queues.\nfunc (r *RDB) ListLeaseExpired(cutoff time.Time, qnames ...string) ([]*base.TaskMessage, error) {\n\tvar op errors.Op = \"rdb.ListLeaseExpired\"\n\tvar msgs []*base.TaskMessage\n\tfor _, qname := range qnames {\n\t\tres, err := listLeaseExpiredCmd.Run(context.Background(), r.client,\n\t\t\t[]string{base.LeaseKey(qname)},\n\t\t\tcutoff.Unix(), base.TaskKeyPrefix(qname)).Result()\n\t\tif err != nil {\n\t\t\treturn nil, errors.E(op, errors.Internal, fmt.Sprintf(\"redis eval error: %v\", err))\n\t\t}\n\t\tdata, err := cast.ToStringSliceE(res)\n\t\tif err != nil {\n\t\t\treturn nil, errors.E(op, errors.Internal, fmt.Sprintf(\"cast error: Lua script returned unexpected value: %v\", res))\n\t\t}\n\t\tfor _, s := range data {\n\t\t\tmsg, err := base.DecodeMessage([]byte(s))\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.E(op, errors.Internal, fmt.Sprintf(\"cannot decode message: %v\", err))\n\t\t\t}\n\t\t\tmsgs = append(msgs, msg)\n\t\t}\n\t}\n\treturn msgs, nil\n}\n\n// ExtendLease extends the lease for the given tasks by LeaseDuration (30s).\n// It returns a new expiration time if the operation was successful.\nfunc (r *RDB) ExtendLease(qname string, ids ...string) (expirationTime time.Time, err error) {\n\texpireAt := r.clock.Now().Add(LeaseDuration)\n\tvar zs []redis.Z\n\tfor _, id := range ids {\n\t\tzs = append(zs, redis.Z{Member: id, Score: float64(expireAt.Unix())})\n\t}\n\t// Use XX option to only update elements that already exist; Don't add new elements\n\t// TODO: Consider adding GT option to ensure we only \"extend\" the lease. Ceveat is that GT is supported from redis v6.2.0 or above.\n\terr = r.client.ZAddXX(context.Background(), base.LeaseKey(qname), zs...).Err()\n\tif err != nil {\n\t\treturn time.Time{}, err\n\t}\n\treturn expireAt, nil\n}\n\n// KEYS[1] -> asynq:servers:{}\n// KEYS[2] -> asynq:workers:{}\n// ARGV[1] -> TTL in seconds\n// ARGV[2] -> server info\n// ARGV[3:] -> alternate key-value pair of (worker id, worker data)\n// Note: Add key to ZSET with expiration time as score.\n// ref: https://github.com/antirez/redis/issues/135#issuecomment-2361996\nvar writeServerStateCmd = redis.NewScript(`\nredis.call(\"SETEX\", KEYS[1], ARGV[1], ARGV[2])\nredis.call(\"DEL\", KEYS[2])\nfor i = 3, table.getn(ARGV)-1, 2 do\n\tredis.call(\"HSET\", KEYS[2], ARGV[i], ARGV[i+1])\nend\nredis.call(\"EXPIRE\", KEYS[2], ARGV[1])\nreturn redis.status_reply(\"OK\")`)\n\n// WriteServerState writes server state data to redis with expiration set to the value ttl.\nfunc (r *RDB) WriteServerState(info *base.ServerInfo, workers []*base.WorkerInfo, ttl time.Duration) error {\n\tvar op errors.Op = \"rdb.WriteServerState\"\n\tctx := context.Background()\n\tbytes, err := base.EncodeServerInfo(info)\n\tif err != nil {\n\t\treturn errors.E(op, errors.Internal, fmt.Sprintf(\"cannot encode server info: %v\", err))\n\t}\n\texp := r.clock.Now().Add(ttl).UTC()\n\targs := []interface{}{ttl.Seconds(), bytes} // args to the lua script\n\tfor _, w := range workers {\n\t\tbytes, err := base.EncodeWorkerInfo(w)\n\t\tif err != nil {\n\t\t\tcontinue // skip bad data\n\t\t}\n\t\targs = append(args, w.ID, bytes)\n\t}\n\tskey := base.ServerInfoKey(info.Host, info.PID, info.ServerID)\n\twkey := base.WorkersKey(info.Host, info.PID, info.ServerID)\n\tif err := r.client.ZAdd(ctx, base.AllServers, redis.Z{Score: float64(exp.Unix()), Member: skey}).Err(); err != nil {\n\t\treturn errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: \"sadd\", Err: err})\n\t}\n\tif err := r.client.ZAdd(ctx, base.AllWorkers, redis.Z{Score: float64(exp.Unix()), Member: wkey}).Err(); err != nil {\n\t\treturn errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: \"zadd\", Err: err})\n\t}\n\treturn r.runScript(ctx, op, writeServerStateCmd, []string{skey, wkey}, args...)\n}\n\n// KEYS[1] -> asynq:servers:{}\n// KEYS[2] -> asynq:workers:{}\nvar clearServerStateCmd = redis.NewScript(`\nredis.call(\"DEL\", KEYS[1])\nredis.call(\"DEL\", KEYS[2])\nreturn redis.status_reply(\"OK\")`)\n\n// ClearServerState deletes server state data from redis.\nfunc (r *RDB) ClearServerState(host string, pid int, serverID string) error {\n\tvar op errors.Op = \"rdb.ClearServerState\"\n\tctx := context.Background()\n\tskey := base.ServerInfoKey(host, pid, serverID)\n\twkey := base.WorkersKey(host, pid, serverID)\n\tif err := r.client.ZRem(ctx, base.AllServers, skey).Err(); err != nil {\n\t\treturn errors.E(op, errors.Internal, &errors.RedisCommandError{Command: \"zrem\", Err: err})\n\t}\n\tif err := r.client.ZRem(ctx, base.AllWorkers, wkey).Err(); err != nil {\n\t\treturn errors.E(op, errors.Internal, &errors.RedisCommandError{Command: \"zrem\", Err: err})\n\t}\n\treturn r.runScript(ctx, op, clearServerStateCmd, []string{skey, wkey})\n}\n\n// KEYS[1] -> asynq:schedulers:{}\n// ARGV[1] -> TTL in seconds\n// ARGV[2:] -> scheduler entries\nvar writeSchedulerEntriesCmd = redis.NewScript(`\nredis.call(\"DEL\", KEYS[1])\nfor i = 2, #ARGV do\n\tredis.call(\"LPUSH\", KEYS[1], ARGV[i])\nend\nredis.call(\"EXPIRE\", KEYS[1], ARGV[1])\nreturn redis.status_reply(\"OK\")`)\n\n// WriteSchedulerEntries writes scheduler entries data to redis with expiration set to the value ttl.\nfunc (r *RDB) WriteSchedulerEntries(schedulerID string, entries []*base.SchedulerEntry, ttl time.Duration) error {\n\tvar op errors.Op = \"rdb.WriteSchedulerEntries\"\n\tctx := context.Background()\n\targs := []interface{}{ttl.Seconds()}\n\tfor _, e := range entries {\n\t\tbytes, err := base.EncodeSchedulerEntry(e)\n\t\tif err != nil {\n\t\t\tcontinue // skip bad data\n\t\t}\n\t\targs = append(args, bytes)\n\t}\n\texp := r.clock.Now().Add(ttl).UTC()\n\tkey := base.SchedulerEntriesKey(schedulerID)\n\terr := r.client.ZAdd(ctx, base.AllSchedulers, redis.Z{Score: float64(exp.Unix()), Member: key}).Err()\n\tif err != nil {\n\t\treturn errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: \"zadd\", Err: err})\n\t}\n\treturn r.runScript(ctx, op, writeSchedulerEntriesCmd, []string{key}, args...)\n}\n\n// ClearSchedulerEntries deletes scheduler entries data from redis.\nfunc (r *RDB) ClearSchedulerEntries(schedulerID string) error {\n\tvar op errors.Op = \"rdb.ClearSchedulerEntries\"\n\tctx := context.Background()\n\tkey := base.SchedulerEntriesKey(schedulerID)\n\tif err := r.client.ZRem(ctx, base.AllSchedulers, key).Err(); err != nil {\n\t\treturn errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: \"zrem\", Err: err})\n\t}\n\tif err := r.client.Del(ctx, key).Err(); err != nil {\n\t\treturn errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: \"del\", Err: err})\n\t}\n\treturn nil\n}\n\n// CancelationPubSub returns a pubsub for cancelation messages.\nfunc (r *RDB) CancelationPubSub() (*redis.PubSub, error) {\n\tvar op errors.Op = \"rdb.CancelationPubSub\"\n\tctx := context.Background()\n\tpubsub := r.client.Subscribe(ctx, base.CancelChannel)\n\t_, err := pubsub.Receive(ctx)\n\tif err != nil {\n\t\treturn nil, errors.E(op, errors.Unknown, fmt.Sprintf(\"redis pubsub receive error: %v\", err))\n\t}\n\treturn pubsub, nil\n}\n\n// PublishCancelation publish cancelation message to all subscribers.\n// The message is the ID for the task to be canceled.\nfunc (r *RDB) PublishCancelation(id string) error {\n\tvar op errors.Op = \"rdb.PublishCancelation\"\n\tctx := context.Background()\n\tif err := r.client.Publish(ctx, base.CancelChannel, id).Err(); err != nil {\n\t\treturn errors.E(op, errors.Unknown, fmt.Sprintf(\"redis pubsub publish error: %v\", err))\n\t}\n\treturn nil\n}\n\n// KEYS[1] -> asynq:scheduler_history:\n// ARGV[1] -> enqueued_at timestamp\n// ARGV[2] -> serialized SchedulerEnqueueEvent data\n// ARGV[3] -> max number of events to be persisted\nvar recordSchedulerEnqueueEventCmd = redis.NewScript(`\nredis.call(\"ZREMRANGEBYRANK\", KEYS[1], 0, -ARGV[3])\nredis.call(\"ZADD\", KEYS[1], ARGV[1], ARGV[2])\nreturn redis.status_reply(\"OK\")`)\n\n// Maximum number of enqueue events to store per entry.\nconst maxEvents = 1000\n\n// RecordSchedulerEnqueueEvent records the time when the given task was enqueued.\nfunc (r *RDB) RecordSchedulerEnqueueEvent(entryID string, event *base.SchedulerEnqueueEvent) error {\n\tvar op errors.Op = \"rdb.RecordSchedulerEnqueueEvent\"\n\tctx := context.Background()\n\tdata, err := base.EncodeSchedulerEnqueueEvent(event)\n\tif err != nil {\n\t\treturn errors.E(op, errors.Internal, fmt.Sprintf(\"cannot encode scheduler enqueue event: %v\", err))\n\t}\n\tkeys := []string{\n\t\tbase.SchedulerHistoryKey(entryID),\n\t}\n\targv := []interface{}{\n\t\tevent.EnqueuedAt.Unix(),\n\t\tdata,\n\t\tmaxEvents,\n\t}\n\treturn r.runScript(ctx, op, recordSchedulerEnqueueEventCmd, keys, argv...)\n}\n\n// ClearSchedulerHistory deletes the enqueue event history for the given scheduler entry.\nfunc (r *RDB) ClearSchedulerHistory(entryID string) error {\n\tvar op errors.Op = \"rdb.ClearSchedulerHistory\"\n\tctx := context.Background()\n\tkey := base.SchedulerHistoryKey(entryID)\n\tif err := r.client.Del(ctx, key).Err(); err != nil {\n\t\treturn errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: \"del\", Err: err})\n\t}\n\treturn nil\n}\n\n// WriteResult writes the given result data for the specified task.\nfunc (r *RDB) WriteResult(qname, taskID string, data []byte) (int, error) {\n\tvar op errors.Op = \"rdb.WriteResult\"\n\tctx := context.Background()\n\ttaskKey := base.TaskKey(qname, taskID)\n\tif err := r.client.HSet(ctx, taskKey, \"result\", data).Err(); err != nil {\n\t\treturn 0, errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: \"hset\", Err: err})\n\t}\n\treturn len(data), nil\n}\n" }, { "path": "internal/rdb/rdb_test.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage rdb\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"flag\"\n\t\"math\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/google/go-cmp/cmp\"\n\t\"github.com/google/go-cmp/cmp/cmpopts\"\n\t\"github.com/google/uuid\"\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/errors\"\n\th \"github.com/hibiken/asynq/internal/testutil\"\n\t\"github.com/hibiken/asynq/internal/timeutil\"\n\t\"github.com/redis/go-redis/v9\"\n)\n\n// variables used for package testing.\nvar (\n\tredisAddr string\n\tredisDB int\n\n\tuseRedisCluster bool\n\tredisClusterAddrs string // comma-separated list of host:port\n)\n\nfunc init() {\n\tflag.StringVar(&redisAddr, \"redis_addr\", \"localhost:6379\", \"redis address to use in testing\")\n\tflag.IntVar(&redisDB, \"redis_db\", 15, \"redis db number to use in testing\")\n\tflag.BoolVar(&useRedisCluster, \"redis_cluster\", false, \"use redis cluster as a broker in testing\")\n\tflag.StringVar(&redisClusterAddrs, \"redis_cluster_addrs\", \"localhost:7000,localhost:7001,localhost:7002\", \"comma separated list of redis server addresses\")\n}\n\nfunc setup(tb testing.TB) (r *RDB) {\n\ttb.Helper()\n\tif useRedisCluster {\n\t\taddrs := strings.Split(redisClusterAddrs, \",\")\n\t\tif len(addrs) == 0 {\n\t\t\ttb.Fatal(\"No redis cluster addresses provided. Please set addresses using --redis_cluster_addrs flag.\")\n\t\t}\n\t\tr = NewRDB(redis.NewClusterClient(&redis.ClusterOptions{\n\t\t\tAddrs: addrs,\n\t\t}))\n\t} else {\n\t\tr = NewRDB(redis.NewClient(&redis.Options{\n\t\t\tAddr: redisAddr,\n\t\t\tDB: redisDB,\n\t\t}))\n\t}\n\t// Start each test with a clean slate.\n\th.FlushDB(tb, r.client)\n\treturn r\n}\n\nfunc TestEnqueue(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tt1 := h.NewTaskMessage(\"send_email\", h.JSON(map[string]interface{}{\"to\": \"exampleuser@gmail.com\", \"from\": \"noreply@example.com\"}))\n\tt2 := h.NewTaskMessageWithQueue(\"generate_csv\", h.JSON(map[string]interface{}{}), \"csv\")\n\tt3 := h.NewTaskMessageWithQueue(\"sync\", nil, \"low\")\n\n\tenqueueTime := time.Now()\n\tr.SetClock(timeutil.NewSimulatedClock(enqueueTime))\n\n\ttests := []struct {\n\t\tmsg *base.TaskMessage\n\t}{\n\t\t{t1},\n\t\t{t2},\n\t\t{t3},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case.\n\n\t\terr := r.Enqueue(context.Background(), tc.msg)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"(*RDB).Enqueue(msg) = %v, want nil\", err)\n\t\t\tcontinue\n\t\t}\n\n\t\t// Check Pending list has task ID.\n\t\tpendingKey := base.PendingKey(tc.msg.Queue)\n\t\tpendingIDs := r.client.LRange(context.Background(), pendingKey, 0, -1).Val()\n\t\tif n := len(pendingIDs); n != 1 {\n\t\t\tt.Errorf(\"Redis LIST %q contains %d IDs, want 1\", pendingKey, n)\n\t\t\tcontinue\n\t\t}\n\t\tif pendingIDs[0] != tc.msg.ID {\n\t\t\tt.Errorf(\"Redis LIST %q: got %v, want %v\", pendingKey, pendingIDs, []string{tc.msg.ID})\n\t\t\tcontinue\n\t\t}\n\n\t\t// Check the value under the task key.\n\t\ttaskKey := base.TaskKey(tc.msg.Queue, tc.msg.ID)\n\t\tencoded := r.client.HGet(context.Background(), taskKey, \"msg\").Val() // \"msg\" field\n\t\tdecoded := h.MustUnmarshal(t, encoded)\n\t\tif diff := cmp.Diff(tc.msg, decoded); diff != \"\" {\n\t\t\tt.Errorf(\"persisted message was %v, want %v; (-want, +got)\\n%s\", decoded, tc.msg, diff)\n\t\t}\n\t\tstate := r.client.HGet(context.Background(), taskKey, \"state\").Val() // \"state\" field\n\t\tif state != \"pending\" {\n\t\t\tt.Errorf(\"state field under task-key is set to %q, want %q\", state, \"pending\")\n\t\t}\n\t\tpendingSince := r.client.HGet(context.Background(), taskKey, \"pending_since\").Val() // \"pending_since\" field\n\t\tif want := strconv.Itoa(int(enqueueTime.UnixNano())); pendingSince != want {\n\t\t\tt.Errorf(\"pending_since field under task-key is set to %v, want %v\", pendingSince, want)\n\t\t}\n\n\t\t// Check queue is in the AllQueues set.\n\t\tif !r.client.SIsMember(context.Background(), base.AllQueues, tc.msg.Queue).Val() {\n\t\t\tt.Errorf(\"%q is not a member of SET %q\", tc.msg.Queue, base.AllQueues)\n\t\t}\n\t}\n}\n\nfunc TestEnqueueTaskIdConflictError(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tm1 := base.TaskMessage{\n\t\tID: \"custom_id\",\n\t\tType: \"foo\",\n\t\tPayload: nil,\n\t}\n\tm2 := base.TaskMessage{\n\t\tID: \"custom_id\",\n\t\tType: \"bar\",\n\t\tPayload: nil,\n\t}\n\n\ttests := []struct {\n\t\tfirstMsg *base.TaskMessage\n\t\tsecondMsg *base.TaskMessage\n\t}{\n\t\t{firstMsg: &m1, secondMsg: &m2},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case.\n\n\t\tif err := r.Enqueue(context.Background(), tc.firstMsg); err != nil {\n\t\t\tt.Errorf(\"First message: Enqueue failed: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tif err := r.Enqueue(context.Background(), tc.secondMsg); !errors.Is(err, errors.ErrTaskIdConflict) {\n\t\t\tt.Errorf(\"Second message: Enqueue returned %v, want %v\", err, errors.ErrTaskIdConflict)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\nfunc TestEnqueueQueueCache(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tt1 := h.NewTaskMessageWithQueue(\"sync1\", nil, \"low\")\n\n\tenqueueTime := time.Now()\n\tclock := timeutil.NewSimulatedClock(enqueueTime)\n\tr.SetClock(clock)\n\n\terr := r.Enqueue(context.Background(), t1)\n\tif err != nil {\n\t\tt.Fatalf(\"(*RDB).Enqueue(msg) = %v, want nil\", err)\n\t}\n\n\t// Check queue is in the AllQueues set.\n\tif !r.client.SIsMember(context.Background(), base.AllQueues, t1.Queue).Val() {\n\t\tt.Fatalf(\"%q is not a member of SET %q\", t1.Queue, base.AllQueues)\n\t}\n\n\tif _, ok := r.queuesPublished.Load(t1.Queue); !ok {\n\t\tt.Fatalf(\"%q is not cached in queuesPublished\", t1.Queue)\n\t}\n\n\tt.Run(\"remove-queue\", func(t *testing.T) {\n\t\terr := r.RemoveQueue(t1.Queue, true)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"(*RDB).RemoveQueue(%q, %t) = %v, want nil\", t1.Queue, true, err)\n\t\t}\n\n\t\tif _, ok := r.queuesPublished.Load(t1.Queue); ok {\n\t\t\tt.Fatalf(\"%q is still cached in queuesPublished\", t1.Queue)\n\t\t}\n\n\t\tif r.client.SIsMember(context.Background(), base.AllQueues, t1.Queue).Val() {\n\t\t\tt.Fatalf(\"%q is a member of SET %q\", t1.Queue, base.AllQueues)\n\t\t}\n\n\t\terr = r.Enqueue(context.Background(), t1)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"(*RDB).Enqueue(msg) = %v, want nil\", err)\n\t\t}\n\n\t\t// Check queue is in the AllQueues set.\n\t\tif !r.client.SIsMember(context.Background(), base.AllQueues, t1.Queue).Val() {\n\t\t\tt.Fatalf(\"%q is not a member of SET %q\", t1.Queue, base.AllQueues)\n\t\t}\n\n\t\tif _, ok := r.queuesPublished.Load(t1.Queue); !ok {\n\t\t\tt.Fatalf(\"%q is not cached in queuesPublished\", t1.Queue)\n\t\t}\n\t})\n}\n\nfunc TestEnqueueUnique(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"email\",\n\t\tPayload: h.JSON(map[string]interface{}{\"user_id\": json.Number(\"123\")}),\n\t\tQueue: base.DefaultQueueName,\n\t\tUniqueKey: base.UniqueKey(base.DefaultQueueName, \"email\", h.JSON(map[string]interface{}{\"user_id\": 123})),\n\t}\n\n\tenqueueTime := time.Now()\n\tr.SetClock(timeutil.NewSimulatedClock(enqueueTime))\n\n\ttests := []struct {\n\t\tmsg *base.TaskMessage\n\t\tttl time.Duration // uniqueness ttl\n\t}{\n\t\t{&m1, time.Minute},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case.\n\n\t\t// Enqueue the first message, should succeed.\n\t\terr := r.EnqueueUnique(context.Background(), tc.msg, tc.ttl)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"First message: (*RDB).EnqueueUnique(%v, %v) = %v, want nil\",\n\t\t\t\ttc.msg, tc.ttl, err)\n\t\t\tcontinue\n\t\t}\n\t\tgotPending := h.GetPendingMessages(t, r.client, tc.msg.Queue)\n\t\tif len(gotPending) != 1 {\n\t\t\tt.Errorf(\"%q has length %d, want 1\", base.PendingKey(tc.msg.Queue), len(gotPending))\n\t\t\tcontinue\n\t\t}\n\t\tif diff := cmp.Diff(tc.msg, gotPending[0]); diff != \"\" {\n\t\t\tt.Errorf(\"persisted data differed from the original input (-want, +got)\\n%s\", diff)\n\t\t}\n\t\tif !r.client.SIsMember(context.Background(), base.AllQueues, tc.msg.Queue).Val() {\n\t\t\tt.Errorf(\"%q is not a member of SET %q\", tc.msg.Queue, base.AllQueues)\n\t\t}\n\n\t\t// Check Pending list has task ID.\n\t\tpendingKey := base.PendingKey(tc.msg.Queue)\n\t\tpendingIDs := r.client.LRange(context.Background(), pendingKey, 0, -1).Val()\n\t\tif len(pendingIDs) != 1 {\n\t\t\tt.Errorf(\"Redis LIST %q contains %d IDs, want 1\", pendingKey, len(pendingIDs))\n\t\t\tcontinue\n\t\t}\n\t\tif pendingIDs[0] != tc.msg.ID {\n\t\t\tt.Errorf(\"Redis LIST %q: got %v, want %v\", pendingKey, pendingIDs, []string{tc.msg.ID})\n\t\t\tcontinue\n\t\t}\n\n\t\t// Check the value under the task key.\n\t\ttaskKey := base.TaskKey(tc.msg.Queue, tc.msg.ID)\n\t\tencoded := r.client.HGet(context.Background(), taskKey, \"msg\").Val() // \"msg\" field\n\t\tdecoded := h.MustUnmarshal(t, encoded)\n\t\tif diff := cmp.Diff(tc.msg, decoded); diff != \"\" {\n\t\t\tt.Errorf(\"persisted message was %v, want %v; (-want, +got)\\n%s\", decoded, tc.msg, diff)\n\t\t}\n\t\tstate := r.client.HGet(context.Background(), taskKey, \"state\").Val() // \"state\" field\n\t\tif state != \"pending\" {\n\t\t\tt.Errorf(\"state field under task-key is set to %q, want %q\", state, \"pending\")\n\t\t}\n\t\tpendingSince := r.client.HGet(context.Background(), taskKey, \"pending_since\").Val() // \"pending_since\" field\n\t\tif want := strconv.Itoa(int(enqueueTime.UnixNano())); pendingSince != want {\n\t\t\tt.Errorf(\"pending_since field under task-key is set to %v, want %v\", pendingSince, want)\n\t\t}\n\t\tuniqueKey := r.client.HGet(context.Background(), taskKey, \"unique_key\").Val() // \"unique_key\" field\n\t\tif uniqueKey != tc.msg.UniqueKey {\n\t\t\tt.Errorf(\"uniqueue_key field under task key is set to %q, want %q\", uniqueKey, tc.msg.UniqueKey)\n\t\t}\n\n\t\t// Check queue is in the AllQueues set.\n\t\tif !r.client.SIsMember(context.Background(), base.AllQueues, tc.msg.Queue).Val() {\n\t\t\tt.Errorf(\"%q is not a member of SET %q\", tc.msg.Queue, base.AllQueues)\n\t\t}\n\n\t\t// Enqueue the second message, should fail.\n\t\tgot := r.EnqueueUnique(context.Background(), tc.msg, tc.ttl)\n\t\tif !errors.Is(got, errors.ErrDuplicateTask) {\n\t\t\tt.Errorf(\"Second message: (*RDB).EnqueueUnique(msg, ttl) = %v, want %v\", got, errors.ErrDuplicateTask)\n\t\t\tcontinue\n\t\t}\n\t\tgotTTL := r.client.TTL(context.Background(), tc.msg.UniqueKey).Val()\n\t\tif !cmp.Equal(tc.ttl.Seconds(), gotTTL.Seconds(), cmpopts.EquateApprox(0, 2)) {\n\t\t\tt.Errorf(\"TTL %q = %v, want %v\", tc.msg.UniqueKey, gotTTL, tc.ttl)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\nfunc TestEnqueueUniqueTaskIdConflictError(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tm1 := base.TaskMessage{\n\t\tID: \"custom_id\",\n\t\tType: \"foo\",\n\t\tPayload: nil,\n\t\tUniqueKey: \"unique_key_one\",\n\t}\n\tm2 := base.TaskMessage{\n\t\tID: \"custom_id\",\n\t\tType: \"bar\",\n\t\tPayload: nil,\n\t\tUniqueKey: \"unique_key_two\",\n\t}\n\tconst ttl = 30 * time.Second\n\n\ttests := []struct {\n\t\tfirstMsg *base.TaskMessage\n\t\tsecondMsg *base.TaskMessage\n\t}{\n\t\t{firstMsg: &m1, secondMsg: &m2},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case.\n\n\t\tif err := r.EnqueueUnique(context.Background(), tc.firstMsg, ttl); err != nil {\n\t\t\tt.Errorf(\"First message: EnqueueUnique failed: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tif err := r.EnqueueUnique(context.Background(), tc.secondMsg, ttl); !errors.Is(err, errors.ErrTaskIdConflict) {\n\t\t\tt.Errorf(\"Second message: EnqueueUnique returned %v, want %v\", err, errors.ErrTaskIdConflict)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\nfunc TestDequeue(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tnow := time.Now()\n\tr.SetClock(timeutil.NewSimulatedClock(now))\n\tt1 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"send_email\",\n\t\tPayload: h.JSON(map[string]interface{}{\"subject\": \"hello!\"}),\n\t\tQueue: \"default\",\n\t\tTimeout: 1800,\n\t\tDeadline: 0,\n\t}\n\tt2 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"export_csv\",\n\t\tPayload: nil,\n\t\tQueue: \"critical\",\n\t\tTimeout: 0,\n\t\tDeadline: 1593021600,\n\t}\n\tt3 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"reindex\",\n\t\tPayload: nil,\n\t\tQueue: \"low\",\n\t\tTimeout: int64((5 * time.Minute).Seconds()),\n\t\tDeadline: time.Now().Add(10 * time.Minute).Unix(),\n\t}\n\n\ttests := []struct {\n\t\tpending map[string][]*base.TaskMessage\n\t\tqnames []string // list of queues to query\n\t\twantMsg *base.TaskMessage\n\t\twantExpirationTime time.Time\n\t\twantPending map[string][]*base.TaskMessage\n\t\twantActive map[string][]*base.TaskMessage\n\t\twantLease map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t},\n\t\t\tqnames: []string{\"default\"},\n\t\t\twantMsg: t1,\n\t\t\twantExpirationTime: now.Add(LeaseDuration),\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t},\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t1, Score: now.Add(LeaseDuration).Unix()}},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t\t\"critical\": {t2},\n\t\t\t\t\"low\": {t3},\n\t\t\t},\n\t\t\tqnames: []string{\"critical\", \"default\", \"low\"},\n\t\t\twantMsg: t2,\n\t\t\twantExpirationTime: now.Add(LeaseDuration),\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {t3},\n\t\t\t},\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {t2},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {{Message: t2, Score: now.Add(LeaseDuration).Unix()}},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {t3},\n\t\t\t},\n\t\t\tqnames: []string{\"critical\", \"default\", \"low\"},\n\t\t\twantMsg: t1,\n\t\t\twantExpirationTime: now.Add(LeaseDuration),\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {t3},\n\t\t\t},\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t1, Score: now.Add(LeaseDuration).Unix()}},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllPendingQueues(t, r.client, tc.pending)\n\n\t\tgotMsg, gotExpirationTime, err := r.Dequeue(tc.qnames...)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"(*RDB).Dequeue(%v) returned error %v\", tc.qnames, err)\n\t\t\tcontinue\n\t\t}\n\t\tif !cmp.Equal(gotMsg, tc.wantMsg) {\n\t\t\tt.Errorf(\"(*RDB).Dequeue(%v) returned message %v; want %v\",\n\t\t\t\ttc.qnames, gotMsg, tc.wantMsg)\n\t\t\tcontinue\n\t\t}\n\t\tif gotExpirationTime != tc.wantExpirationTime {\n\t\t\tt.Errorf(\"(*RDB).Dequeue(%v) returned expiration time %v, want %v\",\n\t\t\t\ttc.qnames, gotExpirationTime, tc.wantExpirationTime)\n\t\t}\n\n\t\tfor queue, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r.client, queue)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q: (-want,+got):\\n%s\", base.PendingKey(queue), diff)\n\t\t\t}\n\t\t}\n\t\tfor queue, want := range tc.wantActive {\n\t\t\tgotActive := h.GetActiveMessages(t, r.client, queue)\n\t\t\tif diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q: (-want,+got):\\n%s\", base.ActiveKey(queue), diff)\n\t\t\t}\n\t\t}\n\t\tfor queue, want := range tc.wantLease {\n\t\t\tgotLease := h.GetLeaseEntries(t, r.client, queue)\n\t\t\tif diff := cmp.Diff(want, gotLease, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q: (-want,+got):\\n%s\", base.LeaseKey(queue), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestDequeueError(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\ttests := []struct {\n\t\tpending map[string][]*base.TaskMessage\n\t\tqnames []string // list of queues to query\n\t\twantErr error\n\t\twantPending map[string][]*base.TaskMessage\n\t\twantActive map[string][]*base.TaskMessage\n\t\twantLease map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tqnames: []string{\"default\"},\n\t\t\twantErr: errors.ErrNoProcessableTask,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t\tqnames: []string{\"critical\", \"default\", \"low\"},\n\t\t\twantErr: errors.ErrNoProcessableTask,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllPendingQueues(t, r.client, tc.pending)\n\n\t\tgotMsg, _, gotErr := r.Dequeue(tc.qnames...)\n\t\tif !errors.Is(gotErr, tc.wantErr) {\n\t\t\tt.Errorf(\"(*RDB).Dequeue(%v) returned error %v; want %v\",\n\t\t\t\ttc.qnames, gotErr, tc.wantErr)\n\t\t\tcontinue\n\t\t}\n\t\tif gotMsg != nil {\n\t\t\tt.Errorf(\"(*RDB).Dequeue(%v) returned message %v; want nil\", tc.qnames, gotMsg)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor queue, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r.client, queue)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q: (-want,+got):\\n%s\", base.PendingKey(queue), diff)\n\t\t\t}\n\t\t}\n\t\tfor queue, want := range tc.wantActive {\n\t\t\tgotActive := h.GetActiveMessages(t, r.client, queue)\n\t\t\tif diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q: (-want,+got):\\n%s\", base.ActiveKey(queue), diff)\n\t\t\t}\n\t\t}\n\t\tfor queue, want := range tc.wantLease {\n\t\t\tgotLease := h.GetLeaseEntries(t, r.client, queue)\n\t\t\tif diff := cmp.Diff(want, gotLease, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q: (-want,+got):\\n%s\", base.LeaseKey(queue), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestDequeueIgnoresPausedQueues(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tt1 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"send_email\",\n\t\tPayload: h.JSON(map[string]interface{}{\"subject\": \"hello!\"}),\n\t\tQueue: \"default\",\n\t\tTimeout: 1800,\n\t\tDeadline: 0,\n\t}\n\tt2 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"export_csv\",\n\t\tPayload: nil,\n\t\tQueue: \"critical\",\n\t\tTimeout: 1800,\n\t\tDeadline: 0,\n\t}\n\n\ttests := []struct {\n\t\tpaused []string // list of paused queues\n\t\tpending map[string][]*base.TaskMessage\n\t\tqnames []string // list of queues to query\n\t\twantMsg *base.TaskMessage\n\t\twantErr error\n\t\twantPending map[string][]*base.TaskMessage\n\t\twantActive map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tpaused: []string{\"default\"},\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t\t\"critical\": {t2},\n\t\t\t},\n\t\t\tqnames: []string{\"default\", \"critical\"},\n\t\t\twantMsg: t2,\n\t\t\twantErr: nil,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {t2},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tpaused: []string{\"default\"},\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t},\n\t\t\tqnames: []string{\"default\"},\n\t\t\twantMsg: nil,\n\t\t\twantErr: errors.ErrNoProcessableTask,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t},\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tpaused: []string{\"critical\", \"default\"},\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t\t\"critical\": {t2},\n\t\t\t},\n\t\t\tqnames: []string{\"default\", \"critical\"},\n\t\t\twantMsg: nil,\n\t\t\twantErr: errors.ErrNoProcessableTask,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t\t\"critical\": {t2},\n\t\t\t},\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\tfor _, qname := range tc.paused {\n\t\t\tif err := r.Pause(qname); err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t}\n\t\th.SeedAllPendingQueues(t, r.client, tc.pending)\n\n\t\tgot, _, err := r.Dequeue(tc.qnames...)\n\t\tif !cmp.Equal(got, tc.wantMsg) || !errors.Is(err, tc.wantErr) {\n\t\t\tt.Errorf(\"Dequeue(%v) = %v, %v; want %v, %v\",\n\t\t\t\ttc.qnames, got, err, tc.wantMsg, tc.wantErr)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor queue, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r.client, queue)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q: (-want,+got):\\n%s\", base.PendingKey(queue), diff)\n\t\t\t}\n\t\t}\n\t\tfor queue, want := range tc.wantActive {\n\t\t\tgotActive := h.GetActiveMessages(t, r.client, queue)\n\t\t\tif diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q: (-want,+got):\\n%s\", base.ActiveKey(queue), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestDone(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tnow := time.Now()\n\tt1 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"send_email\",\n\t\tPayload: nil,\n\t\tTimeout: 1800,\n\t\tDeadline: 0,\n\t\tQueue: \"default\",\n\t}\n\tt2 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"export_csv\",\n\t\tPayload: nil,\n\t\tTimeout: 0,\n\t\tDeadline: 1592485787,\n\t\tQueue: \"custom\",\n\t}\n\tt3 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"reindex\",\n\t\tPayload: nil,\n\t\tTimeout: 1800,\n\t\tDeadline: 0,\n\t\tUniqueKey: \"asynq:{default}:unique:b0804ec967f48520697662a204f5fe72\",\n\t\tQueue: \"default\",\n\t}\n\n\ttests := []struct {\n\t\tdesc string\n\t\tactive map[string][]*base.TaskMessage // initial state of the active list\n\t\tlease map[string][]base.Z // initial state of the lease set\n\t\ttarget *base.TaskMessage // task to remove\n\t\twantActive map[string][]*base.TaskMessage // final state of the active list\n\t\twantLease map[string][]base.Z // final state of the lease set\n\t}{\n\t\t{\n\t\t\tdesc: \"removes message from the correct queue\",\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t\t\"custom\": {t2},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t\t\"custom\": {{Message: t2, Score: now.Add(20 * time.Second).Unix()}},\n\t\t\t},\n\t\t\ttarget: t1,\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {t2},\n\t\t\t},\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {{Message: t2, Score: now.Add(20 * time.Second).Unix()}},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with one queue\",\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t},\n\t\t\ttarget: t1,\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with multiple messages in a queue\",\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t3},\n\t\t\t\t\"custom\": {t2},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t1, Score: now.Add(15 * time.Second).Unix()}, {Message: t3, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t\t\"custom\": {{Message: t2, Score: now.Add(20 * time.Second).Unix()}},\n\t\t\t},\n\t\t\ttarget: t3,\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t\t\"custom\": {t2},\n\t\t\t},\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t1, Score: now.Add(15 * time.Second).Unix()}},\n\t\t\t\t\"custom\": {{Message: t2, Score: now.Add(20 * time.Second).Unix()}},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllLease(t, r.client, tc.lease)\n\t\th.SeedAllActiveQueues(t, r.client, tc.active)\n\t\tfor _, msgs := range tc.active {\n\t\t\tfor _, msg := range msgs {\n\t\t\t\t// Set uniqueness lock if unique key is present.\n\t\t\t\tif len(msg.UniqueKey) > 0 {\n\t\t\t\t\terr := r.client.SetNX(context.Background(), msg.UniqueKey, msg.ID, time.Minute).Err()\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tt.Fatal(err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\terr := r.Done(context.Background(), tc.target)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s; (*RDB).Done(task) = %v, want nil\", tc.desc, err)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor queue, want := range tc.wantActive {\n\t\t\tgotActive := h.GetActiveMessages(t, r.client, queue)\n\t\t\tif diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s; mismatch found in %q: (-want, +got):\\n%s\", tc.desc, base.ActiveKey(queue), diff)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\tfor queue, want := range tc.wantLease {\n\t\t\tgotLease := h.GetLeaseEntries(t, r.client, queue)\n\t\t\tif diff := cmp.Diff(want, gotLease, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s; mismatch found in %q: (-want, +got):\\n%s\", tc.desc, base.LeaseKey(queue), diff)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\tprocessedKey := base.ProcessedKey(tc.target.Queue, time.Now())\n\t\tgotProcessed := r.client.Get(context.Background(), processedKey).Val()\n\t\tif gotProcessed != \"1\" {\n\t\t\tt.Errorf(\"%s; GET %q = %q, want 1\", tc.desc, processedKey, gotProcessed)\n\t\t}\n\t\tgotTTL := r.client.TTL(context.Background(), processedKey).Val()\n\t\tif gotTTL > statsTTL {\n\t\t\tt.Errorf(\"%s; TTL %q = %v, want less than or equal to %v\", tc.desc, processedKey, gotTTL, statsTTL)\n\t\t}\n\n\t\tprocessedTotalKey := base.ProcessedTotalKey(tc.target.Queue)\n\t\tgotProcessedTotal := r.client.Get(context.Background(), processedTotalKey).Val()\n\t\tif gotProcessedTotal != \"1\" {\n\t\t\tt.Errorf(\"%s; GET %q = %q, want 1\", tc.desc, processedTotalKey, gotProcessedTotal)\n\t\t}\n\n\t\tif len(tc.target.UniqueKey) > 0 && r.client.Exists(context.Background(), tc.target.UniqueKey).Val() != 0 {\n\t\t\tt.Errorf(\"%s; Uniqueness lock %q still exists\", tc.desc, tc.target.UniqueKey)\n\t\t}\n\t}\n}\n\n// Make sure that processed_total counter wraps to 1 when reaching int64 max value.\nfunc TestDoneWithMaxCounter(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tmsg := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"foo\",\n\t\tPayload: nil,\n\t\tTimeout: 1800,\n\t\tDeadline: 0,\n\t\tQueue: \"default\",\n\t}\n\n\tz := base.Z{\n\t\tMessage: msg,\n\t\tScore: time.Now().Add(15 * time.Second).Unix(),\n\t}\n\th.SeedLease(t, r.client, []base.Z{z}, msg.Queue)\n\th.SeedActiveQueue(t, r.client, []*base.TaskMessage{msg}, msg.Queue)\n\n\tprocessedTotalKey := base.ProcessedTotalKey(msg.Queue)\n\tctx := context.Background()\n\tif err := r.client.Set(ctx, processedTotalKey, math.MaxInt64, 0).Err(); err != nil {\n\t\tt.Fatalf(\"Redis command failed: SET %q %v\", processedTotalKey, math.MaxInt64)\n\t}\n\n\tif err := r.Done(context.Background(), msg); err != nil {\n\t\tt.Fatalf(\"RDB.Done failed: %v\", err)\n\t}\n\n\tgotProcessedTotal := r.client.Get(ctx, processedTotalKey).Val()\n\tif gotProcessedTotal != \"1\" {\n\t\tt.Errorf(\"GET %q = %v, want 1\", processedTotalKey, gotProcessedTotal)\n\t}\n}\n\nfunc TestMarkAsComplete(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tnow := time.Now()\n\tr.SetClock(timeutil.NewSimulatedClock(now))\n\tt1 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"send_email\",\n\t\tPayload: nil,\n\t\tTimeout: 1800,\n\t\tDeadline: 0,\n\t\tQueue: \"default\",\n\t\tRetention: 3600,\n\t}\n\tt2 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"export_csv\",\n\t\tPayload: nil,\n\t\tTimeout: 0,\n\t\tDeadline: now.Add(2 * time.Hour).Unix(),\n\t\tQueue: \"custom\",\n\t\tRetention: 7200,\n\t}\n\tt3 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"reindex\",\n\t\tPayload: nil,\n\t\tTimeout: 1800,\n\t\tDeadline: 0,\n\t\tUniqueKey: \"asynq:{default}:unique:b0804ec967f48520697662a204f5fe72\",\n\t\tQueue: \"default\",\n\t\tRetention: 1800,\n\t}\n\n\ttests := []struct {\n\t\tdesc string\n\t\tactive map[string][]*base.TaskMessage // initial state of the active list\n\t\tlease map[string][]base.Z // initial state of the lease set\n\t\tcompleted map[string][]base.Z // initial state of the completed set\n\t\ttarget *base.TaskMessage // task to mark as completed\n\t\twantActive map[string][]*base.TaskMessage // final state of the active list\n\t\twantLease map[string][]base.Z // final state of the lease set\n\t\twantCompleted map[string][]base.Z // final state of the completed set\n\t}{\n\t\t{\n\t\t\tdesc: \"select a message from the correct queue\",\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t\t\"custom\": {t2},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t1, Score: now.Add(30 * time.Second).Unix()}},\n\t\t\t\t\"custom\": {{Message: t2, Score: now.Add(20 * time.Second).Unix()}},\n\t\t\t},\n\t\t\tcompleted: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\ttarget: t1,\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {t2},\n\t\t\t},\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {{Message: t2, Score: now.Add(20 * time.Second).Unix()}},\n\t\t\t},\n\t\t\twantCompleted: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: h.TaskMessageWithCompletedAt(*t1, now), Score: now.Unix() + t1.Retention}},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with one queue\",\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t},\n\t\t\tcompleted: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\ttarget: t1,\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantCompleted: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: h.TaskMessageWithCompletedAt(*t1, now), Score: now.Unix() + t1.Retention}},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with multiple messages in a queue\",\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t3},\n\t\t\t\t\"custom\": {t2},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}, {Message: t3, Score: now.Add(12 * time.Second).Unix()}},\n\t\t\t\t\"custom\": {{Message: t2, Score: now.Add(12 * time.Second).Unix()}},\n\t\t\t},\n\t\t\tcompleted: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\ttarget: t3,\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t\t\"custom\": {t2},\n\t\t\t},\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t\t\"custom\": {{Message: t2, Score: now.Add(12 * time.Second).Unix()}},\n\t\t\t},\n\t\t\twantCompleted: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: h.TaskMessageWithCompletedAt(*t3, now), Score: now.Unix() + t3.Retention}},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllLease(t, r.client, tc.lease)\n\t\th.SeedAllActiveQueues(t, r.client, tc.active)\n\t\th.SeedAllCompletedQueues(t, r.client, tc.completed)\n\t\tfor _, msgs := range tc.active {\n\t\t\tfor _, msg := range msgs {\n\t\t\t\t// Set uniqueness lock if unique key is present.\n\t\t\t\tif len(msg.UniqueKey) > 0 {\n\t\t\t\t\terr := r.client.SetNX(context.Background(), msg.UniqueKey, msg.ID, time.Minute).Err()\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tt.Fatal(err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\terr := r.MarkAsComplete(context.Background(), tc.target)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s; (*RDB).MarkAsCompleted(task) = %v, want nil\", tc.desc, err)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor queue, want := range tc.wantActive {\n\t\t\tgotActive := h.GetActiveMessages(t, r.client, queue)\n\t\t\tif diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s; mismatch found in %q: (-want, +got):\\n%s\", tc.desc, base.ActiveKey(queue), diff)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\tfor queue, want := range tc.wantLease {\n\t\t\tgotLease := h.GetLeaseEntries(t, r.client, queue)\n\t\t\tif diff := cmp.Diff(want, gotLease, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s; mismatch found in %q: (-want, +got):\\n%s\", tc.desc, base.LeaseKey(queue), diff)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\tfor queue, want := range tc.wantCompleted {\n\t\t\tgotCompleted := h.GetCompletedEntries(t, r.client, queue)\n\t\t\tif diff := cmp.Diff(want, gotCompleted, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s; mismatch found in %q: (-want, +got):\\n%s\", tc.desc, base.CompletedKey(queue), diff)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\tprocessedKey := base.ProcessedKey(tc.target.Queue, time.Now())\n\t\tgotProcessed := r.client.Get(context.Background(), processedKey).Val()\n\t\tif gotProcessed != \"1\" {\n\t\t\tt.Errorf(\"%s; GET %q = %q, want 1\", tc.desc, processedKey, gotProcessed)\n\t\t}\n\n\t\tgotTTL := r.client.TTL(context.Background(), processedKey).Val()\n\t\tif gotTTL > statsTTL {\n\t\t\tt.Errorf(\"%s; TTL %q = %v, want less than or equal to %v\", tc.desc, processedKey, gotTTL, statsTTL)\n\t\t}\n\n\t\tif len(tc.target.UniqueKey) > 0 && r.client.Exists(context.Background(), tc.target.UniqueKey).Val() != 0 {\n\t\t\tt.Errorf(\"%s; Uniqueness lock %q still exists\", tc.desc, tc.target.UniqueKey)\n\t\t}\n\t}\n}\n\nfunc TestRequeue(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tnow := time.Now()\n\tt1 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"send_email\",\n\t\tPayload: nil,\n\t\tQueue: \"default\",\n\t\tTimeout: 1800,\n\t}\n\tt2 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"export_csv\",\n\t\tPayload: nil,\n\t\tQueue: \"default\",\n\t\tTimeout: 3000,\n\t}\n\tt3 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"send_email\",\n\t\tPayload: nil,\n\t\tQueue: \"critical\",\n\t\tTimeout: 80,\n\t}\n\n\ttests := []struct {\n\t\tpending map[string][]*base.TaskMessage // initial state of queues\n\t\tactive map[string][]*base.TaskMessage // initial state of the active list\n\t\tlease map[string][]base.Z // initial state of the lease set\n\t\ttarget *base.TaskMessage // task to requeue\n\t\twantPending map[string][]*base.TaskMessage // final state of queues\n\t\twantActive map[string][]*base.TaskMessage // final state of the active list\n\t\twantLease map[string][]base.Z // final state of the lease set\n\t}{\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t2},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t\t{Message: t2, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\ttarget: t1,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t},\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t2},\n\t\t\t},\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t2, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t},\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t2},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t2, Score: now.Add(20 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\ttarget: t2,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t2},\n\t\t\t},\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t2},\n\t\t\t\t\"critical\": {t3},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t2, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t\t\"critical\": {{Message: t3, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t},\n\t\t\ttarget: t3,\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t\t\"critical\": {t3},\n\t\t\t},\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t2},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t2, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllPendingQueues(t, r.client, tc.pending)\n\t\th.SeedAllActiveQueues(t, r.client, tc.active)\n\t\th.SeedAllLease(t, r.client, tc.lease)\n\n\t\terr := r.Requeue(context.Background(), tc.target)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"(*RDB).Requeue(task) = %v, want nil\", err)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.PendingKey(qname), diff)\n\t\t\t}\n\t\t}\n\t\tfor qname, want := range tc.wantActive {\n\t\t\tgotActive := h.GetActiveMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q: (-want, +got):\\n%s\", base.ActiveKey(qname), diff)\n\t\t\t}\n\t\t}\n\t\tfor qname, want := range tc.wantLease {\n\t\t\tgotLease := h.GetLeaseEntries(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotLease, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q: (-want, +got):\\n%s\", base.LeaseKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestAddToGroup(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tnow := time.Now()\n\tr.SetClock(timeutil.NewSimulatedClock(now))\n\tmsg := h.NewTaskMessage(\"mytask\", []byte(\"foo\"))\n\tctx := context.Background()\n\n\ttests := []struct {\n\t\tmsg *base.TaskMessage\n\t\tgroupKey string\n\t}{\n\t\t{\n\t\t\tmsg: msg,\n\t\t\tgroupKey: \"mygroup\",\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\n\t\terr := r.AddToGroup(ctx, tc.msg, tc.groupKey)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"r.AddToGroup(ctx, msg, %q) returned error: %v\", tc.groupKey, err)\n\t\t\tcontinue\n\t\t}\n\n\t\t// Check Group zset has task ID\n\t\tgkey := base.GroupKey(tc.msg.Queue, tc.groupKey)\n\t\tzs := r.client.ZRangeWithScores(ctx, gkey, 0, -1).Val()\n\t\tif n := len(zs); n != 1 {\n\t\t\tt.Errorf(\"Redis ZSET %q contains %d elements, want 1\", gkey, n)\n\t\t\tcontinue\n\t\t}\n\t\tif got := zs[0].Member.(string); got != tc.msg.ID {\n\t\t\tt.Errorf(\"Redis ZSET %q member: got %v, want %v\", gkey, got, tc.msg.ID)\n\t\t\tcontinue\n\t\t}\n\t\tif got := int64(zs[0].Score); got != now.Unix() {\n\t\t\tt.Errorf(\"Redis ZSET %q score: got %d, want %d\", gkey, got, now.Unix())\n\t\t\tcontinue\n\t\t}\n\n\t\t// Check the values under the task key.\n\t\ttaskKey := base.TaskKey(tc.msg.Queue, tc.msg.ID)\n\t\tencoded := r.client.HGet(ctx, taskKey, \"msg\").Val() // \"msg\" field\n\t\tdecoded := h.MustUnmarshal(t, encoded)\n\t\tif diff := cmp.Diff(tc.msg, decoded); diff != \"\" {\n\t\t\tt.Errorf(\"persisted message was %v, want %v; (-want, +got)\\n%s\", decoded, tc.msg, diff)\n\t\t}\n\t\tstate := r.client.HGet(ctx, taskKey, \"state\").Val() // \"state\" field\n\t\tif want := \"aggregating\"; state != want {\n\t\t\tt.Errorf(\"state field under task-key is set to %q, want %q\", state, want)\n\t\t}\n\t\tgroup := r.client.HGet(ctx, taskKey, \"group\").Val() // \"group\" field\n\t\tif want := tc.groupKey; group != want {\n\t\t\tt.Errorf(\"group field under task-key is set to %q, want %q\", group, want)\n\t\t}\n\n\t\t// Check queue is in the AllQueues set.\n\t\tif !r.client.SIsMember(context.Background(), base.AllQueues, tc.msg.Queue).Val() {\n\t\t\tt.Errorf(\"%q is not a member of SET %q\", tc.msg.Queue, base.AllQueues)\n\t\t}\n\t}\n}\n\nfunc TestAddToGroupeTaskIdConflictError(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tctx := context.Background()\n\tm1 := base.TaskMessage{\n\t\tID: \"custom_id\",\n\t\tType: \"foo\",\n\t\tPayload: nil,\n\t\tUniqueKey: \"unique_key_one\",\n\t}\n\tm2 := base.TaskMessage{\n\t\tID: \"custom_id\",\n\t\tType: \"bar\",\n\t\tPayload: nil,\n\t\tUniqueKey: \"unique_key_two\",\n\t}\n\tconst groupKey = \"mygroup\"\n\n\ttests := []struct {\n\t\tfirstMsg *base.TaskMessage\n\t\tsecondMsg *base.TaskMessage\n\t}{\n\t\t{firstMsg: &m1, secondMsg: &m2},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case.\n\n\t\tif err := r.AddToGroup(ctx, tc.firstMsg, groupKey); err != nil {\n\t\t\tt.Errorf(\"First message: AddToGroup failed: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tif err := r.AddToGroup(ctx, tc.secondMsg, groupKey); !errors.Is(err, errors.ErrTaskIdConflict) {\n\t\t\tt.Errorf(\"Second message: AddToGroup returned %v, want %v\", err, errors.ErrTaskIdConflict)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\nfunc TestAddToGroupUnique(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tnow := time.Now()\n\tr.SetClock(timeutil.NewSimulatedClock(now))\n\tmsg := h.NewTaskMessage(\"mytask\", []byte(\"foo\"))\n\tmsg.UniqueKey = base.UniqueKey(msg.Queue, msg.Type, msg.Payload)\n\tctx := context.Background()\n\n\ttests := []struct {\n\t\tmsg *base.TaskMessage\n\t\tgroupKey string\n\t\tttl time.Duration\n\t}{\n\t\t{\n\t\t\tmsg: msg,\n\t\t\tgroupKey: \"mygroup\",\n\t\t\tttl: 30 * time.Second,\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\n\t\terr := r.AddToGroupUnique(ctx, tc.msg, tc.groupKey, tc.ttl)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"First message: r.AddToGroupUnique(ctx, msg, %q) returned error: %v\", tc.groupKey, err)\n\t\t\tcontinue\n\t\t}\n\n\t\t// Check Group zset has task ID\n\t\tgkey := base.GroupKey(tc.msg.Queue, tc.groupKey)\n\t\tzs := r.client.ZRangeWithScores(ctx, gkey, 0, -1).Val()\n\t\tif n := len(zs); n != 1 {\n\t\t\tt.Errorf(\"Redis ZSET %q contains %d elements, want 1\", gkey, n)\n\t\t\tcontinue\n\t\t}\n\t\tif got := zs[0].Member.(string); got != tc.msg.ID {\n\t\t\tt.Errorf(\"Redis ZSET %q member: got %v, want %v\", gkey, got, tc.msg.ID)\n\t\t\tcontinue\n\t\t}\n\t\tif got := int64(zs[0].Score); got != now.Unix() {\n\t\t\tt.Errorf(\"Redis ZSET %q score: got %d, want %d\", gkey, got, now.Unix())\n\t\t\tcontinue\n\t\t}\n\n\t\t// Check the values under the task key.\n\t\ttaskKey := base.TaskKey(tc.msg.Queue, tc.msg.ID)\n\t\tencoded := r.client.HGet(ctx, taskKey, \"msg\").Val() // \"msg\" field\n\t\tdecoded := h.MustUnmarshal(t, encoded)\n\t\tif diff := cmp.Diff(tc.msg, decoded); diff != \"\" {\n\t\t\tt.Errorf(\"persisted message was %v, want %v; (-want, +got)\\n%s\", decoded, tc.msg, diff)\n\t\t}\n\t\tstate := r.client.HGet(ctx, taskKey, \"state\").Val() // \"state\" field\n\t\tif want := \"aggregating\"; state != want {\n\t\t\tt.Errorf(\"state field under task-key is set to %q, want %q\", state, want)\n\t\t}\n\t\tgroup := r.client.HGet(ctx, taskKey, \"group\").Val() // \"group\" field\n\t\tif want := tc.groupKey; group != want {\n\t\t\tt.Errorf(\"group field under task-key is set to %q, want %q\", group, want)\n\t\t}\n\n\t\t// Check queue is in the AllQueues set.\n\t\tif !r.client.SIsMember(context.Background(), base.AllQueues, tc.msg.Queue).Val() {\n\t\t\tt.Errorf(\"%q is not a member of SET %q\", tc.msg.Queue, base.AllQueues)\n\t\t}\n\n\t\tgot := r.AddToGroupUnique(ctx, tc.msg, tc.groupKey, tc.ttl)\n\t\tif !errors.Is(got, errors.ErrDuplicateTask) {\n\t\t\tt.Errorf(\"Second message: r.AddGroupUnique(ctx, msg, %q) = %v, want %v\",\n\t\t\t\ttc.groupKey, got, errors.ErrDuplicateTask)\n\t\t\tcontinue\n\t\t}\n\n\t\tgotTTL := r.client.TTL(ctx, tc.msg.UniqueKey).Val()\n\t\tif !cmp.Equal(tc.ttl.Seconds(), gotTTL.Seconds(), cmpopts.EquateApprox(0, 1)) {\n\t\t\tt.Errorf(\"TTL %q = %v, want %v\", tc.msg.UniqueKey, gotTTL, tc.ttl)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\nfunc TestAddToGroupUniqueTaskIdConflictError(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tctx := context.Background()\n\tm1 := base.TaskMessage{\n\t\tID: \"custom_id\",\n\t\tType: \"foo\",\n\t\tPayload: nil,\n\t\tUniqueKey: \"unique_key_one\",\n\t}\n\tm2 := base.TaskMessage{\n\t\tID: \"custom_id\",\n\t\tType: \"bar\",\n\t\tPayload: nil,\n\t\tUniqueKey: \"unique_key_two\",\n\t}\n\tconst groupKey = \"mygroup\"\n\tconst ttl = 30 * time.Second\n\n\ttests := []struct {\n\t\tfirstMsg *base.TaskMessage\n\t\tsecondMsg *base.TaskMessage\n\t}{\n\t\t{firstMsg: &m1, secondMsg: &m2},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case.\n\n\t\tif err := r.AddToGroupUnique(ctx, tc.firstMsg, groupKey, ttl); err != nil {\n\t\t\tt.Errorf(\"First message: AddToGroupUnique failed: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tif err := r.AddToGroupUnique(ctx, tc.secondMsg, groupKey, ttl); !errors.Is(err, errors.ErrTaskIdConflict) {\n\t\t\tt.Errorf(\"Second message: AddToGroupUnique returned %v, want %v\", err, errors.ErrTaskIdConflict)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\nfunc TestSchedule(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tmsg := h.NewTaskMessage(\"send_email\", h.JSON(map[string]interface{}{\"subject\": \"hello\"}))\n\ttests := []struct {\n\t\tmsg *base.TaskMessage\n\t\tprocessAt time.Time\n\t}{\n\t\t{msg, time.Now().Add(15 * time.Minute)},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\n\t\terr := r.Schedule(context.Background(), tc.msg, tc.processAt)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"(*RDB).Schedule(%v, %v) = %v, want nil\", tc.msg, tc.processAt, err)\n\t\t\tcontinue\n\t\t}\n\n\t\t// Check Scheduled zset has task ID.\n\t\tscheduledKey := base.ScheduledKey(tc.msg.Queue)\n\t\tzs := r.client.ZRangeWithScores(context.Background(), scheduledKey, 0, -1).Val()\n\t\tif n := len(zs); n != 1 {\n\t\t\tt.Errorf(\"Redis ZSET %q contains %d elements, want 1\", scheduledKey, n)\n\t\t\tcontinue\n\t\t}\n\t\tif got := zs[0].Member.(string); got != tc.msg.ID {\n\t\t\tt.Errorf(\"Redis ZSET %q member: got %v, want %v\", scheduledKey, got, tc.msg.ID)\n\t\t\tcontinue\n\t\t}\n\t\tif got := int64(zs[0].Score); got != tc.processAt.Unix() {\n\t\t\tt.Errorf(\"Redis ZSET %q score: got %d, want %d\",\n\t\t\t\tscheduledKey, got, tc.processAt.Unix())\n\t\t\tcontinue\n\t\t}\n\n\t\t// Check the values under the task key.\n\t\ttaskKey := base.TaskKey(tc.msg.Queue, tc.msg.ID)\n\t\tencoded := r.client.HGet(context.Background(), taskKey, \"msg\").Val() // \"msg\" field\n\t\tdecoded := h.MustUnmarshal(t, encoded)\n\t\tif diff := cmp.Diff(tc.msg, decoded); diff != \"\" {\n\t\t\tt.Errorf(\"persisted message was %v, want %v; (-want, +got)\\n%s\",\n\t\t\t\tdecoded, tc.msg, diff)\n\t\t}\n\t\tstate := r.client.HGet(context.Background(), taskKey, \"state\").Val() // \"state\" field\n\t\tif want := \"scheduled\"; state != want {\n\t\t\tt.Errorf(\"state field under task-key is set to %q, want %q\",\n\t\t\t\tstate, want)\n\t\t}\n\n\t\t// Check queue is in the AllQueues set.\n\t\tif !r.client.SIsMember(context.Background(), base.AllQueues, tc.msg.Queue).Val() {\n\t\t\tt.Errorf(\"%q is not a member of SET %q\", tc.msg.Queue, base.AllQueues)\n\t\t}\n\t}\n}\n\nfunc TestScheduleTaskIdConflictError(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tm1 := base.TaskMessage{\n\t\tID: \"custom_id\",\n\t\tType: \"foo\",\n\t\tPayload: nil,\n\t\tUniqueKey: \"unique_key_one\",\n\t}\n\tm2 := base.TaskMessage{\n\t\tID: \"custom_id\",\n\t\tType: \"bar\",\n\t\tPayload: nil,\n\t\tUniqueKey: \"unique_key_two\",\n\t}\n\tprocessAt := time.Now().Add(30 * time.Second)\n\n\ttests := []struct {\n\t\tfirstMsg *base.TaskMessage\n\t\tsecondMsg *base.TaskMessage\n\t}{\n\t\t{firstMsg: &m1, secondMsg: &m2},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case.\n\n\t\tif err := r.Schedule(context.Background(), tc.firstMsg, processAt); err != nil {\n\t\t\tt.Errorf(\"First message: Schedule failed: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tif err := r.Schedule(context.Background(), tc.secondMsg, processAt); !errors.Is(err, errors.ErrTaskIdConflict) {\n\t\t\tt.Errorf(\"Second message: Schedule returned %v, want %v\", err, errors.ErrTaskIdConflict)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\nfunc TestScheduleUnique(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tm1 := base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"email\",\n\t\tPayload: h.JSON(map[string]interface{}{\"user_id\": 123}),\n\t\tQueue: base.DefaultQueueName,\n\t\tUniqueKey: base.UniqueKey(base.DefaultQueueName, \"email\", h.JSON(map[string]interface{}{\"user_id\": 123})),\n\t}\n\n\ttests := []struct {\n\t\tmsg *base.TaskMessage\n\t\tprocessAt time.Time\n\t\tttl time.Duration // uniqueness lock ttl\n\t}{\n\t\t{&m1, time.Now().Add(15 * time.Minute), time.Minute},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\n\t\tdesc := \"(*RDB).ScheduleUnique(msg, processAt, ttl)\"\n\t\terr := r.ScheduleUnique(context.Background(), tc.msg, tc.processAt, tc.ttl)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"First task: %s = %v, want nil\", desc, err)\n\t\t\tcontinue\n\t\t}\n\n\t\t// Check Scheduled zset has task ID.\n\t\tscheduledKey := base.ScheduledKey(tc.msg.Queue)\n\t\tzs := r.client.ZRangeWithScores(context.Background(), scheduledKey, 0, -1).Val()\n\t\tif n := len(zs); n != 1 {\n\t\t\tt.Errorf(\"Redis ZSET %q contains %d elements, want 1\",\n\t\t\t\tscheduledKey, n)\n\t\t\tcontinue\n\t\t}\n\t\tif got := zs[0].Member.(string); got != tc.msg.ID {\n\t\t\tt.Errorf(\"Redis ZSET %q member: got %v, want %v\",\n\t\t\t\tscheduledKey, got, tc.msg.ID)\n\t\t\tcontinue\n\t\t}\n\t\tif got := int64(zs[0].Score); got != tc.processAt.Unix() {\n\t\t\tt.Errorf(\"Redis ZSET %q score: got %d, want %d\",\n\t\t\t\tscheduledKey, got, tc.processAt.Unix())\n\t\t\tcontinue\n\t\t}\n\n\t\t// Check the values under the task key.\n\t\ttaskKey := base.TaskKey(tc.msg.Queue, tc.msg.ID)\n\t\tencoded := r.client.HGet(context.Background(), taskKey, \"msg\").Val() // \"msg\" field\n\t\tdecoded := h.MustUnmarshal(t, encoded)\n\t\tif diff := cmp.Diff(tc.msg, decoded); diff != \"\" {\n\t\t\tt.Errorf(\"persisted message was %v, want %v; (-want, +got)\\n%s\",\n\t\t\t\tdecoded, tc.msg, diff)\n\t\t}\n\t\tstate := r.client.HGet(context.Background(), taskKey, \"state\").Val() // \"state\" field\n\t\tif want := \"scheduled\"; state != want {\n\t\t\tt.Errorf(\"state field under task-key is set to %q, want %q\",\n\t\t\t\tstate, want)\n\t\t}\n\t\tuniqueKey := r.client.HGet(context.Background(), taskKey, \"unique_key\").Val() // \"unique_key\" field\n\t\tif uniqueKey != tc.msg.UniqueKey {\n\t\t\tt.Errorf(\"uniqueue_key field under task key is set to %q, want %q\", uniqueKey, tc.msg.UniqueKey)\n\t\t}\n\n\t\t// Check queue is in the AllQueues set.\n\t\tif !r.client.SIsMember(context.Background(), base.AllQueues, tc.msg.Queue).Val() {\n\t\t\tt.Errorf(\"%q is not a member of SET %q\", tc.msg.Queue, base.AllQueues)\n\t\t}\n\n\t\t// Enqueue the second message, should fail.\n\t\tgot := r.ScheduleUnique(context.Background(), tc.msg, tc.processAt, tc.ttl)\n\t\tif !errors.Is(got, errors.ErrDuplicateTask) {\n\t\t\tt.Errorf(\"Second task: %s = %v, want %v\", desc, got, errors.ErrDuplicateTask)\n\t\t\tcontinue\n\t\t}\n\n\t\tgotTTL := r.client.TTL(context.Background(), tc.msg.UniqueKey).Val()\n\t\tif !cmp.Equal(tc.ttl.Seconds(), gotTTL.Seconds(), cmpopts.EquateApprox(0, 1)) {\n\t\t\tt.Errorf(\"TTL %q = %v, want %v\", tc.msg.UniqueKey, gotTTL, tc.ttl)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\nfunc TestScheduleUniqueTaskIdConflictError(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tm1 := base.TaskMessage{\n\t\tID: \"custom_id\",\n\t\tType: \"foo\",\n\t\tPayload: nil,\n\t\tUniqueKey: \"unique_key_one\",\n\t}\n\tm2 := base.TaskMessage{\n\t\tID: \"custom_id\",\n\t\tType: \"bar\",\n\t\tPayload: nil,\n\t\tUniqueKey: \"unique_key_two\",\n\t}\n\tconst ttl = 30 * time.Second\n\tprocessAt := time.Now().Add(30 * time.Second)\n\n\ttests := []struct {\n\t\tfirstMsg *base.TaskMessage\n\t\tsecondMsg *base.TaskMessage\n\t}{\n\t\t{firstMsg: &m1, secondMsg: &m2},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case.\n\n\t\tif err := r.ScheduleUnique(context.Background(), tc.firstMsg, processAt, ttl); err != nil {\n\t\t\tt.Errorf(\"First message: ScheduleUnique failed: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tif err := r.ScheduleUnique(context.Background(), tc.secondMsg, processAt, ttl); !errors.Is(err, errors.ErrTaskIdConflict) {\n\t\t\tt.Errorf(\"Second message: ScheduleUnique returned %v, want %v\", err, errors.ErrTaskIdConflict)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\nfunc TestRetry(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tnow := time.Now()\n\tr.SetClock(timeutil.NewSimulatedClock(now))\n\tt1 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"send_email\",\n\t\tPayload: h.JSON(map[string]interface{}{\"subject\": \"Hola!\"}),\n\t\tRetried: 10,\n\t\tTimeout: 1800,\n\t\tQueue: \"default\",\n\t}\n\tt2 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"gen_thumbnail\",\n\t\tPayload: h.JSON(map[string]interface{}{\"path\": \"some/path/to/image.jpg\"}),\n\t\tTimeout: 3000,\n\t\tQueue: \"default\",\n\t}\n\tt3 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"reindex\",\n\t\tPayload: nil,\n\t\tTimeout: 60,\n\t\tQueue: \"default\",\n\t}\n\tt4 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"send_notification\",\n\t\tPayload: nil,\n\t\tTimeout: 1800,\n\t\tQueue: \"custom\",\n\t}\n\terrMsg := \"SMTP server is not responding\"\n\n\ttests := []struct {\n\t\tactive map[string][]*base.TaskMessage\n\t\tlease map[string][]base.Z\n\t\tretry map[string][]base.Z\n\t\tmsg *base.TaskMessage\n\t\tprocessAt time.Time\n\t\terrMsg string\n\t\twantActive map[string][]*base.TaskMessage\n\t\twantLease map[string][]base.Z\n\t\twantRetry map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t2},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}, {Message: t2, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t},\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t3, Score: now.Add(time.Minute).Unix()}},\n\t\t\t},\n\t\t\tmsg: t1,\n\t\t\tprocessAt: now.Add(5 * time.Minute),\n\t\t\terrMsg: errMsg,\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t2},\n\t\t\t},\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t2, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t},\n\t\t\twantRetry: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: h.TaskMessageAfterRetry(*t1, errMsg, now), Score: now.Add(5 * time.Minute).Unix()},\n\t\t\t\t\t{Message: t3, Score: now.Add(time.Minute).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t2},\n\t\t\t\t\"custom\": {t4},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}, {Message: t2, Score: now.Add(20 * time.Second).Unix()}},\n\t\t\t\t\"custom\": {{Message: t4, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t},\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tmsg: t4,\n\t\t\tprocessAt: now.Add(5 * time.Minute),\n\t\t\terrMsg: errMsg,\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t2},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}, {Message: t2, Score: now.Add(20 * time.Second).Unix()}},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\twantRetry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: h.TaskMessageAfterRetry(*t4, errMsg, now), Score: now.Add(5 * time.Minute).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\th.SeedAllActiveQueues(t, r.client, tc.active)\n\t\th.SeedAllLease(t, r.client, tc.lease)\n\t\th.SeedAllRetryQueues(t, r.client, tc.retry)\n\n\t\terr := r.Retry(context.Background(), tc.msg, tc.processAt, tc.errMsg, true /*isFailure*/)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"(*RDB).Retry = %v, want nil\", err)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor queue, want := range tc.wantActive {\n\t\t\tgotActive := h.GetActiveMessages(t, r.client, queue)\n\t\t\tif diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.ActiveKey(queue), diff)\n\t\t\t}\n\t\t}\n\t\tfor queue, want := range tc.wantLease {\n\t\t\tgotLease := h.GetLeaseEntries(t, r.client, queue)\n\t\t\tif diff := cmp.Diff(want, gotLease, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.LeaseKey(queue), diff)\n\t\t\t}\n\t\t}\n\t\tfor queue, want := range tc.wantRetry {\n\t\t\tgotRetry := h.GetRetryEntries(t, r.client, queue)\n\t\t\tif diff := cmp.Diff(want, gotRetry, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.RetryKey(queue), diff)\n\t\t\t}\n\t\t}\n\n\t\tprocessedKey := base.ProcessedKey(tc.msg.Queue, time.Now())\n\t\tgotProcessed := r.client.Get(context.Background(), processedKey).Val()\n\t\tif gotProcessed != \"1\" {\n\t\t\tt.Errorf(\"GET %q = %q, want 1\", processedKey, gotProcessed)\n\t\t}\n\t\tgotTTL := r.client.TTL(context.Background(), processedKey).Val()\n\t\tif gotTTL > statsTTL {\n\t\t\tt.Errorf(\"TTL %q = %v, want less than or equal to %v\", processedKey, gotTTL, statsTTL)\n\t\t}\n\n\t\tfailedKey := base.FailedKey(tc.msg.Queue, time.Now())\n\t\tgotFailed := r.client.Get(context.Background(), failedKey).Val()\n\t\tif gotFailed != \"1\" {\n\t\t\tt.Errorf(\"GET %q = %q, want 1\", failedKey, gotFailed)\n\t\t}\n\t\tgotTTL = r.client.TTL(context.Background(), failedKey).Val()\n\t\tif gotTTL > statsTTL {\n\t\t\tt.Errorf(\"TTL %q = %v, want less than or equal to %v\", failedKey, gotTTL, statsTTL)\n\t\t}\n\n\t\tprocessedTotalKey := base.ProcessedTotalKey(tc.msg.Queue)\n\t\tgotProcessedTotal := r.client.Get(context.Background(), processedTotalKey).Val()\n\t\tif gotProcessedTotal != \"1\" {\n\t\t\tt.Errorf(\"GET %q = %q, want 1\", processedTotalKey, gotProcessedTotal)\n\t\t}\n\n\t\tfailedTotalKey := base.FailedTotalKey(tc.msg.Queue)\n\t\tgotFailedTotal := r.client.Get(context.Background(), failedTotalKey).Val()\n\t\tif gotFailedTotal != \"1\" {\n\t\t\tt.Errorf(\"GET %q = %q, want 1\", failedTotalKey, gotFailedTotal)\n\t\t}\n\t}\n}\n\nfunc TestRetryWithNonFailureError(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tnow := time.Now()\n\tr.SetClock(timeutil.NewSimulatedClock(now))\n\tt1 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"send_email\",\n\t\tPayload: h.JSON(map[string]interface{}{\"subject\": \"Hola!\"}),\n\t\tRetried: 10,\n\t\tTimeout: 1800,\n\t\tQueue: \"default\",\n\t}\n\tt2 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"gen_thumbnail\",\n\t\tPayload: h.JSON(map[string]interface{}{\"path\": \"some/path/to/image.jpg\"}),\n\t\tTimeout: 3000,\n\t\tQueue: \"default\",\n\t}\n\tt3 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"reindex\",\n\t\tPayload: nil,\n\t\tTimeout: 60,\n\t\tQueue: \"default\",\n\t}\n\tt4 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"send_notification\",\n\t\tPayload: nil,\n\t\tTimeout: 1800,\n\t\tQueue: \"custom\",\n\t}\n\terrMsg := \"SMTP server is not responding\"\n\n\ttests := []struct {\n\t\tactive map[string][]*base.TaskMessage\n\t\tlease map[string][]base.Z\n\t\tretry map[string][]base.Z\n\t\tmsg *base.TaskMessage\n\t\tprocessAt time.Time\n\t\terrMsg string\n\t\twantActive map[string][]*base.TaskMessage\n\t\twantLease map[string][]base.Z\n\t\twantRetry map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t2},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}, {Message: t2, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t},\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t3, Score: now.Add(time.Minute).Unix()}},\n\t\t\t},\n\t\t\tmsg: t1,\n\t\t\tprocessAt: now.Add(5 * time.Minute),\n\t\t\terrMsg: errMsg,\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t2},\n\t\t\t},\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t2, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t},\n\t\t\twantRetry: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t// Task message should include the error message but without incrementing the retry count.\n\t\t\t\t\t{Message: h.TaskMessageWithError(*t1, errMsg, now), Score: now.Add(5 * time.Minute).Unix()},\n\t\t\t\t\t{Message: t3, Score: now.Add(time.Minute).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t2},\n\t\t\t\t\"custom\": {t4},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}, {Message: t2, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t\t\"custom\": {{Message: t4, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t},\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tmsg: t4,\n\t\t\tprocessAt: now.Add(5 * time.Minute),\n\t\t\terrMsg: errMsg,\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t2},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}, {Message: t2, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\twantRetry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t// Task message should include the error message but without incrementing the retry count.\n\t\t\t\t\t{Message: h.TaskMessageWithError(*t4, errMsg, now), Score: now.Add(5 * time.Minute).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\th.SeedAllActiveQueues(t, r.client, tc.active)\n\t\th.SeedAllLease(t, r.client, tc.lease)\n\t\th.SeedAllRetryQueues(t, r.client, tc.retry)\n\n\t\terr := r.Retry(context.Background(), tc.msg, tc.processAt, tc.errMsg, false /*isFailure*/)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"(*RDB).Retry = %v, want nil\", err)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor queue, want := range tc.wantActive {\n\t\t\tgotActive := h.GetActiveMessages(t, r.client, queue)\n\t\t\tif diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.ActiveKey(queue), diff)\n\t\t\t}\n\t\t}\n\t\tfor queue, want := range tc.wantLease {\n\t\t\tgotLease := h.GetLeaseEntries(t, r.client, queue)\n\t\t\tif diff := cmp.Diff(want, gotLease, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.LeaseKey(queue), diff)\n\t\t\t}\n\t\t}\n\t\tfor queue, want := range tc.wantRetry {\n\t\t\tgotRetry := h.GetRetryEntries(t, r.client, queue)\n\t\t\tif diff := cmp.Diff(want, gotRetry, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.RetryKey(queue), diff)\n\t\t\t}\n\t\t}\n\n\t\t// If isFailure is set to false, no stats should be recorded to avoid skewing the error rate.\n\t\tprocessedKey := base.ProcessedKey(tc.msg.Queue, time.Now())\n\t\tgotProcessed := r.client.Get(context.Background(), processedKey).Val()\n\t\tif gotProcessed != \"\" {\n\t\t\tt.Errorf(\"GET %q = %q, want empty\", processedKey, gotProcessed)\n\t\t}\n\n\t\t// If isFailure is set to false, no stats should be recorded to avoid skewing the error rate.\n\t\tfailedKey := base.FailedKey(tc.msg.Queue, time.Now())\n\t\tgotFailed := r.client.Get(context.Background(), failedKey).Val()\n\t\tif gotFailed != \"\" {\n\t\t\tt.Errorf(\"GET %q = %q, want empty\", failedKey, gotFailed)\n\t\t}\n\n\t\tprocessedTotalKey := base.ProcessedTotalKey(tc.msg.Queue)\n\t\tgotProcessedTotal := r.client.Get(context.Background(), processedTotalKey).Val()\n\t\tif gotProcessedTotal != \"\" {\n\t\t\tt.Errorf(\"GET %q = %q, want empty\", processedTotalKey, gotProcessedTotal)\n\t\t}\n\n\t\tfailedTotalKey := base.FailedTotalKey(tc.msg.Queue)\n\t\tgotFailedTotal := r.client.Get(context.Background(), failedTotalKey).Val()\n\t\tif gotFailedTotal != \"\" {\n\t\t\tt.Errorf(\"GET %q = %q, want empty\", failedTotalKey, gotFailedTotal)\n\t\t}\n\t}\n}\n\nfunc TestArchive(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tnow := time.Now()\n\tr.SetClock(timeutil.NewSimulatedClock(now))\n\tt1 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"send_email\",\n\t\tPayload: nil,\n\t\tQueue: \"default\",\n\t\tRetry: 25,\n\t\tRetried: 25,\n\t\tTimeout: 1800,\n\t}\n\tt2 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"reindex\",\n\t\tPayload: nil,\n\t\tQueue: \"default\",\n\t\tRetry: 25,\n\t\tRetried: 0,\n\t\tTimeout: 3000,\n\t}\n\tt3 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"generate_csv\",\n\t\tPayload: nil,\n\t\tQueue: \"default\",\n\t\tRetry: 25,\n\t\tRetried: 0,\n\t\tTimeout: 60,\n\t}\n\tt4 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"send_email\",\n\t\tPayload: nil,\n\t\tQueue: \"custom\",\n\t\tRetry: 25,\n\t\tRetried: 25,\n\t\tTimeout: 1800,\n\t}\n\terrMsg := \"SMTP server not responding\"\n\n\ttests := []struct {\n\t\tactive map[string][]*base.TaskMessage\n\t\tlease map[string][]base.Z\n\t\tarchived map[string][]base.Z\n\t\ttarget *base.TaskMessage // task to archive\n\t\twantActive map[string][]*base.TaskMessage\n\t\twantLease map[string][]base.Z\n\t\twantArchived map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t2},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t\t{Message: t2, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t3, Score: now.Add(-time.Hour).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\ttarget: t1,\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t2},\n\t\t\t},\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t2, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: h.TaskMessageWithError(*t1, errMsg, now), Score: now.Unix()},\n\t\t\t\t\t{Message: t3, Score: now.Add(-time.Hour).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t2, t3},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t\t{Message: t2, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t\t{Message: t3, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\ttarget: t1,\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t2, t3},\n\t\t\t},\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t2, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t\t{Message: t3, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: h.TaskMessageWithError(*t1, errMsg, now), Score: now.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t\t\"custom\": {t4},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: t4, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\ttarget: t4,\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: h.TaskMessageWithError(*t4, errMsg, now), Score: now.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllActiveQueues(t, r.client, tc.active)\n\t\th.SeedAllLease(t, r.client, tc.lease)\n\t\th.SeedAllArchivedQueues(t, r.client, tc.archived)\n\n\t\terr := r.Archive(context.Background(), tc.target, errMsg)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"(*RDB).Archive(%v, %v) = %v, want nil\", tc.target, errMsg, err)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor queue, want := range tc.wantActive {\n\t\t\tgotActive := h.GetActiveMessages(t, r.client, queue)\n\t\t\tif diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q: (-want, +got)\\n%s\", base.ActiveKey(queue), diff)\n\t\t\t}\n\t\t}\n\t\tfor queue, want := range tc.wantLease {\n\t\t\tgotLease := h.GetLeaseEntries(t, r.client, queue)\n\t\t\tif diff := cmp.Diff(want, gotLease, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q after calling (*RDB).Archive: (-want, +got):\\n%s\", base.LeaseKey(queue), diff)\n\t\t\t}\n\t\t}\n\t\tfor queue, want := range tc.wantArchived {\n\t\t\tgotArchived := h.GetArchivedEntries(t, r.client, queue)\n\t\t\tif diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt, zScoreCmpOpt, timeCmpOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q after calling (*RDB).Archive: (-want, +got):\\n%s\", base.ArchivedKey(queue), diff)\n\t\t\t}\n\t\t}\n\n\t\tprocessedKey := base.ProcessedKey(tc.target.Queue, time.Now())\n\t\tgotProcessed := r.client.Get(context.Background(), processedKey).Val()\n\t\tif gotProcessed != \"1\" {\n\t\t\tt.Errorf(\"GET %q = %q, want 1\", processedKey, gotProcessed)\n\t\t}\n\t\tgotTTL := r.client.TTL(context.Background(), processedKey).Val()\n\t\tif gotTTL > statsTTL {\n\t\t\tt.Errorf(\"TTL %q = %v, want less than or equal to %v\", processedKey, gotTTL, statsTTL)\n\t\t}\n\n\t\tfailedKey := base.FailedKey(tc.target.Queue, time.Now())\n\t\tgotFailed := r.client.Get(context.Background(), failedKey).Val()\n\t\tif gotFailed != \"1\" {\n\t\t\tt.Errorf(\"GET %q = %q, want 1\", failedKey, gotFailed)\n\t\t}\n\t\tgotTTL = r.client.TTL(context.Background(), processedKey).Val()\n\t\tif gotTTL > statsTTL {\n\t\t\tt.Errorf(\"TTL %q = %v, want less than or equal to %v\", failedKey, gotTTL, statsTTL)\n\t\t}\n\n\t\tprocessedTotalKey := base.ProcessedTotalKey(tc.target.Queue)\n\t\tgotProcessedTotal := r.client.Get(context.Background(), processedTotalKey).Val()\n\t\tif gotProcessedTotal != \"1\" {\n\t\t\tt.Errorf(\"GET %q = %q, want 1\", processedTotalKey, gotProcessedTotal)\n\t\t}\n\n\t\tfailedTotalKey := base.FailedTotalKey(tc.target.Queue)\n\t\tgotFailedTotal := r.client.Get(context.Background(), failedTotalKey).Val()\n\t\tif gotFailedTotal != \"1\" {\n\t\t\tt.Errorf(\"GET %q = %q, want 1\", failedTotalKey, gotFailedTotal)\n\t\t}\n\t}\n}\n\nfunc TestArchiveTrim(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tnow := time.Now()\n\tr.SetClock(timeutil.NewSimulatedClock(now))\n\n\tt1 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"send_email\",\n\t\tPayload: nil,\n\t\tQueue: \"default\",\n\t\tRetry: 25,\n\t\tRetried: 25,\n\t\tTimeout: 1800,\n\t}\n\tt2 := &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"reindex\",\n\t\tPayload: nil,\n\t\tQueue: \"default\",\n\t\tRetry: 25,\n\t\tRetried: 0,\n\t\tTimeout: 3000,\n\t}\n\terrMsg := \"SMTP server not responding\"\n\n\tmaxArchiveSet := make([]base.Z, 0)\n\tfor i := 0; i < maxArchiveSize-1; i++ {\n\t\tmaxArchiveSet = append(maxArchiveSet, base.Z{Message: &base.TaskMessage{\n\t\t\tID: uuid.NewString(),\n\t\t\tType: \"generate_csv\",\n\t\t\tPayload: nil,\n\t\t\tQueue: \"default\",\n\t\t\tRetry: 25,\n\t\t\tRetried: 0,\n\t\t\tTimeout: 60,\n\t\t}, Score: now.Add(-time.Hour + -time.Second*time.Duration(i)).Unix()})\n\t}\n\n\twantMaxArchiveSet := make([]base.Z, 0)\n\t// newly archived task should be at the front\n\twantMaxArchiveSet = append(wantMaxArchiveSet, base.Z{Message: h.TaskMessageWithError(*t1, errMsg, now), Score: now.Unix()})\n\t// oldest task should be dropped from the set\n\twantMaxArchiveSet = append(wantMaxArchiveSet, maxArchiveSet[:len(maxArchiveSet)-1]...)\n\n\ttests := []struct {\n\t\ttoArchive map[string][]*base.TaskMessage\n\t\tlease map[string][]base.Z\n\t\tarchived map[string][]base.Z\n\t\twantArchived map[string][]base.Z\n\t}{\n\t\t{ // simple, 1 to be archived, 1 already archived, both are in the archive set\n\t\t\ttoArchive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t2, Score: now.Add(-time.Hour).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: h.TaskMessageWithError(*t1, errMsg, now), Score: now.Unix()},\n\t\t\t\t\t{Message: t2, Score: now.Add(-time.Hour).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{ // 1 to be archived, 1 already archived but past expiry, only the newly archived task should be left\n\t\t\ttoArchive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t2, Score: now.Add(-time.Hour * 24 * (archivedExpirationInDays + 1)).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: h.TaskMessageWithError(*t1, errMsg, now), Score: now.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{ // 1 to be archived, maxArchiveSize in archive set, archive set should be trimmed back to maxArchiveSize and newly archived task should be in the set\n\t\t\ttoArchive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": maxArchiveSet,\n\t\t\t},\n\t\t\twantArchived: map[string][]base.Z{\n\t\t\t\t\"default\": wantMaxArchiveSet,\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllActiveQueues(t, r.client, tc.toArchive)\n\t\th.SeedAllLease(t, r.client, tc.lease)\n\t\th.SeedAllArchivedQueues(t, r.client, tc.archived)\n\n\t\tfor _, tasks := range tc.toArchive {\n\t\t\tfor _, target := range tasks {\n\t\t\t\terr := r.Archive(context.Background(), target, errMsg)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Errorf(\"(*RDB).Archive(%v, %v) = %v, want nil\", target, errMsg, err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tfor queue, want := range tc.wantArchived {\n\t\t\tgotArchived := h.GetArchivedEntries(t, r.client, queue)\n\n\t\t\tif diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt, zScoreCmpOpt, timeCmpOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q after calling (*RDB).Archive: (-want, +got):\\n%s\", base.ArchivedKey(queue), diff)\n\t\t\t}\n\n\t\t\t// check that only keys present in the archived set are in rdb\n\t\t\tvals := r.client.Keys(context.Background(), base.TaskKeyPrefix(queue)+\"*\").Val()\n\t\t\tif len(vals) != len(gotArchived) {\n\t\t\t\tt.Errorf(\"len of keys = %v, want %v\", len(vals), len(gotArchived))\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tfor _, val := range vals {\n\t\t\t\tfound := false\n\t\t\t\tfor _, entry := range gotArchived {\n\t\t\t\t\tif strings.Contains(val, entry.Message.ID) {\n\t\t\t\t\t\tfound = true\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tif !found {\n\t\t\t\t\tt.Errorf(\"key %v not found in archived set (it was orphaned by the archive trim)\", val)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestForwardIfReadyWithGroup(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tnow := time.Now()\n\tr.SetClock(timeutil.NewSimulatedClock(now))\n\tt1 := h.NewTaskMessage(\"send_email\", nil)\n\tt2 := h.NewTaskMessage(\"generate_csv\", nil)\n\tt3 := h.NewTaskMessage(\"gen_thumbnail\", nil)\n\tt4 := h.NewTaskMessageWithQueue(\"important_task\", nil, \"critical\")\n\tt5 := h.NewTaskMessageWithQueue(\"minor_task\", nil, \"low\")\n\t// Set group keys for the tasks.\n\tt1.GroupKey = \"notifications\"\n\tt2.GroupKey = \"csv\"\n\tt4.GroupKey = \"critical_task_group\"\n\tt5.GroupKey = \"minor_task_group\"\n\n\tctx := context.Background()\n\tsecondAgo := now.Add(-time.Second)\n\n\ttests := []struct {\n\t\tscheduled map[string][]base.Z\n\t\tretry map[string][]base.Z\n\t\tqnames []string\n\t\twantPending map[string][]*base.TaskMessage\n\t\twantScheduled map[string][]*base.TaskMessage\n\t\twantRetry map[string][]*base.TaskMessage\n\t\twantGroup map[string]map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: secondAgo.Unix()},\n\t\t\t\t\t{Message: t2, Score: secondAgo.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t3, Score: secondAgo.Unix()}},\n\t\t\t},\n\t\t\tqnames: []string{\"default\"},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t3},\n\t\t\t},\n\t\t\twantScheduled: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantRetry: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantGroup: map[string]map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t\"notifications\": {{Message: t1, Score: now.Unix()}},\n\t\t\t\t\t\"csv\": {{Message: t2, Score: now.Unix()}},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t1, Score: secondAgo.Unix()}},\n\t\t\t\t\"critical\": {{Message: t4, Score: secondAgo.Unix()}},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {{Message: t5, Score: secondAgo.Unix()}},\n\t\t\t},\n\t\t\tqnames: []string{\"default\", \"critical\", \"low\"},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t\twantScheduled: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t\twantRetry: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t\twantGroup: map[string]map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t\"notifications\": {{Message: t1, Score: now.Unix()}},\n\t\t\t\t},\n\t\t\t\t\"critical\": {\n\t\t\t\t\t\"critical_task_group\": {{Message: t4, Score: now.Unix()}},\n\t\t\t\t},\n\t\t\t\t\"low\": {\n\t\t\t\t\t\"minor_task_group\": {{Message: t5, Score: now.Unix()}},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllScheduledQueues(t, r.client, tc.scheduled)\n\t\th.SeedAllRetryQueues(t, r.client, tc.retry)\n\n\t\terr := r.ForwardIfReady(tc.qnames...)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"(*RDB).ForwardIfReady(%v) = %v, want nil\", tc.qnames, err)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.PendingKey(qname), diff)\n\t\t\t}\n\t\t\t// Make sure \"pending_since\" field is set\n\t\t\tfor _, msg := range gotPending {\n\t\t\t\tpendingSince := r.client.HGet(ctx, base.TaskKey(msg.Queue, msg.ID), \"pending_since\").Val()\n\t\t\t\tif want := strconv.Itoa(int(now.UnixNano())); pendingSince != want {\n\t\t\t\t\tt.Error(\"pending_since field is not set for newly pending message\")\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tfor qname, want := range tc.wantScheduled {\n\t\t\tgotScheduled := h.GetScheduledMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotScheduled, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.ScheduledKey(qname), diff)\n\t\t\t}\n\t\t}\n\t\tfor qname, want := range tc.wantRetry {\n\t\t\tgotRetry := h.GetRetryMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotRetry, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.RetryKey(qname), diff)\n\t\t\t}\n\t\t}\n\t\tfor qname, groups := range tc.wantGroup {\n\t\t\tfor groupKey, wantGroup := range groups {\n\t\t\t\tgotGroup := h.GetGroupEntries(t, r.client, qname, groupKey)\n\t\t\t\tif diff := cmp.Diff(wantGroup, gotGroup, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.GroupKey(qname, groupKey), diff)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestForwardIfReady(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tt1 := h.NewTaskMessage(\"send_email\", nil)\n\tt2 := h.NewTaskMessage(\"generate_csv\", nil)\n\tt3 := h.NewTaskMessage(\"gen_thumbnail\", nil)\n\tt4 := h.NewTaskMessageWithQueue(\"important_task\", nil, \"critical\")\n\tt5 := h.NewTaskMessageWithQueue(\"minor_task\", nil, \"low\")\n\tsecondAgo := time.Now().Add(-time.Second)\n\thourFromNow := time.Now().Add(time.Hour)\n\n\ttests := []struct {\n\t\tscheduled map[string][]base.Z\n\t\tretry map[string][]base.Z\n\t\tqnames []string\n\t\twantPending map[string][]*base.TaskMessage\n\t\twantScheduled map[string][]*base.TaskMessage\n\t\twantRetry map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: secondAgo.Unix()},\n\t\t\t\t\t{Message: t2, Score: secondAgo.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t3, Score: secondAgo.Unix()}},\n\t\t\t},\n\t\t\tqnames: []string{\"default\"},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t2, t3},\n\t\t\t},\n\t\t\twantScheduled: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantRetry: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: hourFromNow.Unix()},\n\t\t\t\t\t{Message: t2, Score: secondAgo.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t3, Score: secondAgo.Unix()}},\n\t\t\t},\n\t\t\tqnames: []string{\"default\"},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t2, t3},\n\t\t\t},\n\t\t\twantScheduled: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t},\n\t\t\twantRetry: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: hourFromNow.Unix()},\n\t\t\t\t\t{Message: t2, Score: hourFromNow.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t3, Score: hourFromNow.Unix()}},\n\t\t\t},\n\t\t\tqnames: []string{\"default\"},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantScheduled: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t2},\n\t\t\t},\n\t\t\twantRetry: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t3},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscheduled: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t1, Score: secondAgo.Unix()}},\n\t\t\t\t\"critical\": {{Message: t4, Score: secondAgo.Unix()}},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {{Message: t5, Score: secondAgo.Unix()}},\n\t\t\t},\n\t\t\tqnames: []string{\"default\", \"critical\", \"low\"},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t\t\"critical\": {t4},\n\t\t\t\t\"low\": {t5},\n\t\t\t},\n\t\t\twantScheduled: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t\twantRetry: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client) // clean up db before each test case\n\t\th.SeedAllScheduledQueues(t, r.client, tc.scheduled)\n\t\th.SeedAllRetryQueues(t, r.client, tc.retry)\n\n\t\tnow := time.Now()\n\t\tr.SetClock(timeutil.NewSimulatedClock(now))\n\n\t\terr := r.ForwardIfReady(tc.qnames...)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"(*RDB).ForwardIfReady(%v) = %v, want nil\", tc.qnames, err)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.PendingKey(qname), diff)\n\t\t\t}\n\t\t\t// Make sure \"pending_since\" field is set\n\t\t\tfor _, msg := range gotPending {\n\t\t\t\tpendingSince := r.client.HGet(context.Background(), base.TaskKey(msg.Queue, msg.ID), \"pending_since\").Val()\n\t\t\t\tif want := strconv.Itoa(int(now.UnixNano())); pendingSince != want {\n\t\t\t\t\tt.Error(\"pending_since field is not set for newly pending message\")\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tfor qname, want := range tc.wantScheduled {\n\t\t\tgotScheduled := h.GetScheduledMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotScheduled, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.ScheduledKey(qname), diff)\n\t\t\t}\n\t\t}\n\t\tfor qname, want := range tc.wantRetry {\n\t\t\tgotRetry := h.GetRetryMessages(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotRetry, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"mismatch found in %q; (-want, +got)\\n%s\", base.RetryKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc newCompletedTask(qname, typename string, payload []byte, completedAt time.Time) *base.TaskMessage {\n\tmsg := h.NewTaskMessageWithQueue(typename, payload, qname)\n\tmsg.CompletedAt = completedAt.Unix()\n\treturn msg\n}\n\nfunc TestDeleteExpiredCompletedTasks(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tnow := time.Now()\n\tsecondAgo := now.Add(-time.Second)\n\thourFromNow := now.Add(time.Hour)\n\thourAgo := now.Add(-time.Hour)\n\tminuteAgo := now.Add(-time.Minute)\n\n\tt1 := newCompletedTask(\"default\", \"task1\", nil, hourAgo)\n\tt2 := newCompletedTask(\"default\", \"task2\", nil, minuteAgo)\n\tt3 := newCompletedTask(\"default\", \"task3\", nil, secondAgo)\n\tt4 := newCompletedTask(\"critical\", \"critical_task\", nil, hourAgo)\n\tt5 := newCompletedTask(\"low\", \"low_priority_task\", nil, hourAgo)\n\n\ttests := []struct {\n\t\tdesc string\n\t\tcompleted map[string][]base.Z\n\t\tqname string\n\t\twantCompleted map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tdesc: \"deletes expired task from default queue\",\n\t\t\tcompleted: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: secondAgo.Unix()},\n\t\t\t\t\t{Message: t2, Score: hourFromNow.Unix()},\n\t\t\t\t\t{Message: t3, Score: now.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twantCompleted: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t2, Score: hourFromNow.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"deletes expired task from specified queue\",\n\t\t\tcompleted: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t2, Score: secondAgo.Unix()},\n\t\t\t\t},\n\t\t\t\t\"critical\": {\n\t\t\t\t\t{Message: t4, Score: secondAgo.Unix()},\n\t\t\t\t},\n\t\t\t\t\"low\": {\n\t\t\t\t\t{Message: t5, Score: now.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"critical\",\n\t\t\twantCompleted: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t2, Score: secondAgo.Unix()},\n\t\t\t\t},\n\t\t\t\t\"critical\": {},\n\t\t\t\t\"low\": {\n\t\t\t\t\t{Message: t5, Score: now.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\th.SeedAllCompletedQueues(t, r.client, tc.completed)\n\n\t\tif err := r.DeleteExpiredCompletedTasks(tc.qname, 100); err != nil {\n\t\t\tt.Errorf(\"DeleteExpiredCompletedTasks(%q, 100) failed: %v\", tc.qname, err)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor qname, want := range tc.wantCompleted {\n\t\t\tgot := h.GetCompletedEntries(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, got, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s: diff found in %q completed set: want=%v, got=%v\\n%s\", tc.desc, qname, want, got, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestListLeaseExpired(t *testing.T) {\n\tt1 := h.NewTaskMessageWithQueue(\"task1\", nil, \"default\")\n\tt2 := h.NewTaskMessageWithQueue(\"task2\", nil, \"default\")\n\tt3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"critical\")\n\n\tnow := time.Now()\n\n\ttests := []struct {\n\t\tdesc string\n\t\tlease map[string][]base.Z\n\t\tqnames []string\n\t\tcutoff time.Time\n\t\twant []*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tdesc: \"with a single active task\",\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t1, Score: now.Add(-10 * time.Second).Unix()}},\n\t\t\t},\n\t\t\tqnames: []string{\"default\"},\n\t\t\tcutoff: now,\n\t\t\twant: []*base.TaskMessage{t1},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with multiple active tasks, and one expired\",\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: now.Add(-5 * time.Minute).Unix()},\n\t\t\t\t\t{Message: t2, Score: now.Add(20 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t\t\"critical\": {\n\t\t\t\t\t{Message: t3, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqnames: []string{\"default\", \"critical\"},\n\t\t\tcutoff: now,\n\t\t\twant: []*base.TaskMessage{t1},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with multiple expired active tasks\",\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: now.Add(-2 * time.Minute).Unix()},\n\t\t\t\t\t{Message: t2, Score: now.Add(20 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t\t\"critical\": {\n\t\t\t\t\t{Message: t3, Score: now.Add(-30 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqnames: []string{\"default\", \"critical\"},\n\t\t\tcutoff: now,\n\t\t\twant: []*base.TaskMessage{t1, t3},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with empty active queue\",\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\tqnames: []string{\"default\", \"critical\"},\n\t\t\tcutoff: now,\n\t\t\twant: []*base.TaskMessage{},\n\t\t},\n\t}\n\n\tr := setup(t)\n\tdefer r.Close()\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\th.SeedAllLease(t, r.client, tc.lease)\n\n\t\tgot, err := r.ListLeaseExpired(tc.cutoff, tc.qnames...)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s; ListLeaseExpired(%v) returned error: %v\", tc.desc, tc.cutoff, err)\n\t\t\tcontinue\n\t\t}\n\n\t\tif diff := cmp.Diff(tc.want, got, h.SortMsgOpt); diff != \"\" {\n\t\t\tt.Errorf(\"%s; ListLeaseExpired(%v) returned %v, want %v;(-want,+got)\\n%s\",\n\t\t\t\ttc.desc, tc.cutoff, got, tc.want, diff)\n\t\t}\n\t}\n}\n\nfunc TestExtendLease(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\tnow := time.Now()\n\tr.SetClock(timeutil.NewSimulatedClock(now))\n\n\tt1 := h.NewTaskMessageWithQueue(\"task1\", nil, \"default\")\n\tt2 := h.NewTaskMessageWithQueue(\"task2\", nil, \"default\")\n\tt3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"critical\")\n\tt4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"default\")\n\n\ttests := []struct {\n\t\tdesc string\n\t\tlease map[string][]base.Z\n\t\tqname string\n\t\tids []string\n\t\twantExpirationTime time.Time\n\t\twantLease map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tdesc: \"Should extends lease for a single message in a queue\",\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t\t\"critical\": {{Message: t3, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tids: []string{t1.ID},\n\t\t\twantExpirationTime: now.Add(LeaseDuration),\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t1, Score: now.Add(LeaseDuration).Unix()}},\n\t\t\t\t\"critical\": {{Message: t3, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"Should extends lease for multiple message in a queue\",\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t1, Score: now.Add(10 * time.Second).Unix()}, {Message: t2, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t\t\"critical\": {{Message: t3, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tids: []string{t1.ID, t2.ID},\n\t\t\twantExpirationTime: now.Add(LeaseDuration),\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t1, Score: now.Add(LeaseDuration).Unix()}, {Message: t2, Score: now.Add(LeaseDuration).Unix()}},\n\t\t\t\t\"critical\": {{Message: t3, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"Should selectively extends lease for messages in a queue\",\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t\t{Message: t2, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t\t{Message: t4, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t\t\"critical\": {{Message: t3, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tids: []string{t2.ID, t4.ID},\n\t\t\twantExpirationTime: now.Add(LeaseDuration),\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t\t{Message: t2, Score: now.Add(LeaseDuration).Unix()},\n\t\t\t\t\t{Message: t4, Score: now.Add(LeaseDuration).Unix()},\n\t\t\t\t},\n\t\t\t\t\"critical\": {{Message: t3, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"Should not add a new entry in the lease set\",\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tids: []string{t1.ID, t2.ID},\n\t\t\twantExpirationTime: now.Add(LeaseDuration),\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: now.Add(LeaseDuration).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\th.SeedAllLease(t, r.client, tc.lease)\n\n\t\tgotExpirationTime, err := r.ExtendLease(tc.qname, tc.ids...)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"%s: ExtendLease(%q, %v) returned error: %v\", tc.desc, tc.qname, tc.ids, err)\n\t\t}\n\t\tif gotExpirationTime != tc.wantExpirationTime {\n\t\t\tt.Errorf(\"%s: ExtendLease(%q, %v) returned expirationTime %v, want %v\", tc.desc, tc.qname, tc.ids, gotExpirationTime, tc.wantExpirationTime)\n\t\t}\n\n\t\tfor qname, want := range tc.wantLease {\n\t\t\tgotLease := h.GetLeaseEntries(t, r.client, qname)\n\t\t\tif diff := cmp.Diff(want, gotLease, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s: mismatch found in %q: (-want,+got):\\n%s\", tc.desc, base.LeaseKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestWriteServerState(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tvar (\n\t\thost = \"localhost\"\n\t\tpid = 4242\n\t\tserverID = \"server123\"\n\n\t\tttl = 5 * time.Second\n\t)\n\n\tinfo := base.ServerInfo{\n\t\tHost: host,\n\t\tPID: pid,\n\t\tServerID: serverID,\n\t\tConcurrency: 10,\n\t\tQueues: map[string]int{\"default\": 2, \"email\": 5, \"low\": 1},\n\t\tStrictPriority: false,\n\t\tStarted: time.Now().UTC(),\n\t\tStatus: \"active\",\n\t\tActiveWorkerCount: 0,\n\t}\n\n\terr := r.WriteServerState(&info, nil /* workers */, ttl)\n\tif err != nil {\n\t\tt.Errorf(\"r.WriteServerState returned an error: %v\", err)\n\t}\n\n\t// Check ServerInfo was written correctly.\n\tskey := base.ServerInfoKey(host, pid, serverID)\n\tdata := r.client.Get(context.Background(), skey).Val()\n\tgot, err := base.DecodeServerInfo([]byte(data))\n\tif err != nil {\n\t\tt.Fatalf(\"could not decode server info: %v\", err)\n\t}\n\tif diff := cmp.Diff(info, *got); diff != \"\" {\n\t\tt.Errorf(\"persisted ServerInfo was %v, want %v; (-want,+got)\\n%s\",\n\t\t\tgot, info, diff)\n\t}\n\t// Check ServerInfo TTL was set correctly.\n\tgotTTL := r.client.TTL(context.Background(), skey).Val()\n\tif !cmp.Equal(ttl.Seconds(), gotTTL.Seconds(), cmpopts.EquateApprox(0, 1)) {\n\t\tt.Errorf(\"TTL of %q was %v, want %v\", skey, gotTTL, ttl)\n\t}\n\t// Check ServerInfo key was added to the set all server keys correctly.\n\tgotServerKeys := r.client.ZRange(context.Background(), base.AllServers, 0, -1).Val()\n\twantServerKeys := []string{skey}\n\tif diff := cmp.Diff(wantServerKeys, gotServerKeys); diff != \"\" {\n\t\tt.Errorf(\"%q contained %v, want %v\", base.AllServers, gotServerKeys, wantServerKeys)\n\t}\n\n\t// Check WorkersInfo was written correctly.\n\twkey := base.WorkersKey(host, pid, serverID)\n\tworkerExist := r.client.Exists(context.Background(), wkey).Val()\n\tif workerExist != 0 {\n\t\tt.Errorf(\"%q key exists\", wkey)\n\t}\n\t// Check WorkersInfo key was added to the set correctly.\n\tgotWorkerKeys := r.client.ZRange(context.Background(), base.AllWorkers, 0, -1).Val()\n\twantWorkerKeys := []string{wkey}\n\tif diff := cmp.Diff(wantWorkerKeys, gotWorkerKeys); diff != \"\" {\n\t\tt.Errorf(\"%q contained %v, want %v\", base.AllWorkers, gotWorkerKeys, wantWorkerKeys)\n\t}\n}\n\nfunc TestWriteServerStateWithWorkers(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tvar (\n\t\thost = \"127.0.0.1\"\n\t\tpid = 4242\n\t\tserverID = \"server123\"\n\n\t\tmsg1 = h.NewTaskMessage(\"send_email\", h.JSON(map[string]interface{}{\"user_id\": \"123\"}))\n\t\tmsg2 = h.NewTaskMessage(\"gen_thumbnail\", h.JSON(map[string]interface{}{\"path\": \"some/path/to/imgfile\"}))\n\n\t\tttl = 5 * time.Second\n\t)\n\n\tworkers := []*base.WorkerInfo{\n\t\t{\n\t\t\tHost: host,\n\t\t\tPID: pid,\n\t\t\tID: msg1.ID,\n\t\t\tType: msg1.Type,\n\t\t\tQueue: msg1.Queue,\n\t\t\tPayload: msg1.Payload,\n\t\t\tStarted: time.Now().Add(-10 * time.Second),\n\t\t},\n\t\t{\n\t\t\tHost: host,\n\t\t\tPID: pid,\n\t\t\tID: msg2.ID,\n\t\t\tType: msg2.Type,\n\t\t\tQueue: msg2.Queue,\n\t\t\tPayload: msg2.Payload,\n\t\t\tStarted: time.Now().Add(-2 * time.Minute),\n\t\t},\n\t}\n\n\tserverInfo := base.ServerInfo{\n\t\tHost: host,\n\t\tPID: pid,\n\t\tServerID: serverID,\n\t\tConcurrency: 10,\n\t\tQueues: map[string]int{\"default\": 2, \"email\": 5, \"low\": 1},\n\t\tStrictPriority: false,\n\t\tStarted: time.Now().Add(-10 * time.Minute).UTC(),\n\t\tStatus: \"active\",\n\t\tActiveWorkerCount: len(workers),\n\t}\n\n\terr := r.WriteServerState(&serverInfo, workers, ttl)\n\tif err != nil {\n\t\tt.Fatalf(\"r.WriteServerState returned an error: %v\", err)\n\t}\n\n\t// Check ServerInfo was written correctly.\n\tskey := base.ServerInfoKey(host, pid, serverID)\n\tdata := r.client.Get(context.Background(), skey).Val()\n\tgot, err := base.DecodeServerInfo([]byte(data))\n\tif err != nil {\n\t\tt.Fatalf(\"could not decode server info: %v\", err)\n\t}\n\tif diff := cmp.Diff(serverInfo, *got); diff != \"\" {\n\t\tt.Errorf(\"persisted ServerInfo was %v, want %v; (-want,+got)\\n%s\",\n\t\t\tgot, serverInfo, diff)\n\t}\n\t// Check ServerInfo TTL was set correctly.\n\tgotTTL := r.client.TTL(context.Background(), skey).Val()\n\tif !cmp.Equal(ttl.Seconds(), gotTTL.Seconds(), cmpopts.EquateApprox(0, 1)) {\n\t\tt.Errorf(\"TTL of %q was %v, want %v\", skey, gotTTL, ttl)\n\t}\n\t// Check ServerInfo key was added to the set correctly.\n\tgotServerKeys := r.client.ZRange(context.Background(), base.AllServers, 0, -1).Val()\n\twantServerKeys := []string{skey}\n\tif diff := cmp.Diff(wantServerKeys, gotServerKeys); diff != \"\" {\n\t\tt.Errorf(\"%q contained %v, want %v\", base.AllServers, gotServerKeys, wantServerKeys)\n\t}\n\n\t// Check WorkersInfo was written correctly.\n\twkey := base.WorkersKey(host, pid, serverID)\n\twdata := r.client.HGetAll(context.Background(), wkey).Val()\n\tif len(wdata) != 2 {\n\t\tt.Fatalf(\"HGETALL %q returned a hash of size %d, want 2\", wkey, len(wdata))\n\t}\n\tvar gotWorkers []*base.WorkerInfo\n\tfor _, val := range wdata {\n\t\tw, err := base.DecodeWorkerInfo([]byte(val))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"could not unmarshal worker's data: %v\", err)\n\t\t}\n\t\tgotWorkers = append(gotWorkers, w)\n\t}\n\tif diff := cmp.Diff(workers, gotWorkers, h.SortWorkerInfoOpt); diff != \"\" {\n\t\tt.Errorf(\"persisted workers info was %v, want %v; (-want,+got)\\n%s\",\n\t\t\tgotWorkers, workers, diff)\n\t}\n\n\t// Check WorkersInfo TTL was set correctly.\n\tgotTTL = r.client.TTL(context.Background(), wkey).Val()\n\tif !cmp.Equal(ttl.Seconds(), gotTTL.Seconds(), cmpopts.EquateApprox(0, 1)) {\n\t\tt.Errorf(\"TTL of %q was %v, want %v\", wkey, gotTTL, ttl)\n\t}\n\t// Check WorkersInfo key was added to the set correctly.\n\tgotWorkerKeys := r.client.ZRange(context.Background(), base.AllWorkers, 0, -1).Val()\n\twantWorkerKeys := []string{wkey}\n\tif diff := cmp.Diff(wantWorkerKeys, gotWorkerKeys); diff != \"\" {\n\t\tt.Errorf(\"%q contained %v, want %v\", base.AllWorkers, gotWorkerKeys, wantWorkerKeys)\n\t}\n}\n\nfunc TestClearServerState(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tvar (\n\t\thost = \"127.0.0.1\"\n\t\tpid = 1234\n\t\tserverID = \"server123\"\n\n\t\totherHost = \"127.0.0.2\"\n\t\totherPID = 9876\n\t\totherServerID = \"server987\"\n\n\t\tmsg1 = h.NewTaskMessage(\"send_email\", h.JSON(map[string]interface{}{\"user_id\": \"123\"}))\n\t\tmsg2 = h.NewTaskMessage(\"gen_thumbnail\", h.JSON(map[string]interface{}{\"path\": \"some/path/to/imgfile\"}))\n\n\t\tttl = 5 * time.Second\n\t)\n\n\tworkers1 := []*base.WorkerInfo{\n\t\t{\n\t\t\tHost: host,\n\t\t\tPID: pid,\n\t\t\tID: msg1.ID,\n\t\t\tType: msg1.Type,\n\t\t\tQueue: msg1.Queue,\n\t\t\tPayload: msg1.Payload,\n\t\t\tStarted: time.Now().Add(-10 * time.Second),\n\t\t},\n\t}\n\tserverInfo1 := base.ServerInfo{\n\t\tHost: host,\n\t\tPID: pid,\n\t\tServerID: serverID,\n\t\tConcurrency: 10,\n\t\tQueues: map[string]int{\"default\": 2, \"email\": 5, \"low\": 1},\n\t\tStrictPriority: false,\n\t\tStarted: time.Now().Add(-10 * time.Minute),\n\t\tStatus: \"active\",\n\t\tActiveWorkerCount: len(workers1),\n\t}\n\n\tworkers2 := []*base.WorkerInfo{\n\t\t{\n\t\t\tHost: otherHost,\n\t\t\tPID: otherPID,\n\t\t\tID: msg2.ID,\n\t\t\tType: msg2.Type,\n\t\t\tQueue: msg2.Queue,\n\t\t\tPayload: msg2.Payload,\n\t\t\tStarted: time.Now().Add(-30 * time.Second),\n\t\t},\n\t}\n\tserverInfo2 := base.ServerInfo{\n\t\tHost: otherHost,\n\t\tPID: otherPID,\n\t\tServerID: otherServerID,\n\t\tConcurrency: 10,\n\t\tQueues: map[string]int{\"default\": 2, \"email\": 5, \"low\": 1},\n\t\tStrictPriority: false,\n\t\tStarted: time.Now().Add(-15 * time.Minute),\n\t\tStatus: \"active\",\n\t\tActiveWorkerCount: len(workers2),\n\t}\n\n\t// Write server and workers data.\n\tif err := r.WriteServerState(&serverInfo1, workers1, ttl); err != nil {\n\t\tt.Fatalf(\"could not write server state: %v\", err)\n\t}\n\tif err := r.WriteServerState(&serverInfo2, workers2, ttl); err != nil {\n\t\tt.Fatalf(\"could not write server state: %v\", err)\n\t}\n\n\terr := r.ClearServerState(host, pid, serverID)\n\tif err != nil {\n\t\tt.Fatalf(\"(*RDB).ClearServerState failed: %v\", err)\n\t}\n\n\tskey := base.ServerInfoKey(host, pid, serverID)\n\twkey := base.WorkersKey(host, pid, serverID)\n\totherSKey := base.ServerInfoKey(otherHost, otherPID, otherServerID)\n\totherWKey := base.WorkersKey(otherHost, otherPID, otherServerID)\n\t// Check all keys are cleared.\n\tif r.client.Exists(context.Background(), skey).Val() != 0 {\n\t\tt.Errorf(\"Redis key %q exists\", skey)\n\t}\n\tif r.client.Exists(context.Background(), wkey).Val() != 0 {\n\t\tt.Errorf(\"Redis key %q exists\", wkey)\n\t}\n\tgotServerKeys := r.client.ZRange(context.Background(), base.AllServers, 0, -1).Val()\n\twantServerKeys := []string{otherSKey}\n\tif diff := cmp.Diff(wantServerKeys, gotServerKeys); diff != \"\" {\n\t\tt.Errorf(\"%q contained %v, want %v\", base.AllServers, gotServerKeys, wantServerKeys)\n\t}\n\tgotWorkerKeys := r.client.ZRange(context.Background(), base.AllWorkers, 0, -1).Val()\n\twantWorkerKeys := []string{otherWKey}\n\tif diff := cmp.Diff(wantWorkerKeys, gotWorkerKeys); diff != \"\" {\n\t\tt.Errorf(\"%q contained %v, want %v\", base.AllWorkers, gotWorkerKeys, wantWorkerKeys)\n\t}\n}\n\nfunc TestCancelationPubSub(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tpubsub, err := r.CancelationPubSub()\n\tif err != nil {\n\t\tt.Fatalf(\"(*RDB).CancelationPubSub() returned an error: %v\", err)\n\t}\n\n\tcancelCh := pubsub.Channel()\n\n\tvar (\n\t\tmu sync.Mutex\n\t\treceived []string\n\t)\n\n\tgo func() {\n\t\tfor msg := range cancelCh {\n\t\t\tmu.Lock()\n\t\t\treceived = append(received, msg.Payload)\n\t\t\tmu.Unlock()\n\t\t}\n\t}()\n\n\tpublish := []string{\"one\", \"two\", \"three\"}\n\n\tfor _, msg := range publish {\n\t\t_ = r.PublishCancelation(msg)\n\t}\n\n\t// allow for message to reach subscribers.\n\ttime.Sleep(time.Second)\n\n\tpubsub.Close()\n\n\tmu.Lock()\n\tif diff := cmp.Diff(publish, received, h.SortStringSliceOpt); diff != \"\" {\n\t\tt.Errorf(\"subscriber received %v, want %v; (-want,+got)\\n%s\", received, publish, diff)\n\t}\n\tmu.Unlock()\n}\n\nfunc TestWriteResult(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\ttests := []struct {\n\t\tqname string\n\t\ttaskID string\n\t\tdata []byte\n\t}{\n\t\t{\n\t\t\tqname: \"default\",\n\t\t\ttaskID: uuid.NewString(),\n\t\t\tdata: []byte(\"hello\"),\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\n\t\tn, err := r.WriteResult(tc.qname, tc.taskID, tc.data)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"WriteResult failed: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tif n != len(tc.data) {\n\t\t\tt.Errorf(\"WriteResult returned %d, want %d\", n, len(tc.data))\n\t\t}\n\n\t\ttaskKey := base.TaskKey(tc.qname, tc.taskID)\n\t\tgot := r.client.HGet(context.Background(), taskKey, \"result\").Val()\n\t\tif got != string(tc.data) {\n\t\t\tt.Errorf(\"`result` field under %q key is set to %q, want %q\", taskKey, got, string(tc.data))\n\t\t}\n\t}\n}\n\nfunc TestAggregationCheck(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tnow := time.Now()\n\tr.SetClock(timeutil.NewSimulatedClock(now))\n\n\tctx := context.Background()\n\tmsg1 := h.NewTaskMessageBuilder().SetType(\"task1\").SetGroup(\"mygroup\").Build()\n\tmsg2 := h.NewTaskMessageBuilder().SetType(\"task2\").SetGroup(\"mygroup\").Build()\n\tmsg3 := h.NewTaskMessageBuilder().SetType(\"task3\").SetGroup(\"mygroup\").Build()\n\tmsg4 := h.NewTaskMessageBuilder().SetType(\"task4\").SetGroup(\"mygroup\").Build()\n\tmsg5 := h.NewTaskMessageBuilder().SetType(\"task5\").SetGroup(\"mygroup\").Build()\n\n\ttests := []struct {\n\t\tdesc string\n\t\t// initial data\n\t\ttasks []*h.TaskSeedData\n\t\tgroups map[string][]redis.Z\n\t\tallGroups map[string][]string\n\n\t\t// args\n\t\tqname string\n\t\tgname string\n\t\tgracePeriod time.Duration\n\t\tmaxDelay time.Duration\n\t\tmaxSize int\n\n\t\t// expectaions\n\t\tshouldCreateSet bool // whether the check should create a new aggregation set\n\t\twantAggregationSet []*base.TaskMessage\n\t\twantGroups map[string][]redis.Z\n\t\tshouldClearGroup bool // whether the check should clear the group from redis\n\t}{\n\t\t{\n\t\t\tdesc: \"with an empty group\",\n\t\t\ttasks: []*h.TaskSeedData{},\n\t\t\tgroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"mygroup\"): {},\n\t\t\t},\n\t\t\tallGroups: map[string][]string{\n\t\t\t\tbase.AllGroups(\"default\"): {},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tgname: \"mygroup\",\n\t\t\tgracePeriod: 1 * time.Minute,\n\t\t\tmaxDelay: 10 * time.Minute,\n\t\t\tmaxSize: 5,\n\t\t\tshouldCreateSet: false,\n\t\t\twantAggregationSet: nil,\n\t\t\twantGroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"mygroup\"): {},\n\t\t\t},\n\t\t\tshouldClearGroup: true,\n\t\t},\n\t\t{\n\t\t\tdesc: \"with a group size reaching the max size\",\n\t\t\ttasks: []*h.TaskSeedData{\n\t\t\t\t{Msg: msg1, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: msg2, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: msg3, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: msg4, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: msg5, State: base.TaskStateAggregating},\n\t\t\t},\n\t\t\tgroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"mygroup\"): {\n\t\t\t\t\t{Member: msg1.ID, Score: float64(now.Add(-5 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg2.ID, Score: float64(now.Add(-3 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg3.ID, Score: float64(now.Add(-2 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg4.ID, Score: float64(now.Add(-1 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg5.ID, Score: float64(now.Add(-10 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\tallGroups: map[string][]string{\n\t\t\t\tbase.AllGroups(\"default\"): {\"mygroup\"},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tgname: \"mygroup\",\n\t\t\tgracePeriod: 1 * time.Minute,\n\t\t\tmaxDelay: 10 * time.Minute,\n\t\t\tmaxSize: 5,\n\t\t\tshouldCreateSet: true,\n\t\t\twantAggregationSet: []*base.TaskMessage{msg1, msg2, msg3, msg4, msg5},\n\t\t\twantGroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"mygroup\"): {},\n\t\t\t},\n\t\t\tshouldClearGroup: true,\n\t\t},\n\t\t{\n\t\t\tdesc: \"with group size greater than max size\",\n\t\t\ttasks: []*h.TaskSeedData{\n\t\t\t\t{Msg: msg1, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: msg2, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: msg3, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: msg4, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: msg5, State: base.TaskStateAggregating},\n\t\t\t},\n\t\t\tgroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"mygroup\"): {\n\t\t\t\t\t{Member: msg1.ID, Score: float64(now.Add(-5 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg2.ID, Score: float64(now.Add(-3 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg3.ID, Score: float64(now.Add(-2 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg4.ID, Score: float64(now.Add(-1 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg5.ID, Score: float64(now.Add(-10 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\tallGroups: map[string][]string{\n\t\t\t\tbase.AllGroups(\"default\"): {\"mygroup\"},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tgname: \"mygroup\",\n\t\t\tgracePeriod: 2 * time.Minute,\n\t\t\tmaxDelay: 10 * time.Minute,\n\t\t\tmaxSize: 3,\n\t\t\tshouldCreateSet: true,\n\t\t\twantAggregationSet: []*base.TaskMessage{msg1, msg2, msg3},\n\t\t\twantGroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"mygroup\"): {\n\t\t\t\t\t{Member: msg4.ID, Score: float64(now.Add(-1 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg5.ID, Score: float64(now.Add(-10 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\tshouldClearGroup: false,\n\t\t},\n\t\t{\n\t\t\tdesc: \"with the most recent task older than grace period\",\n\t\t\ttasks: []*h.TaskSeedData{\n\t\t\t\t{Msg: msg1, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: msg2, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: msg3, State: base.TaskStateAggregating},\n\t\t\t},\n\t\t\tgroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"mygroup\"): {\n\t\t\t\t\t{Member: msg1.ID, Score: float64(now.Add(-5 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg2.ID, Score: float64(now.Add(-3 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg3.ID, Score: float64(now.Add(-2 * time.Minute).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\tallGroups: map[string][]string{\n\t\t\t\tbase.AllGroups(\"default\"): {\"mygroup\"},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tgname: \"mygroup\",\n\t\t\tgracePeriod: 1 * time.Minute,\n\t\t\tmaxDelay: 10 * time.Minute,\n\t\t\tmaxSize: 5,\n\t\t\tshouldCreateSet: true,\n\t\t\twantAggregationSet: []*base.TaskMessage{msg1, msg2, msg3},\n\t\t\twantGroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"mygroup\"): {},\n\t\t\t},\n\t\t\tshouldClearGroup: true,\n\t\t},\n\t\t{\n\t\t\tdesc: \"with the oldest task older than max delay\",\n\t\t\ttasks: []*h.TaskSeedData{\n\t\t\t\t{Msg: msg1, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: msg2, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: msg3, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: msg4, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: msg5, State: base.TaskStateAggregating},\n\t\t\t},\n\t\t\tgroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"mygroup\"): {\n\t\t\t\t\t{Member: msg1.ID, Score: float64(now.Add(-15 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg2.ID, Score: float64(now.Add(-3 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg3.ID, Score: float64(now.Add(-2 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg4.ID, Score: float64(now.Add(-1 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg5.ID, Score: float64(now.Add(-10 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\tallGroups: map[string][]string{\n\t\t\t\tbase.AllGroups(\"default\"): {\"mygroup\"},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tgname: \"mygroup\",\n\t\t\tgracePeriod: 2 * time.Minute,\n\t\t\tmaxDelay: 10 * time.Minute,\n\t\t\tmaxSize: 30,\n\t\t\tshouldCreateSet: true,\n\t\t\twantAggregationSet: []*base.TaskMessage{msg1, msg2, msg3, msg4, msg5},\n\t\t\twantGroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"mygroup\"): {},\n\t\t\t},\n\t\t\tshouldClearGroup: true,\n\t\t},\n\t\t{\n\t\t\tdesc: \"with unlimited size\",\n\t\t\ttasks: []*h.TaskSeedData{\n\t\t\t\t{Msg: msg1, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: msg2, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: msg3, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: msg4, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: msg5, State: base.TaskStateAggregating},\n\t\t\t},\n\t\t\tgroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"mygroup\"): {\n\t\t\t\t\t{Member: msg1.ID, Score: float64(now.Add(-15 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg2.ID, Score: float64(now.Add(-3 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg3.ID, Score: float64(now.Add(-2 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg4.ID, Score: float64(now.Add(-1 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg5.ID, Score: float64(now.Add(-10 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\tallGroups: map[string][]string{\n\t\t\t\tbase.AllGroups(\"default\"): {\"mygroup\"},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tgname: \"mygroup\",\n\t\t\tgracePeriod: 1 * time.Minute,\n\t\t\tmaxDelay: 30 * time.Minute,\n\t\t\tmaxSize: 0, // maxSize=0 indicates no size limit\n\t\t\tshouldCreateSet: false,\n\t\t\twantAggregationSet: nil,\n\t\t\twantGroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"mygroup\"): {\n\t\t\t\t\t{Member: msg1.ID, Score: float64(now.Add(-15 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg2.ID, Score: float64(now.Add(-3 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg3.ID, Score: float64(now.Add(-2 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg4.ID, Score: float64(now.Add(-1 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg5.ID, Score: float64(now.Add(-10 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\tshouldClearGroup: false,\n\t\t},\n\t\t{\n\t\t\tdesc: \"with unlimited size and passed grace period\",\n\t\t\ttasks: []*h.TaskSeedData{\n\t\t\t\t{Msg: msg1, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: msg2, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: msg3, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: msg4, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: msg5, State: base.TaskStateAggregating},\n\t\t\t},\n\t\t\tgroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"mygroup\"): {\n\t\t\t\t\t{Member: msg1.ID, Score: float64(now.Add(-15 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg2.ID, Score: float64(now.Add(-3 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg3.ID, Score: float64(now.Add(-2 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg4.ID, Score: float64(now.Add(-1 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg5.ID, Score: float64(now.Add(-1 * time.Minute).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\tallGroups: map[string][]string{\n\t\t\t\tbase.AllGroups(\"default\"): {\"mygroup\"},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tgname: \"mygroup\",\n\t\t\tgracePeriod: 30 * time.Second,\n\t\t\tmaxDelay: 30 * time.Minute,\n\t\t\tmaxSize: 0, // maxSize=0 indicates no size limit\n\t\t\tshouldCreateSet: true,\n\t\t\twantAggregationSet: []*base.TaskMessage{msg1, msg2, msg3, msg4, msg5},\n\t\t\twantGroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"mygroup\"): {},\n\t\t\t},\n\t\t\tshouldClearGroup: true,\n\t\t},\n\t\t{\n\t\t\tdesc: \"with unlimited delay\",\n\t\t\ttasks: []*h.TaskSeedData{\n\t\t\t\t{Msg: msg1, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: msg2, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: msg3, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: msg4, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: msg5, State: base.TaskStateAggregating},\n\t\t\t},\n\t\t\tgroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"mygroup\"): {\n\t\t\t\t\t{Member: msg1.ID, Score: float64(now.Add(-15 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg2.ID, Score: float64(now.Add(-3 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg3.ID, Score: float64(now.Add(-2 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg4.ID, Score: float64(now.Add(-1 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg5.ID, Score: float64(now.Add(-10 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\tallGroups: map[string][]string{\n\t\t\t\tbase.AllGroups(\"default\"): {\"mygroup\"},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\tgname: \"mygroup\",\n\t\t\tgracePeriod: 1 * time.Minute,\n\t\t\tmaxDelay: 0, // maxDelay=0 indicates no limit\n\t\t\tmaxSize: 10,\n\t\t\tshouldCreateSet: false,\n\t\t\twantAggregationSet: nil,\n\t\t\twantGroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"mygroup\"): {\n\t\t\t\t\t{Member: msg1.ID, Score: float64(now.Add(-15 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg2.ID, Score: float64(now.Add(-3 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg3.ID, Score: float64(now.Add(-2 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg4.ID, Score: float64(now.Add(-1 * time.Minute).Unix())},\n\t\t\t\t\t{Member: msg5.ID, Score: float64(now.Add(-10 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\tshouldClearGroup: false,\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\n\t\tt.Run(tc.desc, func(t *testing.T) {\n\t\t\th.SeedTasks(t, r.client, tc.tasks)\n\t\t\th.SeedRedisZSets(t, r.client, tc.groups)\n\t\t\th.SeedRedisSets(t, r.client, tc.allGroups)\n\n\t\t\taggregationSetID, err := r.AggregationCheck(tc.qname, tc.gname, now, tc.gracePeriod, tc.maxDelay, tc.maxSize)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"AggregationCheck returned error: %v\", err)\n\t\t\t}\n\n\t\t\tif !tc.shouldCreateSet && aggregationSetID != \"\" {\n\t\t\t\tt.Fatal(\"AggregationCheck returned non empty set ID. want empty ID\")\n\t\t\t}\n\t\t\tif tc.shouldCreateSet && aggregationSetID == \"\" {\n\t\t\t\tt.Fatal(\"AggregationCheck returned empty set ID. want non empty ID\")\n\t\t\t}\n\n\t\t\tif tc.shouldCreateSet {\n\t\t\t\tmsgs, deadline, err := r.ReadAggregationSet(tc.qname, tc.gname, aggregationSetID)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"Failed to read aggregation set %q: %v\", aggregationSetID, err)\n\t\t\t\t}\n\t\t\t\tif diff := cmp.Diff(tc.wantAggregationSet, msgs, h.SortMsgOpt); diff != \"\" {\n\t\t\t\t\tt.Errorf(\"Mismatch found in aggregation set: (-want,+got)\\n%s\", diff)\n\t\t\t\t}\n\n\t\t\t\tif wantDeadline := now.Add(aggregationTimeout); deadline.Unix() != wantDeadline.Unix() {\n\t\t\t\t\tt.Errorf(\"ReadAggregationSet returned deadline=%v, want=%v\", deadline, wantDeadline)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\th.AssertRedisZSets(t, r.client, tc.wantGroups)\n\n\t\t\tif tc.shouldClearGroup {\n\t\t\t\tif key := base.GroupKey(tc.qname, tc.gname); r.client.Exists(ctx, key).Val() != 0 {\n\t\t\t\t\tt.Errorf(\"group key %q still exists\", key)\n\t\t\t\t}\n\t\t\t\tif r.client.SIsMember(ctx, base.AllGroups(tc.qname), tc.gname).Val() {\n\t\t\t\t\tt.Errorf(\"all-group set %q still contains the group name %q\", base.AllGroups(tc.qname), tc.gname)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif key := base.GroupKey(tc.qname, tc.gname); r.client.Exists(ctx, key).Val() == 0 {\n\t\t\t\t\tt.Errorf(\"group key %q does not exists\", key)\n\t\t\t\t}\n\t\t\t\tif !r.client.SIsMember(ctx, base.AllGroups(tc.qname), tc.gname).Val() {\n\t\t\t\t\tt.Errorf(\"all-group set %q doesn't contains the group name %q\", base.AllGroups(tc.qname), tc.gname)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestDeleteAggregationSet(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tnow := time.Now()\n\tsetID := uuid.NewString()\n\totherSetID := uuid.NewString()\n\tm1 := h.NewTaskMessageBuilder().SetQueue(\"default\").SetGroup(\"mygroup\").Build()\n\tm2 := h.NewTaskMessageBuilder().SetQueue(\"default\").SetGroup(\"mygroup\").Build()\n\tm3 := h.NewTaskMessageBuilder().SetQueue(\"default\").SetGroup(\"mygroup\").Build()\n\n\ttests := []struct {\n\t\tdesc string\n\t\t// initial data\n\t\ttasks []*h.TaskSeedData\n\t\taggregationSets map[string][]redis.Z\n\t\tallAggregationSets map[string][]redis.Z\n\n\t\t// args\n\t\tctx context.Context\n\t\tqname string\n\t\tgname string\n\t\tsetID string\n\n\t\t// expectations\n\t\twantDeletedKeys []string // redis key to check for non existence\n\t\twantAggregationSets map[string][]redis.Z\n\t\twantAllAggregationSets map[string][]redis.Z\n\t}{\n\t\t{\n\t\t\tdesc: \"with a sigle active aggregation set\",\n\t\t\ttasks: []*h.TaskSeedData{\n\t\t\t\t{Msg: m1, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: m2, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: m3, State: base.TaskStateAggregating},\n\t\t\t},\n\t\t\taggregationSets: map[string][]redis.Z{\n\t\t\t\tbase.AggregationSetKey(\"default\", \"mygroup\", setID): {\n\t\t\t\t\t{Member: m1.ID, Score: float64(now.Add(-5 * time.Minute).Unix())},\n\t\t\t\t\t{Member: m2.ID, Score: float64(now.Add(-4 * time.Minute).Unix())},\n\t\t\t\t\t{Member: m3.ID, Score: float64(now.Add(-3 * time.Minute).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\tallAggregationSets: map[string][]redis.Z{\n\t\t\t\tbase.AllAggregationSets(\"default\"): {\n\t\t\t\t\t{Member: base.AggregationSetKey(\"default\", \"mygroup\", setID), Score: float64(now.Add(aggregationTimeout).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\tctx: context.Background(),\n\t\t\tqname: \"default\",\n\t\t\tgname: \"mygroup\",\n\t\t\tsetID: setID,\n\t\t\twantDeletedKeys: []string{\n\t\t\t\tbase.AggregationSetKey(\"default\", \"mygroup\", setID),\n\t\t\t\tbase.TaskKey(m1.Queue, m1.ID),\n\t\t\t\tbase.TaskKey(m2.Queue, m2.ID),\n\t\t\t\tbase.TaskKey(m3.Queue, m3.ID),\n\t\t\t},\n\t\t\twantAggregationSets: map[string][]redis.Z{},\n\t\t\twantAllAggregationSets: map[string][]redis.Z{\n\t\t\t\tbase.AllAggregationSets(\"default\"): {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with multiple active aggregation sets\",\n\t\t\ttasks: []*h.TaskSeedData{\n\t\t\t\t{Msg: m1, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: m2, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: m3, State: base.TaskStateAggregating},\n\t\t\t},\n\t\t\taggregationSets: map[string][]redis.Z{\n\t\t\t\tbase.AggregationSetKey(\"default\", \"mygroup\", setID): {\n\t\t\t\t\t{Member: m1.ID, Score: float64(now.Add(-5 * time.Minute).Unix())},\n\t\t\t\t},\n\t\t\t\tbase.AggregationSetKey(\"default\", \"mygroup\", otherSetID): {\n\t\t\t\t\t{Member: m2.ID, Score: float64(now.Add(-4 * time.Minute).Unix())},\n\t\t\t\t\t{Member: m3.ID, Score: float64(now.Add(-3 * time.Minute).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\tallAggregationSets: map[string][]redis.Z{\n\t\t\t\tbase.AllAggregationSets(\"default\"): {\n\t\t\t\t\t{Member: base.AggregationSetKey(\"default\", \"mygroup\", setID), Score: float64(now.Add(aggregationTimeout).Unix())},\n\t\t\t\t\t{Member: base.AggregationSetKey(\"default\", \"mygroup\", otherSetID), Score: float64(now.Add(aggregationTimeout).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\tctx: context.Background(),\n\t\t\tqname: \"default\",\n\t\t\tgname: \"mygroup\",\n\t\t\tsetID: setID,\n\t\t\twantDeletedKeys: []string{\n\t\t\t\tbase.AggregationSetKey(\"default\", \"mygroup\", setID),\n\t\t\t\tbase.TaskKey(m1.Queue, m1.ID),\n\t\t\t},\n\t\t\twantAggregationSets: map[string][]redis.Z{\n\t\t\t\tbase.AggregationSetKey(\"default\", \"mygroup\", otherSetID): {\n\t\t\t\t\t{Member: m2.ID, Score: float64(now.Add(-4 * time.Minute).Unix())},\n\t\t\t\t\t{Member: m3.ID, Score: float64(now.Add(-3 * time.Minute).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\twantAllAggregationSets: map[string][]redis.Z{\n\t\t\t\tbase.AllAggregationSets(\"default\"): {\n\t\t\t\t\t{Member: base.AggregationSetKey(\"default\", \"mygroup\", otherSetID), Score: float64(now.Add(aggregationTimeout).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\tt.Run(tc.desc, func(t *testing.T) {\n\t\t\th.SeedTasks(t, r.client, tc.tasks)\n\t\t\th.SeedRedisZSets(t, r.client, tc.aggregationSets)\n\t\t\th.SeedRedisZSets(t, r.client, tc.allAggregationSets)\n\n\t\t\tif err := r.DeleteAggregationSet(tc.ctx, tc.qname, tc.gname, tc.setID); err != nil {\n\t\t\t\tt.Fatalf(\"DeleteAggregationSet returned error: %v\", err)\n\t\t\t}\n\n\t\t\tfor _, key := range tc.wantDeletedKeys {\n\t\t\t\tif r.client.Exists(context.Background(), key).Val() != 0 {\n\t\t\t\t\tt.Errorf(\"key=%q still exists, want deleted\", key)\n\t\t\t\t}\n\t\t\t}\n\t\t\th.AssertRedisZSets(t, r.client, tc.wantAllAggregationSets)\n\t\t})\n\t}\n}\n\nfunc TestDeleteAggregationSetError(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tnow := time.Now()\n\tsetID := uuid.NewString()\n\tm1 := h.NewTaskMessageBuilder().SetQueue(\"default\").SetGroup(\"mygroup\").Build()\n\tm2 := h.NewTaskMessageBuilder().SetQueue(\"default\").SetGroup(\"mygroup\").Build()\n\tm3 := h.NewTaskMessageBuilder().SetQueue(\"default\").SetGroup(\"mygroup\").Build()\n\tdeadlineExceededCtx, cancel := context.WithDeadline(context.Background(), now.Add(-10*time.Second))\n\tdefer cancel()\n\n\ttests := []struct {\n\t\tdesc string\n\t\t// initial data\n\t\ttasks []*h.TaskSeedData\n\t\taggregationSets map[string][]redis.Z\n\t\tallAggregationSets map[string][]redis.Z\n\n\t\t// args\n\t\tctx context.Context\n\t\tqname string\n\t\tgname string\n\t\tsetID string\n\n\t\t// expectations\n\t\twantAggregationSets map[string][]redis.Z\n\t\twantAllAggregationSets map[string][]redis.Z\n\t}{\n\t\t{\n\t\t\tdesc: \"with deadline exceeded context\",\n\t\t\ttasks: []*h.TaskSeedData{\n\t\t\t\t{Msg: m1, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: m2, State: base.TaskStateAggregating},\n\t\t\t\t{Msg: m3, State: base.TaskStateAggregating},\n\t\t\t},\n\t\t\taggregationSets: map[string][]redis.Z{\n\t\t\t\tbase.AggregationSetKey(\"default\", \"mygroup\", setID): {\n\t\t\t\t\t{Member: m1.ID, Score: float64(now.Add(-5 * time.Minute).Unix())},\n\t\t\t\t\t{Member: m2.ID, Score: float64(now.Add(-4 * time.Minute).Unix())},\n\t\t\t\t\t{Member: m3.ID, Score: float64(now.Add(-3 * time.Minute).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\tallAggregationSets: map[string][]redis.Z{\n\t\t\t\tbase.AllAggregationSets(\"default\"): {\n\t\t\t\t\t{Member: base.AggregationSetKey(\"default\", \"mygroup\", setID), Score: float64(now.Add(aggregationTimeout).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\tctx: deadlineExceededCtx,\n\t\t\tqname: \"default\",\n\t\t\tgname: \"mygroup\",\n\t\t\tsetID: setID,\n\t\t\t// want data unchanged.\n\t\t\twantAggregationSets: map[string][]redis.Z{\n\t\t\t\tbase.AggregationSetKey(\"default\", \"mygroup\", setID): {\n\t\t\t\t\t{Member: m1.ID, Score: float64(now.Add(-5 * time.Minute).Unix())},\n\t\t\t\t\t{Member: m2.ID, Score: float64(now.Add(-4 * time.Minute).Unix())},\n\t\t\t\t\t{Member: m3.ID, Score: float64(now.Add(-3 * time.Minute).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\t// want data unchanged.\n\t\t\twantAllAggregationSets: map[string][]redis.Z{\n\t\t\t\tbase.AllAggregationSets(\"default\"): {\n\t\t\t\t\t{Member: base.AggregationSetKey(\"default\", \"mygroup\", setID), Score: float64(now.Add(aggregationTimeout).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\tt.Run(tc.desc, func(t *testing.T) {\n\t\t\th.SeedTasks(t, r.client, tc.tasks)\n\t\t\th.SeedRedisZSets(t, r.client, tc.aggregationSets)\n\t\t\th.SeedRedisZSets(t, r.client, tc.allAggregationSets)\n\n\t\t\tif err := r.DeleteAggregationSet(tc.ctx, tc.qname, tc.gname, tc.setID); err == nil {\n\t\t\t\tt.Fatal(\"DeleteAggregationSet returned nil, want non-nil error\")\n\t\t\t}\n\n\t\t\t// Make sure zsets are unchanged.\n\t\t\th.AssertRedisZSets(t, r.client, tc.wantAggregationSets)\n\t\t\th.AssertRedisZSets(t, r.client, tc.wantAllAggregationSets)\n\t\t})\n\t}\n}\n\nfunc TestReclaimStaleAggregationSets(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tnow := time.Now()\n\tr.SetClock(timeutil.NewSimulatedClock(now))\n\n\tm1 := h.NewTaskMessageBuilder().SetQueue(\"default\").SetGroup(\"foo\").Build()\n\tm2 := h.NewTaskMessageBuilder().SetQueue(\"default\").SetGroup(\"foo\").Build()\n\tm3 := h.NewTaskMessageBuilder().SetQueue(\"default\").SetGroup(\"bar\").Build()\n\tm4 := h.NewTaskMessageBuilder().SetQueue(\"default\").SetGroup(\"qux\").Build()\n\n\t// Note: In this test, we're trying out a new way to test RDB by exactly describing how\n\t// keys and values are represented in Redis.\n\ttests := []struct {\n\t\tgroups map[string][]redis.Z // map redis-key to redis-zset\n\t\taggregationSets map[string][]redis.Z\n\t\tallAggregationSets map[string][]redis.Z\n\t\tqname string\n\t\twantGroups map[string][]redis.Z\n\t\twantAggregationSets map[string][]redis.Z\n\t\twantAllAggregationSets map[string][]redis.Z\n\t}{\n\t\t{\n\t\t\tgroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"foo\"): {},\n\t\t\t\tbase.GroupKey(\"default\", \"bar\"): {},\n\t\t\t\tbase.GroupKey(\"default\", \"qux\"): {\n\t\t\t\t\t{Member: m4.ID, Score: float64(now.Add(-10 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\taggregationSets: map[string][]redis.Z{\n\t\t\t\tbase.AggregationSetKey(\"default\", \"foo\", \"set1\"): {\n\t\t\t\t\t{Member: m1.ID, Score: float64(now.Add(-3 * time.Minute).Unix())},\n\t\t\t\t\t{Member: m2.ID, Score: float64(now.Add(-4 * time.Minute).Unix())},\n\t\t\t\t},\n\t\t\t\tbase.AggregationSetKey(\"default\", \"bar\", \"set2\"): {\n\t\t\t\t\t{Member: m3.ID, Score: float64(now.Add(-1 * time.Minute).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\tallAggregationSets: map[string][]redis.Z{\n\t\t\t\tbase.AllAggregationSets(\"default\"): {\n\t\t\t\t\t{Member: base.AggregationSetKey(\"default\", \"foo\", \"set1\"), Score: float64(now.Add(-10 * time.Second).Unix())}, // set1 is expired\n\t\t\t\t\t{Member: base.AggregationSetKey(\"default\", \"bar\", \"set2\"), Score: float64(now.Add(40 * time.Second).Unix())}, // set2 is not expired\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twantGroups: map[string][]redis.Z{\n\t\t\t\tbase.GroupKey(\"default\", \"foo\"): {\n\t\t\t\t\t{Member: m1.ID, Score: float64(now.Add(-3 * time.Minute).Unix())},\n\t\t\t\t\t{Member: m2.ID, Score: float64(now.Add(-4 * time.Minute).Unix())},\n\t\t\t\t},\n\t\t\t\tbase.GroupKey(\"default\", \"bar\"): {},\n\t\t\t\tbase.GroupKey(\"default\", \"qux\"): {\n\t\t\t\t\t{Member: m4.ID, Score: float64(now.Add(-10 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\twantAggregationSets: map[string][]redis.Z{\n\t\t\t\tbase.AggregationSetKey(\"default\", \"bar\", \"set2\"): {\n\t\t\t\t\t{Member: m3.ID, Score: float64(now.Add(-1 * time.Minute).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t\twantAllAggregationSets: map[string][]redis.Z{\n\t\t\t\tbase.AllAggregationSets(\"default\"): {\n\t\t\t\t\t{Member: base.AggregationSetKey(\"default\", \"bar\", \"set2\"), Score: float64(now.Add(40 * time.Second).Unix())},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\th.SeedRedisZSets(t, r.client, tc.groups)\n\t\th.SeedRedisZSets(t, r.client, tc.aggregationSets)\n\t\th.SeedRedisZSets(t, r.client, tc.allAggregationSets)\n\n\t\tif err := r.ReclaimStaleAggregationSets(tc.qname); err != nil {\n\t\t\tt.Errorf(\"ReclaimStaleAggregationSets returned error: %v\", err)\n\t\t\tcontinue\n\t\t}\n\n\t\th.AssertRedisZSets(t, r.client, tc.wantGroups)\n\t\th.AssertRedisZSets(t, r.client, tc.wantAggregationSets)\n\t\th.AssertRedisZSets(t, r.client, tc.wantAllAggregationSets)\n\t}\n}\n\nfunc TestListGroups(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\n\tnow := time.Now()\n\tm1 := h.NewTaskMessageBuilder().SetQueue(\"default\").SetGroup(\"foo\").Build()\n\tm2 := h.NewTaskMessageBuilder().SetQueue(\"default\").SetGroup(\"bar\").Build()\n\tm3 := h.NewTaskMessageBuilder().SetQueue(\"custom\").SetGroup(\"baz\").Build()\n\tm4 := h.NewTaskMessageBuilder().SetQueue(\"custom\").SetGroup(\"qux\").Build()\n\n\ttests := []struct {\n\t\tgroups map[string]map[string][]base.Z\n\t\tqname string\n\t\twant []string\n\t}{\n\t\t{\n\t\t\tgroups: map[string]map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t\"foo\": {{Message: m1, Score: now.Add(-10 * time.Second).Unix()}},\n\t\t\t\t\t\"bar\": {{Message: m2, Score: now.Add(-10 * time.Second).Unix()}},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t\"baz\": {{Message: m3, Score: now.Add(-10 * time.Second).Unix()}},\n\t\t\t\t\t\"qux\": {{Message: m4, Score: now.Add(-10 * time.Second).Unix()}},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"default\",\n\t\t\twant: []string{\"foo\", \"bar\"},\n\t\t},\n\t\t{\n\t\t\tgroups: map[string]map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t\"foo\": {{Message: m1, Score: now.Add(-10 * time.Second).Unix()}},\n\t\t\t\t\t\"bar\": {{Message: m2, Score: now.Add(-10 * time.Second).Unix()}},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t\"baz\": {{Message: m3, Score: now.Add(-10 * time.Second).Unix()}},\n\t\t\t\t\t\"qux\": {{Message: m4, Score: now.Add(-10 * time.Second).Unix()}},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqname: \"custom\",\n\t\t\twant: []string{\"baz\", \"qux\"},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r.client)\n\t\th.SeedAllGroups(t, r.client, tc.groups)\n\n\t\tgot, err := r.ListGroups(tc.qname)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"ListGroups returned error: %v\", err)\n\t\t\tcontinue\n\t\t}\n\n\t\tif diff := cmp.Diff(tc.want, got, h.SortStringSliceOpt); diff != \"\" {\n\t\t\tt.Errorf(\"ListGroups=%v, want=%v; (-want,+got)\\n%s\", got, tc.want, diff)\n\t\t}\n\t}\n}\n" }, { "path": "internal/testbroker/testbroker.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\n// Package testbroker exports a broker implementation that should be used in package testing.\npackage testbroker\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/redis/go-redis/v9\"\n)\n\nvar errRedisDown = errors.New(\"testutil: redis is down\")\n\n// TestBroker is a broker implementation which enables\n// to simulate Redis failure in tests.\ntype TestBroker struct {\n\tmu sync.Mutex\n\tsleeping bool\n\n\t// real broker\n\treal base.Broker\n}\n\n// Make sure TestBroker implements Broker interface at compile time.\nvar _ base.Broker = (*TestBroker)(nil)\n\nfunc NewTestBroker(b base.Broker) *TestBroker {\n\treturn &TestBroker{real: b}\n}\n\nfunc (tb *TestBroker) Sleep() {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\ttb.sleeping = true\n}\n\nfunc (tb *TestBroker) Wakeup() {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\ttb.sleeping = false\n}\n\nfunc (tb *TestBroker) Enqueue(ctx context.Context, msg *base.TaskMessage) error {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn errRedisDown\n\t}\n\treturn tb.real.Enqueue(ctx, msg)\n}\n\nfunc (tb *TestBroker) EnqueueUnique(ctx context.Context, msg *base.TaskMessage, ttl time.Duration) error {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn errRedisDown\n\t}\n\treturn tb.real.EnqueueUnique(ctx, msg, ttl)\n}\n\nfunc (tb *TestBroker) Dequeue(qnames ...string) (*base.TaskMessage, time.Time, error) {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn nil, time.Time{}, errRedisDown\n\t}\n\treturn tb.real.Dequeue(qnames...)\n}\n\nfunc (tb *TestBroker) Done(ctx context.Context, msg *base.TaskMessage) error {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn errRedisDown\n\t}\n\treturn tb.real.Done(ctx, msg)\n}\n\nfunc (tb *TestBroker) MarkAsComplete(ctx context.Context, msg *base.TaskMessage) error {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn errRedisDown\n\t}\n\treturn tb.real.MarkAsComplete(ctx, msg)\n}\n\nfunc (tb *TestBroker) Requeue(ctx context.Context, msg *base.TaskMessage) error {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn errRedisDown\n\t}\n\treturn tb.real.Requeue(ctx, msg)\n}\n\nfunc (tb *TestBroker) Schedule(ctx context.Context, msg *base.TaskMessage, processAt time.Time) error {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn errRedisDown\n\t}\n\treturn tb.real.Schedule(ctx, msg, processAt)\n}\n\nfunc (tb *TestBroker) ScheduleUnique(ctx context.Context, msg *base.TaskMessage, processAt time.Time, ttl time.Duration) error {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn errRedisDown\n\t}\n\treturn tb.real.ScheduleUnique(ctx, msg, processAt, ttl)\n}\n\nfunc (tb *TestBroker) Retry(ctx context.Context, msg *base.TaskMessage, processAt time.Time, errMsg string, isFailure bool) error {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn errRedisDown\n\t}\n\treturn tb.real.Retry(ctx, msg, processAt, errMsg, isFailure)\n}\n\nfunc (tb *TestBroker) Archive(ctx context.Context, msg *base.TaskMessage, errMsg string) error {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn errRedisDown\n\t}\n\treturn tb.real.Archive(ctx, msg, errMsg)\n}\n\nfunc (tb *TestBroker) ForwardIfReady(qnames ...string) error {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn errRedisDown\n\t}\n\treturn tb.real.ForwardIfReady(qnames...)\n}\n\nfunc (tb *TestBroker) DeleteExpiredCompletedTasks(qname string, batchSize int) error {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn errRedisDown\n\t}\n\treturn tb.real.DeleteExpiredCompletedTasks(qname, batchSize)\n}\n\nfunc (tb *TestBroker) ListLeaseExpired(cutoff time.Time, qnames ...string) ([]*base.TaskMessage, error) {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn nil, errRedisDown\n\t}\n\treturn tb.real.ListLeaseExpired(cutoff, qnames...)\n}\n\nfunc (tb *TestBroker) ExtendLease(qname string, ids ...string) (time.Time, error) {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn time.Time{}, errRedisDown\n\t}\n\treturn tb.real.ExtendLease(qname, ids...)\n}\n\nfunc (tb *TestBroker) WriteServerState(info *base.ServerInfo, workers []*base.WorkerInfo, ttl time.Duration) error {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn errRedisDown\n\t}\n\treturn tb.real.WriteServerState(info, workers, ttl)\n}\n\nfunc (tb *TestBroker) ClearServerState(host string, pid int, serverID string) error {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn errRedisDown\n\t}\n\treturn tb.real.ClearServerState(host, pid, serverID)\n}\n\nfunc (tb *TestBroker) CancelationPubSub() (*redis.PubSub, error) {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn nil, errRedisDown\n\t}\n\treturn tb.real.CancelationPubSub()\n}\n\nfunc (tb *TestBroker) PublishCancelation(id string) error {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn errRedisDown\n\t}\n\treturn tb.real.PublishCancelation(id)\n}\n\nfunc (tb *TestBroker) WriteResult(qname, id string, data []byte) (int, error) {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn 0, errRedisDown\n\t}\n\treturn tb.real.WriteResult(qname, id, data)\n}\n\nfunc (tb *TestBroker) Ping() error {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn errRedisDown\n\t}\n\treturn tb.real.Ping()\n}\n\nfunc (tb *TestBroker) Close() error {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn errRedisDown\n\t}\n\treturn tb.real.Close()\n}\n\nfunc (tb *TestBroker) AddToGroup(ctx context.Context, msg *base.TaskMessage, gname string) error {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn errRedisDown\n\t}\n\treturn tb.real.AddToGroup(ctx, msg, gname)\n}\n\nfunc (tb *TestBroker) AddToGroupUnique(ctx context.Context, msg *base.TaskMessage, gname string, ttl time.Duration) error {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn errRedisDown\n\t}\n\treturn tb.real.AddToGroupUnique(ctx, msg, gname, ttl)\n}\n\nfunc (tb *TestBroker) ListGroups(qname string) ([]string, error) {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn nil, errRedisDown\n\t}\n\treturn tb.real.ListGroups(qname)\n}\n\nfunc (tb *TestBroker) AggregationCheck(qname, gname string, t time.Time, gracePeriod, maxDelay time.Duration, maxSize int) (aggregationSetID string, err error) {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn \"\", errRedisDown\n\t}\n\treturn tb.real.AggregationCheck(qname, gname, t, gracePeriod, maxDelay, maxSize)\n}\n\nfunc (tb *TestBroker) ReadAggregationSet(qname, gname, aggregationSetID string) ([]*base.TaskMessage, time.Time, error) {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn nil, time.Time{}, errRedisDown\n\t}\n\treturn tb.real.ReadAggregationSet(qname, gname, aggregationSetID)\n}\n\nfunc (tb *TestBroker) DeleteAggregationSet(ctx context.Context, qname, gname, aggregationSetID string) error {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn errRedisDown\n\t}\n\treturn tb.real.DeleteAggregationSet(ctx, qname, gname, aggregationSetID)\n}\n\nfunc (tb *TestBroker) ReclaimStaleAggregationSets(qname string) error {\n\ttb.mu.Lock()\n\tdefer tb.mu.Unlock()\n\tif tb.sleeping {\n\t\treturn errRedisDown\n\t}\n\treturn tb.real.ReclaimStaleAggregationSets(qname)\n}\n" }, { "path": "internal/testutil/builder.go", "content": "// Copyright 2022 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage testutil\n\nimport (\n\t\"time\"\n\n\t\"github.com/google/uuid\"\n\t\"github.com/hibiken/asynq/internal/base\"\n)\n\nfunc makeDefaultTaskMessage() *base.TaskMessage {\n\treturn &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: \"default_task\",\n\t\tQueue: \"default\",\n\t\tRetry: 25,\n\t\tTimeout: 1800, // default timeout of 30 mins\n\t\tDeadline: 0, // no deadline\n\t}\n}\n\ntype TaskMessageBuilder struct {\n\tmsg *base.TaskMessage\n}\n\nfunc NewTaskMessageBuilder() *TaskMessageBuilder {\n\treturn &TaskMessageBuilder{}\n}\n\nfunc (b *TaskMessageBuilder) lazyInit() {\n\tif b.msg == nil {\n\t\tb.msg = makeDefaultTaskMessage()\n\t}\n}\n\nfunc (b *TaskMessageBuilder) Build() *base.TaskMessage {\n\tb.lazyInit()\n\treturn b.msg\n}\n\nfunc (b *TaskMessageBuilder) SetType(typename string) *TaskMessageBuilder {\n\tb.lazyInit()\n\tb.msg.Type = typename\n\treturn b\n}\n\nfunc (b *TaskMessageBuilder) SetPayload(payload []byte) *TaskMessageBuilder {\n\tb.lazyInit()\n\tb.msg.Payload = payload\n\treturn b\n}\n\nfunc (b *TaskMessageBuilder) SetQueue(qname string) *TaskMessageBuilder {\n\tb.lazyInit()\n\tb.msg.Queue = qname\n\treturn b\n}\n\nfunc (b *TaskMessageBuilder) SetRetry(n int) *TaskMessageBuilder {\n\tb.lazyInit()\n\tb.msg.Retry = n\n\treturn b\n}\n\nfunc (b *TaskMessageBuilder) SetTimeout(timeout time.Duration) *TaskMessageBuilder {\n\tb.lazyInit()\n\tb.msg.Timeout = int64(timeout.Seconds())\n\treturn b\n}\n\nfunc (b *TaskMessageBuilder) SetDeadline(deadline time.Time) *TaskMessageBuilder {\n\tb.lazyInit()\n\tb.msg.Deadline = deadline.Unix()\n\treturn b\n}\n\nfunc (b *TaskMessageBuilder) SetGroup(gname string) *TaskMessageBuilder {\n\tb.lazyInit()\n\tb.msg.GroupKey = gname\n\treturn b\n}\n" }, { "path": "internal/testutil/builder_test.go", "content": "// Copyright 2022 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage testutil\n\nimport (\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/google/go-cmp/cmp\"\n\t\"github.com/google/go-cmp/cmp/cmpopts\"\n\t\"github.com/hibiken/asynq/internal/base\"\n)\n\nfunc TestTaskMessageBuilder(t *testing.T) {\n\ttests := []struct {\n\t\tdesc string\n\t\tops func(b *TaskMessageBuilder) // operations to perform on the builder\n\t\twant *base.TaskMessage\n\t}{\n\t\t{\n\t\t\tdesc: \"zero value and build\",\n\t\t\tops: nil,\n\t\t\twant: &base.TaskMessage{\n\t\t\t\tType: \"default_task\",\n\t\t\t\tQueue: \"default\",\n\t\t\t\tPayload: nil,\n\t\t\t\tRetry: 25,\n\t\t\t\tTimeout: 1800, // 30m\n\t\t\t\tDeadline: 0,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with type, payload, and queue\",\n\t\t\tops: func(b *TaskMessageBuilder) {\n\t\t\t\tb.SetType(\"foo\").SetPayload([]byte(\"hello\")).SetQueue(\"myqueue\")\n\t\t\t},\n\t\t\twant: &base.TaskMessage{\n\t\t\t\tType: \"foo\",\n\t\t\t\tQueue: \"myqueue\",\n\t\t\t\tPayload: []byte(\"hello\"),\n\t\t\t\tRetry: 25,\n\t\t\t\tTimeout: 1800, // 30m\n\t\t\t\tDeadline: 0,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with retry, timeout, and deadline\",\n\t\t\tops: func(b *TaskMessageBuilder) {\n\t\t\t\tb.SetRetry(1).\n\t\t\t\t\tSetTimeout(20 * time.Second).\n\t\t\t\t\tSetDeadline(time.Date(2017, 3, 6, 0, 0, 0, 0, time.UTC))\n\t\t\t},\n\t\t\twant: &base.TaskMessage{\n\t\t\t\tType: \"default_task\",\n\t\t\t\tQueue: \"default\",\n\t\t\t\tPayload: nil,\n\t\t\t\tRetry: 1,\n\t\t\t\tTimeout: 20,\n\t\t\t\tDeadline: time.Date(2017, 3, 6, 0, 0, 0, 0, time.UTC).Unix(),\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with group\",\n\t\t\tops: func(b *TaskMessageBuilder) {\n\t\t\t\tb.SetGroup(\"mygroup\")\n\t\t\t},\n\t\t\twant: &base.TaskMessage{\n\t\t\t\tType: \"default_task\",\n\t\t\t\tQueue: \"default\",\n\t\t\t\tPayload: nil,\n\t\t\t\tRetry: 25,\n\t\t\t\tTimeout: 1800,\n\t\t\t\tDeadline: 0,\n\t\t\t\tGroupKey: \"mygroup\",\n\t\t\t},\n\t\t},\n\t}\n\tcmpOpts := []cmp.Option{cmpopts.IgnoreFields(base.TaskMessage{}, \"ID\")}\n\n\tfor _, tc := range tests {\n\t\tvar b TaskMessageBuilder\n\t\tif tc.ops != nil {\n\t\t\ttc.ops(&b)\n\t\t}\n\n\t\tgot := b.Build()\n\t\tif diff := cmp.Diff(tc.want, got, cmpOpts...); diff != \"\" {\n\t\t\tt.Errorf(\"%s: TaskMessageBuilder.Build() = %+v, want %+v;\\n(-want,+got)\\n%s\",\n\t\t\t\ttc.desc, got, tc.want, diff)\n\t\t}\n\t}\n}\n" }, { "path": "internal/testutil/testutil.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\n// Package testutil defines test helpers for asynq and its internal packages.\npackage testutil\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"math\"\n\t\"sort\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/google/go-cmp/cmp\"\n\t\"github.com/google/go-cmp/cmp/cmpopts\"\n\t\"github.com/google/uuid\"\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/timeutil\"\n\t\"github.com/redis/go-redis/v9\"\n)\n\n// EquateInt64Approx returns a Comparer option that treats int64 values\n// to be equal if they are within the given margin.\nfunc EquateInt64Approx(margin int64) cmp.Option {\n\treturn cmp.Comparer(func(a, b int64) bool {\n\t\treturn math.Abs(float64(a-b)) <= float64(margin)\n\t})\n}\n\n// SortMsgOpt is a cmp.Option to sort base.TaskMessage for comparing slice of task messages.\nvar SortMsgOpt = cmp.Transformer(\"SortTaskMessages\", func(in []*base.TaskMessage) []*base.TaskMessage {\n\tout := append([]*base.TaskMessage(nil), in...) // Copy input to avoid mutating it\n\tsort.Slice(out, func(i, j int) bool {\n\t\treturn out[i].ID < out[j].ID\n\t})\n\treturn out\n})\n\n// SortZSetEntryOpt is an cmp.Option to sort ZSetEntry for comparing slice of zset entries.\nvar SortZSetEntryOpt = cmp.Transformer(\"SortZSetEntries\", func(in []base.Z) []base.Z {\n\tout := append([]base.Z(nil), in...) // Copy input to avoid mutating it\n\tsort.Slice(out, func(i, j int) bool {\n\t\treturn out[i].Message.ID < out[j].Message.ID\n\t})\n\treturn out\n})\n\n// SortServerInfoOpt is a cmp.Option to sort base.ServerInfo for comparing slice of process info.\nvar SortServerInfoOpt = cmp.Transformer(\"SortServerInfo\", func(in []*base.ServerInfo) []*base.ServerInfo {\n\tout := append([]*base.ServerInfo(nil), in...) // Copy input to avoid mutating it\n\tsort.Slice(out, func(i, j int) bool {\n\t\tif out[i].Host != out[j].Host {\n\t\t\treturn out[i].Host < out[j].Host\n\t\t}\n\t\treturn out[i].PID < out[j].PID\n\t})\n\treturn out\n})\n\n// SortWorkerInfoOpt is a cmp.Option to sort base.WorkerInfo for comparing slice of worker info.\nvar SortWorkerInfoOpt = cmp.Transformer(\"SortWorkerInfo\", func(in []*base.WorkerInfo) []*base.WorkerInfo {\n\tout := append([]*base.WorkerInfo(nil), in...) // Copy input to avoid mutating it\n\tsort.Slice(out, func(i, j int) bool {\n\t\treturn out[i].ID < out[j].ID\n\t})\n\treturn out\n})\n\n// SortSchedulerEntryOpt is a cmp.Option to sort base.SchedulerEntry for comparing slice of entries.\nvar SortSchedulerEntryOpt = cmp.Transformer(\"SortSchedulerEntry\", func(in []*base.SchedulerEntry) []*base.SchedulerEntry {\n\tout := append([]*base.SchedulerEntry(nil), in...) // Copy input to avoid mutating it\n\tsort.Slice(out, func(i, j int) bool {\n\t\treturn out[i].Spec < out[j].Spec\n\t})\n\treturn out\n})\n\n// SortSchedulerEnqueueEventOpt is a cmp.Option to sort base.SchedulerEnqueueEvent for comparing slice of events.\nvar SortSchedulerEnqueueEventOpt = cmp.Transformer(\"SortSchedulerEnqueueEvent\", func(in []*base.SchedulerEnqueueEvent) []*base.SchedulerEnqueueEvent {\n\tout := append([]*base.SchedulerEnqueueEvent(nil), in...)\n\tsort.Slice(out, func(i, j int) bool {\n\t\treturn out[i].EnqueuedAt.Unix() < out[j].EnqueuedAt.Unix()\n\t})\n\treturn out\n})\n\n// SortStringSliceOpt is a cmp.Option to sort string slice.\nvar SortStringSliceOpt = cmp.Transformer(\"SortStringSlice\", func(in []string) []string {\n\tout := append([]string(nil), in...)\n\tsort.Strings(out)\n\treturn out\n})\n\nvar SortRedisZSetEntryOpt = cmp.Transformer(\"SortZSetEntries\", func(in []redis.Z) []redis.Z {\n\tout := append([]redis.Z(nil), in...) // Copy input to avoid mutating it\n\tsort.Slice(out, func(i, j int) bool {\n\t\t// TODO: If member is a comparable type (int, string, etc) compare by the member\n\t\t// Use generic comparable type here once update to go1.18\n\t\tif _, ok := out[i].Member.(string); ok {\n\t\t\t// If member is a string, compare the member\n\t\t\treturn out[i].Member.(string) < out[j].Member.(string)\n\t\t}\n\t\treturn out[i].Score < out[j].Score\n\t})\n\treturn out\n})\n\n// IgnoreIDOpt is an cmp.Option to ignore ID field in task messages when comparing.\nvar IgnoreIDOpt = cmpopts.IgnoreFields(base.TaskMessage{}, \"ID\")\n\n// NewTaskMessage returns a new instance of TaskMessage given a task type and payload.\nfunc NewTaskMessage(taskType string, payload []byte) *base.TaskMessage {\n\treturn NewTaskMessageWithQueue(taskType, payload, base.DefaultQueueName)\n}\n\n// NewTaskMessageWithQueue returns a new instance of TaskMessage given a\n// task type, payload and queue name.\nfunc NewTaskMessageWithQueue(taskType string, payload []byte, qname string) *base.TaskMessage {\n\treturn &base.TaskMessage{\n\t\tID: uuid.NewString(),\n\t\tType: taskType,\n\t\tQueue: qname,\n\t\tRetry: 25,\n\t\tPayload: payload,\n\t\tTimeout: 1800, // default timeout of 30 mins\n\t\tDeadline: 0, // no deadline\n\t}\n}\n\n// NewLeaseWithClock returns a new lease with the given expiration time and clock.\nfunc NewLeaseWithClock(expirationTime time.Time, clock timeutil.Clock) *base.Lease {\n\tl := base.NewLease(expirationTime)\n\tl.Clock = clock\n\treturn l\n}\n\n// JSON serializes the given key-value pairs into stream of bytes in JSON.\nfunc JSON(kv map[string]interface{}) []byte {\n\tb, err := json.Marshal(kv)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\treturn b\n}\n\n// TaskMessageAfterRetry returns an updated copy of t after retry.\n// It increments retry count and sets the error message and last_failed_at time.\nfunc TaskMessageAfterRetry(t base.TaskMessage, errMsg string, failedAt time.Time) *base.TaskMessage {\n\tt.Retried = t.Retried + 1\n\tt.ErrorMsg = errMsg\n\tt.LastFailedAt = failedAt.Unix()\n\treturn &t\n}\n\n// TaskMessageWithError returns an updated copy of t with the given error message.\nfunc TaskMessageWithError(t base.TaskMessage, errMsg string, failedAt time.Time) *base.TaskMessage {\n\tt.ErrorMsg = errMsg\n\tt.LastFailedAt = failedAt.Unix()\n\treturn &t\n}\n\n// TaskMessageWithCompletedAt returns an updated copy of t after completion.\nfunc TaskMessageWithCompletedAt(t base.TaskMessage, completedAt time.Time) *base.TaskMessage {\n\tt.CompletedAt = completedAt.Unix()\n\treturn &t\n}\n\n// MustMarshal marshals given task message and returns a json string.\n// Calling test will fail if marshaling errors out.\nfunc MustMarshal(tb testing.TB, msg *base.TaskMessage) string {\n\ttb.Helper()\n\tdata, err := base.EncodeMessage(msg)\n\tif err != nil {\n\t\ttb.Fatal(err)\n\t}\n\treturn string(data)\n}\n\n// MustUnmarshal unmarshals given string into task message struct.\n// Calling test will fail if unmarshaling errors out.\nfunc MustUnmarshal(tb testing.TB, data string) *base.TaskMessage {\n\ttb.Helper()\n\tmsg, err := base.DecodeMessage([]byte(data))\n\tif err != nil {\n\t\ttb.Fatal(err)\n\t}\n\treturn msg\n}\n\n// FlushDB deletes all the keys of the currently selected DB.\nfunc FlushDB(tb testing.TB, r redis.UniversalClient) {\n\ttb.Helper()\n\tswitch r := r.(type) {\n\tcase *redis.Client:\n\t\tif err := r.FlushDB(context.Background()).Err(); err != nil {\n\t\t\ttb.Fatal(err)\n\t\t}\n\tcase *redis.ClusterClient:\n\t\terr := r.ForEachMaster(context.Background(), func(ctx context.Context, c *redis.Client) error {\n\t\t\tif err := c.FlushAll(ctx).Err(); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\tif err != nil {\n\t\t\ttb.Fatal(err)\n\t\t}\n\t}\n}\n\n// SeedPendingQueue initializes the specified queue with the given messages.\nfunc SeedPendingQueue(tb testing.TB, r redis.UniversalClient, msgs []*base.TaskMessage, qname string) {\n\ttb.Helper()\n\tr.SAdd(context.Background(), base.AllQueues, qname)\n\tseedRedisList(tb, r, base.PendingKey(qname), msgs, base.TaskStatePending)\n}\n\n// SeedActiveQueue initializes the active queue with the given messages.\nfunc SeedActiveQueue(tb testing.TB, r redis.UniversalClient, msgs []*base.TaskMessage, qname string) {\n\ttb.Helper()\n\tr.SAdd(context.Background(), base.AllQueues, qname)\n\tseedRedisList(tb, r, base.ActiveKey(qname), msgs, base.TaskStateActive)\n}\n\n// SeedScheduledQueue initializes the scheduled queue with the given messages.\nfunc SeedScheduledQueue(tb testing.TB, r redis.UniversalClient, entries []base.Z, qname string) {\n\ttb.Helper()\n\tr.SAdd(context.Background(), base.AllQueues, qname)\n\tseedRedisZSet(tb, r, base.ScheduledKey(qname), entries, base.TaskStateScheduled)\n}\n\n// SeedRetryQueue initializes the retry queue with the given messages.\nfunc SeedRetryQueue(tb testing.TB, r redis.UniversalClient, entries []base.Z, qname string) {\n\ttb.Helper()\n\tr.SAdd(context.Background(), base.AllQueues, qname)\n\tseedRedisZSet(tb, r, base.RetryKey(qname), entries, base.TaskStateRetry)\n}\n\n// SeedArchivedQueue initializes the archived queue with the given messages.\nfunc SeedArchivedQueue(tb testing.TB, r redis.UniversalClient, entries []base.Z, qname string) {\n\ttb.Helper()\n\tr.SAdd(context.Background(), base.AllQueues, qname)\n\tseedRedisZSet(tb, r, base.ArchivedKey(qname), entries, base.TaskStateArchived)\n}\n\n// SeedLease initializes the lease set with the given entries.\nfunc SeedLease(tb testing.TB, r redis.UniversalClient, entries []base.Z, qname string) {\n\ttb.Helper()\n\tr.SAdd(context.Background(), base.AllQueues, qname)\n\tseedRedisZSet(tb, r, base.LeaseKey(qname), entries, base.TaskStateActive)\n}\n\n// SeedCompletedQueue initializes the completed set with the given entries.\nfunc SeedCompletedQueue(tb testing.TB, r redis.UniversalClient, entries []base.Z, qname string) {\n\ttb.Helper()\n\tr.SAdd(context.Background(), base.AllQueues, qname)\n\tseedRedisZSet(tb, r, base.CompletedKey(qname), entries, base.TaskStateCompleted)\n}\n\n// SeedGroup initializes the group with the given entries.\nfunc SeedGroup(tb testing.TB, r redis.UniversalClient, entries []base.Z, qname, gname string) {\n\ttb.Helper()\n\tctx := context.Background()\n\tr.SAdd(ctx, base.AllQueues, qname)\n\tr.SAdd(ctx, base.AllGroups(qname), gname)\n\tseedRedisZSet(tb, r, base.GroupKey(qname, gname), entries, base.TaskStateAggregating)\n}\n\nfunc SeedAggregationSet(tb testing.TB, r redis.UniversalClient, entries []base.Z, qname, gname, setID string) {\n\ttb.Helper()\n\tr.SAdd(context.Background(), base.AllQueues, qname)\n\tseedRedisZSet(tb, r, base.AggregationSetKey(qname, gname, setID), entries, base.TaskStateAggregating)\n}\n\n// SeedAllPendingQueues initializes all of the specified queues with the given messages.\n//\n// pending maps a queue name to a list of messages.\nfunc SeedAllPendingQueues(tb testing.TB, r redis.UniversalClient, pending map[string][]*base.TaskMessage) {\n\ttb.Helper()\n\tfor q, msgs := range pending {\n\t\tSeedPendingQueue(tb, r, msgs, q)\n\t}\n}\n\n// SeedAllActiveQueues initializes all of the specified active queues with the given messages.\nfunc SeedAllActiveQueues(tb testing.TB, r redis.UniversalClient, active map[string][]*base.TaskMessage) {\n\ttb.Helper()\n\tfor q, msgs := range active {\n\t\tSeedActiveQueue(tb, r, msgs, q)\n\t}\n}\n\n// SeedAllScheduledQueues initializes all of the specified scheduled queues with the given entries.\nfunc SeedAllScheduledQueues(tb testing.TB, r redis.UniversalClient, scheduled map[string][]base.Z) {\n\ttb.Helper()\n\tfor q, entries := range scheduled {\n\t\tSeedScheduledQueue(tb, r, entries, q)\n\t}\n}\n\n// SeedAllRetryQueues initializes all of the specified retry queues with the given entries.\nfunc SeedAllRetryQueues(tb testing.TB, r redis.UniversalClient, retry map[string][]base.Z) {\n\ttb.Helper()\n\tfor q, entries := range retry {\n\t\tSeedRetryQueue(tb, r, entries, q)\n\t}\n}\n\n// SeedAllArchivedQueues initializes all of the specified archived queues with the given entries.\nfunc SeedAllArchivedQueues(tb testing.TB, r redis.UniversalClient, archived map[string][]base.Z) {\n\ttb.Helper()\n\tfor q, entries := range archived {\n\t\tSeedArchivedQueue(tb, r, entries, q)\n\t}\n}\n\n// SeedAllLease initializes all of the lease sets with the given entries.\nfunc SeedAllLease(tb testing.TB, r redis.UniversalClient, lease map[string][]base.Z) {\n\ttb.Helper()\n\tfor q, entries := range lease {\n\t\tSeedLease(tb, r, entries, q)\n\t}\n}\n\n// SeedAllCompletedQueues initializes all of the completed queues with the given entries.\nfunc SeedAllCompletedQueues(tb testing.TB, r redis.UniversalClient, completed map[string][]base.Z) {\n\ttb.Helper()\n\tfor q, entries := range completed {\n\t\tSeedCompletedQueue(tb, r, entries, q)\n\t}\n}\n\n// SeedAllGroups initializes all groups in all queues.\n// The map maps queue names to group names which maps to a list of task messages and the time it was\n// added to the group.\nfunc SeedAllGroups(tb testing.TB, r redis.UniversalClient, groups map[string]map[string][]base.Z) {\n\ttb.Helper()\n\tfor qname, g := range groups {\n\t\tfor gname, entries := range g {\n\t\t\tSeedGroup(tb, r, entries, qname, gname)\n\t\t}\n\t}\n}\n\nfunc seedRedisList(tb testing.TB, c redis.UniversalClient, key string,\n\tmsgs []*base.TaskMessage, state base.TaskState) {\n\ttb.Helper()\n\tfor _, msg := range msgs {\n\t\tencoded := MustMarshal(tb, msg)\n\t\tif err := c.LPush(context.Background(), key, msg.ID).Err(); err != nil {\n\t\t\ttb.Fatal(err)\n\t\t}\n\t\ttaskKey := base.TaskKey(msg.Queue, msg.ID)\n\t\tdata := map[string]interface{}{\n\t\t\t\"msg\": encoded,\n\t\t\t\"state\": state.String(),\n\t\t\t\"unique_key\": msg.UniqueKey,\n\t\t\t\"group\": msg.GroupKey,\n\t\t}\n\t\tif err := c.HSet(context.Background(), taskKey, data).Err(); err != nil {\n\t\t\ttb.Fatal(err)\n\t\t}\n\t\tif len(msg.UniqueKey) > 0 {\n\t\t\terr := c.SetNX(context.Background(), msg.UniqueKey, msg.ID, 1*time.Minute).Err()\n\t\t\tif err != nil {\n\t\t\t\ttb.Fatalf(\"Failed to set unique lock in redis: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc seedRedisZSet(tb testing.TB, c redis.UniversalClient, key string,\n\titems []base.Z, state base.TaskState) {\n\ttb.Helper()\n\tfor _, item := range items {\n\t\tmsg := item.Message\n\t\tencoded := MustMarshal(tb, msg)\n\t\tz := redis.Z{Member: msg.ID, Score: float64(item.Score)}\n\t\tif err := c.ZAdd(context.Background(), key, z).Err(); err != nil {\n\t\t\ttb.Fatal(err)\n\t\t}\n\t\ttaskKey := base.TaskKey(msg.Queue, msg.ID)\n\t\tdata := map[string]interface{}{\n\t\t\t\"msg\": encoded,\n\t\t\t\"state\": state.String(),\n\t\t\t\"unique_key\": msg.UniqueKey,\n\t\t\t\"group\": msg.GroupKey,\n\t\t}\n\t\tif err := c.HSet(context.Background(), taskKey, data).Err(); err != nil {\n\t\t\ttb.Fatal(err)\n\t\t}\n\t\tif len(msg.UniqueKey) > 0 {\n\t\t\terr := c.SetNX(context.Background(), msg.UniqueKey, msg.ID, 1*time.Minute).Err()\n\t\t\tif err != nil {\n\t\t\t\ttb.Fatalf(\"Failed to set unique lock in redis: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// GetPendingMessages returns all pending messages in the given queue.\n// It also asserts the state field of the task.\nfunc GetPendingMessages(tb testing.TB, r redis.UniversalClient, qname string) []*base.TaskMessage {\n\ttb.Helper()\n\treturn getMessagesFromList(tb, r, qname, base.PendingKey, base.TaskStatePending)\n}\n\n// GetActiveMessages returns all active messages in the given queue.\n// It also asserts the state field of the task.\nfunc GetActiveMessages(tb testing.TB, r redis.UniversalClient, qname string) []*base.TaskMessage {\n\ttb.Helper()\n\treturn getMessagesFromList(tb, r, qname, base.ActiveKey, base.TaskStateActive)\n}\n\n// GetScheduledMessages returns all scheduled task messages in the given queue.\n// It also asserts the state field of the task.\nfunc GetScheduledMessages(tb testing.TB, r redis.UniversalClient, qname string) []*base.TaskMessage {\n\ttb.Helper()\n\treturn getMessagesFromZSet(tb, r, qname, base.ScheduledKey, base.TaskStateScheduled)\n}\n\n// GetRetryMessages returns all retry messages in the given queue.\n// It also asserts the state field of the task.\nfunc GetRetryMessages(tb testing.TB, r redis.UniversalClient, qname string) []*base.TaskMessage {\n\ttb.Helper()\n\treturn getMessagesFromZSet(tb, r, qname, base.RetryKey, base.TaskStateRetry)\n}\n\n// GetArchivedMessages returns all archived messages in the given queue.\n// It also asserts the state field of the task.\nfunc GetArchivedMessages(tb testing.TB, r redis.UniversalClient, qname string) []*base.TaskMessage {\n\ttb.Helper()\n\treturn getMessagesFromZSet(tb, r, qname, base.ArchivedKey, base.TaskStateArchived)\n}\n\n// GetCompletedMessages returns all completed task messages in the given queue.\n// It also asserts the state field of the task.\nfunc GetCompletedMessages(tb testing.TB, r redis.UniversalClient, qname string) []*base.TaskMessage {\n\ttb.Helper()\n\treturn getMessagesFromZSet(tb, r, qname, base.CompletedKey, base.TaskStateCompleted)\n}\n\n// GetScheduledEntries returns all scheduled messages and its score in the given queue.\n// It also asserts the state field of the task.\nfunc GetScheduledEntries(tb testing.TB, r redis.UniversalClient, qname string) []base.Z {\n\ttb.Helper()\n\treturn getMessagesFromZSetWithScores(tb, r, qname, base.ScheduledKey, base.TaskStateScheduled)\n}\n\n// GetRetryEntries returns all retry messages and its score in the given queue.\n// It also asserts the state field of the task.\nfunc GetRetryEntries(tb testing.TB, r redis.UniversalClient, qname string) []base.Z {\n\ttb.Helper()\n\treturn getMessagesFromZSetWithScores(tb, r, qname, base.RetryKey, base.TaskStateRetry)\n}\n\n// GetArchivedEntries returns all archived messages and its score in the given queue.\n// It also asserts the state field of the task.\nfunc GetArchivedEntries(tb testing.TB, r redis.UniversalClient, qname string) []base.Z {\n\ttb.Helper()\n\treturn getMessagesFromZSetWithScores(tb, r, qname, base.ArchivedKey, base.TaskStateArchived)\n}\n\n// GetLeaseEntries returns all task IDs and its score in the lease set for the given queue.\n// It also asserts the state field of the task.\nfunc GetLeaseEntries(tb testing.TB, r redis.UniversalClient, qname string) []base.Z {\n\ttb.Helper()\n\treturn getMessagesFromZSetWithScores(tb, r, qname, base.LeaseKey, base.TaskStateActive)\n}\n\n// GetCompletedEntries returns all completed messages and its score in the given queue.\n// It also asserts the state field of the task.\nfunc GetCompletedEntries(tb testing.TB, r redis.UniversalClient, qname string) []base.Z {\n\ttb.Helper()\n\treturn getMessagesFromZSetWithScores(tb, r, qname, base.CompletedKey, base.TaskStateCompleted)\n}\n\n// GetGroupEntries returns all scheduled messages and its score in the given queue.\n// It also asserts the state field of the task.\nfunc GetGroupEntries(tb testing.TB, r redis.UniversalClient, qname, groupKey string) []base.Z {\n\ttb.Helper()\n\treturn getMessagesFromZSetWithScores(tb, r, qname,\n\t\tfunc(qname string) string { return base.GroupKey(qname, groupKey) }, base.TaskStateAggregating)\n}\n\n// Retrieves all messages stored under `keyFn(qname)` key in redis list.\nfunc getMessagesFromList(tb testing.TB, r redis.UniversalClient, qname string,\n\tkeyFn func(qname string) string, state base.TaskState) []*base.TaskMessage {\n\ttb.Helper()\n\tids := r.LRange(context.Background(), keyFn(qname), 0, -1).Val()\n\tvar msgs []*base.TaskMessage\n\tfor _, id := range ids {\n\t\ttaskKey := base.TaskKey(qname, id)\n\t\tdata := r.HGet(context.Background(), taskKey, \"msg\").Val()\n\t\tmsgs = append(msgs, MustUnmarshal(tb, data))\n\t\tif gotState := r.HGet(context.Background(), taskKey, \"state\").Val(); gotState != state.String() {\n\t\t\ttb.Errorf(\"task (id=%q) is in %q state, want %v\", id, gotState, state)\n\t\t}\n\t}\n\treturn msgs\n}\n\n// Retrieves all messages stored under `keyFn(qname)` key in redis zset (sorted-set).\nfunc getMessagesFromZSet(tb testing.TB, r redis.UniversalClient, qname string,\n\tkeyFn func(qname string) string, state base.TaskState) []*base.TaskMessage {\n\ttb.Helper()\n\tids := r.ZRange(context.Background(), keyFn(qname), 0, -1).Val()\n\tvar msgs []*base.TaskMessage\n\tfor _, id := range ids {\n\t\ttaskKey := base.TaskKey(qname, id)\n\t\tmsg := r.HGet(context.Background(), taskKey, \"msg\").Val()\n\t\tmsgs = append(msgs, MustUnmarshal(tb, msg))\n\t\tif gotState := r.HGet(context.Background(), taskKey, \"state\").Val(); gotState != state.String() {\n\t\t\ttb.Errorf(\"task (id=%q) is in %q state, want %v\", id, gotState, state)\n\t\t}\n\t}\n\treturn msgs\n}\n\n// Retrieves all messages along with their scores stored under `keyFn(qname)` key in redis zset (sorted-set).\nfunc getMessagesFromZSetWithScores(tb testing.TB, r redis.UniversalClient,\n\tqname string, keyFn func(qname string) string, state base.TaskState) []base.Z {\n\ttb.Helper()\n\tzs := r.ZRangeWithScores(context.Background(), keyFn(qname), 0, -1).Val()\n\tvar res []base.Z\n\tfor _, z := range zs {\n\t\ttaskID := z.Member.(string)\n\t\ttaskKey := base.TaskKey(qname, taskID)\n\t\tmsg := r.HGet(context.Background(), taskKey, \"msg\").Val()\n\t\tres = append(res, base.Z{Message: MustUnmarshal(tb, msg), Score: int64(z.Score)})\n\t\tif gotState := r.HGet(context.Background(), taskKey, \"state\").Val(); gotState != state.String() {\n\t\t\ttb.Errorf(\"task (id=%q) is in %q state, want %v\", taskID, gotState, state)\n\t\t}\n\t}\n\treturn res\n}\n\n// TaskSeedData holds the data required to seed tasks under the task key in test.\ntype TaskSeedData struct {\n\tMsg *base.TaskMessage\n\tState base.TaskState\n\tPendingSince time.Time\n}\n\nfunc SeedTasks(tb testing.TB, r redis.UniversalClient, taskData []*TaskSeedData) {\n\tfor _, data := range taskData {\n\t\tmsg := data.Msg\n\t\tctx := context.Background()\n\t\tkey := base.TaskKey(msg.Queue, msg.ID)\n\t\tv := map[string]interface{}{\n\t\t\t\"msg\": MustMarshal(tb, msg),\n\t\t\t\"state\": data.State.String(),\n\t\t\t\"unique_key\": msg.UniqueKey,\n\t\t\t\"group\": msg.GroupKey,\n\t\t}\n\t\tif !data.PendingSince.IsZero() {\n\t\t\tv[\"pending_since\"] = data.PendingSince.Unix()\n\t\t}\n\t\tif err := r.HSet(ctx, key, v).Err(); err != nil {\n\t\t\ttb.Fatalf(\"Failed to write task data in redis: %v\", err)\n\t\t}\n\t\tif len(msg.UniqueKey) > 0 {\n\t\t\terr := r.SetNX(ctx, msg.UniqueKey, msg.ID, 1*time.Minute).Err()\n\t\t\tif err != nil {\n\t\t\t\ttb.Fatalf(\"Failed to set unique lock in redis: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc SeedRedisZSets(tb testing.TB, r redis.UniversalClient, zsets map[string][]redis.Z) {\n\tfor key, zs := range zsets {\n\t\t// FIXME: How come we can't simply do ZAdd(ctx, key, zs...) here?\n\t\tfor _, z := range zs {\n\t\t\tif err := r.ZAdd(context.Background(), key, z).Err(); err != nil {\n\t\t\t\ttb.Fatalf(\"Failed to seed zset (key=%q): %v\", key, err)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc SeedRedisSets(tb testing.TB, r redis.UniversalClient, sets map[string][]string) {\n\tfor key, set := range sets {\n\t\tSeedRedisSet(tb, r, key, set)\n\t}\n}\n\nfunc SeedRedisSet(tb testing.TB, r redis.UniversalClient, key string, members []string) {\n\tfor _, mem := range members {\n\t\tif err := r.SAdd(context.Background(), key, mem).Err(); err != nil {\n\t\t\ttb.Fatalf(\"Failed to seed set (key=%q): %v\", key, err)\n\t\t}\n\t}\n}\n\nfunc SeedRedisLists(tb testing.TB, r redis.UniversalClient, lists map[string][]string) {\n\tfor key, vals := range lists {\n\t\tfor _, v := range vals {\n\t\t\tif err := r.LPush(context.Background(), key, v).Err(); err != nil {\n\t\t\t\ttb.Fatalf(\"Failed to seed list (key=%q): %v\", key, err)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc AssertRedisLists(t *testing.T, r redis.UniversalClient, wantLists map[string][]string) {\n\tfor key, want := range wantLists {\n\t\tgot, err := r.LRange(context.Background(), key, 0, -1).Result()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Failed to read list (key=%q): %v\", key, err)\n\t\t}\n\t\tif diff := cmp.Diff(want, got, SortStringSliceOpt); diff != \"\" {\n\t\t\tt.Errorf(\"mismatch found in list (key=%q): (-want,+got)\\n%s\", key, diff)\n\t\t}\n\t}\n}\n\nfunc AssertRedisSets(t *testing.T, r redis.UniversalClient, wantSets map[string][]string) {\n\tfor key, want := range wantSets {\n\t\tgot, err := r.SMembers(context.Background(), key).Result()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Failed to read set (key=%q): %v\", key, err)\n\t\t}\n\t\tif diff := cmp.Diff(want, got, SortStringSliceOpt); diff != \"\" {\n\t\t\tt.Errorf(\"mismatch found in set (key=%q): (-want,+got)\\n%s\", key, diff)\n\t\t}\n\t}\n}\n\nfunc AssertRedisZSets(t *testing.T, r redis.UniversalClient, wantZSets map[string][]redis.Z) {\n\tfor key, want := range wantZSets {\n\t\tgot, err := r.ZRangeWithScores(context.Background(), key, 0, -1).Result()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Failed to read zset (key=%q): %v\", key, err)\n\t\t}\n\t\tif diff := cmp.Diff(want, got, SortRedisZSetEntryOpt); diff != \"\" {\n\t\t\tt.Errorf(\"mismatch found in zset (key=%q): (-want,+got)\\n%s\", key, diff)\n\t\t}\n\t}\n}\n" }, { "path": "internal/timeutil/timeutil.go", "content": "// Copyright 2022 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\n// Package timeutil exports functions and types related to time and date.\npackage timeutil\n\nimport (\n\t\"sync\"\n\t\"time\"\n)\n\n// A Clock is an object that can tell you the current time.\n//\n// This interface allows decoupling code that uses time from the code that creates\n// a point in time. You can use this to your advantage by injecting Clocks into interfaces\n// rather than having implementations call time.Now() directly.\n//\n// Use RealClock() in production.\n// Use SimulatedClock() in test.\ntype Clock interface {\n\tNow() time.Time\n}\n\nfunc NewRealClock() Clock { return &realTimeClock{} }\n\ntype realTimeClock struct{}\n\nfunc (_ *realTimeClock) Now() time.Time { return time.Now() }\n\n// A SimulatedClock is a concrete Clock implementation that doesn't \"tick\" on its own.\n// Time is advanced by explicit call to the AdvanceTime() or SetTime() functions.\n// This object is concurrency safe.\ntype SimulatedClock struct {\n\tmu sync.Mutex\n\tt time.Time // guarded by mu\n}\n\nfunc NewSimulatedClock(t time.Time) *SimulatedClock {\n\treturn &SimulatedClock{t: t}\n}\n\nfunc (c *SimulatedClock) Now() time.Time {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\treturn c.t\n}\n\nfunc (c *SimulatedClock) SetTime(t time.Time) {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\tc.t = t\n}\n\nfunc (c *SimulatedClock) AdvanceTime(d time.Duration) {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\tc.t = c.t.Add(d)\n}\n" }, { "path": "internal/timeutil/timeutil_test.go", "content": "// Copyright 2022 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage timeutil\n\nimport (\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestSimulatedClock(t *testing.T) {\n\tnow := time.Now()\n\n\ttests := []struct {\n\t\tdesc string\n\t\tinitTime time.Time\n\t\tadvanceBy time.Duration\n\t\twantTime time.Time\n\t}{\n\t\t{\n\t\t\tdesc: \"advance time forward\",\n\t\t\tinitTime: now,\n\t\t\tadvanceBy: 30 * time.Second,\n\t\t\twantTime: now.Add(30 * time.Second),\n\t\t},\n\t\t{\n\t\t\tdesc: \"advance time backward\",\n\t\t\tinitTime: now,\n\t\t\tadvanceBy: -10 * time.Second,\n\t\t\twantTime: now.Add(-10 * time.Second),\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tc := NewSimulatedClock(tc.initTime)\n\n\t\tif c.Now() != tc.initTime {\n\t\t\tt.Errorf(\"%s: Before Advance; SimulatedClock.Now() = %v, want %v\", tc.desc, c.Now(), tc.initTime)\n\t\t}\n\n\t\tc.AdvanceTime(tc.advanceBy)\n\n\t\tif c.Now() != tc.wantTime {\n\t\t\tt.Errorf(\"%s: After Advance; SimulatedClock.Now() = %v, want %v\", tc.desc, c.Now(), tc.wantTime)\n\t\t}\n\t}\n}\n" }, { "path": "janitor.go", "content": "// Copyright 2021 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/log\"\n)\n\n// A janitor is responsible for deleting expired completed tasks from the specified\n// queues. It periodically checks for any expired tasks in the completed set, and\n// deletes them.\ntype janitor struct {\n\tlogger *log.Logger\n\tbroker base.Broker\n\n\t// channel to communicate back to the long running \"janitor\" goroutine.\n\tdone chan struct{}\n\n\t// list of queue names to check.\n\tqueues []string\n\n\t// average interval between checks.\n\tavgInterval time.Duration\n\n\t// number of tasks to be deleted when janitor runs to delete the expired completed tasks.\n\tbatchSize int\n}\n\ntype janitorParams struct {\n\tlogger *log.Logger\n\tbroker base.Broker\n\tqueues []string\n\tinterval time.Duration\n\tbatchSize int\n}\n\nfunc newJanitor(params janitorParams) *janitor {\n\treturn &janitor{\n\t\tlogger: params.logger,\n\t\tbroker: params.broker,\n\t\tdone: make(chan struct{}),\n\t\tqueues: params.queues,\n\t\tavgInterval: params.interval,\n\t\tbatchSize: params.batchSize,\n\t}\n}\n\nfunc (j *janitor) shutdown() {\n\tj.logger.Debug(\"Janitor shutting down...\")\n\t// Signal the janitor goroutine to stop.\n\tj.done <- struct{}{}\n}\n\n// start starts the \"janitor\" goroutine.\nfunc (j *janitor) start(wg *sync.WaitGroup) {\n\twg.Add(1)\n\ttimer := time.NewTimer(j.avgInterval) // randomize this interval with margin of 1s\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-j.done:\n\t\t\t\tj.logger.Debug(\"Janitor done\")\n\t\t\t\treturn\n\t\t\tcase <-timer.C:\n\t\t\t\tj.exec()\n\t\t\t\ttimer.Reset(j.avgInterval)\n\t\t\t}\n\t\t}\n\t}()\n}\n\nfunc (j *janitor) exec() {\n\tfor _, qname := range j.queues {\n\t\tif err := j.broker.DeleteExpiredCompletedTasks(qname, j.batchSize); err != nil {\n\t\t\tj.logger.Errorf(\"Failed to delete expired completed tasks from queue %q: %v\",\n\t\t\t\tqname, err)\n\t\t}\n\t}\n}\n" }, { "path": "janitor_test.go", "content": "// Copyright 2021 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/google/go-cmp/cmp\"\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/rdb\"\n\th \"github.com/hibiken/asynq/internal/testutil\"\n)\n\nfunc newCompletedTask(qname, tasktype string, payload []byte, completedAt time.Time) *base.TaskMessage {\n\tmsg := h.NewTaskMessageWithQueue(tasktype, payload, qname)\n\tmsg.CompletedAt = completedAt.Unix()\n\treturn msg\n}\n\nfunc TestJanitor(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\trdbClient := rdb.NewRDB(r)\n\tconst interval = 1 * time.Second\n\tconst batchSize = 100\n\tjanitor := newJanitor(janitorParams{\n\t\tlogger: testLogger,\n\t\tbroker: rdbClient,\n\t\tqueues: []string{\"default\", \"custom\"},\n\t\tinterval: interval,\n\t\tbatchSize: batchSize,\n\t})\n\n\tnow := time.Now()\n\thourAgo := now.Add(-1 * time.Hour)\n\tminuteAgo := now.Add(-1 * time.Minute)\n\thalfHourAgo := now.Add(-30 * time.Minute)\n\thalfHourFromNow := now.Add(30 * time.Minute)\n\tfiveMinFromNow := now.Add(5 * time.Minute)\n\tmsg1 := newCompletedTask(\"default\", \"task1\", nil, hourAgo)\n\tmsg2 := newCompletedTask(\"default\", \"task2\", nil, minuteAgo)\n\tmsg3 := newCompletedTask(\"custom\", \"task3\", nil, hourAgo)\n\tmsg4 := newCompletedTask(\"custom\", \"task4\", nil, minuteAgo)\n\n\ttests := []struct {\n\t\tcompleted map[string][]base.Z // initial completed sets\n\t\twantCompleted map[string][]base.Z // expected completed sets after janitor runs\n\t}{\n\t\t{\n\t\t\tcompleted: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: msg1, Score: halfHourAgo.Unix()},\n\t\t\t\t\t{Message: msg2, Score: fiveMinFromNow.Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: msg3, Score: halfHourFromNow.Unix()},\n\t\t\t\t\t{Message: msg4, Score: minuteAgo.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\twantCompleted: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: msg2, Score: fiveMinFromNow.Unix()},\n\t\t\t\t},\n\t\t\t\t\"custom\": {\n\t\t\t\t\t{Message: msg3, Score: halfHourFromNow.Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllCompletedQueues(t, r, tc.completed)\n\n\t\tvar wg sync.WaitGroup\n\t\tjanitor.start(&wg)\n\t\ttime.Sleep(2 * interval) // make sure to let janitor run at least one time\n\t\tjanitor.shutdown()\n\n\t\tfor qname, want := range tc.wantCompleted {\n\t\t\tgot := h.GetCompletedEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, got, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"diff found in %q after running janitor: (-want, +got)\\n%s\", base.CompletedKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n" }, { "path": "periodic_task_manager.go", "content": "// Copyright 2022 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"crypto/sha256\"\n\t\"fmt\"\n\t\"sort\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/redis/go-redis/v9\"\n)\n\n// PeriodicTaskManager manages scheduling of periodic tasks.\n// It syncs scheduler's entries by calling the config provider periodically.\ntype PeriodicTaskManager struct {\n\ts *Scheduler\n\tp PeriodicTaskConfigProvider\n\tsyncInterval time.Duration\n\tdone chan (struct{})\n\twg sync.WaitGroup\n\tm map[string]string // map[hash]entryID\n}\n\ntype PeriodicTaskManagerOpts struct {\n\t// Required: must be non nil\n\tPeriodicTaskConfigProvider PeriodicTaskConfigProvider\n\n\t// Optional: if RedisUniversalClient is nil must be non nil\n\tRedisConnOpt RedisConnOpt\n\n\t// Optional: if RedisUniversalClient is non nil, RedisConnOpt is ignored.\n\tRedisUniversalClient redis.UniversalClient\n\n\t// Optional: scheduler options\n\t*SchedulerOpts\n\n\t// Optional: default is 3m\n\tSyncInterval time.Duration\n}\n\nconst defaultSyncInterval = 3 * time.Minute\n\n// NewPeriodicTaskManager returns a new PeriodicTaskManager instance.\n// The given opts should specify the RedisConnOp and PeriodicTaskConfigProvider at minimum.\nfunc NewPeriodicTaskManager(opts PeriodicTaskManagerOpts) (*PeriodicTaskManager, error) {\n\tif opts.PeriodicTaskConfigProvider == nil {\n\t\treturn nil, fmt.Errorf(\"PeriodicTaskConfigProvider cannot be nil\")\n\t}\n\tif opts.RedisConnOpt == nil && opts.RedisUniversalClient == nil {\n\t\treturn nil, fmt.Errorf(\"RedisConnOpt/RedisUniversalClient cannot be nil\")\n\t}\n\tvar scheduler *Scheduler\n\tif opts.RedisUniversalClient != nil {\n\t\tscheduler = NewSchedulerFromRedisClient(opts.RedisUniversalClient, opts.SchedulerOpts)\n\t} else {\n\t\tscheduler = NewScheduler(opts.RedisConnOpt, opts.SchedulerOpts)\n\t}\n\n\tsyncInterval := opts.SyncInterval\n\tif syncInterval == 0 {\n\t\tsyncInterval = defaultSyncInterval\n\t}\n\treturn &PeriodicTaskManager{\n\t\ts: scheduler,\n\t\tp: opts.PeriodicTaskConfigProvider,\n\t\tsyncInterval: syncInterval,\n\t\tdone: make(chan struct{}),\n\t\tm: make(map[string]string),\n\t}, nil\n}\n\n// PeriodicTaskConfigProvider provides configs for periodic tasks.\n// GetConfigs will be called by a PeriodicTaskManager periodically to\n// sync the scheduler's entries with the configs returned by the provider.\ntype PeriodicTaskConfigProvider interface {\n\tGetConfigs() ([]*PeriodicTaskConfig, error)\n}\n\n// PeriodicTaskConfig specifies the details of a periodic task.\ntype PeriodicTaskConfig struct {\n\tCronspec string // required: must be non empty string\n\tTask *Task // required: must be non nil\n\tOpts []Option // optional: can be nil\n}\n\nfunc (c *PeriodicTaskConfig) hash() string {\n\th := sha256.New()\n\t_, _ = h.Write([]byte(c.Cronspec))\n\t_, _ = h.Write([]byte(c.Task.Type()))\n\th.Write(c.Task.Payload())\n\topts := stringifyOptions(c.Opts)\n\tsort.Strings(opts)\n\tfor _, opt := range opts {\n\t\t_, _ = h.Write([]byte(opt))\n\t}\n\treturn fmt.Sprintf(\"%x\", h.Sum(nil))\n}\n\nfunc validatePeriodicTaskConfig(c *PeriodicTaskConfig) error {\n\tif c == nil {\n\t\treturn fmt.Errorf(\"PeriodicTaskConfig cannot be nil\")\n\t}\n\tif c.Task == nil {\n\t\treturn fmt.Errorf(\"PeriodicTaskConfig.Task cannot be nil\")\n\t}\n\tif c.Cronspec == \"\" {\n\t\treturn fmt.Errorf(\"PeriodicTaskConfig.Cronspec cannot be empty\")\n\t}\n\treturn nil\n}\n\n// Start starts a scheduler and background goroutine to sync the scheduler with the configs\n// returned by the provider.\n//\n// Start returns any error encountered at start up time.\nfunc (mgr *PeriodicTaskManager) Start() error {\n\tif mgr.s == nil || mgr.p == nil {\n\t\tpanic(\"asynq: cannot start uninitialized PeriodicTaskManager; use NewPeriodicTaskManager to initialize\")\n\t}\n\tif err := mgr.initialSync(); err != nil {\n\t\treturn fmt.Errorf(\"asynq: %w\", err)\n\t}\n\tif err := mgr.s.Start(); err != nil {\n\t\treturn fmt.Errorf(\"asynq: %w\", err)\n\t}\n\tmgr.wg.Add(1)\n\tgo func() {\n\t\tdefer mgr.wg.Done()\n\t\tticker := time.NewTicker(mgr.syncInterval)\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-mgr.done:\n\t\t\t\tmgr.s.logger.Debugf(\"Stopping syncer goroutine\")\n\t\t\t\tticker.Stop()\n\t\t\t\treturn\n\t\t\tcase <-ticker.C:\n\t\t\t\tmgr.sync()\n\t\t\t}\n\t\t}\n\t}()\n\treturn nil\n}\n\n// Shutdown gracefully shuts down the manager.\n// It notifies a background syncer goroutine to stop and stops scheduler.\nfunc (mgr *PeriodicTaskManager) Shutdown() {\n\tclose(mgr.done)\n\tmgr.wg.Wait()\n\tmgr.s.Shutdown()\n}\n\n// Run starts the manager and blocks until an os signal to exit the program is received.\n// Once it receives a signal, it gracefully shuts down the manager.\nfunc (mgr *PeriodicTaskManager) Run() error {\n\tif err := mgr.Start(); err != nil {\n\t\treturn err\n\t}\n\tmgr.s.waitForSignals()\n\tmgr.Shutdown()\n\tmgr.s.logger.Debugf(\"PeriodicTaskManager exiting\")\n\treturn nil\n}\n\nfunc (mgr *PeriodicTaskManager) initialSync() error {\n\tconfigs, err := mgr.p.GetConfigs()\n\tif err != nil {\n\t\treturn fmt.Errorf(\"initial call to GetConfigs failed: %w\", err)\n\t}\n\tfor _, c := range configs {\n\t\tif err := validatePeriodicTaskConfig(c); err != nil {\n\t\t\treturn fmt.Errorf(\"initial call to GetConfigs contained an invalid config: %w\", err)\n\t\t}\n\t}\n\tmgr.add(configs)\n\treturn nil\n}\n\nfunc (mgr *PeriodicTaskManager) add(configs []*PeriodicTaskConfig) {\n\tfor _, c := range configs {\n\t\tentryID, err := mgr.s.Register(c.Cronspec, c.Task, c.Opts...)\n\t\tif err != nil {\n\t\t\tmgr.s.logger.Errorf(\"Failed to register periodic task: cronspec=%q task=%q err=%v\",\n\t\t\t\tc.Cronspec, c.Task.Type(), err)\n\t\t\tcontinue\n\t\t}\n\t\tmgr.m[c.hash()] = entryID\n\t\tmgr.s.logger.Infof(\"Successfully registered periodic task: cronspec=%q task=%q, entryID=%s\",\n\t\t\tc.Cronspec, c.Task.Type(), entryID)\n\t}\n}\n\nfunc (mgr *PeriodicTaskManager) remove(removed map[string]string) {\n\tfor hash, entryID := range removed {\n\t\tif err := mgr.s.Unregister(entryID); err != nil {\n\t\t\tmgr.s.logger.Errorf(\"Failed to unregister periodic task: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tdelete(mgr.m, hash)\n\t\tmgr.s.logger.Infof(\"Successfully unregistered periodic task: entryID=%s\", entryID)\n\t}\n}\n\nfunc (mgr *PeriodicTaskManager) sync() {\n\tconfigs, err := mgr.p.GetConfigs()\n\tif err != nil {\n\t\tmgr.s.logger.Errorf(\"Failed to get periodic task configs: %v\", err)\n\t\treturn\n\t}\n\tfor _, c := range configs {\n\t\tif err := validatePeriodicTaskConfig(c); err != nil {\n\t\t\tmgr.s.logger.Errorf(\"Failed to sync: GetConfigs returned an invalid config: %v\", err)\n\t\t\treturn\n\t\t}\n\t}\n\t// Diff and only register/unregister the newly added/removed entries.\n\tremoved := mgr.diffRemoved(configs)\n\tadded := mgr.diffAdded(configs)\n\tmgr.remove(removed)\n\tmgr.add(added)\n}\n\n// diffRemoved diffs the incoming configs with the registered config and returns\n// a map containing hash and entryID of each config that was removed.\nfunc (mgr *PeriodicTaskManager) diffRemoved(configs []*PeriodicTaskConfig) map[string]string {\n\tnewConfigs := make(map[string]string)\n\tfor _, c := range configs {\n\t\tnewConfigs[c.hash()] = \"\" // empty value since we don't have entryID yet\n\t}\n\tremoved := make(map[string]string)\n\tfor k, v := range mgr.m {\n\t\t// test whether existing config is present in the incoming configs\n\t\tif _, found := newConfigs[k]; !found {\n\t\t\tremoved[k] = v\n\t\t}\n\t}\n\treturn removed\n}\n\n// diffAdded diffs the incoming configs with the registered configs and returns\n// a list of configs that were added.\nfunc (mgr *PeriodicTaskManager) diffAdded(configs []*PeriodicTaskConfig) []*PeriodicTaskConfig {\n\tvar added []*PeriodicTaskConfig\n\tfor _, c := range configs {\n\t\tif _, found := mgr.m[c.hash()]; !found {\n\t\t\tadded = append(added, c)\n\t\t}\n\t}\n\treturn added\n}\n" }, { "path": "periodic_task_manager_test.go", "content": "// Copyright 2022 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"sort\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/google/go-cmp/cmp\"\n)\n\n// Trivial implementation of PeriodicTaskConfigProvider for testing purpose.\ntype FakeConfigProvider struct {\n\tmu sync.Mutex\n\tcfgs []*PeriodicTaskConfig\n}\n\nfunc (p *FakeConfigProvider) SetConfigs(cfgs []*PeriodicTaskConfig) {\n\tp.mu.Lock()\n\tdefer p.mu.Unlock()\n\tp.cfgs = cfgs\n}\n\nfunc (p *FakeConfigProvider) GetConfigs() ([]*PeriodicTaskConfig, error) {\n\tp.mu.Lock()\n\tdefer p.mu.Unlock()\n\treturn p.cfgs, nil\n}\n\nfunc TestNewPeriodicTaskManager(t *testing.T) {\n\tredisConnOpt := getRedisConnOpt(t)\n\tcfgs := []*PeriodicTaskConfig{\n\t\t{Cronspec: \"* * * * *\", Task: NewTask(\"foo\", nil)},\n\t\t{Cronspec: \"* * * * *\", Task: NewTask(\"bar\", nil)},\n\t}\n\ttests := []struct {\n\t\tdesc string\n\t\topts PeriodicTaskManagerOpts\n\t}{\n\t\t{\n\t\t\tdesc: \"with provider and redisConnOpt\",\n\t\t\topts: PeriodicTaskManagerOpts{\n\t\t\t\tRedisConnOpt: redisConnOpt,\n\t\t\t\tPeriodicTaskConfigProvider: &FakeConfigProvider{cfgs: cfgs},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with sync option\",\n\t\t\topts: PeriodicTaskManagerOpts{\n\t\t\t\tRedisConnOpt: redisConnOpt,\n\t\t\t\tPeriodicTaskConfigProvider: &FakeConfigProvider{cfgs: cfgs},\n\t\t\t\tSyncInterval: 5 * time.Minute,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with scheduler option\",\n\t\t\topts: PeriodicTaskManagerOpts{\n\t\t\t\tRedisConnOpt: redisConnOpt,\n\t\t\t\tPeriodicTaskConfigProvider: &FakeConfigProvider{cfgs: cfgs},\n\t\t\t\tSyncInterval: 5 * time.Minute,\n\t\t\t\tSchedulerOpts: &SchedulerOpts{\n\t\t\t\t\tLogLevel: DebugLevel,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\t_, err := NewPeriodicTaskManager(tc.opts)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"%s; NewPeriodicTaskManager returned error: %v\", tc.desc, err)\n\t\t}\n\t}\n\n\tt.Run(\"error\", func(t *testing.T) {\n\t\ttests := []struct {\n\t\t\tdesc string\n\t\t\topts PeriodicTaskManagerOpts\n\t\t}{\n\t\t\t{\n\t\t\t\tdesc: \"without provider\",\n\t\t\t\topts: PeriodicTaskManagerOpts{\n\t\t\t\t\tRedisConnOpt: redisConnOpt,\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tdesc: \"without redisConOpt\",\n\t\t\t\topts: PeriodicTaskManagerOpts{\n\t\t\t\t\tPeriodicTaskConfigProvider: &FakeConfigProvider{cfgs: cfgs},\n\t\t\t\t},\n\t\t\t},\n\t\t}\n\n\t\tfor _, tc := range tests {\n\t\t\t_, err := NewPeriodicTaskManager(tc.opts)\n\t\t\tif err == nil {\n\t\t\t\tt.Errorf(\"%s; NewPeriodicTaskManager did not return error\", tc.desc)\n\t\t\t}\n\t\t}\n\t})\n}\n\nfunc TestPeriodicTaskConfigHash(t *testing.T) {\n\ttests := []struct {\n\t\tdesc string\n\t\ta *PeriodicTaskConfig\n\t\tb *PeriodicTaskConfig\n\t\tisSame bool\n\t}{\n\t\t{\n\t\t\tdesc: \"basic identity test\",\n\t\t\ta: &PeriodicTaskConfig{\n\t\t\t\tCronspec: \"* * * * *\",\n\t\t\t\tTask: NewTask(\"foo\", nil),\n\t\t\t},\n\t\t\tb: &PeriodicTaskConfig{\n\t\t\t\tCronspec: \"* * * * *\",\n\t\t\t\tTask: NewTask(\"foo\", nil),\n\t\t\t},\n\t\t\tisSame: true,\n\t\t},\n\t\t{\n\t\t\tdesc: \"with a option\",\n\t\t\ta: &PeriodicTaskConfig{\n\t\t\t\tCronspec: \"* * * * *\",\n\t\t\t\tTask: NewTask(\"foo\", nil),\n\t\t\t\tOpts: []Option{Queue(\"myqueue\")},\n\t\t\t},\n\t\t\tb: &PeriodicTaskConfig{\n\t\t\t\tCronspec: \"* * * * *\",\n\t\t\t\tTask: NewTask(\"foo\", nil),\n\t\t\t\tOpts: []Option{Queue(\"myqueue\")},\n\t\t\t},\n\t\t\tisSame: true,\n\t\t},\n\t\t{\n\t\t\tdesc: \"with multiple options (different order)\",\n\t\t\ta: &PeriodicTaskConfig{\n\t\t\t\tCronspec: \"* * * * *\",\n\t\t\t\tTask: NewTask(\"foo\", nil),\n\t\t\t\tOpts: []Option{Unique(5 * time.Minute), Queue(\"myqueue\")},\n\t\t\t},\n\t\t\tb: &PeriodicTaskConfig{\n\t\t\t\tCronspec: \"* * * * *\",\n\t\t\t\tTask: NewTask(\"foo\", nil),\n\t\t\t\tOpts: []Option{Queue(\"myqueue\"), Unique(5 * time.Minute)},\n\t\t\t},\n\t\t\tisSame: true,\n\t\t},\n\t\t{\n\t\t\tdesc: \"with payload\",\n\t\t\ta: &PeriodicTaskConfig{\n\t\t\t\tCronspec: \"* * * * *\",\n\t\t\t\tTask: NewTask(\"foo\", []byte(\"hello world!\")),\n\t\t\t\tOpts: []Option{Queue(\"myqueue\")},\n\t\t\t},\n\t\t\tb: &PeriodicTaskConfig{\n\t\t\t\tCronspec: \"* * * * *\",\n\t\t\t\tTask: NewTask(\"foo\", []byte(\"hello world!\")),\n\t\t\t\tOpts: []Option{Queue(\"myqueue\")},\n\t\t\t},\n\t\t\tisSame: true,\n\t\t},\n\t\t{\n\t\t\tdesc: \"with different cronspecs\",\n\t\t\ta: &PeriodicTaskConfig{\n\t\t\t\tCronspec: \"* * * * *\",\n\t\t\t\tTask: NewTask(\"foo\", nil),\n\t\t\t},\n\t\t\tb: &PeriodicTaskConfig{\n\t\t\t\tCronspec: \"5 * * * *\",\n\t\t\t\tTask: NewTask(\"foo\", nil),\n\t\t\t},\n\t\t\tisSame: false,\n\t\t},\n\t\t{\n\t\t\tdesc: \"with different task type\",\n\t\t\ta: &PeriodicTaskConfig{\n\t\t\t\tCronspec: \"* * * * *\",\n\t\t\t\tTask: NewTask(\"foo\", nil),\n\t\t\t},\n\t\t\tb: &PeriodicTaskConfig{\n\t\t\t\tCronspec: \"* * * * *\",\n\t\t\t\tTask: NewTask(\"bar\", nil),\n\t\t\t},\n\t\t\tisSame: false,\n\t\t},\n\t\t{\n\t\t\tdesc: \"with different options\",\n\t\t\ta: &PeriodicTaskConfig{\n\t\t\t\tCronspec: \"* * * * *\",\n\t\t\t\tTask: NewTask(\"foo\", nil),\n\t\t\t\tOpts: []Option{Queue(\"myqueue\")},\n\t\t\t},\n\t\t\tb: &PeriodicTaskConfig{\n\t\t\t\tCronspec: \"* * * * *\",\n\t\t\t\tTask: NewTask(\"foo\", nil),\n\t\t\t\tOpts: []Option{Unique(10 * time.Minute)},\n\t\t\t},\n\t\t\tisSame: false,\n\t\t},\n\t\t{\n\t\t\tdesc: \"with different options (one is subset of the other)\",\n\t\t\ta: &PeriodicTaskConfig{\n\t\t\t\tCronspec: \"* * * * *\",\n\t\t\t\tTask: NewTask(\"foo\", nil),\n\t\t\t\tOpts: []Option{Queue(\"myqueue\")},\n\t\t\t},\n\t\t\tb: &PeriodicTaskConfig{\n\t\t\t\tCronspec: \"* * * * *\",\n\t\t\t\tTask: NewTask(\"foo\", nil),\n\t\t\t\tOpts: []Option{Queue(\"myqueue\"), Unique(10 * time.Minute)},\n\t\t\t},\n\t\t\tisSame: false,\n\t\t},\n\t\t{\n\t\t\tdesc: \"with different payload\",\n\t\t\ta: &PeriodicTaskConfig{\n\t\t\t\tCronspec: \"* * * * *\",\n\t\t\t\tTask: NewTask(\"foo\", []byte(\"hello!\")),\n\t\t\t\tOpts: []Option{Queue(\"myqueue\")},\n\t\t\t},\n\t\t\tb: &PeriodicTaskConfig{\n\t\t\t\tCronspec: \"* * * * *\",\n\t\t\t\tTask: NewTask(\"foo\", []byte(\"HELLO!\")),\n\t\t\t\tOpts: []Option{Queue(\"myqueue\"), Unique(10 * time.Minute)},\n\t\t\t},\n\t\t\tisSame: false,\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tif tc.isSame && tc.a.hash() != tc.b.hash() {\n\t\t\tt.Errorf(\"%s: a.hash=%s b.hash=%s expected to be equal\",\n\t\t\t\ttc.desc, tc.a.hash(), tc.b.hash())\n\t\t}\n\t\tif !tc.isSame && tc.a.hash() == tc.b.hash() {\n\t\t\tt.Errorf(\"%s: a.hash=%s b.hash=%s expected to be not equal\",\n\t\t\t\ttc.desc, tc.a.hash(), tc.b.hash())\n\t\t}\n\t}\n}\n\n// Things to test.\n// - Run the manager\n// - Change provider to return new configs\n// - Verify that the scheduler synced with the new config\nfunc TestPeriodicTaskManager(t *testing.T) {\n\t// Note: In this test, we'll use task type as an ID for each config.\n\tcfgs := []*PeriodicTaskConfig{\n\t\t{Task: NewTask(\"task1\", nil), Cronspec: \"* * * * 1\"},\n\t\t{Task: NewTask(\"task2\", nil), Cronspec: \"* * * * 2\"},\n\t}\n\tconst syncInterval = 3 * time.Second\n\tprovider := &FakeConfigProvider{cfgs: cfgs}\n\tmgr, err := NewPeriodicTaskManager(PeriodicTaskManagerOpts{\n\t\tRedisConnOpt: getRedisConnOpt(t),\n\t\tPeriodicTaskConfigProvider: provider,\n\t\tSyncInterval: syncInterval,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to initialize PeriodicTaskManager: %v\", err)\n\t}\n\n\tif err := mgr.Start(); err != nil {\n\t\tt.Fatalf(\"Failed to start PeriodicTaskManager: %v\", err)\n\t}\n\tdefer mgr.Shutdown()\n\n\tgot := extractCronEntries(mgr.s)\n\twant := []*cronEntry{\n\t\t{Cronspec: \"* * * * 1\", TaskType: \"task1\"},\n\t\t{Cronspec: \"* * * * 2\", TaskType: \"task2\"},\n\t}\n\tif diff := cmp.Diff(want, got, sortCronEntry); diff != \"\" {\n\t\tt.Errorf(\"Diff found in scheduler's registered entries: %s\", diff)\n\t}\n\n\t// Change the underlying configs\n\t// - task2 removed\n\t// - task3 added\n\tprovider.SetConfigs([]*PeriodicTaskConfig{\n\t\t{Task: NewTask(\"task1\", nil), Cronspec: \"* * * * 1\"},\n\t\t{Task: NewTask(\"task3\", nil), Cronspec: \"* * * * 3\"},\n\t})\n\n\t// Wait for the next sync\n\ttime.Sleep(syncInterval * 2)\n\n\t// Verify the entries are synced\n\tgot = extractCronEntries(mgr.s)\n\twant = []*cronEntry{\n\t\t{Cronspec: \"* * * * 1\", TaskType: \"task1\"},\n\t\t{Cronspec: \"* * * * 3\", TaskType: \"task3\"},\n\t}\n\tif diff := cmp.Diff(want, got, sortCronEntry); diff != \"\" {\n\t\tt.Errorf(\"Diff found in scheduler's registered entries: %s\", diff)\n\t}\n\n\t// Change the underlying configs\n\t// All configs removed, empty set.\n\tprovider.SetConfigs([]*PeriodicTaskConfig{})\n\n\t// Wait for the next sync\n\ttime.Sleep(syncInterval * 2)\n\n\t// Verify the entries are synced\n\tgot = extractCronEntries(mgr.s)\n\twant = []*cronEntry{}\n\tif diff := cmp.Diff(want, got, sortCronEntry); diff != \"\" {\n\t\tt.Errorf(\"Diff found in scheduler's registered entries: %s\", diff)\n\t}\n}\n\nfunc extractCronEntries(s *Scheduler) []*cronEntry {\n\tvar out []*cronEntry\n\tfor _, e := range s.cron.Entries() {\n\t\tjob := e.Job.(*enqueueJob)\n\t\tout = append(out, &cronEntry{Cronspec: job.cronspec, TaskType: job.task.Type()})\n\t}\n\treturn out\n}\n\nvar sortCronEntry = cmp.Transformer(\"sortCronEntry\", func(in []*cronEntry) []*cronEntry {\n\tout := append([]*cronEntry(nil), in...)\n\tsort.Slice(out, func(i, j int) bool {\n\t\treturn out[i].TaskType < out[j].TaskType\n\t})\n\treturn out\n})\n\n// A simple struct to allow for simpler comparison in test.\ntype cronEntry struct {\n\tCronspec string\n\tTaskType string\n}\n" }, { "path": "processor.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"math\"\n\t\"math/rand/v2\"\n\t\"runtime\"\n\t\"runtime/debug\"\n\t\"sort\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/hibiken/asynq/internal/base\"\n\tasynqcontext \"github.com/hibiken/asynq/internal/context\"\n\t\"github.com/hibiken/asynq/internal/errors\"\n\t\"github.com/hibiken/asynq/internal/log\"\n\t\"github.com/hibiken/asynq/internal/timeutil\"\n\t\"golang.org/x/time/rate\"\n)\n\ntype processor struct {\n\tlogger *log.Logger\n\tbroker base.Broker\n\tclock timeutil.Clock\n\n\thandler Handler\n\tbaseCtxFn func() context.Context\n\n\tqueueConfig map[string]int\n\n\t// orderedQueues is set only in strict-priority mode.\n\torderedQueues []string\n\n\ttaskCheckInterval time.Duration\n\tretryDelayFunc RetryDelayFunc\n\tisFailureFunc func(error) bool\n\n\terrHandler ErrorHandler\n\tshutdownTimeout time.Duration\n\n\t// channel via which to send sync requests to syncer.\n\tsyncRequestCh chan<- *syncRequest\n\n\t// rate limiter to prevent spamming logs with a bunch of errors.\n\terrLogLimiter *rate.Limiter\n\n\t// sema is a counting semaphore to ensure the number of active workers\n\t// does not exceed the limit.\n\tsema chan struct{}\n\n\t// channel to communicate back to the long running \"processor\" goroutine.\n\t// once is used to send value to the channel only once.\n\tdone chan struct{}\n\tonce sync.Once\n\n\t// quit channel is closed when the shutdown of the \"processor\" goroutine starts.\n\tquit chan struct{}\n\n\t// abort channel communicates to the in-flight worker goroutines to stop.\n\tabort chan struct{}\n\n\t// cancelations is a set of cancel functions for all active tasks.\n\tcancelations *base.Cancelations\n\n\tstarting chan<- *workerInfo\n\tfinished chan<- *base.TaskMessage\n}\n\ntype processorParams struct {\n\tlogger *log.Logger\n\tbroker base.Broker\n\tbaseCtxFn func() context.Context\n\tretryDelayFunc RetryDelayFunc\n\ttaskCheckInterval time.Duration\n\tisFailureFunc func(error) bool\n\tsyncCh chan<- *syncRequest\n\tcancelations *base.Cancelations\n\tconcurrency int\n\tqueues map[string]int\n\tstrictPriority bool\n\terrHandler ErrorHandler\n\tshutdownTimeout time.Duration\n\tstarting chan<- *workerInfo\n\tfinished chan<- *base.TaskMessage\n}\n\n// newProcessor constructs a new processor.\nfunc newProcessor(params processorParams) *processor {\n\tqueues := normalizeQueues(params.queues)\n\torderedQueues := []string(nil)\n\tif params.strictPriority {\n\t\torderedQueues = sortByPriority(queues)\n\t}\n\treturn &processor{\n\t\tlogger: params.logger,\n\t\tbroker: params.broker,\n\t\tbaseCtxFn: params.baseCtxFn,\n\t\tclock: timeutil.NewRealClock(),\n\t\tqueueConfig: queues,\n\t\torderedQueues: orderedQueues,\n\t\ttaskCheckInterval: params.taskCheckInterval,\n\t\tretryDelayFunc: params.retryDelayFunc,\n\t\tisFailureFunc: params.isFailureFunc,\n\t\tsyncRequestCh: params.syncCh,\n\t\tcancelations: params.cancelations,\n\t\terrLogLimiter: rate.NewLimiter(rate.Every(3*time.Second), 1),\n\t\tsema: make(chan struct{}, params.concurrency),\n\t\tdone: make(chan struct{}),\n\t\tquit: make(chan struct{}),\n\t\tabort: make(chan struct{}),\n\t\terrHandler: params.errHandler,\n\t\thandler: HandlerFunc(func(ctx context.Context, t *Task) error { return fmt.Errorf(\"handler not set\") }),\n\t\tshutdownTimeout: params.shutdownTimeout,\n\t\tstarting: params.starting,\n\t\tfinished: params.finished,\n\t}\n}\n\n// Note: stops only the \"processor\" goroutine, does not stop workers.\n// It's safe to call this method multiple times.\nfunc (p *processor) stop() {\n\tp.once.Do(func() {\n\t\tp.logger.Debug(\"Processor shutting down...\")\n\t\t// Unblock if processor is waiting for sema token.\n\t\tclose(p.quit)\n\t\t// Signal the processor goroutine to stop processing tasks\n\t\t// from the queue.\n\t\tp.done <- struct{}{}\n\t})\n}\n\n// NOTE: once shutdown, processor cannot be re-started.\nfunc (p *processor) shutdown() {\n\tp.stop()\n\n\ttime.AfterFunc(p.shutdownTimeout, func() { close(p.abort) })\n\n\tp.logger.Info(\"Waiting for all workers to finish...\")\n\t// block until all workers have released the token\n\tfor i := 0; i < cap(p.sema); i++ {\n\t\tp.sema <- struct{}{}\n\t}\n\tp.logger.Info(\"All workers have finished\")\n}\n\nfunc (p *processor) start(wg *sync.WaitGroup) {\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-p.done:\n\t\t\t\tp.logger.Debug(\"Processor done\")\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t\tp.exec()\n\t\t\t}\n\t\t}\n\t}()\n}\n\n// exec pulls a task out of the queue and starts a worker goroutine to\n// process the task.\nfunc (p *processor) exec() {\n\tselect {\n\tcase <-p.quit:\n\t\treturn\n\tcase p.sema <- struct{}{}: // acquire token\n\t\tqnames := p.queues()\n\t\tmsg, leaseExpirationTime, err := p.broker.Dequeue(qnames...)\n\t\tswitch {\n\t\tcase errors.Is(err, errors.ErrNoProcessableTask):\n\t\t\tp.logger.Debug(\"All queues are empty\")\n\t\t\t// Queues are empty, this is a normal behavior.\n\t\t\t// Sleep to avoid slamming redis and let scheduler move tasks into queues.\n\t\t\t// Note: We are not using blocking pop operation and polling queues instead.\n\t\t\t// This adds significant load to redis.\n\t\t\tjitter := rand.N(p.taskCheckInterval)\n\t\t\ttime.Sleep(p.taskCheckInterval/2 + jitter)\n\t\t\t<-p.sema // release token\n\t\t\treturn\n\t\tcase err != nil:\n\t\t\tif p.errLogLimiter.Allow() {\n\t\t\t\tp.logger.Errorf(\"Dequeue error: %v\", err)\n\t\t\t}\n\t\t\t<-p.sema // release token\n\t\t\treturn\n\t\t}\n\n\t\tlease := base.NewLease(leaseExpirationTime)\n\t\tdeadline := p.computeDeadline(msg)\n\t\tp.starting <- &workerInfo{msg, time.Now(), deadline, lease}\n\t\tgo func() {\n\t\t\tdefer func() {\n\t\t\t\tp.finished <- msg\n\t\t\t\t<-p.sema // release token\n\t\t\t}()\n\n\t\t\tctx, cancel := asynqcontext.New(p.baseCtxFn(), msg, deadline)\n\t\t\tp.cancelations.Add(msg.ID, cancel)\n\t\t\tdefer func() {\n\t\t\t\tcancel()\n\t\t\t\tp.cancelations.Delete(msg.ID)\n\t\t\t}()\n\n\t\t\t// check context before starting a worker goroutine.\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\t// already canceled (e.g. deadline exceeded).\n\t\t\t\tp.handleFailedMessage(ctx, lease, msg, ctx.Err())\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t}\n\n\t\t\tresCh := make(chan error, 1)\n\t\t\tgo func() {\n\t\t\t\ttask := newTask(\n\t\t\t\t\tmsg.Type,\n\t\t\t\t\tmsg.Payload,\n\t\t\t\t\t&ResultWriter{\n\t\t\t\t\t\tid: msg.ID,\n\t\t\t\t\t\tqname: msg.Queue,\n\t\t\t\t\t\tbroker: p.broker,\n\t\t\t\t\t\tctx: ctx,\n\t\t\t\t\t},\n\t\t\t\t)\n\t\t\t\ttask.headers = msg.Headers\n\t\t\t\tresCh <- p.perform(ctx, task)\n\t\t\t}()\n\n\t\t\tselect {\n\t\t\tcase <-p.abort:\n\t\t\t\t// time is up, push the message back to queue and quit this worker goroutine.\n\t\t\t\tp.logger.Warnf(\"Quitting worker. task id=%s\", msg.ID)\n\t\t\t\tp.requeue(lease, msg)\n\t\t\t\treturn\n\t\t\tcase <-lease.Done():\n\t\t\t\tcancel()\n\t\t\t\tp.handleFailedMessage(ctx, lease, msg, ErrLeaseExpired)\n\t\t\t\treturn\n\t\t\tcase <-ctx.Done():\n\t\t\t\tp.handleFailedMessage(ctx, lease, msg, ctx.Err())\n\t\t\t\treturn\n\t\t\tcase resErr := <-resCh:\n\t\t\t\tif resErr != nil {\n\t\t\t\t\tp.handleFailedMessage(ctx, lease, msg, resErr)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tp.handleSucceededMessage(lease, msg)\n\t\t\t}\n\t\t}()\n\t}\n}\n\nfunc (p *processor) requeue(l *base.Lease, msg *base.TaskMessage) {\n\tif !l.IsValid() {\n\t\t// If lease is not valid, do not write to redis; Let recoverer take care of it.\n\t\treturn\n\t}\n\tctx, cancel := context.WithDeadline(context.Background(), l.Deadline())\n\tdefer cancel()\n\terr := p.broker.Requeue(ctx, msg)\n\tif err != nil {\n\t\tp.logger.Errorf(\"Could not push task id=%s back to queue: %v\", msg.ID, err)\n\t} else {\n\t\tp.logger.Infof(\"Pushed task id=%s back to queue\", msg.ID)\n\t}\n}\n\nfunc (p *processor) handleSucceededMessage(l *base.Lease, msg *base.TaskMessage) {\n\tif msg.Retention > 0 {\n\t\tp.markAsComplete(l, msg)\n\t} else {\n\t\tp.markAsDone(l, msg)\n\t}\n}\n\nfunc (p *processor) markAsComplete(l *base.Lease, msg *base.TaskMessage) {\n\tif !l.IsValid() {\n\t\t// If lease is not valid, do not write to redis; Let recoverer take care of it.\n\t\treturn\n\t}\n\tctx, cancel := context.WithDeadline(context.Background(), l.Deadline())\n\tdefer cancel()\n\terr := p.broker.MarkAsComplete(ctx, msg)\n\tif err != nil {\n\t\terrMsg := fmt.Sprintf(\"Could not move task id=%s type=%q from %q to %q: %+v\",\n\t\t\tmsg.ID, msg.Type, base.ActiveKey(msg.Queue), base.CompletedKey(msg.Queue), err)\n\t\tp.logger.Warnf(\"%s; Will retry syncing\", errMsg)\n\t\tp.syncRequestCh <- &syncRequest{\n\t\t\tfn: func() error {\n\t\t\t\treturn p.broker.MarkAsComplete(ctx, msg)\n\t\t\t},\n\t\t\terrMsg: errMsg,\n\t\t\tdeadline: l.Deadline(),\n\t\t}\n\t}\n}\n\nfunc (p *processor) markAsDone(l *base.Lease, msg *base.TaskMessage) {\n\tif !l.IsValid() {\n\t\t// If lease is not valid, do not write to redis; Let recoverer take care of it.\n\t\treturn\n\t}\n\tctx, cancel := context.WithDeadline(context.Background(), l.Deadline())\n\tdefer cancel()\n\terr := p.broker.Done(ctx, msg)\n\tif err != nil {\n\t\terrMsg := fmt.Sprintf(\"Could not remove task id=%s type=%q from %q err: %+v\", msg.ID, msg.Type, base.ActiveKey(msg.Queue), err)\n\t\tp.logger.Warnf(\"%s; Will retry syncing\", errMsg)\n\t\tp.syncRequestCh <- &syncRequest{\n\t\t\tfn: func() error {\n\t\t\t\treturn p.broker.Done(ctx, msg)\n\t\t\t},\n\t\t\terrMsg: errMsg,\n\t\t\tdeadline: l.Deadline(),\n\t\t}\n\t}\n}\n\n// SkipRetry is used as a return value from Handler.ProcessTask to indicate that\n// the task should not be retried and should be archived instead.\nvar SkipRetry = errors.New(\"skip retry for the task\")\n\n// RevokeTask is used as a return value from Handler.ProcessTask to indicate that\n// the task should not be retried or archived.\nvar RevokeTask = errors.New(\"revoke task\")\n\nfunc (p *processor) handleFailedMessage(ctx context.Context, l *base.Lease, msg *base.TaskMessage, err error) {\n\tif p.errHandler != nil {\n\t\tp.errHandler.HandleError(ctx, NewTaskWithHeaders(msg.Type, msg.Payload, msg.Headers), err)\n\t}\n\tswitch {\n\tcase errors.Is(err, RevokeTask):\n\t\tp.logger.Warnf(\"revoke task id=%s\", msg.ID)\n\t\tp.markAsDone(l, msg)\n\tcase msg.Retried >= msg.Retry || errors.Is(err, SkipRetry):\n\t\tp.logger.Warnf(\"Retry exhausted for task id=%s\", msg.ID)\n\t\tp.archive(l, msg, err)\n\tdefault:\n\t\tp.retry(l, msg, err, p.isFailureFunc(err))\n\t}\n}\n\nfunc (p *processor) retry(l *base.Lease, msg *base.TaskMessage, e error, isFailure bool) {\n\tif !l.IsValid() {\n\t\t// If lease is not valid, do not write to redis; Let recoverer take care of it.\n\t\treturn\n\t}\n\tctx, cancel := context.WithDeadline(context.Background(), l.Deadline())\n\tdefer cancel()\n\td := p.retryDelayFunc(msg.Retried, e, NewTaskWithHeaders(msg.Type, msg.Payload, msg.Headers))\n\tretryAt := time.Now().Add(d)\n\terr := p.broker.Retry(ctx, msg, retryAt, e.Error(), isFailure)\n\tif err != nil {\n\t\terrMsg := fmt.Sprintf(\"Could not move task id=%s from %q to %q\", msg.ID, base.ActiveKey(msg.Queue), base.RetryKey(msg.Queue))\n\t\tp.logger.Warnf(\"%s; Will retry syncing\", errMsg)\n\t\tp.syncRequestCh <- &syncRequest{\n\t\t\tfn: func() error {\n\t\t\t\treturn p.broker.Retry(ctx, msg, retryAt, e.Error(), isFailure)\n\t\t\t},\n\t\t\terrMsg: errMsg,\n\t\t\tdeadline: l.Deadline(),\n\t\t}\n\t}\n}\n\nfunc (p *processor) archive(l *base.Lease, msg *base.TaskMessage, e error) {\n\tif !l.IsValid() {\n\t\t// If lease is not valid, do not write to redis; Let recoverer take care of it.\n\t\treturn\n\t}\n\tctx, cancel := context.WithDeadline(context.Background(), l.Deadline())\n\tdefer cancel()\n\terr := p.broker.Archive(ctx, msg, e.Error())\n\tif err != nil {\n\t\terrMsg := fmt.Sprintf(\"Could not move task id=%s from %q to %q\", msg.ID, base.ActiveKey(msg.Queue), base.ArchivedKey(msg.Queue))\n\t\tp.logger.Warnf(\"%s; Will retry syncing\", errMsg)\n\t\tp.syncRequestCh <- &syncRequest{\n\t\t\tfn: func() error {\n\t\t\t\treturn p.broker.Archive(ctx, msg, e.Error())\n\t\t\t},\n\t\t\terrMsg: errMsg,\n\t\t\tdeadline: l.Deadline(),\n\t\t}\n\t}\n}\n\n// queues returns a list of queues to query.\n// Order of the queue names is based on the priority of each queue.\n// Queue names is sorted by their priority level if strict-priority is true.\n// If strict-priority is false, then the order of queue names are roughly based on\n// the priority level but randomized in order to avoid starving low priority queues.\nfunc (p *processor) queues() []string {\n\t// skip the overhead of generating a list of queue names\n\t// if we are processing one queue.\n\tif len(p.queueConfig) == 1 {\n\t\tfor qname := range p.queueConfig {\n\t\t\treturn []string{qname}\n\t\t}\n\t}\n\tif p.orderedQueues != nil {\n\t\treturn p.orderedQueues\n\t}\n\tvar names []string\n\tfor qname, priority := range p.queueConfig {\n\t\tfor i := 0; i < priority; i++ {\n\t\t\tnames = append(names, qname)\n\t\t}\n\t}\n\trand.Shuffle(len(names), func(i, j int) { names[i], names[j] = names[j], names[i] })\n\treturn uniq(names, len(p.queueConfig))\n}\n\n// perform calls the handler with the given task.\n// If the call returns without panic, it simply returns the value,\n// otherwise, it recovers from panic and returns an error.\nfunc (p *processor) perform(ctx context.Context, task *Task) (err error) {\n\tdefer func() {\n\t\tif x := recover(); x != nil {\n\t\t\tp.logger.Errorf(\"recovering from panic. See the stack trace below for details:\\n%s\", string(debug.Stack()))\n\t\t\t_, file, line, ok := runtime.Caller(1) // skip the first frame (panic itself)\n\t\t\tif ok && strings.Contains(file, \"runtime/\") {\n\t\t\t\t// The panic came from the runtime, most likely due to incorrect\n\t\t\t\t// map/slice usage. The parent frame should have the real trigger.\n\t\t\t\t_, file, line, ok = runtime.Caller(2)\n\t\t\t}\n\t\t\tvar errMsg string\n\t\t\t// Include the file and line number info in the error, if runtime.Caller returned ok.\n\t\t\tif ok {\n\t\t\t\terrMsg = fmt.Sprintf(\"panic [%s:%d]: %v\", file, line, x)\n\t\t\t} else {\n\t\t\t\terrMsg = fmt.Sprintf(\"panic: %v\", x)\n\t\t\t}\n\t\t\terr = &errors.PanicError{\n\t\t\t\tErrMsg: errMsg,\n\t\t\t}\n\t\t}\n\t}()\n\treturn p.handler.ProcessTask(ctx, task)\n}\n\n// uniq dedupes elements and returns a slice of unique names of length l.\n// Order of the output slice is based on the input list.\nfunc uniq(names []string, l int) []string {\n\tvar res []string\n\tseen := make(map[string]struct{})\n\tfor _, s := range names {\n\t\tif _, ok := seen[s]; !ok {\n\t\t\tseen[s] = struct{}{}\n\t\t\tres = append(res, s)\n\t\t}\n\t\tif len(res) == l {\n\t\t\tbreak\n\t\t}\n\t}\n\treturn res\n}\n\n// sortByPriority returns a list of queue names sorted by\n// their priority level in descending order.\nfunc sortByPriority(qcfg map[string]int) []string {\n\tvar queues []*queue\n\tfor qname, n := range qcfg {\n\t\tqueues = append(queues, &queue{qname, n})\n\t}\n\tsort.Sort(sort.Reverse(byPriority(queues)))\n\tvar res []string\n\tfor _, q := range queues {\n\t\tres = append(res, q.name)\n\t}\n\treturn res\n}\n\ntype queue struct {\n\tname string\n\tpriority int\n}\n\ntype byPriority []*queue\n\nfunc (x byPriority) Len() int { return len(x) }\nfunc (x byPriority) Less(i, j int) bool { return x[i].priority < x[j].priority }\nfunc (x byPriority) Swap(i, j int) { x[i], x[j] = x[j], x[i] }\n\n// normalizeQueues divides priority numbers by their greatest common divisor.\nfunc normalizeQueues(queues map[string]int) map[string]int {\n\tvar xs []int\n\tfor _, x := range queues {\n\t\txs = append(xs, x)\n\t}\n\td := gcd(xs...)\n\tres := make(map[string]int)\n\tfor q, x := range queues {\n\t\tres[q] = x / d\n\t}\n\treturn res\n}\n\nfunc gcd(xs ...int) int {\n\tfn := func(x, y int) int {\n\t\tfor y > 0 {\n\t\t\tx, y = y, x%y\n\t\t}\n\t\treturn x\n\t}\n\tres := xs[0]\n\tfor i := 0; i < len(xs); i++ {\n\t\tres = fn(xs[i], res)\n\t\tif res == 1 {\n\t\t\treturn 1\n\t\t}\n\t}\n\treturn res\n}\n\n// computeDeadline returns the given task's deadline,\nfunc (p *processor) computeDeadline(msg *base.TaskMessage) time.Time {\n\tif msg.Timeout == 0 && msg.Deadline == 0 {\n\t\tp.logger.Errorf(\"asynq: internal error: both timeout and deadline are not set for the task message: %s\", msg.ID)\n\t\treturn p.clock.Now().Add(defaultTimeout)\n\t}\n\tif msg.Timeout != 0 && msg.Deadline != 0 {\n\t\tdeadlineUnix := math.Min(float64(p.clock.Now().Unix()+msg.Timeout), float64(msg.Deadline))\n\t\treturn time.Unix(int64(deadlineUnix), 0)\n\t}\n\tif msg.Timeout != 0 {\n\t\treturn p.clock.Now().Add(time.Duration(msg.Timeout) * time.Second)\n\t}\n\treturn time.Unix(msg.Deadline, 0)\n}\n\nfunc IsPanicError(err error) bool {\n\treturn errors.IsPanicError(err)\n}\n" }, { "path": "processor_test.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"sort\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/google/go-cmp/cmp\"\n\t\"github.com/google/go-cmp/cmp/cmpopts\"\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/errors\"\n\t\"github.com/hibiken/asynq/internal/log\"\n\t\"github.com/hibiken/asynq/internal/rdb\"\n\th \"github.com/hibiken/asynq/internal/testutil\"\n\t\"github.com/hibiken/asynq/internal/timeutil\"\n)\n\nvar taskCmpOpts = []cmp.Option{\n\tsortTaskOpt, // sort the tasks\n\tcmp.AllowUnexported(Task{}), // allow typename, payload fields to be compared\n\tcmpopts.IgnoreFields(Task{}, \"opts\", \"w\"), // ignore opts, w fields\n}\n\n// fakeHeartbeater receives from starting and finished channels and do nothing.\nfunc fakeHeartbeater(starting <-chan *workerInfo, finished <-chan *base.TaskMessage, done <-chan struct{}) {\n\tfor {\n\t\tselect {\n\t\tcase <-starting:\n\t\tcase <-finished:\n\t\tcase <-done:\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// fakeSyncer receives from sync channel and do nothing.\nfunc fakeSyncer(syncCh <-chan *syncRequest, done <-chan struct{}) {\n\tfor {\n\t\tselect {\n\t\tcase <-syncCh:\n\t\tcase <-done:\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// Returns a processor instance configured for testing purpose.\nfunc newProcessorForTest(t *testing.T, r *rdb.RDB, h Handler) *processor {\n\tstarting := make(chan *workerInfo)\n\tfinished := make(chan *base.TaskMessage)\n\tsyncCh := make(chan *syncRequest)\n\tdone := make(chan struct{})\n\tt.Cleanup(func() { close(done) })\n\tgo fakeHeartbeater(starting, finished, done)\n\tgo fakeSyncer(syncCh, done)\n\tp := newProcessor(processorParams{\n\t\tlogger: testLogger,\n\t\tbroker: r,\n\t\tbaseCtxFn: context.Background,\n\t\tretryDelayFunc: DefaultRetryDelayFunc,\n\t\ttaskCheckInterval: defaultTaskCheckInterval,\n\t\tisFailureFunc: defaultIsFailureFunc,\n\t\tsyncCh: syncCh,\n\t\tcancelations: base.NewCancelations(),\n\t\tconcurrency: 10,\n\t\tqueues: defaultQueueConfig,\n\t\tstrictPriority: false,\n\t\terrHandler: nil,\n\t\tshutdownTimeout: defaultShutdownTimeout,\n\t\tstarting: starting,\n\t\tfinished: finished,\n\t})\n\tp.handler = h\n\treturn p\n}\n\nfunc TestProcessorSuccessWithSingleQueue(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\trdbClient := rdb.NewRDB(r)\n\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\tm2 := h.NewTaskMessage(\"task2\", nil)\n\tm3 := h.NewTaskMessage(\"task3\", nil)\n\tm4 := h.NewTaskMessage(\"task4\", nil)\n\n\tt1 := NewTask(m1.Type, m1.Payload)\n\tt2 := NewTask(m2.Type, m2.Payload)\n\tt3 := NewTask(m3.Type, m3.Payload)\n\tt4 := NewTask(m4.Type, m4.Payload)\n\n\ttests := []struct {\n\t\tpending []*base.TaskMessage // initial default queue state\n\t\tincoming []*base.TaskMessage // tasks to be enqueued during run\n\t\twantProcessed []*Task // tasks to be processed at the end\n\t}{\n\t\t{\n\t\t\tpending: []*base.TaskMessage{m1},\n\t\t\tincoming: []*base.TaskMessage{m2, m3, m4},\n\t\t\twantProcessed: []*Task{t1, t2, t3, t4},\n\t\t},\n\t\t{\n\t\t\tpending: []*base.TaskMessage{},\n\t\t\tincoming: []*base.TaskMessage{m1},\n\t\t\twantProcessed: []*Task{t1},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r) // clean up db before each test case.\n\t\th.SeedPendingQueue(t, r, tc.pending, base.DefaultQueueName) // initialize default queue.\n\n\t\t// instantiate a new processor\n\t\tvar mu sync.Mutex\n\t\tvar processed []*Task\n\t\thandler := func(ctx context.Context, task *Task) error {\n\t\t\tmu.Lock()\n\t\t\tdefer mu.Unlock()\n\t\t\tprocessed = append(processed, task)\n\t\t\treturn nil\n\t\t}\n\t\tp := newProcessorForTest(t, rdbClient, HandlerFunc(handler))\n\n\t\tp.start(&sync.WaitGroup{})\n\t\tfor _, msg := range tc.incoming {\n\t\t\terr := rdbClient.Enqueue(context.Background(), msg)\n\t\t\tif err != nil {\n\t\t\t\tp.shutdown()\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t}\n\t\ttime.Sleep(2 * time.Second) // wait for two second to allow all pending tasks to be processed.\n\t\tif l := r.LLen(context.Background(), base.ActiveKey(base.DefaultQueueName)).Val(); l != 0 {\n\t\t\tt.Errorf(\"%q has %d tasks, want 0\", base.ActiveKey(base.DefaultQueueName), l)\n\t\t}\n\t\tp.shutdown()\n\n\t\tmu.Lock()\n\t\tif diff := cmp.Diff(tc.wantProcessed, processed, taskCmpOpts...); diff != \"\" {\n\t\t\tt.Errorf(\"mismatch found in processed tasks; (-want, +got)\\n%s\", diff)\n\t\t}\n\t\tmu.Unlock()\n\t}\n}\n\nfunc TestProcessorSuccessWithMultipleQueues(t *testing.T) {\n\tvar (\n\t\tr = setup(t)\n\t\trdbClient = rdb.NewRDB(r)\n\n\t\tm1 = h.NewTaskMessage(\"task1\", nil)\n\t\tm2 = h.NewTaskMessage(\"task2\", nil)\n\t\tm3 = h.NewTaskMessageWithQueue(\"task3\", nil, \"high\")\n\t\tm4 = h.NewTaskMessageWithQueue(\"task4\", nil, \"low\")\n\n\t\tt1 = NewTask(m1.Type, m1.Payload)\n\t\tt2 = NewTask(m2.Type, m2.Payload)\n\t\tt3 = NewTask(m3.Type, m3.Payload)\n\t\tt4 = NewTask(m4.Type, m4.Payload)\n\t)\n\tdefer r.Close()\n\n\ttests := []struct {\n\t\tpending map[string][]*base.TaskMessage\n\t\tqueues []string // list of queues to consume the tasks from\n\t\twantProcessed []*Task // tasks to be processed at the end\n\t}{\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {m1, m2},\n\t\t\t\t\"high\": {m3},\n\t\t\t\t\"low\": {m4},\n\t\t\t},\n\t\t\tqueues: []string{\"default\", \"high\", \"low\"},\n\t\t\twantProcessed: []*Task{t1, t2, t3, t4},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\t// Set up test case.\n\t\th.FlushDB(t, r)\n\t\th.SeedAllPendingQueues(t, r, tc.pending)\n\n\t\t// Instantiate a new processor.\n\t\tvar mu sync.Mutex\n\t\tvar processed []*Task\n\t\thandler := func(ctx context.Context, task *Task) error {\n\t\t\tmu.Lock()\n\t\t\tdefer mu.Unlock()\n\t\t\tprocessed = append(processed, task)\n\t\t\treturn nil\n\t\t}\n\t\tp := newProcessorForTest(t, rdbClient, HandlerFunc(handler))\n\t\tp.queueConfig = map[string]int{\n\t\t\t\"default\": 2,\n\t\t\t\"high\": 3,\n\t\t\t\"low\": 1,\n\t\t}\n\n\t\tp.start(&sync.WaitGroup{})\n\t\t// Wait for two second to allow all pending tasks to be processed.\n\t\ttime.Sleep(2 * time.Second)\n\t\t// Make sure no messages are stuck in active list.\n\t\tfor _, qname := range tc.queues {\n\t\t\tif l := r.LLen(context.Background(), base.ActiveKey(qname)).Val(); l != 0 {\n\t\t\t\tt.Errorf(\"%q has %d tasks, want 0\", base.ActiveKey(qname), l)\n\t\t\t}\n\t\t}\n\t\tp.shutdown()\n\n\t\tmu.Lock()\n\t\tif diff := cmp.Diff(tc.wantProcessed, processed, taskCmpOpts...); diff != \"\" {\n\t\t\tt.Errorf(\"mismatch found in processed tasks; (-want, +got)\\n%s\", diff)\n\t\t}\n\t\tmu.Unlock()\n\t}\n}\n\n// https://github.com/hibiken/asynq/issues/166\nfunc TestProcessTasksWithLargeNumberInPayload(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\trdbClient := rdb.NewRDB(r)\n\n\tm1 := h.NewTaskMessage(\"large_number\", h.JSON(map[string]interface{}{\"data\": 111111111111111111}))\n\tt1 := NewTask(m1.Type, m1.Payload)\n\n\ttests := []struct {\n\t\tpending []*base.TaskMessage // initial default queue state\n\t\twantProcessed []*Task // tasks to be processed at the end\n\t}{\n\t\t{\n\t\t\tpending: []*base.TaskMessage{m1},\n\t\t\twantProcessed: []*Task{t1},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r) // clean up db before each test case.\n\t\th.SeedPendingQueue(t, r, tc.pending, base.DefaultQueueName) // initialize default queue.\n\n\t\tvar mu sync.Mutex\n\t\tvar processed []*Task\n\t\thandler := func(ctx context.Context, task *Task) error {\n\t\t\tmu.Lock()\n\t\t\tdefer mu.Unlock()\n\t\t\tvar payload map[string]int\n\t\t\tif err := json.Unmarshal(task.Payload(), &payload); err != nil {\n\t\t\t\tt.Errorf(\"coult not decode payload: %v\", err)\n\t\t\t}\n\t\t\tif data, ok := payload[\"data\"]; ok {\n\t\t\t\tt.Logf(\"data == %d\", data)\n\t\t\t} else {\n\t\t\t\tt.Errorf(\"could not get data from payload\")\n\t\t\t}\n\t\t\tprocessed = append(processed, task)\n\t\t\treturn nil\n\t\t}\n\t\tp := newProcessorForTest(t, rdbClient, HandlerFunc(handler))\n\n\t\tp.start(&sync.WaitGroup{})\n\t\ttime.Sleep(2 * time.Second) // wait for two second to allow all pending tasks to be processed.\n\t\tif l := r.LLen(context.Background(), base.ActiveKey(base.DefaultQueueName)).Val(); l != 0 {\n\t\t\tt.Errorf(\"%q has %d tasks, want 0\", base.ActiveKey(base.DefaultQueueName), l)\n\t\t}\n\t\tp.shutdown()\n\n\t\tmu.Lock()\n\t\tif diff := cmp.Diff(tc.wantProcessed, processed, taskCmpOpts...); diff != \"\" {\n\t\t\tt.Errorf(\"mismatch found in processed tasks; (-want, +got)\\n%s\", diff)\n\t\t}\n\t\tmu.Unlock()\n\t}\n}\n\nfunc TestProcessorRetry(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\trdbClient := rdb.NewRDB(r)\n\n\tm1 := h.NewTaskMessage(\"send_email\", nil)\n\tm1.Retried = m1.Retry // m1 has reached its max retry count\n\tm2 := h.NewTaskMessage(\"gen_thumbnail\", nil)\n\tm3 := h.NewTaskMessage(\"reindex\", nil)\n\tm4 := h.NewTaskMessage(\"sync\", nil)\n\n\terrMsg := \"something went wrong\"\n\twrappedSkipRetry := fmt.Errorf(\"%s:%w\", errMsg, SkipRetry)\n\twrappedRevokeTask := fmt.Errorf(\"%s:%w\", errMsg, RevokeTask)\n\n\ttests := []struct {\n\t\tdesc string // test description\n\t\tpending []*base.TaskMessage // initial default queue state\n\t\tdelay time.Duration // retry delay duration\n\t\thandler Handler // task handler\n\t\twait time.Duration // wait duration between starting and stopping processor for this test case\n\t\twantErrMsg string // error message the task should record\n\t\twantRetry []*base.TaskMessage // tasks in retry queue at the end\n\t\twantArchived []*base.TaskMessage // tasks in archived queue at the end\n\t\twantErrCount int // number of times error handler should be called\n\t}{\n\t\t{\n\t\t\tdesc: \"Should automatically retry errored tasks\",\n\t\t\tpending: []*base.TaskMessage{m1, m2, m3, m4},\n\t\t\tdelay: time.Minute,\n\t\t\thandler: HandlerFunc(func(ctx context.Context, task *Task) error {\n\t\t\t\treturn errors.New(errMsg)\n\t\t\t}),\n\t\t\twait: 2 * time.Second,\n\t\t\twantErrMsg: errMsg,\n\t\t\twantRetry: []*base.TaskMessage{m2, m3, m4},\n\t\t\twantArchived: []*base.TaskMessage{m1},\n\t\t\twantErrCount: 4,\n\t\t},\n\t\t{\n\t\t\tdesc: \"Should skip retry errored tasks\",\n\t\t\tpending: []*base.TaskMessage{m1, m2},\n\t\t\tdelay: time.Minute,\n\t\t\thandler: HandlerFunc(func(ctx context.Context, task *Task) error {\n\t\t\t\treturn SkipRetry // return SkipRetry without wrapping\n\t\t\t}),\n\t\t\twait: 2 * time.Second,\n\t\t\twantErrMsg: SkipRetry.Error(),\n\t\t\twantRetry: []*base.TaskMessage{},\n\t\t\twantArchived: []*base.TaskMessage{m1, m2},\n\t\t\twantErrCount: 2, // ErrorHandler should still be called with SkipRetry error\n\t\t},\n\t\t{\n\t\t\tdesc: \"Should skip retry errored tasks (with error wrapping)\",\n\t\t\tpending: []*base.TaskMessage{m1, m2},\n\t\t\tdelay: time.Minute,\n\t\t\thandler: HandlerFunc(func(ctx context.Context, task *Task) error {\n\t\t\t\treturn wrappedSkipRetry\n\t\t\t}),\n\t\t\twait: 2 * time.Second,\n\t\t\twantErrMsg: wrappedSkipRetry.Error(),\n\t\t\twantRetry: []*base.TaskMessage{},\n\t\t\twantArchived: []*base.TaskMessage{m1, m2},\n\t\t\twantErrCount: 2, // ErrorHandler should still be called with SkipRetry error\n\t\t},\n\t\t{\n\t\t\tdesc: \"Should revoke task\",\n\t\t\tpending: []*base.TaskMessage{m1, m2},\n\t\t\tdelay: time.Minute,\n\t\t\thandler: HandlerFunc(func(ctx context.Context, task *Task) error {\n\t\t\t\treturn RevokeTask // return RevokeTask without wrapping\n\t\t\t}),\n\t\t\twait: 2 * time.Second,\n\t\t\twantErrMsg: RevokeTask.Error(),\n\t\t\twantRetry: []*base.TaskMessage{},\n\t\t\twantArchived: []*base.TaskMessage{},\n\t\t\twantErrCount: 2, // ErrorHandler should still be called with RevokeTask error\n\t\t},\n\t\t{\n\t\t\tdesc: \"Should revoke task (with error wrapping)\",\n\t\t\tpending: []*base.TaskMessage{m1, m2},\n\t\t\tdelay: time.Minute,\n\t\t\thandler: HandlerFunc(func(ctx context.Context, task *Task) error {\n\t\t\t\treturn wrappedRevokeTask\n\t\t\t}),\n\t\t\twait: 2 * time.Second,\n\t\t\twantErrMsg: wrappedRevokeTask.Error(),\n\t\t\twantRetry: []*base.TaskMessage{},\n\t\t\twantArchived: []*base.TaskMessage{},\n\t\t\twantErrCount: 2, // ErrorHandler should still be called with RevokeTask error\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r) // clean up db before each test case.\n\t\th.SeedPendingQueue(t, r, tc.pending, base.DefaultQueueName) // initialize default queue.\n\n\t\t// instantiate a new processor\n\t\tdelayFunc := func(n int, e error, t *Task) time.Duration {\n\t\t\treturn tc.delay\n\t\t}\n\t\tvar (\n\t\t\tmu sync.Mutex // guards n\n\t\t\tn int // number of times error handler is called\n\t\t)\n\t\terrHandler := func(ctx context.Context, t *Task, err error) {\n\t\t\tmu.Lock()\n\t\t\tdefer mu.Unlock()\n\t\t\tn++\n\t\t}\n\t\tp := newProcessorForTest(t, rdbClient, tc.handler)\n\t\tp.errHandler = ErrorHandlerFunc(errHandler)\n\t\tp.retryDelayFunc = delayFunc\n\n\t\tp.start(&sync.WaitGroup{})\n\t\trunTime := time.Now() // time when processor is running\n\t\ttime.Sleep(tc.wait) // FIXME: This makes test flaky.\n\t\tp.shutdown()\n\n\t\tcmpOpt := h.EquateInt64Approx(int64(tc.wait.Seconds())) // allow up to a wait-second difference in zset score\n\t\tgotRetry := h.GetRetryEntries(t, r, base.DefaultQueueName)\n\t\tvar wantRetry []base.Z // Note: construct wantRetry here since `LastFailedAt` and ZSCORE is relative to each test run.\n\t\tfor _, msg := range tc.wantRetry {\n\t\t\twantRetry = append(wantRetry,\n\t\t\t\tbase.Z{\n\t\t\t\t\tMessage: h.TaskMessageAfterRetry(*msg, tc.wantErrMsg, runTime),\n\t\t\t\t\tScore: runTime.Add(tc.delay).Unix(),\n\t\t\t\t})\n\t\t}\n\t\tif diff := cmp.Diff(wantRetry, gotRetry, h.SortZSetEntryOpt, cmpOpt); diff != \"\" {\n\t\t\tt.Errorf(\"%s: mismatch found in %q after running processor; (-want, +got)\\n%s\", tc.desc, base.RetryKey(base.DefaultQueueName), diff)\n\t\t}\n\n\t\tgotArchived := h.GetArchivedEntries(t, r, base.DefaultQueueName)\n\t\tvar wantArchived []base.Z // Note: construct wantArchived here since `LastFailedAt` and ZSCORE is relative to each test run.\n\t\tfor _, msg := range tc.wantArchived {\n\t\t\twantArchived = append(wantArchived,\n\t\t\t\tbase.Z{\n\t\t\t\t\tMessage: h.TaskMessageWithError(*msg, tc.wantErrMsg, runTime),\n\t\t\t\t\tScore: runTime.Unix(),\n\t\t\t\t})\n\t\t}\n\t\tif diff := cmp.Diff(wantArchived, gotArchived, h.SortZSetEntryOpt, cmpOpt); diff != \"\" {\n\t\t\tt.Errorf(\"%s: mismatch found in %q after running processor; (-want, +got)\\n%s\", tc.desc, base.ArchivedKey(base.DefaultQueueName), diff)\n\t\t}\n\n\t\tif l := r.LLen(context.Background(), base.ActiveKey(base.DefaultQueueName)).Val(); l != 0 {\n\t\t\tt.Errorf(\"%s: %q has %d tasks, want 0\", base.ActiveKey(base.DefaultQueueName), tc.desc, l)\n\t\t}\n\n\t\tif n != tc.wantErrCount {\n\t\t\tt.Errorf(\"error handler was called %d times, want %d\", n, tc.wantErrCount)\n\t\t}\n\t}\n}\n\nfunc TestProcessorMarkAsComplete(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\trdbClient := rdb.NewRDB(r)\n\n\tmsg1 := h.NewTaskMessage(\"one\", nil)\n\tmsg2 := h.NewTaskMessage(\"two\", nil)\n\tmsg3 := h.NewTaskMessageWithQueue(\"three\", nil, \"custom\")\n\tmsg1.Retention = 3600\n\tmsg3.Retention = 7200\n\n\thandler := func(ctx context.Context, task *Task) error { return nil }\n\n\ttests := []struct {\n\t\tpending map[string][]*base.TaskMessage\n\t\tcompleted map[string][]base.Z\n\t\tqueueCfg map[string]int\n\t\twantPending map[string][]*base.TaskMessage\n\t\twantCompleted func(completedAt time.Time) map[string][]base.Z\n\t}{\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {msg1, msg2},\n\t\t\t\t\"custom\": {msg3},\n\t\t\t},\n\t\t\tcompleted: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\tqueueCfg: map[string]int{\n\t\t\t\t\"default\": 1,\n\t\t\t\t\"custom\": 1,\n\t\t\t},\n\t\t\twantPending: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"custom\": {},\n\t\t\t},\n\t\t\twantCompleted: func(completedAt time.Time) map[string][]base.Z {\n\t\t\t\treturn map[string][]base.Z{\n\t\t\t\t\t\"default\": {{Message: h.TaskMessageWithCompletedAt(*msg1, completedAt), Score: completedAt.Unix() + msg1.Retention}},\n\t\t\t\t\t\"custom\": {{Message: h.TaskMessageWithCompletedAt(*msg3, completedAt), Score: completedAt.Unix() + msg3.Retention}},\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllPendingQueues(t, r, tc.pending)\n\t\th.SeedAllCompletedQueues(t, r, tc.completed)\n\n\t\tp := newProcessorForTest(t, rdbClient, HandlerFunc(handler))\n\t\tp.queueConfig = tc.queueCfg\n\n\t\tp.start(&sync.WaitGroup{})\n\t\trunTime := time.Now() // time when processor is running\n\t\ttime.Sleep(2 * time.Second)\n\t\tp.shutdown()\n\n\t\tfor qname, want := range tc.wantPending {\n\t\t\tgotPending := h.GetPendingMessages(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotPending, cmpopts.EquateEmpty()); diff != \"\" {\n\t\t\t\tt.Errorf(\"diff found in %q pending set; want=%v, got=%v\\n%s\", qname, want, gotPending, diff)\n\t\t\t}\n\t\t}\n\n\t\tfor qname, want := range tc.wantCompleted(runTime) {\n\t\t\tgotCompleted := h.GetCompletedEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotCompleted, cmpopts.EquateEmpty()); diff != \"\" {\n\t\t\t\tt.Errorf(\"diff found in %q completed set; want=%v, got=%v\\n%s\", qname, want, gotCompleted, diff)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Test a scenario where the worker server cannot communicate with redis due to a network failure\n// and the lease expires\nfunc TestProcessorWithExpiredLease(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\trdbClient := rdb.NewRDB(r)\n\n\tm1 := h.NewTaskMessage(\"task1\", nil)\n\n\ttests := []struct {\n\t\tpending []*base.TaskMessage\n\t\thandler Handler\n\t\twantErrCount int\n\t}{\n\t\t{\n\t\t\tpending: []*base.TaskMessage{m1},\n\t\t\thandler: HandlerFunc(func(ctx context.Context, task *Task) error {\n\t\t\t\t// make sure the task processing time exceeds lease duration\n\t\t\t\t// to test expired lease.\n\t\t\t\ttime.Sleep(rdb.LeaseDuration + 10*time.Second)\n\t\t\t\treturn nil\n\t\t\t}),\n\t\t\twantErrCount: 1, // ErrorHandler should still be called with ErrLeaseExpired\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedPendingQueue(t, r, tc.pending, base.DefaultQueueName)\n\n\t\tstarting := make(chan *workerInfo)\n\t\tfinished := make(chan *base.TaskMessage)\n\t\tsyncCh := make(chan *syncRequest)\n\t\tdone := make(chan struct{})\n\t\tt.Cleanup(func() { close(done) })\n\t\t// fake heartbeater which notifies lease expiration\n\t\tgo func() {\n\t\t\tfor {\n\t\t\t\tselect {\n\t\t\t\tcase w := <-starting:\n\t\t\t\t\t// simulate expiration by resetting to some time in the past\n\t\t\t\t\tw.lease.Reset(time.Now().Add(-5 * time.Second))\n\t\t\t\t\tif !w.lease.NotifyExpiration() {\n\t\t\t\t\t\tpanic(\"Failed to notifiy lease expiration\")\n\t\t\t\t\t}\n\t\t\t\tcase <-finished:\n\t\t\t\t\t// do nothing\n\t\t\t\tcase <-done:\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t\tgo fakeSyncer(syncCh, done)\n\t\tp := newProcessor(processorParams{\n\t\t\tlogger: testLogger,\n\t\t\tbroker: rdbClient,\n\t\t\tbaseCtxFn: context.Background,\n\t\t\ttaskCheckInterval: defaultTaskCheckInterval,\n\t\t\tretryDelayFunc: DefaultRetryDelayFunc,\n\t\t\tisFailureFunc: defaultIsFailureFunc,\n\t\t\tsyncCh: syncCh,\n\t\t\tcancelations: base.NewCancelations(),\n\t\t\tconcurrency: 10,\n\t\t\tqueues: defaultQueueConfig,\n\t\t\tstrictPriority: false,\n\t\t\terrHandler: nil,\n\t\t\tshutdownTimeout: defaultShutdownTimeout,\n\t\t\tstarting: starting,\n\t\t\tfinished: finished,\n\t\t})\n\t\tp.handler = tc.handler\n\t\tvar (\n\t\t\tmu sync.Mutex // guards n and errs\n\t\t\tn int // number of times error handler is called\n\t\t\terrs []error // error passed to error handler\n\t\t)\n\t\tp.errHandler = ErrorHandlerFunc(func(ctx context.Context, t *Task, err error) {\n\t\t\tmu.Lock()\n\t\t\tdefer mu.Unlock()\n\t\t\tn++\n\t\t\terrs = append(errs, err)\n\t\t})\n\n\t\tp.start(&sync.WaitGroup{})\n\t\ttime.Sleep(4 * time.Second)\n\t\tp.shutdown()\n\n\t\tif n != tc.wantErrCount {\n\t\t\tt.Errorf(\"Unexpected number of error count: got %d, want %d\", n, tc.wantErrCount)\n\t\t\tcontinue\n\t\t}\n\t\tfor i := 0; i < tc.wantErrCount; i++ {\n\t\t\tif !errors.Is(errs[i], ErrLeaseExpired) {\n\t\t\t\tt.Errorf(\"Unexpected error was passed to ErrorHandler: got %v want %v\", errs[i], ErrLeaseExpired)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestProcessorQueues(t *testing.T) {\n\tsortOpt := cmp.Transformer(\"SortStrings\", func(in []string) []string {\n\t\tout := append([]string(nil), in...) // Copy input to avoid mutating it\n\t\tsort.Strings(out)\n\t\treturn out\n\t})\n\n\ttests := []struct {\n\t\tqueueCfg map[string]int\n\t\twant []string\n\t}{\n\t\t{\n\t\t\tqueueCfg: map[string]int{\n\t\t\t\t\"high\": 6,\n\t\t\t\t\"default\": 3,\n\t\t\t\t\"low\": 1,\n\t\t\t},\n\t\t\twant: []string{\"high\", \"default\", \"low\"},\n\t\t},\n\t\t{\n\t\t\tqueueCfg: map[string]int{\n\t\t\t\t\"default\": 1,\n\t\t\t},\n\t\t\twant: []string{\"default\"},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\t// Note: rdb and handler not needed for this test.\n\t\tp := newProcessorForTest(t, nil, nil)\n\t\tp.queueConfig = tc.queueCfg\n\n\t\tgot := p.queues()\n\t\tif diff := cmp.Diff(tc.want, got, sortOpt); diff != \"\" {\n\t\t\tt.Errorf(\"with queue config: %v\\n(*processor).queues() = %v, want %v\\n(-want,+got):\\n%s\",\n\t\t\t\ttc.queueCfg, got, tc.want, diff)\n\t\t}\n\t}\n}\n\nfunc TestProcessorWithStrictPriority(t *testing.T) {\n\tvar (\n\t\tr = setup(t)\n\n\t\trdbClient = rdb.NewRDB(r)\n\n\t\tm1 = h.NewTaskMessageWithQueue(\"task1\", nil, \"critical\")\n\t\tm2 = h.NewTaskMessageWithQueue(\"task2\", nil, \"critical\")\n\t\tm3 = h.NewTaskMessageWithQueue(\"task3\", nil, \"critical\")\n\t\tm4 = h.NewTaskMessageWithQueue(\"task4\", nil, base.DefaultQueueName)\n\t\tm5 = h.NewTaskMessageWithQueue(\"task5\", nil, base.DefaultQueueName)\n\t\tm6 = h.NewTaskMessageWithQueue(\"task6\", nil, \"low\")\n\t\tm7 = h.NewTaskMessageWithQueue(\"task7\", nil, \"low\")\n\n\t\tt1 = NewTask(m1.Type, m1.Payload)\n\t\tt2 = NewTask(m2.Type, m2.Payload)\n\t\tt3 = NewTask(m3.Type, m3.Payload)\n\t\tt4 = NewTask(m4.Type, m4.Payload)\n\t\tt5 = NewTask(m5.Type, m5.Payload)\n\t\tt6 = NewTask(m6.Type, m6.Payload)\n\t\tt7 = NewTask(m7.Type, m7.Payload)\n\t)\n\tdefer r.Close()\n\n\ttests := []struct {\n\t\tpending map[string][]*base.TaskMessage // initial queues state\n\t\tqueues []string // list of queues to consume tasks from\n\t\twait time.Duration // wait duration between starting and stopping processor for this test case\n\t\twantProcessed []*Task // tasks to be processed at the end\n\t}{\n\t\t{\n\t\t\tpending: map[string][]*base.TaskMessage{\n\t\t\t\tbase.DefaultQueueName: {m4, m5},\n\t\t\t\t\"critical\": {m1, m2, m3},\n\t\t\t\t\"low\": {m6, m7},\n\t\t\t},\n\t\t\tqueues: []string{base.DefaultQueueName, \"critical\", \"low\"},\n\t\t\twait: time.Second,\n\t\t\twantProcessed: []*Task{t1, t2, t3, t4, t5, t6, t7},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r) // clean up db before each test case.\n\t\tfor qname, msgs := range tc.pending {\n\t\t\th.SeedPendingQueue(t, r, msgs, qname)\n\t\t}\n\n\t\t// instantiate a new processor\n\t\tvar mu sync.Mutex\n\t\tvar processed []*Task\n\t\thandler := func(ctx context.Context, task *Task) error {\n\t\t\tmu.Lock()\n\t\t\tdefer mu.Unlock()\n\t\t\tprocessed = append(processed, task)\n\t\t\treturn nil\n\t\t}\n\t\tqueueCfg := map[string]int{\n\t\t\tbase.DefaultQueueName: 2,\n\t\t\t\"critical\": 3,\n\t\t\t\"low\": 1,\n\t\t}\n\t\tstarting := make(chan *workerInfo)\n\t\tfinished := make(chan *base.TaskMessage)\n\t\tsyncCh := make(chan *syncRequest)\n\t\tdone := make(chan struct{})\n\t\tdefer func() { close(done) }()\n\t\tgo fakeHeartbeater(starting, finished, done)\n\t\tgo fakeSyncer(syncCh, done)\n\t\tp := newProcessor(processorParams{\n\t\t\tlogger: testLogger,\n\t\t\tbroker: rdbClient,\n\t\t\tbaseCtxFn: context.Background,\n\t\t\ttaskCheckInterval: defaultTaskCheckInterval,\n\t\t\tretryDelayFunc: DefaultRetryDelayFunc,\n\t\t\tisFailureFunc: defaultIsFailureFunc,\n\t\t\tsyncCh: syncCh,\n\t\t\tcancelations: base.NewCancelations(),\n\t\t\tconcurrency: 1, // Set concurrency to 1 to make sure tasks are processed one at a time.\n\t\t\tqueues: queueCfg,\n\t\t\tstrictPriority: true,\n\t\t\terrHandler: nil,\n\t\t\tshutdownTimeout: defaultShutdownTimeout,\n\t\t\tstarting: starting,\n\t\t\tfinished: finished,\n\t\t})\n\t\tp.handler = HandlerFunc(handler)\n\n\t\tp.start(&sync.WaitGroup{})\n\t\ttime.Sleep(tc.wait)\n\t\t// Make sure no tasks are stuck in active list.\n\t\tfor _, qname := range tc.queues {\n\t\t\tif l := r.LLen(context.Background(), base.ActiveKey(qname)).Val(); l != 0 {\n\t\t\t\tt.Errorf(\"%q has %d tasks, want 0\", base.ActiveKey(qname), l)\n\t\t\t}\n\t\t}\n\t\tp.shutdown()\n\n\t\tif diff := cmp.Diff(tc.wantProcessed, processed, taskCmpOpts...); diff != \"\" {\n\t\t\tt.Errorf(\"mismatch found in processed tasks; (-want, +got)\\n%s\", diff)\n\t\t}\n\n\t}\n}\n\nfunc TestProcessorPerform(t *testing.T) {\n\ttests := []struct {\n\t\tdesc string\n\t\thandler HandlerFunc\n\t\ttask *Task\n\t\twantErr bool\n\t}{\n\t\t{\n\t\t\tdesc: \"handler returns nil\",\n\t\t\thandler: func(ctx context.Context, t *Task) error {\n\t\t\t\treturn nil\n\t\t\t},\n\t\t\ttask: NewTask(\"gen_thumbnail\", h.JSON(map[string]interface{}{\"src\": \"some/img/path\"})),\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tdesc: \"handler returns error\",\n\t\t\thandler: func(ctx context.Context, t *Task) error {\n\t\t\t\treturn fmt.Errorf(\"something went wrong\")\n\t\t\t},\n\t\t\ttask: NewTask(\"gen_thumbnail\", h.JSON(map[string]interface{}{\"src\": \"some/img/path\"})),\n\t\t\twantErr: true,\n\t\t},\n\t\t{\n\t\t\tdesc: \"handler panics\",\n\t\t\thandler: func(ctx context.Context, t *Task) error {\n\t\t\t\tpanic(\"something went terribly wrong\")\n\t\t\t},\n\t\t\ttask: NewTask(\"gen_thumbnail\", h.JSON(map[string]interface{}{\"src\": \"some/img/path\"})),\n\t\t\twantErr: true,\n\t\t},\n\t}\n\t// Note: We don't need to fully initialized the processor since we are only testing\n\t// perform method.\n\tp := newProcessorForTest(t, nil, nil)\n\n\tfor _, tc := range tests {\n\t\tp.handler = tc.handler\n\t\tgot := p.perform(context.Background(), tc.task)\n\t\tif !tc.wantErr && got != nil {\n\t\t\tt.Errorf(\"%s: perform() = %v, want nil\", tc.desc, got)\n\t\t\tcontinue\n\t\t}\n\t\tif tc.wantErr && got == nil {\n\t\t\tt.Errorf(\"%s: perform() = nil, want non-nil error\", tc.desc)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\nfunc TestGCD(t *testing.T) {\n\ttests := []struct {\n\t\tinput []int\n\t\twant int\n\t}{\n\t\t{[]int{6, 2, 12}, 2},\n\t\t{[]int{3, 3, 3}, 3},\n\t\t{[]int{6, 3, 1}, 1},\n\t\t{[]int{1}, 1},\n\t\t{[]int{1, 0, 2}, 1},\n\t\t{[]int{8, 0, 4}, 4},\n\t\t{[]int{9, 12, 18, 30}, 3},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := gcd(tc.input...)\n\t\tif got != tc.want {\n\t\t\tt.Errorf(\"gcd(%v) = %d, want %d\", tc.input, got, tc.want)\n\t\t}\n\t}\n}\n\nfunc TestNormalizeQueues(t *testing.T) {\n\ttests := []struct {\n\t\tinput map[string]int\n\t\twant map[string]int\n\t}{\n\t\t{\n\t\t\tinput: map[string]int{\n\t\t\t\t\"high\": 100,\n\t\t\t\t\"default\": 20,\n\t\t\t\t\"low\": 5,\n\t\t\t},\n\t\t\twant: map[string]int{\n\t\t\t\t\"high\": 20,\n\t\t\t\t\"default\": 4,\n\t\t\t\t\"low\": 1,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: map[string]int{\n\t\t\t\t\"default\": 10,\n\t\t\t},\n\t\t\twant: map[string]int{\n\t\t\t\t\"default\": 1,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: map[string]int{\n\t\t\t\t\"critical\": 5,\n\t\t\t\t\"default\": 1,\n\t\t\t},\n\t\t\twant: map[string]int{\n\t\t\t\t\"critical\": 5,\n\t\t\t\t\"default\": 1,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: map[string]int{\n\t\t\t\t\"critical\": 6,\n\t\t\t\t\"default\": 3,\n\t\t\t\t\"low\": 0,\n\t\t\t},\n\t\t\twant: map[string]int{\n\t\t\t\t\"critical\": 2,\n\t\t\t\t\"default\": 1,\n\t\t\t\t\"low\": 0,\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := normalizeQueues(tc.input)\n\t\tif diff := cmp.Diff(tc.want, got); diff != \"\" {\n\t\t\tt.Errorf(\"normalizeQueues(%v) = %v, want %v; (-want, +got):\\n%s\",\n\t\t\t\ttc.input, got, tc.want, diff)\n\t\t}\n\t}\n}\n\nfunc TestProcessorComputeDeadline(t *testing.T) {\n\tnow := time.Now()\n\tp := processor{\n\t\tlogger: log.NewLogger(nil),\n\t\tclock: timeutil.NewSimulatedClock(now),\n\t}\n\n\ttests := []struct {\n\t\tdesc string\n\t\tmsg *base.TaskMessage\n\t\twant time.Time\n\t}{\n\t\t{\n\t\t\tdesc: \"message with only timeout specified\",\n\t\t\tmsg: &base.TaskMessage{\n\t\t\t\tTimeout: int64((30 * time.Minute).Seconds()),\n\t\t\t},\n\t\t\twant: now.Add(30 * time.Minute),\n\t\t},\n\t\t{\n\t\t\tdesc: \"message with only deadline specified\",\n\t\t\tmsg: &base.TaskMessage{\n\t\t\t\tDeadline: now.Add(24 * time.Hour).Unix(),\n\t\t\t},\n\t\t\twant: now.Add(24 * time.Hour),\n\t\t},\n\t\t{\n\t\t\tdesc: \"message with both timeout and deadline set (now+timeout < deadline)\",\n\t\t\tmsg: &base.TaskMessage{\n\t\t\t\tDeadline: now.Add(24 * time.Hour).Unix(),\n\t\t\t\tTimeout: int64((30 * time.Minute).Seconds()),\n\t\t\t},\n\t\t\twant: now.Add(30 * time.Minute),\n\t\t},\n\t\t{\n\t\t\tdesc: \"message with both timeout and deadline set (now+timeout > deadline)\",\n\t\t\tmsg: &base.TaskMessage{\n\t\t\t\tDeadline: now.Add(10 * time.Minute).Unix(),\n\t\t\t\tTimeout: int64((30 * time.Minute).Seconds()),\n\t\t\t},\n\t\t\twant: now.Add(10 * time.Minute),\n\t\t},\n\t\t{\n\t\t\tdesc: \"message with both timeout and deadline set (now+timeout == deadline)\",\n\t\t\tmsg: &base.TaskMessage{\n\t\t\t\tDeadline: now.Add(30 * time.Minute).Unix(),\n\t\t\t\tTimeout: int64((30 * time.Minute).Seconds()),\n\t\t\t},\n\t\t\twant: now.Add(30 * time.Minute),\n\t\t},\n\t\t{\n\t\t\tdesc: \"message without timeout and deadline\",\n\t\t\tmsg: &base.TaskMessage{},\n\t\t\twant: now.Add(defaultTimeout),\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := p.computeDeadline(tc.msg)\n\t\t// Compare the Unix epoch with seconds granularity\n\t\tif got.Unix() != tc.want.Unix() {\n\t\t\tt.Errorf(\"%s: got=%v, want=%v\", tc.desc, got.Unix(), tc.want.Unix())\n\t\t}\n\t}\n}\n\nfunc TestReturnPanicError(t *testing.T) {\n\n\ttask := NewTask(\"gen_thumbnail\", h.JSON(map[string]interface{}{\"src\": \"some/img/path\"}))\n\n\ttests := []struct {\n\t\tname string\n\t\thandler HandlerFunc\n\t\tIsPanicError bool\n\t}{\n\t\t{\n\t\t\tname: \"should return panic error when occurred panic recovery\",\n\t\t\thandler: func(ctx context.Context, t *Task) error {\n\t\t\t\tpanic(\"something went terribly wrong\")\n\t\t\t},\n\t\t\tIsPanicError: true,\n\t\t},\n\t\t{\n\t\t\tname: \"should return normal error when don't occur panic recovery\",\n\t\t\thandler: func(ctx context.Context, t *Task) error {\n\t\t\t\treturn fmt.Errorf(\"something went terribly wrong\")\n\t\t\t},\n\t\t\tIsPanicError: false,\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tp := processor{\n\t\t\t\tlogger: log.NewLogger(nil),\n\t\t\t\thandler: tc.handler,\n\t\t\t}\n\t\t\tgot := p.perform(context.Background(), task)\n\t\t\tif tc.IsPanicError != IsPanicError(got) {\n\t\t\t\tt.Errorf(\"%s: got=%t, want=%t\", tc.name, IsPanicError(got), tc.IsPanicError)\n\t\t\t}\n\t\t\tif tc.IsPanicError && !strings.HasPrefix(got.Error(), \"panic error cause by:\") {\n\t\t\t\tt.Error(\"wrong text msg for panic error\")\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "recoverer.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"context\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/errors\"\n\t\"github.com/hibiken/asynq/internal/log\"\n)\n\ntype recoverer struct {\n\tlogger *log.Logger\n\tbroker base.Broker\n\tretryDelayFunc RetryDelayFunc\n\tisFailureFunc func(error) bool\n\n\t// channel to communicate back to the long running \"recoverer\" goroutine.\n\tdone chan struct{}\n\n\t// list of queues to check for deadline.\n\tqueues []string\n\n\t// poll interval.\n\tinterval time.Duration\n}\n\ntype recovererParams struct {\n\tlogger *log.Logger\n\tbroker base.Broker\n\tqueues []string\n\tinterval time.Duration\n\tretryDelayFunc RetryDelayFunc\n\tisFailureFunc func(error) bool\n}\n\nfunc newRecoverer(params recovererParams) *recoverer {\n\treturn &recoverer{\n\t\tlogger: params.logger,\n\t\tbroker: params.broker,\n\t\tdone: make(chan struct{}),\n\t\tqueues: params.queues,\n\t\tinterval: params.interval,\n\t\tretryDelayFunc: params.retryDelayFunc,\n\t\tisFailureFunc: params.isFailureFunc,\n\t}\n}\n\nfunc (r *recoverer) shutdown() {\n\tr.logger.Debug(\"Recoverer shutting down...\")\n\t// Signal the recoverer goroutine to stop polling.\n\tr.done <- struct{}{}\n}\n\nfunc (r *recoverer) start(wg *sync.WaitGroup) {\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tr.recover()\n\t\ttimer := time.NewTimer(r.interval)\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-r.done:\n\t\t\t\tr.logger.Debug(\"Recoverer done\")\n\t\t\t\ttimer.Stop()\n\t\t\t\treturn\n\t\t\tcase <-timer.C:\n\t\t\t\tr.recover()\n\t\t\t\ttimer.Reset(r.interval)\n\t\t\t}\n\t\t}\n\t}()\n}\n\n// ErrLeaseExpired error indicates that the task failed because the worker working on the task\n// could not extend its lease due to missing heartbeats. The worker may have crashed or got cutoff from the network.\nvar ErrLeaseExpired = errors.New(\"asynq: task lease expired\")\n\nfunc (r *recoverer) recover() {\n\tr.recoverLeaseExpiredTasks()\n\tr.recoverStaleAggregationSets()\n}\n\nfunc (r *recoverer) recoverLeaseExpiredTasks() {\n\t// Get all tasks which have expired 30 seconds ago or earlier to accommodate certain amount of clock skew.\n\tcutoff := time.Now().Add(-30 * time.Second)\n\tmsgs, err := r.broker.ListLeaseExpired(cutoff, r.queues...)\n\tif err != nil {\n\t\tr.logger.Warnf(\"recoverer: could not list lease expired tasks: %v\", err)\n\t\treturn\n\t}\n\tfor _, msg := range msgs {\n\t\tif msg.Retried >= msg.Retry {\n\t\t\tr.archive(msg, ErrLeaseExpired)\n\t\t} else {\n\t\t\tr.retry(msg, ErrLeaseExpired)\n\t\t}\n\t}\n}\n\nfunc (r *recoverer) recoverStaleAggregationSets() {\n\tfor _, qname := range r.queues {\n\t\tif err := r.broker.ReclaimStaleAggregationSets(qname); err != nil {\n\t\t\tr.logger.Warnf(\"recoverer: could not reclaim stale aggregation sets in queue %q: %v\", qname, err)\n\t\t}\n\t}\n}\n\nfunc (r *recoverer) retry(msg *base.TaskMessage, err error) {\n\tdelay := r.retryDelayFunc(msg.Retried, err, NewTaskWithHeaders(msg.Type, msg.Payload, msg.Headers))\n\tretryAt := time.Now().Add(delay)\n\tif err := r.broker.Retry(context.Background(), msg, retryAt, err.Error(), r.isFailureFunc(err)); err != nil {\n\t\tr.logger.Warnf(\"recoverer: could not retry lease expired task: %v\", err)\n\t}\n}\n\nfunc (r *recoverer) archive(msg *base.TaskMessage, err error) {\n\tif err := r.broker.Archive(context.Background(), msg, err.Error()); err != nil {\n\t\tr.logger.Warnf(\"recoverer: could not move task to archive: %v\", err)\n\t}\n}\n" }, { "path": "recoverer_test.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/google/go-cmp/cmp\"\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/rdb\"\n\th \"github.com/hibiken/asynq/internal/testutil\"\n)\n\nfunc TestRecoverer(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\trdbClient := rdb.NewRDB(r)\n\n\tt1 := h.NewTaskMessageWithQueue(\"task1\", nil, \"default\")\n\tt2 := h.NewTaskMessageWithQueue(\"task2\", nil, \"default\")\n\tt3 := h.NewTaskMessageWithQueue(\"task3\", nil, \"critical\")\n\tt4 := h.NewTaskMessageWithQueue(\"task4\", nil, \"default\")\n\tt4.Retried = t4.Retry // t4 has reached its max retry count\n\n\tnow := time.Now()\n\n\ttests := []struct {\n\t\tdesc string\n\t\tactive map[string][]*base.TaskMessage\n\t\tlease map[string][]base.Z\n\t\tretry map[string][]base.Z\n\t\tarchived map[string][]base.Z\n\t\twantActive map[string][]*base.TaskMessage\n\t\twantLease map[string][]base.Z\n\t\twantRetry map[string][]*base.TaskMessage\n\t\twantArchived map[string][]*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tdesc: \"with one active task\",\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t1, Score: now.Add(-1 * time.Minute).Unix()}},\n\t\t\t},\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t\twantRetry: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t},\n\t\t\twantArchived: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with a task with max-retry reached\",\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t4},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t4, Score: now.Add(-40 * time.Second).Unix()}},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\twantRetry: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\twantArchived: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t4},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with multiple active tasks, and one expired\",\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t2},\n\t\t\t\t\"critical\": {t3},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: now.Add(-2 * time.Minute).Unix()},\n\t\t\t\t\t{Message: t2, Score: now.Add(20 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t\t\"critical\": {\n\t\t\t\t\t{Message: t3, Score: now.Add(20 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t2},\n\t\t\t\t\"critical\": {t3},\n\t\t\t},\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t2, Score: now.Add(20 * time.Second).Unix()}},\n\t\t\t\t\"critical\": {{Message: t3, Score: now.Add(20 * time.Second).Unix()}},\n\t\t\t},\n\t\t\twantRetry: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\twantArchived: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with multiple expired active tasks\",\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1, t2},\n\t\t\t\t\"critical\": {t3},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {\n\t\t\t\t\t{Message: t1, Score: now.Add(-1 * time.Minute).Unix()},\n\t\t\t\t\t{Message: t2, Score: now.Add(10 * time.Second).Unix()},\n\t\t\t\t},\n\t\t\t\t\"critical\": {\n\t\t\t\t\t{Message: t3, Score: now.Add(-1 * time.Minute).Unix()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"cricial\": {},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"cricial\": {},\n\t\t\t},\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t2},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {{Message: t2, Score: now.Add(10 * time.Second).Unix()}},\n\t\t\t},\n\t\t\twantRetry: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {t1},\n\t\t\t\t\"critical\": {t3},\n\t\t\t},\n\t\t\twantArchived: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"with empty active queue\",\n\t\t\tactive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\tlease: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\tretry: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\tarchived: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\twantActive: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\twantLease: map[string][]base.Z{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\twantRetry: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t\twantArchived: map[string][]*base.TaskMessage{\n\t\t\t\t\"default\": {},\n\t\t\t\t\"critical\": {},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\th.FlushDB(t, r)\n\t\th.SeedAllActiveQueues(t, r, tc.active)\n\t\th.SeedAllLease(t, r, tc.lease)\n\t\th.SeedAllRetryQueues(t, r, tc.retry)\n\t\th.SeedAllArchivedQueues(t, r, tc.archived)\n\n\t\trecoverer := newRecoverer(recovererParams{\n\t\t\tlogger: testLogger,\n\t\t\tbroker: rdbClient,\n\t\t\tqueues: []string{\"default\", \"critical\"},\n\t\t\tinterval: 1 * time.Second,\n\t\t\tretryDelayFunc: func(n int, err error, task *Task) time.Duration { return 30 * time.Second },\n\t\t\tisFailureFunc: defaultIsFailureFunc,\n\t\t})\n\n\t\tvar wg sync.WaitGroup\n\t\trecoverer.start(&wg)\n\t\trunTime := time.Now() // time when recoverer is running\n\t\ttime.Sleep(2 * time.Second)\n\t\trecoverer.shutdown()\n\n\t\tfor qname, want := range tc.wantActive {\n\t\t\tgotActive := h.GetActiveMessages(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotActive, h.SortMsgOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s; mismatch found in %q; (-want,+got)\\n%s\", tc.desc, base.ActiveKey(qname), diff)\n\t\t\t}\n\t\t}\n\t\tfor qname, want := range tc.wantLease {\n\t\t\tgotLease := h.GetLeaseEntries(t, r, qname)\n\t\t\tif diff := cmp.Diff(want, gotLease, h.SortZSetEntryOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s; mismatch found in %q; (-want,+got)\\n%s\", tc.desc, base.LeaseKey(qname), diff)\n\t\t\t}\n\t\t}\n\t\tcmpOpt := h.EquateInt64Approx(2) // allow up to two-second difference in `LastFailedAt`\n\t\tfor qname, msgs := range tc.wantRetry {\n\t\t\tgotRetry := h.GetRetryMessages(t, r, qname)\n\t\t\tvar wantRetry []*base.TaskMessage // Note: construct message here since `LastFailedAt` is relative to each test run\n\t\t\tfor _, msg := range msgs {\n\t\t\t\twantRetry = append(wantRetry, h.TaskMessageAfterRetry(*msg, ErrLeaseExpired.Error(), runTime))\n\t\t\t}\n\t\t\tif diff := cmp.Diff(wantRetry, gotRetry, h.SortMsgOpt, cmpOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s; mismatch found in %q: (-want, +got)\\n%s\", tc.desc, base.RetryKey(qname), diff)\n\t\t\t}\n\t\t}\n\t\tfor qname, msgs := range tc.wantArchived {\n\t\t\tgotArchived := h.GetArchivedMessages(t, r, qname)\n\t\t\tvar wantArchived []*base.TaskMessage\n\t\t\tfor _, msg := range msgs {\n\t\t\t\twantArchived = append(wantArchived, h.TaskMessageWithError(*msg, ErrLeaseExpired.Error(), runTime))\n\t\t\t}\n\t\t\tif diff := cmp.Diff(wantArchived, gotArchived, h.SortMsgOpt, cmpOpt); diff != \"\" {\n\t\t\t\tt.Errorf(\"%s; mismatch found in %q: (-want, +got)\\n%s\", tc.desc, base.ArchivedKey(qname), diff)\n\t\t\t}\n\t\t}\n\t}\n}\n" }, { "path": "scheduler.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/google/uuid\"\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/log\"\n\t\"github.com/hibiken/asynq/internal/rdb\"\n\t\"github.com/redis/go-redis/v9\"\n\t\"github.com/robfig/cron/v3\"\n)\n\n// A Scheduler kicks off tasks at regular intervals based on the user defined schedule.\n//\n// Schedulers are safe for concurrent use by multiple goroutines.\ntype Scheduler struct {\n\tid string\n\n\tstate *serverState\n\n\theartbeatInterval time.Duration\n\tlogger *log.Logger\n\tclient *Client\n\trdb *rdb.RDB\n\tcron *cron.Cron\n\tlocation *time.Location\n\tdone chan struct{}\n\twg sync.WaitGroup\n\tpreEnqueueFunc func(task *Task, opts []Option)\n\tpostEnqueueFunc func(info *TaskInfo, err error)\n\terrHandler func(task *Task, opts []Option, err error)\n\n\t// guards idmap\n\tmu sync.Mutex\n\t// idmap maps Scheduler's entry ID to cron.EntryID\n\t// to avoid using cron.EntryID as the public API of\n\t// the Scheduler.\n\tidmap map[string]cron.EntryID\n}\n\nconst defaultHeartbeatInterval = 10 * time.Second\n\n// NewScheduler returns a new Scheduler instance given the redis connection option.\n// The parameter opts is optional, defaults will be used if opts is set to nil\nfunc NewScheduler(r RedisConnOpt, opts *SchedulerOpts) *Scheduler {\n\tscheduler := newScheduler(opts)\n\n\tredisClient, ok := r.MakeRedisClient().(redis.UniversalClient)\n\tif !ok {\n\t\tpanic(fmt.Sprintf(\"asynq: unsupported RedisConnOpt type %T\", r))\n\t}\n\n\trdb := rdb.NewRDB(redisClient)\n\n\tscheduler.rdb = rdb\n\tscheduler.client = &Client{broker: rdb, sharedConnection: false}\n\n\treturn scheduler\n}\n\n// NewSchedulerFromRedisClient returns a new instance of Scheduler given a redis.UniversalClient\n// The parameter opts is optional, defaults will be used if opts is set to nil.\n// Warning: The underlying redis connection pool will not be closed by Asynq, you are responsible for closing it.\nfunc NewSchedulerFromRedisClient(c redis.UniversalClient, opts *SchedulerOpts) *Scheduler {\n\tscheduler := newScheduler(opts)\n\n\tscheduler.rdb = rdb.NewRDB(c)\n\tscheduler.client = NewClientFromRedisClient(c)\n\n\treturn scheduler\n}\n\nfunc newScheduler(opts *SchedulerOpts) *Scheduler {\n\tif opts == nil {\n\t\topts = &SchedulerOpts{}\n\t}\n\n\theartbeatInterval := opts.HeartbeatInterval\n\tif heartbeatInterval <= 0 {\n\t\theartbeatInterval = defaultHeartbeatInterval\n\t}\n\n\tlogger := log.NewLogger(opts.Logger)\n\tloglevel := opts.LogLevel\n\tif loglevel == level_unspecified {\n\t\tloglevel = InfoLevel\n\t}\n\tlogger.SetLevel(toInternalLogLevel(loglevel))\n\n\tloc := opts.Location\n\tif loc == nil {\n\t\tloc = time.UTC\n\t}\n\n\treturn &Scheduler{\n\t\tid: generateSchedulerID(),\n\t\tstate: &serverState{value: srvStateNew},\n\t\theartbeatInterval: heartbeatInterval,\n\t\tlogger: logger,\n\t\tcron: cron.New(cron.WithLocation(loc)),\n\t\tlocation: loc,\n\t\tdone: make(chan struct{}),\n\t\tpreEnqueueFunc: opts.PreEnqueueFunc,\n\t\tpostEnqueueFunc: opts.PostEnqueueFunc,\n\t\terrHandler: opts.EnqueueErrorHandler,\n\t\tidmap: make(map[string]cron.EntryID),\n\t}\n}\n\nfunc generateSchedulerID() string {\n\thost, err := os.Hostname()\n\tif err != nil {\n\t\thost = \"unknown-host\"\n\t}\n\treturn fmt.Sprintf(\"%s:%d:%v\", host, os.Getpid(), uuid.New())\n}\n\n// SchedulerOpts specifies scheduler options.\ntype SchedulerOpts struct {\n\t// HeartbeatInterval specifies the interval between scheduler heartbeats.\n\t//\n\t// If unset, zero or a negative value, the interval is set to 10 second.\n\t//\n\t// Note: Setting this value too low may add significant load to redis.\n\t//\n\t// By default, HeartbeatInterval is set to 10 seconds.\n\tHeartbeatInterval time.Duration\n\n\t// Logger specifies the logger used by the scheduler instance.\n\t//\n\t// If unset, the default logger is used.\n\tLogger Logger\n\n\t// LogLevel specifies the minimum log level to enable.\n\t//\n\t// If unset, InfoLevel is used by default.\n\tLogLevel LogLevel\n\n\t// Location specifies the time zone location.\n\t//\n\t// If unset, the UTC time zone (time.UTC) is used.\n\tLocation *time.Location\n\n\t// PreEnqueueFunc, if provided, is called before a task gets enqueued by Scheduler.\n\t// The callback function should return quickly to not block the current thread.\n\tPreEnqueueFunc func(task *Task, opts []Option)\n\n\t// PostEnqueueFunc, if provided, is called after a task gets enqueued by Scheduler.\n\t// The callback function should return quickly to not block the current thread.\n\tPostEnqueueFunc func(info *TaskInfo, err error)\n\n\t// Deprecated: Use PostEnqueueFunc instead\n\t// EnqueueErrorHandler gets called when scheduler cannot enqueue a registered task\n\t// due to an error.\n\tEnqueueErrorHandler func(task *Task, opts []Option, err error)\n}\n\n// enqueueJob encapsulates the job of enqueuing a task and recording the event.\ntype enqueueJob struct {\n\tid uuid.UUID\n\tcronspec string\n\ttask *Task\n\topts []Option\n\tlocation *time.Location\n\tlogger *log.Logger\n\tclient *Client\n\trdb *rdb.RDB\n\tpreEnqueueFunc func(task *Task, opts []Option)\n\tpostEnqueueFunc func(info *TaskInfo, err error)\n\terrHandler func(task *Task, opts []Option, err error)\n}\n\nfunc (j *enqueueJob) Run() {\n\tif j.preEnqueueFunc != nil {\n\t\tj.preEnqueueFunc(j.task, j.opts)\n\t}\n\tinfo, err := j.client.Enqueue(j.task, j.opts...)\n\tif j.postEnqueueFunc != nil {\n\t\tj.postEnqueueFunc(info, err)\n\t}\n\tif err != nil {\n\t\tif j.errHandler != nil {\n\t\t\tj.errHandler(j.task, j.opts, err)\n\t\t}\n\t\treturn\n\t}\n\tj.logger.Debugf(\"scheduler enqueued a task: %+v\", info)\n\tevent := &base.SchedulerEnqueueEvent{\n\t\tTaskID: info.ID,\n\t\tEnqueuedAt: time.Now().In(j.location),\n\t}\n\terr = j.rdb.RecordSchedulerEnqueueEvent(j.id.String(), event)\n\tif err != nil {\n\t\tj.logger.Warnf(\"scheduler could not record enqueue event of enqueued task %s: %v\", info.ID, err)\n\t}\n}\n\n// Register registers a task to be enqueued on the given schedule specified by the cronspec.\n// It returns an ID of the newly registered entry.\nfunc (s *Scheduler) Register(cronspec string, task *Task, opts ...Option) (entryID string, err error) {\n\tjob := &enqueueJob{\n\t\tid: uuid.New(),\n\t\tcronspec: cronspec,\n\t\ttask: task,\n\t\topts: opts,\n\t\tlocation: s.location,\n\t\tclient: s.client,\n\t\trdb: s.rdb,\n\t\tlogger: s.logger,\n\t\tpreEnqueueFunc: s.preEnqueueFunc,\n\t\tpostEnqueueFunc: s.postEnqueueFunc,\n\t\terrHandler: s.errHandler,\n\t}\n\tcronID, err := s.cron.AddJob(cronspec, job)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\ts.mu.Lock()\n\ts.idmap[job.id.String()] = cronID\n\ts.mu.Unlock()\n\treturn job.id.String(), nil\n}\n\n// Unregister removes a registered entry by entry ID.\n// Unregister returns a non-nil error if no entries were found for the given entryID.\nfunc (s *Scheduler) Unregister(entryID string) error {\n\ts.mu.Lock()\n\tdefer s.mu.Unlock()\n\tcronID, ok := s.idmap[entryID]\n\tif !ok {\n\t\treturn fmt.Errorf(\"asynq: no scheduler entry found\")\n\t}\n\tdelete(s.idmap, entryID)\n\ts.cron.Remove(cronID)\n\treturn nil\n}\n\n// Run starts the scheduler until an os signal to exit the program is received.\n// It returns an error if scheduler is already running or has been shutdown.\nfunc (s *Scheduler) Run() error {\n\tif err := s.Start(); err != nil {\n\t\treturn err\n\t}\n\ts.waitForSignals()\n\ts.Shutdown()\n\treturn nil\n}\n\n// Start starts the scheduler.\n// It returns an error if the scheduler is already running or has been shutdown.\nfunc (s *Scheduler) Start() error {\n\tif err := s.start(); err != nil {\n\t\treturn err\n\t}\n\ts.logger.Info(\"Scheduler starting\")\n\ts.logger.Infof(\"Scheduler timezone is set to %v\", s.location)\n\ts.cron.Start()\n\ts.wg.Add(1)\n\tgo s.runHeartbeater()\n\treturn nil\n}\n\n// Checks server state and returns an error if pre-condition is not met.\n// Otherwise it sets the server state to active.\nfunc (s *Scheduler) start() error {\n\ts.state.mu.Lock()\n\tdefer s.state.mu.Unlock()\n\tswitch s.state.value {\n\tcase srvStateActive:\n\t\treturn fmt.Errorf(\"asynq: the scheduler is already running\")\n\tcase srvStateClosed:\n\t\treturn fmt.Errorf(\"asynq: the scheduler has already been stopped\")\n\t}\n\ts.state.value = srvStateActive\n\treturn nil\n}\n\n// Shutdown stops and shuts down the scheduler.\nfunc (s *Scheduler) Shutdown() {\n\ts.state.mu.Lock()\n\tif s.state.value == srvStateNew || s.state.value == srvStateClosed {\n\t\t// scheduler is not running, do nothing and return.\n\t\ts.state.mu.Unlock()\n\t\treturn\n\t}\n\ts.state.value = srvStateClosed\n\ts.state.mu.Unlock()\n\n\ts.logger.Info(\"Scheduler shutting down\")\n\tclose(s.done) // signal heartbeater to stop\n\tctx := s.cron.Stop()\n\t<-ctx.Done()\n\ts.wg.Wait()\n\n\ts.clearHistory()\n\tif err := s.client.Close(); err != nil {\n\t\ts.logger.Errorf(\"Failed to close redis client connection: %v\", err)\n\t}\n\ts.logger.Info(\"Scheduler stopped\")\n}\n\nfunc (s *Scheduler) runHeartbeater() {\n\tdefer s.wg.Done()\n\tticker := time.NewTicker(s.heartbeatInterval)\n\tfor {\n\t\tselect {\n\t\tcase <-s.done:\n\t\t\ts.logger.Debugf(\"Scheduler heatbeater shutting down\")\n\t\t\tif err := s.rdb.ClearSchedulerEntries(s.id); err != nil {\n\t\t\t\ts.logger.Errorf(\"Failed to clear the scheduler entries: %v\", err)\n\t\t\t}\n\t\t\tticker.Stop()\n\t\t\treturn\n\t\tcase <-ticker.C:\n\t\t\ts.beat()\n\t\t}\n\t}\n}\n\n// beat writes a snapshot of entries to redis.\nfunc (s *Scheduler) beat() {\n\tvar entries []*base.SchedulerEntry\n\tfor _, entry := range s.cron.Entries() {\n\t\tjob := entry.Job.(*enqueueJob)\n\t\te := &base.SchedulerEntry{\n\t\t\tID: job.id.String(),\n\t\t\tSpec: job.cronspec,\n\t\t\tType: job.task.Type(),\n\t\t\tPayload: job.task.Payload(),\n\t\t\tOpts: stringifyOptions(job.opts),\n\t\t\tNext: entry.Next,\n\t\t\tPrev: entry.Prev,\n\t\t}\n\t\tentries = append(entries, e)\n\t}\n\tif err := s.rdb.WriteSchedulerEntries(s.id, entries, s.heartbeatInterval*2); err != nil {\n\t\ts.logger.Warnf(\"Scheduler could not write heartbeat data: %v\", err)\n\t}\n}\n\nfunc stringifyOptions(opts []Option) []string {\n\tvar res []string\n\tfor _, opt := range opts {\n\t\tres = append(res, opt.String())\n\t}\n\treturn res\n}\n\nfunc (s *Scheduler) clearHistory() {\n\tfor _, entry := range s.cron.Entries() {\n\t\tjob := entry.Job.(*enqueueJob)\n\t\tif err := s.rdb.ClearSchedulerHistory(job.id.String()); err != nil {\n\t\t\ts.logger.Warnf(\"Could not clear scheduler history for entry %q: %v\", job.id.String(), err)\n\t\t}\n\t}\n}\n\n// Ping performs a ping against the redis connection.\nfunc (s *Scheduler) Ping() error {\n\ts.state.mu.Lock()\n\tdefer s.state.mu.Unlock()\n\tif s.state.value == srvStateClosed {\n\t\treturn nil\n\t}\n\n\treturn s.rdb.Ping()\n}\n" }, { "path": "scheduler_test.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/google/go-cmp/cmp\"\n\t\"github.com/redis/go-redis/v9\"\n\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/testutil\"\n)\n\nfunc TestSchedulerRegister(t *testing.T) {\n\ttests := []struct {\n\t\tcronspec string\n\t\ttask *Task\n\t\topts []Option\n\t\twait time.Duration\n\t\tqueue string\n\t\twant []*base.TaskMessage\n\t}{\n\t\t{\n\t\t\tcronspec: \"@every 3s\",\n\t\t\ttask: NewTask(\"task1\", nil),\n\t\t\topts: []Option{MaxRetry(10)},\n\t\t\twait: 10 * time.Second,\n\t\t\tqueue: \"default\",\n\t\t\twant: []*base.TaskMessage{\n\t\t\t\t{\n\t\t\t\t\tType: \"task1\",\n\t\t\t\t\tPayload: nil,\n\t\t\t\t\tRetry: 10,\n\t\t\t\t\tTimeout: int64(defaultTimeout.Seconds()),\n\t\t\t\t\tQueue: \"default\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tType: \"task1\",\n\t\t\t\t\tPayload: nil,\n\t\t\t\t\tRetry: 10,\n\t\t\t\t\tTimeout: int64(defaultTimeout.Seconds()),\n\t\t\t\t\tQueue: \"default\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tType: \"task1\",\n\t\t\t\t\tPayload: nil,\n\t\t\t\t\tRetry: 10,\n\t\t\t\t\tTimeout: int64(defaultTimeout.Seconds()),\n\t\t\t\t\tQueue: \"default\",\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tr := setup(t)\n\n\t// Tests for new redis connection.\n\tfor _, tc := range tests {\n\t\tscheduler := NewScheduler(getRedisConnOpt(t), nil)\n\t\tif _, err := scheduler.Register(tc.cronspec, tc.task, tc.opts...); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tif err := scheduler.Start(); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\ttime.Sleep(tc.wait)\n\t\tscheduler.Shutdown()\n\n\t\tgot := testutil.GetPendingMessages(t, r, tc.queue)\n\t\tif diff := cmp.Diff(tc.want, got, testutil.IgnoreIDOpt); diff != \"\" {\n\t\t\tt.Errorf(\"mismatch found in queue %q: (-want,+got)\\n%s\", tc.queue, diff)\n\t\t}\n\t}\n\n\tr = setup(t)\n\n\t// Tests for existing redis connection.\n\tfor _, tc := range tests {\n\t\tredisClient := getRedisConnOpt(t).MakeRedisClient().(redis.UniversalClient)\n\t\tscheduler := NewSchedulerFromRedisClient(redisClient, nil)\n\t\tif _, err := scheduler.Register(tc.cronspec, tc.task, tc.opts...); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tif err := scheduler.Start(); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\ttime.Sleep(tc.wait)\n\t\tscheduler.Shutdown()\n\n\t\tgot := testutil.GetPendingMessages(t, r, tc.queue)\n\t\tif diff := cmp.Diff(tc.want, got, testutil.IgnoreIDOpt); diff != \"\" {\n\t\t\tt.Errorf(\"mismatch found in queue %q: (-want,+got)\\n%s\", tc.queue, diff)\n\t\t}\n\t}\n}\n\nfunc TestSchedulerWhenRedisDown(t *testing.T) {\n\tvar (\n\t\tmu sync.Mutex\n\t\tcounter int\n\t)\n\terrorHandler := func(task *Task, opts []Option, err error) {\n\t\tmu.Lock()\n\t\tcounter++\n\t\tmu.Unlock()\n\t}\n\n\t// Connect to non-existent redis instance to simulate a redis server being down.\n\tscheduler := NewScheduler(\n\t\tRedisClientOpt{Addr: \":9876\"}, // no Redis listening to this port.\n\t\t&SchedulerOpts{EnqueueErrorHandler: errorHandler},\n\t)\n\n\ttask := NewTask(\"test\", nil)\n\n\tif _, err := scheduler.Register(\"@every 3s\", task); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif err := scheduler.Start(); err != nil {\n\t\tt.Fatal(err)\n\t}\n\t// Scheduler should attempt to enqueue the task three times (every 3s).\n\ttime.Sleep(10 * time.Second)\n\tscheduler.Shutdown()\n\n\tmu.Lock()\n\tif counter != 3 {\n\t\tt.Errorf(\"EnqueueErrorHandler was called %d times, want 3\", counter)\n\t}\n\tmu.Unlock()\n}\n\nfunc TestSchedulerUnregister(t *testing.T) {\n\ttests := []struct {\n\t\tcronspec string\n\t\ttask *Task\n\t\topts []Option\n\t\twait time.Duration\n\t\tqueue string\n\t}{\n\t\t{\n\t\t\tcronspec: \"@every 3s\",\n\t\t\ttask: NewTask(\"task1\", nil),\n\t\t\topts: []Option{MaxRetry(10)},\n\t\t\twait: 10 * time.Second,\n\t\t\tqueue: \"default\",\n\t\t},\n\t}\n\n\tr := setup(t)\n\n\tfor _, tc := range tests {\n\t\tscheduler := NewScheduler(getRedisConnOpt(t), nil)\n\t\tentryID, err := scheduler.Register(tc.cronspec, tc.task, tc.opts...)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif err := scheduler.Unregister(entryID); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tif err := scheduler.Start(); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\ttime.Sleep(tc.wait)\n\t\tscheduler.Shutdown()\n\n\t\tgot := testutil.GetPendingMessages(t, r, tc.queue)\n\t\tif len(got) != 0 {\n\t\t\tt.Errorf(\"%d tasks were enqueued, want zero\", len(got))\n\t\t}\n\t}\n}\n\nfunc TestSchedulerPostAndPreEnqueueHandler(t *testing.T) {\n\tvar (\n\t\tpreMu sync.Mutex\n\t\tpreCounter int\n\t\tpostMu sync.Mutex\n\t\tpostCounter int\n\t)\n\tpreHandler := func(task *Task, opts []Option) {\n\t\tpreMu.Lock()\n\t\tpreCounter++\n\t\tpreMu.Unlock()\n\t}\n\tpostHandler := func(info *TaskInfo, err error) {\n\t\tpostMu.Lock()\n\t\tpostCounter++\n\t\tpostMu.Unlock()\n\t}\n\n\t// Connect to non-existent redis instance to simulate a redis server being down.\n\tscheduler := NewScheduler(\n\t\tgetRedisConnOpt(t),\n\t\t&SchedulerOpts{\n\t\t\tPreEnqueueFunc: preHandler,\n\t\t\tPostEnqueueFunc: postHandler,\n\t\t},\n\t)\n\n\ttask := NewTask(\"test\", nil)\n\n\tif _, err := scheduler.Register(\"@every 3s\", task); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif err := scheduler.Start(); err != nil {\n\t\tt.Fatal(err)\n\t}\n\t// Scheduler should attempt to enqueue the task three times (every 3s).\n\ttime.Sleep(10 * time.Second)\n\tscheduler.Shutdown()\n\n\tpreMu.Lock()\n\tif preCounter != 3 {\n\t\tt.Errorf(\"PreEnqueueFunc was called %d times, want 3\", preCounter)\n\t}\n\tpreMu.Unlock()\n\n\tpostMu.Lock()\n\tif postCounter != 3 {\n\t\tt.Errorf(\"PostEnqueueFunc was called %d times, want 3\", postCounter)\n\t}\n\tpostMu.Unlock()\n}\n" }, { "path": "servemux.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"sort\"\n\t\"strings\"\n\t\"sync\"\n)\n\n// ErrHandlerNotFound indicates that no task handler was found for a given pattern.\nvar ErrHandlerNotFound = errors.New(\"handler not found for task\")\n\n// ServeMux is a multiplexer for asynchronous tasks.\n// It matches the type of each task against a list of registered patterns\n// and calls the handler for the pattern that most closely matches the\n// task's type name.\n//\n// Longer patterns take precedence over shorter ones, so that if there are\n// handlers registered for both \"images\" and \"images:thumbnails\",\n// the latter handler will be called for tasks with a type name beginning with\n// \"images:thumbnails\" and the former will receive tasks with type name beginning\n// with \"images\".\ntype ServeMux struct {\n\tmu sync.RWMutex\n\tm map[string]muxEntry\n\tes []muxEntry // slice of entries sorted from longest to shortest.\n\tmws []MiddlewareFunc\n}\n\ntype muxEntry struct {\n\th Handler\n\tpattern string\n}\n\n// MiddlewareFunc is a function which receives an asynq.Handler and returns another asynq.Handler.\n// Typically, the returned handler is a closure which does something with the context and task passed\n// to it, and then calls the handler passed as parameter to the MiddlewareFunc.\ntype MiddlewareFunc func(Handler) Handler\n\n// NewServeMux allocates and returns a new ServeMux.\nfunc NewServeMux() *ServeMux {\n\treturn new(ServeMux)\n}\n\n// ProcessTask dispatches the task to the handler whose\n// pattern most closely matches the task type.\nfunc (mux *ServeMux) ProcessTask(ctx context.Context, task *Task) error {\n\th, _ := mux.Handler(task)\n\treturn h.ProcessTask(ctx, task)\n}\n\n// Handler returns the handler to use for the given task.\n// It always return a non-nil handler.\n//\n// Handler also returns the registered pattern that matches the task.\n//\n// If there is no registered handler that applies to the task,\n// handler returns a 'not found' handler which returns an error.\nfunc (mux *ServeMux) Handler(t *Task) (h Handler, pattern string) {\n\tmux.mu.RLock()\n\tdefer mux.mu.RUnlock()\n\n\th, pattern = mux.match(t.Type())\n\tif h == nil {\n\t\th, pattern = NotFoundHandler(), \"\"\n\t}\n\tfor i := len(mux.mws) - 1; i >= 0; i-- {\n\t\th = mux.mws[i](h)\n\t}\n\treturn h, pattern\n}\n\n// Find a handler on a handler map given a typename string.\n// Most-specific (longest) pattern wins.\nfunc (mux *ServeMux) match(typename string) (h Handler, pattern string) {\n\t// Check for exact match first.\n\tv, ok := mux.m[typename]\n\tif ok {\n\t\treturn v.h, v.pattern\n\t}\n\n\t// Check for longest valid match.\n\t// mux.es contains all patterns from longest to shortest.\n\tfor _, e := range mux.es {\n\t\tif strings.HasPrefix(typename, e.pattern) {\n\t\t\treturn e.h, e.pattern\n\t\t}\n\t}\n\treturn nil, \"\"\n\n}\n\n// Handle registers the handler for the given pattern.\n// If a handler already exists for pattern, Handle panics.\nfunc (mux *ServeMux) Handle(pattern string, handler Handler) {\n\tmux.mu.Lock()\n\tdefer mux.mu.Unlock()\n\n\tif strings.TrimSpace(pattern) == \"\" {\n\t\tpanic(\"asynq: invalid pattern\")\n\t}\n\tif handler == nil {\n\t\tpanic(\"asynq: nil handler\")\n\t}\n\tif _, exist := mux.m[pattern]; exist {\n\t\tpanic(\"asynq: multiple registrations for \" + pattern)\n\t}\n\n\tif mux.m == nil {\n\t\tmux.m = make(map[string]muxEntry)\n\t}\n\te := muxEntry{h: handler, pattern: pattern}\n\tmux.m[pattern] = e\n\tmux.es = appendSorted(mux.es, e)\n}\n\nfunc appendSorted(es []muxEntry, e muxEntry) []muxEntry {\n\tn := len(es)\n\ti := sort.Search(n, func(i int) bool {\n\t\treturn len(es[i].pattern) < len(e.pattern)\n\t})\n\tif i == n {\n\t\treturn append(es, e)\n\t}\n\t// we now know that i points at where we want to insert.\n\tes = append(es, muxEntry{}) // try to grow the slice in place, any entry works.\n\tcopy(es[i+1:], es[i:]) // shift shorter entries down.\n\tes[i] = e\n\treturn es\n}\n\n// HandleFunc registers the handler function for the given pattern.\nfunc (mux *ServeMux) HandleFunc(pattern string, handler func(context.Context, *Task) error) {\n\tif handler == nil {\n\t\tpanic(\"asynq: nil handler\")\n\t}\n\tmux.Handle(pattern, HandlerFunc(handler))\n}\n\n// Use appends a MiddlewareFunc to the chain.\n// Middlewares are executed in the order that they are applied to the ServeMux.\nfunc (mux *ServeMux) Use(mws ...MiddlewareFunc) {\n\tmux.mu.Lock()\n\tdefer mux.mu.Unlock()\n\tmux.mws = append(mux.mws, mws...)\n}\n\n// NotFound returns an error indicating that the handler was not found for the given task.\nfunc NotFound(ctx context.Context, task *Task) error {\n\treturn fmt.Errorf(\"%w %q\", ErrHandlerNotFound, task.Type())\n}\n\n// NotFoundHandler returns a simple task handler that returns a “not found“ error.\nfunc NotFoundHandler() Handler { return HandlerFunc(NotFound) }\n" }, { "path": "servemux_test.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\t\"github.com/google/go-cmp/cmp\"\n)\n\nvar called string // identity of the handler that was called.\nvar invoked []string // list of middlewares in the order they were invoked.\n\n// makeFakeHandler returns a handler that updates the global called variable\n// to the given identity.\nfunc makeFakeHandler(identity string) Handler {\n\treturn HandlerFunc(func(ctx context.Context, t *Task) error {\n\t\tcalled = identity\n\t\treturn nil\n\t})\n}\n\n// makeFakeMiddleware returns a middleware function that appends the given identity\n// to the global invoked slice.\nfunc makeFakeMiddleware(identity string) MiddlewareFunc {\n\treturn func(next Handler) Handler {\n\t\treturn HandlerFunc(func(ctx context.Context, t *Task) error {\n\t\t\tinvoked = append(invoked, identity)\n\t\t\treturn next.ProcessTask(ctx, t)\n\t\t})\n\t}\n}\n\n// A list of pattern, handler pair that is registered with mux.\nvar serveMuxRegister = []struct {\n\tpattern string\n\th Handler\n}{\n\t{\"email:\", makeFakeHandler(\"default email handler\")},\n\t{\"email:signup\", makeFakeHandler(\"signup email handler\")},\n\t{\"csv:export\", makeFakeHandler(\"csv export handler\")},\n}\n\nvar serveMuxTests = []struct {\n\ttypename string // task's type name\n\twant string // identifier of the handler that should be called\n}{\n\t{\"email:signup\", \"signup email handler\"},\n\t{\"csv:export\", \"csv export handler\"},\n\t{\"email:daily\", \"default email handler\"},\n}\n\nfunc TestServeMux(t *testing.T) {\n\tmux := NewServeMux()\n\tfor _, e := range serveMuxRegister {\n\t\tmux.Handle(e.pattern, e.h)\n\t}\n\n\tfor _, tc := range serveMuxTests {\n\t\tcalled = \"\" // reset to zero value\n\n\t\ttask := NewTask(tc.typename, nil)\n\t\tif err := mux.ProcessTask(context.Background(), task); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tif called != tc.want {\n\t\t\tt.Errorf(\"%q handler was called for task %q, want %q to be called\", called, task.Type(), tc.want)\n\t\t}\n\t}\n}\n\nfunc TestServeMuxRegisterNilHandler(t *testing.T) {\n\tdefer func() {\n\t\tif err := recover(); err == nil {\n\t\t\tt.Error(\"expected call to mux.HandleFunc to panic\")\n\t\t}\n\t}()\n\n\tmux := NewServeMux()\n\tmux.HandleFunc(\"email:signup\", nil)\n}\n\nfunc TestServeMuxRegisterEmptyPattern(t *testing.T) {\n\tdefer func() {\n\t\tif err := recover(); err == nil {\n\t\t\tt.Error(\"expected call to mux.HandleFunc to panic\")\n\t\t}\n\t}()\n\n\tmux := NewServeMux()\n\tmux.Handle(\"\", makeFakeHandler(\"email\"))\n}\n\nfunc TestServeMuxRegisterDuplicatePattern(t *testing.T) {\n\tdefer func() {\n\t\tif err := recover(); err == nil {\n\t\t\tt.Error(\"expected call to mux.HandleFunc to panic\")\n\t\t}\n\t}()\n\n\tmux := NewServeMux()\n\tmux.Handle(\"email\", makeFakeHandler(\"email\"))\n\tmux.Handle(\"email\", makeFakeHandler(\"email:default\"))\n}\n\nvar notFoundTests = []struct {\n\ttypename string // task's type name\n}{\n\t{\"image:minimize\"},\n\t{\"csv:\"}, // registered patterns match the task's type prefix, not the other way around.\n}\n\nfunc TestServeMuxNotFound(t *testing.T) {\n\tmux := NewServeMux()\n\tfor _, e := range serveMuxRegister {\n\t\tmux.Handle(e.pattern, e.h)\n\t}\n\n\tfor _, tc := range notFoundTests {\n\t\ttask := NewTask(tc.typename, nil)\n\t\terr := mux.ProcessTask(context.Background(), task)\n\t\tif err == nil {\n\t\t\tt.Errorf(\"ProcessTask did not return error for task %q, should return 'not found' error\", task.Type())\n\t\t}\n\t}\n}\n\nvar middlewareTests = []struct {\n\ttypename string // task's type name\n\tmiddlewares []string // middlewares to use. They should be called in this order.\n\twant string // identifier of the handler that should be called\n}{\n\t{\"email:signup\", []string{\"logging\", \"expiration\"}, \"signup email handler\"},\n\t{\"csv:export\", []string{}, \"csv export handler\"},\n\t{\"email:daily\", []string{\"expiration\", \"logging\"}, \"default email handler\"},\n}\n\nfunc TestServeMuxMiddlewares(t *testing.T) {\n\tfor _, tc := range middlewareTests {\n\t\tmux := NewServeMux()\n\t\tfor _, e := range serveMuxRegister {\n\t\t\tmux.Handle(e.pattern, e.h)\n\t\t}\n\t\tvar mws []MiddlewareFunc\n\t\tfor _, s := range tc.middlewares {\n\t\t\tmws = append(mws, makeFakeMiddleware(s))\n\t\t}\n\t\tmux.Use(mws...)\n\n\t\tinvoked = []string{} // reset to empty slice\n\t\tcalled = \"\" // reset to zero value\n\n\t\ttask := NewTask(tc.typename, nil)\n\t\tif err := mux.ProcessTask(context.Background(), task); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tif diff := cmp.Diff(invoked, tc.middlewares); diff != \"\" {\n\t\t\tt.Errorf(\"invoked middlewares were %v, want %v\", invoked, tc.middlewares)\n\t\t}\n\n\t\tif called != tc.want {\n\t\t\tt.Errorf(\"%q handler was called for task %q, want %q to be called\", called, task.Type(), tc.want)\n\t\t}\n\t}\n}\n" }, { "path": "server.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"math/rand/v2\"\n\t\"runtime\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/log\"\n\t\"github.com/hibiken/asynq/internal/rdb\"\n\t\"github.com/redis/go-redis/v9\"\n)\n\n// Server is responsible for task processing and task lifecycle management.\n//\n// Server pulls tasks off queues and processes them.\n// If the processing of a task is unsuccessful, server will schedule it for a retry.\n//\n// A task will be retried until either the task gets processed successfully\n// or until it reaches its max retry count.\n//\n// If a task exhausts its retries, it will be moved to the archive and\n// will be kept in the archive set.\n// Note that the archive size is finite and once it reaches its max size,\n// oldest tasks in the archive will be deleted.\ntype Server struct {\n\tlogger *log.Logger\n\n\tbroker base.Broker\n\t// When a Server has been created with an existing Redis connection, we do\n\t// not want to close it.\n\tsharedConnection bool\n\n\tstate *serverState\n\n\t// wait group to wait for all goroutines to finish.\n\twg sync.WaitGroup\n\tforwarder *forwarder\n\tprocessor *processor\n\tsyncer *syncer\n\theartbeater *heartbeater\n\tsubscriber *subscriber\n\trecoverer *recoverer\n\thealthchecker *healthchecker\n\tjanitor *janitor\n\taggregator *aggregator\n}\n\ntype serverState struct {\n\tmu sync.Mutex\n\tvalue serverStateValue\n}\n\ntype serverStateValue int\n\nconst (\n\t// StateNew represents a new server. Server begins in\n\t// this state and then transition to StatusActive when\n\t// Start or Run is callled.\n\tsrvStateNew serverStateValue = iota\n\n\t// StateActive indicates the server is up and active.\n\tsrvStateActive\n\n\t// StateStopped indicates the server is up but no longer processing new tasks.\n\tsrvStateStopped\n\n\t// StateClosed indicates the server has been shutdown.\n\tsrvStateClosed\n)\n\nvar serverStates = []string{\n\t\"new\",\n\t\"active\",\n\t\"stopped\",\n\t\"closed\",\n}\n\nfunc (s serverStateValue) String() string {\n\tif srvStateNew <= s && s <= srvStateClosed {\n\t\treturn serverStates[s]\n\t}\n\treturn \"unknown status\"\n}\n\n// Config specifies the server's background-task processing behavior.\ntype Config struct {\n\t// Maximum number of concurrent processing of tasks.\n\t//\n\t// If set to a zero or negative value, NewServer will overwrite the value\n\t// to the number of CPUs usable by the current process.\n\tConcurrency int\n\n\t// BaseContext optionally specifies a function that returns the base context for Handler invocations on this server.\n\t//\n\t// If BaseContext is nil, the default is context.Background().\n\t// If this is defined, then it MUST return a non-nil context\n\tBaseContext func() context.Context\n\n\t// TaskCheckInterval specifies the interval between checks for new tasks to process when all queues are empty.\n\t//\n\t// If unset, zero or a negative value, the interval is set to 1 second.\n\t//\n\t// Note: Setting this value too low may add significant load to redis.\n\t//\n\t// By default, TaskCheckInterval is set to 1 seconds.\n\tTaskCheckInterval time.Duration\n\n\t// Function to calculate retry delay for a failed task.\n\t//\n\t// By default, it uses exponential backoff algorithm to calculate the delay.\n\tRetryDelayFunc RetryDelayFunc\n\n\t// Predicate function to determine whether the error returned from Handler is a failure.\n\t// If the function returns false, Server will not increment the retried counter for the task,\n\t// and Server won't record the queue stats (processed and failed stats) to avoid skewing the error\n\t// rate of the queue.\n\t//\n\t// By default, if the given error is non-nil the function returns true.\n\tIsFailure func(error) bool\n\n\t// List of queues to process with given priority value. Keys are the names of the\n\t// queues and values are associated priority value.\n\t//\n\t// If set to nil or not specified, the server will process only the \"default\" queue.\n\t//\n\t// Priority is treated as follows to avoid starving low priority queues.\n\t//\n\t// Example:\n\t//\n\t// Queues: map[string]int{\n\t// \"critical\": 6,\n\t// \"default\": 3,\n\t// \"low\": 1,\n\t// }\n\t//\n\t// With the above config and given that all queues are not empty, the tasks\n\t// in \"critical\", \"default\", \"low\" should be processed 60%, 30%, 10% of\n\t// the time respectively.\n\t//\n\t// If a queue has a zero or negative priority value, the queue will be ignored.\n\tQueues map[string]int\n\n\t// StrictPriority indicates whether the queue priority should be treated strictly.\n\t//\n\t// If set to true, tasks in the queue with the highest priority is processed first.\n\t// The tasks in lower priority queues are processed only when those queues with\n\t// higher priorities are empty.\n\tStrictPriority bool\n\n\t// ErrorHandler handles errors returned by the task handler.\n\t//\n\t// HandleError is invoked only if the task handler returns a non-nil error.\n\t//\n\t// Example:\n\t//\n\t// func reportError(ctx context, task *asynq.Task, err error) {\n\t// retried, _ := asynq.GetRetryCount(ctx)\n\t// maxRetry, _ := asynq.GetMaxRetry(ctx)\n\t// \t if retried >= maxRetry {\n\t// err = fmt.Errorf(\"retry exhausted for task %s: %w\", task.Type, err)\n\t// \t }\n\t// errorReportingService.Notify(err)\n\t// })\n\t//\n\t// ErrorHandler: asynq.ErrorHandlerFunc(reportError)\n\t//\n\t// we can also handle panic error like:\n\t// func reportError(ctx context, task *asynq.Task, err error) {\n\t// if asynq.IsPanicError(err) {\n\t//\t errorReportingService.Notify(err)\n\t// \t }\n\t// })\n\t//\n\t// ErrorHandler: asynq.ErrorHandlerFunc(reportError)\n\tErrorHandler ErrorHandler\n\n\t// Logger specifies the logger used by the server instance.\n\t//\n\t// If unset, default logger is used.\n\tLogger Logger\n\n\t// LogLevel specifies the minimum log level to enable.\n\t//\n\t// If unset, InfoLevel is used by default.\n\tLogLevel LogLevel\n\n\t// ShutdownTimeout specifies the duration to wait to let workers finish their tasks\n\t// before forcing them to abort when stopping the server.\n\t//\n\t// If unset or zero, default timeout of 8 seconds is used.\n\tShutdownTimeout time.Duration\n\n\t// HealthCheckFunc is called periodically with any errors encountered during ping to the\n\t// connected redis server.\n\tHealthCheckFunc func(error)\n\n\t// HealthCheckInterval specifies the interval between healthchecks.\n\t//\n\t// If unset or zero, the interval is set to 15 seconds.\n\tHealthCheckInterval time.Duration\n\n\t// DelayedTaskCheckInterval specifies the interval between checks run on 'scheduled' and 'retry'\n\t// tasks, and forwarding them to 'pending' state if they are ready to be processed.\n\t//\n\t// If unset or zero, the interval is set to 5 seconds.\n\tDelayedTaskCheckInterval time.Duration\n\n\t// GroupGracePeriod specifies the amount of time the server will wait for an incoming task before aggregating\n\t// the tasks in a group. If an incoming task is received within this period, the server will wait for another\n\t// period of the same length, up to GroupMaxDelay if specified.\n\t//\n\t// If unset or zero, the grace period is set to 1 minute.\n\t// Minimum duration for GroupGracePeriod is 1 second. If value specified is less than a second, the call to\n\t// NewServer will panic.\n\tGroupGracePeriod time.Duration\n\n\t// GroupMaxDelay specifies the maximum amount of time the server will wait for incoming tasks before aggregating\n\t// the tasks in a group.\n\t//\n\t// If unset or zero, no delay limit is used.\n\tGroupMaxDelay time.Duration\n\n\t// GroupMaxSize specifies the maximum number of tasks that can be aggregated into a single task within a group.\n\t// If GroupMaxSize is reached, the server will aggregate the tasks into one immediately.\n\t//\n\t// If unset or zero, no size limit is used.\n\tGroupMaxSize int\n\n\t// GroupAggregator specifies the aggregation function used to aggregate multiple tasks in a group into one task.\n\t//\n\t// If unset or nil, the group aggregation feature will be disabled on the server.\n\tGroupAggregator GroupAggregator\n\n\t// JanitorInterval specifies the average interval of janitor checks for expired completed tasks.\n\t//\n\t// If unset or zero, default interval of 8 seconds is used.\n\tJanitorInterval time.Duration\n\n\t// JanitorBatchSize specifies the number of expired completed tasks to be deleted in one run.\n\t//\n\t// If unset or zero, default batch size of 100 is used.\n\t// Make sure to not put a big number as the batch size to prevent a long-running script.\n\tJanitorBatchSize int\n}\n\n// GroupAggregator aggregates a group of tasks into one before the tasks are passed to the Handler.\ntype GroupAggregator interface {\n\t// Aggregate aggregates the given tasks in a group with the given group name,\n\t// and returns a new task which is the aggregation of those tasks.\n\t//\n\t// Use NewTask(typename, payload, opts...) to set any options for the aggregated task.\n\t// The Queue option, if provided, will be ignored and the aggregated task will always be enqueued\n\t// to the same queue the group belonged.\n\tAggregate(group string, tasks []*Task) *Task\n}\n\n// The GroupAggregatorFunc type is an adapter to allow the use of ordinary functions as a GroupAggregator.\n// If f is a function with the appropriate signature, GroupAggregatorFunc(f) is a GroupAggregator that calls f.\ntype GroupAggregatorFunc func(group string, tasks []*Task) *Task\n\n// Aggregate calls fn(group, tasks)\nfunc (fn GroupAggregatorFunc) Aggregate(group string, tasks []*Task) *Task {\n\treturn fn(group, tasks)\n}\n\n// An ErrorHandler handles an error occurred during task processing.\ntype ErrorHandler interface {\n\tHandleError(ctx context.Context, task *Task, err error)\n}\n\n// The ErrorHandlerFunc type is an adapter to allow the use of ordinary functions as a ErrorHandler.\n// If f is a function with the appropriate signature, ErrorHandlerFunc(f) is a ErrorHandler that calls f.\ntype ErrorHandlerFunc func(ctx context.Context, task *Task, err error)\n\n// HandleError calls fn(ctx, task, err)\nfunc (fn ErrorHandlerFunc) HandleError(ctx context.Context, task *Task, err error) {\n\tfn(ctx, task, err)\n}\n\n// RetryDelayFunc calculates the retry delay duration for a failed task given\n// the retry count, error, and the task.\n//\n// n is the number of times the task has been retried.\n// e is the error returned by the task handler.\n// t is the task in question.\ntype RetryDelayFunc func(n int, e error, t *Task) time.Duration\n\n// Logger supports logging at various log levels.\ntype Logger interface {\n\t// Debug logs a message at Debug level.\n\tDebug(args ...interface{})\n\n\t// Info logs a message at Info level.\n\tInfo(args ...interface{})\n\n\t// Warn logs a message at Warning level.\n\tWarn(args ...interface{})\n\n\t// Error logs a message at Error level.\n\tError(args ...interface{})\n\n\t// Fatal logs a message at Fatal level\n\t// and process will exit with status set to 1.\n\tFatal(args ...interface{})\n}\n\n// LogLevel represents logging level.\n//\n// It satisfies flag.Value interface.\ntype LogLevel int32\n\nconst (\n\t// Note: reserving value zero to differentiate unspecified case.\n\tlevel_unspecified LogLevel = iota\n\n\t// DebugLevel is the lowest level of logging.\n\t// Debug logs are intended for debugging and development purposes.\n\tDebugLevel\n\n\t// InfoLevel is used for general informational log messages.\n\tInfoLevel\n\n\t// WarnLevel is used for undesired but relatively expected events,\n\t// which may indicate a problem.\n\tWarnLevel\n\n\t// ErrorLevel is used for undesired and unexpected events that\n\t// the program can recover from.\n\tErrorLevel\n\n\t// FatalLevel is used for undesired and unexpected events that\n\t// the program cannot recover from.\n\tFatalLevel\n)\n\n// String is part of the flag.Value interface.\nfunc (l *LogLevel) String() string {\n\tswitch *l {\n\tcase DebugLevel:\n\t\treturn \"debug\"\n\tcase InfoLevel:\n\t\treturn \"info\"\n\tcase WarnLevel:\n\t\treturn \"warn\"\n\tcase ErrorLevel:\n\t\treturn \"error\"\n\tcase FatalLevel:\n\t\treturn \"fatal\"\n\t}\n\tpanic(fmt.Sprintf(\"asynq: unexpected log level: %v\", *l))\n}\n\n// Set is part of the flag.Value interface.\nfunc (l *LogLevel) Set(val string) error {\n\tswitch strings.ToLower(val) {\n\tcase \"debug\":\n\t\t*l = DebugLevel\n\tcase \"info\":\n\t\t*l = InfoLevel\n\tcase \"warn\", \"warning\":\n\t\t*l = WarnLevel\n\tcase \"error\":\n\t\t*l = ErrorLevel\n\tcase \"fatal\":\n\t\t*l = FatalLevel\n\tdefault:\n\t\treturn fmt.Errorf(\"asynq: unsupported log level %q\", val)\n\t}\n\treturn nil\n}\n\nfunc toInternalLogLevel(l LogLevel) log.Level {\n\tswitch l {\n\tcase DebugLevel:\n\t\treturn log.DebugLevel\n\tcase InfoLevel:\n\t\treturn log.InfoLevel\n\tcase WarnLevel:\n\t\treturn log.WarnLevel\n\tcase ErrorLevel:\n\t\treturn log.ErrorLevel\n\tcase FatalLevel:\n\t\treturn log.FatalLevel\n\t}\n\tpanic(fmt.Sprintf(\"asynq: unexpected log level: %v\", l))\n}\n\n// DefaultRetryDelayFunc is the default RetryDelayFunc used if one is not specified in Config.\n// It uses exponential back-off strategy to calculate the retry delay.\nfunc DefaultRetryDelayFunc(n int, e error, t *Task) time.Duration {\n\t// Formula taken from https://github.com/mperham/sidekiq.\n\ts := int(math.Pow(float64(n), 4)) + 15 + (rand.IntN(30) * (n + 1))\n\treturn time.Duration(s) * time.Second\n}\n\nfunc defaultIsFailureFunc(err error) bool { return err != nil }\n\nvar defaultQueueConfig = map[string]int{\n\tbase.DefaultQueueName: 1,\n}\n\nconst (\n\tdefaultTaskCheckInterval = 1 * time.Second\n\n\tdefaultShutdownTimeout = 8 * time.Second\n\n\tdefaultHealthCheckInterval = 15 * time.Second\n\n\tdefaultDelayedTaskCheckInterval = 5 * time.Second\n\n\tdefaultGroupGracePeriod = 1 * time.Minute\n\n\tdefaultJanitorInterval = 8 * time.Second\n\n\tdefaultJanitorBatchSize = 100\n)\n\n// NewServer returns a new Server given a redis connection option\n// and server configuration.\nfunc NewServer(r RedisConnOpt, cfg Config) *Server {\n\tredisClient, ok := r.MakeRedisClient().(redis.UniversalClient)\n\tif !ok {\n\t\tpanic(fmt.Sprintf(\"asynq: unsupported RedisConnOpt type %T\", r))\n\t}\n\tserver := NewServerFromRedisClient(redisClient, cfg)\n\tserver.sharedConnection = false\n\treturn server\n}\n\n// NewServerFromRedisClient returns a new instance of Server given a redis.UniversalClient\n// and server configuration\n// Warning: The underlying redis connection pool will not be closed by Asynq, you are responsible for closing it.\nfunc NewServerFromRedisClient(c redis.UniversalClient, cfg Config) *Server {\n\tbaseCtxFn := cfg.BaseContext\n\tif baseCtxFn == nil {\n\t\tbaseCtxFn = context.Background\n\t}\n\tn := cfg.Concurrency\n\tif n < 1 {\n\t\tn = runtime.NumCPU()\n\t}\n\n\ttaskCheckInterval := cfg.TaskCheckInterval\n\tif taskCheckInterval <= 0 {\n\t\ttaskCheckInterval = defaultTaskCheckInterval\n\t}\n\n\tdelayFunc := cfg.RetryDelayFunc\n\tif delayFunc == nil {\n\t\tdelayFunc = DefaultRetryDelayFunc\n\t}\n\tisFailureFunc := cfg.IsFailure\n\tif isFailureFunc == nil {\n\t\tisFailureFunc = defaultIsFailureFunc\n\t}\n\tqueues := make(map[string]int)\n\tfor qname, p := range cfg.Queues {\n\t\tif err := base.ValidateQueueName(qname); err != nil {\n\t\t\tcontinue // ignore invalid queue names\n\t\t}\n\t\tif p > 0 {\n\t\t\tqueues[qname] = p\n\t\t}\n\t}\n\tif len(queues) == 0 {\n\t\tqueues = defaultQueueConfig\n\t}\n\tvar qnames []string\n\tfor q := range queues {\n\t\tqnames = append(qnames, q)\n\t}\n\tshutdownTimeout := cfg.ShutdownTimeout\n\tif shutdownTimeout == 0 {\n\t\tshutdownTimeout = defaultShutdownTimeout\n\t}\n\thealthcheckInterval := cfg.HealthCheckInterval\n\tif healthcheckInterval == 0 {\n\t\thealthcheckInterval = defaultHealthCheckInterval\n\t}\n\t// TODO: Create a helper to check for zero value and fall back to default (e.g. getDurationOrDefault())\n\tgroupGracePeriod := cfg.GroupGracePeriod\n\tif groupGracePeriod == 0 {\n\t\tgroupGracePeriod = defaultGroupGracePeriod\n\t}\n\tif groupGracePeriod < time.Second {\n\t\tpanic(\"GroupGracePeriod cannot be less than a second\")\n\t}\n\tlogger := log.NewLogger(cfg.Logger)\n\tloglevel := cfg.LogLevel\n\tif loglevel == level_unspecified {\n\t\tloglevel = InfoLevel\n\t}\n\tlogger.SetLevel(toInternalLogLevel(loglevel))\n\n\trdb := rdb.NewRDB(c)\n\tstarting := make(chan *workerInfo)\n\tfinished := make(chan *base.TaskMessage)\n\tsyncCh := make(chan *syncRequest)\n\tsrvState := &serverState{value: srvStateNew}\n\tcancels := base.NewCancelations()\n\n\tsyncer := newSyncer(syncerParams{\n\t\tlogger: logger,\n\t\trequestsCh: syncCh,\n\t\tinterval: 5 * time.Second,\n\t})\n\theartbeater := newHeartbeater(heartbeaterParams{\n\t\tlogger: logger,\n\t\tbroker: rdb,\n\t\tinterval: 5 * time.Second,\n\t\tconcurrency: n,\n\t\tqueues: queues,\n\t\tstrictPriority: cfg.StrictPriority,\n\t\tstate: srvState,\n\t\tstarting: starting,\n\t\tfinished: finished,\n\t})\n\tdelayedTaskCheckInterval := cfg.DelayedTaskCheckInterval\n\tif delayedTaskCheckInterval == 0 {\n\t\tdelayedTaskCheckInterval = defaultDelayedTaskCheckInterval\n\t}\n\tforwarder := newForwarder(forwarderParams{\n\t\tlogger: logger,\n\t\tbroker: rdb,\n\t\tqueues: qnames,\n\t\tinterval: delayedTaskCheckInterval,\n\t})\n\tsubscriber := newSubscriber(subscriberParams{\n\t\tlogger: logger,\n\t\tbroker: rdb,\n\t\tcancelations: cancels,\n\t})\n\tprocessor := newProcessor(processorParams{\n\t\tlogger: logger,\n\t\tbroker: rdb,\n\t\tretryDelayFunc: delayFunc,\n\t\ttaskCheckInterval: taskCheckInterval,\n\t\tbaseCtxFn: baseCtxFn,\n\t\tisFailureFunc: isFailureFunc,\n\t\tsyncCh: syncCh,\n\t\tcancelations: cancels,\n\t\tconcurrency: n,\n\t\tqueues: queues,\n\t\tstrictPriority: cfg.StrictPriority,\n\t\terrHandler: cfg.ErrorHandler,\n\t\tshutdownTimeout: shutdownTimeout,\n\t\tstarting: starting,\n\t\tfinished: finished,\n\t})\n\trecoverer := newRecoverer(recovererParams{\n\t\tlogger: logger,\n\t\tbroker: rdb,\n\t\tretryDelayFunc: delayFunc,\n\t\tisFailureFunc: isFailureFunc,\n\t\tqueues: qnames,\n\t\tinterval: 1 * time.Minute,\n\t})\n\thealthchecker := newHealthChecker(healthcheckerParams{\n\t\tlogger: logger,\n\t\tbroker: rdb,\n\t\tinterval: healthcheckInterval,\n\t\thealthcheckFunc: cfg.HealthCheckFunc,\n\t})\n\n\tjanitorInterval := cfg.JanitorInterval\n\tif janitorInterval == 0 {\n\t\tjanitorInterval = defaultJanitorInterval\n\t}\n\n\tjanitorBatchSize := cfg.JanitorBatchSize\n\tif janitorBatchSize == 0 {\n\t\tjanitorBatchSize = defaultJanitorBatchSize\n\t}\n\tif janitorBatchSize > defaultJanitorBatchSize {\n\t\tlogger.Warnf(\"Janitor batch size of %d is greater than the recommended batch size of %d. \"+\n\t\t\t\"This might cause a long-running script\", janitorBatchSize, defaultJanitorBatchSize)\n\t}\n\tjanitor := newJanitor(janitorParams{\n\t\tlogger: logger,\n\t\tbroker: rdb,\n\t\tqueues: qnames,\n\t\tinterval: janitorInterval,\n\t\tbatchSize: janitorBatchSize,\n\t})\n\taggregator := newAggregator(aggregatorParams{\n\t\tlogger: logger,\n\t\tbroker: rdb,\n\t\tqueues: qnames,\n\t\tgracePeriod: groupGracePeriod,\n\t\tmaxDelay: cfg.GroupMaxDelay,\n\t\tmaxSize: cfg.GroupMaxSize,\n\t\tgroupAggregator: cfg.GroupAggregator,\n\t})\n\treturn &Server{\n\t\tlogger: logger,\n\t\tbroker: rdb,\n\t\tsharedConnection: true,\n\t\tstate: srvState,\n\t\tforwarder: forwarder,\n\t\tprocessor: processor,\n\t\tsyncer: syncer,\n\t\theartbeater: heartbeater,\n\t\tsubscriber: subscriber,\n\t\trecoverer: recoverer,\n\t\thealthchecker: healthchecker,\n\t\tjanitor: janitor,\n\t\taggregator: aggregator,\n\t}\n}\n\n// A Handler processes tasks.\n//\n// ProcessTask should return nil if the processing of a task\n// is successful.\n//\n// If ProcessTask returns a non-nil error or panics, the task\n// will be retried after delay if retry-count is remaining,\n// otherwise the task will be archived.\n//\n// One exception to this rule is when ProcessTask returns a SkipRetry error.\n// If the returned error is SkipRetry or an error wraps SkipRetry, retry is\n// skipped and the task will be immediately archived instead.\n//\n// Another exception to this rule is when ProcessTask returns a RevokeTask error.\n// If the returned error is RevokeTask or an error wraps RevokeTask, the task\n// will not be retried or archived.\ntype Handler interface {\n\tProcessTask(context.Context, *Task) error\n}\n\n// The HandlerFunc type is an adapter to allow the use of\n// ordinary functions as a Handler. If f is a function\n// with the appropriate signature, HandlerFunc(f) is a\n// Handler that calls f.\ntype HandlerFunc func(context.Context, *Task) error\n\n// ProcessTask calls fn(ctx, task)\nfunc (fn HandlerFunc) ProcessTask(ctx context.Context, task *Task) error {\n\treturn fn(ctx, task)\n}\n\n// ErrServerClosed indicates that the operation is now illegal because of the server has been shutdown.\nvar ErrServerClosed = errors.New(\"asynq: Server closed\")\n\n// Run starts the task processing and blocks until\n// an os signal to exit the program is received. Once it receives\n// a signal, it gracefully shuts down all active workers and other\n// goroutines to process the tasks.\n//\n// Run returns any error encountered at server startup time.\n// If the server has already been shutdown, ErrServerClosed is returned.\nfunc (srv *Server) Run(handler Handler) error {\n\tif err := srv.Start(handler); err != nil {\n\t\treturn err\n\t}\n\tsrv.waitForSignals()\n\tsrv.Shutdown()\n\treturn nil\n}\n\n// Start starts the worker server. Once the server has started,\n// it pulls tasks off queues and starts a worker goroutine for each task\n// and then call Handler to process it.\n// Tasks are processed concurrently by the workers up to the number of\n// concurrency specified in Config.Concurrency.\n//\n// Start returns any error encountered at server startup time.\n// If the server has already been shutdown, ErrServerClosed is returned.\nfunc (srv *Server) Start(handler Handler) error {\n\tif handler == nil {\n\t\treturn fmt.Errorf(\"asynq: server cannot run with nil handler\")\n\t}\n\tsrv.processor.handler = handler\n\n\tif err := srv.start(); err != nil {\n\t\treturn err\n\t}\n\tsrv.logger.Info(\"Starting processing\")\n\n\tsrv.heartbeater.start(&srv.wg)\n\tsrv.healthchecker.start(&srv.wg)\n\tsrv.subscriber.start(&srv.wg)\n\tsrv.syncer.start(&srv.wg)\n\tsrv.recoverer.start(&srv.wg)\n\tsrv.forwarder.start(&srv.wg)\n\tsrv.processor.start(&srv.wg)\n\tsrv.janitor.start(&srv.wg)\n\tsrv.aggregator.start(&srv.wg)\n\treturn nil\n}\n\n// Checks server state and returns an error if pre-condition is not met.\n// Otherwise it sets the server state to active.\nfunc (srv *Server) start() error {\n\tsrv.state.mu.Lock()\n\tdefer srv.state.mu.Unlock()\n\tswitch srv.state.value {\n\tcase srvStateActive:\n\t\treturn fmt.Errorf(\"asynq: the server is already running\")\n\tcase srvStateStopped:\n\t\treturn fmt.Errorf(\"asynq: the server is in the stopped state. Waiting for shutdown.\")\n\tcase srvStateClosed:\n\t\treturn ErrServerClosed\n\t}\n\tsrv.state.value = srvStateActive\n\treturn nil\n}\n\n// Shutdown gracefully shuts down the server.\n// It gracefully closes all active workers. The server will wait for\n// active workers to finish processing tasks for duration specified in Config.ShutdownTimeout.\n// If worker didn't finish processing a task during the timeout, the task will be pushed back to Redis.\nfunc (srv *Server) Shutdown() {\n\tsrv.state.mu.Lock()\n\tif srv.state.value == srvStateNew || srv.state.value == srvStateClosed {\n\t\tsrv.state.mu.Unlock()\n\t\t// server is not running, do nothing and return.\n\t\treturn\n\t}\n\tsrv.state.value = srvStateClosed\n\tsrv.state.mu.Unlock()\n\n\tsrv.logger.Info(\"Starting graceful shutdown\")\n\t// Note: The order of shutdown is important.\n\t// Sender goroutines should be terminated before the receiver goroutines.\n\t// processor -> syncer (via syncCh)\n\t// processor -> heartbeater (via starting, finished channels)\n\tsrv.forwarder.shutdown()\n\tsrv.processor.shutdown()\n\tsrv.recoverer.shutdown()\n\tsrv.syncer.shutdown()\n\tsrv.subscriber.shutdown()\n\tsrv.janitor.shutdown()\n\tsrv.aggregator.shutdown()\n\tsrv.healthchecker.shutdown()\n\tsrv.heartbeater.shutdown()\n\tsrv.wg.Wait()\n\n\tif !srv.sharedConnection {\n\t\tsrv.broker.Close()\n\t}\n\tsrv.logger.Info(\"Exiting\")\n}\n\n// Stop signals the server to stop pulling new tasks off queues.\n// Stop can be used before shutting down the server to ensure that all\n// currently active tasks are processed before server shutdown.\n//\n// Stop does not shutdown the server, make sure to call Shutdown before exit.\nfunc (srv *Server) Stop() {\n\tsrv.state.mu.Lock()\n\tif srv.state.value != srvStateActive {\n\t\t// Invalid call to Stop, server can only go from Active state to Stopped state.\n\t\tsrv.state.mu.Unlock()\n\t\treturn\n\t}\n\tsrv.state.value = srvStateStopped\n\tsrv.state.mu.Unlock()\n\n\tsrv.logger.Info(\"Stopping processor\")\n\tsrv.processor.stop()\n\tsrv.logger.Info(\"Processor stopped\")\n}\n\n// Ping performs a ping against the redis connection.\n//\n// This is an alternative to the HealthCheckFunc available in the Config object.\nfunc (srv *Server) Ping() error {\n\tsrv.state.mu.Lock()\n\tdefer srv.state.mu.Unlock()\n\tif srv.state.value == srvStateClosed {\n\t\treturn nil\n\t}\n\n\treturn srv.broker.Ping()\n}\n" }, { "path": "server_test.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"syscall\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/hibiken/asynq/internal/rdb\"\n\t\"github.com/hibiken/asynq/internal/testbroker\"\n\t\"github.com/hibiken/asynq/internal/testutil\"\n\n\t\"github.com/redis/go-redis/v9\"\n\t\"go.uber.org/goleak\"\n)\n\nfunc testServer(t *testing.T, c *Client, srv *Server) {\n\t// no-op handler\n\th := func(ctx context.Context, task *Task) error {\n\t\treturn nil\n\t}\n\n\terr := srv.Start(HandlerFunc(h))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t_, err = c.Enqueue(NewTask(\"send_email\", testutil.JSON(map[string]interface{}{\"recipient_id\": 123})))\n\tif err != nil {\n\t\tt.Errorf(\"could not enqueue a task: %v\", err)\n\t}\n\n\t_, err = c.Enqueue(NewTask(\"send_email\", testutil.JSON(map[string]interface{}{\"recipient_id\": 456})), ProcessIn(1*time.Hour))\n\tif err != nil {\n\t\tt.Errorf(\"could not enqueue a task: %v\", err)\n\t}\n\n\tsrv.Shutdown()\n}\n\nfunc TestServer(t *testing.T) {\n\t// https://github.com/go-redis/redis/issues/1029\n\tignoreOpt := goleak.IgnoreTopFunction(\"github.com/redis/go-redis/v9/internal/pool.(*ConnPool).reaper\")\n\tdefer goleak.VerifyNone(t, ignoreOpt)\n\n\tredisConnOpt := getRedisConnOpt(t)\n\tc := NewClient(redisConnOpt)\n\tdefer c.Close()\n\tsrv := NewServer(redisConnOpt, Config{\n\t\tConcurrency: 10,\n\t\tLogLevel: testLogLevel,\n\t})\n\n\ttestServer(t, c, srv)\n}\n\nfunc TestServerFromRedisClient(t *testing.T) {\n\t// https://github.com/go-redis/redis/issues/1029\n\tignoreOpt := goleak.IgnoreTopFunction(\"github.com/redis/go-redis/v9/internal/pool.(*ConnPool).reaper\")\n\tdefer goleak.VerifyNone(t, ignoreOpt)\n\n\tredisConnOpt := getRedisConnOpt(t)\n\tredisClient := redisConnOpt.MakeRedisClient().(redis.UniversalClient)\n\tc := NewClientFromRedisClient(redisClient)\n\tsrv := NewServerFromRedisClient(redisClient, Config{\n\t\tConcurrency: 10,\n\t\tLogLevel: testLogLevel,\n\t})\n\n\ttestServer(t, c, srv)\n\n\terr := c.Close()\n\tif err == nil {\n\t\tt.Error(\"client.Close() should have failed because of a shared client but it didn't\")\n\t}\n}\n\nfunc TestServerRun(t *testing.T) {\n\t// https://github.com/go-redis/redis/issues/1029\n\tignoreOpt := goleak.IgnoreTopFunction(\"github.com/redis/go-redis/v9/internal/pool.(*ConnPool).reaper\")\n\tdefer goleak.VerifyNone(t, ignoreOpt)\n\n\tsrv := NewServer(getRedisConnOpt(t), Config{LogLevel: testLogLevel})\n\n\tdone := make(chan struct{})\n\t// Make sure server exits when receiving TERM signal.\n\tgo func() {\n\t\ttime.Sleep(2 * time.Second)\n\t\t_ = syscall.Kill(syscall.Getpid(), syscall.SIGTERM)\n\t\tdone <- struct{}{}\n\t}()\n\n\tgo func() {\n\t\tselect {\n\t\tcase <-time.After(10 * time.Second):\n\t\t\tpanic(\"server did not stop after receiving TERM signal\")\n\t\tcase <-done:\n\t\t}\n\t}()\n\n\tmux := NewServeMux()\n\tif err := srv.Run(mux); err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestServerErrServerClosed(t *testing.T) {\n\tsrv := NewServer(getRedisConnOpt(t), Config{LogLevel: testLogLevel})\n\thandler := NewServeMux()\n\tif err := srv.Start(handler); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tsrv.Shutdown()\n\terr := srv.Start(handler)\n\tif err != ErrServerClosed {\n\t\tt.Errorf(\"Restarting server: (*Server).Start(handler) = %v, want ErrServerClosed error\", err)\n\t}\n}\n\nfunc TestServerErrNilHandler(t *testing.T) {\n\tsrv := NewServer(getRedisConnOpt(t), Config{LogLevel: testLogLevel})\n\terr := srv.Start(nil)\n\tif err == nil {\n\t\tt.Error(\"Starting server with nil handler: (*Server).Start(nil) did not return error\")\n\t\tsrv.Shutdown()\n\t}\n}\n\nfunc TestServerErrServerRunning(t *testing.T) {\n\tsrv := NewServer(getRedisConnOpt(t), Config{LogLevel: testLogLevel})\n\thandler := NewServeMux()\n\tif err := srv.Start(handler); err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr := srv.Start(handler)\n\tif err == nil {\n\t\tt.Error(\"Calling (*Server).Start(handler) on already running server did not return error\")\n\t}\n\tsrv.Shutdown()\n}\n\nfunc TestServerWithRedisDown(t *testing.T) {\n\t// Make sure that server does not panic and exit if redis is down.\n\tdefer func() {\n\t\tif r := recover(); r != nil {\n\t\t\tt.Errorf(\"panic occurred: %v\", r)\n\t\t}\n\t}()\n\tr := rdb.NewRDB(setup(t))\n\ttestBroker := testbroker.NewTestBroker(r)\n\tsrv := NewServer(getRedisConnOpt(t), Config{LogLevel: testLogLevel})\n\tsrv.broker = testBroker\n\tsrv.forwarder.broker = testBroker\n\tsrv.heartbeater.broker = testBroker\n\tsrv.processor.broker = testBroker\n\tsrv.subscriber.broker = testBroker\n\ttestBroker.Sleep()\n\n\t// no-op handler\n\th := func(ctx context.Context, task *Task) error {\n\t\treturn nil\n\t}\n\n\terr := srv.Start(HandlerFunc(h))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttime.Sleep(3 * time.Second)\n\n\tsrv.Shutdown()\n}\n\nfunc TestServerWithFlakyBroker(t *testing.T) {\n\t// Make sure that server does not panic and exit if redis is down.\n\tdefer func() {\n\t\tif r := recover(); r != nil {\n\t\t\tt.Errorf(\"panic occurred: %v\", r)\n\t\t}\n\t}()\n\tr := rdb.NewRDB(setup(t))\n\ttestBroker := testbroker.NewTestBroker(r)\n\tredisConnOpt := getRedisConnOpt(t)\n\tsrv := NewServer(redisConnOpt, Config{LogLevel: testLogLevel})\n\tsrv.broker = testBroker\n\tsrv.forwarder.broker = testBroker\n\tsrv.heartbeater.broker = testBroker\n\tsrv.processor.broker = testBroker\n\tsrv.subscriber.broker = testBroker\n\n\tc := NewClient(redisConnOpt)\n\n\th := func(ctx context.Context, task *Task) error {\n\t\t// force task retry.\n\t\tif task.Type() == \"bad_task\" {\n\t\t\treturn fmt.Errorf(\"could not process %q\", task.Type())\n\t\t}\n\t\ttime.Sleep(2 * time.Second)\n\t\treturn nil\n\t}\n\n\terr := srv.Start(HandlerFunc(h))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfor i := 0; i < 10; i++ {\n\t\t_, err := c.Enqueue(NewTask(\"enqueued\", nil), MaxRetry(i))\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\t_, err = c.Enqueue(NewTask(\"bad_task\", nil))\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\t_, err = c.Enqueue(NewTask(\"scheduled\", nil), ProcessIn(time.Duration(i)*time.Second))\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\t// simulate redis going down.\n\ttestBroker.Sleep()\n\n\ttime.Sleep(3 * time.Second)\n\n\t// simulate redis comes back online.\n\ttestBroker.Wakeup()\n\n\ttime.Sleep(3 * time.Second)\n\n\tsrv.Shutdown()\n}\n\nfunc TestLogLevel(t *testing.T) {\n\ttests := []struct {\n\t\tflagVal string\n\t\twant LogLevel\n\t\twantStr string\n\t}{\n\t\t{\"debug\", DebugLevel, \"debug\"},\n\t\t{\"Info\", InfoLevel, \"info\"},\n\t\t{\"WARN\", WarnLevel, \"warn\"},\n\t\t{\"warning\", WarnLevel, \"warn\"},\n\t\t{\"Error\", ErrorLevel, \"error\"},\n\t\t{\"fatal\", FatalLevel, \"fatal\"},\n\t}\n\n\tfor _, tc := range tests {\n\t\tlevel := new(LogLevel)\n\t\tif err := level.Set(tc.flagVal); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif *level != tc.want {\n\t\t\tt.Errorf(\"Set(%q): got %v, want %v\", tc.flagVal, level, &tc.want)\n\t\t\tcontinue\n\t\t}\n\t\tif got := level.String(); got != tc.wantStr {\n\t\t\tt.Errorf(\"String() returned %q, want %q\", got, tc.wantStr)\n\t\t}\n\t}\n}\n" }, { "path": "signals_unix.go", "content": "//go:build linux || dragonfly || freebsd || netbsd || openbsd || darwin\n\npackage asynq\n\nimport (\n\t\"os\"\n\t\"os/signal\"\n\n\t\"golang.org/x/sys/unix\"\n)\n\n// waitForSignals waits for signals and handles them.\n// It handles SIGTERM, SIGINT, and SIGTSTP.\n// SIGTERM and SIGINT will signal the process to exit.\n// SIGTSTP will signal the process to stop processing new tasks.\nfunc (srv *Server) waitForSignals() {\n\tsrv.logger.Info(\"Send signal TSTP to stop processing new tasks\")\n\tsrv.logger.Info(\"Send signal TERM or INT to terminate the process\")\n\n\tsigs := make(chan os.Signal, 1)\n\tsignal.Notify(sigs, unix.SIGTERM, unix.SIGINT, unix.SIGTSTP)\n\tfor {\n\t\tsig := <-sigs\n\t\tif sig == unix.SIGTSTP {\n\t\t\tsrv.Stop()\n\t\t\tcontinue\n\t\t} else {\n\t\t\tsrv.Stop()\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nfunc (s *Scheduler) waitForSignals() {\n\ts.logger.Info(\"Send signal TERM or INT to stop the scheduler\")\n\tsigs := make(chan os.Signal, 1)\n\tsignal.Notify(sigs, unix.SIGTERM, unix.SIGINT)\n\t<-sigs\n}\n" }, { "path": "signals_windows.go", "content": "//go:build windows\n\npackage asynq\n\nimport (\n\t\"os\"\n\t\"os/signal\"\n\n\t\"golang.org/x/sys/windows\"\n)\n\n// waitForSignals waits for signals and handles them.\n// It handles SIGTERM and SIGINT.\n// SIGTERM and SIGINT will signal the process to exit.\n//\n// Note: Currently SIGTSTP is not supported for windows build.\nfunc (srv *Server) waitForSignals() {\n\tsrv.logger.Info(\"Send signal TERM or INT to terminate the process\")\n\tsigs := make(chan os.Signal, 1)\n\tsignal.Notify(sigs, windows.SIGTERM, windows.SIGINT)\n\t<-sigs\n}\n\nfunc (s *Scheduler) waitForSignals() {\n\ts.logger.Info(\"Send signal TERM or INT to stop the scheduler\")\n\tsigs := make(chan os.Signal, 1)\n\tsignal.Notify(sigs, windows.SIGTERM, windows.SIGINT)\n\t<-sigs\n}\n" }, { "path": "subscriber.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/log\"\n\t\"github.com/redis/go-redis/v9\"\n)\n\ntype subscriber struct {\n\tlogger *log.Logger\n\tbroker base.Broker\n\n\t// channel to communicate back to the long running \"subscriber\" goroutine.\n\tdone chan struct{}\n\n\t// cancelations hold cancel functions for all active tasks.\n\tcancelations *base.Cancelations\n\n\t// time to wait before retrying to connect to redis.\n\tretryTimeout time.Duration\n}\n\ntype subscriberParams struct {\n\tlogger *log.Logger\n\tbroker base.Broker\n\tcancelations *base.Cancelations\n}\n\nfunc newSubscriber(params subscriberParams) *subscriber {\n\treturn &subscriber{\n\t\tlogger: params.logger,\n\t\tbroker: params.broker,\n\t\tdone: make(chan struct{}),\n\t\tcancelations: params.cancelations,\n\t\tretryTimeout: 5 * time.Second,\n\t}\n}\n\nfunc (s *subscriber) shutdown() {\n\ts.logger.Debug(\"Subscriber shutting down...\")\n\t// Signal the subscriber goroutine to stop.\n\ts.done <- struct{}{}\n}\n\nfunc (s *subscriber) start(wg *sync.WaitGroup) {\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tvar (\n\t\t\tpubsub *redis.PubSub\n\t\t\terr error\n\t\t)\n\t\t// Try until successfully connect to Redis.\n\t\tfor {\n\t\t\tpubsub, err = s.broker.CancelationPubSub()\n\t\t\tif err != nil {\n\t\t\t\ts.logger.Errorf(\"cannot subscribe to cancelation channel: %v\", err)\n\t\t\t\tselect {\n\t\t\t\tcase <-time.After(s.retryTimeout):\n\t\t\t\t\tcontinue\n\t\t\t\tcase <-s.done:\n\t\t\t\t\ts.logger.Debug(\"Subscriber done\")\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t\tcancelCh := pubsub.Channel()\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-s.done:\n\t\t\t\tpubsub.Close()\n\t\t\t\ts.logger.Debug(\"Subscriber done\")\n\t\t\t\treturn\n\t\t\tcase msg := <-cancelCh:\n\t\t\t\tcancel, ok := s.cancelations.Get(msg.Payload)\n\t\t\t\tif ok {\n\t\t\t\t\tcancel()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}()\n}\n" }, { "path": "subscriber_test.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/rdb\"\n\t\"github.com/hibiken/asynq/internal/testbroker\"\n)\n\nfunc TestSubscriber(t *testing.T) {\n\tr := setup(t)\n\tdefer r.Close()\n\trdbClient := rdb.NewRDB(r)\n\n\ttests := []struct {\n\t\tregisteredID string // ID for which cancel func is registered\n\t\tpublishID string // ID to be published\n\t\twantCalled bool // whether cancel func should be called\n\t}{\n\t\t{\"abc123\", \"abc123\", true},\n\t\t{\"abc456\", \"abc123\", false},\n\t}\n\n\tfor _, tc := range tests {\n\t\tvar mu sync.Mutex\n\t\tcalled := false\n\t\tfakeCancelFunc := func() {\n\t\t\tmu.Lock()\n\t\t\tdefer mu.Unlock()\n\t\t\tcalled = true\n\t\t}\n\t\tcancelations := base.NewCancelations()\n\t\tcancelations.Add(tc.registeredID, fakeCancelFunc)\n\n\t\tsubscriber := newSubscriber(subscriberParams{\n\t\t\tlogger: testLogger,\n\t\t\tbroker: rdbClient,\n\t\t\tcancelations: cancelations,\n\t\t})\n\t\tvar wg sync.WaitGroup\n\t\tsubscriber.start(&wg)\n\t\tdefer subscriber.shutdown()\n\n\t\t// wait for subscriber to establish connection to pubsub channel\n\t\ttime.Sleep(time.Second)\n\n\t\tif err := rdbClient.PublishCancelation(tc.publishID); err != nil {\n\t\t\tt.Fatalf(\"could not publish cancelation message: %v\", err)\n\t\t}\n\n\t\t// wait for redis to publish message\n\t\ttime.Sleep(time.Second)\n\n\t\tmu.Lock()\n\t\tif called != tc.wantCalled {\n\t\t\tif tc.wantCalled {\n\t\t\t\tt.Errorf(\"fakeCancelFunc was not called, want the function to be called\")\n\t\t\t} else {\n\t\t\t\tt.Errorf(\"fakeCancelFunc was called, want the function to not be called\")\n\t\t\t}\n\t\t}\n\t\tmu.Unlock()\n\t}\n}\n\nfunc TestSubscriberWithRedisDown(t *testing.T) {\n\tdefer func() {\n\t\tif r := recover(); r != nil {\n\t\t\tt.Errorf(\"panic occurred: %v\", r)\n\t\t}\n\t}()\n\tr := rdb.NewRDB(setup(t))\n\tdefer r.Close()\n\ttestBroker := testbroker.NewTestBroker(r)\n\n\tcancelations := base.NewCancelations()\n\tsubscriber := newSubscriber(subscriberParams{\n\t\tlogger: testLogger,\n\t\tbroker: testBroker,\n\t\tcancelations: cancelations,\n\t})\n\tsubscriber.retryTimeout = 1 * time.Second // set shorter retry timeout for testing purpose.\n\n\ttestBroker.Sleep() // simulate a situation where subscriber cannot connect to redis.\n\tvar wg sync.WaitGroup\n\tsubscriber.start(&wg)\n\tdefer subscriber.shutdown()\n\n\ttime.Sleep(2 * time.Second) // subscriber should wait and retry connecting to redis.\n\n\ttestBroker.Wakeup() // simulate a situation where redis server is back online.\n\n\ttime.Sleep(2 * time.Second) // allow subscriber to establish pubsub channel.\n\n\tconst id = \"test\"\n\tvar (\n\t\tmu sync.Mutex\n\t\tcalled bool\n\t)\n\tcancelations.Add(id, func() {\n\t\tmu.Lock()\n\t\tdefer mu.Unlock()\n\t\tcalled = true\n\t})\n\n\tif err := r.PublishCancelation(id); err != nil {\n\t\tt.Fatalf(\"could not publish cancelation message: %v\", err)\n\t}\n\n\ttime.Sleep(time.Second) // wait for redis to publish message.\n\n\tmu.Lock()\n\tif !called {\n\t\tt.Errorf(\"cancel function was not called\")\n\t}\n\tmu.Unlock()\n}\n" }, { "path": "syncer.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/hibiken/asynq/internal/log\"\n)\n\n// syncer is responsible for queuing up failed requests to redis and retry\n// those requests to sync state between the background process and redis.\ntype syncer struct {\n\tlogger *log.Logger\n\n\trequestsCh <-chan *syncRequest\n\n\t// channel to communicate back to the long running \"syncer\" goroutine.\n\tdone chan struct{}\n\n\t// interval between sync operations.\n\tinterval time.Duration\n}\n\ntype syncRequest struct {\n\tfn func() error // sync operation\n\terrMsg string // error message\n\tdeadline time.Time // request should be dropped if deadline has been exceeded\n}\n\ntype syncerParams struct {\n\tlogger *log.Logger\n\trequestsCh <-chan *syncRequest\n\tinterval time.Duration\n}\n\nfunc newSyncer(params syncerParams) *syncer {\n\treturn &syncer{\n\t\tlogger: params.logger,\n\t\trequestsCh: params.requestsCh,\n\t\tdone: make(chan struct{}),\n\t\tinterval: params.interval,\n\t}\n}\n\nfunc (s *syncer) shutdown() {\n\ts.logger.Debug(\"Syncer shutting down...\")\n\t// Signal the syncer goroutine to stop.\n\ts.done <- struct{}{}\n}\n\nfunc (s *syncer) start(wg *sync.WaitGroup) {\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tvar requests []*syncRequest\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-s.done:\n\t\t\t\t// Try sync one last time before shutting down.\n\t\t\t\tfor _, req := range requests {\n\t\t\t\t\tif err := req.fn(); err != nil {\n\t\t\t\t\t\ts.logger.Error(req.errMsg)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\ts.logger.Debug(\"Syncer done\")\n\t\t\t\treturn\n\t\t\tcase req := <-s.requestsCh:\n\t\t\t\trequests = append(requests, req)\n\t\t\tcase <-time.After(s.interval):\n\t\t\t\tvar temp []*syncRequest\n\t\t\t\tfor _, req := range requests {\n\t\t\t\t\tif req.deadline.Before(time.Now()) {\n\t\t\t\t\t\tcontinue // drop stale request\n\t\t\t\t\t}\n\t\t\t\t\tif err := req.fn(); err != nil {\n\t\t\t\t\t\ttemp = append(temp, req)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\trequests = temp\n\t\t\t}\n\t\t}\n\t}()\n}\n" }, { "path": "syncer_test.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage asynq\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/rdb\"\n\th \"github.com/hibiken/asynq/internal/testutil\"\n)\n\nfunc TestSyncer(t *testing.T) {\n\tinProgress := []*base.TaskMessage{\n\t\th.NewTaskMessage(\"send_email\", nil),\n\t\th.NewTaskMessage(\"reindex\", nil),\n\t\th.NewTaskMessage(\"gen_thumbnail\", nil),\n\t}\n\tr := setup(t)\n\tdefer r.Close()\n\trdbClient := rdb.NewRDB(r)\n\th.SeedActiveQueue(t, r, inProgress, base.DefaultQueueName)\n\n\tconst interval = time.Second\n\tsyncRequestCh := make(chan *syncRequest)\n\tsyncer := newSyncer(syncerParams{\n\t\tlogger: testLogger,\n\t\trequestsCh: syncRequestCh,\n\t\tinterval: interval,\n\t})\n\tvar wg sync.WaitGroup\n\tsyncer.start(&wg)\n\tdefer syncer.shutdown()\n\n\tfor _, msg := range inProgress {\n\t\tm := msg\n\t\tsyncRequestCh <- &syncRequest{\n\t\t\tfn: func() error {\n\t\t\t\treturn rdbClient.Done(context.Background(), m)\n\t\t\t},\n\t\t\tdeadline: time.Now().Add(5 * time.Minute),\n\t\t}\n\t}\n\n\ttime.Sleep(2 * interval) // ensure that syncer runs at least once\n\n\tgotActive := h.GetActiveMessages(t, r, base.DefaultQueueName)\n\tif l := len(gotActive); l != 0 {\n\t\tt.Errorf(\"%q has length %d; want 0\", base.ActiveKey(base.DefaultQueueName), l)\n\t}\n}\n\nfunc TestSyncerRetry(t *testing.T) {\n\tconst interval = time.Second\n\tsyncRequestCh := make(chan *syncRequest)\n\tsyncer := newSyncer(syncerParams{\n\t\tlogger: testLogger,\n\t\trequestsCh: syncRequestCh,\n\t\tinterval: interval,\n\t})\n\n\tvar wg sync.WaitGroup\n\tsyncer.start(&wg)\n\tdefer syncer.shutdown()\n\n\tvar (\n\t\tmu sync.Mutex\n\t\tcounter int\n\t)\n\n\t// Increment the counter for each call.\n\t// Initial call will fail and second call will succeed.\n\trequestFunc := func() error {\n\t\tmu.Lock()\n\t\tdefer mu.Unlock()\n\t\tif counter == 0 {\n\t\t\tcounter++\n\t\t\treturn fmt.Errorf(\"zero\")\n\t\t}\n\t\tcounter++\n\t\treturn nil\n\t}\n\n\tsyncRequestCh <- &syncRequest{\n\t\tfn: requestFunc,\n\t\terrMsg: \"error\",\n\t\tdeadline: time.Now().Add(5 * time.Minute),\n\t}\n\n\t// allow syncer to retry\n\ttime.Sleep(3 * interval)\n\n\tmu.Lock()\n\tif counter != 2 {\n\t\tt.Errorf(\"counter = %d, want 2\", counter)\n\t}\n\tmu.Unlock()\n}\n\nfunc TestSyncerDropsStaleRequests(t *testing.T) {\n\tconst interval = time.Second\n\tsyncRequestCh := make(chan *syncRequest)\n\tsyncer := newSyncer(syncerParams{\n\t\tlogger: testLogger,\n\t\trequestsCh: syncRequestCh,\n\t\tinterval: interval,\n\t})\n\tvar wg sync.WaitGroup\n\tsyncer.start(&wg)\n\n\tvar (\n\t\tmu sync.Mutex\n\t\tn int // number of times request has been processed\n\t)\n\n\tfor i := 0; i < 10; i++ {\n\t\tsyncRequestCh <- &syncRequest{\n\t\t\tfn: func() error {\n\t\t\t\tmu.Lock()\n\t\t\t\tn++\n\t\t\t\tmu.Unlock()\n\t\t\t\treturn nil\n\t\t\t},\n\t\t\tdeadline: time.Now().Add(time.Duration(-i) * time.Second), // already exceeded deadline\n\t\t}\n\t}\n\n\ttime.Sleep(2 * interval) // ensure that syncer runs at least once\n\tsyncer.shutdown()\n\n\tmu.Lock()\n\tif n != 0 {\n\t\tt.Errorf(\"requests has been processed %d times, want 0\", n)\n\t}\n\tmu.Unlock()\n}\n" }, { "path": "tools/asynq/README.md", "content": "# Asynq CLI\n\nAsynq CLI is a command line tool to monitor the queues and tasks managed by `asynq` package.\n\n## Table of Contents\n\n- [Installation](#installation)\n- [Usage](#usage)\n- [Config File](#config-file)\n\n## Installation\n\nIn order to use the tool, compile it using the following command:\n\n go install github.com/hibiken/asynq/tools/asynq@latest\n\nThis will create the asynq executable under your `$GOPATH/bin` directory.\n\n## Usage\n\n### Commands\n\nTo view details on any command, use `asynq help `.\n\n- `asynq dash`\n- `asynq stats`\n- `asynq queue [ls inspect history rm pause unpause]`\n- `asynq task [ls cancel delete archive run deleteall archiveall runall]`\n- `asynq server [ls]`\n\n### Global flags\n\nAsynq CLI needs to connect to a redis-server to inspect the state of queues and tasks. Use flags to specify the options to connect to the redis-server used by your application.\nTo connect to a redis cluster, pass `--cluster` and `--cluster_addrs` flags.\n\nBy default, CLI will try to connect to a redis server running at `localhost:6379`.\n\n```\n --config string config file to set flag defaut values (default is $HOME/.asynq.yaml)\n -n, --db int redis database number (default is 0)\n -h, --help help for asynq\n -U, --username string username to use when connecting to redis server\n -p, --password string password to use when connecting to redis server\n -u, --uri string redis server URI (default \"127.0.0.1:6379\")\n\n --cluster connect to redis cluster\n --cluster_addrs string list of comma-separated redis server addresses\n```\n\n## Config File\n\nYou can use a config file to set default values for the flags.\n\nBy default, `asynq` will try to read config file located in\n`$HOME/.asynq.(yml|json)`. You can specify the file location via `--config` flag.\n\nConfig file example:\n\n```yaml\nuri: 127.0.0.1:6379\ndb: 2\npassword: mypassword\n```\n\nThis will set the default values for `--uri`, `--db`, and `--password` flags.\n" }, { "path": "tools/asynq/cmd/cron.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"sort\"\n\t\"time\"\n\n\t\"github.com/MakeNowJust/heredoc/v2\"\n\t\"github.com/hibiken/asynq\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc init() {\n\trootCmd.AddCommand(cronCmd)\n\tcronCmd.AddCommand(cronListCmd)\n\tcronCmd.AddCommand(cronHistoryCmd)\n\tcronHistoryCmd.Flags().Int(\"page\", 1, \"page number\")\n\tcronHistoryCmd.Flags().Int(\"size\", 30, \"page size\")\n}\n\nvar cronCmd = &cobra.Command{\n\tUse: \"cron [flags]\",\n\tShort: \"Manage cron\",\n\tExample: heredoc.Doc(`\n\t\t$ asynq cron ls\n\t\t$ asynq cron history 7837f142-6337-4217-9276-8f27281b67d1`),\n}\n\nvar cronListCmd = &cobra.Command{\n\tUse: \"list\",\n\tAliases: []string{\"ls\"},\n\tShort: \"List cron entries\",\n\tRun: cronList,\n}\n\nvar cronHistoryCmd = &cobra.Command{\n\tUse: \"history [...]\",\n\tShort: \"Show history of each cron tasks\",\n\tArgs: cobra.MinimumNArgs(1),\n\tRun: cronHistory,\n\tExample: heredoc.Doc(`\n\t\t$ asynq cron history 7837f142-6337-4217-9276-8f27281b67d1\n\t\t$ asynq cron history 7837f142-6337-4217-9276-8f27281b67d1 bf6a8594-cd03-4968-b36a-8572c5e160dd\n\t\t$ asynq cron history 7837f142-6337-4217-9276-8f27281b67d1 --size=100\n\t\t$ asynq cron history 7837f142-6337-4217-9276-8f27281b67d1 --page=2`),\n}\n\nfunc cronList(cmd *cobra.Command, args []string) {\n\tinspector := createInspector()\n\n\tentries, err := inspector.SchedulerEntries()\n\tif err != nil {\n\t\tfmt.Println(err)\n\t\tos.Exit(1)\n\t}\n\tif len(entries) == 0 {\n\t\tfmt.Println(\"No scheduler entries\")\n\t\treturn\n\t}\n\n\t// Sort entries by spec.\n\tsort.Slice(entries, func(i, j int) bool {\n\t\tx, y := entries[i], entries[j]\n\t\treturn x.Spec < y.Spec\n\t})\n\n\tcols := []string{\"EntryID\", \"Spec\", \"Type\", \"Payload\", \"Options\", \"Next\", \"Prev\"}\n\tprintRows := func(w io.Writer, tmpl string) {\n\t\tfor _, e := range entries {\n\t\t\tfmt.Fprintf(w, tmpl, e.ID, e.Spec, e.Task.Type(), sprintBytes(e.Task.Payload()), e.Opts,\n\t\t\t\tnextEnqueue(e.Next), prevEnqueue(e.Prev))\n\t\t}\n\t}\n\tprintTable(cols, printRows)\n}\n\n// Returns a string describing when the next enqueue will happen.\nfunc nextEnqueue(nextEnqueueAt time.Time) string {\n\td := nextEnqueueAt.Sub(time.Now()).Round(time.Second)\n\tif d < 0 {\n\t\treturn \"Now\"\n\t}\n\treturn fmt.Sprintf(\"In %v\", d)\n}\n\n// Returns a string describing when the previous enqueue was.\nfunc prevEnqueue(prevEnqueuedAt time.Time) string {\n\tif prevEnqueuedAt.IsZero() {\n\t\treturn \"N/A\"\n\t}\n\treturn fmt.Sprintf(\"%v ago\", time.Since(prevEnqueuedAt).Round(time.Second))\n}\n\nfunc cronHistory(cmd *cobra.Command, args []string) {\n\tpageNum, err := cmd.Flags().GetInt(\"page\")\n\tif err != nil {\n\t\tfmt.Println(err)\n\t\tos.Exit(1)\n\t}\n\tpageSize, err := cmd.Flags().GetInt(\"size\")\n\tif err != nil {\n\t\tfmt.Println(err)\n\t\tos.Exit(1)\n\t}\n\tinspector := createInspector()\n\tfor i, entryID := range args {\n\t\tif i > 0 {\n\t\t\tfmt.Printf(\"\\n%s\\n\", separator)\n\t\t}\n\t\tfmt.Println()\n\n\t\tfmt.Printf(\"Entry: %s\\n\\n\", entryID)\n\n\t\tevents, err := inspector.ListSchedulerEnqueueEvents(\n\t\t\tentryID, asynq.PageSize(pageSize), asynq.Page(pageNum))\n\t\tif err != nil {\n\t\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\t\tcontinue\n\t\t}\n\t\tif len(events) == 0 {\n\t\t\tfmt.Printf(\"No scheduler enqueue events found for entry: %s\\n\", entryID)\n\t\t\tcontinue\n\t\t}\n\n\t\tcols := []string{\"TaskID\", \"EnqueuedAt\"}\n\t\tprintRows := func(w io.Writer, tmpl string) {\n\t\t\tfor _, e := range events {\n\t\t\t\tfmt.Fprintf(w, tmpl, e.TaskID, e.EnqueuedAt)\n\t\t\t}\n\t\t}\n\t\tprintTable(cols, printRows)\n\t}\n}\n" }, { "path": "tools/asynq/cmd/dash/dash.go", "content": "// Copyright 2022 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage dash\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/gdamore/tcell/v2\"\n\t\"github.com/hibiken/asynq\"\n)\n\n// viewType is an enum for dashboard views.\ntype viewType int\n\nconst (\n\tviewTypeQueues viewType = iota\n\tviewTypeQueueDetails\n\tviewTypeHelp\n)\n\n// State holds dashboard state.\ntype State struct {\n\tqueues []*asynq.QueueInfo\n\ttasks []*asynq.TaskInfo\n\tgroups []*asynq.GroupInfo\n\terr error\n\n\t// Note: index zero corresponds to the table header; index=1 correctponds to the first element\n\tqueueTableRowIdx int // highlighted row in queue table\n\ttaskTableRowIdx int // highlighted row in task table\n\tgroupTableRowIdx int // highlighted row in group table\n\ttaskState asynq.TaskState // highlighted task state in queue details view\n\ttaskID string // selected task ID\n\n\tselectedQueue *asynq.QueueInfo // queue shown on queue details view\n\tselectedGroup *asynq.GroupInfo\n\tselectedTask *asynq.TaskInfo\n\n\tpageNum int // pagination page number\n\n\tview viewType // current view type\n\tprevView viewType // to support \"go back\"\n}\n\nfunc (s *State) DebugString() string {\n\tvar b strings.Builder\n\tb.WriteString(fmt.Sprintf(\"len(queues)=%d \", len(s.queues)))\n\tb.WriteString(fmt.Sprintf(\"len(tasks)=%d \", len(s.tasks)))\n\tb.WriteString(fmt.Sprintf(\"len(groups)=%d \", len(s.groups)))\n\tb.WriteString(fmt.Sprintf(\"err=%v \", s.err))\n\n\tif s.taskState != 0 {\n\t\tb.WriteString(fmt.Sprintf(\"taskState=%s \", s.taskState.String()))\n\t} else {\n\t\tb.WriteString(fmt.Sprintf(\"taskState=0\"))\n\t}\n\tb.WriteString(fmt.Sprintf(\"taskID=%s \", s.taskID))\n\n\tb.WriteString(fmt.Sprintf(\"queueTableRowIdx=%d \", s.queueTableRowIdx))\n\tb.WriteString(fmt.Sprintf(\"taskTableRowIdx=%d \", s.taskTableRowIdx))\n\tb.WriteString(fmt.Sprintf(\"groupTableRowIdx=%d \", s.groupTableRowIdx))\n\n\tif s.selectedQueue != nil {\n\t\tb.WriteString(fmt.Sprintf(\"selectedQueue={Queue:%s} \", s.selectedQueue.Queue))\n\t} else {\n\t\tb.WriteString(\"selectedQueue=nil \")\n\t}\n\n\tif s.selectedGroup != nil {\n\t\tb.WriteString(fmt.Sprintf(\"selectedGroup={Group:%s} \", s.selectedGroup.Group))\n\t} else {\n\t\tb.WriteString(\"selectedGroup=nil \")\n\t}\n\n\tif s.selectedTask != nil {\n\t\tb.WriteString(fmt.Sprintf(\"selectedTask={ID:%s} \", s.selectedTask.ID))\n\t} else {\n\t\tb.WriteString(\"selectedTask=nil \")\n\t}\n\n\tb.WriteString(fmt.Sprintf(\"pageNum=%d\", s.pageNum))\n\treturn b.String()\n}\n\ntype Options struct {\n\tDebugMode bool\n\tPollInterval time.Duration\n\tRedisConnOpt asynq.RedisConnOpt\n}\n\nfunc Run(opts Options) {\n\ts, err := tcell.NewScreen()\n\tif err != nil {\n\t\tfmt.Printf(\"failed to create a screen: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\tif err := s.Init(); err != nil {\n\t\tfmt.Printf(\"failed to initialize screen: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\ts.SetStyle(baseStyle) // set default text style\n\n\tvar (\n\t\tstate = State{} // confined in this goroutine only; DO NOT SHARE\n\n\t\tinspector = asynq.NewInspector(opts.RedisConnOpt)\n\t\tticker = time.NewTicker(opts.PollInterval)\n\n\t\teventCh = make(chan tcell.Event)\n\t\tdone = make(chan struct{})\n\n\t\t// channels to send/receive data fetched asynchronously\n\t\terrorCh = make(chan error)\n\t\tqueueCh = make(chan *asynq.QueueInfo)\n\t\ttaskCh = make(chan *asynq.TaskInfo)\n\t\tqueuesCh = make(chan []*asynq.QueueInfo)\n\t\tgroupsCh = make(chan []*asynq.GroupInfo)\n\t\ttasksCh = make(chan []*asynq.TaskInfo)\n\t)\n\tdefer ticker.Stop()\n\n\tf := dataFetcher{\n\t\tinspector,\n\t\topts,\n\t\ts,\n\t\terrorCh,\n\t\tqueueCh,\n\t\ttaskCh,\n\t\tqueuesCh,\n\t\tgroupsCh,\n\t\ttasksCh,\n\t}\n\n\td := dashDrawer{\n\t\ts,\n\t\topts,\n\t}\n\n\th := keyEventHandler{\n\t\ts: s,\n\t\tfetcher: &f,\n\t\tdrawer: &d,\n\t\tstate: &state,\n\t\tdone: done,\n\t\tticker: ticker,\n\t\tpollInterval: opts.PollInterval,\n\t}\n\n\tgo fetchQueues(inspector, queuesCh, errorCh, opts)\n\tgo s.ChannelEvents(eventCh, done) // TODO: Double check that we are not leaking goroutine with this one.\n\td.Draw(&state) // draw initial screen\n\n\tfor {\n\t\t// Update screen\n\t\ts.Show()\n\n\t\tselect {\n\t\tcase ev := <-eventCh:\n\t\t\t// Process event\n\t\t\tswitch ev := ev.(type) {\n\t\t\tcase *tcell.EventResize:\n\t\t\t\ts.Sync()\n\t\t\tcase *tcell.EventKey:\n\t\t\t\th.HandleKeyEvent(ev)\n\t\t\t}\n\n\t\tcase <-ticker.C:\n\t\t\tf.Fetch(&state)\n\n\t\tcase queues := <-queuesCh:\n\t\t\tstate.queues = queues\n\t\t\tstate.err = nil\n\t\t\tif len(queues) < state.queueTableRowIdx {\n\t\t\t\tstate.queueTableRowIdx = len(queues)\n\t\t\t}\n\t\t\td.Draw(&state)\n\n\t\tcase q := <-queueCh:\n\t\t\tstate.selectedQueue = q\n\t\t\tstate.err = nil\n\t\t\td.Draw(&state)\n\n\t\tcase groups := <-groupsCh:\n\t\t\tstate.groups = groups\n\t\t\tstate.err = nil\n\t\t\tif len(groups) < state.groupTableRowIdx {\n\t\t\t\tstate.groupTableRowIdx = len(groups)\n\t\t\t}\n\t\t\td.Draw(&state)\n\n\t\tcase tasks := <-tasksCh:\n\t\t\tstate.tasks = tasks\n\t\t\tstate.err = nil\n\t\t\tif len(tasks) < state.taskTableRowIdx {\n\t\t\t\tstate.taskTableRowIdx = len(tasks)\n\t\t\t}\n\t\t\td.Draw(&state)\n\n\t\tcase t := <-taskCh:\n\t\t\tstate.selectedTask = t\n\t\t\tstate.err = nil\n\t\t\td.Draw(&state)\n\n\t\tcase err := <-errorCh:\n\t\t\tif errors.Is(err, asynq.ErrTaskNotFound) {\n\t\t\t\tstate.selectedTask = nil\n\t\t\t} else {\n\t\t\t\tstate.err = err\n\t\t\t}\n\t\t\td.Draw(&state)\n\t\t}\n\t}\n\n}\n" }, { "path": "tools/asynq/cmd/dash/draw.go", "content": "// Copyright 2022 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage dash\n\nimport (\n\t\"fmt\"\n\t\"math\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\t\"unicode\"\n\t\"unicode/utf8\"\n\n\t\"github.com/gdamore/tcell/v2\"\n\t\"github.com/hibiken/asynq\"\n\t\"github.com/mattn/go-runewidth\"\n)\n\nvar (\n\tbaseStyle = tcell.StyleDefault.Background(tcell.ColorReset).Foreground(tcell.ColorReset)\n\tlabelStyle = baseStyle.Foreground(tcell.ColorLightGray)\n\n\t// styles for bar graph\n\tactiveStyle = baseStyle.Foreground(tcell.ColorBlue)\n\tpendingStyle = baseStyle.Foreground(tcell.ColorGreen)\n\taggregatingStyle = baseStyle.Foreground(tcell.ColorLightGreen)\n\tscheduledStyle = baseStyle.Foreground(tcell.ColorYellow)\n\tretryStyle = baseStyle.Foreground(tcell.ColorPink)\n\tarchivedStyle = baseStyle.Foreground(tcell.ColorPurple)\n\tcompletedStyle = baseStyle.Foreground(tcell.ColorDarkGreen)\n)\n\n// drawer draws UI with the given state.\ntype drawer interface {\n\tDraw(state *State)\n}\n\ntype dashDrawer struct {\n\ts tcell.Screen\n\topts Options\n}\n\nfunc (dd *dashDrawer) Draw(state *State) {\n\ts, opts := dd.s, dd.opts\n\ts.Clear()\n\t// Simulate data update on every render\n\td := NewScreenDrawer(s)\n\tswitch state.view {\n\tcase viewTypeQueues:\n\t\td.Println(\"=== Queues ===\", baseStyle.Bold(true))\n\t\td.NL()\n\t\tdrawQueueSizeGraphs(d, state)\n\t\td.NL()\n\t\tdrawQueueTable(d, baseStyle, state)\n\tcase viewTypeQueueDetails:\n\t\td.Println(\"=== Queue Summary ===\", baseStyle.Bold(true))\n\t\td.NL()\n\t\tdrawQueueSummary(d, state)\n\t\td.NL()\n\t\td.NL()\n\t\td.Println(\"=== Tasks ===\", baseStyle.Bold(true))\n\t\td.NL()\n\t\tdrawTaskStateBreakdown(d, baseStyle, state)\n\t\td.NL()\n\t\tdrawTaskTable(d, state)\n\t\tdrawTaskModal(d, state)\n\tcase viewTypeHelp:\n\t\tdrawHelp(d)\n\t}\n\td.GoToBottom()\n\tif opts.DebugMode {\n\t\tdrawDebugInfo(d, state)\n\t} else {\n\t\tdrawFooter(d, state)\n\t}\n}\n\nfunc drawQueueSizeGraphs(d *ScreenDrawer, state *State) {\n\tvar qnames []string\n\tvar qsizes []string // queue size in strings\n\tmaxSize := 1 // not zero to avoid division by zero\n\tfor _, q := range state.queues {\n\t\tqnames = append(qnames, q.Queue)\n\t\tqsizes = append(qsizes, strconv.Itoa(q.Size))\n\t\tif q.Size > maxSize {\n\t\t\tmaxSize = q.Size\n\t\t}\n\t}\n\tqnameWidth := maxwidth(qnames)\n\tqsizeWidth := maxwidth(qsizes)\n\n\t// Calculate the multipler to scale the graph\n\tscreenWidth, _ := d.Screen().Size()\n\tgraphMaxWidth := screenWidth - (qnameWidth + qsizeWidth + 3) // | \n\tmultipiler := 1.0\n\tif graphMaxWidth < maxSize {\n\t\tmultipiler = float64(graphMaxWidth) / float64(maxSize)\n\t}\n\n\tconst tick = '▇'\n\tfor _, q := range state.queues {\n\t\td.Print(q.Queue, baseStyle)\n\t\td.Print(strings.Repeat(\" \", qnameWidth-runewidth.StringWidth(q.Queue)+1), baseStyle) // padding between qname and graph\n\t\td.Print(\"|\", baseStyle)\n\t\td.Print(strings.Repeat(string(tick), int(math.Floor(float64(q.Active)*multipiler))), activeStyle)\n\t\td.Print(strings.Repeat(string(tick), int(math.Floor(float64(q.Pending)*multipiler))), pendingStyle)\n\t\td.Print(strings.Repeat(string(tick), int(math.Floor(float64(q.Aggregating)*multipiler))), aggregatingStyle)\n\t\td.Print(strings.Repeat(string(tick), int(math.Floor(float64(q.Scheduled)*multipiler))), scheduledStyle)\n\t\td.Print(strings.Repeat(string(tick), int(math.Floor(float64(q.Retry)*multipiler))), retryStyle)\n\t\td.Print(strings.Repeat(string(tick), int(math.Floor(float64(q.Archived)*multipiler))), archivedStyle)\n\t\td.Print(strings.Repeat(string(tick), int(math.Floor(float64(q.Completed)*multipiler))), completedStyle)\n\t\td.Print(fmt.Sprintf(\" %d\", q.Size), baseStyle)\n\t\td.NL()\n\t}\n\td.NL()\n\td.Print(\"active=\", baseStyle)\n\td.Print(string(tick), activeStyle)\n\td.Print(\" pending=\", baseStyle)\n\td.Print(string(tick), pendingStyle)\n\td.Print(\" aggregating=\", baseStyle)\n\td.Print(string(tick), aggregatingStyle)\n\td.Print(\" scheduled=\", baseStyle)\n\td.Print(string(tick), scheduledStyle)\n\td.Print(\" retry=\", baseStyle)\n\td.Print(string(tick), retryStyle)\n\td.Print(\" archived=\", baseStyle)\n\td.Print(string(tick), archivedStyle)\n\td.Print(\" completed=\", baseStyle)\n\td.Print(string(tick), completedStyle)\n\td.NL()\n}\n\nfunc drawFooter(d *ScreenDrawer, state *State) {\n\tif state.err != nil {\n\t\tstyle := baseStyle.Background(tcell.ColorDarkRed)\n\t\td.Print(state.err.Error(), style)\n\t\td.FillLine(' ', style)\n\t\treturn\n\t}\n\tstyle := baseStyle.Background(tcell.ColorDarkSlateGray).Foreground(tcell.ColorWhite)\n\tswitch state.view {\n\tcase viewTypeHelp:\n\t\td.Print(\": GoBack\", style)\n\tdefault:\n\t\td.Print(\": Help : Exit \", style)\n\t}\n\td.FillLine(' ', style)\n}\n\n// returns the maximum width from the given list of names\nfunc maxwidth(names []string) int {\n\tmax := 0\n\tfor _, s := range names {\n\t\tif w := runewidth.StringWidth(s); w > max {\n\t\t\tmax = w\n\t\t}\n\t}\n\treturn max\n}\n\n// rpad adds padding to the right of a string.\nfunc rpad(s string, padding int) string {\n\ttmpl := fmt.Sprintf(\"%%-%ds \", padding)\n\treturn fmt.Sprintf(tmpl, s)\n\n}\n\n// lpad adds padding to the left of a string.\nfunc lpad(s string, padding int) string {\n\ttmpl := fmt.Sprintf(\"%%%ds \", padding)\n\treturn fmt.Sprintf(tmpl, s)\n}\n\n// byteCount converts the given bytes into human readable string\nfunc byteCount(b int64) string {\n\tconst unit = 1000\n\tif b < unit {\n\t\treturn fmt.Sprintf(\"%d B\", b)\n\n\t}\n\tdiv, exp := int64(unit), 0\n\tfor n := b / unit; n >= unit; n /= unit {\n\t\tdiv *= unit\n\t\texp++\n\n\t}\n\treturn fmt.Sprintf(\"%.1f %cB\", float64(b)/float64(div), \"kMGTPE\"[exp])\n}\n\nvar queueColumnConfigs = []*columnConfig[*asynq.QueueInfo]{\n\t{\"Queue\", alignLeft, func(q *asynq.QueueInfo) string { return q.Queue }},\n\t{\"State\", alignLeft, func(q *asynq.QueueInfo) string { return formatQueueState(q) }},\n\t{\"Size\", alignRight, func(q *asynq.QueueInfo) string { return strconv.Itoa(q.Size) }},\n\t{\"Latency\", alignRight, func(q *asynq.QueueInfo) string { return q.Latency.Round(time.Second).String() }},\n\t{\"MemoryUsage\", alignRight, func(q *asynq.QueueInfo) string { return byteCount(q.MemoryUsage) }},\n\t{\"Processed\", alignRight, func(q *asynq.QueueInfo) string { return strconv.Itoa(q.Processed) }},\n\t{\"Failed\", alignRight, func(q *asynq.QueueInfo) string { return strconv.Itoa(q.Failed) }},\n\t{\"ErrorRate\", alignRight, func(q *asynq.QueueInfo) string { return formatErrorRate(q.Processed, q.Failed) }},\n}\n\nfunc formatQueueState(q *asynq.QueueInfo) string {\n\tif q.Paused {\n\t\treturn \"PAUSED\"\n\t}\n\treturn \"RUN\"\n}\n\nfunc formatErrorRate(processed, failed int) string {\n\tif processed == 0 {\n\t\treturn \"-\"\n\t}\n\treturn fmt.Sprintf(\"%.2f\", float64(failed)/float64(processed))\n}\n\nfunc formatNextProcessTime(t time.Time) string {\n\tnow := time.Now()\n\tif t.Before(now) {\n\t\treturn \"now\"\n\t}\n\treturn fmt.Sprintf(\"in %v\", (t.Sub(now).Round(time.Second)))\n}\n\nfunc formatPastTime(t time.Time) string {\n\tnow := time.Now()\n\tif t.After(now) || t.Equal(now) {\n\t\treturn \"just now\"\n\t}\n\treturn fmt.Sprintf(\"%v ago\", time.Since(t).Round(time.Second))\n}\n\nfunc drawQueueTable(d *ScreenDrawer, style tcell.Style, state *State) {\n\tdrawTable(d, style, queueColumnConfigs, state.queues, state.queueTableRowIdx-1)\n}\n\nfunc drawQueueSummary(d *ScreenDrawer, state *State) {\n\tq := state.selectedQueue\n\tif q == nil {\n\t\td.Println(\"ERROR: Press q to go back\", baseStyle)\n\t\treturn\n\t}\n\td.Print(\"Name \", labelStyle)\n\td.Println(q.Queue, baseStyle)\n\td.Print(\"Size \", labelStyle)\n\td.Println(strconv.Itoa(q.Size), baseStyle)\n\td.Print(\"Latency \", labelStyle)\n\td.Println(q.Latency.Round(time.Second).String(), baseStyle)\n\td.Print(\"MemUsage \", labelStyle)\n\td.Println(byteCount(q.MemoryUsage), baseStyle)\n}\n\n// Returns the max number of groups that can be displayed.\nfunc groupPageSize(s tcell.Screen) int {\n\t_, h := s.Size()\n\treturn h - 16 // height - (# of rows used)\n}\n\n// Returns the number of tasks to fetch.\nfunc taskPageSize(s tcell.Screen) int {\n\t_, h := s.Size()\n\treturn h - 15 // height - (# of rows used)\n}\n\nfunc shouldShowGroupTable(state *State) bool {\n\treturn state.taskState == asynq.TaskStateAggregating && state.selectedGroup == nil\n}\n\nfunc getTaskTableColumnConfig(taskState asynq.TaskState) []*columnConfig[*asynq.TaskInfo] {\n\tswitch taskState {\n\tcase asynq.TaskStateActive:\n\t\treturn activeTaskTableColumns\n\tcase asynq.TaskStatePending:\n\t\treturn pendingTaskTableColumns\n\tcase asynq.TaskStateAggregating:\n\t\treturn aggregatingTaskTableColumns\n\tcase asynq.TaskStateScheduled:\n\t\treturn scheduledTaskTableColumns\n\tcase asynq.TaskStateRetry:\n\t\treturn retryTaskTableColumns\n\tcase asynq.TaskStateArchived:\n\t\treturn archivedTaskTableColumns\n\tcase asynq.TaskStateCompleted:\n\t\treturn completedTaskTableColumns\n\t}\n\tpanic(\"unknown task state\")\n}\n\nvar activeTaskTableColumns = []*columnConfig[*asynq.TaskInfo]{\n\t{\"ID\", alignLeft, func(t *asynq.TaskInfo) string { return t.ID }},\n\t{\"Type\", alignLeft, func(t *asynq.TaskInfo) string { return t.Type }},\n\t{\"Retried\", alignRight, func(t *asynq.TaskInfo) string { return strconv.Itoa(t.Retried) }},\n\t{\"Max Retry\", alignRight, func(t *asynq.TaskInfo) string { return strconv.Itoa(t.MaxRetry) }},\n\t{\"Payload\", alignLeft, func(t *asynq.TaskInfo) string { return formatByteSlice(t.Payload) }},\n}\n\nvar pendingTaskTableColumns = []*columnConfig[*asynq.TaskInfo]{\n\t{\"ID\", alignLeft, func(t *asynq.TaskInfo) string { return t.ID }},\n\t{\"Type\", alignLeft, func(t *asynq.TaskInfo) string { return t.Type }},\n\t{\"Retried\", alignRight, func(t *asynq.TaskInfo) string { return strconv.Itoa(t.Retried) }},\n\t{\"Max Retry\", alignRight, func(t *asynq.TaskInfo) string { return strconv.Itoa(t.MaxRetry) }},\n\t{\"Payload\", alignLeft, func(t *asynq.TaskInfo) string { return formatByteSlice(t.Payload) }},\n}\n\nvar aggregatingTaskTableColumns = []*columnConfig[*asynq.TaskInfo]{\n\t{\"ID\", alignLeft, func(t *asynq.TaskInfo) string { return t.ID }},\n\t{\"Type\", alignLeft, func(t *asynq.TaskInfo) string { return t.Type }},\n\t{\"Payload\", alignLeft, func(t *asynq.TaskInfo) string { return formatByteSlice(t.Payload) }},\n\t{\"Group\", alignLeft, func(t *asynq.TaskInfo) string { return t.Group }},\n}\n\nvar scheduledTaskTableColumns = []*columnConfig[*asynq.TaskInfo]{\n\t{\"ID\", alignLeft, func(t *asynq.TaskInfo) string { return t.ID }},\n\t{\"Type\", alignLeft, func(t *asynq.TaskInfo) string { return t.Type }},\n\t{\"Next Process Time\", alignLeft, func(t *asynq.TaskInfo) string {\n\t\treturn formatNextProcessTime(t.NextProcessAt)\n\t}},\n\t{\"Payload\", alignLeft, func(t *asynq.TaskInfo) string { return formatByteSlice(t.Payload) }},\n}\n\nvar retryTaskTableColumns = []*columnConfig[*asynq.TaskInfo]{\n\t{\"ID\", alignLeft, func(t *asynq.TaskInfo) string { return t.ID }},\n\t{\"Type\", alignLeft, func(t *asynq.TaskInfo) string { return t.Type }},\n\t{\"Retry\", alignRight, func(t *asynq.TaskInfo) string { return fmt.Sprintf(\"%d/%d\", t.Retried, t.MaxRetry) }},\n\t{\"Last Failure\", alignLeft, func(t *asynq.TaskInfo) string { return t.LastErr }},\n\t{\"Last Failure Time\", alignLeft, func(t *asynq.TaskInfo) string { return formatPastTime(t.LastFailedAt) }},\n\t{\"Next Process Time\", alignLeft, func(t *asynq.TaskInfo) string {\n\t\treturn formatNextProcessTime(t.NextProcessAt)\n\t}},\n\t{\"Payload\", alignLeft, func(t *asynq.TaskInfo) string { return formatByteSlice(t.Payload) }},\n}\n\nvar archivedTaskTableColumns = []*columnConfig[*asynq.TaskInfo]{\n\t{\"ID\", alignLeft, func(t *asynq.TaskInfo) string { return t.ID }},\n\t{\"Type\", alignLeft, func(t *asynq.TaskInfo) string { return t.Type }},\n\t{\"Retry\", alignRight, func(t *asynq.TaskInfo) string { return fmt.Sprintf(\"%d/%d\", t.Retried, t.MaxRetry) }},\n\t{\"Last Failure\", alignLeft, func(t *asynq.TaskInfo) string { return t.LastErr }},\n\t{\"Last Failure Time\", alignLeft, func(t *asynq.TaskInfo) string { return formatPastTime(t.LastFailedAt) }},\n\t{\"Payload\", alignLeft, func(t *asynq.TaskInfo) string { return formatByteSlice(t.Payload) }},\n}\n\nvar completedTaskTableColumns = []*columnConfig[*asynq.TaskInfo]{\n\t{\"ID\", alignLeft, func(t *asynq.TaskInfo) string { return t.ID }},\n\t{\"Type\", alignLeft, func(t *asynq.TaskInfo) string { return t.Type }},\n\t{\"Completion Time\", alignLeft, func(t *asynq.TaskInfo) string { return formatPastTime(t.CompletedAt) }},\n\t{\"Payload\", alignLeft, func(t *asynq.TaskInfo) string { return formatByteSlice(t.Payload) }},\n\t{\"Result\", alignLeft, func(t *asynq.TaskInfo) string { return formatByteSlice(t.Result) }},\n}\n\nfunc drawTaskTable(d *ScreenDrawer, state *State) {\n\tif shouldShowGroupTable(state) {\n\t\tdrawGroupTable(d, state)\n\t\treturn\n\t}\n\tif len(state.tasks) == 0 {\n\t\treturn // print nothing\n\t}\n\tdrawTable(d, baseStyle, getTaskTableColumnConfig(state.taskState), state.tasks, state.taskTableRowIdx-1)\n\n\t// Pagination\n\tpageSize := taskPageSize(d.Screen())\n\ttotalCount := getTaskCount(state.selectedQueue, state.taskState)\n\tif state.taskState == asynq.TaskStateAggregating {\n\t\t// aggregating tasks are scoped to each group when shown in the table.\n\t\ttotalCount = state.selectedGroup.Size\n\t}\n\tif pageSize < totalCount {\n\t\tstart := (state.pageNum-1)*pageSize + 1\n\t\tend := start + len(state.tasks) - 1\n\t\tpaginationStyle := baseStyle.Foreground(tcell.ColorLightGray)\n\t\td.Print(fmt.Sprintf(\"Showing %d-%d out of %d\", start, end, totalCount), paginationStyle)\n\t\tif isNextTaskPageAvailable(d.Screen(), state) {\n\t\t\td.Print(\" n=NextPage\", paginationStyle)\n\t\t}\n\t\tif state.pageNum > 1 {\n\t\t\td.Print(\" p=PrevPage\", paginationStyle)\n\t\t}\n\t\td.FillLine(' ', paginationStyle)\n\t}\n}\n\nfunc isNextTaskPageAvailable(s tcell.Screen, state *State) bool {\n\ttotalCount := getTaskCount(state.selectedQueue, state.taskState)\n\tend := (state.pageNum-1)*taskPageSize(s) + len(state.tasks)\n\treturn end < totalCount\n}\n\nfunc drawGroupTable(d *ScreenDrawer, state *State) {\n\tif len(state.groups) == 0 {\n\t\treturn // print nothing\n\t}\n\td.Println(\"<<< Select group >>>\", baseStyle)\n\tcolConfigs := []*columnConfig[*asynq.GroupInfo]{\n\t\t{\"Name\", alignLeft, func(g *asynq.GroupInfo) string { return g.Group }},\n\t\t{\"Size\", alignRight, func(g *asynq.GroupInfo) string { return strconv.Itoa(g.Size) }},\n\t}\n\t// pagination\n\tpageSize := groupPageSize(d.Screen())\n\ttotal := len(state.groups)\n\tstart := (state.pageNum - 1) * pageSize\n\tend := min(start+pageSize, total)\n\tdrawTable(d, baseStyle, colConfigs, state.groups[start:end], state.groupTableRowIdx-1)\n\n\tif pageSize < total {\n\t\td.Print(fmt.Sprintf(\"Showing %d-%d out of %d\", start+1, end, total), labelStyle)\n\t\tif end < total {\n\t\t\td.Print(\" n=NextPage\", labelStyle)\n\t\t}\n\t\tif start > 0 {\n\t\t\td.Print(\" p=PrevPage\", labelStyle)\n\t\t}\n\t}\n\td.FillLine(' ', labelStyle)\n}\n\ntype number interface {\n\tint | int64 | float64\n}\n\n// min returns the smaller of x and y. if x==y, returns x\nfunc min[V number](x, y V) V {\n\tif x > y {\n\t\treturn y\n\t}\n\treturn x\n}\n\n// Define the order of states to show\nvar taskStates = []asynq.TaskState{\n\tasynq.TaskStateActive,\n\tasynq.TaskStatePending,\n\tasynq.TaskStateAggregating,\n\tasynq.TaskStateScheduled,\n\tasynq.TaskStateRetry,\n\tasynq.TaskStateArchived,\n\tasynq.TaskStateCompleted,\n}\n\nfunc nextTaskState(current asynq.TaskState) asynq.TaskState {\n\tfor i, ts := range taskStates {\n\t\tif current == ts {\n\t\t\tif i == len(taskStates)-1 {\n\t\t\t\treturn taskStates[0]\n\t\t\t} else {\n\t\t\t\treturn taskStates[i+1]\n\t\t\t}\n\t\t}\n\t}\n\tpanic(\"unknown task state\")\n}\n\nfunc prevTaskState(current asynq.TaskState) asynq.TaskState {\n\tfor i, ts := range taskStates {\n\t\tif current == ts {\n\t\t\tif i == 0 {\n\t\t\t\treturn taskStates[len(taskStates)-1]\n\t\t\t} else {\n\t\t\t\treturn taskStates[i-1]\n\t\t\t}\n\t\t}\n\t}\n\tpanic(\"unknown task state\")\n}\n\nfunc getTaskCount(queue *asynq.QueueInfo, taskState asynq.TaskState) int {\n\tswitch taskState {\n\tcase asynq.TaskStateActive:\n\t\treturn queue.Active\n\tcase asynq.TaskStatePending:\n\t\treturn queue.Pending\n\tcase asynq.TaskStateAggregating:\n\t\treturn queue.Aggregating\n\tcase asynq.TaskStateScheduled:\n\t\treturn queue.Scheduled\n\tcase asynq.TaskStateRetry:\n\t\treturn queue.Retry\n\tcase asynq.TaskStateArchived:\n\t\treturn queue.Archived\n\tcase asynq.TaskStateCompleted:\n\t\treturn queue.Completed\n\t}\n\tpanic(\"unkonwn task state\")\n}\n\nfunc drawTaskStateBreakdown(d *ScreenDrawer, style tcell.Style, state *State) {\n\tconst pad = \" \" // padding between states\n\tfor _, ts := range taskStates {\n\t\ts := style\n\t\tif state.taskState == ts {\n\t\t\ts = s.Bold(true).Underline(true)\n\t\t}\n\t\td.Print(fmt.Sprintf(\"%s:%d\", strings.Title(ts.String()), getTaskCount(state.selectedQueue, ts)), s)\n\t\td.Print(pad, style)\n\t}\n\td.NL()\n}\n\nfunc drawTaskModal(d *ScreenDrawer, state *State) {\n\tif state.taskID == \"\" {\n\t\treturn\n\t}\n\ttask := state.selectedTask\n\tif task == nil {\n\t\t// task no longer found\n\t\tfns := []func(d *modalRowDrawer){\n\t\t\tfunc(d *modalRowDrawer) { d.Print(\"=== Task Info ===\", baseStyle.Bold(true)) },\n\t\t\tfunc(d *modalRowDrawer) { d.Print(\"\", baseStyle) },\n\t\t\tfunc(d *modalRowDrawer) {\n\t\t\t\td.Print(fmt.Sprintf(\"Task %q no longer exists\", state.taskID), baseStyle)\n\t\t\t},\n\t\t}\n\t\twithModal(d, fns)\n\t\treturn\n\t}\n\tfns := []func(d *modalRowDrawer){\n\t\tfunc(d *modalRowDrawer) { d.Print(\"=== Task Info ===\", baseStyle.Bold(true)) },\n\t\tfunc(d *modalRowDrawer) { d.Print(\"\", baseStyle) },\n\t\tfunc(d *modalRowDrawer) {\n\t\t\td.Print(\"ID: \", labelStyle)\n\t\t\td.Print(task.ID, baseStyle)\n\t\t},\n\t\tfunc(d *modalRowDrawer) {\n\t\t\td.Print(\"Type: \", labelStyle)\n\t\t\td.Print(task.Type, baseStyle)\n\t\t},\n\t\tfunc(d *modalRowDrawer) {\n\t\t\td.Print(\"State: \", labelStyle)\n\t\t\td.Print(task.State.String(), baseStyle)\n\t\t},\n\t\tfunc(d *modalRowDrawer) {\n\t\t\td.Print(\"Queue: \", labelStyle)\n\t\t\td.Print(task.Queue, baseStyle)\n\t\t},\n\t\tfunc(d *modalRowDrawer) {\n\t\t\td.Print(\"Retry: \", labelStyle)\n\t\t\td.Print(fmt.Sprintf(\"%d/%d\", task.Retried, task.MaxRetry), baseStyle)\n\t\t},\n\t}\n\tif task.LastErr != \"\" {\n\t\tfns = append(fns, func(d *modalRowDrawer) {\n\t\t\td.Print(\"Last Failure: \", labelStyle)\n\t\t\td.Print(task.LastErr, baseStyle)\n\t\t})\n\t\tfns = append(fns, func(d *modalRowDrawer) {\n\t\t\td.Print(\"Last Failure Time: \", labelStyle)\n\t\t\td.Print(fmt.Sprintf(\"%v (%s)\", task.LastFailedAt, formatPastTime(task.LastFailedAt)), baseStyle)\n\t\t})\n\t}\n\tif !task.NextProcessAt.IsZero() {\n\t\tfns = append(fns, func(d *modalRowDrawer) {\n\t\t\td.Print(\"Next Process Time: \", labelStyle)\n\t\t\td.Print(fmt.Sprintf(\"%v (%s)\", task.NextProcessAt, formatNextProcessTime(task.NextProcessAt)), baseStyle)\n\t\t})\n\t}\n\tif !task.CompletedAt.IsZero() {\n\t\tfns = append(fns, func(d *modalRowDrawer) {\n\t\t\td.Print(\"Completion Time: \", labelStyle)\n\t\t\td.Print(fmt.Sprintf(\"%v (%s)\", task.CompletedAt, formatPastTime(task.CompletedAt)), baseStyle)\n\t\t})\n\t}\n\tfns = append(fns, func(d *modalRowDrawer) {\n\t\td.Print(\"Payload: \", labelStyle)\n\t\td.Print(formatByteSlice(task.Payload), baseStyle)\n\t})\n\tif task.Result != nil {\n\t\tfns = append(fns, func(d *modalRowDrawer) {\n\t\t\td.Print(\"Result: \", labelStyle)\n\t\t\td.Print(formatByteSlice(task.Result), baseStyle)\n\t\t})\n\t}\n\twithModal(d, fns)\n}\n\n// Reports whether the given byte slice is printable (i.e. human readable)\nfunc isPrintable(data []byte) bool {\n\tif !utf8.Valid(data) {\n\t\treturn false\n\t}\n\tisAllSpace := true\n\tfor _, r := range string(data) {\n\t\tif !unicode.IsGraphic(r) {\n\t\t\treturn false\n\t\t}\n\t\tif !unicode.IsSpace(r) {\n\t\t\tisAllSpace = false\n\t\t}\n\t}\n\treturn !isAllSpace\n}\n\nfunc formatByteSlice(data []byte) string {\n\tif data == nil {\n\t\treturn \"\"\n\t}\n\tif !isPrintable(data) {\n\t\treturn \"\"\n\t}\n\treturn strings.ReplaceAll(string(data), \"\\n\", \" \")\n}\n\ntype modalRowDrawer struct {\n\td *ScreenDrawer\n\twidth int // current width occupied by content\n\tmaxWidth int\n}\n\n// Note: s should not include newline\nfunc (d *modalRowDrawer) Print(s string, style tcell.Style) {\n\tif d.width >= d.maxWidth {\n\t\treturn // no longer write to this row\n\t}\n\tif d.width+runewidth.StringWidth(s) > d.maxWidth {\n\t\ts = truncate(s, d.maxWidth-d.width)\n\t}\n\td.d.Print(s, style)\n}\n\n// withModal draws a modal with the given functions row by row.\nfunc withModal(d *ScreenDrawer, rowPrintFns []func(d *modalRowDrawer)) {\n\tw, h := d.Screen().Size()\n\tvar (\n\t\tmodalWidth = int(math.Floor(float64(w) * 0.6))\n\t\tmodalHeight = int(math.Floor(float64(h) * 0.6))\n\t\trowOffset = int(math.Floor(float64(h) * 0.2)) // 20% from the top\n\t\tcolOffset = int(math.Floor(float64(w) * 0.2)) // 20% from the left\n\t)\n\tif modalHeight < 3 {\n\t\treturn // no content can be shown\n\t}\n\td.Goto(colOffset, rowOffset)\n\td.Print(string(tcell.RuneULCorner), baseStyle)\n\td.Print(strings.Repeat(string(tcell.RuneHLine), modalWidth-2), baseStyle)\n\td.Print(string(tcell.RuneURCorner), baseStyle)\n\td.NL()\n\trowDrawer := modalRowDrawer{\n\t\td: d,\n\t\twidth: 0,\n\t\tmaxWidth: modalWidth - 4, /* borders + paddings */\n\t}\n\tfor i := 1; i < modalHeight-1; i++ {\n\t\td.Goto(colOffset, rowOffset+i)\n\t\td.Print(fmt.Sprintf(\"%c \", tcell.RuneVLine), baseStyle)\n\t\tif i <= len(rowPrintFns) {\n\t\t\trowPrintFns[i-1](&rowDrawer)\n\t\t}\n\t\td.FillUntil(' ', baseStyle, colOffset+modalWidth-2)\n\t\td.Print(fmt.Sprintf(\" %c\", tcell.RuneVLine), baseStyle)\n\t\td.NL()\n\t}\n\td.Goto(colOffset, rowOffset+modalHeight-1)\n\td.Print(string(tcell.RuneLLCorner), baseStyle)\n\td.Print(strings.Repeat(string(tcell.RuneHLine), modalWidth-2), baseStyle)\n\td.Print(string(tcell.RuneLRCorner), baseStyle)\n\td.NL()\n}\n\nfunc adjustWidth(s string, width int) string {\n\tsw := runewidth.StringWidth(s)\n\tif sw > width {\n\t\treturn truncate(s, width)\n\t}\n\tvar b strings.Builder\n\tb.WriteString(s)\n\tb.WriteString(strings.Repeat(\" \", width-sw))\n\treturn b.String()\n}\n\n// truncates s if s exceeds max length.\nfunc truncate(s string, max int) string {\n\tif runewidth.StringWidth(s) <= max {\n\t\treturn s\n\t}\n\treturn string([]rune(s)[:max-1]) + \"…\"\n}\n\nfunc drawDebugInfo(d *ScreenDrawer, state *State) {\n\td.Println(state.DebugString(), baseStyle)\n}\n\nfunc drawHelp(d *ScreenDrawer) {\n\tkeyStyle := labelStyle.Bold(true)\n\twithModal(d, []func(*modalRowDrawer){\n\t\tfunc(d *modalRowDrawer) { d.Print(\"=== Help ===\", baseStyle.Bold(true)) },\n\t\tfunc(d *modalRowDrawer) { d.Print(\"\", baseStyle) },\n\t\tfunc(d *modalRowDrawer) {\n\t\t\td.Print(\"\", keyStyle)\n\t\t\td.Print(\" to select\", baseStyle)\n\t\t},\n\t\tfunc(d *modalRowDrawer) {\n\t\t\td.Print(\"\", keyStyle)\n\t\t\td.Print(\" or \", baseStyle)\n\t\t\td.Print(\"\", keyStyle)\n\t\t\td.Print(\" to go back\", baseStyle)\n\t\t},\n\t\tfunc(d *modalRowDrawer) {\n\t\t\td.Print(\"\", keyStyle)\n\t\t\td.Print(\" or \", baseStyle)\n\t\t\td.Print(\"\", keyStyle)\n\t\t\td.Print(\" to move up\", baseStyle)\n\t\t},\n\t\tfunc(d *modalRowDrawer) {\n\t\t\td.Print(\"\", keyStyle)\n\t\t\td.Print(\" or \", baseStyle)\n\t\t\td.Print(\"\", keyStyle)\n\t\t\td.Print(\" to move down\", baseStyle)\n\t\t},\n\t\tfunc(d *modalRowDrawer) {\n\t\t\td.Print(\"\", keyStyle)\n\t\t\td.Print(\" or \", baseStyle)\n\t\t\td.Print(\"\", keyStyle)\n\t\t\td.Print(\" to move left\", baseStyle)\n\t\t},\n\t\tfunc(d *modalRowDrawer) {\n\t\t\td.Print(\"\", keyStyle)\n\t\t\td.Print(\" or \", baseStyle)\n\t\t\td.Print(\"\", keyStyle)\n\t\t\td.Print(\" to move right\", baseStyle)\n\t\t},\n\t\tfunc(d *modalRowDrawer) {\n\t\t\td.Print(\"\", keyStyle)\n\t\t\td.Print(\" to quit\", baseStyle)\n\t\t},\n\t})\n}\n" }, { "path": "tools/asynq/cmd/dash/draw_test.go", "content": "// Copyright 2022 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage dash\n\nimport \"testing\"\n\nfunc TestTruncate(t *testing.T) {\n\ttests := []struct {\n\t\ts string\n\t\tmax int\n\t\twant string\n\t}{\n\t\t{\n\t\t\ts: \"hello world!\",\n\t\t\tmax: 15,\n\t\t\twant: \"hello world!\",\n\t\t},\n\t\t{\n\t\t\ts: \"hello world!\",\n\t\t\tmax: 6,\n\t\t\twant: \"hello…\",\n\t\t},\n\t}\n\n\tfor _, tc := range tests {\n\t\tgot := truncate(tc.s, tc.max)\n\t\tif tc.want != got {\n\t\t\tt.Errorf(\"truncate(%q, %d) = %q, want %q\", tc.s, tc.max, got, tc.want)\n\t\t}\n\t}\n}\n" }, { "path": "tools/asynq/cmd/dash/fetch.go", "content": "// Copyright 2022 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage dash\n\nimport (\n\t\"sort\"\n\n\t\"github.com/gdamore/tcell/v2\"\n\t\"github.com/hibiken/asynq\"\n)\n\ntype fetcher interface {\n\t// Fetch retries data required by the given state of the dashboard.\n\tFetch(state *State)\n}\n\ntype dataFetcher struct {\n\tinspector *asynq.Inspector\n\topts Options\n\ts tcell.Screen\n\n\terrorCh chan<- error\n\tqueueCh chan<- *asynq.QueueInfo\n\ttaskCh chan<- *asynq.TaskInfo\n\tqueuesCh chan<- []*asynq.QueueInfo\n\tgroupsCh chan<- []*asynq.GroupInfo\n\ttasksCh chan<- []*asynq.TaskInfo\n}\n\nfunc (f *dataFetcher) Fetch(state *State) {\n\tswitch state.view {\n\tcase viewTypeQueues:\n\t\tf.fetchQueues()\n\tcase viewTypeQueueDetails:\n\t\tif shouldShowGroupTable(state) {\n\t\t\tf.fetchGroups(state.selectedQueue.Queue)\n\t\t} else if state.taskState == asynq.TaskStateAggregating {\n\t\t\tf.fetchAggregatingTasks(state.selectedQueue.Queue, state.selectedGroup.Group, taskPageSize(f.s), state.pageNum)\n\t\t} else {\n\t\t\tf.fetchTasks(state.selectedQueue.Queue, state.taskState, taskPageSize(f.s), state.pageNum)\n\t\t}\n\t\t// if the task modal is open, additionally fetch the selected task's info\n\t\tif state.taskID != \"\" {\n\t\t\tf.fetchTaskInfo(state.selectedQueue.Queue, state.taskID)\n\t\t}\n\t}\n}\n\nfunc (f *dataFetcher) fetchQueues() {\n\tvar (\n\t\tinspector = f.inspector\n\t\tqueuesCh = f.queuesCh\n\t\terrorCh = f.errorCh\n\t\topts = f.opts\n\t)\n\tgo fetchQueues(inspector, queuesCh, errorCh, opts)\n}\n\nfunc fetchQueues(i *asynq.Inspector, queuesCh chan<- []*asynq.QueueInfo, errorCh chan<- error, opts Options) {\n\tqueues, err := i.Queues()\n\tif err != nil {\n\t\terrorCh <- err\n\t\treturn\n\t}\n\tsort.Strings(queues)\n\tvar res []*asynq.QueueInfo\n\tfor _, q := range queues {\n\t\tinfo, err := i.GetQueueInfo(q)\n\t\tif err != nil {\n\t\t\terrorCh <- err\n\t\t\treturn\n\t\t}\n\t\tres = append(res, info)\n\t}\n\tqueuesCh <- res\n}\n\nfunc fetchQueueInfo(i *asynq.Inspector, qname string, queueCh chan<- *asynq.QueueInfo, errorCh chan<- error) {\n\tq, err := i.GetQueueInfo(qname)\n\tif err != nil {\n\t\terrorCh <- err\n\t\treturn\n\t}\n\tqueueCh <- q\n}\n\nfunc (f *dataFetcher) fetchGroups(qname string) {\n\tvar (\n\t\ti = f.inspector\n\t\tgroupsCh = f.groupsCh\n\t\terrorCh = f.errorCh\n\t\tqueueCh = f.queueCh\n\t)\n\tgo fetchGroups(i, qname, groupsCh, errorCh)\n\tgo fetchQueueInfo(i, qname, queueCh, errorCh)\n}\n\nfunc fetchGroups(i *asynq.Inspector, qname string, groupsCh chan<- []*asynq.GroupInfo, errorCh chan<- error) {\n\tgroups, err := i.Groups(qname)\n\tif err != nil {\n\t\terrorCh <- err\n\t\treturn\n\t}\n\tgroupsCh <- groups\n}\n\nfunc (f *dataFetcher) fetchAggregatingTasks(qname, group string, pageSize, pageNum int) {\n\tvar (\n\t\ti = f.inspector\n\t\ttasksCh = f.tasksCh\n\t\terrorCh = f.errorCh\n\t\tqueueCh = f.queueCh\n\t)\n\tgo fetchAggregatingTasks(i, qname, group, pageSize, pageNum, tasksCh, errorCh)\n\tgo fetchQueueInfo(i, qname, queueCh, errorCh)\n}\n\nfunc fetchAggregatingTasks(i *asynq.Inspector, qname, group string, pageSize, pageNum int,\n\ttasksCh chan<- []*asynq.TaskInfo, errorCh chan<- error) {\n\ttasks, err := i.ListAggregatingTasks(qname, group, asynq.PageSize(pageSize), asynq.Page(pageNum))\n\tif err != nil {\n\t\terrorCh <- err\n\t\treturn\n\t}\n\ttasksCh <- tasks\n}\n\nfunc (f *dataFetcher) fetchTasks(qname string, taskState asynq.TaskState, pageSize, pageNum int) {\n\tvar (\n\t\ti = f.inspector\n\t\ttasksCh = f.tasksCh\n\t\terrorCh = f.errorCh\n\t\tqueueCh = f.queueCh\n\t)\n\tgo fetchTasks(i, qname, taskState, pageSize, pageNum, tasksCh, errorCh)\n\tgo fetchQueueInfo(i, qname, queueCh, errorCh)\n}\n\nfunc fetchTasks(i *asynq.Inspector, qname string, taskState asynq.TaskState, pageSize, pageNum int,\n\ttasksCh chan<- []*asynq.TaskInfo, errorCh chan<- error) {\n\tvar (\n\t\ttasks []*asynq.TaskInfo\n\t\terr error\n\t)\n\topts := []asynq.ListOption{asynq.PageSize(pageSize), asynq.Page(pageNum)}\n\tswitch taskState {\n\tcase asynq.TaskStateActive:\n\t\ttasks, err = i.ListActiveTasks(qname, opts...)\n\tcase asynq.TaskStatePending:\n\t\ttasks, err = i.ListPendingTasks(qname, opts...)\n\tcase asynq.TaskStateScheduled:\n\t\ttasks, err = i.ListScheduledTasks(qname, opts...)\n\tcase asynq.TaskStateRetry:\n\t\ttasks, err = i.ListRetryTasks(qname, opts...)\n\tcase asynq.TaskStateArchived:\n\t\ttasks, err = i.ListArchivedTasks(qname, opts...)\n\tcase asynq.TaskStateCompleted:\n\t\ttasks, err = i.ListCompletedTasks(qname, opts...)\n\t}\n\tif err != nil {\n\t\terrorCh <- err\n\t\treturn\n\t}\n\ttasksCh <- tasks\n}\n\nfunc (f *dataFetcher) fetchTaskInfo(qname, taskID string) {\n\tvar (\n\t\ti = f.inspector\n\t\ttaskCh = f.taskCh\n\t\terrorCh = f.errorCh\n\t)\n\tgo fetchTaskInfo(i, qname, taskID, taskCh, errorCh)\n}\n\nfunc fetchTaskInfo(i *asynq.Inspector, qname, taskID string, taskCh chan<- *asynq.TaskInfo, errorCh chan<- error) {\n\tinfo, err := i.GetTaskInfo(qname, taskID)\n\tif err != nil {\n\t\terrorCh <- err\n\t\treturn\n\t}\n\ttaskCh <- info\n}\n" }, { "path": "tools/asynq/cmd/dash/key_event.go", "content": "// Copyright 2022 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage dash\n\nimport (\n\t\"os\"\n\t\"time\"\n\n\t\"github.com/gdamore/tcell/v2\"\n\t\"github.com/hibiken/asynq\"\n)\n\n// keyEventHandler handles keyboard events and updates the state.\n// It delegates data fetching to fetcher and UI rendering to drawer.\ntype keyEventHandler struct {\n\ts tcell.Screen\n\tstate *State\n\tdone chan struct{}\n\n\tfetcher fetcher\n\tdrawer drawer\n\n\tticker *time.Ticker\n\tpollInterval time.Duration\n}\n\nfunc (h *keyEventHandler) quit() {\n\th.s.Fini()\n\tclose(h.done)\n\tos.Exit(0)\n}\n\nfunc (h *keyEventHandler) HandleKeyEvent(ev *tcell.EventKey) {\n\tif ev.Key() == tcell.KeyEscape || ev.Rune() == 'q' {\n\t\th.goBack() // Esc and 'q' key have \"go back\" semantics\n\t} else if ev.Key() == tcell.KeyCtrlC {\n\t\th.quit()\n\t} else if ev.Key() == tcell.KeyCtrlL {\n\t\th.s.Sync()\n\t} else if ev.Key() == tcell.KeyDown || ev.Rune() == 'j' {\n\t\th.handleDownKey()\n\t} else if ev.Key() == tcell.KeyUp || ev.Rune() == 'k' {\n\t\th.handleUpKey()\n\t} else if ev.Key() == tcell.KeyRight || ev.Rune() == 'l' {\n\t\th.handleRightKey()\n\t} else if ev.Key() == tcell.KeyLeft || ev.Rune() == 'h' {\n\t\th.handleLeftKey()\n\t} else if ev.Key() == tcell.KeyEnter {\n\t\th.handleEnterKey()\n\t} else if ev.Rune() == '?' {\n\t\th.showHelp()\n\t} else if ev.Rune() == 'n' {\n\t\th.nextPage()\n\t} else if ev.Rune() == 'p' {\n\t\th.prevPage()\n\t}\n}\n\nfunc (h *keyEventHandler) goBack() {\n\tvar (\n\t\tstate = h.state\n\t\td = h.drawer\n\t\tf = h.fetcher\n\t)\n\tif state.view == viewTypeHelp {\n\t\tstate.view = state.prevView // exit help\n\t\tf.Fetch(state)\n\t\th.resetTicker()\n\t\td.Draw(state)\n\t} else if state.view == viewTypeQueueDetails {\n\t\t// if task modal is open close it; otherwise go back to the previous view\n\t\tif state.taskID != \"\" {\n\t\t\tstate.taskID = \"\"\n\t\t\tstate.selectedTask = nil\n\t\t\td.Draw(state)\n\t\t} else {\n\t\t\tstate.view = viewTypeQueues\n\t\t\tf.Fetch(state)\n\t\t\th.resetTicker()\n\t\t\td.Draw(state)\n\t\t}\n\t} else {\n\t\th.quit()\n\t}\n}\n\nfunc (h *keyEventHandler) handleDownKey() {\n\tswitch h.state.view {\n\tcase viewTypeQueues:\n\t\th.downKeyQueues()\n\tcase viewTypeQueueDetails:\n\t\th.downKeyQueueDetails()\n\t}\n}\n\nfunc (h *keyEventHandler) downKeyQueues() {\n\tif h.state.queueTableRowIdx < len(h.state.queues) {\n\t\th.state.queueTableRowIdx++\n\t} else {\n\t\th.state.queueTableRowIdx = 0 // loop back\n\t}\n\th.drawer.Draw(h.state)\n}\n\nfunc (h *keyEventHandler) downKeyQueueDetails() {\n\ts, state := h.s, h.state\n\tif shouldShowGroupTable(state) {\n\t\tif state.groupTableRowIdx < groupPageSize(s) {\n\t\t\tstate.groupTableRowIdx++\n\t\t} else {\n\t\t\tstate.groupTableRowIdx = 0 // loop back\n\t\t}\n\t} else if state.taskID == \"\" {\n\t\tif state.taskTableRowIdx < len(state.tasks) {\n\t\t\tstate.taskTableRowIdx++\n\t\t} else {\n\t\t\tstate.taskTableRowIdx = 0 // loop back\n\t\t}\n\t}\n\th.drawer.Draw(state)\n}\n\nfunc (h *keyEventHandler) handleUpKey() {\n\tswitch h.state.view {\n\tcase viewTypeQueues:\n\t\th.upKeyQueues()\n\tcase viewTypeQueueDetails:\n\t\th.upKeyQueueDetails()\n\t}\n}\n\nfunc (h *keyEventHandler) upKeyQueues() {\n\tstate := h.state\n\tif state.queueTableRowIdx == 0 {\n\t\tstate.queueTableRowIdx = len(state.queues)\n\t} else {\n\t\tstate.queueTableRowIdx--\n\t}\n\th.drawer.Draw(state)\n}\n\nfunc (h *keyEventHandler) upKeyQueueDetails() {\n\ts, state := h.s, h.state\n\tif shouldShowGroupTable(state) {\n\t\tif state.groupTableRowIdx == 0 {\n\t\t\tstate.groupTableRowIdx = groupPageSize(s)\n\t\t} else {\n\t\t\tstate.groupTableRowIdx--\n\t\t}\n\t} else if state.taskID == \"\" {\n\t\tif state.taskTableRowIdx == 0 {\n\t\t\tstate.taskTableRowIdx = len(state.tasks)\n\t\t} else {\n\t\t\tstate.taskTableRowIdx--\n\t\t}\n\t}\n\th.drawer.Draw(state)\n}\n\nfunc (h *keyEventHandler) handleEnterKey() {\n\tswitch h.state.view {\n\tcase viewTypeQueues:\n\t\th.enterKeyQueues()\n\tcase viewTypeQueueDetails:\n\t\th.enterKeyQueueDetails()\n\t}\n}\n\nfunc (h *keyEventHandler) resetTicker() {\n\th.ticker.Reset(h.pollInterval)\n}\n\nfunc (h *keyEventHandler) enterKeyQueues() {\n\tvar (\n\t\tstate = h.state\n\t\tf = h.fetcher\n\t\td = h.drawer\n\t)\n\tif state.queueTableRowIdx != 0 {\n\t\tstate.selectedQueue = state.queues[state.queueTableRowIdx-1]\n\t\tstate.view = viewTypeQueueDetails\n\t\tstate.taskState = asynq.TaskStateActive\n\t\tstate.tasks = nil\n\t\tstate.pageNum = 1\n\t\tf.Fetch(state)\n\t\th.resetTicker()\n\t\td.Draw(state)\n\t}\n}\n\nfunc (h *keyEventHandler) enterKeyQueueDetails() {\n\tvar (\n\t\tstate = h.state\n\t\tf = h.fetcher\n\t\td = h.drawer\n\t)\n\tif shouldShowGroupTable(state) && state.groupTableRowIdx != 0 {\n\t\tstate.selectedGroup = state.groups[state.groupTableRowIdx-1]\n\t\tstate.tasks = nil\n\t\tstate.pageNum = 1\n\t\tf.Fetch(state)\n\t\th.resetTicker()\n\t\td.Draw(state)\n\t} else if !shouldShowGroupTable(state) && state.taskTableRowIdx != 0 {\n\t\ttask := state.tasks[state.taskTableRowIdx-1]\n\t\tstate.selectedTask = task\n\t\tstate.taskID = task.ID\n\t\tf.Fetch(state)\n\t\th.resetTicker()\n\t\td.Draw(state)\n\t}\n\n}\n\nfunc (h *keyEventHandler) handleLeftKey() {\n\tvar (\n\t\tstate = h.state\n\t\tf = h.fetcher\n\t\td = h.drawer\n\t)\n\tif state.view == viewTypeQueueDetails && state.taskID == \"\" {\n\t\tstate.taskState = prevTaskState(state.taskState)\n\t\tstate.pageNum = 1\n\t\tstate.taskTableRowIdx = 0\n\t\tstate.tasks = nil\n\t\tstate.selectedGroup = nil\n\t\tf.Fetch(state)\n\t\th.resetTicker()\n\t\td.Draw(state)\n\t}\n}\n\nfunc (h *keyEventHandler) handleRightKey() {\n\tvar (\n\t\tstate = h.state\n\t\tf = h.fetcher\n\t\td = h.drawer\n\t)\n\tif state.view == viewTypeQueueDetails && state.taskID == \"\" {\n\t\tstate.taskState = nextTaskState(state.taskState)\n\t\tstate.pageNum = 1\n\t\tstate.taskTableRowIdx = 0\n\t\tstate.tasks = nil\n\t\tstate.selectedGroup = nil\n\t\tf.Fetch(state)\n\t\th.resetTicker()\n\t\td.Draw(state)\n\t}\n}\n\nfunc (h *keyEventHandler) nextPage() {\n\tvar (\n\t\ts = h.s\n\t\tstate = h.state\n\t\tf = h.fetcher\n\t\td = h.drawer\n\t)\n\tif state.view == viewTypeQueueDetails {\n\t\tif shouldShowGroupTable(state) {\n\t\t\tpageSize := groupPageSize(s)\n\t\t\ttotal := len(state.groups)\n\t\t\tstart := (state.pageNum - 1) * pageSize\n\t\t\tend := start + pageSize\n\t\t\tif end <= total {\n\t\t\t\tstate.pageNum++\n\t\t\t\td.Draw(state)\n\t\t\t}\n\t\t} else {\n\t\t\tpageSize := taskPageSize(s)\n\t\t\ttotalCount := getTaskCount(state.selectedQueue, state.taskState)\n\t\t\tif (state.pageNum-1)*pageSize+len(state.tasks) < totalCount {\n\t\t\t\tstate.pageNum++\n\t\t\t\tf.Fetch(state)\n\t\t\t\th.resetTicker()\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (h *keyEventHandler) prevPage() {\n\tvar (\n\t\ts = h.s\n\t\tstate = h.state\n\t\tf = h.fetcher\n\t\td = h.drawer\n\t)\n\tif state.view == viewTypeQueueDetails {\n\t\tif shouldShowGroupTable(state) {\n\t\t\tpageSize := groupPageSize(s)\n\t\t\tstart := (state.pageNum - 1) * pageSize\n\t\t\tif start > 0 {\n\t\t\t\tstate.pageNum--\n\t\t\t\td.Draw(state)\n\t\t\t}\n\t\t} else {\n\t\t\tif state.pageNum > 1 {\n\t\t\t\tstate.pageNum--\n\t\t\t\tf.Fetch(state)\n\t\t\t\th.resetTicker()\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (h *keyEventHandler) showHelp() {\n\tvar (\n\t\tstate = h.state\n\t\td = h.drawer\n\t)\n\tif state.view != viewTypeHelp {\n\t\tstate.prevView = state.view\n\t\tstate.view = viewTypeHelp\n\t\td.Draw(state)\n\t}\n}\n" }, { "path": "tools/asynq/cmd/dash/key_event_test.go", "content": "// Copyright 2022 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage dash\n\nimport (\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/gdamore/tcell/v2\"\n\t\"github.com/google/go-cmp/cmp\"\n\t\"github.com/hibiken/asynq\"\n)\n\nfunc makeKeyEventHandler(t *testing.T, state *State) *keyEventHandler {\n\tticker := time.NewTicker(time.Second)\n\tt.Cleanup(func() { ticker.Stop() })\n\treturn &keyEventHandler{\n\t\ts: tcell.NewSimulationScreen(\"UTF-8\"),\n\t\tstate: state,\n\t\tdone: make(chan struct{}),\n\t\tfetcher: &fakeFetcher{},\n\t\tdrawer: &fakeDrawer{},\n\t\tticker: ticker,\n\t\tpollInterval: time.Second,\n\t}\n}\n\ntype keyEventHandlerTest struct {\n\tdesc string // test description\n\tstate *State // initial state, to be mutated by the handler\n\tevents []*tcell.EventKey // keyboard events\n\twantState State // expected state after the events\n}\n\nfunc TestKeyEventHandler(t *testing.T) {\n\ttests := []*keyEventHandlerTest{\n\t\t{\n\t\t\tdesc: \"navigates to help view\",\n\t\t\tstate: &State{view: viewTypeQueues},\n\t\t\tevents: []*tcell.EventKey{tcell.NewEventKey(tcell.KeyRune, '?', tcell.ModNone)},\n\t\t\twantState: State{view: viewTypeHelp},\n\t\t},\n\t\t{\n\t\t\tdesc: \"navigates to queue details view\",\n\t\t\tstate: &State{\n\t\t\t\tview: viewTypeQueues,\n\t\t\t\tqueues: []*asynq.QueueInfo{\n\t\t\t\t\t{Queue: \"default\", Size: 100, Active: 10, Pending: 40, Scheduled: 40, Completed: 10},\n\t\t\t\t},\n\t\t\t\tqueueTableRowIdx: 0,\n\t\t\t},\n\t\t\tevents: []*tcell.EventKey{\n\t\t\t\ttcell.NewEventKey(tcell.KeyRune, 'j', tcell.ModNone), // down\n\t\t\t\ttcell.NewEventKey(tcell.KeyEnter, '\\n', tcell.ModNone), // Enter\n\t\t\t},\n\t\t\twantState: State{\n\t\t\t\tview: viewTypeQueueDetails,\n\t\t\t\tqueues: []*asynq.QueueInfo{\n\t\t\t\t\t{Queue: \"default\", Size: 100, Active: 10, Pending: 40, Scheduled: 40, Completed: 10},\n\t\t\t\t},\n\t\t\t\tselectedQueue: &asynq.QueueInfo{Queue: \"default\", Size: 100, Active: 10, Pending: 40, Scheduled: 40, Completed: 10},\n\t\t\t\tqueueTableRowIdx: 1,\n\t\t\t\ttaskState: asynq.TaskStateActive,\n\t\t\t\tpageNum: 1,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"does nothing if no queues are present\",\n\t\t\tstate: &State{\n\t\t\t\tview: viewTypeQueues,\n\t\t\t\tqueues: []*asynq.QueueInfo{}, // empty\n\t\t\t\tqueueTableRowIdx: 0,\n\t\t\t},\n\t\t\tevents: []*tcell.EventKey{\n\t\t\t\ttcell.NewEventKey(tcell.KeyRune, 'j', tcell.ModNone), // down\n\t\t\t\ttcell.NewEventKey(tcell.KeyEnter, '\\n', tcell.ModNone), // Enter\n\t\t\t},\n\t\t\twantState: State{\n\t\t\t\tview: viewTypeQueues,\n\t\t\t\tqueues: []*asynq.QueueInfo{},\n\t\t\t\tqueueTableRowIdx: 0,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"opens task info modal\",\n\t\t\tstate: &State{\n\t\t\t\tview: viewTypeQueueDetails,\n\t\t\t\tqueues: []*asynq.QueueInfo{\n\t\t\t\t\t{Queue: \"default\", Size: 500, Active: 10, Pending: 40},\n\t\t\t\t},\n\t\t\t\tqueueTableRowIdx: 1,\n\t\t\t\tselectedQueue: &asynq.QueueInfo{Queue: \"default\", Size: 50, Active: 10, Pending: 40},\n\t\t\t\ttaskState: asynq.TaskStatePending,\n\t\t\t\tpageNum: 1,\n\t\t\t\ttasks: []*asynq.TaskInfo{\n\t\t\t\t\t{ID: \"xxxx\", Type: \"foo\"},\n\t\t\t\t\t{ID: \"yyyy\", Type: \"bar\"},\n\t\t\t\t\t{ID: \"zzzz\", Type: \"baz\"},\n\t\t\t\t},\n\t\t\t\ttaskTableRowIdx: 2,\n\t\t\t},\n\t\t\tevents: []*tcell.EventKey{\n\t\t\t\ttcell.NewEventKey(tcell.KeyEnter, '\\n', tcell.ModNone), // Enter\n\t\t\t},\n\t\t\twantState: State{\n\t\t\t\tview: viewTypeQueueDetails,\n\t\t\t\tqueues: []*asynq.QueueInfo{\n\t\t\t\t\t{Queue: \"default\", Size: 500, Active: 10, Pending: 40},\n\t\t\t\t},\n\t\t\t\tqueueTableRowIdx: 1,\n\t\t\t\tselectedQueue: &asynq.QueueInfo{Queue: \"default\", Size: 50, Active: 10, Pending: 40},\n\t\t\t\ttaskState: asynq.TaskStatePending,\n\t\t\t\tpageNum: 1,\n\t\t\t\ttasks: []*asynq.TaskInfo{\n\t\t\t\t\t{ID: \"xxxx\", Type: \"foo\"},\n\t\t\t\t\t{ID: \"yyyy\", Type: \"bar\"},\n\t\t\t\t\t{ID: \"zzzz\", Type: \"baz\"},\n\t\t\t\t},\n\t\t\t\ttaskTableRowIdx: 2,\n\t\t\t\t// new states\n\t\t\t\ttaskID: \"yyyy\",\n\t\t\t\tselectedTask: &asynq.TaskInfo{ID: \"yyyy\", Type: \"bar\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"Esc closes task info modal\",\n\t\t\tstate: &State{\n\t\t\t\tview: viewTypeQueueDetails,\n\t\t\t\tqueues: []*asynq.QueueInfo{\n\t\t\t\t\t{Queue: \"default\", Size: 500, Active: 10, Pending: 40},\n\t\t\t\t},\n\t\t\t\tqueueTableRowIdx: 1,\n\t\t\t\tselectedQueue: &asynq.QueueInfo{Queue: \"default\", Size: 50, Active: 10, Pending: 40},\n\t\t\t\ttaskState: asynq.TaskStatePending,\n\t\t\t\tpageNum: 1,\n\t\t\t\ttasks: []*asynq.TaskInfo{\n\t\t\t\t\t{ID: \"xxxx\", Type: \"foo\"},\n\t\t\t\t\t{ID: \"yyyy\", Type: \"bar\"},\n\t\t\t\t\t{ID: \"zzzz\", Type: \"baz\"},\n\t\t\t\t},\n\t\t\t\ttaskTableRowIdx: 2,\n\t\t\t\ttaskID: \"yyyy\", // presence of this field opens the modal\n\t\t\t},\n\t\t\tevents: []*tcell.EventKey{\n\t\t\t\ttcell.NewEventKey(tcell.KeyEscape, ' ', tcell.ModNone), // Esc\n\t\t\t},\n\t\t\twantState: State{\n\t\t\t\tview: viewTypeQueueDetails,\n\t\t\t\tqueues: []*asynq.QueueInfo{\n\t\t\t\t\t{Queue: \"default\", Size: 500, Active: 10, Pending: 40},\n\t\t\t\t},\n\t\t\t\tqueueTableRowIdx: 1,\n\t\t\t\tselectedQueue: &asynq.QueueInfo{Queue: \"default\", Size: 50, Active: 10, Pending: 40},\n\t\t\t\ttaskState: asynq.TaskStatePending,\n\t\t\t\tpageNum: 1,\n\t\t\t\ttasks: []*asynq.TaskInfo{\n\t\t\t\t\t{ID: \"xxxx\", Type: \"foo\"},\n\t\t\t\t\t{ID: \"yyyy\", Type: \"bar\"},\n\t\t\t\t\t{ID: \"zzzz\", Type: \"baz\"},\n\t\t\t\t},\n\t\t\t\ttaskTableRowIdx: 2,\n\t\t\t\ttaskID: \"\", // this field should be unset\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"Arrow keys are disabled while task info modal is open\",\n\t\t\tstate: &State{\n\t\t\t\tview: viewTypeQueueDetails,\n\t\t\t\tqueues: []*asynq.QueueInfo{\n\t\t\t\t\t{Queue: \"default\", Size: 500, Active: 10, Pending: 40},\n\t\t\t\t},\n\t\t\t\tqueueTableRowIdx: 1,\n\t\t\t\tselectedQueue: &asynq.QueueInfo{Queue: \"default\", Size: 50, Active: 10, Pending: 40},\n\t\t\t\ttaskState: asynq.TaskStatePending,\n\t\t\t\tpageNum: 1,\n\t\t\t\ttasks: []*asynq.TaskInfo{\n\t\t\t\t\t{ID: \"xxxx\", Type: \"foo\"},\n\t\t\t\t\t{ID: \"yyyy\", Type: \"bar\"},\n\t\t\t\t\t{ID: \"zzzz\", Type: \"baz\"},\n\t\t\t\t},\n\t\t\t\ttaskTableRowIdx: 2,\n\t\t\t\ttaskID: \"yyyy\", // presence of this field opens the modal\n\t\t\t},\n\t\t\tevents: []*tcell.EventKey{\n\t\t\t\ttcell.NewEventKey(tcell.KeyLeft, ' ', tcell.ModNone),\n\t\t\t},\n\n\t\t\t// no change\n\t\t\twantState: State{\n\t\t\t\tview: viewTypeQueueDetails,\n\t\t\t\tqueues: []*asynq.QueueInfo{\n\t\t\t\t\t{Queue: \"default\", Size: 500, Active: 10, Pending: 40},\n\t\t\t\t},\n\t\t\t\tqueueTableRowIdx: 1,\n\t\t\t\tselectedQueue: &asynq.QueueInfo{Queue: \"default\", Size: 50, Active: 10, Pending: 40},\n\t\t\t\ttaskState: asynq.TaskStatePending,\n\t\t\t\tpageNum: 1,\n\t\t\t\ttasks: []*asynq.TaskInfo{\n\t\t\t\t\t{ID: \"xxxx\", Type: \"foo\"},\n\t\t\t\t\t{ID: \"yyyy\", Type: \"bar\"},\n\t\t\t\t\t{ID: \"zzzz\", Type: \"baz\"},\n\t\t\t\t},\n\t\t\t\ttaskTableRowIdx: 2,\n\t\t\t\ttaskID: \"yyyy\", // presence of this field opens the modal\n\t\t\t},\n\t\t},\n\t\t// TODO: Add more tests\n\t}\n\n\tfor _, tc := range tests {\n\t\tt.Run(tc.desc, func(t *testing.T) {\n\t\t\th := makeKeyEventHandler(t, tc.state)\n\t\t\tfor _, e := range tc.events {\n\t\t\t\th.HandleKeyEvent(e)\n\t\t\t}\n\t\t\tif diff := cmp.Diff(tc.wantState, *tc.state, cmp.AllowUnexported(State{})); diff != \"\" {\n\t\t\t\tt.Errorf(\"after state was %+v, want %+v: (-want,+got)\\n%s\", *tc.state, tc.wantState, diff)\n\t\t\t}\n\t\t})\n\t}\n\n}\n\n/*** fake implementation for tests ***/\n\ntype fakeFetcher struct{}\n\nfunc (f *fakeFetcher) Fetch(s *State) {}\n\ntype fakeDrawer struct{}\n\nfunc (d *fakeDrawer) Draw(s *State) {}\n" }, { "path": "tools/asynq/cmd/dash/screen_drawer.go", "content": "// Copyright 2022 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage dash\n\nimport (\n\t\"strings\"\n\n\t\"github.com/gdamore/tcell/v2\"\n\t\"github.com/mattn/go-runewidth\"\n)\n\n/*** Screen Drawer ***/\n\n// ScreenDrawer is used to draw contents on screen.\n//\n// Usage example:\n//\n//\td := NewScreenDrawer(s)\n//\td.Println(\"Hello world\", mystyle)\n//\td.NL() // adds newline\n//\td.Print(\"foo\", mystyle.Bold(true))\n//\td.Print(\"bar\", mystyle.Italic(true))\ntype ScreenDrawer struct {\n\tl *LineDrawer\n}\n\nfunc NewScreenDrawer(s tcell.Screen) *ScreenDrawer {\n\treturn &ScreenDrawer{l: NewLineDrawer(0, s)}\n}\n\nfunc (d *ScreenDrawer) Print(s string, style tcell.Style) {\n\td.l.Draw(s, style)\n}\n\nfunc (d *ScreenDrawer) Println(s string, style tcell.Style) {\n\td.Print(s, style)\n\td.NL()\n}\n\n// FillLine prints the given rune until the end of the current line\n// and adds a newline.\nfunc (d *ScreenDrawer) FillLine(r rune, style tcell.Style) {\n\tw, _ := d.Screen().Size()\n\tif w-d.l.col < 0 {\n\t\td.NL()\n\t\treturn\n\t}\n\ts := strings.Repeat(string(r), w-d.l.col)\n\td.Print(s, style)\n\td.NL()\n}\n\nfunc (d *ScreenDrawer) FillUntil(r rune, style tcell.Style, limit int) {\n\tif d.l.col > limit {\n\t\treturn // already passed the limit\n\t}\n\ts := strings.Repeat(string(r), limit-d.l.col)\n\td.Print(s, style)\n}\n\n// NL adds a newline (i.e., moves to the next line).\nfunc (d *ScreenDrawer) NL() {\n\td.l.row++\n\td.l.col = 0\n}\n\nfunc (d *ScreenDrawer) Screen() tcell.Screen {\n\treturn d.l.s\n}\n\n// Goto moves the screendrawer to the specified cell.\nfunc (d *ScreenDrawer) Goto(x, y int) {\n\td.l.row = y\n\td.l.col = x\n}\n\n// Go to the bottom of the screen.\nfunc (d *ScreenDrawer) GoToBottom() {\n\t_, h := d.Screen().Size()\n\td.l.row = h - 1\n\td.l.col = 0\n}\n\ntype LineDrawer struct {\n\ts tcell.Screen\n\trow int\n\tcol int\n}\n\nfunc NewLineDrawer(row int, s tcell.Screen) *LineDrawer {\n\treturn &LineDrawer{row: row, col: 0, s: s}\n}\n\nfunc (d *LineDrawer) Draw(s string, style tcell.Style) {\n\tfor _, r := range s {\n\t\td.s.SetContent(d.col, d.row, r, nil, style)\n\t\td.col += runewidth.RuneWidth(r)\n\t}\n}\n" }, { "path": "tools/asynq/cmd/dash/table.go", "content": "// Copyright 2022 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage dash\n\nimport (\n\t\"github.com/gdamore/tcell/v2\"\n\t\"github.com/mattn/go-runewidth\"\n)\n\ntype columnAlignment int\n\nconst (\n\talignRight columnAlignment = iota\n\talignLeft\n)\n\ntype columnConfig[V any] struct {\n\tname string\n\talignment columnAlignment\n\tdisplayFn func(v V) string\n}\n\ntype column[V any] struct {\n\t*columnConfig[V]\n\twidth int\n}\n\n// Helper to draw a table.\nfunc drawTable[V any](d *ScreenDrawer, style tcell.Style, configs []*columnConfig[V], data []V, highlightRowIdx int) {\n\tconst colBuffer = \" \" // extra buffer between columns\n\tcols := make([]*column[V], len(configs))\n\tfor i, cfg := range configs {\n\t\tcols[i] = &column[V]{cfg, runewidth.StringWidth(cfg.name)}\n\t}\n\t// adjust the column width to accommodate the widest value.\n\tfor _, v := range data {\n\t\tfor _, col := range cols {\n\t\t\tif w := runewidth.StringWidth(col.displayFn(v)); col.width < w {\n\t\t\t\tcol.width = w\n\t\t\t}\n\t\t}\n\t}\n\t// print header\n\theaderStyle := style.Background(tcell.ColorDimGray).Foreground(tcell.ColorWhite)\n\tfor _, col := range cols {\n\t\tif col.alignment == alignLeft {\n\t\t\td.Print(rpad(col.name, col.width)+colBuffer, headerStyle)\n\t\t} else {\n\t\t\td.Print(lpad(col.name, col.width)+colBuffer, headerStyle)\n\t\t}\n\t}\n\td.FillLine(' ', headerStyle)\n\t// print body\n\tfor i, v := range data {\n\t\trowStyle := style\n\t\tif highlightRowIdx == i {\n\t\t\trowStyle = style.Background(tcell.ColorDarkOliveGreen)\n\t\t}\n\t\tfor _, col := range cols {\n\t\t\tif col.alignment == alignLeft {\n\t\t\t\td.Print(rpad(col.displayFn(v), col.width)+colBuffer, rowStyle)\n\t\t\t} else {\n\t\t\t\td.Print(lpad(col.displayFn(v), col.width)+colBuffer, rowStyle)\n\t\t\t}\n\t\t}\n\t\td.FillLine(' ', rowStyle)\n\t}\n}\n" }, { "path": "tools/asynq/cmd/dash.go", "content": "// Copyright 2022 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"time\"\n\n\t\"github.com/MakeNowJust/heredoc/v2\"\n\t\"github.com/hibiken/asynq/tools/asynq/cmd/dash\"\n\t\"github.com/spf13/cobra\"\n)\n\nvar (\n\tflagPollInterval = 8 * time.Second\n)\n\nfunc init() {\n\trootCmd.AddCommand(dashCmd)\n\tdashCmd.Flags().DurationVar(&flagPollInterval, \"refresh\", 8*time.Second, \"Interval between data refresh (default: 8s, min allowed: 1s)\")\n}\n\nvar dashCmd = &cobra.Command{\n\tUse: \"dash\",\n\tShort: \"View dashboard\",\n\tLong: heredoc.Doc(`\n\t\tDisplay interactive dashboard.`),\n\tArgs: cobra.NoArgs,\n\tExample: heredoc.Doc(`\n $ asynq dash\n $ asynq dash --refresh=3s`),\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\tif flagPollInterval < 1*time.Second {\n\t\t\tfmt.Println(\"error: --refresh cannot be less than 1s\")\n\t\t\tos.Exit(1)\n\t\t}\n\t\tdash.Run(dash.Options{\n\t\t\tPollInterval: flagPollInterval,\n\t\t\tRedisConnOpt: getRedisConnOpt(),\n\t\t})\n\t},\n}\n" }, { "path": "tools/asynq/cmd/group.go", "content": "// Copyright 2022 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\n\t\"github.com/MakeNowJust/heredoc/v2\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc init() {\n\trootCmd.AddCommand(groupCmd)\n\tgroupCmd.AddCommand(groupListCmd)\n\tgroupListCmd.Flags().StringP(\"queue\", \"q\", \"\", \"queue to inspect\")\n\tgroupListCmd.MarkFlagRequired(\"queue\")\n}\n\nvar groupCmd = &cobra.Command{\n\tUse: \"group [flags]\",\n\tShort: \"Manage groups\",\n\tExample: heredoc.Doc(`\n\t\t$ asynq group list --queue=myqueue`),\n}\n\nvar groupListCmd = &cobra.Command{\n\tUse: \"list\",\n\tAliases: []string{\"ls\"},\n\tShort: \"List groups\",\n\tArgs: cobra.NoArgs,\n\tRun: groupLists,\n}\n\nfunc groupLists(cmd *cobra.Command, args []string) {\n\tqname, err := cmd.Flags().GetString(\"queue\")\n\tif err != nil {\n\t\tfmt.Println(err)\n\t\tos.Exit(1)\n\t}\n\tinspector := createInspector()\n\tgroups, err := inspector.Groups(qname)\n\tif len(groups) == 0 {\n\t\tfmt.Printf(\"No groups found in queue %q\\n\", qname)\n\t\treturn\n\t}\n\tfor _, g := range groups {\n\t\tfmt.Println(g.Group)\n\t}\n}\n" }, { "path": "tools/asynq/cmd/queue.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\n\t\"github.com/MakeNowJust/heredoc/v2\"\n\t\"github.com/fatih/color\"\n\t\"github.com/hibiken/asynq\"\n\t\"github.com/hibiken/asynq/internal/errors\"\n\t\"github.com/spf13/cobra\"\n)\n\nconst separator = \"=================================================\"\n\nfunc init() {\n\trootCmd.AddCommand(queueCmd)\n\tqueueCmd.AddCommand(queueListCmd)\n\tqueueCmd.AddCommand(queueInspectCmd)\n\tqueueCmd.AddCommand(queueHistoryCmd)\n\tqueueHistoryCmd.Flags().IntP(\"days\", \"x\", 10, \"show data from last x days\")\n\n\tqueueCmd.AddCommand(queuePauseCmd)\n\tqueueCmd.AddCommand(queueUnpauseCmd)\n\tqueueCmd.AddCommand(queueRemoveCmd)\n\tqueueRemoveCmd.Flags().BoolP(\"force\", \"f\", false, \"remove the queue regardless of its size\")\n}\n\nvar queueCmd = &cobra.Command{\n\tUse: \"queue [flags]\",\n\tShort: \"Manage queues\",\n\tExample: heredoc.Doc(`\n\t $ asynq queue ls\n\t $ asynq queue inspect myqueue\n\t $ asynq queue pause myqueue`),\n}\n\nvar queueListCmd = &cobra.Command{\n\tUse: \"list\",\n\tShort: \"List queues\",\n\tAliases: []string{\"ls\"},\n\t// TODO: Use RunE instead?\n\tRun: queueList,\n}\n\nvar queueInspectCmd = &cobra.Command{\n\tUse: \"inspect [...]\",\n\tShort: \"Display detailed information on one or more queues\",\n\tArgs: cobra.MinimumNArgs(1),\n\t// TODO: Use RunE instead?\n\tRun: queueInspect,\n\tExample: heredoc.Doc(`\n\t\t$ asynq queue inspect myqueue\n\t\t$ asynq queue inspect queue1 queue2 queue3`),\n}\n\nvar queueHistoryCmd = &cobra.Command{\n\tUse: \"history [...]\",\n\tShort: \"Display historical aggregate data from one or more queues\",\n\tArgs: cobra.MinimumNArgs(1),\n\tRun: queueHistory,\n\tExample: heredoc.Doc(`\n\t\t$ asynq queue history myqueue\n\t\t$ asynq queue history queue1 queue2 queue3\n\t\t$ asynq queue history myqueue --days=90`),\n}\n\nvar queuePauseCmd = &cobra.Command{\n\tUse: \"pause [...]\",\n\tShort: \"Pause one or more queues\",\n\tArgs: cobra.MinimumNArgs(1),\n\tRun: queuePause,\n\tExample: heredoc.Doc(`\n\t\t$ asynq queue pause myqueue\n\t\t$ asynq queue pause queue1 queue2 queue3`),\n}\n\nvar queueUnpauseCmd = &cobra.Command{\n\tUse: \"resume [...]\",\n\tShort: \"Resume (unpause) one or more queues\",\n\tArgs: cobra.MinimumNArgs(1),\n\tAliases: []string{\"unpause\"},\n\tRun: queueUnpause,\n\tExample: heredoc.Doc(`\n\t\t$ asynq queue resume myqueue\n\t\t$ asynq queue resume queue1 queue2 queue3`),\n}\n\nvar queueRemoveCmd = &cobra.Command{\n\tUse: \"remove [...]\",\n\tShort: \"Remove one or more queues\",\n\tAliases: []string{\"rm\", \"delete\"},\n\tArgs: cobra.MinimumNArgs(1),\n\tRun: queueRemove,\n\tExample: heredoc.Doc(`\n\t\t$ asynq queue rm myqueue\n\t\t$ asynq queue rm queue1 queue2 queue3\n\t\t$ asynq queue rm myqueue --force`),\n}\n\nfunc queueList(cmd *cobra.Command, args []string) {\n\ttype queueInfo struct {\n\t\tname string\n\t\tkeyslot int64\n\t\tnodes []*asynq.ClusterNode\n\t}\n\tinspector := createInspector()\n\tqueues, err := inspector.Queues()\n\tif err != nil {\n\t\tfmt.Printf(\"error: Could not fetch list of queues: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\tvar qs []*queueInfo\n\tfor _, qname := range queues {\n\t\tq := queueInfo{name: qname}\n\t\tif useRedisCluster {\n\t\t\tkeyslot, err := inspector.ClusterKeySlot(qname)\n\t\t\tif err != nil {\n\t\t\t\tfmt.Errorf(\"error: Could not get cluster keyslot for %q\\n\", qname)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tq.keyslot = keyslot\n\t\t\tnodes, err := inspector.ClusterNodes(qname)\n\t\t\tif err != nil {\n\t\t\t\tfmt.Errorf(\"error: Could not get cluster nodes for %q\\n\", qname)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tq.nodes = nodes\n\t\t}\n\t\tqs = append(qs, &q)\n\t}\n\tif useRedisCluster {\n\t\tprintTable(\n\t\t\t[]string{\"Queue\", \"Cluster KeySlot\", \"Cluster Nodes\"},\n\t\t\tfunc(w io.Writer, tmpl string) {\n\t\t\t\tfor _, q := range qs {\n\t\t\t\t\tfmt.Fprintf(w, tmpl, q.name, q.keyslot, q.nodes)\n\t\t\t\t}\n\t\t\t},\n\t\t)\n\t} else {\n\t\tfor _, q := range qs {\n\t\t\tfmt.Println(q.name)\n\t\t}\n\t}\n}\n\nfunc queueInspect(cmd *cobra.Command, args []string) {\n\tinspector := createInspector()\n\tfor i, qname := range args {\n\t\tif i > 0 {\n\t\t\tfmt.Printf(\"\\n%s\\n\\n\", separator)\n\t\t}\n\t\tinfo, err := inspector.GetQueueInfo(qname)\n\t\tif err != nil {\n\t\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\t\tcontinue\n\t\t}\n\t\tprintQueueInfo(info)\n\t}\n}\n\nfunc printQueueInfo(info *asynq.QueueInfo) {\n\tbold := color.New(color.Bold)\n\tbold.Println(\"Queue Info\")\n\tfmt.Printf(\"Name: %s\\n\", info.Queue)\n\tfmt.Printf(\"Size: %d\\n\", info.Size)\n\tfmt.Printf(\"Groups: %d\\n\", info.Groups)\n\tfmt.Printf(\"Paused: %t\\n\\n\", info.Paused)\n\tbold.Println(\"Task Count by State\")\n\tprintTable(\n\t\t[]string{\"active\", \"pending\", \"aggregating\", \"scheduled\", \"retry\", \"archived\", \"completed\"},\n\t\tfunc(w io.Writer, tmpl string) {\n\t\t\tfmt.Fprintf(w, tmpl, info.Active, info.Pending, info.Aggregating, info.Scheduled, info.Retry, info.Archived, info.Completed)\n\t\t},\n\t)\n\tfmt.Println()\n\tbold.Printf(\"Daily Stats %s UTC\\n\", info.Timestamp.UTC().Format(\"2006-01-02\"))\n\tprintTable(\n\t\t[]string{\"processed\", \"failed\", \"error rate\"},\n\t\tfunc(w io.Writer, tmpl string) {\n\t\t\tvar errRate string\n\t\t\tif info.Processed == 0 {\n\t\t\t\terrRate = \"N/A\"\n\t\t\t} else {\n\t\t\t\terrRate = fmt.Sprintf(\"%.2f%%\", float64(info.Failed)/float64(info.Processed)*100)\n\t\t\t}\n\t\t\tfmt.Fprintf(w, tmpl, info.Processed, info.Failed, errRate)\n\t\t},\n\t)\n}\n\nfunc queueHistory(cmd *cobra.Command, args []string) {\n\tdays, err := cmd.Flags().GetInt(\"days\")\n\tif err != nil {\n\t\tfmt.Printf(\"error: Internal error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\tinspector := createInspector()\n\tfor i, qname := range args {\n\t\tif i > 0 {\n\t\t\tfmt.Printf(\"\\n%s\\n\\n\", separator)\n\t\t}\n\t\tfmt.Printf(\"Queue: %s\\n\\n\", qname)\n\t\tstats, err := inspector.History(qname, days)\n\t\tif err != nil {\n\t\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\t\tcontinue\n\t\t}\n\t\tprintDailyStats(stats)\n\t}\n}\n\nfunc printDailyStats(stats []*asynq.DailyStats) {\n\tprintTable(\n\t\t[]string{\"date (UTC)\", \"processed\", \"failed\", \"error rate\"},\n\t\tfunc(w io.Writer, tmpl string) {\n\t\t\tfor _, s := range stats {\n\t\t\t\tvar errRate string\n\t\t\t\tif s.Processed == 0 {\n\t\t\t\t\terrRate = \"N/A\"\n\t\t\t\t} else {\n\t\t\t\t\terrRate = fmt.Sprintf(\"%.2f%%\", float64(s.Failed)/float64(s.Processed)*100)\n\t\t\t\t}\n\t\t\t\tfmt.Fprintf(w, tmpl, s.Date.Format(\"2006-01-02\"), s.Processed, s.Failed, errRate)\n\t\t\t}\n\t\t},\n\t)\n}\n\nfunc queuePause(cmd *cobra.Command, args []string) {\n\tinspector := createInspector()\n\tfor _, qname := range args {\n\t\terr := inspector.PauseQueue(qname)\n\t\tif err != nil {\n\t\t\tfmt.Println(err)\n\t\t\tcontinue\n\t\t}\n\t\tfmt.Printf(\"Successfully paused queue %q\\n\", qname)\n\t}\n}\n\nfunc queueUnpause(cmd *cobra.Command, args []string) {\n\tinspector := createInspector()\n\tfor _, qname := range args {\n\t\terr := inspector.UnpauseQueue(qname)\n\t\tif err != nil {\n\t\t\tfmt.Println(err)\n\t\t\tcontinue\n\t\t}\n\t\tfmt.Printf(\"Successfully unpaused queue %q\\n\", qname)\n\t}\n}\n\nfunc queueRemove(cmd *cobra.Command, args []string) {\n\t// TODO: Use inspector once RemoveQueue become public API.\n\tforce, err := cmd.Flags().GetBool(\"force\")\n\tif err != nil {\n\t\tfmt.Printf(\"error: Internal error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\n\tr := createRDB()\n\tfor _, qname := range args {\n\t\terr = r.RemoveQueue(qname, force)\n\t\tif err != nil {\n\t\t\tif errors.IsQueueNotEmpty(err) {\n\t\t\t\tfmt.Printf(\"error: %v\\nIf you are sure you want to delete it, run 'asynq queue rm --force %s'\\n\", err, qname)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\t\tcontinue\n\t\t}\n\t\tfmt.Printf(\"Successfully removed queue %q\\n\", qname)\n\t}\n}\n" }, { "path": "tools/asynq/cmd/root.go", "content": "//\n// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage cmd\n\nimport (\n\t\"crypto/tls\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"strings\"\n\t\"text/tabwriter\"\n\t\"time\"\n\t\"unicode\"\n\t\"unicode/utf8\"\n\n\t\"github.com/MakeNowJust/heredoc/v2\"\n\t\"github.com/fatih/color\"\n\t\"github.com/hibiken/asynq\"\n\t\"github.com/hibiken/asynq/internal/base\"\n\t\"github.com/hibiken/asynq/internal/rdb\"\n\t\"github.com/redis/go-redis/v9\"\n\t\"github.com/spf13/cobra\"\n\t\"github.com/spf13/pflag\"\n\t\"golang.org/x/exp/utf8string\"\n\n\thomedir \"github.com/mitchellh/go-homedir\"\n\t\"github.com/spf13/viper\"\n)\n\nvar cfgFile string\n\n// Global flag variables\nvar (\n\turi string\n\tdb int\n\tpassword string\n\tusername string\n\n\tuseRedisCluster bool\n\tclusterAddrs string\n\ttlsServerName string\n\tinsecure bool\n\tuseTLS bool\n)\n\n// rootCmd represents the base command when called without any subcommands\nvar rootCmd = &cobra.Command{\n\tUse: \"asynq [flags]\",\n\tShort: \"Asynq CLI\",\n\tLong: `Command line tool to inspect tasks and queues managed by Asynq`,\n\tVersion: base.Version,\n\n\tSilenceUsage: true,\n\tSilenceErrors: true,\n\n\tExample: heredoc.Doc(`\n\t\t$ asynq stats\n\t\t$ asynq queue pause myqueue\n\t\t$ asynq task list --queue=myqueue --state=archived`),\n\tAnnotations: map[string]string{\n\t\t\"help:feedback\": heredoc.Doc(`\n\t\t\tOpen an issue at https://github.com/hibiken/asynq/issues/new/choose`),\n\t},\n}\n\nvar versionOutput = fmt.Sprintf(\"asynq version %s\\n\", base.Version)\n\nvar versionCmd = &cobra.Command{\n\tUse: \"version\",\n\tHidden: true,\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\tfmt.Print(versionOutput)\n\t},\n}\n\n// Execute adds all child commands to the root command and sets flags appropriately.\n// This is called by main.main(). It only needs to happen once to the rootCmd.\nfunc Execute() {\n\tif err := rootCmd.Execute(); err != nil {\n\t\tfmt.Println(err)\n\t\tos.Exit(1)\n\t}\n}\n\nfunc isRootCmd(cmd *cobra.Command) bool {\n\treturn cmd != nil && !cmd.HasParent()\n}\n\n// displayLine represents a line displayed in the output as ' ',\n// where pad is used to pad the name from desc.\ntype displayLine struct {\n\tname string\n\tdesc string\n\tpad int // number of rpad\n}\n\nfunc (l *displayLine) String() string {\n\treturn rpad(l.name, l.pad) + l.desc\n}\n\ntype displayLines []*displayLine\n\nfunc (dls displayLines) String() string {\n\tvar lines []string\n\tfor _, dl := range dls {\n\t\tlines = append(lines, dl.String())\n\t}\n\treturn strings.Join(lines, \"\\n\")\n}\n\n// Capitalize the first word in the given string.\nfunc capitalize(s string) string {\n\tstr := utf8string.NewString(s)\n\tif str.RuneCount() == 0 {\n\t\treturn \"\"\n\t}\n\tvar b strings.Builder\n\tb.WriteString(strings.ToUpper(string(str.At(0))))\n\tb.WriteString(str.Slice(1, str.RuneCount()))\n\treturn b.String()\n}\n\nfunc rootHelpFunc(cmd *cobra.Command, args []string) {\n\t// Display helpful error message when user mistypes a subcommand (e.g. 'asynq queue lst').\n\tif isRootCmd(cmd.Parent()) && len(args) >= 2 && args[1] != \"--help\" && args[1] != \"-h\" {\n\t\tprintSubcommandSuggestions(cmd, args[1])\n\t\treturn\n\t}\n\n\tvar lines []*displayLine\n\tvar commands []*displayLine\n\tfor _, c := range cmd.Commands() {\n\t\tif c.Hidden || c.Short == \"\" || c.Name() == \"help\" {\n\t\t\tcontinue\n\t\t}\n\t\tl := &displayLine{name: c.Name() + \":\", desc: capitalize(c.Short)}\n\t\tcommands = append(commands, l)\n\t\tlines = append(lines, l)\n\t}\n\tvar localFlags []*displayLine\n\tcmd.LocalFlags().VisitAll(func(f *pflag.Flag) {\n\t\tl := &displayLine{name: \"--\" + f.Name, desc: capitalize(f.Usage)}\n\t\tlocalFlags = append(localFlags, l)\n\t\tlines = append(lines, l)\n\t})\n\tvar inheritedFlags []*displayLine\n\tcmd.InheritedFlags().VisitAll(func(f *pflag.Flag) {\n\t\tl := &displayLine{name: \"--\" + f.Name, desc: capitalize(f.Usage)}\n\t\tinheritedFlags = append(inheritedFlags, l)\n\t\tlines = append(lines, l)\n\t})\n\tadjustPadding(lines...)\n\n\ttype helpEntry struct {\n\t\tTitle string\n\t\tBody string\n\t}\n\tvar helpEntries []*helpEntry\n\tdesc := cmd.Long\n\tif desc == \"\" {\n\t\tdesc = cmd.Short\n\t}\n\tif desc != \"\" {\n\t\thelpEntries = append(helpEntries, &helpEntry{\"\", desc})\n\t}\n\thelpEntries = append(helpEntries, &helpEntry{\"USAGE\", cmd.UseLine()})\n\tif len(commands) > 0 {\n\t\thelpEntries = append(helpEntries, &helpEntry{\"COMMANDS\", displayLines(commands).String()})\n\t}\n\tif cmd.LocalFlags().HasFlags() {\n\t\thelpEntries = append(helpEntries, &helpEntry{\"FLAGS\", displayLines(localFlags).String()})\n\t}\n\tif cmd.InheritedFlags().HasFlags() {\n\t\thelpEntries = append(helpEntries, &helpEntry{\"INHERITED FLAGS\", displayLines(inheritedFlags).String()})\n\t}\n\tif cmd.Example != \"\" {\n\t\thelpEntries = append(helpEntries, &helpEntry{\"EXAMPLES\", cmd.Example})\n\t}\n\thelpEntries = append(helpEntries, &helpEntry{\"LEARN MORE\", heredoc.Doc(`\n\t\tUse 'asynq --help' for more information about a command.`)})\n\tif s, ok := cmd.Annotations[\"help:feedback\"]; ok {\n\t\thelpEntries = append(helpEntries, &helpEntry{\"FEEDBACK\", s})\n\t}\n\n\tout := cmd.OutOrStdout()\n\tbold := color.New(color.Bold)\n\tfor _, e := range helpEntries {\n\t\tif e.Title != \"\" {\n\t\t\t// If there is a title, add indentation to each line in the body\n\t\t\tbold.Fprintln(out, e.Title)\n\t\t\tfmt.Fprintln(out, indent(e.Body, 2 /* spaces */))\n\t\t} else {\n\t\t\t// If there is no title, print the body as is\n\t\t\tfmt.Fprintln(out, e.Body)\n\t\t}\n\t\tfmt.Fprintln(out)\n\t}\n}\n\nfunc rootUsageFunc(cmd *cobra.Command) error {\n\tout := cmd.OutOrStdout()\n\tfmt.Fprintf(out, \"Usage: %s\", cmd.UseLine())\n\tif subcmds := cmd.Commands(); len(subcmds) > 0 {\n\t\tfmt.Fprint(out, \"\\n\\nAvailable commands:\\n\")\n\t\tfor _, c := range subcmds {\n\t\t\tif c.Hidden {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tfmt.Fprintf(out, \" %s\\n\", c.Name())\n\t\t}\n\t}\n\n\tvar localFlags []*displayLine\n\tcmd.LocalFlags().VisitAll(func(f *pflag.Flag) {\n\t\tlocalFlags = append(localFlags, &displayLine{name: \"--\" + f.Name, desc: capitalize(f.Usage)})\n\t})\n\tadjustPadding(localFlags...)\n\tif len(localFlags) > 0 {\n\t\tfmt.Fprint(out, \"\\n\\nFlags:\\n\")\n\t\tfor _, l := range localFlags {\n\t\t\tfmt.Fprintf(out, \" %s\\n\", l.String())\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc printSubcommandSuggestions(cmd *cobra.Command, arg string) {\n\tout := cmd.OutOrStdout()\n\tfmt.Fprintf(out, \"unknown command %q for %q\\n\", arg, cmd.CommandPath())\n\tif cmd.SuggestionsMinimumDistance <= 0 {\n\t\tcmd.SuggestionsMinimumDistance = 2\n\t}\n\tcandidates := cmd.SuggestionsFor(arg)\n\tif len(candidates) > 0 {\n\t\tfmt.Fprint(out, \"\\nDid you mean this?\\n\")\n\t\tfor _, c := range candidates {\n\t\t\tfmt.Fprintf(out, \"\\t%s\\n\", c)\n\t\t}\n\t}\n\tfmt.Fprintln(out)\n\trootUsageFunc(cmd)\n}\n\nfunc adjustPadding(lines ...*displayLine) {\n\t// find the maximum width of the name\n\tmax := 0\n\tfor _, l := range lines {\n\t\tif n := utf8.RuneCountInString(l.name); n > max {\n\t\t\tmax = n\n\t\t}\n\t}\n\tfor _, l := range lines {\n\t\tl.pad = max\n\t}\n}\n\n// rpad adds padding to the right of a string.\nfunc rpad(s string, padding int) string {\n\ttmpl := fmt.Sprintf(\"%%-%ds \", padding)\n\treturn fmt.Sprintf(tmpl, s)\n}\n\n// lpad adds padding to the left of a string.\nfunc lpad(s string, padding int) string {\n\ttmpl := fmt.Sprintf(\"%%%ds \", padding)\n\treturn fmt.Sprintf(tmpl, s)\n}\n\n// indent indents the given text by given spaces.\nfunc indent(text string, space int) string {\n\tif len(text) == 0 {\n\t\treturn \"\"\n\t}\n\tvar b strings.Builder\n\tindentation := strings.Repeat(\" \", space)\n\tlastRune := '\\n'\n\tfor _, r := range text {\n\t\tif lastRune == '\\n' {\n\t\t\tb.WriteString(indentation)\n\t\t}\n\t\tb.WriteRune(r)\n\t\tlastRune = r\n\t}\n\treturn b.String()\n}\n\n// dedent removes any indentation from the given text.\nfunc dedent(text string) string {\n\tlines := strings.Split(text, \"\\n\")\n\tvar b strings.Builder\n\tfor _, l := range lines {\n\t\tb.WriteString(strings.TrimLeftFunc(l, unicode.IsSpace))\n\t\tb.WriteRune('\\n')\n\t}\n\treturn b.String()\n}\n\nfunc init() {\n\tcobra.OnInitialize(initConfig)\n\n\trootCmd.SetHelpFunc(rootHelpFunc)\n\trootCmd.SetUsageFunc(rootUsageFunc)\n\n\trootCmd.AddCommand(versionCmd)\n\trootCmd.SetVersionTemplate(versionOutput)\n\n\trootCmd.PersistentFlags().StringVar(&cfgFile, \"config\", \"\", \"Config file to set flag defaut values (default is $HOME/.asynq.yaml)\")\n\trootCmd.PersistentFlags().StringVarP(&uri, \"uri\", \"u\", \"127.0.0.1:6379\", \"Redis server URI\")\n\trootCmd.PersistentFlags().IntVarP(&db, \"db\", \"n\", 0, \"Redis database number (default is 0)\")\n\trootCmd.PersistentFlags().StringVarP(&password, \"password\", \"p\", \"\", \"Password to use when connecting to redis server\")\n\trootCmd.PersistentFlags().StringVarP(&username, \"username\", \"U\", \"\", \"Username to use when connecting to Redis (ACL username)\")\n\trootCmd.PersistentFlags().BoolVar(&useRedisCluster, \"cluster\", false, \"Connect to redis cluster\")\n\trootCmd.PersistentFlags().StringVar(&clusterAddrs, \"cluster_addrs\",\n\t\t\"127.0.0.1:7000,127.0.0.1:7001,127.0.0.1:7002,127.0.0.1:7003,127.0.0.1:7004,127.0.0.1:7005\",\n\t\t\"List of comma-separated redis server addresses\")\n\trootCmd.PersistentFlags().BoolVar(&useTLS, \"tls\", false, \"Enable TLS connection\")\n\trootCmd.PersistentFlags().StringVar(&tlsServerName, \"tls_server\",\n\t\t\"\", \"Server name for TLS validation\")\n\trootCmd.PersistentFlags().BoolVar(&insecure, \"insecure\",\n\t\tfalse, \"Allow insecure TLS connection by skipping cert validation\")\n\t// Bind flags with config.\n\tviper.BindPFlag(\"uri\", rootCmd.PersistentFlags().Lookup(\"uri\"))\n\tviper.BindPFlag(\"db\", rootCmd.PersistentFlags().Lookup(\"db\"))\n\tviper.BindPFlag(\"password\", rootCmd.PersistentFlags().Lookup(\"password\"))\n\tviper.BindPFlag(\"username\", rootCmd.PersistentFlags().Lookup(\"username\"))\n\tviper.BindPFlag(\"cluster\", rootCmd.PersistentFlags().Lookup(\"cluster\"))\n\tviper.BindPFlag(\"cluster_addrs\", rootCmd.PersistentFlags().Lookup(\"cluster_addrs\"))\n\tviper.BindPFlag(\"tls\", rootCmd.PersistentFlags().Lookup(\"tls\"))\n\tviper.BindPFlag(\"tls_server\", rootCmd.PersistentFlags().Lookup(\"tls_server\"))\n\tviper.BindPFlag(\"insecure\", rootCmd.PersistentFlags().Lookup(\"insecure\"))\n}\n\n// initConfig reads in config file and ENV variables if set.\nfunc initConfig() {\n\tif cfgFile != \"\" {\n\t\t// Use config file from the flag.\n\t\tviper.SetConfigFile(cfgFile)\n\t} else {\n\t\t// Find home directory.\n\t\thome, err := homedir.Dir()\n\t\tif err != nil {\n\t\t\tfmt.Println(err)\n\t\t\tos.Exit(1)\n\t\t}\n\n\t\t// Search config in home directory with name \".asynq\" (without extension).\n\t\tviper.AddConfigPath(home)\n\t\tviper.SetConfigName(\".asynq\")\n\t}\n\n\tviper.AutomaticEnv() // read in environment variables that match\n\n\t// If a config file is found, read it in.\n\tif err := viper.ReadInConfig(); err == nil {\n\t\tfmt.Println(\"Using config file:\", viper.ConfigFileUsed())\n\t}\n}\n\n// createRDB creates a RDB instance using flag values and returns it.\nfunc createRDB() *rdb.RDB {\n\tvar c redis.UniversalClient\n\tif viper.GetBool(\"cluster\") {\n\t\taddrs := strings.Split(viper.GetString(\"cluster_addrs\"), \",\")\n\t\tc = redis.NewClusterClient(&redis.ClusterOptions{\n\t\t\tAddrs: addrs,\n\t\t\tPassword: viper.GetString(\"password\"),\n\t\t\tUsername: viper.GetString(\"username\"),\n\t\t\tTLSConfig: getTLSConfig(),\n\t\t})\n\t} else {\n\t\tc = redis.NewClient(&redis.Options{\n\t\t\tAddr: viper.GetString(\"uri\"),\n\t\t\tDB: viper.GetInt(\"db\"),\n\t\t\tPassword: viper.GetString(\"password\"),\n\t\t\tUsername: viper.GetString(\"username\"),\n\t\t\tTLSConfig: getTLSConfig(),\n\t\t})\n\t}\n\treturn rdb.NewRDB(c)\n}\n\n// createClient creates a Client instance using flag values and returns it.\nfunc createClient() *asynq.Client {\n\treturn asynq.NewClient(getRedisConnOpt())\n}\n\n// createInspector creates a Inspector instance using flag values and returns it.\nfunc createInspector() *asynq.Inspector {\n\treturn asynq.NewInspector(getRedisConnOpt())\n}\n\nfunc getRedisConnOpt() asynq.RedisConnOpt {\n\tif viper.GetBool(\"cluster\") {\n\t\taddrs := strings.Split(viper.GetString(\"cluster_addrs\"), \",\")\n\t\treturn asynq.RedisClusterClientOpt{\n\t\t\tAddrs: addrs,\n\t\t\tPassword: viper.GetString(\"password\"),\n\t\t\tUsername: viper.GetString(\"username\"),\n\t\t\tTLSConfig: getTLSConfig(),\n\t\t}\n\t}\n\treturn asynq.RedisClientOpt{\n\t\tAddr: viper.GetString(\"uri\"),\n\t\tDB: viper.GetInt(\"db\"),\n\t\tPassword: viper.GetString(\"password\"),\n\t\tUsername: viper.GetString(\"username\"),\n\t\tTLSConfig: getTLSConfig(),\n\t}\n}\n\nfunc getTLSConfig() *tls.Config {\n\ttlsServer := viper.GetString(\"tls_server\")\n\tif tlsServer != \"\" {\n\t\treturn &tls.Config{ServerName: tlsServer, InsecureSkipVerify: viper.GetBool(\"insecure\")}\n\t}\n\n\tif viper.GetBool(\"tls\") {\n\t\treturn &tls.Config{InsecureSkipVerify: viper.GetBool(\"insecure\")}\n\t}\n\n\treturn nil\n}\n\n// printTable is a helper function to print data in table format.\n//\n// cols is a list of headers and printRow specifies how to print rows.\n//\n// Example:\n//\n//\ttype User struct {\n//\t Name string\n//\t Addr string\n//\t Age int\n//\t}\n//\n// data := []*User{{\"user1\", \"addr1\", 24}, {\"user2\", \"addr2\", 42}, ...}\n// cols := []string{\"Name\", \"Addr\", \"Age\"}\n//\n//\tprintRows := func(w io.Writer, tmpl string) {\n//\t for _, u := range data {\n//\t fmt.Fprintf(w, tmpl, u.Name, u.Addr, u.Age)\n//\t }\n//\t}\n//\n// printTable(cols, printRows)\nfunc printTable(cols []string, printRows func(w io.Writer, tmpl string)) {\n\tformat := strings.Repeat(\"%v\\t\", len(cols)) + \"\\n\"\n\ttw := new(tabwriter.Writer).Init(os.Stdout, 0, 8, 2, ' ', 0)\n\tvar headers []interface{}\n\tvar seps []interface{}\n\tfor _, name := range cols {\n\t\theaders = append(headers, name)\n\t\tseps = append(seps, strings.Repeat(\"-\", len(name)))\n\t}\n\tfmt.Fprintf(tw, format, headers...)\n\tfmt.Fprintf(tw, format, seps...)\n\tprintRows(tw, format)\n\ttw.Flush()\n}\n\n// sprintBytes returns a string representation of the given byte slice if data is printable.\n// If data is not printable, it returns a string describing it is not printable.\nfunc sprintBytes(payload []byte) string {\n\tif !isPrintable(payload) {\n\t\treturn \"non-printable bytes\"\n\t}\n\treturn string(payload)\n}\n\nfunc isPrintable(data []byte) bool {\n\tif !utf8.Valid(data) {\n\t\treturn false\n\t}\n\tisAllSpace := true\n\tfor _, r := range string(data) {\n\t\tif !unicode.IsPrint(r) {\n\t\t\treturn false\n\t\t}\n\t\tif !unicode.IsSpace(r) {\n\t\t\tisAllSpace = false\n\t\t}\n\t}\n\treturn !isAllSpace\n}\n\n// Helper to turn a command line flag into a duration\nfunc getDuration(cmd *cobra.Command, arg string) time.Duration {\n\tdurationStr, err := cmd.Flags().GetString(arg)\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\n\tduration, err := time.ParseDuration(durationStr)\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\n\treturn duration\n}\n\n// Helper to turn a command line flag into a time\nfunc getTime(cmd *cobra.Command, arg string) time.Time {\n\ttimeStr, err := cmd.Flags().GetString(arg)\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\n\ttimeVal, err := time.Parse(time.RFC3339, timeStr)\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\n\treturn timeVal\n}\n" }, { "path": "tools/asynq/cmd/server.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"sort\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/MakeNowJust/heredoc/v2\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc init() {\n\trootCmd.AddCommand(serverCmd)\n\tserverCmd.AddCommand(serverListCmd)\n}\n\nvar serverCmd = &cobra.Command{\n\tUse: \"server [flags]\",\n\tShort: \"Manage servers\",\n\tExample: heredoc.Doc(`\n\t\t$ asynq server list`),\n}\n\nvar serverListCmd = &cobra.Command{\n\tUse: \"list\",\n\tAliases: []string{\"ls\"},\n\tShort: \"List servers\",\n\tLong: `Server list (asynq server ls) shows all running worker servers\npulling tasks from the given redis instance.\n\nThe command shows the following for each server:\n* Host and PID of the process in which the server is running\n* Number of active workers out of worker pool\n* Queue configuration\n* State of the worker server (\"active\" | \"stopped\")\n* Time the server was started\n\nA \"active\" server is pulling tasks from queues and processing them.\nA \"stopped\" server is no longer pulling new tasks from queues`,\n\tRun: serverList,\n}\n\nfunc serverList(cmd *cobra.Command, args []string) {\n\tr := createRDB()\n\n\tservers, err := r.ListServers()\n\tif err != nil {\n\t\tfmt.Println(err)\n\t\tos.Exit(1)\n\t}\n\tif len(servers) == 0 {\n\t\tfmt.Println(\"No running servers\")\n\t\treturn\n\t}\n\n\t// sort by hostname and pid\n\tsort.Slice(servers, func(i, j int) bool {\n\t\tx, y := servers[i], servers[j]\n\t\tif x.Host != y.Host {\n\t\t\treturn x.Host < y.Host\n\t\t}\n\t\treturn x.PID < y.PID\n\t})\n\n\t// print server info\n\tcols := []string{\"Host\", \"PID\", \"State\", \"Active Workers\", \"Queues\", \"Started\"}\n\tprintRows := func(w io.Writer, tmpl string) {\n\t\tfor _, info := range servers {\n\t\t\tfmt.Fprintf(w, tmpl,\n\t\t\t\tinfo.Host, info.PID, info.Status,\n\t\t\t\tfmt.Sprintf(\"%d/%d\", info.ActiveWorkerCount, info.Concurrency),\n\t\t\t\tformatQueues(info.Queues), timeAgo(info.Started))\n\t\t}\n\t}\n\tprintTable(cols, printRows)\n}\n\nfunc formatQueues(qmap map[string]int) string {\n\t// sort queues by priority and name\n\ttype queue struct {\n\t\tname string\n\t\tpriority int\n\t}\n\tvar queues []*queue\n\tfor qname, p := range qmap {\n\t\tqueues = append(queues, &queue{qname, p})\n\t}\n\tsort.Slice(queues, func(i, j int) bool {\n\t\tx, y := queues[i], queues[j]\n\t\tif x.priority != y.priority {\n\t\t\treturn x.priority > y.priority\n\t\t}\n\t\treturn x.name < y.name\n\t})\n\n\tvar b strings.Builder\n\tl := len(queues)\n\tfor _, q := range queues {\n\t\tfmt.Fprintf(&b, \"%s:%d\", q.name, q.priority)\n\t\tl--\n\t\tif l > 0 {\n\t\t\tb.WriteString(\" \")\n\t\t}\n\t}\n\treturn b.String()\n}\n\n// timeAgo takes a time and returns a string of the format \" ago\".\nfunc timeAgo(since time.Time) string {\n\td := time.Since(since).Round(time.Second)\n\treturn fmt.Sprintf(\"%v ago\", d)\n}\n" }, { "path": "tools/asynq/cmd/stats.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage cmd\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"os\"\n\t\"strconv\"\n\t\"strings\"\n\t\"text/tabwriter\"\n\t\"time\"\n\t\"unicode/utf8\"\n\n\t\"github.com/MakeNowJust/heredoc/v2\"\n\t\"github.com/fatih/color\"\n\t\"github.com/hibiken/asynq/internal/rdb\"\n\t\"github.com/spf13/cobra\"\n)\n\n// statsCmd represents the stats command\nvar statsCmd = &cobra.Command{\n\tUse: \"stats\",\n\tShort: \"View current state\",\n\tLong: heredoc.Doc(`\n\t Stats shows the overview of tasks and queues at that instant.\n\n\t The command shows the following:\n\t * Number of tasks in each state\n\t * Number of tasks in each queue\n\t * Aggregate data for the current day\n\t * Basic information about the running redis instance`),\n\tArgs: cobra.NoArgs,\n\tRun: stats,\n}\n\nvar jsonFlag bool\n\nfunc init() {\n\trootCmd.AddCommand(statsCmd)\n\tstatsCmd.Flags().BoolVar(&jsonFlag, \"json\", false, \"Output stats in JSON format.\")\n\n\t// Here you will define your flags and configuration settings.\n\n\t// Cobra supports Persistent Flags which will work for this command\n\t// and all subcommands, e.g.:\n\t// statsCmd.PersistentFlags().String(\"foo\", \"\", \"A help for foo\")\n\n\t// Cobra supports local flags which will only run when this command\n\t// is called directly, e.g.:\n\t// statsCmd.Flags().BoolP(\"toggle\", \"t\", false, \"Help message for toggle\")\n}\n\ntype AggregateStats struct {\n\tActive int `json:\"active\"`\n\tPending int `json:\"pending\"`\n\tAggregating int `json:\"aggregating\"`\n\tScheduled int `json:\"scheduled\"`\n\tRetry int `json:\"retry\"`\n\tArchived int `json:\"archived\"`\n\tCompleted int `json:\"completed\"`\n\tProcessed int `json:\"processed\"`\n\tFailed int `json:\"failed\"`\n\tTimestamp time.Time `json:\"timestamp\"`\n}\n\ntype FullStats struct {\n\tAggregate AggregateStats `json:\"aggregate\"`\n\tQueueStats []*rdb.Stats `json:\"queues\"`\n\tRedisInfo map[string]string `json:\"redis\"`\n}\n\nfunc stats(cmd *cobra.Command, args []string) {\n\tr := createRDB()\n\n\tqueues, err := r.AllQueues()\n\tif err != nil {\n\t\tfmt.Println(err)\n\t\tos.Exit(1)\n\t}\n\n\tvar aggStats AggregateStats\n\tvar stats []*rdb.Stats\n\tfor _, qname := range queues {\n\t\ts, err := r.CurrentStats(qname)\n\t\tif err != nil {\n\t\t\tfmt.Println(err)\n\t\t\tos.Exit(1)\n\t\t}\n\t\taggStats.Active += s.Active\n\t\taggStats.Pending += s.Pending\n\t\taggStats.Aggregating += s.Aggregating\n\t\taggStats.Scheduled += s.Scheduled\n\t\taggStats.Retry += s.Retry\n\t\taggStats.Archived += s.Archived\n\t\taggStats.Completed += s.Completed\n\t\taggStats.Processed += s.Processed\n\t\taggStats.Failed += s.Failed\n\t\taggStats.Timestamp = s.Timestamp\n\t\tstats = append(stats, s)\n\t}\n\tvar info map[string]string\n\tif useRedisCluster {\n\t\tinfo, err = r.RedisClusterInfo()\n\t} else {\n\t\tinfo, err = r.RedisInfo()\n\t}\n\tif err != nil {\n\t\tfmt.Println(err)\n\t\tos.Exit(1)\n\t}\n\n\tif jsonFlag {\n\t\tstatsJSON, err := json.Marshal(FullStats{\n\t\t\tAggregate: aggStats,\n\t\t\tQueueStats: stats,\n\t\t\tRedisInfo: info,\n\t\t})\n\n\t\tif err != nil {\n\t\t\tfmt.Println(err)\n\t\t\tos.Exit(1)\n\t\t}\n\n\t\tfmt.Println(string(statsJSON))\n\t\treturn\n\t}\n\n\tbold := color.New(color.Bold)\n\tbold.Println(\"Task Count by State\")\n\tprintStatsByState(&aggStats)\n\tfmt.Println()\n\n\tbold.Println(\"Task Count by Queue\")\n\tprintStatsByQueue(stats)\n\tfmt.Println()\n\n\tbold.Printf(\"Daily Stats %s UTC\\n\", aggStats.Timestamp.UTC().Format(\"2006-01-02\"))\n\tprintSuccessFailureStats(&aggStats)\n\tfmt.Println()\n\n\tif useRedisCluster {\n\t\tbold.Println(\"Redis Cluster Info\")\n\t\tprintClusterInfo(info)\n\t} else {\n\t\tbold.Println(\"Redis Info\")\n\t\tprintInfo(info)\n\t}\n\tfmt.Println()\n}\n\nfunc printStatsByState(s *AggregateStats) {\n\tformat := strings.Repeat(\"%v\\t\", 7) + \"\\n\"\n\ttw := new(tabwriter.Writer).Init(os.Stdout, 0, 8, 2, ' ', 0)\n\tfmt.Fprintf(tw, format, \"active\", \"pending\", \"aggregating\", \"scheduled\", \"retry\", \"archived\", \"completed\")\n\twidth := maxInt(9 /* defaultWidth */, maxWidthOf(s.Active, s.Pending, s.Aggregating, s.Scheduled, s.Retry, s.Archived, s.Completed)) // length of widest column\n\tsep := strings.Repeat(\"-\", width)\n\tfmt.Fprintf(tw, format, sep, sep, sep, sep, sep, sep, sep)\n\tfmt.Fprintf(tw, format, s.Active, s.Pending, s.Aggregating, s.Scheduled, s.Retry, s.Archived, s.Completed)\n\ttw.Flush()\n}\n\n// numDigits returns the number of digits in n.\nfunc numDigits(n int) int {\n\treturn len(strconv.Itoa(n))\n}\n\n// maxWidthOf returns the max number of digits amount the provided vals.\nfunc maxWidthOf(vals ...int) int {\n\tmax := 0\n\tfor _, v := range vals {\n\t\tif vw := numDigits(v); vw > max {\n\t\t\tmax = vw\n\t\t}\n\t}\n\treturn max\n}\n\nfunc maxInt(a, b int) int {\n\treturn int(math.Max(float64(a), float64(b)))\n}\n\nfunc printStatsByQueue(stats []*rdb.Stats) {\n\tvar headers, seps, counts []string\n\tmaxHeaderWidth := 0\n\tfor _, s := range stats {\n\t\ttitle := queueTitle(s)\n\t\theaders = append(headers, title)\n\t\tif w := utf8.RuneCountInString(title); w > maxHeaderWidth {\n\t\t\tmaxHeaderWidth = w\n\t\t}\n\t\tcounts = append(counts, strconv.Itoa(s.Size))\n\t}\n\tfor i := 0; i < len(headers); i++ {\n\t\tseps = append(seps, strings.Repeat(\"-\", maxHeaderWidth))\n\t}\n\tformat := strings.Repeat(\"%v\\t\", len(headers)) + \"\\n\"\n\ttw := new(tabwriter.Writer).Init(os.Stdout, 0, 8, 2, ' ', 0)\n\tfmt.Fprintf(tw, format, toInterfaceSlice(headers)...)\n\tfmt.Fprintf(tw, format, toInterfaceSlice(seps)...)\n\tfmt.Fprintf(tw, format, toInterfaceSlice(counts)...)\n\ttw.Flush()\n}\n\nfunc queueTitle(s *rdb.Stats) string {\n\tvar b strings.Builder\n\tb.WriteString(s.Queue)\n\tif s.Paused {\n\t\tb.WriteString(\" (paused)\")\n\t}\n\treturn b.String()\n}\n\nfunc printSuccessFailureStats(s *AggregateStats) {\n\tformat := strings.Repeat(\"%v\\t\", 3) + \"\\n\"\n\ttw := new(tabwriter.Writer).Init(os.Stdout, 0, 8, 2, ' ', 0)\n\tfmt.Fprintf(tw, format, \"processed\", \"failed\", \"error rate\")\n\tfmt.Fprintf(tw, format, \"---------\", \"------\", \"----------\")\n\tvar errrate string\n\tif s.Processed == 0 {\n\t\terrrate = \"N/A\"\n\t} else {\n\t\terrrate = fmt.Sprintf(\"%.2f%%\", float64(s.Failed)/float64(s.Processed)*100)\n\t}\n\tfmt.Fprintf(tw, format, s.Processed, s.Failed, errrate)\n\ttw.Flush()\n}\n\nfunc printInfo(info map[string]string) {\n\tformat := strings.Repeat(\"%v\\t\", 5) + \"\\n\"\n\ttw := new(tabwriter.Writer).Init(os.Stdout, 0, 8, 2, ' ', 0)\n\tfmt.Fprintf(tw, format, \"version\", \"uptime\", \"connections\", \"memory usage\", \"peak memory usage\")\n\tfmt.Fprintf(tw, format, \"-------\", \"------\", \"-----------\", \"------------\", \"-----------------\")\n\tfmt.Fprintf(tw, format,\n\t\tinfo[\"redis_version\"],\n\t\tfmt.Sprintf(\"%s days\", info[\"uptime_in_days\"]),\n\t\tinfo[\"connected_clients\"],\n\t\tfmt.Sprintf(\"%sB\", info[\"used_memory_human\"]),\n\t\tfmt.Sprintf(\"%sB\", info[\"used_memory_peak_human\"]),\n\t)\n\ttw.Flush()\n}\n\nfunc printClusterInfo(info map[string]string) {\n\tprintTable(\n\t\t[]string{\"State\", \"Known Nodes\", \"Cluster Size\"},\n\t\tfunc(w io.Writer, tmpl string) {\n\t\t\tfmt.Fprintf(w, tmpl,\n\t\t\t\tstrings.ToUpper(info[\"cluster_state\"]),\n\t\t\t\tinfo[\"cluster_known_nodes\"],\n\t\t\t\tinfo[\"cluster_size\"],\n\t\t\t)\n\t\t},\n\t)\n}\n\nfunc toInterfaceSlice(strs []string) []interface{} {\n\tvar res []interface{}\n\tfor _, s := range strs {\n\t\tres = append(res, s)\n\t}\n\treturn res\n}\n" }, { "path": "tools/asynq/cmd/task.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"time\"\n\n\t\"github.com/MakeNowJust/heredoc/v2\"\n\t\"github.com/fatih/color\"\n\t\"github.com/hibiken/asynq\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc init() {\n\trootCmd.AddCommand(taskCmd)\n\ttaskCmd.AddCommand(taskListCmd)\n\ttaskListCmd.Flags().StringP(\"queue\", \"q\", \"\", \"queue to inspect (required)\")\n\ttaskListCmd.Flags().StringP(\"state\", \"s\", \"\", \"state of the tasks; one of { active | pending | aggregating | scheduled | retry | archived | completed } (required)\")\n\ttaskListCmd.Flags().Int(\"page\", 1, \"page number\")\n\ttaskListCmd.Flags().Int(\"size\", 30, \"page size\")\n\ttaskListCmd.Flags().StringP(\"group\", \"g\", \"\", \"group to inspect (required for listing aggregating tasks)\")\n\ttaskListCmd.MarkFlagRequired(\"queue\")\n\ttaskListCmd.MarkFlagRequired(\"state\")\n\n\ttaskCmd.AddCommand(taskCancelCmd)\n\n\ttaskCmd.AddCommand(taskInspectCmd)\n\ttaskInspectCmd.Flags().StringP(\"queue\", \"q\", \"\", \"queue to which the task belongs (required)\")\n\ttaskInspectCmd.Flags().StringP(\"id\", \"i\", \"\", \"id of the task (required)\")\n\ttaskInspectCmd.MarkFlagRequired(\"queue\")\n\ttaskInspectCmd.MarkFlagRequired(\"id\")\n\n\ttaskCmd.AddCommand(taskArchiveCmd)\n\ttaskArchiveCmd.Flags().StringP(\"queue\", \"q\", \"\", \"queue to which the task belongs (required)\")\n\ttaskArchiveCmd.Flags().StringP(\"id\", \"i\", \"\", \"id of the task (required)\")\n\ttaskArchiveCmd.MarkFlagRequired(\"queue\")\n\ttaskArchiveCmd.MarkFlagRequired(\"id\")\n\n\ttaskCmd.AddCommand(taskDeleteCmd)\n\ttaskDeleteCmd.Flags().StringP(\"queue\", \"q\", \"\", \"queue to which the task belongs (required)\")\n\ttaskDeleteCmd.Flags().StringP(\"id\", \"i\", \"\", \"id of the task (required)\")\n\ttaskDeleteCmd.MarkFlagRequired(\"queue\")\n\ttaskDeleteCmd.MarkFlagRequired(\"id\")\n\n\ttaskCmd.AddCommand(taskRunCmd)\n\ttaskRunCmd.Flags().StringP(\"queue\", \"q\", \"\", \"queue to which the task belongs (required)\")\n\ttaskRunCmd.Flags().StringP(\"id\", \"i\", \"\", \"id of the task (required)\")\n\ttaskRunCmd.MarkFlagRequired(\"queue\")\n\ttaskRunCmd.MarkFlagRequired(\"id\")\n\n\ttaskCmd.AddCommand(taskEnqueueCmd)\n\ttaskEnqueueCmd.Flags().StringP(\"type_name\", \"t\", \"\", \"type name to enqueue the task as (required)\")\n\ttaskEnqueueCmd.Flags().StringP(\"payload\", \"l\", \"\", \"payload to enqueue (required)\")\n\t// The following are the various OptionTypes; if not specified we won't pass them so that composeOptions()\n\t// can apply its own defaults\n\ttaskEnqueueCmd.Flags().Int(\"retry\", 0, \"maximum retries\")\n\ttaskEnqueueCmd.Flags().String(\"queue\", \"\", \"queue to enqueue the task to\")\n\ttaskEnqueueCmd.Flags().String(\"id\", \"\", \"id to enqueue the task as\")\n\ttaskEnqueueCmd.Flags().String(\"timeout\", \"\", \"timeout for the task (how long it can run); must be parseable as a time.Duration\")\n\ttaskEnqueueCmd.Flags().String(\"deadline\", \"\", \"deadline for the task; must be in RFC3339 format\")\n\ttaskEnqueueCmd.Flags().String(\"unique\", \"\", \"unique period for the task (duration within which it is guaranteed to be unique); must be parseable as a time.Duration\")\n\ttaskEnqueueCmd.Flags().String(\"process_at\", \"\", \"process at time for the task; must be in RFC3339 format\")\n\ttaskEnqueueCmd.Flags().String(\"process_in\", \"\", \"process in window for the task; must be parseable as a time.Duration\")\n\ttaskEnqueueCmd.Flags().String(\"retention\", \"\", \"retention window for the task; must be parseable as a time.Duration\")\n\ttaskEnqueueCmd.Flags().String(\"group\", \"\", \"group for the task\")\n\ttaskEnqueueCmd.MarkFlagRequired(\"type_name\")\n\ttaskEnqueueCmd.MarkFlagRequired(\"payload\")\n\n\ttaskCmd.AddCommand(taskArchiveAllCmd)\n\ttaskArchiveAllCmd.Flags().StringP(\"queue\", \"q\", \"\", \"queue to which the tasks belong (required)\")\n\ttaskArchiveAllCmd.Flags().StringP(\"state\", \"s\", \"\", \"state of the tasks; one of { pending | aggregating | scheduled | retry } (required)\")\n\ttaskArchiveAllCmd.MarkFlagRequired(\"queue\")\n\ttaskArchiveAllCmd.MarkFlagRequired(\"state\")\n\ttaskArchiveAllCmd.Flags().StringP(\"group\", \"g\", \"\", \"group to which the tasks belong (required for archiving aggregating tasks)\")\n\n\ttaskCmd.AddCommand(taskDeleteAllCmd)\n\ttaskDeleteAllCmd.Flags().StringP(\"queue\", \"q\", \"\", \"queue to which the tasks belong (required)\")\n\ttaskDeleteAllCmd.Flags().StringP(\"state\", \"s\", \"\", \"state of the tasks; one of { pending | aggregating | scheduled | retry | archived | completed } (required)\")\n\ttaskDeleteAllCmd.MarkFlagRequired(\"queue\")\n\ttaskDeleteAllCmd.MarkFlagRequired(\"state\")\n\ttaskDeleteAllCmd.Flags().StringP(\"group\", \"g\", \"\", \"group to which the tasks belong (required for deleting aggregating tasks)\")\n\n\ttaskCmd.AddCommand(taskRunAllCmd)\n\ttaskRunAllCmd.Flags().StringP(\"queue\", \"q\", \"\", \"queue to which the tasks belong (required)\")\n\ttaskRunAllCmd.Flags().StringP(\"state\", \"s\", \"\", \"state of the tasks; one of { scheduled | retry | archived } (required)\")\n\ttaskRunAllCmd.MarkFlagRequired(\"queue\")\n\ttaskRunAllCmd.MarkFlagRequired(\"state\")\n\ttaskRunAllCmd.Flags().StringP(\"group\", \"g\", \"\", \"group to which the tasks belong (required for running aggregating tasks)\")\n}\n\nvar taskCmd = &cobra.Command{\n\tUse: \"task [flags]\",\n\tShort: \"Manage tasks\",\n\tExample: heredoc.Doc(`\n\t\t$ asynq task list --queue=myqueue --state=scheduled\n\t\t$ asynq task inspect --queue=myqueue --id=7837f142-6337-4217-9276-8f27281b67d1\n\t\t$ asynq task delete --queue=myqueue --id=7837f142-6337-4217-9276-8f27281b67d1\n\t\t$ asynq task deleteall --queue=myqueue --state=archived`),\n}\n\nvar taskListCmd = &cobra.Command{\n\tUse: \"list --queue= --state= [flags]\",\n\tAliases: []string{\"ls\"},\n\tShort: \"List tasks\",\n\tLong: heredoc.Doc(`\n\tList tasks of the given state from the specified queue.\n\n\tThe --queue and --state flags are required.\n\n\tNote: For aggregating tasks, additional --group flag is required.\n\n\tList opeartion paginates the result set. By default, the command fetches the first 30 tasks.\n\tUse --page and --size flags to specify the page number and size.`),\n\tExample: heredoc.Doc(`\n\t\t$ asynq task list --queue=myqueue --state=pending\n\t\t$ asynq task list --queue=myqueue --state=aggregating --group=mygroup\n\t\t$ asynq task list --queue=myqueue --state=scheduled --page=2`),\n\tRun: taskList,\n}\n\nvar taskInspectCmd = &cobra.Command{\n\tUse: \"inspect --queue= --id=\",\n\tShort: \"Display detailed information on the specified task\",\n\tArgs: cobra.NoArgs,\n\tRun: taskInspect,\n\tExample: heredoc.Doc(`\n\t\t$ asynq task inspect --queue=myqueue --id=f1720682-f5a6-4db1-8953-4f48ae541d0f`),\n}\n\nvar taskCancelCmd = &cobra.Command{\n\tUse: \"cancel [...]\",\n\tShort: \"Cancel one or more active tasks\",\n\tArgs: cobra.MinimumNArgs(1),\n\tRun: taskCancel,\n\tExample: heredoc.Doc(`\n\t\t$ asynq task cancel f1720682-f5a6-4db1-8953-4f48ae541d0f`),\n}\n\nvar taskArchiveCmd = &cobra.Command{\n\tUse: \"archive --queue= --id=\",\n\tShort: \"Archive a task with the given id\",\n\tArgs: cobra.NoArgs,\n\tRun: taskArchive,\n\tExample: heredoc.Doc(`\n\t\t$ asynq task archive --queue=myqueue --id=f1720682-f5a6-4db1-8953-4f48ae541d0f`),\n}\n\nvar taskDeleteCmd = &cobra.Command{\n\tUse: \"delete --queue= --id=\",\n\tAliases: []string{\"remove\", \"rm\"},\n\tShort: \"Delete a task with the given id\",\n\tArgs: cobra.NoArgs,\n\tRun: taskDelete,\n\tExample: heredoc.Doc(`\n\t\t$ asynq task delete --queue=myqueue --id=f1720682-f5a6-4db1-8953-4f48ae541d0f`),\n}\n\nvar taskRunCmd = &cobra.Command{\n\tUse: \"run --queue= --id=\",\n\tShort: \"Run a task with the given id\",\n\tArgs: cobra.NoArgs,\n\tRun: taskRun,\n\tExample: heredoc.Doc(`\n\t\t$ asynq task run --queue=myqueue --id=f1720682-f5a6-4db1-8953-4f48ae541d0f`),\n}\n\nvar taskEnqueueCmd = &cobra.Command{\n\tUse: \"enqueue --type_name=footype --payload=barpayload\",\n\tShort: \"Enqueue a task\",\n\tArgs: cobra.NoArgs,\n\tRun: taskEnqueue,\n\tExample: heredoc.Doc(`\n\t\t$ asynq task enqueue -t footype -l barpayload\n\t\t$ asynq task enqueue -t footask -l barpayload --retry 3 --id f1720682-f5a6-4db1-8953-4f48ae541d0f --queue bazqueue --timeout 100s --deadline 2024-12-14T01:23:45Z --unique 100s --process_at 2024-12-14T01:22:05Z --process_in 100s --retention 5h --group baygroup`),\n}\n\nvar taskArchiveAllCmd = &cobra.Command{\n\tUse: \"archiveall --queue= --state=\",\n\tShort: \"Archive all tasks in the given state\",\n\tArgs: cobra.NoArgs,\n\tRun: taskArchiveAll,\n\tExample: heredoc.Doc(`\n\t\t$ asynq task archiveall --queue=myqueue --state=retry\n\t\t$ asynq task archiveall --queue=myqueue --state=aggregating --group=mygroup`),\n}\n\nvar taskDeleteAllCmd = &cobra.Command{\n\tUse: \"deleteall --queue= --state=\",\n\tShort: \"Delete all tasks in the given state\",\n\tArgs: cobra.NoArgs,\n\tRun: taskDeleteAll,\n\tExample: heredoc.Doc(`\n\t\t$ asynq task deleteall --queue=myqueue --state=archived\n\t\t$ asynq task deleteall --queue=myqueue --state=aggregating --group=mygroup`),\n}\n\nvar taskRunAllCmd = &cobra.Command{\n\tUse: \"runall --queue= --state=\",\n\tShort: \"Run all tasks in the given state\",\n\tArgs: cobra.NoArgs,\n\tRun: taskRunAll,\n\tExample: heredoc.Doc(`\n\t\t$ asynq task runall --queue=myqueue --state=retry\n\t\t$ asynq task runall --queue=myqueue --state=aggregating --group=mygroup`),\n}\n\nfunc taskList(cmd *cobra.Command, args []string) {\n\tqname, err := cmd.Flags().GetString(\"queue\")\n\tif err != nil {\n\t\tfmt.Println(err)\n\t\tos.Exit(1)\n\t}\n\tstate, err := cmd.Flags().GetString(\"state\")\n\tif err != nil {\n\t\tfmt.Println(err)\n\t\tos.Exit(1)\n\t}\n\tpageNum, err := cmd.Flags().GetInt(\"page\")\n\tif err != nil {\n\t\tfmt.Println(err)\n\t\tos.Exit(1)\n\t}\n\tpageSize, err := cmd.Flags().GetInt(\"size\")\n\tif err != nil {\n\t\tfmt.Println(err)\n\t\tos.Exit(1)\n\t}\n\n\tswitch state {\n\tcase \"active\":\n\t\tlistActiveTasks(qname, pageNum, pageSize)\n\tcase \"pending\":\n\t\tlistPendingTasks(qname, pageNum, pageSize)\n\tcase \"scheduled\":\n\t\tlistScheduledTasks(qname, pageNum, pageSize)\n\tcase \"retry\":\n\t\tlistRetryTasks(qname, pageNum, pageSize)\n\tcase \"archived\":\n\t\tlistArchivedTasks(qname, pageNum, pageSize)\n\tcase \"completed\":\n\t\tlistCompletedTasks(qname, pageNum, pageSize)\n\tcase \"aggregating\":\n\t\tgroup, err := cmd.Flags().GetString(\"group\")\n\t\tif err != nil {\n\t\t\tfmt.Println(err)\n\t\t\tos.Exit(1)\n\t\t}\n\t\tif group == \"\" {\n\t\t\tfmt.Println(\"Flag --group is required for listing aggregating tasks\")\n\t\t\tos.Exit(1)\n\t\t}\n\t\tlistAggregatingTasks(qname, group, pageNum, pageSize)\n\tdefault:\n\t\tfmt.Printf(\"error: state=%q is not supported\\n\", state)\n\t\tos.Exit(1)\n\t}\n}\n\nfunc listActiveTasks(qname string, pageNum, pageSize int) {\n\ti := createInspector()\n\ttasks, err := i.ListActiveTasks(qname, asynq.PageSize(pageSize), asynq.Page(pageNum))\n\tif err != nil {\n\t\tfmt.Println(err)\n\t\tos.Exit(1)\n\t}\n\tif len(tasks) == 0 {\n\t\tfmt.Printf(\"No active tasks in %q queue\\n\", qname)\n\t\treturn\n\t}\n\tprintTable(\n\t\t[]string{\"ID\", \"Type\", \"Payload\"},\n\t\tfunc(w io.Writer, tmpl string) {\n\t\t\tfor _, t := range tasks {\n\t\t\t\tfmt.Fprintf(w, tmpl, t.ID, t.Type, sprintBytes(t.Payload))\n\t\t\t}\n\t\t},\n\t)\n}\n\nfunc listPendingTasks(qname string, pageNum, pageSize int) {\n\ti := createInspector()\n\ttasks, err := i.ListPendingTasks(qname, asynq.PageSize(pageSize), asynq.Page(pageNum))\n\tif err != nil {\n\t\tfmt.Println(err)\n\t\tos.Exit(1)\n\t}\n\tif len(tasks) == 0 {\n\t\tfmt.Printf(\"No pending tasks in %q queue\\n\", qname)\n\t\treturn\n\t}\n\tprintTable(\n\t\t[]string{\"ID\", \"Type\", \"Payload\"},\n\t\tfunc(w io.Writer, tmpl string) {\n\t\t\tfor _, t := range tasks {\n\t\t\t\tfmt.Fprintf(w, tmpl, t.ID, t.Type, sprintBytes(t.Payload))\n\t\t\t}\n\t\t},\n\t)\n}\n\nfunc listScheduledTasks(qname string, pageNum, pageSize int) {\n\ti := createInspector()\n\ttasks, err := i.ListScheduledTasks(qname, asynq.PageSize(pageSize), asynq.Page(pageNum))\n\tif err != nil {\n\t\tfmt.Println(err)\n\t\tos.Exit(1)\n\t}\n\tif len(tasks) == 0 {\n\t\tfmt.Printf(\"No scheduled tasks in %q queue\\n\", qname)\n\t\treturn\n\t}\n\tprintTable(\n\t\t[]string{\"ID\", \"Type\", \"Payload\", \"Process In\"},\n\t\tfunc(w io.Writer, tmpl string) {\n\t\t\tfor _, t := range tasks {\n\t\t\t\tfmt.Fprintf(w, tmpl, t.ID, t.Type, sprintBytes(t.Payload), formatProcessAt(t.NextProcessAt))\n\t\t\t}\n\t\t},\n\t)\n}\n\n// formatProcessAt formats next process at time to human friendly string.\n// If processAt time is in the past, returns \"right now\".\n// If processAt time is in the future, returns \"in xxx\" where xxx is the duration from now.\nfunc formatProcessAt(processAt time.Time) string {\n\td := processAt.Sub(time.Now())\n\tif d < 0 {\n\t\treturn \"right now\"\n\t}\n\treturn fmt.Sprintf(\"in %v\", d.Round(time.Second))\n}\n\nfunc listRetryTasks(qname string, pageNum, pageSize int) {\n\ti := createInspector()\n\ttasks, err := i.ListRetryTasks(qname, asynq.PageSize(pageSize), asynq.Page(pageNum))\n\tif err != nil {\n\t\tfmt.Println(err)\n\t\tos.Exit(1)\n\t}\n\tif len(tasks) == 0 {\n\t\tfmt.Printf(\"No retry tasks in %q queue\\n\", qname)\n\t\treturn\n\t}\n\tprintTable(\n\t\t[]string{\"ID\", \"Type\", \"Payload\", \"Next Retry\", \"Last Error\", \"Last Failed\", \"Retried\", \"Max Retry\"},\n\t\tfunc(w io.Writer, tmpl string) {\n\t\t\tfor _, t := range tasks {\n\t\t\t\tfmt.Fprintf(w, tmpl, t.ID, t.Type, sprintBytes(t.Payload), formatProcessAt(t.NextProcessAt),\n\t\t\t\t\tt.LastErr, formatPastTime(t.LastFailedAt), t.Retried, t.MaxRetry)\n\t\t\t}\n\t\t},\n\t)\n}\n\nfunc listArchivedTasks(qname string, pageNum, pageSize int) {\n\ti := createInspector()\n\ttasks, err := i.ListArchivedTasks(qname, asynq.PageSize(pageSize), asynq.Page(pageNum))\n\tif err != nil {\n\t\tfmt.Println(err)\n\t\tos.Exit(1)\n\t}\n\tif len(tasks) == 0 {\n\t\tfmt.Printf(\"No archived tasks in %q queue\\n\", qname)\n\t\treturn\n\t}\n\tprintTable(\n\t\t[]string{\"ID\", \"Type\", \"Payload\", \"Last Failed\", \"Last Error\"},\n\t\tfunc(w io.Writer, tmpl string) {\n\t\t\tfor _, t := range tasks {\n\t\t\t\tfmt.Fprintf(w, tmpl, t.ID, t.Type, sprintBytes(t.Payload), formatPastTime(t.LastFailedAt), t.LastErr)\n\t\t\t}\n\t\t})\n}\n\nfunc listCompletedTasks(qname string, pageNum, pageSize int) {\n\ti := createInspector()\n\ttasks, err := i.ListCompletedTasks(qname, asynq.PageSize(pageSize), asynq.Page(pageNum))\n\tif err != nil {\n\t\tfmt.Println(err)\n\t\tos.Exit(1)\n\t}\n\tif len(tasks) == 0 {\n\t\tfmt.Printf(\"No completed tasks in %q queue\\n\", qname)\n\t\treturn\n\t}\n\tprintTable(\n\t\t[]string{\"ID\", \"Type\", \"Payload\", \"CompletedAt\", \"Result\"},\n\t\tfunc(w io.Writer, tmpl string) {\n\t\t\tfor _, t := range tasks {\n\t\t\t\tfmt.Fprintf(w, tmpl, t.ID, t.Type, sprintBytes(t.Payload), formatPastTime(t.CompletedAt), sprintBytes(t.Result))\n\t\t\t}\n\t\t})\n}\n\nfunc listAggregatingTasks(qname, group string, pageNum, pageSize int) {\n\ti := createInspector()\n\ttasks, err := i.ListAggregatingTasks(qname, group, asynq.PageSize(pageSize), asynq.Page(pageNum))\n\tif err != nil {\n\t\tfmt.Println(err)\n\t\tos.Exit(1)\n\t}\n\tif len(tasks) == 0 {\n\t\tfmt.Printf(\"No aggregating tasks in group %q \\n\", group)\n\t\treturn\n\t}\n\tprintTable(\n\t\t[]string{\"ID\", \"Type\", \"Payload\", \"Group\"},\n\t\tfunc(w io.Writer, tmpl string) {\n\t\t\tfor _, t := range tasks {\n\t\t\t\tfmt.Fprintf(w, tmpl, t.ID, t.Type, sprintBytes(t.Payload), t.Group)\n\t\t\t}\n\t\t},\n\t)\n}\n\nfunc taskCancel(cmd *cobra.Command, args []string) {\n\ti := createInspector()\n\tfor _, id := range args {\n\t\tif err := i.CancelProcessing(id); err != nil {\n\t\t\tfmt.Printf(\"error: could not send cancelation signal: %v\\n\", err)\n\t\t\tcontinue\n\t\t}\n\t\tfmt.Printf(\"Sent cancelation signal for task %s\\n\", id)\n\t}\n}\n\nfunc taskInspect(cmd *cobra.Command, args []string) {\n\tqname, err := cmd.Flags().GetString(\"queue\")\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\tid, err := cmd.Flags().GetString(\"id\")\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\n\ti := createInspector()\n\tinfo, err := i.GetTaskInfo(qname, id)\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\tprintTaskInfo(info)\n}\n\nfunc printTaskInfo(info *asynq.TaskInfo) {\n\tbold := color.New(color.Bold)\n\tbold.Println(\"Task Info\")\n\tfmt.Printf(\"Queue: %s\\n\", info.Queue)\n\tfmt.Printf(\"ID: %s\\n\", info.ID)\n\tfmt.Printf(\"Type: %s\\n\", info.Type)\n\tfmt.Printf(\"State: %v\\n\", info.State)\n\tfmt.Printf(\"Retried: %d/%d\\n\", info.Retried, info.MaxRetry)\n\tfmt.Println()\n\tfmt.Printf(\"Next process time: %s\\n\", formatNextProcessAt(info.NextProcessAt))\n\tif len(info.LastErr) != 0 {\n\t\tfmt.Println()\n\t\tbold.Println(\"Last Failure\")\n\t\tfmt.Printf(\"Failed at: %s\\n\", formatPastTime(info.LastFailedAt))\n\t\tfmt.Printf(\"Error message: %s\\n\", info.LastErr)\n\t}\n}\n\nfunc formatNextProcessAt(processAt time.Time) string {\n\tif processAt.IsZero() || processAt.Unix() == 0 {\n\t\treturn \"n/a\"\n\t}\n\tif processAt.Before(time.Now()) {\n\t\treturn \"now\"\n\t}\n\treturn fmt.Sprintf(\"%s (in %v)\", processAt.Format(time.UnixDate), processAt.Sub(time.Now()).Round(time.Second))\n}\n\n// formatPastTime takes t which is time in the past and returns a user-friendly string.\nfunc formatPastTime(t time.Time) string {\n\tif t.IsZero() || t.Unix() == 0 {\n\t\treturn \"\"\n\t}\n\treturn t.Format(time.UnixDate)\n}\n\nfunc taskArchive(cmd *cobra.Command, args []string) {\n\tqname, err := cmd.Flags().GetString(\"queue\")\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\tid, err := cmd.Flags().GetString(\"id\")\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\n\ti := createInspector()\n\terr = i.ArchiveTask(qname, id)\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\tfmt.Println(\"task archived\")\n}\n\nfunc taskDelete(cmd *cobra.Command, args []string) {\n\tqname, err := cmd.Flags().GetString(\"queue\")\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\tid, err := cmd.Flags().GetString(\"id\")\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\n\ti := createInspector()\n\terr = i.DeleteTask(qname, id)\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\tfmt.Println(\"task deleted\")\n}\n\nfunc taskRun(cmd *cobra.Command, args []string) {\n\tqname, err := cmd.Flags().GetString(\"queue\")\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\tid, err := cmd.Flags().GetString(\"id\")\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\n\ti := createInspector()\n\terr = i.RunTask(qname, id)\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\tfmt.Println(\"task is now pending\")\n}\n\nfunc taskEnqueue(cmd *cobra.Command, args []string) {\n\ttypeName, err := cmd.Flags().GetString(\"type_name\")\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\n\tpayload, err := cmd.Flags().GetString(\"payload\")\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\n\t// For all of the optional flags, we need to explicitly check whether they were set or\n\t// not; for consistency we want to use the defaults set in composeOptions() rather than\n\t// the ones in the flag definitions.\n\topts := []asynq.Option{}\n\tif cmd.Flags().Changed(\"retry\") {\n\t\tretry, err := cmd.Flags().GetInt(\"retry\")\n\t\tif err != nil {\n\t\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\t\tos.Exit(1)\n\t\t}\n\t\topts = append(opts, asynq.MaxRetry(retry))\n\t}\n\n\tif cmd.Flags().Changed(\"queue\") {\n\t\tqueue, err := cmd.Flags().GetString(\"queue\")\n\t\tif err != nil {\n\t\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\t\tos.Exit(1)\n\t\t}\n\t\topts = append(opts, asynq.Queue(queue))\n\t}\n\n\tif cmd.Flags().Changed(\"id\") {\n\t\tid, err := cmd.Flags().GetString(\"id\")\n\t\tif err != nil {\n\t\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\t\tos.Exit(1)\n\t\t}\n\t\topts = append(opts, asynq.TaskID(id))\n\t}\n\n\tif cmd.Flags().Changed(\"timeout\") {\n\t\topts = append(opts, asynq.Timeout(getDuration(cmd, \"timeout\")))\n\t}\n\n\tif cmd.Flags().Changed(\"deadline\") {\n\t\topts = append(opts, asynq.Deadline(getTime(cmd, \"deadline\")))\n\t}\n\n\tif cmd.Flags().Changed(\"unique\") {\n\t\topts = append(opts, asynq.Unique(getDuration(cmd, \"unique\")))\n\t}\n\n\tif cmd.Flags().Changed(\"process_at\") {\n\t\topts = append(opts, asynq.ProcessAt(getTime(cmd, \"process_at\")))\n\t}\n\n\tif cmd.Flags().Changed(\"process_in\") {\n\t\topts = append(opts, asynq.ProcessIn(getDuration(cmd, \"process_in\")))\n\t}\n\n\tif cmd.Flags().Changed(\"retention\") {\n\t\topts = append(opts, asynq.Retention(getDuration(cmd, \"retention\")))\n\t}\n\n\tif cmd.Flags().Changed(\"group\") {\n\t\tgroup, err := cmd.Flags().GetString(\"group\")\n\t\tif err != nil {\n\t\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\t\tos.Exit(1)\n\t\t}\n\t\topts = append(opts, asynq.Group(group))\n\t}\n\n\tc := createClient()\n\ttask := asynq.NewTask(typeName, []byte(payload), opts...)\n\n\ttaskInfo, err := c.Enqueue(task)\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\n\tfmt.Printf(\"Enqueued task %s to queue %s\\n\", taskInfo.ID, taskInfo.Queue)\n}\n\nfunc taskArchiveAll(cmd *cobra.Command, args []string) {\n\tqname, err := cmd.Flags().GetString(\"queue\")\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\tstate, err := cmd.Flags().GetString(\"state\")\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\n\ti := createInspector()\n\tvar n int\n\tswitch state {\n\tcase \"pending\":\n\t\tn, err = i.ArchiveAllPendingTasks(qname)\n\tcase \"scheduled\":\n\t\tn, err = i.ArchiveAllScheduledTasks(qname)\n\tcase \"retry\":\n\t\tn, err = i.ArchiveAllRetryTasks(qname)\n\tcase \"aggregating\":\n\t\tgroup, err := cmd.Flags().GetString(\"group\")\n\t\tif err != nil {\n\t\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\t\tos.Exit(1)\n\t\t}\n\t\tif group == \"\" {\n\t\t\tfmt.Println(\"error: Flag --group is required for aggregating tasks\")\n\t\t\tos.Exit(1)\n\t\t}\n\t\tn, err = i.ArchiveAllAggregatingTasks(qname, group)\n\tdefault:\n\t\tfmt.Printf(\"error: unsupported state %q\\n\", state)\n\t\tos.Exit(1)\n\t}\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\tfmt.Printf(\"%d tasks archived\\n\", n)\n}\n\nfunc taskDeleteAll(cmd *cobra.Command, args []string) {\n\tqname, err := cmd.Flags().GetString(\"queue\")\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\tstate, err := cmd.Flags().GetString(\"state\")\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\n\ti := createInspector()\n\tvar n int\n\tswitch state {\n\tcase \"pending\":\n\t\tn, err = i.DeleteAllPendingTasks(qname)\n\tcase \"scheduled\":\n\t\tn, err = i.DeleteAllScheduledTasks(qname)\n\tcase \"retry\":\n\t\tn, err = i.DeleteAllRetryTasks(qname)\n\tcase \"archived\":\n\t\tn, err = i.DeleteAllArchivedTasks(qname)\n\tcase \"completed\":\n\t\tn, err = i.DeleteAllCompletedTasks(qname)\n\tcase \"aggregating\":\n\t\tgroup, err := cmd.Flags().GetString(\"group\")\n\t\tif err != nil {\n\t\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\t\tos.Exit(1)\n\t\t}\n\t\tif group == \"\" {\n\t\t\tfmt.Println(\"error: Flag --group is required for aggregating tasks\")\n\t\t\tos.Exit(1)\n\t\t}\n\t\tn, err = i.DeleteAllAggregatingTasks(qname, group)\n\tdefault:\n\t\tfmt.Printf(\"error: unsupported state %q\\n\", state)\n\t\tos.Exit(1)\n\t}\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\tfmt.Printf(\"%d tasks deleted\\n\", n)\n}\n\nfunc taskRunAll(cmd *cobra.Command, args []string) {\n\tqname, err := cmd.Flags().GetString(\"queue\")\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\tstate, err := cmd.Flags().GetString(\"state\")\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\n\ti := createInspector()\n\tvar n int\n\tswitch state {\n\tcase \"scheduled\":\n\t\tn, err = i.RunAllScheduledTasks(qname)\n\tcase \"retry\":\n\t\tn, err = i.RunAllRetryTasks(qname)\n\tcase \"archived\":\n\t\tn, err = i.RunAllArchivedTasks(qname)\n\tcase \"aggregating\":\n\t\tgroup, err := cmd.Flags().GetString(\"group\")\n\t\tif err != nil {\n\t\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\t\tos.Exit(1)\n\t\t}\n\t\tif group == \"\" {\n\t\t\tfmt.Println(\"error: Flag --group is required for aggregating tasks\")\n\t\t\tos.Exit(1)\n\t\t}\n\t\tn, err = i.RunAllAggregatingTasks(qname, group)\n\tdefault:\n\t\tfmt.Printf(\"error: unsupported state %q\\n\", state)\n\t\tos.Exit(1)\n\t}\n\tif err != nil {\n\t\tfmt.Printf(\"error: %v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\tfmt.Printf(\"%d tasks are now pending\\n\", n)\n}\n" }, { "path": "tools/asynq/main.go", "content": "// Copyright 2020 Kentaro Hibino. All rights reserved.\n// Use of this source code is governed by a MIT license\n// that can be found in the LICENSE file.\n\npackage main\n\nimport \"github.com/hibiken/asynq/tools/asynq/cmd\"\n\nfunc main() {\n\tcmd.Execute()\n}\n" }, { "path": "tools/go.mod", "content": "module github.com/hibiken/asynq/tools\n\ngo 1.22\n\nrequire (\n\tgithub.com/MakeNowJust/heredoc/v2 v2.0.1\n\tgithub.com/fatih/color v1.18.0\n\tgithub.com/gdamore/tcell/v2 v2.5.1\n\tgithub.com/google/go-cmp v0.6.0\n\tgithub.com/hibiken/asynq v0.25.0\n\tgithub.com/hibiken/asynq/x v0.0.0-20220131170841-349f4c50fb1d\n\tgithub.com/mattn/go-runewidth v0.0.16\n\tgithub.com/mitchellh/go-homedir v1.1.0\n\tgithub.com/prometheus/client_golang v1.11.1\n\tgithub.com/redis/go-redis/v9 v9.7.0\n\tgithub.com/spf13/cobra v1.1.1\n\tgithub.com/spf13/pflag v1.0.5\n\tgithub.com/spf13/viper v1.7.0\n\tgolang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136\n)\n\nrequire (\n\tgithub.com/beorn7/perks v1.0.1 // indirect\n\tgithub.com/cespare/xxhash/v2 v2.2.0 // indirect\n\tgithub.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect\n\tgithub.com/fsnotify/fsnotify v1.4.9 // indirect\n\tgithub.com/gdamore/encoding v1.0.0 // indirect\n\tgithub.com/golang/protobuf v1.5.3 // indirect\n\tgithub.com/google/uuid v1.6.0 // indirect\n\tgithub.com/hashicorp/hcl v1.0.0 // indirect\n\tgithub.com/inconshreveable/mousetrap v1.0.0 // indirect\n\tgithub.com/lucasb-eyer/go-colorful v1.2.0 // indirect\n\tgithub.com/magiconair/properties v1.8.1 // indirect\n\tgithub.com/mattn/go-colorable v0.1.13 // indirect\n\tgithub.com/mattn/go-isatty v0.0.20 // indirect\n\tgithub.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect\n\tgithub.com/mitchellh/mapstructure v1.1.2 // indirect\n\tgithub.com/pelletier/go-toml v1.2.0 // indirect\n\tgithub.com/prometheus/client_model v0.2.0 // indirect\n\tgithub.com/prometheus/common v0.26.0 // indirect\n\tgithub.com/prometheus/procfs v0.6.0 // indirect\n\tgithub.com/rivo/uniseg v0.2.0 // indirect\n\tgithub.com/robfig/cron/v3 v3.0.1 // indirect\n\tgithub.com/spf13/afero v1.1.2 // indirect\n\tgithub.com/spf13/cast v1.7.0 // indirect\n\tgithub.com/spf13/jwalterweatherman v1.0.0 // indirect\n\tgithub.com/subosito/gotenv v1.2.0 // indirect\n\tgolang.org/x/sys v0.26.0 // indirect\n\tgolang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect\n\tgolang.org/x/text v0.3.8 // indirect\n\tgolang.org/x/time v0.7.0 // indirect\n\tgoogle.golang.org/protobuf v1.35.1 // indirect\n\tgopkg.in/ini.v1 v1.51.0 // indirect\n\tgopkg.in/yaml.v2 v2.4.0 // indirect\n)\n" }, { "path": "tools/go.sum", "content": "cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=\ncloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=\ncloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=\ncloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=\ncloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=\ncloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=\ncloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=\ncloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=\ncloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=\ncloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=\ncloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=\ncloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=\ndmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=\ngithub.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=\ngithub.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=\ngithub.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=\ngithub.com/MakeNowJust/heredoc/v2 v2.0.1 h1:rlCHh70XXXv7toz95ajQWOWQnN4WNLt0TdpZYIR/J6A=\ngithub.com/MakeNowJust/heredoc/v2 v2.0.1/go.mod h1:6/2Abh5s+hc3g9nbWLe9ObDIOhaRrqsyY9MWy+4JdRM=\ngithub.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=\ngithub.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=\ngithub.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=\ngithub.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=\ngithub.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=\ngithub.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=\ngithub.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=\ngithub.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=\ngithub.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=\ngithub.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=\ngithub.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=\ngithub.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=\ngithub.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=\ngithub.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=\ngithub.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=\ngithub.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=\ngithub.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=\ngithub.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=\ngithub.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=\ngithub.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=\ngithub.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=\ngithub.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=\ngithub.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=\ngithub.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=\ngithub.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=\ngithub.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=\ngithub.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=\ngithub.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=\ngithub.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=\ngithub.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=\ngithub.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=\ngithub.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=\ngithub.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=\ngithub.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=\ngithub.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=\ngithub.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=\ngithub.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=\ngithub.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=\ngithub.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=\ngithub.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=\ngithub.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=\ngithub.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=\ngithub.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=\ngithub.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=\ngithub.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=\ngithub.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=\ngithub.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=\ngithub.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko=\ngithub.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=\ngithub.com/gdamore/tcell/v2 v2.5.1 h1:zc3LPdpK184lBW7syF2a5C6MV827KmErk9jGVnmsl/I=\ngithub.com/gdamore/tcell/v2 v2.5.1/go.mod h1:wSkrPaXoiIWZqW/g7Px4xc79di6FTcpB8tvaKJ6uGBo=\ngithub.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=\ngithub.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=\ngithub.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=\ngithub.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=\ngithub.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=\ngithub.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=\ngithub.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=\ngithub.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=\ngithub.com/go-redis/redis/v8 v8.11.2/go.mod h1:DLomh7y2e3ggQXQLd1YgmvIfecPJoFl7WU5SOQ/r06M=\ngithub.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w=\ngithub.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=\ngithub.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=\ngithub.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=\ngithub.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=\ngithub.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=\ngithub.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=\ngithub.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=\ngithub.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=\ngithub.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=\ngithub.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=\ngithub.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=\ngithub.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=\ngithub.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=\ngithub.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=\ngithub.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=\ngithub.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=\ngithub.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=\ngithub.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=\ngithub.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=\ngithub.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=\ngithub.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=\ngithub.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=\ngithub.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=\ngithub.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=\ngithub.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=\ngithub.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=\ngithub.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=\ngithub.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=\ngithub.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=\ngithub.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=\ngithub.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=\ngithub.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=\ngithub.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=\ngithub.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=\ngithub.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=\ngithub.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=\ngithub.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=\ngithub.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=\ngithub.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=\ngithub.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=\ngithub.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=\ngithub.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=\ngithub.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=\ngithub.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=\ngithub.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=\ngithub.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=\ngithub.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=\ngithub.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=\ngithub.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=\ngithub.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=\ngithub.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=\ngithub.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=\ngithub.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=\ngithub.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=\ngithub.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=\ngithub.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=\ngithub.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=\ngithub.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=\ngithub.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=\ngithub.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=\ngithub.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=\ngithub.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=\ngithub.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=\ngithub.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=\ngithub.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=\ngithub.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=\ngithub.com/hibiken/asynq v0.19.0/go.mod h1:tyc63ojaW8SJ5SBm8mvI4DDONsguP5HE85EEl4Qr5Ig=\ngithub.com/hibiken/asynq v0.25.0 h1:VCPyRRrrjFChsTSI8x5OCPu51MlEz6Rk+1p0kHKnZug=\ngithub.com/hibiken/asynq v0.25.0/go.mod h1:DYQ1etBEl2Y+uSkqFElGYbk3M0ujLVwCfWE+TlvxtEk=\ngithub.com/hibiken/asynq/x v0.0.0-20220131170841-349f4c50fb1d h1:Er+U+9PmnyRHRDQjSjRQ24HoWvOY7w9Pk7bUPYM3Ags=\ngithub.com/hibiken/asynq/x v0.0.0-20220131170841-349f4c50fb1d/go.mod h1:VmxwMfMKyb6gyv8xG0oOBMXIhquWKPx+zPtbVBd2Q1s=\ngithub.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=\ngithub.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=\ngithub.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=\ngithub.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=\ngithub.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=\ngithub.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=\ngithub.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=\ngithub.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=\ngithub.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=\ngithub.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=\ngithub.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=\ngithub.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=\ngithub.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=\ngithub.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=\ngithub.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=\ngithub.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=\ngithub.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=\ngithub.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=\ngithub.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=\ngithub.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=\ngithub.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=\ngithub.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=\ngithub.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=\ngithub.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=\ngithub.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=\ngithub.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=\ngithub.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=\ngithub.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=\ngithub.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=\ngithub.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=\ngithub.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=\ngithub.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=\ngithub.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=\ngithub.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=\ngithub.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=\ngithub.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=\ngithub.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=\ngithub.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=\ngithub.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=\ngithub.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=\ngithub.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=\ngithub.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=\ngithub.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=\ngithub.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=\ngithub.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=\ngithub.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=\ngithub.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=\ngithub.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=\ngithub.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=\ngithub.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=\ngithub.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=\ngithub.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=\ngithub.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=\ngithub.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=\ngithub.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=\ngithub.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=\ngithub.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=\ngithub.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=\ngithub.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=\ngithub.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=\ngithub.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=\ngithub.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=\ngithub.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=\ngithub.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg=\ngithub.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=\ngithub.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=\ngithub.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=\ngithub.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48=\ngithub.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=\ngithub.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=\ngithub.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=\ngithub.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=\ngithub.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=\ngithub.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=\ngithub.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=\ngithub.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=\ngithub.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=\ngithub.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=\ngithub.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=\ngithub.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s=\ngithub.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=\ngithub.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=\ngithub.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=\ngithub.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=\ngithub.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=\ngithub.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=\ngithub.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=\ngithub.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=\ngithub.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=\ngithub.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=\ngithub.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ=\ngithub.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=\ngithub.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=\ngithub.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=\ngithub.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=\ngithub.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=\ngithub.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=\ngithub.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=\ngithub.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=\ngithub.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=\ngithub.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=\ngithub.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=\ngithub.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=\ngithub.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=\ngithub.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=\ngithub.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=\ngithub.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=\ngithub.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=\ngithub.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=\ngithub.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=\ngithub.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=\ngithub.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=\ngithub.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=\ngithub.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=\ngithub.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=\ngithub.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=\ngithub.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=\ngithub.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=\ngithub.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=\ngithub.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=\ngithub.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=\ngithub.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=\ngithub.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=\ngithub.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=\ngithub.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=\ngithub.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=\ngithub.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w=\ngithub.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=\ngithub.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4=\ngithub.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=\ngithub.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=\ngithub.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=\ngithub.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=\ngithub.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=\ngithub.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=\ngithub.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM=\ngithub.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=\ngithub.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=\ngithub.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=\ngithub.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=\ngithub.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=\ngithub.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=\ngithub.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=\ngithub.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=\ngithub.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=\ngithub.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=\ngithub.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=\ngo.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=\ngo.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=\ngo.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=\ngo.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=\ngo.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI=\ngo.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=\ngo.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=\ngo.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=\ngo.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=\ngolang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=\ngolang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=\ngolang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=\ngolang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=\ngolang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=\ngolang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=\ngolang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=\ngolang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=\ngolang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136 h1:A1gGSx58LAGVHUUsOf7IiR0u8Xb6W51gRwfDBhkdcaw=\ngolang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=\ngolang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=\ngolang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=\ngolang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=\ngolang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=\ngolang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=\ngolang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=\ngolang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=\ngolang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=\ngolang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=\ngolang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=\ngolang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=\ngolang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=\ngolang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=\ngolang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=\ngolang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=\ngolang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=\ngolang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=\ngolang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=\ngolang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=\ngolang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=\ngolang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=\ngolang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=\ngolang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220318055525-2edf467146b5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=\ngolang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=\ngolang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=\ngolang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M=\ngolang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=\ngolang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=\ngolang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=\ngolang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=\ngolang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=\ngolang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=\ngolang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=\ngolang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ=\ngolang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=\ngolang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=\ngolang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=\ngolang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=\ngolang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=\ngolang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=\ngolang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=\ngolang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=\ngolang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=\ngolang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngoogle.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=\ngoogle.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=\ngoogle.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=\ngoogle.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=\ngoogle.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=\ngoogle.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=\ngoogle.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=\ngoogle.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=\ngoogle.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=\ngoogle.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=\ngoogle.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=\ngoogle.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=\ngoogle.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=\ngoogle.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=\ngoogle.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=\ngoogle.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=\ngoogle.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=\ngoogle.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=\ngoogle.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=\ngoogle.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=\ngoogle.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=\ngoogle.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=\ngoogle.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=\ngoogle.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=\ngoogle.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=\ngoogle.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=\ngoogle.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=\ngoogle.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=\ngoogle.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=\ngoogle.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=\ngoogle.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=\ngoogle.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=\ngoogle.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA=\ngoogle.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=\ngopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=\ngopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=\ngopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=\ngopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=\ngopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=\ngopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=\ngopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=\ngopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=\ngopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=\ngopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\nhonnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nhonnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nhonnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nhonnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nhonnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=\nrsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=\n" }, { "path": "tools/metrics_exporter/main.go", "content": "package main\n\nimport (\n\t\"flag\"\n\t\"fmt\"\n\t\"log\"\n\t\"net/http\"\n\n\t\"github.com/hibiken/asynq\"\n\t\"github.com/hibiken/asynq/x/metrics\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n\t\"github.com/prometheus/client_golang/prometheus/collectors\"\n\t\"github.com/prometheus/client_golang/prometheus/promhttp\"\n)\n\n// Declare command-line flags.\n// These variables are binded to flags in init().\nvar (\n\tflagRedisAddr string\n\tflagRedisDB int\n\tflagRedisPassword string\n\tflagRedisUsername string\n\tflagPort int\n)\n\nfunc init() {\n\tflag.StringVar(&flagRedisAddr, \"redis-addr\", \"127.0.0.1:6379\", \"host:port of redis server to connect to\")\n\tflag.IntVar(&flagRedisDB, \"redis-db\", 0, \"redis DB number to use\")\n\tflag.StringVar(&flagRedisPassword, \"redis-password\", \"\", \"password used to connect to redis server\")\n\tflag.StringVar(&flagRedisUsername, \"redis-username\", \"\", \"username used to connect to redis server\")\n\tflag.IntVar(&flagPort, \"port\", 9876, \"port to use for the HTTP server\")\n}\n\nfunc main() {\n\tflag.Parse()\n\t// Using NewPedanticRegistry here to test the implementation of Collectors and Metrics.\n\treg := prometheus.NewPedanticRegistry()\n\n\tinspector := asynq.NewInspector(asynq.RedisClientOpt{\n\t\tAddr: flagRedisAddr,\n\t\tDB: flagRedisDB,\n\t\tPassword: flagRedisPassword,\n\t\tUsername: flagRedisUsername,\n\t})\n\n\treg.MustRegister(\n\t\tmetrics.NewQueueMetricsCollector(inspector),\n\t\t// Add the standard process and go metrics to the registry\n\t\tcollectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),\n\t\tcollectors.NewGoCollector(),\n\t)\n\n\thttp.Handle(\"/metrics\", promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))\n\tlog.Printf(\"exporter server is listening on port: %d\\n\", flagPort)\n\tlog.Fatal(http.ListenAndServe(fmt.Sprintf(\":%d\", flagPort), nil))\n}\n" }, { "path": "x/go.mod", "content": "module github.com/hibiken/asynq/x\n\ngo 1.22\n\nrequire (\n\tgithub.com/google/uuid v1.6.0\n\tgithub.com/hibiken/asynq v0.25.0\n\tgithub.com/prometheus/client_golang v1.20.5\n\tgithub.com/redis/go-redis/v9 v9.7.0\n)\n\nrequire (\n\tgithub.com/beorn7/perks v1.0.1 // indirect\n\tgithub.com/cespare/xxhash/v2 v2.3.0 // indirect\n\tgithub.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect\n\tgithub.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect\n\tgithub.com/prometheus/client_model v0.6.1 // indirect\n\tgithub.com/prometheus/common v0.55.0 // indirect\n\tgithub.com/prometheus/procfs v0.15.1 // indirect\n\tgithub.com/robfig/cron/v3 v3.0.1 // indirect\n\tgithub.com/spf13/cast v1.7.0 // indirect\n\tgolang.org/x/sys v0.26.0 // indirect\n\tgolang.org/x/time v0.7.0 // indirect\n\tgoogle.golang.org/protobuf v1.35.1 // indirect\n)\n" }, { "path": "x/go.sum", "content": "github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=\ngithub.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=\ngithub.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=\ngithub.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=\ngithub.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=\ngithub.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=\ngithub.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=\ngithub.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=\ngithub.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=\ngithub.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=\ngithub.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=\ngithub.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=\ngithub.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=\ngithub.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=\ngithub.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=\ngithub.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=\ngithub.com/hibiken/asynq v0.25.0 h1:VCPyRRrrjFChsTSI8x5OCPu51MlEz6Rk+1p0kHKnZug=\ngithub.com/hibiken/asynq v0.25.0/go.mod h1:DYQ1etBEl2Y+uSkqFElGYbk3M0ujLVwCfWE+TlvxtEk=\ngithub.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=\ngithub.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=\ngithub.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=\ngithub.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=\ngithub.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=\ngithub.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=\ngithub.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=\ngithub.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=\ngithub.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=\ngithub.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=\ngithub.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=\ngithub.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=\ngithub.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=\ngithub.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=\ngithub.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=\ngithub.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=\ngithub.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=\ngithub.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=\ngithub.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=\ngithub.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=\ngithub.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=\ngithub.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=\ngithub.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w=\ngithub.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=\ngo.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=\ngo.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=\ngolang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=\ngolang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=\ngolang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ=\ngolang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=\ngoogle.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA=\ngoogle.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=\n" }, { "path": "x/metrics/metrics.go", "content": "// Package metrics provides implementations of prometheus.Collector to collect Asynq queue metrics.\npackage metrics\n\nimport (\n\t\"fmt\"\n\t\"log\"\n\n\t\"github.com/hibiken/asynq\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n)\n\n// Namespace used in fully-qualified metrics names.\nconst namespace = \"asynq\"\n\n// QueueMetricsCollector gathers queue metrics.\n// It implements prometheus.Collector interface.\n//\n// All metrics exported from this collector have prefix \"asynq\".\ntype QueueMetricsCollector struct {\n\tinspector *asynq.Inspector\n}\n\n// collectQueueInfo gathers QueueInfo of all queues.\n// Since this operation is expensive, it must be called once per collection.\nfunc (qmc *QueueMetricsCollector) collectQueueInfo() ([]*asynq.QueueInfo, error) {\n\tqnames, err := qmc.inspector.Queues()\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to get queue names: %w\", err)\n\t}\n\tinfos := make([]*asynq.QueueInfo, len(qnames))\n\tfor i, qname := range qnames {\n\t\tqinfo, err := qmc.inspector.GetQueueInfo(qname)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"failed to get queue info: %w\", err)\n\t\t}\n\t\tinfos[i] = qinfo\n\t}\n\treturn infos, nil\n}\n\n// Descriptors used by QueueMetricsCollector\nvar (\n\ttasksQueuedDesc = prometheus.NewDesc(\n\t\tprometheus.BuildFQName(namespace, \"\", \"tasks_enqueued_total\"),\n\t\t\"Number of tasks enqueued; broken down by queue and state.\",\n\t\t[]string{\"queue\", \"state\"}, nil,\n\t)\n\n\tqueueSizeDesc = prometheus.NewDesc(\n\t\tprometheus.BuildFQName(namespace, \"\", \"queue_size\"),\n\t\t\"Number of tasks in a queue\",\n\t\t[]string{\"queue\"}, nil,\n\t)\n\n\tqueueLatencyDesc = prometheus.NewDesc(\n\t\tprometheus.BuildFQName(namespace, \"\", \"queue_latency_seconds\"),\n\t\t\"Number of seconds the oldest pending task is waiting in pending state to be processed.\",\n\t\t[]string{\"queue\"}, nil,\n\t)\n\n\tqueueMemUsgDesc = prometheus.NewDesc(\n\t\tprometheus.BuildFQName(namespace, \"\", \"queue_memory_usage_approx_bytes\"),\n\t\t\"Number of memory used by a given queue (approximated number by sampling).\",\n\t\t[]string{\"queue\"}, nil,\n\t)\n\n\ttasksProcessedTotalDesc = prometheus.NewDesc(\n\t\tprometheus.BuildFQName(namespace, \"\", \"tasks_processed_total\"),\n\t\t\"Number of tasks processed (both succeeded and failed); broken down by queue\",\n\t\t[]string{\"queue\"}, nil,\n\t)\n\n\ttasksFailedTotalDesc = prometheus.NewDesc(\n\t\tprometheus.BuildFQName(namespace, \"\", \"tasks_failed_total\"),\n\t\t\"Number of tasks failed; broken down by queue\",\n\t\t[]string{\"queue\"}, nil,\n\t)\n\n\tpausedQueues = prometheus.NewDesc(\n\t\tprometheus.BuildFQName(namespace, \"\", \"queue_paused_total\"),\n\t\t\"Number of queues paused\",\n\t\t[]string{\"queue\"}, nil,\n\t)\n)\n\nfunc (qmc *QueueMetricsCollector) Describe(ch chan<- *prometheus.Desc) {\n\tprometheus.DescribeByCollect(qmc, ch)\n}\n\nfunc (qmc *QueueMetricsCollector) Collect(ch chan<- prometheus.Metric) {\n\tqueueInfos, err := qmc.collectQueueInfo()\n\tif err != nil {\n\t\tlog.Printf(\"Failed to collect metrics data: %v\", err)\n\t}\n\tfor _, info := range queueInfos {\n\t\tch <- prometheus.MustNewConstMetric(\n\t\t\ttasksQueuedDesc,\n\t\t\tprometheus.GaugeValue,\n\t\t\tfloat64(info.Active),\n\t\t\tinfo.Queue,\n\t\t\t\"active\",\n\t\t)\n\t\tch <- prometheus.MustNewConstMetric(\n\t\t\ttasksQueuedDesc,\n\t\t\tprometheus.GaugeValue,\n\t\t\tfloat64(info.Pending),\n\t\t\tinfo.Queue,\n\t\t\t\"pending\",\n\t\t)\n\t\tch <- prometheus.MustNewConstMetric(\n\t\t\ttasksQueuedDesc,\n\t\t\tprometheus.GaugeValue,\n\t\t\tfloat64(info.Scheduled),\n\t\t\tinfo.Queue,\n\t\t\t\"scheduled\",\n\t\t)\n\t\tch <- prometheus.MustNewConstMetric(\n\t\t\ttasksQueuedDesc,\n\t\t\tprometheus.GaugeValue,\n\t\t\tfloat64(info.Retry),\n\t\t\tinfo.Queue,\n\t\t\t\"retry\",\n\t\t)\n\t\tch <- prometheus.MustNewConstMetric(\n\t\t\ttasksQueuedDesc,\n\t\t\tprometheus.GaugeValue,\n\t\t\tfloat64(info.Archived),\n\t\t\tinfo.Queue,\n\t\t\t\"archived\",\n\t\t)\n\t\tch <- prometheus.MustNewConstMetric(\n\t\t\ttasksQueuedDesc,\n\t\t\tprometheus.GaugeValue,\n\t\t\tfloat64(info.Completed),\n\t\t\tinfo.Queue,\n\t\t\t\"completed\",\n\t\t)\n\n\t\tch <- prometheus.MustNewConstMetric(\n\t\t\tqueueSizeDesc,\n\t\t\tprometheus.GaugeValue,\n\t\t\tfloat64(info.Size),\n\t\t\tinfo.Queue,\n\t\t)\n\n\t\tch <- prometheus.MustNewConstMetric(\n\t\t\tqueueLatencyDesc,\n\t\t\tprometheus.GaugeValue,\n\t\t\tinfo.Latency.Seconds(),\n\t\t\tinfo.Queue,\n\t\t)\n\n\t\tch <- prometheus.MustNewConstMetric(\n\t\t\tqueueMemUsgDesc,\n\t\t\tprometheus.GaugeValue,\n\t\t\tfloat64(info.MemoryUsage),\n\t\t\tinfo.Queue,\n\t\t)\n\n\t\tch <- prometheus.MustNewConstMetric(\n\t\t\ttasksProcessedTotalDesc,\n\t\t\tprometheus.CounterValue,\n\t\t\tfloat64(info.ProcessedTotal),\n\t\t\tinfo.Queue,\n\t\t)\n\n\t\tch <- prometheus.MustNewConstMetric(\n\t\t\ttasksFailedTotalDesc,\n\t\t\tprometheus.CounterValue,\n\t\t\tfloat64(info.FailedTotal),\n\t\t\tinfo.Queue,\n\t\t)\n\n\t\tpausedValue := 0 // zero to indicate \"not paused\"\n\t\tif info.Paused {\n\t\t\tpausedValue = 1\n\t\t}\n\t\tch <- prometheus.MustNewConstMetric(\n\t\t\tpausedQueues,\n\t\t\tprometheus.GaugeValue,\n\t\t\tfloat64(pausedValue),\n\t\t\tinfo.Queue,\n\t\t)\n\t}\n}\n\n// NewQueueMetricsCollector returns a collector that exports metrics about Asynq queues.\nfunc NewQueueMetricsCollector(inspector *asynq.Inspector) *QueueMetricsCollector {\n\treturn &QueueMetricsCollector{inspector: inspector}\n}\n" }, { "path": "x/rate/example_test.go", "content": "package rate_test\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"time\"\n\n\t\"github.com/hibiken/asynq\"\n\t\"github.com/hibiken/asynq/x/rate\"\n)\n\ntype RateLimitError struct {\n\tRetryIn time.Duration\n}\n\nfunc (e *RateLimitError) Error() string {\n\treturn fmt.Sprintf(\"rate limited (retry in %v)\", e.RetryIn)\n}\n\nfunc ExampleNewSemaphore() {\n\tredisConnOpt := asynq.RedisClientOpt{Addr: \":6379\"}\n\tsema := rate.NewSemaphore(redisConnOpt, \"my_queue\", 10)\n\t// call sema.Close() when appropriate\n\n\t_ = asynq.HandlerFunc(func(ctx context.Context, task *asynq.Task) error {\n\t\tok, err := sema.Acquire(ctx)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !ok {\n\t\t\treturn &RateLimitError{RetryIn: 30 * time.Second}\n\t\t}\n\n\t\t// Make sure to release the token once we're done.\n\t\tdefer sema.Release(ctx)\n\n\t\t// Process task\n\t\treturn nil\n\t})\n}\n" }, { "path": "x/rate/semaphore.go", "content": "// Package rate contains rate limiting strategies for asynq.Handler(s).\npackage rate\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/hibiken/asynq\"\n\tasynqcontext \"github.com/hibiken/asynq/internal/context\"\n\t\"github.com/redis/go-redis/v9\"\n)\n\n// NewSemaphore creates a counting Semaphore for the given scope with the given number of tokens.\nfunc NewSemaphore(rco asynq.RedisConnOpt, scope string, maxTokens int) *Semaphore {\n\trc, ok := rco.MakeRedisClient().(redis.UniversalClient)\n\tif !ok {\n\t\tpanic(fmt.Sprintf(\"rate.NewSemaphore: unsupported RedisConnOpt type %T\", rco))\n\t}\n\n\tif maxTokens < 1 {\n\t\tpanic(\"rate.NewSemaphore: maxTokens cannot be less than 1\")\n\t}\n\n\tif len(strings.TrimSpace(scope)) == 0 {\n\t\tpanic(\"rate.NewSemaphore: scope should not be empty\")\n\t}\n\n\treturn &Semaphore{\n\t\trc: rc,\n\t\tscope: scope,\n\t\tmaxTokens: maxTokens,\n\t}\n}\n\n// Semaphore is a distributed counting semaphore which can be used to set maxTokens across multiple asynq servers.\ntype Semaphore struct {\n\trc redis.UniversalClient\n\tmaxTokens int\n\tscope string\n}\n\n// KEYS[1] -> asynq:sema:\n// ARGV[1] -> max concurrency\n// ARGV[2] -> current time in unix time\n// ARGV[3] -> deadline in unix time\n// ARGV[4] -> task ID\nvar acquireCmd = redis.NewScript(`\nredis.call(\"ZREMRANGEBYSCORE\", KEYS[1], \"-inf\", tonumber(ARGV[2])-1)\nlocal count = redis.call(\"ZCARD\", KEYS[1])\n\nif (count < tonumber(ARGV[1])) then\n redis.call(\"ZADD\", KEYS[1], ARGV[3], ARGV[4])\n return 'true'\nelse\n return 'false'\nend\n`)\n\n// Acquire attempts to acquire a token from the semaphore.\n// - Returns (true, nil), iff semaphore key exists and current value is less than maxTokens\n// - Returns (false, nil) when token cannot be acquired\n// - Returns (false, error) otherwise\n//\n// The context.Context passed to Acquire must have a deadline set,\n// this ensures that token is released if the job goroutine crashes and does not call Release.\nfunc (s *Semaphore) Acquire(ctx context.Context) (bool, error) {\n\td, ok := ctx.Deadline()\n\tif !ok {\n\t\treturn false, fmt.Errorf(\"provided context must have a deadline\")\n\t}\n\n\ttaskID, ok := asynqcontext.GetTaskID(ctx)\n\tif !ok {\n\t\treturn false, fmt.Errorf(\"provided context is missing task ID value\")\n\t}\n\n\treturn acquireCmd.Run(ctx, s.rc,\n\t\t[]string{semaphoreKey(s.scope)},\n\t\ts.maxTokens,\n\t\ttime.Now().Unix(),\n\t\td.Unix(),\n\t\ttaskID,\n\t).Bool()\n}\n\n// Release will release the token on the counting semaphore.\nfunc (s *Semaphore) Release(ctx context.Context) error {\n\ttaskID, ok := asynqcontext.GetTaskID(ctx)\n\tif !ok {\n\t\treturn fmt.Errorf(\"provided context is missing task ID value\")\n\t}\n\n\tn, err := s.rc.ZRem(ctx, semaphoreKey(s.scope), taskID).Result()\n\tif err != nil {\n\t\treturn fmt.Errorf(\"redis command failed: %w\", err)\n\t}\n\n\tif n == 0 {\n\t\treturn fmt.Errorf(\"no token found for task %q\", taskID)\n\t}\n\n\treturn nil\n}\n\n// Close closes the connection to redis.\nfunc (s *Semaphore) Close() error {\n\treturn s.rc.Close()\n}\n\nfunc semaphoreKey(scope string) string {\n\treturn \"asynq:sema:\" + scope\n}\n" }, { "path": "x/rate/semaphore_test.go", "content": "package rate\n\nimport (\n\t\"context\"\n\t\"flag\"\n\t\"fmt\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/google/uuid\"\n\t\"github.com/hibiken/asynq\"\n\t\"github.com/hibiken/asynq/internal/base\"\n\tasynqcontext \"github.com/hibiken/asynq/internal/context\"\n\t\"github.com/redis/go-redis/v9\"\n)\n\nvar (\n\tredisAddr string\n\tredisDB int\n\n\tuseRedisCluster bool\n\tredisClusterAddrs string // comma-separated list of host:port\n)\n\nfunc init() {\n\tflag.StringVar(&redisAddr, \"redis_addr\", \"localhost:6379\", \"redis address to use in testing\")\n\tflag.IntVar(&redisDB, \"redis_db\", 14, \"redis db number to use in testing\")\n\tflag.BoolVar(&useRedisCluster, \"redis_cluster\", false, \"use redis cluster as a broker in testing\")\n\tflag.StringVar(&redisClusterAddrs, \"redis_cluster_addrs\", \"localhost:7000,localhost:7001,localhost:7002\", \"comma separated list of redis server addresses\")\n}\n\nfunc TestNewSemaphore(t *testing.T) {\n\ttests := []struct {\n\t\tdesc string\n\t\tname string\n\t\tmaxConcurrency int\n\t\twantPanic string\n\t\tconnOpt asynq.RedisConnOpt\n\t}{\n\t\t{\n\t\t\tdesc: \"Bad RedisConnOpt\",\n\t\t\twantPanic: \"rate.NewSemaphore: unsupported RedisConnOpt type *rate.badConnOpt\",\n\t\t\tconnOpt: &badConnOpt{},\n\t\t},\n\t\t{\n\t\t\tdesc: \"Zero maxTokens should panic\",\n\t\t\twantPanic: \"rate.NewSemaphore: maxTokens cannot be less than 1\",\n\t\t},\n\t\t{\n\t\t\tdesc: \"Empty scope should panic\",\n\t\t\tmaxConcurrency: 2,\n\t\t\tname: \" \",\n\t\t\twantPanic: \"rate.NewSemaphore: scope should not be empty\",\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.desc, func(t *testing.T) {\n\t\t\tif tt.wantPanic != \"\" {\n\t\t\t\tdefer func() {\n\t\t\t\t\tif r := recover(); r.(string) != tt.wantPanic {\n\t\t\t\t\t\tt.Errorf(\"%s;\\nNewSemaphore should panic with msg: %s, got %s\", tt.desc, tt.wantPanic, r.(string))\n\t\t\t\t\t}\n\t\t\t\t}()\n\t\t\t}\n\n\t\t\topt := tt.connOpt\n\t\t\tif tt.connOpt == nil {\n\t\t\t\topt = getRedisConnOpt(t)\n\t\t\t}\n\n\t\t\tsema := NewSemaphore(opt, tt.name, tt.maxConcurrency)\n\t\t\tdefer sema.Close()\n\t\t})\n\t}\n}\n\nfunc TestNewSemaphore_Acquire(t *testing.T) {\n\ttests := []struct {\n\t\tdesc string\n\t\tname string\n\t\tmaxConcurrency int\n\t\ttaskIDs []string\n\t\tctxFunc func(string) (context.Context, context.CancelFunc)\n\t\twant []bool\n\t}{\n\t\t{\n\t\t\tdesc: \"Should acquire token when current token count is less than maxTokens\",\n\t\t\tname: \"task-1\",\n\t\t\tmaxConcurrency: 3,\n\t\t\ttaskIDs: []string{uuid.NewString(), uuid.NewString()},\n\t\t\tctxFunc: func(id string) (context.Context, context.CancelFunc) {\n\t\t\t\treturn asynqcontext.New(context.Background(), &base.TaskMessage{\n\t\t\t\t\tID: id,\n\t\t\t\t\tQueue: \"task-1\",\n\t\t\t\t}, time.Now().Add(time.Second))\n\t\t\t},\n\t\t\twant: []bool{true, true},\n\t\t},\n\t\t{\n\t\t\tdesc: \"Should fail acquiring token when current token count is equal to maxTokens\",\n\t\t\tname: \"task-2\",\n\t\t\tmaxConcurrency: 3,\n\t\t\ttaskIDs: []string{uuid.NewString(), uuid.NewString(), uuid.NewString(), uuid.NewString()},\n\t\t\tctxFunc: func(id string) (context.Context, context.CancelFunc) {\n\t\t\t\treturn asynqcontext.New(context.Background(), &base.TaskMessage{\n\t\t\t\t\tID: id,\n\t\t\t\t\tQueue: \"task-2\",\n\t\t\t\t}, time.Now().Add(time.Second))\n\t\t\t},\n\t\t\twant: []bool{true, true, true, false},\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.desc, func(t *testing.T) {\n\t\t\topt := getRedisConnOpt(t)\n\t\t\trc := opt.MakeRedisClient().(redis.UniversalClient)\n\t\t\tdefer rc.Close()\n\n\t\t\tif err := rc.Del(context.Background(), semaphoreKey(tt.name)).Err(); err != nil {\n\t\t\t\tt.Errorf(\"%s;\\nredis.UniversalClient.Del() got error %v\", tt.desc, err)\n\t\t\t}\n\n\t\t\tsema := NewSemaphore(opt, tt.name, tt.maxConcurrency)\n\t\t\tdefer sema.Close()\n\n\t\t\tfor i := 0; i < len(tt.taskIDs); i++ {\n\t\t\t\tctx, cancel := tt.ctxFunc(tt.taskIDs[i])\n\n\t\t\t\tgot, err := sema.Acquire(ctx)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Errorf(\"%s;\\nSemaphore.Acquire() got error %v\", tt.desc, err)\n\t\t\t\t}\n\n\t\t\t\tif got != tt.want[i] {\n\t\t\t\t\tt.Errorf(\"%s;\\nSemaphore.Acquire(ctx) returned %v, want %v\", tt.desc, got, tt.want[i])\n\t\t\t\t}\n\n\t\t\t\tcancel()\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestNewSemaphore_Acquire_Error(t *testing.T) {\n\ttests := []struct {\n\t\tdesc string\n\t\tname string\n\t\tmaxConcurrency int\n\t\ttaskIDs []string\n\t\tctxFunc func(string) (context.Context, context.CancelFunc)\n\t\terrStr string\n\t}{\n\t\t{\n\t\t\tdesc: \"Should return error if context has no deadline\",\n\t\t\tname: \"task-3\",\n\t\t\tmaxConcurrency: 1,\n\t\t\ttaskIDs: []string{uuid.NewString(), uuid.NewString()},\n\t\t\tctxFunc: func(id string) (context.Context, context.CancelFunc) {\n\t\t\t\treturn context.Background(), func() {}\n\t\t\t},\n\t\t\terrStr: \"provided context must have a deadline\",\n\t\t},\n\t\t{\n\t\t\tdesc: \"Should return error when context is missing taskID\",\n\t\t\tname: \"task-4\",\n\t\t\tmaxConcurrency: 1,\n\t\t\ttaskIDs: []string{uuid.NewString()},\n\t\t\tctxFunc: func(_ string) (context.Context, context.CancelFunc) {\n\t\t\t\treturn context.WithTimeout(context.Background(), time.Second)\n\t\t\t},\n\t\t\terrStr: \"provided context is missing task ID value\",\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.desc, func(t *testing.T) {\n\t\t\topt := getRedisConnOpt(t)\n\t\t\trc := opt.MakeRedisClient().(redis.UniversalClient)\n\t\t\tdefer rc.Close()\n\n\t\t\tif err := rc.Del(context.Background(), semaphoreKey(tt.name)).Err(); err != nil {\n\t\t\t\tt.Errorf(\"%s;\\nredis.UniversalClient.Del() got error %v\", tt.desc, err)\n\t\t\t}\n\n\t\t\tsema := NewSemaphore(opt, tt.name, tt.maxConcurrency)\n\t\t\tdefer sema.Close()\n\n\t\t\tfor i := 0; i < len(tt.taskIDs); i++ {\n\t\t\t\tctx, cancel := tt.ctxFunc(tt.taskIDs[i])\n\n\t\t\t\t_, err := sema.Acquire(ctx)\n\t\t\t\tif err == nil || err.Error() != tt.errStr {\n\t\t\t\t\tt.Errorf(\"%s;\\nSemaphore.Acquire() got error %v want error %v\", tt.desc, err, tt.errStr)\n\t\t\t\t}\n\n\t\t\t\tcancel()\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestNewSemaphore_Acquire_StaleToken(t *testing.T) {\n\topt := getRedisConnOpt(t)\n\trc := opt.MakeRedisClient().(redis.UniversalClient)\n\tdefer rc.Close()\n\n\ttaskID := uuid.NewString()\n\n\t// adding a set member to mimic the case where token is acquired but the goroutine crashed,\n\t// in which case, the token will not be explicitly removed and should be present already\n\trc.ZAdd(context.Background(), semaphoreKey(\"stale-token\"), redis.Z{\n\t\tScore: float64(time.Now().Add(-10 * time.Second).Unix()),\n\t\tMember: taskID,\n\t})\n\n\tsema := NewSemaphore(opt, \"stale-token\", 1)\n\tdefer sema.Close()\n\n\tctx, cancel := asynqcontext.New(context.Background(), &base.TaskMessage{\n\t\tID: taskID,\n\t\tQueue: \"task-1\",\n\t}, time.Now().Add(time.Second))\n\tdefer cancel()\n\n\tgot, err := sema.Acquire(ctx)\n\tif err != nil {\n\t\tt.Errorf(\"Acquire_StaleToken;\\nSemaphore.Acquire() got error %v\", err)\n\t}\n\n\tif !got {\n\t\tt.Error(\"Acquire_StaleToken;\\nSemaphore.Acquire() got false want true\")\n\t}\n}\n\nfunc TestNewSemaphore_Release(t *testing.T) {\n\ttests := []struct {\n\t\tdesc string\n\t\tname string\n\t\ttaskIDs []string\n\t\tctxFunc func(string) (context.Context, context.CancelFunc)\n\t\twantCount int64\n\t}{\n\t\t{\n\t\t\tdesc: \"Should decrease token count\",\n\t\t\tname: \"task-5\",\n\t\t\ttaskIDs: []string{uuid.NewString()},\n\t\t\tctxFunc: func(id string) (context.Context, context.CancelFunc) {\n\t\t\t\treturn asynqcontext.New(context.Background(), &base.TaskMessage{\n\t\t\t\t\tID: id,\n\t\t\t\t\tQueue: \"task-3\",\n\t\t\t\t}, time.Now().Add(time.Second))\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdesc: \"Should decrease token count by 2\",\n\t\t\tname: \"task-6\",\n\t\t\ttaskIDs: []string{uuid.NewString(), uuid.NewString()},\n\t\t\tctxFunc: func(id string) (context.Context, context.CancelFunc) {\n\t\t\t\treturn asynqcontext.New(context.Background(), &base.TaskMessage{\n\t\t\t\t\tID: id,\n\t\t\t\t\tQueue: \"task-4\",\n\t\t\t\t}, time.Now().Add(time.Second))\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.desc, func(t *testing.T) {\n\t\t\topt := getRedisConnOpt(t)\n\t\t\trc := opt.MakeRedisClient().(redis.UniversalClient)\n\t\t\tdefer rc.Close()\n\n\t\t\tif err := rc.Del(context.Background(), semaphoreKey(tt.name)).Err(); err != nil {\n\t\t\t\tt.Errorf(\"%s;\\nredis.UniversalClient.Del() got error %v\", tt.desc, err)\n\t\t\t}\n\n\t\t\tvar members []redis.Z\n\t\t\tfor i := 0; i < len(tt.taskIDs); i++ {\n\t\t\t\tmembers = append(members, redis.Z{\n\t\t\t\t\tScore: float64(time.Now().Add(time.Duration(i) * time.Second).Unix()),\n\t\t\t\t\tMember: tt.taskIDs[i],\n\t\t\t\t})\n\t\t\t}\n\t\t\tif err := rc.ZAdd(context.Background(), semaphoreKey(tt.name), members...).Err(); err != nil {\n\t\t\t\tt.Errorf(\"%s;\\nredis.UniversalClient.ZAdd() got error %v\", tt.desc, err)\n\t\t\t}\n\n\t\t\tsema := NewSemaphore(opt, tt.name, 3)\n\t\t\tdefer sema.Close()\n\n\t\t\tfor i := 0; i < len(tt.taskIDs); i++ {\n\t\t\t\tctx, cancel := tt.ctxFunc(tt.taskIDs[i])\n\n\t\t\t\tif err := sema.Release(ctx); err != nil {\n\t\t\t\t\tt.Errorf(\"%s;\\nSemaphore.Release() got error %v\", tt.desc, err)\n\t\t\t\t}\n\n\t\t\t\tcancel()\n\t\t\t}\n\n\t\t\ti, err := rc.ZCount(context.Background(), semaphoreKey(tt.name), \"-inf\", \"+inf\").Result()\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"%s;\\nredis.UniversalClient.ZCount() got error %v\", tt.desc, err)\n\t\t\t}\n\n\t\t\tif i != tt.wantCount {\n\t\t\t\tt.Errorf(\"%s;\\nSemaphore.Release(ctx) didn't release token, got %v want 0\", tt.desc, i)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestNewSemaphore_Release_Error(t *testing.T) {\n\ttestID := uuid.NewString()\n\n\ttests := []struct {\n\t\tdesc string\n\t\tname string\n\t\ttaskIDs []string\n\t\tctxFunc func(string) (context.Context, context.CancelFunc)\n\t\terrStr string\n\t}{\n\t\t{\n\t\t\tdesc: \"Should return error when context is missing taskID\",\n\t\t\tname: \"task-7\",\n\t\t\ttaskIDs: []string{uuid.NewString()},\n\t\t\tctxFunc: func(_ string) (context.Context, context.CancelFunc) {\n\t\t\t\treturn context.WithTimeout(context.Background(), time.Second)\n\t\t\t},\n\t\t\terrStr: \"provided context is missing task ID value\",\n\t\t},\n\t\t{\n\t\t\tdesc: \"Should return error when context has taskID which never acquired token\",\n\t\t\tname: \"task-8\",\n\t\t\ttaskIDs: []string{uuid.NewString()},\n\t\t\tctxFunc: func(_ string) (context.Context, context.CancelFunc) {\n\t\t\t\treturn asynqcontext.New(context.Background(), &base.TaskMessage{\n\t\t\t\t\tID: testID,\n\t\t\t\t\tQueue: \"task-4\",\n\t\t\t\t}, time.Now().Add(time.Second))\n\t\t\t},\n\t\t\terrStr: fmt.Sprintf(\"no token found for task %q\", testID),\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.desc, func(t *testing.T) {\n\t\t\topt := getRedisConnOpt(t)\n\t\t\trc := opt.MakeRedisClient().(redis.UniversalClient)\n\t\t\tdefer rc.Close()\n\n\t\t\tif err := rc.Del(context.Background(), semaphoreKey(tt.name)).Err(); err != nil {\n\t\t\t\tt.Errorf(\"%s;\\nredis.UniversalClient.Del() got error %v\", tt.desc, err)\n\t\t\t}\n\n\t\t\tvar members []redis.Z\n\t\t\tfor i := 0; i < len(tt.taskIDs); i++ {\n\t\t\t\tmembers = append(members, redis.Z{\n\t\t\t\t\tScore: float64(time.Now().Add(time.Duration(i) * time.Second).Unix()),\n\t\t\t\t\tMember: tt.taskIDs[i],\n\t\t\t\t})\n\t\t\t}\n\t\t\tif err := rc.ZAdd(context.Background(), semaphoreKey(tt.name), members...).Err(); err != nil {\n\t\t\t\tt.Errorf(\"%s;\\nredis.UniversalClient.ZAdd() got error %v\", tt.desc, err)\n\t\t\t}\n\n\t\t\tsema := NewSemaphore(opt, tt.name, 3)\n\t\t\tdefer sema.Close()\n\n\t\t\tfor i := 0; i < len(tt.taskIDs); i++ {\n\t\t\t\tctx, cancel := tt.ctxFunc(tt.taskIDs[i])\n\n\t\t\t\tif err := sema.Release(ctx); err == nil || err.Error() != tt.errStr {\n\t\t\t\t\tt.Errorf(\"%s;\\nSemaphore.Release() got error %v want error %v\", tt.desc, err, tt.errStr)\n\t\t\t\t}\n\n\t\t\t\tcancel()\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc getRedisConnOpt(tb testing.TB) asynq.RedisConnOpt {\n\ttb.Helper()\n\tif useRedisCluster {\n\t\taddrs := strings.Split(redisClusterAddrs, \",\")\n\t\tif len(addrs) == 0 {\n\t\t\ttb.Fatal(\"No redis cluster addresses provided. Please set addresses using --redis_cluster_addrs flag.\")\n\t\t}\n\t\treturn asynq.RedisClusterClientOpt{\n\t\t\tAddrs: addrs,\n\t\t}\n\t}\n\treturn asynq.RedisClientOpt{\n\t\tAddr: redisAddr,\n\t\tDB: redisDB,\n\t}\n}\n\ntype badConnOpt struct {\n}\n\nfunc (b badConnOpt) MakeRedisClient() interface{} {\n\treturn nil\n}\n" } ]