Repository: RichardKnop/machinery Branch: master Commit: 26dbe0308408 Files: 209 Total size: 966.3 KB Directory structure: gitextract_kvdc2vnc/ ├── .deepsource.toml ├── .gitignore ├── .travis.yml ├── Dockerfile.gcppubsub ├── Dockerfile.test ├── LICENSE ├── Makefile ├── README.md ├── docker-compose.test.yml ├── example/ │ ├── amqp/ │ │ └── main.go │ ├── redis/ │ │ └── main.go │ ├── tasks/ │ │ └── tasks.go │ └── tracers/ │ └── jaeger.go ├── go.mod ├── go.sum ├── instruction-notes/ │ └── dynamodb.md ├── integration-tests/ │ ├── amqp_amqp_test.go │ ├── amqp_get_pending_tasks_test.go │ ├── amqp_memcache_test.go │ ├── amqp_mongodb_test.go │ ├── amqp_redis_test.go │ ├── eager_eager_test.go │ ├── gcppubsub_redis_test.go │ ├── redis_get_pending_tasks_test.go │ ├── redis_memcache_test.go │ ├── redis_mongodb_test.go │ ├── redis_redis_test.go │ ├── redis_socket_test.go │ ├── sqs_amqp_test.go │ ├── sqs_mongodb_test.go │ ├── suite_test.go │ └── worker_only_consumes_registered_tasks_test.go ├── v1/ │ ├── backends/ │ │ ├── amqp/ │ │ │ ├── amqp.go │ │ │ └── amqp_test.go │ │ ├── dynamodb/ │ │ │ ├── dynamodb.go │ │ │ ├── dynamodb_export_test.go │ │ │ └── dynamodb_test.go │ │ ├── eager/ │ │ │ ├── eager.go │ │ │ └── eager_test.go │ │ ├── iface/ │ │ │ └── interfaces.go │ │ ├── memcache/ │ │ │ ├── memcache.go │ │ │ └── memcache_test.go │ │ ├── mongo/ │ │ │ ├── mongodb.go │ │ │ └── mongodb_test.go │ │ ├── null/ │ │ │ └── null.go │ │ ├── package.go │ │ ├── redis/ │ │ │ ├── goredis.go │ │ │ ├── goredis_test.go │ │ │ ├── redis.go │ │ │ └── redis_test.go │ │ └── result/ │ │ └── async_result.go │ ├── brokers/ │ │ ├── amqp/ │ │ │ ├── amqp.go │ │ │ ├── amqp_concurrence_test.go │ │ │ └── amqp_test.go │ │ ├── eager/ │ │ │ └── eager.go │ │ ├── errs/ │ │ │ └── errors.go │ │ ├── gcppubsub/ │ │ │ └── gcp_pubsub.go │ │ ├── iface/ │ │ │ └── interfaces.go │ │ ├── package.go │ │ ├── redis/ │ │ │ ├── goredis.go │ │ │ └── redis.go │ │ └── sqs/ │ │ ├── sqs.go │ │ ├── sqs_export_test.go │ │ └── sqs_test.go │ ├── common/ │ │ ├── amqp.go │ │ ├── backend.go │ │ ├── broker.go │ │ ├── broker_test.go │ │ └── redis.go │ ├── config/ │ │ ├── config.go │ │ ├── env.go │ │ ├── env_test.go │ │ ├── file.go │ │ ├── file_test.go │ │ ├── test.env │ │ └── testconfig.yml │ ├── factories.go │ ├── factories_test.go │ ├── locks/ │ │ ├── eager/ │ │ │ ├── eager.go │ │ │ └── eager_test.go │ │ ├── iface/ │ │ │ └── interfaces.go │ │ └── redis/ │ │ └── redis.go │ ├── log/ │ │ ├── log.go │ │ └── log_test.go │ ├── package.go │ ├── retry/ │ │ ├── fibonacci.go │ │ ├── fibonacci_test.go │ │ └── retry.go │ ├── server.go │ ├── server_test.go │ ├── tasks/ │ │ ├── errors.go │ │ ├── reflect.go │ │ ├── reflect_test.go │ │ ├── result.go │ │ ├── result_test.go │ │ ├── signature.go │ │ ├── state.go │ │ ├── state_test.go │ │ ├── task.go │ │ ├── task_test.go │ │ ├── validate.go │ │ ├── validate_test.go │ │ ├── workflow.go │ │ └── workflow_test.go │ ├── tracing/ │ │ └── tracing.go │ ├── utils/ │ │ ├── deepcopy.go │ │ ├── deepcopy_test.go │ │ ├── utils.go │ │ ├── utils_test.go │ │ ├── uuid.go │ │ └── uuid_test.go │ ├── worker.go │ └── worker_test.go ├── v2/ │ ├── Dockerfile.test │ ├── Makefile │ ├── backends/ │ │ ├── amqp/ │ │ │ ├── amqp.go │ │ │ └── amqp_test.go │ │ ├── dynamodb/ │ │ │ ├── dynamodb.go │ │ │ ├── dynamodb_export_test.go │ │ │ └── dynamodb_test.go │ │ ├── eager/ │ │ │ ├── eager.go │ │ │ └── eager_test.go │ │ ├── iface/ │ │ │ ├── dynamodb/ │ │ │ │ └── api.go │ │ │ └── interfaces.go │ │ ├── memcache/ │ │ │ ├── memcache.go │ │ │ └── memcache_test.go │ │ ├── mongo/ │ │ │ ├── mongodb.go │ │ │ └── mongodb_test.go │ │ ├── null/ │ │ │ └── null.go │ │ ├── package.go │ │ ├── redis/ │ │ │ ├── goredis.go │ │ │ ├── goredis_test.go │ │ │ ├── redis.go │ │ │ └── redis_test.go │ │ └── result/ │ │ └── async_result.go │ ├── brokers/ │ │ ├── amqp/ │ │ │ ├── amqp.go │ │ │ ├── amqp_concurrence_test.go │ │ │ └── amqp_test.go │ │ ├── eager/ │ │ │ └── eager.go │ │ ├── errs/ │ │ │ └── errors.go │ │ ├── gcppubsub/ │ │ │ └── gcp_pubsub.go │ │ ├── iface/ │ │ │ ├── interfaces.go │ │ │ └── sqs/ │ │ │ └── api.go │ │ ├── package.go │ │ ├── redis/ │ │ │ ├── goredis.go │ │ │ └── redis.go │ │ └── sqs/ │ │ ├── sqs.go │ │ ├── sqs_export_test.go │ │ └── sqs_test.go │ ├── common/ │ │ ├── amqp.go │ │ ├── backend.go │ │ ├── broker.go │ │ ├── broker_test.go │ │ └── redis.go │ ├── config/ │ │ ├── config.go │ │ ├── env.go │ │ ├── env_test.go │ │ ├── file.go │ │ ├── file_test.go │ │ ├── test.env │ │ └── testconfig.yml │ ├── docker-compose.test.yml │ ├── example/ │ │ ├── amqp/ │ │ │ └── main.go │ │ ├── go-redis/ │ │ │ └── main.go │ │ ├── redigo/ │ │ │ └── main.go │ │ ├── tasks/ │ │ │ └── tasks.go │ │ └── tracers/ │ │ └── jaeger.go │ ├── go.mod │ ├── go.sum │ ├── integration-tests/ │ │ ├── amqp_amqp_test.go │ │ ├── redis_redis_test.go │ │ └── suite_test.go │ ├── locks/ │ │ ├── eager/ │ │ │ ├── eager.go │ │ │ └── eager_test.go │ │ ├── iface/ │ │ │ └── interfaces.go │ │ └── redis/ │ │ └── redis.go │ ├── log/ │ │ ├── log.go │ │ └── log_test.go │ ├── package.go │ ├── retry/ │ │ ├── fibonacci.go │ │ ├── fibonacci_test.go │ │ └── retry.go │ ├── server.go │ ├── server_test.go │ ├── tasks/ │ │ ├── errors.go │ │ ├── reflect.go │ │ ├── reflect_test.go │ │ ├── result.go │ │ ├── result_test.go │ │ ├── signature.go │ │ ├── state.go │ │ ├── state_test.go │ │ ├── task.go │ │ ├── task_test.go │ │ ├── validate.go │ │ ├── validate_test.go │ │ ├── workflow.go │ │ └── workflow_test.go │ ├── tracing/ │ │ └── tracing.go │ ├── utils/ │ │ ├── deepcopy.go │ │ ├── deepcopy_test.go │ │ ├── utils.go │ │ ├── utils_test.go │ │ ├── uuid.go │ │ └── uuid_test.go │ ├── wait-for-it.sh │ ├── worker.go │ └── worker_test.go └── wait-for-it.sh ================================================ FILE CONTENTS ================================================ ================================================ FILE: .deepsource.toml ================================================ version = 1 test_patterns = ["**/*_test.go"] exclude_patterns = ["example/**"] [[analyzers]] name = "go" enabled = true [analyzers.meta] import_paths = ["github.com/RichardKnop/machinery"] ================================================ FILE: .gitignore ================================================ coverage* _vendor-* .idea/ .env .DS_Store dump.rdb ================================================ FILE: .travis.yml ================================================ --- language: go go: - 1.13.x env: - GO111MODULE=on services: - docker script: - make ci after_success: - bash <(curl -s https://codecov.io/bash) ================================================ FILE: Dockerfile.gcppubsub ================================================ FROM google/cloud-sdk:530.0.0-alpine RUN apk --update add openjdk8-jre RUN gcloud components install --quiet beta pubsub-emulator RUN mkdir -p /var/pubsub EXPOSE 8085 CMD [ "gcloud", "beta", "emulators", "pubsub", "start", "--host-port=0.0.0.0:8085"] ================================================ FILE: Dockerfile.test ================================================ # Start from a Debian image with the latest version of Go installed # and a workspace (GOPATH) configured at /go. FROM golang # Set environment variables ENV PATH /go/bin:$PATH # Cd into the source code directory WORKDIR /go/src/github.com/RichardKnop/machinery # Copy the local package files to the container's workspace. ADD . /go/src/github.com/RichardKnop/machinery # Run integration tests as default command CMD /go/src/github.com/RichardKnop/machinery/wait-for-it.sh rabbitmq:5672 -- make test-with-coverage ================================================ FILE: LICENSE ================================================ Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. ================================================ FILE: Makefile ================================================ .PHONY: fmt lint golint test test-with-coverage ci # TODO: When Go 1.9 is released vendor folder should be ignored automatically PACKAGES=`go list ./... | grep -v vendor | grep -v mocks` fmt: for pkg in ${PACKAGES}; do \ go fmt $$pkg; \ done; lint: gometalinter --tests --disable-all --deadline=120s -E vet -E gofmt -E misspell -E ineffassign -E goimports -E deadcode ./... golint: for pkg in ${PACKAGES}; do \ golint -set_exit_status $$pkg || GOLINT_FAILED=1; \ done; \ [ -z "$$GOLINT_FAILED" ] test: TEST_FAILED= ; \ for pkg in ${PACKAGES}; do \ go test $$pkg || TEST_FAILED=1; \ done; \ [ -z "$$TEST_FAILED" ] test-with-coverage: echo "" > coverage.out echo "mode: set" > coverage-all.out TEST_FAILED= ; \ for pkg in ${PACKAGES}; do \ go test -coverprofile=coverage.out -covermode=set $$pkg || TEST_FAILED=1; \ tail -n +2 coverage.out >> coverage-all.out; \ done; \ [ -z "$$TEST_FAILED" ] #go tool cover -html=coverage-all.out ci: bash -c 'docker-compose -f docker-compose.test.yml -p machinery_ci up --build --abort-on-container-exit --exit-code-from sut' ================================================ FILE: README.md ================================================ [1]: https://raw.githubusercontent.com/RichardKnop/assets/master/machinery/example_worker.png [2]: https://raw.githubusercontent.com/RichardKnop/assets/master/machinery/example_worker_receives_tasks.png [3]: http://patreon_public_assets.s3.amazonaws.com/sized/becomeAPatronBanner.png ## Machinery Machinery is an asynchronous task queue/job queue based on distributed message passing. [![godoc for RichardKnop/machinery](https://godoc.org/github.com/nathany/looper?status.svg)](http://godoc.org/github.com/RichardKnop/machinery/v1) [![codecov for RichardKnop/machinery](https://codecov.io/gh/RichardKnop/machinery/branch/master/graph/badge.svg)](https://codecov.io/gh/RichardKnop/machinery) [![Go Report Card](https://goreportcard.com/badge/github.com/RichardKnop/machinery)](https://goreportcard.com/report/github.com/RichardKnop/machinery) [![OpenTracing Badge](https://img.shields.io/badge/OpenTracing-enabled-blue.svg)](http://opentracing.io) [![Sourcegraph for RichardKnop/machinery](https://sourcegraph.com/github.com/RichardKnop/machinery/-/badge.svg)](https://sourcegraph.com/github.com/RichardKnop/machinery?badge) [![Donate Bitcoin](https://img.shields.io/badge/donate-bitcoin-orange.svg)](https://richardknop.github.io/donate/) --- * [V2 Experiment](#v2-experiment) * [First Steps](#first-steps) * [Configuration](#configuration) * [Lock](#lock) * [Broker](#broker) * [DefaultQueue](#defaultqueue) * [ResultBackend](#resultbackend) * [ResultsExpireIn](#resultsexpirein) * [AMQP](#amqp-2) * [DynamoDB](#dynamodb) * [Redis](#redis-2) * [GCPPubSub](#gcppubsub) * [Custom Logger](#custom-logger) * [Server](#server) * [Workers](#workers) * [Tasks](#tasks) * [Registering Tasks](#registering-tasks) * [Signatures](#signatures) * [Supported Types](#supported-types) * [Sending Tasks](#sending-tasks) * [Delayed Tasks](#delayed-tasks) * [Retry Tasks](#retry-tasks) * [Get Pending Tasks](#get-pending-tasks) * [Keeping Results](#keeping-results) * [Workflows](#workflows) * [Groups](#groups) * [Chords](#chords) * [Chains](#chains) * [Periodic Tasks & Workflows](#periodic-tasks--workflows) * [Periodic Tasks](#periodic-tasks) * [Periodic Groups](#periodic-groups) * [Periodic Chains](#periodic-chains) * [Periodic Chords](#periodic-chords) * [Development](#development) * [Requirements](#requirements) * [Dependencies](#dependencies) * [Testing](#testing) ### V2 I recommend using V2 in order to avoid having to import all dependencies for brokers and backends you are not using. Instead of factory, you will need to inject broker and backend objects to the server constructor: ```go import ( "github.com/RichardKnop/machinery/v2" backendsiface "github.com/RichardKnop/machinery/v2/backends/iface" brokersiface "github.com/RichardKnop/machinery/v2/brokers/iface" locksiface "github.com/RichardKnop/machinery/v2/locks/iface" ) var broker brokersiface.Broker var backend backendsiface.Backend var lock locksiface.Lock server := machinery.NewServer(cnf, broker, backend, lock) // server.NewWorker("machinery", 10) ``` ### First Steps To install recommended v2 release: ```sh go get github.com/RichardKnop/machinery/v2 ``` If you want to use legacy v1 version, you still can: ```sh go get github.com/RichardKnop/machinery ``` First, you will need to define some tasks. Look at sample tasks in `v2/example/tasks/tasks.go` to see a few examples. Second, you will need to launch a worker process with one of these commands (v2 is recommended since it doesn't import dependencies for all brokers / backends, only those you actually need): ```sh cd v2/ go run example/amqp/main.go worker go run example/redigo/main.go worker // Redis with redigo driver go run example/go-redis/main.go worker // Redis with Go Redis driver go run example/amqp/main.go worker go run example/redis/main.go worker ``` ![Example worker][1] Finally, once you have a worker running and waiting for tasks to consume, send some tasks with one of these commands (v2 is recommended since it doesn't import dependencies for all brokers / backends, only those you actually need): ```sh cd v2 go run v2/example/amqp/main.go send go run v2/example/redigo/main.go send // Redis with redigo driver go run v2/example/go-redis/main.go send // Redis with Go Redis driver ``` You will be able to see the tasks being processed asynchronously by the worker: ![Example worker receives tasks][2] ### Configuration The [config](/v2/config/config.go) package has convenience methods for loading configuration from environment variables or a YAML file. For example, load configuration from environment variables: ```go cnf, err := config.NewFromEnvironment() ``` Or load from YAML file: ```go cnf, err := config.NewFromYaml("config.yml", true) ``` Second boolean flag enables live reloading of configuration every 10 seconds. Use `false` to disable live reloading. Machinery configuration is encapsulated by a `Config` struct and injected as a dependency to objects that need it. #### Lock ##### Redis Use Redis URL in one of these formats: ``` redis://[password@]host[port][/db_num] ``` For example: 1. `redis://localhost:6379`, or with password `redis://password@localhost:6379` #### Broker A message broker. Currently supported brokers are: ##### AMQP Use AMQP URL in the format: ``` amqp://[username:password@]@host[:port] ``` For example: 1. `amqp://guest:guest@localhost:5672` AMQP also supports multiples brokers urls. You need to specify the URL separator in the `MultipleBrokerSeparator` field. ##### Redis Use Redis URL in one of these formats: ``` redis://[password@]host[port][/db_num] redis+socket://[password@]/path/to/file.sock[:/db_num] ``` For example: 1. `redis://localhost:6379`, or with password `redis://password@localhost:6379` 2. `redis+socket://password@/path/to/file.sock:/0` ##### AWS SQS Use AWS SQS URL in the format: ``` https://sqs.us-east-2.amazonaws.com/123456789012 ``` See [AWS SQS docs](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html) for more information. Also, configuring `AWS_REGION` is required, or an error would be thrown. To use a manually configured SQS Client: ```go var sqsClient = sqs.New(session.Must(session.NewSession(&aws.Config{ Region: aws.String("YOUR_AWS_REGION"), Credentials: credentials.NewStaticCredentials("YOUR_AWS_ACCESS_KEY", "YOUR_AWS_ACCESS_SECRET", ""), HTTPClient: &http.Client{ Timeout: time.Second * 120, }, }))) var visibilityTimeout = 20 var cnf = &config.Config{ Broker: "YOUR_SQS_URL" DefaultQueue: "machinery_tasks", ResultBackend: "YOUR_BACKEND_URL", SQS: &config.SQSConfig{ Client: sqsClient, // if VisibilityTimeout is nil default to the overall visibility timeout setting for the queue // https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html VisibilityTimeout: &visibilityTimeout, WaitTimeSeconds: 30, }, } ``` ##### GCP Pub/Sub Use GCP Pub/Sub URL in the format: ``` gcppubsub://YOUR_GCP_PROJECT_ID/YOUR_PUBSUB_SUBSCRIPTION_NAME ``` To use a manually configured Pub/Sub Client: ```go pubsubClient, err := pubsub.NewClient( context.Background(), "YOUR_GCP_PROJECT_ID", option.WithServiceAccountFile("YOUR_GCP_SERVICE_ACCOUNT_FILE"), ) cnf := &config.Config{ Broker: "gcppubsub://YOUR_GCP_PROJECT_ID/YOUR_PUBSUB_SUBSCRIPTION_NAME" DefaultQueue: "YOUR_PUBSUB_TOPIC_NAME", ResultBackend: "YOUR_BACKEND_URL", GCPPubSub: config.GCPPubSubConfig{ Client: pubsubClient, }, } ``` #### DefaultQueue Default queue name, e.g. `machinery_tasks`. #### ResultBackend Result backend to use for keeping task states and results. Currently supported backends are: ##### Redis Use Redis URL in one of these formats: ``` redis://[password@]host[port][/db_num] redis+socket://[password@]/path/to/file.sock[:/db_num] ``` For example: 1. `redis://localhost:6379`, or with password `redis://password@localhost:6379` 2. `redis+socket://password@/path/to/file.sock:/0` 3. cluster/sentinel `redis://host1:port1,host2:port2,host3:port3/0` 4. cluster/sentinel with password `redis://pass@host1:port1,host2:port2,host3:port3/0` ##### Memcache Use Memcache URL in the format: ``` memcache://host1[:port1][,host2[:port2],...[,hostN[:portN]]] ``` For example: 1. `memcache://localhost:11211` for a single instance, or 2. `memcache://10.0.0.1:11211,10.0.0.2:11211` for a cluster ##### AMQP Use AMQP URL in the format: ``` amqp://[username:password@]@host[:port] ``` For example: 1. `amqp://guest:guest@localhost:5672` > Keep in mind AMQP is not recommended as a result backend. See [Keeping Results](https://github.com/RichardKnop/machinery#keeping-results) ##### MongoDB Use Mongodb URL in the format: ``` mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]] ``` For example: 1. `mongodb://localhost:27017/taskresults` See [MongoDB docs](https://docs.mongodb.org/manual/reference/connection-string/) for more information. #### ResultsExpireIn How long to store task results for in seconds. Defaults to `3600` (1 hour). #### AMQP RabbitMQ related configuration. Not necessary if you are using other broker/backend. * `Exchange`: exchange name, e.g. `machinery_exchange` * `ExchangeType`: exchange type, e.g. `direct` * `QueueBindingArguments`: an optional map of additional arguments used when binding to an AMQP queue * `BindingKey`: The queue is bind to the exchange with this key, e.g. `machinery_task` * `PrefetchCount`: How many tasks to prefetch (set to `1` if you have long running tasks) * `DelayedQueue`: delayed queue name to be used for task retry or delayed task (if empty it will follow auto create and delate delayed queues) #### DynamoDB DynamoDB related configuration. Not necessary if you are using other backend. * `TaskStatesTable`: Custom table name for saving task states. Default one is `task_states`, and make sure to create this table in your AWS admin first, using `TaskUUID` as table's primary key. * `GroupMetasTable`: Custom table name for saving group metas. Default one is `group_metas`, and make sure to create this table in your AWS admin first, using `GroupUUID` as table's primary key. For example: ``` dynamodb: task_states_table: 'task_states' group_metas_table: 'group_metas' ``` If these tables are not found, an fatal error would be thrown. If you wish to expire the records, you can configure the `TTL` field in AWS admin for these tables. The `TTL` field is set based on the `ResultsExpireIn` value in the Server's config. See https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/howitworks-ttl.html for more information. #### Redis Redis related configuration. Not necessary if you are using other backend. See: [config](/v1/config/config.go) (TODO) #### GCPPubSub GCPPubSub related configuration. Not necessary if you are using other backend. See: [config](/v1/config/config.go) (TODO) ### Custom Logger You can define a custom logger by implementing the following interface: ```go type Interface interface { Print(...interface{}) Printf(string, ...interface{}) Println(...interface{}) Fatal(...interface{}) Fatalf(string, ...interface{}) Fatalln(...interface{}) Panic(...interface{}) Panicf(string, ...interface{}) Panicln(...interface{}) } ``` Then just set the logger in your setup code by calling `Set` function exported by `github.com/RichardKnop/machinery/v1/log` package: ```go log.Set(myCustomLogger) ``` ### Server A Machinery library must be instantiated before use. The way this is done is by creating a `Server` instance. `Server` is a base object which stores Machinery configuration and registered tasks. E.g.: ```go import ( "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1" ) var cnf = &config.Config{ Broker: "amqp://guest:guest@localhost:5672/", DefaultQueue: "machinery_tasks", ResultBackend: "amqp://guest:guest@localhost:5672/", AMQP: &config.AMQPConfig{ Exchange: "machinery_exchange", ExchangeType: "direct", BindingKey: "machinery_task", }, } server, err := machinery.NewServer(cnf) if err != nil { // do something with the error } ``` ### Workers In order to consume tasks, you need to have one or more workers running. All you need to run a worker is a `Server` instance with registered tasks. E.g.: ```go worker := server.NewWorker("worker_name", 10) err := worker.Launch() if err != nil { // do something with the error } ``` Each worker will only consume registered tasks. For each task on the queue the Worker.Process() method will be run in a goroutine. Use the second parameter of `server.NewWorker` to limit the number of concurrently running Worker.Process() calls (per worker). Example: 1 will serialize task execution while 0 makes the number of concurrently executed tasks unlimited (default). ### Tasks Tasks are a building block of Machinery applications. A task is a function which defines what happens when a worker receives a message. Each task needs to return an error as a last return value. In addition to error tasks can now return any number of arguments. Examples of valid tasks: ```go func Add(args ...int64) (int64, error) { sum := int64(0) for _, arg := range args { sum += arg } return sum, nil } func Multiply(args ...int64) (int64, error) { sum := int64(1) for _, arg := range args { sum *= arg } return sum, nil } // You can use context.Context as first argument to tasks, useful for open tracing func TaskWithContext(ctx context.Context, arg Arg) error { // ... use ctx ... return nil } // Tasks need to return at least error as a minimal requirement func DummyTask(arg string) error { return errors.New(arg) } // You can also return multiple results from the task func DummyTask2(arg1, arg2 string) (string, string, error) { return arg1, arg2, nil } ``` #### Registering Tasks Before your workers can consume a task, you need to register it with the server. This is done by assigning a task a unique name: ```go server.RegisterTasks(map[string]interface{}{ "add": Add, "multiply": Multiply, }) ``` Tasks can also be registered one by one: ```go server.RegisterTask("add", Add) server.RegisterTask("multiply", Multiply) ``` Simply put, when a worker receives a message like this: ```json { "UUID": "48760a1a-8576-4536-973b-da09048c2ac5", "Name": "add", "RoutingKey": "", "ETA": null, "GroupUUID": "", "GroupTaskCount": 0, "Args": [ { "Type": "int64", "Value": 1, }, { "Type": "int64", "Value": 1, } ], "Immutable": false, "RetryCount": 0, "RetryTimeout": 0, "OnSuccess": null, "OnError": null, "ChordCallback": null } ``` It will call Add(1, 1). Each task should return an error as well so we can handle failures. Ideally, tasks should be idempotent which means there will be no unintended consequences when a task is called multiple times with the same arguments. #### Signatures A signature wraps calling arguments, execution options (such as immutability) and success/error callbacks of a task so it can be sent across the wire to workers. Task signatures implement a simple interface: ```go // Arg represents a single argument passed to invocation fo a task type Arg struct { Type string Value interface{} } // Headers represents the headers which should be used to direct the task type Headers map[string]interface{} // Signature represents a single task invocation type Signature struct { UUID string Name string RoutingKey string ETA *time.Time GroupUUID string GroupTaskCount int Args []Arg Headers Headers Immutable bool RetryCount int RetryTimeout int OnSuccess []*Signature OnError []*Signature ChordCallback *Signature } ``` `UUID` is a unique ID of a task. You can either set it yourself or it will be automatically generated. `Name` is the unique task name by which it is registered against a Server instance. `RoutingKey` is used for routing a task to correct queue. If you leave it empty, the default behaviour will be to set it to the default queue's binding key for direct exchange type and to the default queue name for other exchange types. `ETA` is a timestamp used for delaying a task. if it's nil, the task will be published for workers to consume immediately. If it is set, the task will be delayed until the ETA timestamp. `GroupUUID`, `GroupTaskCount` are useful for creating groups of tasks. `Args` is a list of arguments that will be passed to the task when it is executed by a worker. `Headers` is a list of headers that will be used when publishing the task to AMQP queue. `Immutable` is a flag which defines whether a result of the executed task can be modified or not. This is important with `OnSuccess` callbacks. Immutable task will not pass its result to its success callbacks while a mutable task will prepend its result to args sent to callback tasks. Long story short, set Immutable to false if you want to pass result of the first task in a chain to the second task. `RetryCount` specifies how many times a failed task should be retried (defaults to 0). Retry attempts will be spaced out in time, after each failure another attempt will be scheduled further to the future. `RetryTimeout` specifies how long to wait before resending task to the queue for retry attempt. Default behaviour is to use fibonacci sequence to increase the timeout after each failed retry attempt. `OnSuccess` defines tasks which will be called after the task has executed successfully. It is a slice of task signature structs. `OnError` defines tasks which will be called after the task execution fails. The first argument passed to error callbacks will be the error string returned from the failed task. `ChordCallback` is used to create a callback to a group of tasks. #### Supported Types Machinery encodes tasks to JSON before sending them to the broker. Task results are also stored in the backend as JSON encoded strings. Therefor only types with native JSON representation can be supported. Currently supported types are: * `bool` * `int` * `int8` * `int16` * `int32` * `int64` * `uint` * `uint8` * `uint16` * `uint32` * `uint64` * `float32` * `float64` * `string` * `[]bool` * `[]int` * `[]int8` * `[]int16` * `[]int32` * `[]int64` * `[]uint` * `[]uint8` * `[]uint16` * `[]uint32` * `[]uint64` * `[]float32` * `[]float64` * `[]string` #### Sending Tasks Tasks can be called by passing an instance of `Signature` to an `Server` instance. E.g: ```go import ( "github.com/RichardKnop/machinery/v1/tasks" ) signature := &tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 1, }, { Type: "int64", Value: 1, }, }, } asyncResult, err := server.SendTask(signature) if err != nil { // failed to send the task // do something with the error } ``` #### Delayed Tasks You can delay a task by setting the `ETA` timestamp field on the task signature. ```go // Delay the task by 5 seconds eta := time.Now().UTC().Add(time.Second * 5) signature.ETA = &eta ``` #### Retry Tasks You can set a number of retry attempts before declaring task as failed. Fibonacci sequence will be used to space out retry requests over time. (See `RetryTimeout` for details.) ```go // If the task fails, retry it up to 3 times signature.RetryCount = 3 ``` Alternatively, you can return `tasks.ErrRetryTaskLater` from your task and specify duration after which the task should be retried, e.g.: ```go return tasks.NewErrRetryTaskLater("some error", 4 * time.Hour) ``` #### Get Pending Tasks Tasks currently waiting in the queue to be consumed by workers can be inspected, e.g.: ```go server.GetBroker().GetPendingTasks("some_queue") ``` > Currently only supported by Redis broker. #### Keeping Results If you configure a result backend, the task states and results will be persisted. Possible states: ```go const ( // StatePending - initial state of a task StatePending = "PENDING" // StateReceived - when task is received by a worker StateReceived = "RECEIVED" // StateStarted - when the worker starts processing the task StateStarted = "STARTED" // StateRetry - when failed task has been scheduled for retry StateRetry = "RETRY" // StateSuccess - when the task is processed successfully StateSuccess = "SUCCESS" // StateFailure - when processing of the task fails StateFailure = "FAILURE" ) ``` > When using AMQP as a result backend, task states will be persisted in separate queues for each task. Although RabbitMQ can scale up to thousands of queues, it is strongly advised to use a better suited result backend (e.g. Memcache) when you are expecting to run a large number of parallel tasks. ```go // TaskResult represents an actual return value of a processed task type TaskResult struct { Type string `bson:"type"` Value interface{} `bson:"value"` } // TaskState represents a state of a task type TaskState struct { TaskUUID string `bson:"_id"` State string `bson:"state"` Results []*TaskResult `bson:"results"` Error string `bson:"error"` } // GroupMeta stores useful metadata about tasks within the same group // E.g. UUIDs of all tasks which are used in order to check if all tasks // completed successfully or not and thus whether to trigger chord callback type GroupMeta struct { GroupUUID string `bson:"_id"` TaskUUIDs []string `bson:"task_uuids"` ChordTriggered bool `bson:"chord_triggered"` Lock bool `bson:"lock"` } ``` `TaskResult` represents a slice of return values of a processed task. `TaskState` struct will be serialized and stored every time a task state changes. `GroupMeta` stores useful metadata about tasks within the same group. E.g. UUIDs of all tasks which are used in order to check if all tasks completed successfully or not and thus whether to trigger chord callback. `AsyncResult` object allows you to check for the state of a task: ```go taskState := asyncResult.GetState() fmt.Printf("Current state of %v task is:\n", taskState.TaskUUID) fmt.Println(taskState.State) ``` There are couple of convenient methods to inspect the task status: ```go asyncResult.GetState().IsCompleted() asyncResult.GetState().IsSuccess() asyncResult.GetState().IsFailure() ``` You can also do a synchronous blocking call to wait for a task result: ```go results, err := asyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { // getting result of a task failed // do something with the error } for _, result := range results { fmt.Println(result.Interface()) } ``` #### Error Handling When a task returns with an error, the default behavior is to first attempty to retry the task if it's retriable, otherwise log the error and then eventually call any error callbacks. To customize this, you can set a custom error handler on the worker which can do more than just logging after retries fail and error callbacks are trigerred: ```go worker.SetErrorHandler(func (err error) { customHandler(err) }) ``` ### Workflows Running a single asynchronous task is fine but often you will want to design a workflow of tasks to be executed in an orchestrated way. There are couple of useful functions to help you design workflows. #### Groups `Group` is a set of tasks which will be executed in parallel, independent of each other. E.g.: ```go import ( "github.com/RichardKnop/machinery/v1/tasks" "github.com/RichardKnop/machinery/v1" ) signature1 := tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 1, }, { Type: "int64", Value: 1, }, }, } signature2 := tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 5, }, { Type: "int64", Value: 5, }, }, } group, _ := tasks.NewGroup(&signature1, &signature2) asyncResults, err := server.SendGroup(group, 0) //The second parameter specifies the number of concurrent sending tasks. 0 means unlimited. if err != nil { // failed to send the group // do something with the error } ``` `SendGroup` returns a slice of `AsyncResult` objects. So you can do a blocking call and wait for the result of groups tasks: ```go for _, asyncResult := range asyncResults { results, err := asyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { // getting result of a task failed // do something with the error } for _, result := range results { fmt.Println(result.Interface()) } } ``` #### Chords `Chord` allows you to define a callback to be executed after all tasks in a group finished processing, e.g.: ```go import ( "github.com/RichardKnop/machinery/v1/tasks" "github.com/RichardKnop/machinery/v1" ) signature1 := tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 1, }, { Type: "int64", Value: 1, }, }, } signature2 := tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 5, }, { Type: "int64", Value: 5, }, }, } signature3 := tasks.Signature{ Name: "multiply", } group := tasks.NewGroup(&signature1, &signature2) chord, _ := tasks.NewChord(group, &signature3) chordAsyncResult, err := server.SendChord(chord, 0) //The second parameter specifies the number of concurrent sending tasks. 0 means unlimited. if err != nil { // failed to send the chord // do something with the error } ``` The above example executes task1 and task2 in parallel, aggregates their results and passes them to task3. Therefore what would end up happening is: ``` multiply(add(1, 1), add(5, 5)) ``` More explicitly: ``` (1 + 1) * (5 + 5) = 2 * 10 = 20 ``` `SendChord` returns `ChordAsyncResult` which follows AsyncResult's interface. So you can do a blocking call and wait for the result of the callback: ```go results, err := chordAsyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { // getting result of a chord failed // do something with the error } for _, result := range results { fmt.Println(result.Interface()) } ``` #### Chains `Chain` is simply a set of tasks which will be executed one by one, each successful task triggering the next task in the chain. E.g.: ```go import ( "github.com/RichardKnop/machinery/v1/tasks" "github.com/RichardKnop/machinery/v1" ) signature1 := tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 1, }, { Type: "int64", Value: 1, }, }, } signature2 := tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 5, }, { Type: "int64", Value: 5, }, }, } signature3 := tasks.Signature{ Name: "multiply", Args: []tasks.Arg{ { Type: "int64", Value: 4, }, }, } chain, _ := tasks.NewChain(&signature1, &signature2, &signature3) chainAsyncResult, err := server.SendChain(chain) if err != nil { // failed to send the chain // do something with the error } ``` The above example executes task1, then task2 and then task3. When a task is completed successfully, the result is appended to the end of list of arguments for the next task in the chain. Therefore what would end up happening is: ``` multiply(4, add(5, 5, add(1, 1))) ``` More explicitly: ``` 4 * (5 + 5 + (1 + 1)) # task1: add(1, 1) returns 2 = 4 * (5 + 5 + 2) # task2: add(5, 5, 2) returns 12 = 4 * (12) # task3: multiply(4, 12) returns 48 = 48 ``` `SendChain` returns `ChainAsyncResult` which follows AsyncResult's interface. So you can do a blocking call and wait for the result of the whole chain: ```go results, err := chainAsyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { // getting result of a chain failed // do something with the error } for _, result := range results { fmt.Println(result.Interface()) } ``` ### Periodic Tasks & Workflows Machinery now supports scheduling periodic tasks and workflows. See examples bellow. #### Periodic Tasks ```go import ( "github.com/RichardKnop/machinery/v1/tasks" ) signature := &tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 1, }, { Type: "int64", Value: 1, }, }, } err := server.RegisterPeriodicTask("0 6 * * ?", "periodic-task", signature) if err != nil { // failed to register periodic task } ``` #### Periodic Groups ```go import ( "github.com/RichardKnop/machinery/v1/tasks" "github.com/RichardKnop/machinery/v1" ) signature1 := tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 1, }, { Type: "int64", Value: 1, }, }, } signature2 := tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 5, }, { Type: "int64", Value: 5, }, }, } group, _ := tasks.NewGroup(&signature1, &signature2) err := server.RegisterPeriodicGroup("0 6 * * ?", "periodic-group", group) if err != nil { // failed to register periodic group } ``` #### Periodic Chains ```go import ( "github.com/RichardKnop/machinery/v1/tasks" "github.com/RichardKnop/machinery/v1" ) signature1 := tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 1, }, { Type: "int64", Value: 1, }, }, } signature2 := tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 5, }, { Type: "int64", Value: 5, }, }, } signature3 := tasks.Signature{ Name: "multiply", Args: []tasks.Arg{ { Type: "int64", Value: 4, }, }, } chain, _ := tasks.NewChain(&signature1, &signature2, &signature3) err := server.RegisterPeriodicChain("0 6 * * ?", "periodic-chain", chain) if err != nil { // failed to register periodic chain } ``` #### Chord ```go import ( "github.com/RichardKnop/machinery/v1/tasks" "github.com/RichardKnop/machinery/v1" ) signature1 := tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 1, }, { Type: "int64", Value: 1, }, }, } signature2 := tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 5, }, { Type: "int64", Value: 5, }, }, } signature3 := tasks.Signature{ Name: "multiply", } group := tasks.NewGroup(&signature1, &signature2) chord, _ := tasks.NewChord(group, &signature3) err := server.RegisterPeriodicChord("0 6 * * ?", "periodic-chord", chord) if err != nil { // failed to register periodic chord } ``` ### Development #### Requirements * Go * RabbitMQ (optional) * Redis * Memcached (optional) * MongoDB (optional) On OS X systems, you can install requirements using [Homebrew](http://brew.sh/): ```sh brew install go brew install rabbitmq brew install redis brew install memcached brew install mongodb ``` Or optionally use the corresponding [Docker](http://docker.io/) containers: ``` docker run -d -p 5672:5672 rabbitmq docker run -d -p 6379:6379 redis docker run -d -p 11211:11211 memcached docker run -d -p 27017:27017 mongo docker run -d -p 6831:6831/udp -p 16686:16686 jaegertracing/all-in-one:latest ``` #### Dependencies Since Go 1.11, a new recommended dependency management system is via [modules](https://github.com/golang/go/wiki/Modules). This is one of slight weaknesses of Go as dependency management is not a solved problem. Previously Go was officially recommending to use the [dep tool](https://github.com/golang/dep) but that has been abandoned now in favor of modules. #### Testing Easiest (and platform agnostic) way to run tests is via `docker-compose`: ```sh make ci ``` This will basically run docker-compose command: ```sh (docker-compose -f docker-compose.test.yml -p machinery_ci up --build -d) && (docker logs -f machinery_sut &) && (docker wait machinery_sut) ``` Alternative approach is to setup a development environment on your machine. In order to enable integration tests, you will need to install all required services (RabbitMQ, Redis, Memcache, MongoDB) and export these environment variables: ```sh export AMQP_URL=amqp://guest:guest@localhost:5672/ export REDIS_URL=localhost:6379 export MEMCACHE_URL=localhost:11211 export MONGODB_URL=localhost:27017 ``` To run integration tests against an SQS instance, you will need to create a "test_queue" in SQS and export these environment variables: ```sh export SQS_URL=https://YOUR_SQS_URL export AWS_ACCESS_KEY_ID=YOUR_AWS_ACCESS_KEY_ID export AWS_SECRET_ACCESS_KEY=YOUR_AWS_SECRET_ACCESS_KEY export AWS_DEFAULT_REGION=YOUR_AWS_DEFAULT_REGION ``` Then just run: ```sh make test ``` If the environment variables are not exported, `make test` will only run unit tests. ================================================ FILE: docker-compose.test.yml ================================================ version: "2" services: sut: container_name: machinery_sut image: machinery_sut:latest volumes: - "./:/go/src/github.com/RichardKnop/machinery" depends_on: - rabbitmq - redis - memcached - mongo - gcppubsub links: - rabbitmq - redis - memcached - mongo - gcppubsub build: context: . dockerfile: ./Dockerfile.test environment: AMQP_URLS: 'amqp://guest:guest@dummy:5672/,amqp://guest:guest@rabbitmq:5672/' AMQP_URLS_SEPARATOR: ',' REDIS_URL: 'redis:6379' MEMCACHE_URL: 'memcached:11211' MONGODB_URL: 'mongo:27017' SQS_URL: ${SQS_URL} AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION} AWS_REGION: 'us-west-2' GCPPUBSUB_URL: 'gcppubsub://example-project/test_subscription_queue' GCPPUBSUB_TOPIC: 'test_topic_queue' PUBSUB_EMULATOR_HOST: 'gcppubsub:8085' rabbitmq: container_name: machinery_sut_rabbitmq image: rabbitmq environment: - RABBITMQ_DEFAULT_USER=guest - RABBITMQ_DEFAULT_PASS=guest logging: driver: none redis: container_name: machinery_sut_redis image: redis logging: driver: none memcached: container_name: machinery_sut_memcached image: memcached logging: driver: none mongo: container_name: machinery_sut_mongo image: mongo logging: driver: none gcppubsub: container_name: machinery_sut_gcppubsub build: context: . dockerfile: ./Dockerfile.gcppubsub logging: driver: none ================================================ FILE: example/amqp/main.go ================================================ package main import ( "context" "errors" "fmt" "os" "time" "github.com/google/uuid" "github.com/urfave/cli" "github.com/RichardKnop/machinery/v1" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/log" "github.com/RichardKnop/machinery/v1/tasks" exampletasks "github.com/RichardKnop/machinery/example/tasks" tracers "github.com/RichardKnop/machinery/example/tracers" opentracing "github.com/opentracing/opentracing-go" opentracing_log "github.com/opentracing/opentracing-go/log" ) var ( app *cli.App ) func init() { // Initialise a CLI app app = cli.NewApp() app.Name = "machinery" app.Usage = "machinery worker and send example tasks with machinery send" app.Version = "0.0.0" } func main() { // Set the CLI app commands app.Commands = []cli.Command{ { Name: "worker", Usage: "launch machinery worker", Action: func(c *cli.Context) error { if err := worker(); err != nil { return cli.NewExitError(err.Error(), 1) } return nil }, }, { Name: "send", Usage: "send example tasks ", Action: func(c *cli.Context) error { if err := send(); err != nil { return cli.NewExitError(err.Error(), 1) } return nil }, }, } // Run the CLI app app.Run(os.Args) } func startServer() (*machinery.Server, error) { cnf := &config.Config{ Broker: "amqp://guest:guest@localhost:5672/", DefaultQueue: "machinery_tasks", ResultBackend: "amqp://guest:guest@localhost:5672/", ResultsExpireIn: 3600, AMQP: &config.AMQPConfig{ Exchange: "machinery_exchange", ExchangeType: "direct", BindingKey: "machinery_task", PrefetchCount: 3, }, } server, err := machinery.NewServer(cnf) if err != nil { return nil, err } // Register tasks tasks := map[string]interface{}{ "add": exampletasks.Add, "multiply": exampletasks.Multiply, "sum_ints": exampletasks.SumInts, "sum_floats": exampletasks.SumFloats, "concat": exampletasks.Concat, "split": exampletasks.Split, "panic_task": exampletasks.PanicTask, "long_running_task": exampletasks.LongRunningTask, } return server, server.RegisterTasks(tasks) } func worker() error { consumerTag := "machinery_worker" cleanup, err := tracers.SetupTracer(consumerTag) if err != nil { log.FATAL.Fatalln("Unable to instantiate a tracer:", err) } defer cleanup() server, err := startServer() if err != nil { return err } // The second argument is a consumer tag // Ideally, each worker should have a unique tag (worker1, worker2 etc) worker := server.NewWorker(consumerTag, 0) // Here we inject some custom code for error handling, // start and end of task hooks, useful for metrics for example. errorhandler := func(err error) { log.ERROR.Println("I am an error handler:", err) } pretaskhandler := func(signature *tasks.Signature) { log.INFO.Println("I am a start of task handler for:", signature.Name) } posttaskhandler := func(signature *tasks.Signature) { log.INFO.Println("I am an end of task handler for:", signature.Name) } worker.SetPostTaskHandler(posttaskhandler) worker.SetErrorHandler(errorhandler) worker.SetPreTaskHandler(pretaskhandler) return worker.Launch() } func send() error { cleanup, err := tracers.SetupTracer("sender") if err != nil { log.FATAL.Fatalln("Unable to instantiate a tracer:", err) } defer cleanup() server, err := startServer() if err != nil { return err } var ( addTask0, addTask1, addTask2 tasks.Signature multiplyTask0, multiplyTask1 tasks.Signature sumIntsTask, sumFloatsTask, concatTask, splitTask tasks.Signature panicTask tasks.Signature longRunningTask tasks.Signature ) var initTasks = func() { addTask0 = tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 1, }, { Type: "int64", Value: 1, }, }, } addTask1 = tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 2, }, { Type: "int64", Value: 2, }, }, } addTask2 = tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 5, }, { Type: "int64", Value: 6, }, }, } multiplyTask0 = tasks.Signature{ Name: "multiply", Args: []tasks.Arg{ { Type: "int64", Value: 4, }, }, } multiplyTask1 = tasks.Signature{ Name: "multiply", } sumIntsTask = tasks.Signature{ Name: "sum_ints", Args: []tasks.Arg{ { Type: "[]int64", Value: []int64{1, 2}, }, }, } sumFloatsTask = tasks.Signature{ Name: "sum_floats", Args: []tasks.Arg{ { Type: "[]float64", Value: []float64{1.5, 2.7}, }, }, } concatTask = tasks.Signature{ Name: "concat", Args: []tasks.Arg{ { Type: "[]string", Value: []string{"foo", "bar"}, }, }, } splitTask = tasks.Signature{ Name: "split", Args: []tasks.Arg{ { Type: "string", Value: "foo", }, }, } panicTask = tasks.Signature{ Name: "panic_task", } longRunningTask = tasks.Signature{ Name: "long_running_task", } } /* * Lets start a span representing this run of the `send` command and * set a batch id as baggage so it can travel all the way into * the worker functions. */ span, ctx := opentracing.StartSpanFromContext(context.Background(), "send") defer span.Finish() batchID := uuid.New().String() span.SetBaggageItem("batch.id", batchID) span.LogFields(opentracing_log.String("batch.id", batchID)) log.INFO.Println("Starting batch:", batchID) /* * First, let's try sending a single task */ initTasks() log.INFO.Println("Single task:") asyncResult, err := server.SendTaskWithContext(ctx, &addTask0) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err := asyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf("1 + 1 = %v\n", tasks.HumanReadableResults(results)) /* * Try couple of tasks with a slice argument and slice return value */ asyncResult, err = server.SendTaskWithContext(ctx, &sumIntsTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err = asyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf("sum([1, 2]) = %v\n", tasks.HumanReadableResults(results)) asyncResult, err = server.SendTaskWithContext(ctx, &sumFloatsTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err = asyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf("sum([1.5, 2.7]) = %v\n", tasks.HumanReadableResults(results)) asyncResult, err = server.SendTaskWithContext(ctx, &concatTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err = asyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf("concat([\"foo\", \"bar\"]) = %v\n", tasks.HumanReadableResults(results)) asyncResult, err = server.SendTaskWithContext(ctx, &splitTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err = asyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf("split([\"foo\"]) = %v\n", tasks.HumanReadableResults(results)) /* * Now let's explore ways of sending multiple tasks */ // Now let's try a parallel execution initTasks() log.INFO.Println("Group of tasks (parallel execution):") group, err := tasks.NewGroup(&addTask0, &addTask1, &addTask2) if err != nil { return fmt.Errorf("Error creating group: %s", err.Error()) } asyncResults, err := server.SendGroupWithContext(ctx, group, 10) if err != nil { return fmt.Errorf("Could not send group: %s", err.Error()) } for _, asyncResult := range asyncResults { results, err = asyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf( "%v + %v = %v\n", asyncResult.Signature.Args[0].Value, asyncResult.Signature.Args[1].Value, tasks.HumanReadableResults(results), ) } // Now let's try a group with a chord initTasks() log.INFO.Println("Group of tasks with a callback (chord):") group, err = tasks.NewGroup(&addTask0, &addTask1, &addTask2) if err != nil { return fmt.Errorf("Error creating group: %s", err.Error()) } chord, err := tasks.NewChord(group, &multiplyTask1) if err != nil { return fmt.Errorf("Error creating chord: %s", err) } chordAsyncResult, err := server.SendChordWithContext(ctx, chord, 10) if err != nil { return fmt.Errorf("Could not send chord: %s", err.Error()) } results, err = chordAsyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { return fmt.Errorf("Getting chord result failed with error: %s", err.Error()) } log.INFO.Printf("(1 + 1) * (2 + 2) * (5 + 6) = %v\n", tasks.HumanReadableResults(results)) // Now let's try chaining task results initTasks() log.INFO.Println("Chain of tasks:") chain, err := tasks.NewChain(&addTask0, &addTask1, &addTask2, &multiplyTask0) if err != nil { return fmt.Errorf("Error creating chain: %s", err) } chainAsyncResult, err := server.SendChainWithContext(ctx, chain) if err != nil { return fmt.Errorf("Could not send chain: %s", err.Error()) } results, err = chainAsyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { return fmt.Errorf("Getting chain result failed with error: %s", err.Error()) } log.INFO.Printf("(((1 + 1) + (2 + 2)) + (5 + 6)) * 4 = %v\n", tasks.HumanReadableResults(results)) // Let's try a task which throws panic to make sure stack trace is not lost initTasks() asyncResult, err = server.SendTaskWithContext(ctx, &panicTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } _, err = asyncResult.Get(time.Duration(time.Millisecond * 5)) if err == nil { return errors.New("Error should not be nil if task panicked") } log.INFO.Printf("Task panicked and returned error = %v\n", err.Error()) // Let's try a long running task initTasks() asyncResult, err = server.SendTaskWithContext(ctx, &longRunningTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err = asyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { return fmt.Errorf("Getting long running task result failed with error: %s", err.Error()) } log.INFO.Printf("Long running task returned = %v\n", tasks.HumanReadableResults(results)) return nil } ================================================ FILE: example/redis/main.go ================================================ package main import ( "context" "errors" "fmt" "os" "time" "github.com/google/uuid" "github.com/urfave/cli" "github.com/RichardKnop/machinery/v1" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/log" "github.com/RichardKnop/machinery/v1/tasks" exampletasks "github.com/RichardKnop/machinery/example/tasks" tracers "github.com/RichardKnop/machinery/example/tracers" opentracing "github.com/opentracing/opentracing-go" opentracing_log "github.com/opentracing/opentracing-go/log" ) var ( app *cli.App ) func init() { // Initialise a CLI app app = cli.NewApp() app.Name = "machinery" app.Usage = "machinery worker and send example tasks with machinery send" app.Version = "0.0.0" } func main() { // Set the CLI app commands app.Commands = []cli.Command{ { Name: "worker", Usage: "launch machinery worker", Action: func(c *cli.Context) error { if err := worker(); err != nil { return cli.NewExitError(err.Error(), 1) } return nil }, }, { Name: "send", Usage: "send example tasks ", Action: func(c *cli.Context) error { if err := send(); err != nil { return cli.NewExitError(err.Error(), 1) } return nil }, }, } // Run the CLI app app.Run(os.Args) } func startServer() (*machinery.Server, error) { cnf := &config.Config{ DefaultQueue: "machinery_tasks", ResultsExpireIn: 3600, Broker: "redis://localhost:6379", ResultBackend: "redis://localhost:6379", Redis: &config.RedisConfig{ MaxIdle: 3, IdleTimeout: 240, ReadTimeout: 15, WriteTimeout: 15, ConnectTimeout: 15, NormalTasksPollPeriod: 1000, DelayedTasksPollPeriod: 500, }, } server, err := machinery.NewServer(cnf) if err != nil { return nil, err } // Register tasks tasks := map[string]interface{}{ "add": exampletasks.Add, "multiply": exampletasks.Multiply, "sum_ints": exampletasks.SumInts, "sum_floats": exampletasks.SumFloats, "concat": exampletasks.Concat, "split": exampletasks.Split, "panic_task": exampletasks.PanicTask, "long_running_task": exampletasks.LongRunningTask, } return server, server.RegisterTasks(tasks) } func worker() error { consumerTag := "machinery_worker" cleanup, err := tracers.SetupTracer(consumerTag) if err != nil { log.FATAL.Fatalln("Unable to instantiate a tracer:", err) } defer cleanup() server, err := startServer() if err != nil { return err } // The second argument is a consumer tag // Ideally, each worker should have a unique tag (worker1, worker2 etc) worker := server.NewWorker(consumerTag, 0) // Here we inject some custom code for error handling, // start and end of task hooks, useful for metrics for example. errorhandler := func(err error) { log.ERROR.Println("I am an error handler:", err) } pretaskhandler := func(signature *tasks.Signature) { log.INFO.Println("I am a start of task handler for:", signature.Name) } posttaskhandler := func(signature *tasks.Signature) { log.INFO.Println("I am an end of task handler for:", signature.Name) } worker.SetPostTaskHandler(posttaskhandler) worker.SetErrorHandler(errorhandler) worker.SetPreTaskHandler(pretaskhandler) return worker.Launch() } func send() error { cleanup, err := tracers.SetupTracer("sender") if err != nil { log.FATAL.Fatalln("Unable to instantiate a tracer:", err) } defer cleanup() server, err := startServer() if err != nil { return err } var ( addTask0, addTask1, addTask2 tasks.Signature multiplyTask0, multiplyTask1 tasks.Signature sumIntsTask, sumFloatsTask, concatTask, splitTask tasks.Signature panicTask tasks.Signature longRunningTask tasks.Signature ) var initTasks = func() { addTask0 = tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 1, }, { Type: "int64", Value: 1, }, }, } addTask1 = tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 2, }, { Type: "int64", Value: 2, }, }, } addTask2 = tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 5, }, { Type: "int64", Value: 6, }, }, } multiplyTask0 = tasks.Signature{ Name: "multiply", Args: []tasks.Arg{ { Type: "int64", Value: 4, }, }, } multiplyTask1 = tasks.Signature{ Name: "multiply", } sumIntsTask = tasks.Signature{ Name: "sum_ints", Args: []tasks.Arg{ { Type: "[]int64", Value: []int64{1, 2}, }, }, } sumFloatsTask = tasks.Signature{ Name: "sum_floats", Args: []tasks.Arg{ { Type: "[]float64", Value: []float64{1.5, 2.7}, }, }, } concatTask = tasks.Signature{ Name: "concat", Args: []tasks.Arg{ { Type: "[]string", Value: []string{"foo", "bar"}, }, }, } splitTask = tasks.Signature{ Name: "split", Args: []tasks.Arg{ { Type: "string", Value: "foo", }, }, } panicTask = tasks.Signature{ Name: "panic_task", } longRunningTask = tasks.Signature{ Name: "long_running_task", } } /* * Lets start a span representing this run of the `send` command and * set a batch id as baggage so it can travel all the way into * the worker functions. */ span, ctx := opentracing.StartSpanFromContext(context.Background(), "send") defer span.Finish() batchID := uuid.New().String() span.SetBaggageItem("batch.id", batchID) span.LogFields(opentracing_log.String("batch.id", batchID)) log.INFO.Println("Starting batch:", batchID) /* * First, let's try sending a single task */ initTasks() log.INFO.Println("Single task:") asyncResult, err := server.SendTaskWithContext(ctx, &addTask0) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err := asyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf("1 + 1 = %v\n", tasks.HumanReadableResults(results)) /* * Try couple of tasks with a slice argument and slice return value */ asyncResult, err = server.SendTaskWithContext(ctx, &sumIntsTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err = asyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf("sum([1, 2]) = %v\n", tasks.HumanReadableResults(results)) asyncResult, err = server.SendTaskWithContext(ctx, &sumFloatsTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err = asyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf("sum([1.5, 2.7]) = %v\n", tasks.HumanReadableResults(results)) asyncResult, err = server.SendTaskWithContext(ctx, &concatTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err = asyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf("concat([\"foo\", \"bar\"]) = %v\n", tasks.HumanReadableResults(results)) asyncResult, err = server.SendTaskWithContext(ctx, &splitTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err = asyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf("split([\"foo\"]) = %v\n", tasks.HumanReadableResults(results)) /* * Now let's explore ways of sending multiple tasks */ // Now let's try a parallel execution initTasks() log.INFO.Println("Group of tasks (parallel execution):") group, err := tasks.NewGroup(&addTask0, &addTask1, &addTask2) if err != nil { return fmt.Errorf("Error creating group: %s", err.Error()) } asyncResults, err := server.SendGroupWithContext(ctx, group, 10) if err != nil { return fmt.Errorf("Could not send group: %s", err.Error()) } for _, asyncResult := range asyncResults { results, err = asyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf( "%v + %v = %v\n", asyncResult.Signature.Args[0].Value, asyncResult.Signature.Args[1].Value, tasks.HumanReadableResults(results), ) } // Now let's try a group with a chord initTasks() log.INFO.Println("Group of tasks with a callback (chord):") group, err = tasks.NewGroup(&addTask0, &addTask1, &addTask2) if err != nil { return fmt.Errorf("Error creating group: %s", err.Error()) } chord, err := tasks.NewChord(group, &multiplyTask1) if err != nil { return fmt.Errorf("Error creating chord: %s", err) } chordAsyncResult, err := server.SendChordWithContext(ctx, chord, 10) if err != nil { return fmt.Errorf("Could not send chord: %s", err.Error()) } results, err = chordAsyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { return fmt.Errorf("Getting chord result failed with error: %s", err.Error()) } log.INFO.Printf("(1 + 1) * (2 + 2) * (5 + 6) = %v\n", tasks.HumanReadableResults(results)) // Now let's try chaining task results initTasks() log.INFO.Println("Chain of tasks:") chain, err := tasks.NewChain(&addTask0, &addTask1, &addTask2, &multiplyTask0) if err != nil { return fmt.Errorf("Error creating chain: %s", err) } chainAsyncResult, err := server.SendChainWithContext(ctx, chain) if err != nil { return fmt.Errorf("Could not send chain: %s", err.Error()) } results, err = chainAsyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { return fmt.Errorf("Getting chain result failed with error: %s", err.Error()) } log.INFO.Printf("(((1 + 1) + (2 + 2)) + (5 + 6)) * 4 = %v\n", tasks.HumanReadableResults(results)) // Let's try a task which throws panic to make sure stack trace is not lost initTasks() asyncResult, err = server.SendTaskWithContext(ctx, &panicTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } _, err = asyncResult.Get(time.Duration(time.Millisecond * 5)) if err == nil { return errors.New("Error should not be nil if task panicked") } log.INFO.Printf("Task panicked and returned error = %v\n", err.Error()) // Let's try a long running task initTasks() asyncResult, err = server.SendTaskWithContext(ctx, &longRunningTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err = asyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { return fmt.Errorf("Getting long running task result failed with error: %s", err.Error()) } log.INFO.Printf("Long running task returned = %v\n", tasks.HumanReadableResults(results)) return nil } ================================================ FILE: example/tasks/tasks.go ================================================ package exampletasks import ( "errors" "strings" "time" "github.com/RichardKnop/machinery/v1/log" ) // Add ... func Add(args ...int64) (int64, error) { sum := int64(0) for _, arg := range args { sum += arg } return sum, nil } // Multiply ... func Multiply(args ...int64) (int64, error) { sum := int64(1) for _, arg := range args { sum *= arg } return sum, nil } // SumInts ... func SumInts(numbers []int64) (int64, error) { var sum int64 for _, num := range numbers { sum += num } return sum, nil } // SumFloats ... func SumFloats(numbers []float64) (float64, error) { var sum float64 for _, num := range numbers { sum += num } return sum, nil } // Concat ... func Concat(strs []string) (string, error) { var res string for _, s := range strs { res += s } return res, nil } // Split ... func Split(str string) ([]string, error) { return strings.Split(str, ""), nil } // PanicTask ... func PanicTask() (string, error) { panic(errors.New("oops")) } // LongRunningTask ... func LongRunningTask() error { log.INFO.Print("Long running task started") for i := 0; i < 10; i++ { log.INFO.Print(10 - i) time.Sleep(1 * time.Second) } log.INFO.Print("Long running task finished") return nil } ================================================ FILE: example/tracers/jaeger.go ================================================ package tracers // Uncomment the import statement for the jaeger tracer. // make sure you run dep ensure to pull in the jaeger client // // import ( // jaeger "github.com/uber/jaeger-client-go" // jaegercfg "github.com/uber/jaeger-client-go/config" // ) // SetupTracer is the place where you'd setup your specific tracer. // The jaeger tracer is given as an example. // To capture the jaeger traces you should run the jaeger backend. // This can be done using the following docker command: // // `docker run -ti --rm -p6831:6831/udp -p16686:16686 jaegertracing/all-in-one:latest` // // The collector will be listening on localhost:6831 // and the query UI is reachable on localhost:16686. func SetupTracer(serviceName string) (func(), error) { // Jaeger setup code // // config := jaegercfg.Configuration{ // Sampler: &jaegercfg.SamplerConfig{ // Type: jaeger.SamplerTypeConst, // Param: 1, // }, // } // closer, err := config.InitGlobalTracer(serviceName) // if err != nil { // return nil, err // } cleanupFunc := func() { // closer.Close() } return cleanupFunc, nil } ================================================ FILE: go.mod ================================================ module github.com/RichardKnop/machinery go 1.22 toolchain go1.24.1 require ( cloud.google.com/go/pubsub v1.10.0 github.com/RichardKnop/logging v0.0.0-20190827224416-1a693bdd4fae github.com/aws/aws-sdk-go v1.55.6 github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b github.com/go-redsync/redsync/v4 v4.8.1 github.com/gomodule/redigo v1.9.2 github.com/google/uuid v1.2.0 github.com/kelseyhightower/envconfig v1.4.0 github.com/opentracing/opentracing-go v1.2.0 github.com/pkg/errors v0.9.1 github.com/rabbitmq/amqp091-go v1.9.0 github.com/redis/go-redis/v9 v9.0.5 github.com/robfig/cron/v3 v3.0.1 github.com/stretchr/testify v1.8.4 github.com/urfave/cli v1.22.5 go.mongodb.org/mongo-driver v1.17.0 gopkg.in/yaml.v2 v2.4.0 ) require ( cloud.google.com/go v0.76.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/googleapis/gax-go/v2 v2.0.5 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jstemmer/go-junit-report v0.9.1 // indirect github.com/klauspost/compress v1.13.6 // indirect github.com/montanaflynn/stats v0.7.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect go.opencensus.io v0.22.6 // indirect golang.org/x/crypto v0.26.0 // indirect golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/net v0.25.0 // indirect golang.org/x/oauth2 v0.0.0-20210201163806-010130855d6c // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.23.0 // indirect golang.org/x/text v0.17.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect google.golang.org/api v0.39.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20210207032614-bba0dbe2a9ea // indirect google.golang.org/grpc v1.35.0 // indirect google.golang.org/protobuf v1.26.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) replace git.apache.org/thrift.git => github.com/apache/thrift v0.0.0-20180902110319-2566ecd5d999 ================================================ FILE: go.sum ================================================ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go v0.76.0 h1:Ckw+E/QYZgd/5bpI4wz4h6f+jmpvh9S9uSrKNnbicJI= cloud.google.com/go v0.76.0/go.mod h1:r9EvIAvLrunusnetGdQ50M/gKui1x3zdGW/VELGkdpw= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/pubsub v1.10.0 h1:JK22g5uNpscGPthjJE/D0siWtA6UlU4Cb6pLcyJkzyQ= cloud.google.com/go/pubsub v1.10.0/go.mod h1:eNpTrkOy7dCpkNyaSNetMa6udbgecJMd0ZsTJS/cuNo= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/RichardKnop/logging v0.0.0-20190827224416-1a693bdd4fae h1:DcFpTQBYQ9Ct2d6sC7ol0/ynxc2pO1cpGUM+f4t5adg= github.com/RichardKnop/logging v0.0.0-20190827224416-1a693bdd4fae/go.mod h1:rJJ84PyA/Wlmw1hO+xTzV2wsSUon6J5ktg0g8BF2PuU= github.com/aws/aws-sdk-go v1.55.6 h1:cSg4pvZ3m8dgYcgqB97MrcdjUmZ1BeMYKUxMMB89IPk= github.com/aws/aws-sdk-go v1.55.6/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b h1:L/QXpzIa3pOvUGt1D1lA5KjYhPBAN/3iWdP7xeFS9F0= github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= github.com/bsm/ginkgo/v2 v2.5.0/go.mod h1:AiKlXPm7ItEHNc/2+OkrNG4E0ITzojb9/xWzvQ9XZ9w= github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao= github.com/bsm/ginkgo/v2 v2.7.0/go.mod h1:AiKlXPm7ItEHNc/2+OkrNG4E0ITzojb9/xWzvQ9XZ9w= github.com/bsm/gomega v1.20.0/go.mod h1:JifAceMQ4crZIWYUKrlGcmbN3bqHogVTADMD2ATsbwk= github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= github.com/bsm/gomega v1.26.0/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-redis/redis/v7 v7.4.0 h1:7obg6wUoj05T0EpY0o8B59S9w5yeMWql7sw2kwNW1x4= github.com/go-redis/redis/v7 v7.4.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg= github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg= github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= github.com/go-redsync/redsync/v4 v4.8.1 h1:rq2RvdTI0obznMdxKUWGdmmulo7lS9yCzb8fgDKOlbM= github.com/go-redsync/redsync/v4 v4.8.1/go.mod h1:LmUAsQuQxhzZAoGY7JS6+dNhNmZyonMZiiEDY9plotM= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/gomodule/redigo v1.9.2 h1:HrutZBLhSIU8abiSfW8pj8mPhOyMYjZT/wcA4/L9L9s= github.com/gomodule/redigo v1.9.2/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rabbitmq/amqp091-go v1.9.0 h1:qrQtyzB4H8BQgEuJwhmVQqVHB9O4+MNDJCCAcpc3Aoo= github.com/rabbitmq/amqp091-go v1.9.0/go.mod h1:+jPrT9iY2eLjRaMSRHUhc3z14E/l85kv/f+6luSD3pc= github.com/redis/go-redis/v9 v9.0.2/go.mod h1:/xDTe9EF1LM61hek62Poq2nzQSGj0xSrEtEHbBQevps= github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o= github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 h1:QVqDTf3h2WHt08YuiTGPZLls0Wq99X9bWd0Q5ZSBesM= github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203/go.mod h1:oqN97ltKNihBbwlX8dLpwxCl3+HnXKV/R0e+sRLd9C8= github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mongodb.org/mongo-driver v1.17.0 h1:Hp4q2MCjvY19ViwimTs00wHi7G4yzxh4/2+nTx8r40k= go.mongodb.org/mongo-driver v1.17.0/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.22.6 h1:BdkrbWrzDlV9dnbzoP7sfN+dHheJ4J9JOaYxcUDL+ok= go.opencensus.io v0.22.6/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210113205817-d3ed898aa8a3/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210201163806-010130855d6c h1:HiAZXo96zOhVhtFHchj/ojzoxCFiPrp9/j0GtS38V3g= golang.org/x/oauth2 v0.0.0-20210201163806-010130855d6c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.38.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.39.0 h1:zHCTXf0NeDdKTgcSQpT+ZflWAqHsEp1GmdpxW09f3YM= google.golang.org/api v0.39.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210202153253-cf70463f6119/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210207032614-bba0dbe2a9ea h1:N98SvVh7Hdle2lgUVFuIkf0B3u29CUakMUQa7Hwz8Wc= google.golang.org/genproto v0.0.0-20210207032614-bba0dbe2a9ea/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= ================================================ FILE: instruction-notes/dynamodb.md ================================================ # Using DynamoDB as a result backend ## What is DynamoDB Amazon DynamoDB is a fast and flexible NoSQL database service. Check this [official website](https://aws.amazon.com/dynamodb/ ) for details. ## How to use DynamoDB as a result backend in Machinery ### Create two tables first There will be two tables required now(2018-01-12): * group_metas: A table which saves group tasks' meta data. The primary key for this table is ```GroupUUID```, and it should be set properly when creating this table. * task_states: A table which saves every task's states. The primary key for this table is ```TaskUUID```, and it should be set properly when creating this table. ### Add DynamoDB config to the config file #### example config ```yaml broker: 'https://sqs.us-west-1.amazonaws.com/123456789012' default_queue: machinery-queue result_backend: 'https://dynamodb.us-west-1.amazonaws.com/123456789012' results_expire_in: 3600 dynamodb: task_states_table: 'task_states' group_metas_table: 'group_metas' ``` Then DynamoDB will be used as a result backend. ================================================ FILE: integration-tests/amqp_amqp_test.go ================================================ package integration_test import ( "os" "testing" "github.com/RichardKnop/machinery/v1" "github.com/RichardKnop/machinery/v1/config" ) func TestAmqpAmqp(t *testing.T) { amqpURL := os.Getenv("AMQP_URL") if amqpURL == "" { t.Skip("AMQP_URL is not defined") } finalAmqpURL := amqpURL var finalSeparator string amqpURLs := os.Getenv("AMQP_URLS") if amqpURLs != "" { separator := os.Getenv("AMQP_URLS_SEPARATOR") if separator == "" { return } finalSeparator = separator finalAmqpURL = amqpURLs } // AMQP broker, AMQP result backend server := testSetup(&config.Config{ Broker: finalAmqpURL, MultipleBrokerSeparator: finalSeparator, DefaultQueue: "test_queue", ResultBackend: amqpURL, Lock: "eager", AMQP: &config.AMQPConfig{ Exchange: "test_exchange", ExchangeType: "direct", BindingKey: "test_task", PrefetchCount: 1, }, }) worker := server.(*machinery.Server).NewWorker("test_worker", 0) defer worker.Quit() go worker.Launch() testAll(server, t) } ================================================ FILE: integration-tests/amqp_get_pending_tasks_test.go ================================================ package integration_test import ( "fmt" "os" "testing" "time" "github.com/RichardKnop/machinery/v1" "github.com/RichardKnop/machinery/v1/backends/result" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/tasks" ) func TestAmqpGetPendingTasks(t *testing.T) { amqpURL := os.Getenv("AMQP_URL") if amqpURL == "" { t.Skip("AMQP_URL is not defined") } finalAmqpURL := amqpURL var finalSeparator string amqpURLs := os.Getenv("AMQP_URLS") if amqpURLs != "" { separator := os.Getenv("AMQP_URLS_SEPARATOR") if separator == "" { return } finalSeparator = separator finalAmqpURL = amqpURLs } redisURL := os.Getenv("REDIS_URL") if redisURL == "" { t.Skip("REDIS_URL is not defined") } // AMQP broker, AMQP result backend server := testSetup(&config.Config{ Broker: finalAmqpURL, MultipleBrokerSeparator: finalSeparator, DefaultQueue: "test_queue", ResultBackend: amqpURL, Lock: fmt.Sprintf("redis://%v", redisURL), AMQP: &config.AMQPConfig{ Exchange: "test_exchange", ExchangeType: "direct", BindingKey: "test_task", PrefetchCount: 1, }, }) var results []*result.AsyncResult signatures := []*tasks.Signature{newAddTask(1, 2), newAddTask(3, 5), newAddTask(6, 7)} for _, s := range signatures { ar, err := server.SendTask(s) if err != nil { t.Error(err) } results = append(results, ar) } pendingMessages, err := server.GetBroker().GetPendingTasks(server.GetConfig().DefaultQueue) if err != nil { t.Error(err) } if len(pendingMessages) != len(signatures) { t.Errorf( "%d pending messages, should be %d", len(pendingMessages), len(signatures), ) } for i := 0; i < len(signatures); i++ { compareSigs(t, signatures[i], pendingMessages[i]) } worker := server.(*machinery.Server).NewWorker("test_worker", 0) go worker.Launch() defer worker.Quit() for _, r := range results { r.Get(time.Duration(time.Millisecond * 5)) } pendingMessages, err = server.GetBroker().GetPendingTasks(server.GetConfig().DefaultQueue) if err != nil { t.Error(err) } if len(pendingMessages) != 0 { t.Errorf( "%d pending messages, should be 0", len(pendingMessages), ) } } func compareSigs(t *testing.T, a *tasks.Signature, b *tasks.Signature) { if a.UUID != b.UUID { t.Errorf("UUID mismatch, %v != %v", a.UUID, b.UUID) } if a.Name != b.Name { t.Errorf("UUID mismatch, %v != %v", a.Name, b.Name) } if len(a.Args) != len(b.Args) { t.Errorf("Arg length mismatch, %v != %v", len(a.Args), len(b.Args)) } } ================================================ FILE: integration-tests/amqp_memcache_test.go ================================================ package integration_test import ( "fmt" "os" "testing" "github.com/RichardKnop/machinery/v1" "github.com/RichardKnop/machinery/v1/config" ) func TestAmqpMemcache(t *testing.T) { amqpURL := os.Getenv("AMQP_URL") memcacheURL := os.Getenv("MEMCACHE_URL") if amqpURL == "" { t.Skip("AMQP_URL is not defined") } if memcacheURL == "" { t.Skip("MEMCACHE_URL is not defined") } redisURL := os.Getenv("REDIS_URL") if redisURL == "" { t.Skip("REDIS_URL is not defined") } // AMQP broker, Memcache result backend server := testSetup(&config.Config{ Broker: amqpURL, DefaultQueue: "test_queue", ResultBackend: fmt.Sprintf("memcache://%v", memcacheURL), Lock: fmt.Sprintf("redis://%v", redisURL), AMQP: &config.AMQPConfig{ Exchange: "test_exchange", ExchangeType: "direct", BindingKey: "test_task", PrefetchCount: 1, }, }) worker := server.(*machinery.Server).NewWorker("test_worker", 0) defer worker.Quit() go worker.Launch() testAll(server, t) } ================================================ FILE: integration-tests/amqp_mongodb_test.go ================================================ package integration_test import ( "fmt" "os" "testing" "github.com/RichardKnop/machinery/v1" "github.com/RichardKnop/machinery/v1/config" ) func TestAmqpMongodb(t *testing.T) { amqpURL := os.Getenv("AMQP_URL") mongodbURL := os.Getenv("MONGODB_URL") if amqpURL == "" { t.Skip("AMQP_URL is not defined") } if mongodbURL == "" { t.Skip("MONGODB_URL is not defined") } // AMQP broker, MongoDB result backend server := testSetup(&config.Config{ Broker: amqpURL, DefaultQueue: "test_queue", ResultsExpireIn: 30, ResultBackend: fmt.Sprintf("mongodb://%v", mongodbURL), Lock: "eager", AMQP: &config.AMQPConfig{ Exchange: "test_exchange", ExchangeType: "direct", BindingKey: "test_task", PrefetchCount: 1, }, }) worker := server.(*machinery.Server).NewWorker("test_worker", 0) defer worker.Quit() go worker.Launch() testAll(server, t) } ================================================ FILE: integration-tests/amqp_redis_test.go ================================================ package integration_test import ( "fmt" "os" "testing" "github.com/RichardKnop/machinery/v1" "github.com/RichardKnop/machinery/v1/config" ) func TestAmqpRedis(t *testing.T) { amqpURL := os.Getenv("AMQP_URL") redisURL := os.Getenv("REDIS_URL") if amqpURL == "" { t.Skip("AMQP_URL is not defined") } if redisURL == "" { t.Skip("REDIS_URL is not defined") } // AMQP broker, Redis result backend server := testSetup(&config.Config{ Broker: amqpURL, DefaultQueue: "test_queue", ResultBackend: fmt.Sprintf("redis://%v", redisURL), Lock: "eager", AMQP: &config.AMQPConfig{ Exchange: "test_exchange", ExchangeType: "direct", BindingKey: "test_task", PrefetchCount: 1, }, }) worker := server.(*machinery.Server).NewWorker("test_worker", 0) defer worker.Quit() go worker.Launch() testAll(server, t) } ================================================ FILE: integration-tests/eager_eager_test.go ================================================ package integration_test import ( "reflect" "testing" "time" "github.com/RichardKnop/machinery/v1" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/tasks" "github.com/stretchr/testify/suite" ) type EagerIntegrationTestSuite struct { suite.Suite srv *machinery.Server called float64 } func TestEagerIntegrationTestSuite(t *testing.T) { suite.Run(t, &EagerIntegrationTestSuite{}) } func (s *EagerIntegrationTestSuite) SetupSuite() { var err error // init server cnf := config.Config{ Broker: "eager", ResultBackend: "eager", Lock: "eager", } s.srv, err = machinery.NewServer(&cnf) s.Nil(err) s.NotNil(s.srv) // register task s.called = 0 s.srv.RegisterTask("float_called", func(i float64) (float64, error) { s.called = i return s.called, nil }) s.srv.RegisterTask("float_result", func(i float64) (float64, error) { return i + 100.0, nil }) s.srv.RegisterTask("int_result", func(i int64) (int64, error) { return i + 100, nil }) } func (s *EagerIntegrationTestSuite) TestCalled() { _, err := s.srv.SendTask(&tasks.Signature{ Name: "float_called", Args: []tasks.Arg{ { Type: "float64", Value: 100.0, }, }, }) s.Nil(err) s.Equal(100.0, s.called) } func (s *EagerIntegrationTestSuite) TestSuccessResult() { // float64 { asyncResult, err := s.srv.SendTask(&tasks.Signature{ Name: "float_result", Args: []tasks.Arg{ { Type: "float64", Value: 100.0, }, }, }) s.NotNil(asyncResult) s.Nil(err) s.True(asyncResult.GetState().IsCompleted()) s.True(asyncResult.GetState().IsSuccess()) results, err := asyncResult.Get(time.Duration(time.Millisecond * 5)) if s.NoError(err) { if len(results) != 1 { s.T().Errorf("Number of results returned = %d. Wanted %d", len(results), 1) } s.Equal(reflect.Float64, results[0].Kind()) if results[0].Kind() == reflect.Float64 { s.Equal(200.0, results[0].Float()) } } } // int { asyncResult, err := s.srv.SendTask(&tasks.Signature{ Name: "int_result", Args: []tasks.Arg{ { Type: "int64", Value: 100, }, }, }) s.NotNil(asyncResult) s.Nil(err) s.True(asyncResult.GetState().IsCompleted()) s.True(asyncResult.GetState().IsSuccess()) results, err := asyncResult.Get(time.Duration(time.Millisecond * 5)) if s.NoError(err) { if len(results) != 1 { s.T().Errorf("Number of results returned = %d. Wanted %d", len(results), 1) } s.Equal(reflect.Int64, results[0].Kind()) if results[0].Kind() == reflect.Int64 { s.Equal(int64(200), results[0].Int()) } } } } ================================================ FILE: integration-tests/gcppubsub_redis_test.go ================================================ package integration_test import ( "context" "fmt" "os" "testing" "time" "cloud.google.com/go/pubsub" "github.com/RichardKnop/machinery/v1" "github.com/RichardKnop/machinery/v1/config" ) func createGCPPubSubTopicAndSubscription(cli *pubsub.Client, topicName, subscriptionName string) { ctx := context.Background() var topic *pubsub.Topic topic = cli.Topic(topicName) topicExists, err := topic.Exists(ctx) if err != nil { panic(err) } if !topicExists { topic, err = cli.CreateTopic(ctx, topicName) if err != nil { panic(err) } } var sub *pubsub.Subscription sub = cli.Subscription(subscriptionName) subExists, err := sub.Exists(ctx) if err != nil { panic(err) } if !subExists { _, err = cli.CreateSubscription(ctx, subscriptionName, pubsub.SubscriptionConfig{ Topic: topic, AckDeadline: 10 * time.Second, }) if err != nil { panic(err) } } } func TestGCPPubSubRedis(t *testing.T) { // start Cloud Pub/Sub emulator // $ LANG=C gcloud beta emulators pubsub start // $ eval $(LANG=C gcloud beta emulators pubsub env-init) pubsubURL := os.Getenv("GCPPUBSUB_URL") if pubsubURL == "" { t.Skip("GCPPUBSUB_URL is not defined") } topicName := os.Getenv("GCPPUBSUB_TOPIC") if topicName == "" { t.Skip("GCPPUBSUB_TOPIC is not defined") } _, subscriptionName, err := machinery.ParseGCPPubSubURL(pubsubURL) if err != nil { t.Fatal(err) } redisURL := os.Getenv("REDIS_URL") if redisURL == "" { t.Skip("REDIS_URL is not defined") } pubsubClient, err := pubsub.NewClient(context.Background(), "") if err != nil { t.Fatal(err) } // Create Cloud Pub/Sub Topic and Subscription createGCPPubSubTopicAndSubscription(pubsubClient, topicName, subscriptionName) // Redis broker, Redis result backend server := testSetup(&config.Config{ Broker: pubsubURL, DefaultQueue: topicName, ResultBackend: fmt.Sprintf("redis://%v", redisURL), Lock: fmt.Sprintf("redis://%v", redisURL), GCPPubSub: &config.GCPPubSubConfig{ Client: pubsubClient, }, }) worker := server.(*machinery.Server).NewWorker("test_worker", 0) defer worker.Quit() go worker.Launch() testAll(server, t) } ================================================ FILE: integration-tests/redis_get_pending_tasks_test.go ================================================ package integration_test import ( "fmt" "os" "testing" "github.com/RichardKnop/machinery/v1/config" ) func TestRedisGetPendingTasks(t *testing.T) { redisURL := os.Getenv("REDIS_URL") if redisURL == "" { t.Skip("REDIS_URL is not defined") } // Redis broker, Redis result backend server := testSetup(&config.Config{ Broker: fmt.Sprintf("redis://%v", redisURL), DefaultQueue: "test_queue", ResultBackend: fmt.Sprintf("redis://%v", redisURL), Lock: fmt.Sprintf("redis://%v", redisURL), }) pendingMessages, err := server.GetBroker().GetPendingTasks(server.GetConfig().DefaultQueue) if err != nil { t.Error(err) } if len(pendingMessages) != 0 { t.Errorf( "%d pending messages, should be %d", len(pendingMessages), 0, ) } } ================================================ FILE: integration-tests/redis_memcache_test.go ================================================ package integration_test import ( "fmt" "os" "testing" "github.com/RichardKnop/machinery/v1" "github.com/RichardKnop/machinery/v1/config" ) func TestRedisMemcache(t *testing.T) { redisURL := os.Getenv("REDIS_URL") memcacheURL := os.Getenv("MEMCACHE_URL") if redisURL == "" { t.Skip("REDIS_URL is not defined") } if memcacheURL == "" { t.Skip("MEMCACHE_URL is not defined") } // Redis broker, Redis result backend server := testSetup(&config.Config{ Broker: fmt.Sprintf("redis://%v", redisURL), DefaultQueue: "test_queue", ResultBackend: fmt.Sprintf("memcache://%v", memcacheURL), Lock: fmt.Sprintf("redis://%v", redisURL), }) worker := server.(*machinery.Server).NewWorker("test_worker", 0) defer worker.Quit() go worker.Launch() testAll(server, t) } ================================================ FILE: integration-tests/redis_mongodb_test.go ================================================ package integration_test import ( "fmt" "os" "testing" "github.com/RichardKnop/machinery/v1" "github.com/RichardKnop/machinery/v1/config" ) func TestRedisMongodb(t *testing.T) { redisURL := os.Getenv("REDIS_URL") mongodbURL := os.Getenv("MONGODB_URL") if redisURL == "" { t.Skip("REDIS_URL is not defined") } if mongodbURL == "" { t.Skip("MONGODB_URL is not defined") } // Redis broker, MongoDB result backend server := testSetup(&config.Config{ Broker: fmt.Sprintf("redis://%v", redisURL), DefaultQueue: "test_queue", ResultsExpireIn: 30, ResultBackend: fmt.Sprintf("mongodb://%v", mongodbURL), Lock: fmt.Sprintf("redis://%v", redisURL), }) worker := server.(*machinery.Server).NewWorker("test_worker", 0) defer worker.Quit() go worker.Launch() testAll(server, t) } ================================================ FILE: integration-tests/redis_redis_test.go ================================================ package integration_test import ( "errors" "fmt" "os" "testing" "time" "github.com/RichardKnop/machinery/v1" "github.com/RichardKnop/machinery/v1/config" ) func TestRedisRedis_Redigo(t *testing.T) { redisURL := os.Getenv("REDIS_URL") if redisURL == "" { t.Skip("REDIS_URL is not defined") } // Redis broker, Redis result backend server := testSetup(&config.Config{ Broker: fmt.Sprintf("redis://%v", redisURL), DefaultQueue: "test_queue", ResultBackend: fmt.Sprintf("redis://%v", redisURL), Lock: fmt.Sprintf("redis://%v", redisURL), }) worker := server.(*machinery.Server).NewWorker("test_worker", 0) defer worker.Quit() go worker.Launch() testAll(server, t) } func TestRedisRedisNormalTaskPollPeriodLessThan1SecondShouldNotFailNextTask(t *testing.T) { redisURL := os.Getenv("REDIS_URL") if redisURL == "" { t.Skip("REDIS_URL is not defined") } // Redis broker, Redis result backend server := testSetup(&config.Config{ Broker: fmt.Sprintf("redis://%v", redisURL), DefaultQueue: "test_queue", ResultBackend: fmt.Sprintf("redis://%v", redisURL), Lock: fmt.Sprintf("redis://%v", redisURL), Redis: &config.RedisConfig{ NormalTasksPollPeriod: 10, // 10 milliseconds }, }) worker := server.(*machinery.Server).NewWorker("test_worker", 0) go worker.Launch() defer worker.Quit() testSendTask(server, t) } func TestRedisRedisWorkerQuitRaceCondition(t *testing.T) { repeat := 3 for i := 0; i < repeat; i++ { redisURL := os.Getenv("REDIS_URL") if redisURL == "" { t.Skip("REDIS_URL is not defined") } // Redis broker, Redis result backend cnf := &config.Config{ Broker: fmt.Sprintf("redis://%v", redisURL), DefaultQueue: "test_queue", ResultBackend: fmt.Sprintf("redis://%v", redisURL), Lock: fmt.Sprintf("redis://%v", redisURL), } server, _ := machinery.NewServer(cnf) worker := server.NewWorker("test_worker", 0) errorsChan := make(chan error, 1) // Check Quit() immediately after LaunchAsync() will shutdown gracefully // and not panic on close(b.stopChan) worker.LaunchAsync(errorsChan) worker.Quit() if err := <-errorsChan; err != nil { t.Errorf("Error shutting down machinery worker gracefully %+v", err) continue } } } func TestRedisRedisWorkerQuickQuit(t *testing.T) { redisURL := os.Getenv("REDIS_URL") if redisURL == "" { t.Skip("REDIS_URL is not defined") } // Redis broker, Redis result backend pollPeriod := 1 cnf := &config.Config{ Broker: fmt.Sprintf("redis://%v", redisURL), DefaultQueue: "test_queue", ResultBackend: fmt.Sprintf("redis://%v", redisURL), Lock: fmt.Sprintf("redis://%v", redisURL), Redis: &config.RedisConfig{ NormalTasksPollPeriod: pollPeriod, // default: 1000 }, } server, _ := machinery.NewServer(cnf) worker := server.NewWorker("test_worker", 0) errorsChan := make(chan error, 1) // Check Quit() immediately after LaunchAsync() will shutdown gracefully // and not panic worker.LaunchAsync(errorsChan) before := time.Now() worker.Quit() delta := time.Since(before) threshold := time.Duration(pollPeriod)*time.Millisecond + 1000 // add 1 second as buffer if delta.Nanoseconds() > threshold.Nanoseconds() { t.Error("Worker quit() exceeded timeout") } } func TestRedisRedisWorkerPreConsumeHandler(t *testing.T) { redisURL := os.Getenv("REDIS_URL") if redisURL == "" { t.Skip("REDIS_URL is not defined") } // Redis broker, Redis result backend pollPeriod := 1 cnf := &config.Config{ Broker: fmt.Sprintf("redis://%v", redisURL), DefaultQueue: "test_queue", ResultBackend: fmt.Sprintf("redis://%v", redisURL), Lock: fmt.Sprintf("redis://%v", redisURL), Redis: &config.RedisConfig{ NormalTasksPollPeriod: pollPeriod, // default: 1000 }, } server, _ := machinery.NewServer(cnf) worker := server.NewWorker("test_worker", 0) errorsChan := make(chan error) err := errors.New("PreConsumeHandler is invoked") worker.SetPreConsumeHandler(func(*machinery.Worker) bool { errorsChan <- err return true }) worker.LaunchAsync(errorsChan) if err != <-errorsChan { t.Error("PreConsumeHandler was not invoked") } } ================================================ FILE: integration-tests/redis_socket_test.go ================================================ package integration_test import ( "fmt" "os" "testing" "github.com/RichardKnop/machinery/v1" "github.com/RichardKnop/machinery/v1/config" ) func TestRedisSocket(t *testing.T) { redisSocket := os.Getenv("REDIS_SOCKET") if redisSocket == "" { t.Skip("REDIS_SOCKET is not defined") } // Redis broker, Redis result backend server := testSetup(&config.Config{ Broker: fmt.Sprintf("redis+socket://%v", redisSocket), DefaultQueue: "test_queue", ResultBackend: fmt.Sprintf("redis+socket://%v", redisSocket), Lock: "eager", }) worker := server.(*machinery.Server).NewWorker("test_worker", 0) defer worker.Quit() go worker.Launch() testAll(server, t) } ================================================ FILE: integration-tests/sqs_amqp_test.go ================================================ package integration_test import ( "os" "testing" "github.com/RichardKnop/machinery/v1" "github.com/RichardKnop/machinery/v1/config" ) func TestSQSAmqp(t *testing.T) { sqsURL := os.Getenv("SQS_URL") if sqsURL == "" { t.Skip("SQS_URL is not defined") } amqpURL := os.Getenv("AMQP_URL") if amqpURL == "" { t.Skip("AMQP_URL is not defined") } // AMQP broker, AMQP result backend server := testSetup(&config.Config{ Broker: sqsURL, DefaultQueue: "test_queue", ResultBackend: amqpURL, Lock: "eager", AMQP: &config.AMQPConfig{ Exchange: "test_exchange", ExchangeType: "direct", BindingKey: "test_task", PrefetchCount: 1, }, }) worker := server.(*machinery.Server).NewWorker("test_worker", 0) defer worker.Quit() go worker.Launch() testAll(server, t) } ================================================ FILE: integration-tests/sqs_mongodb_test.go ================================================ package integration_test import ( "fmt" "os" "testing" "github.com/RichardKnop/machinery/v1" "github.com/RichardKnop/machinery/v1/config" ) func TestSQSMongodb(t *testing.T) { sqsURL := os.Getenv("SQS_URL") mongodbURL := os.Getenv("MONGODB_URL") if sqsURL == "" { t.Skip("SQS_URL is not defined") } if mongodbURL == "" { t.Skip("MONGODB_URL is not defined") } // AMQP broker, MongoDB result backend server := testSetup(&config.Config{ Broker: sqsURL, DefaultQueue: "test_queue", ResultsExpireIn: 30, ResultBackend: fmt.Sprintf("mongodb://%v", mongodbURL), Lock: "eager", }) worker := server.(*machinery.Server).NewWorker("test_worker", 0) go worker.Launch() testAll(server, t) worker.Quit() } ================================================ FILE: integration-tests/suite_test.go ================================================ package integration_test import ( "context" "errors" "log" "reflect" "sort" "testing" "time" "github.com/stretchr/testify/assert" "github.com/RichardKnop/machinery/v1" "github.com/RichardKnop/machinery/v1/backends/result" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/tasks" brokersiface "github.com/RichardKnop/machinery/v1/brokers/iface" ) type ascendingInt64s []int64 func (a ascendingInt64s) Len() int { return len(a) } func (a ascendingInt64s) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a ascendingInt64s) Less(i, j int) bool { return a[i] < a[j] } type Server interface { GetBroker() brokersiface.Broker GetConfig() *config.Config RegisterTasks(namedTaskFuncs map[string]interface{}) error SendTaskWithContext(ctx context.Context, signature *tasks.Signature) (*result.AsyncResult, error) SendTask(signature *tasks.Signature) (*result.AsyncResult, error) SendChainWithContext(ctx context.Context, chain *tasks.Chain) (*result.ChainAsyncResult, error) SendChain(chain *tasks.Chain) (*result.ChainAsyncResult, error) SendGroupWithContext(ctx context.Context, group *tasks.Group, sendConcurrency int) ([]*result.AsyncResult, error) SendGroup(group *tasks.Group, sendConcurrency int) ([]*result.AsyncResult, error) SendChordWithContext(ctx context.Context, chord *tasks.Chord, sendConcurrency int) (*result.ChordAsyncResult, error) SendChord(chord *tasks.Chord, sendConcurrency int) (*result.ChordAsyncResult, error) } func testAll(server Server, t *testing.T) { testSendTask(server, t) testSendGroup(server, t, 0) // with unlimited concurrency testSendGroup(server, t, 2) // with limited concurrency (2 parallel tasks at the most) testSendChord(server, t) testSendChain(server, t) testReturnJustError(server, t) testReturnMultipleValues(server, t) testPanic(server, t) testDelay(server, t) } func testSendTask(server Server, t *testing.T) { addTask := newAddTask(1, 1) asyncResult, err := server.SendTask(addTask) if err != nil { t.Error(err) } results, err := asyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { t.Error(err) } if len(results) != 1 { t.Errorf("Number of results returned = %d. Wanted %d", len(results), 1) } if results[0].Interface() != int64(2) { t.Errorf( "result = %v(%v), want int64(2)", results[0].Type().String(), results[0].Interface(), ) } sumTask := newSumTask([]int64{1, 2}) asyncResult, err = server.SendTask(sumTask) if err != nil { t.Error(err) } results, err = asyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { t.Error(err) } if len(results) != 1 { t.Errorf("Number of results returned = %d. Wanted %d", len(results), 1) } if results[0].Interface() != int64(3) { t.Errorf( "result = %v(%v), want int64(3)", results[0].Type().String(), results[0].Interface(), ) } } func testSendGroup(server Server, t *testing.T, sendConcurrency int) { t1, t2, t3 := newAddTask(1, 1), newAddTask(2, 2), newAddTask(5, 6) group, err := tasks.NewGroup(t1, t2, t3) if err != nil { t.Fatal(err) } asyncResults, err := server.SendGroup(group, sendConcurrency) if err != nil { t.Error(err) } expectedResults := []int64{2, 4, 11} actualResults := make([]int64, 3) for i, asyncResult := range asyncResults { results, err := asyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { t.Error(err) } if len(results) != 1 { t.Errorf("Number of results returned = %d. Wanted %d", len(results), 1) } intResult, ok := results[0].Interface().(int64) if !ok { t.Errorf("Could not convert %v to int64", results[0].Interface()) } actualResults[i] = intResult } sort.Sort(ascendingInt64s(actualResults)) if !reflect.DeepEqual(expectedResults, actualResults) { t.Errorf( "expected results = %v, actual results = %v", expectedResults, actualResults, ) } } func testSendChain(server Server, t *testing.T) { t1, t2, t3 := newAddTask(2, 2), newAddTask(5, 6), newMultipleTask(4) chain, err := tasks.NewChain(t1, t2, t3) if err != nil { t.Fatal(err) } chainAsyncResult, err := server.SendChain(chain) if err != nil { t.Error(err) } results, err := chainAsyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { t.Error(err) } if len(results) != 1 { t.Errorf("Number of results returned = %d. Wanted %d", len(results), 1) } if results[0].Interface() != int64(60) { t.Errorf( "result = %v(%v), want int64(60)", results[0].Type().String(), results[0].Interface(), ) } } func testSendChord(server Server, t *testing.T) { t1, t2, t3, t4 := newAddTask(1, 1), newAddTask(2, 2), newAddTask(5, 6), newMultipleTask() group, err := tasks.NewGroup(t1, t2, t3) if err != nil { t.Fatal(err) } chord, err := tasks.NewChord(group, t4) if err != nil { t.Fatal(err) } chordAsyncResult, err := server.SendChord(chord, 10) if err != nil { t.Error(err) } results, err := chordAsyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { t.Error(err) } if len(results) != 1 { t.Errorf("Number of results returned = %d. Wanted %d", len(results), 1) } if results[0].Interface() != int64(88) { t.Errorf( "result = %v(%v), want int64(88)", results[0].Type().String(), results[0].Interface(), ) } } func testReturnJustError(server Server, t *testing.T) { // Fails, returns error as the only value task := newErrorTask("Test error", true) asyncResult, err := server.SendTask(task) if err != nil { t.Error(err) } results, err := asyncResult.Get(time.Duration(time.Millisecond * 5)) if len(results) != 0 { t.Errorf("Number of results returned = %d. Wanted %d", len(results), 0) } assert.Equal(t, "Test error", err.Error()) // Successful, returns nil as the only value task = newErrorTask("", false) asyncResult, err = server.SendTask(task) if err != nil { t.Error(err) } results, err = asyncResult.Get(time.Duration(time.Millisecond * 5)) if len(results) != 0 { t.Errorf("Number of results returned = %d. Wanted %d", len(results), 0) } assert.NoError(t, err) } func testReturnMultipleValues(server Server, t *testing.T) { // Successful task with multiple return values task := newMultipleReturnTask("foo", "bar", false) asyncResult, err := server.SendTask(task) if err != nil { t.Error(err) } results, err := asyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { t.Error(err) } if len(results) != 2 { t.Errorf("Number of results returned = %d. Wanted %d", len(results), 2) } if results[0].Interface() != "foo" { t.Errorf( "result = %v(%v), want string(\"foo\":)", results[0].Type().String(), results[0].Interface(), ) } if results[1].Interface() != "bar" { t.Errorf( "result = %v(%v), want string(\"bar\":)", results[1].Type().String(), results[1].Interface(), ) } // Failed task with multiple return values task = newMultipleReturnTask("", "", true) asyncResult, err = server.SendTask(task) if err != nil { t.Error(err) } results, err = asyncResult.Get(time.Duration(time.Millisecond * 5)) if len(results) != 0 { t.Errorf("Number of results returned = %d. Wanted %d", len(results), 0) } assert.Error(t, err) } func testPanic(server Server, t *testing.T) { task := &tasks.Signature{Name: "panic"} asyncResult, err := server.SendTask(task) if err != nil { t.Error(err) } results, err := asyncResult.Get(time.Duration(time.Millisecond * 5)) if len(results) != 0 { t.Errorf("Number of results returned = %d. Wanted %d", len(results), 0) } assert.Equal(t, "oops", err.Error()) } func testDelay(server Server, t *testing.T) { now := time.Now().UTC() eta := now.Add(100 * time.Millisecond) task := newDelayTask(eta) asyncResult, err := server.SendTask(task) if err != nil { t.Error(err) } results, err := asyncResult.Get(time.Duration(5 * time.Millisecond)) if err != nil { t.Error(err) } if len(results) != 1 { t.Errorf("Number of results returned = %d. Wanted %d", len(results), 1) } tm, ok := results[0].Interface().(int64) if !ok { t.Errorf( "Could not type assert = %v(%v) to int64", results[0].Type().String(), results[0].Interface(), ) } if tm < eta.UnixNano() { t.Errorf( "result = %v(%v), want >= int64(%d)", results[0].Type().String(), results[0].Interface(), eta.UnixNano(), ) } } func registerTestTasks(server Server) { tasks := map[string]interface{}{ "add": func(args ...int64) (int64, error) { sum := int64(0) for _, arg := range args { sum += arg } return sum, nil }, "multiply": func(args ...int64) (int64, error) { sum := int64(1) for _, arg := range args { sum *= arg } return sum, nil }, "sum": func(numbers []int64) (int64, error) { var sum int64 for _, num := range numbers { sum += num } return sum, nil }, "return_just_error": func(msg string, fail bool) (err error) { if fail { err = errors.New(msg) } return err }, "return_multiple_values": func(arg1, arg2 string, fail bool) (r1 string, r2 string, err error) { if fail { err = errors.New("some error") } else { r1 = arg1 r2 = arg2 } return r1, r2, err }, "panic": func() (string, error) { panic(errors.New("oops")) }, "delay_test": func() (int64, error) { return time.Now().UTC().UnixNano(), nil }, } server.RegisterTasks(tasks) } func testSetup(cnf *config.Config) Server { server, err := machinery.NewServer(cnf) if err != nil { log.Fatal(err, "Could not initialize server") } registerTestTasks(server) return server } func newAddTask(a, b int) *tasks.Signature { return &tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: a, }, { Type: "int64", Value: b, }, }, } } func newMultipleTask(nums ...int) *tasks.Signature { args := make([]tasks.Arg, len(nums)) for i, n := range nums { args[i] = tasks.Arg{ Type: "int64", Value: n, } } return &tasks.Signature{ Name: "multiply", Args: args, } } func newSumTask(nums []int64) *tasks.Signature { return &tasks.Signature{ Name: "sum", Args: []tasks.Arg{ { Type: "[]int64", Value: nums, }, }, } } func newErrorTask(msg string, fail bool) *tasks.Signature { return &tasks.Signature{ Name: "return_just_error", Args: []tasks.Arg{ { Type: "string", Value: msg, }, { Type: "bool", Value: fail, }, }, } } func newMultipleReturnTask(arg1, arg2 string, fail bool) *tasks.Signature { return &tasks.Signature{ Name: "return_multiple_values", Args: []tasks.Arg{ { Type: "string", Value: arg1, }, { Type: "string", Value: arg2, }, { Type: "bool", Value: fail, }, }, } } func newDelayTask(eta time.Time) *tasks.Signature { return &tasks.Signature{ Name: "delay_test", ETA: &eta, } } ================================================ FILE: integration-tests/worker_only_consumes_registered_tasks_test.go ================================================ package integration_test import ( "fmt" "os" "reflect" "sort" "testing" "time" "github.com/RichardKnop/machinery/v1" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/tasks" ) func TestWorkerOnlyConsumesRegisteredTaskAMQP(t *testing.T) { amqpURL := os.Getenv("AMQP_URL") if amqpURL == "" { t.Skip("AMQP_URL is not defined") } cnf := config.Config{ Broker: amqpURL, DefaultQueue: "test_queue", ResultBackend: amqpURL, Lock: "eager", AMQP: &config.AMQPConfig{ Exchange: "test_exchange", ExchangeType: "direct", BindingKey: "test_task", PrefetchCount: 3, }, } server1, err := machinery.NewServer(&cnf) if err != nil { t.Fatal(err, "Could not initialize server") } server1.RegisterTask("add", func(args ...int64) (int64, error) { sum := int64(0) for _, arg := range args { sum += arg } return sum, nil }) server2, err := machinery.NewServer(&cnf) if err != nil { t.Fatal(err, "Could not initialize server") } server2.RegisterTask("multiply", func(args ...int64) (int64, error) { sum := int64(1) for _, arg := range args { sum *= arg } return sum, nil }) task1 := tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 2, }, { Type: "int64", Value: 3, }, }, } task2 := tasks.Signature{ Name: "multiply", Args: []tasks.Arg{ { Type: "int64", Value: 4, }, { Type: "int64", Value: 5, }, }, } worker1 := server1.NewWorker("test_worker", 0) worker2 := server2.NewWorker("test_worker2", 0) go worker1.Launch() go worker2.Launch() group, err := tasks.NewGroup(&task2, &task1) if err != nil { t.Fatal(err) } asyncResults, err := server1.SendGroup(group, 10) if err != nil { t.Error(err) } expectedResults := []int64{5, 20} actualResults := make([]int64, 2) for i, asyncResult := range asyncResults { results, err := asyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { t.Error(err) } if len(results) != 1 { t.Errorf("Number of results returned = %d. Wanted %d", len(results), 1) } intResult, ok := results[0].Interface().(int64) if !ok { t.Errorf("Could not convert %v to int64", results[0].Interface()) } actualResults[i] = intResult } worker1.Quit() worker2.Quit() sort.Sort(ascendingInt64s(actualResults)) if !reflect.DeepEqual(expectedResults, actualResults) { t.Errorf( "expected results = %v, actual results = %v", expectedResults, actualResults, ) } } func TestWorkerOnlyConsumesRegisteredTaskRedis(t *testing.T) { redisURL := os.Getenv("REDIS_URL") if redisURL == "" { t.Skip("REDIS_URL is not defined") } cnf := config.Config{ Broker: fmt.Sprintf("redis://%v", redisURL), DefaultQueue: "test_queue", ResultBackend: fmt.Sprintf("redis://%v", redisURL), Lock: fmt.Sprintf("redis://%v", redisURL), } server1, err := machinery.NewServer(&cnf) if err != nil { t.Fatal(err) } server1.RegisterTask("add", func(args ...int64) (int64, error) { sum := int64(0) for _, arg := range args { sum += arg } return sum, nil }) server2, err := machinery.NewServer(&cnf) if err != nil { t.Fatal(err) } server2.RegisterTask("multiply", func(args ...int64) (int64, error) { sum := int64(1) for _, arg := range args { sum *= arg } return sum, nil }) task1 := tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 2, }, { Type: "int64", Value: 3, }, }, } task2 := tasks.Signature{ Name: "multiply", Args: []tasks.Arg{ { Type: "int64", Value: 4, }, { Type: "int64", Value: 5, }, }, } worker1 := server1.NewWorker("test_worker", 0) worker2 := server2.NewWorker("test_worker2", 0) go worker1.Launch() go worker2.Launch() group, err := tasks.NewGroup(&task2, &task1) if err != nil { t.Fatal(err) } asyncResults, err := server1.SendGroup(group, 10) if err != nil { t.Error(err) } expectedResults := []int64{5, 20} actualResults := make([]int64, 2) for i, asyncResult := range asyncResults { results, err := asyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { t.Error(err) } if len(results) != 1 { t.Errorf("Number of results returned = %d. Wanted %d", len(results), 1) } intResult, ok := results[0].Interface().(int64) if !ok { t.Errorf("Could not convert %v to int64", results[0].Interface()) } actualResults[i] = intResult } worker1.Quit() worker2.Quit() sort.Sort(ascendingInt64s(actualResults)) if !reflect.DeepEqual(expectedResults, actualResults) { t.Errorf( "expected results = %v, actual results = %v", expectedResults, actualResults, ) } } ================================================ FILE: v1/backends/amqp/amqp.go ================================================ package amqp // NOTE: Using AMQP as a result backend is quite tricky since every time we // read a message from the queue keeping task states, the message is removed // from the queue. This leads to problems with keeping a reliable state of a // group of tasks since concurrent processes updating the group state cause // race conditions and inconsistent state. // // This is avoided by a "clever" hack. A special queue identified by a group // UUID is created and we store serialised TaskState objects of successfully // completed tasks. By inspecting the queue we can then say: // 1) If all group tasks finished (number of unacked messages = group task count) // 2) If all group tasks finished AND succeeded (by consuming the queue) // // It is important to consume the queue exclusively to avoid race conditions. import ( "bytes" "encoding/json" "errors" "fmt" "github.com/RichardKnop/machinery/v1/backends/iface" "github.com/RichardKnop/machinery/v1/common" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/log" "github.com/RichardKnop/machinery/v1/tasks" amqp "github.com/rabbitmq/amqp091-go" ) // Backend represents an AMQP result backend type Backend struct { common.Backend common.AMQPConnector } // New creates Backend instance func New(cnf *config.Config) iface.Backend { return &Backend{Backend: common.NewBackend(cnf), AMQPConnector: common.AMQPConnector{}} } // InitGroup creates and saves a group meta data object func (b *Backend) InitGroup(groupUUID string, taskUUIDs []string) error { return nil } // GroupCompleted returns true if all tasks in a group finished // NOTE: Given AMQP limitation this will only return true if all finished // tasks were successful as we do not keep track of completed failed tasks func (b *Backend) GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) { conn, channel, err := b.Open(b.GetConfig().ResultBackend, b.GetConfig().TLSConfig) if err != nil { return false, err } defer b.Close(channel, conn) queueState, err := b.InspectQueue(channel, groupUUID) if err != nil { return false, nil } return queueState.Messages == groupTaskCount, nil } // GroupTaskStates returns states of all tasks in the group func (b *Backend) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error) { conn, channel, err := b.Open(b.GetConfig().ResultBackend, b.GetConfig().TLSConfig) if err != nil { return nil, err } defer b.Close(channel, conn) queueState, err := b.InspectQueue(channel, groupUUID) if err != nil { return nil, err } if queueState.Messages != groupTaskCount { return nil, fmt.Errorf("Already consumed: %v", err) } deliveries, err := channel.Consume( groupUUID, // queue name "", // consumer tag false, // auto-ack true, // exclusive false, // no-local false, // no-wait nil, // arguments ) if err != nil { return nil, fmt.Errorf("Queue consume error: %s", err) } states := make([]*tasks.TaskState, groupTaskCount) for i := 0; i < groupTaskCount; i++ { d := <-deliveries state := new(tasks.TaskState) decoder := json.NewDecoder(bytes.NewReader([]byte(d.Body))) decoder.UseNumber() if err := decoder.Decode(state); err != nil { d.Nack(false, false) // multiple, requeue return nil, err } d.Ack(false) // multiple states[i] = state } return states, nil } // TriggerChord flags chord as triggered in the backend storage to make sure // chord is never trigerred multiple times. Returns a boolean flag to indicate // whether the worker should trigger chord (true) or no if it has been triggered // already (false) func (b *Backend) TriggerChord(groupUUID string) (bool, error) { conn, channel, err := b.Open(b.GetConfig().ResultBackend, b.GetConfig().TLSConfig) if err != nil { return false, err } defer b.Close(channel, conn) _, err = b.InspectQueue(channel, amqmChordTriggeredQueue(groupUUID)) if err != nil { return true, nil } return false, nil } // SetStatePending updates task state to PENDING func (b *Backend) SetStatePending(signature *tasks.Signature) error { taskState := tasks.NewPendingTaskState(signature) return b.updateState(taskState) } // SetStateReceived updates task state to RECEIVED func (b *Backend) SetStateReceived(signature *tasks.Signature) error { taskState := tasks.NewReceivedTaskState(signature) return b.updateState(taskState) } // SetStateStarted updates task state to STARTED func (b *Backend) SetStateStarted(signature *tasks.Signature) error { taskState := tasks.NewStartedTaskState(signature) return b.updateState(taskState) } // SetStateRetry updates task state to RETRY func (b *Backend) SetStateRetry(signature *tasks.Signature) error { state := tasks.NewRetryTaskState(signature) return b.updateState(state) } // SetStateSuccess updates task state to SUCCESS func (b *Backend) SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error { taskState := tasks.NewSuccessTaskState(signature, results) if err := b.updateState(taskState); err != nil { return err } if signature.GroupUUID == "" { return nil } return b.markTaskCompleted(signature, taskState) } // SetStateFailure updates task state to FAILURE func (b *Backend) SetStateFailure(signature *tasks.Signature, err string) error { taskState := tasks.NewFailureTaskState(signature, err) if err := b.updateState(taskState); err != nil { return err } if signature.GroupUUID == "" { return nil } return b.markTaskCompleted(signature, taskState) } // GetState returns the latest task state. It will only return the status once // as the message will get consumed and removed from the queue. func (b *Backend) GetState(taskUUID string) (*tasks.TaskState, error) { declareQueueArgs := amqp.Table{ // Time in milliseconds // after that message will expire "x-message-ttl": int32(b.getExpiresIn()), // Time after that the queue will be deleted. "x-expires": int32(b.getExpiresIn()), } conn, channel, _, _, _, err := b.Connect( b.GetConfig().ResultBackend, "", b.GetConfig().TLSConfig, b.GetConfig().AMQP.Exchange, // exchange name b.GetConfig().AMQP.ExchangeType, // exchange type taskUUID, // queue name false, // queue durable true, // queue delete when unused taskUUID, // queue binding key nil, // exchange declare args declareQueueArgs, // queue declare args nil, // queue binding args ) if err != nil { return nil, err } defer b.Close(channel, conn) d, ok, err := channel.Get( taskUUID, // queue name false, // multiple ) if err != nil { return nil, err } if !ok { return nil, errors.New("No state ready") } d.Ack(false) state := new(tasks.TaskState) decoder := json.NewDecoder(bytes.NewReader([]byte(d.Body))) decoder.UseNumber() if err := decoder.Decode(state); err != nil { log.ERROR.Printf("Failed to unmarshal task state: %s", string(d.Body)) log.ERROR.Print(err) return nil, err } return state, nil } // PurgeState deletes stored task state func (b *Backend) PurgeState(taskUUID string) error { conn, channel, err := b.Open(b.GetConfig().ResultBackend, b.GetConfig().TLSConfig) if err != nil { return err } defer b.Close(channel, conn) return b.DeleteQueue(channel, taskUUID) } // PurgeGroupMeta deletes stored group meta data func (b *Backend) PurgeGroupMeta(groupUUID string) error { conn, channel, err := b.Open(b.GetConfig().ResultBackend, b.GetConfig().TLSConfig) if err != nil { return err } defer b.Close(channel, conn) b.DeleteQueue(channel, amqmChordTriggeredQueue(groupUUID)) return b.DeleteQueue(channel, groupUUID) } // updateState saves current task state func (b *Backend) updateState(taskState *tasks.TaskState) error { message, err := json.Marshal(taskState) if err != nil { return fmt.Errorf("JSON marshal error: %s", err) } declareQueueArgs := amqp.Table{ // Time in milliseconds // after that message will expire "x-message-ttl": int32(b.getExpiresIn()), // Time after that the queue will be deleted. "x-expires": int32(b.getExpiresIn()), } conn, channel, queue, confirmsChan, _, err := b.Connect( b.GetConfig().ResultBackend, "", b.GetConfig().TLSConfig, b.GetConfig().AMQP.Exchange, // exchange name b.GetConfig().AMQP.ExchangeType, // exchange type taskState.TaskUUID, // queue name false, // queue durable true, // queue delete when unused taskState.TaskUUID, // queue binding key nil, // exchange declare args declareQueueArgs, // queue declare args nil, // queue binding args ) if err != nil { return err } defer b.Close(channel, conn) if err := channel.Publish( b.GetConfig().AMQP.Exchange, // exchange queue.Name, // routing key false, // mandatory false, // immediate amqp.Publishing{ ContentType: "application/json", Body: message, DeliveryMode: amqp.Persistent, // Persistent // Transient }, ); err != nil { return err } confirmed := <-confirmsChan if confirmed.Ack { return nil } return fmt.Errorf("Failed delivery of delivery tag: %d", confirmed.DeliveryTag) } // getExpiresIn returns expiration time func (b *Backend) getExpiresIn() int { resultsExpireIn := b.GetConfig().ResultsExpireIn * 1000 if resultsExpireIn == 0 { // // expire results after 1 hour by default resultsExpireIn = config.DefaultResultsExpireIn * 1000 } return resultsExpireIn } // markTaskCompleted marks task as completed in either groupdUUID_success // or groupUUID_failure queue. This is important for GroupCompleted and // GroupSuccessful methods func (b *Backend) markTaskCompleted(signature *tasks.Signature, taskState *tasks.TaskState) error { if signature.GroupUUID == "" || signature.GroupTaskCount == 0 { return nil } message, err := json.Marshal(taskState) if err != nil { return fmt.Errorf("JSON marshal error: %s", err) } declareQueueArgs := amqp.Table{ // Time in milliseconds // after that message will expire "x-message-ttl": int32(b.getExpiresIn()), // Time after that the queue will be deleted. "x-expires": int32(b.getExpiresIn()), } conn, channel, queue, confirmsChan, _, err := b.Connect( b.GetConfig().ResultBackend, "", b.GetConfig().TLSConfig, b.GetConfig().AMQP.Exchange, // exchange name b.GetConfig().AMQP.ExchangeType, // exchange type signature.GroupUUID, // queue name false, // queue durable true, // queue delete when unused signature.GroupUUID, // queue binding key nil, // exchange declare args declareQueueArgs, // queue declare args nil, // queue binding args ) if err != nil { return err } defer b.Close(channel, conn) if err := channel.Publish( b.GetConfig().AMQP.Exchange, // exchange queue.Name, // routing key false, // mandatory false, // immediate amqp.Publishing{ ContentType: "application/json", Body: message, DeliveryMode: amqp.Persistent, // Persistent // Transient }, ); err != nil { return err } confirmed := <-confirmsChan if !confirmed.Ack { return fmt.Errorf("Failed delivery of delivery tag: %v", confirmed.DeliveryTag) } return nil } func amqmChordTriggeredQueue(groupUUID string) string { return fmt.Sprintf("%s_chord_triggered", groupUUID) } ================================================ FILE: v1/backends/amqp/amqp_test.go ================================================ package amqp_test import ( "os" "testing" "time" "github.com/RichardKnop/machinery/v1/backends/amqp" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/tasks" "github.com/stretchr/testify/assert" ) var ( amqpConfig *config.Config ) func init() { amqpURL := os.Getenv("AMQP_URL") if amqpURL == "" { return } finalAmqpURL := amqpURL var finalSeparator string amqpURLs := os.Getenv("AMQP_URLS") if amqpURLs != "" { separator := os.Getenv("AMQP_URLS_SEPARATOR") if separator == "" { return } finalSeparator = separator finalAmqpURL = amqpURLs } amqp2URL := os.Getenv("AMQP2_URL") if amqp2URL == "" { amqp2URL = amqpURL } amqpConfig = &config.Config{ Broker: finalAmqpURL, MultipleBrokerSeparator: finalSeparator, DefaultQueue: "test_queue", ResultBackend: amqp2URL, AMQP: &config.AMQPConfig{ Exchange: "test_exchange", ExchangeType: "direct", BindingKey: "test_task", PrefetchCount: 1, }, } } func TestGroupCompleted(t *testing.T) { if os.Getenv("AMQP_URL") == "" { t.Skip("AMQP_URL is not defined") } groupUUID := "testGroupUUID" groupTaskCount := 2 task1 := &tasks.Signature{ UUID: "testTaskUUID1", GroupUUID: groupUUID, GroupTaskCount: groupTaskCount, } task2 := &tasks.Signature{ UUID: "testTaskUUID2", GroupUUID: groupUUID, GroupTaskCount: groupTaskCount, } backend := amqp.New(amqpConfig) // Cleanup before the test backend.PurgeState(task1.UUID) backend.PurgeState(task2.UUID) backend.PurgeGroupMeta(groupUUID) groupCompleted, err := backend.GroupCompleted(groupUUID, groupTaskCount) if assert.NoError(t, err) { assert.False(t, groupCompleted) } backend.InitGroup(groupUUID, []string{task1.UUID, task2.UUID}) groupCompleted, err = backend.GroupCompleted(groupUUID, groupTaskCount) if assert.NoError(t, err) { assert.False(t, groupCompleted) } backend.SetStatePending(task1) backend.SetStateStarted(task2) groupCompleted, err = backend.GroupCompleted(groupUUID, groupTaskCount) if assert.NoError(t, err) { assert.False(t, groupCompleted) } taskResults := []*tasks.TaskResult{new(tasks.TaskResult)} backend.SetStateSuccess(task1, taskResults) backend.SetStateSuccess(task2, taskResults) groupCompleted, err = backend.GroupCompleted(groupUUID, groupTaskCount) if assert.NoError(t, err) { assert.True(t, groupCompleted) } } func TestGetState(t *testing.T) { if os.Getenv("AMQP_URL") == "" { t.Skip("AMQP_URL is not defined") } signature := &tasks.Signature{ UUID: "testTaskUUID", GroupUUID: "testGroupUUID", } go func() { backend := amqp.New(amqpConfig) backend.SetStatePending(signature) time.Sleep(2 * time.Millisecond) backend.SetStateReceived(signature) time.Sleep(2 * time.Millisecond) backend.SetStateStarted(signature) time.Sleep(2 * time.Millisecond) taskResults := []*tasks.TaskResult{ { Type: "float64", Value: 2, }, } backend.SetStateSuccess(signature, taskResults) }() backend := amqp.New(amqpConfig) var ( taskState *tasks.TaskState err error ) for { taskState, err = backend.GetState(signature.UUID) if taskState == nil { assert.Equal(t, "No state ready", err.Error()) continue } assert.NoError(t, err) if taskState.IsCompleted() { break } } } func TestPurgeState(t *testing.T) { if os.Getenv("AMQP_URL") == "" { t.Skip("AMQP_URL is not defined") } signature := &tasks.Signature{ UUID: "testTaskUUID", GroupUUID: "testGroupUUID", } backend := amqp.New(amqpConfig) backend.SetStatePending(signature) backend.SetStateReceived(signature) taskState, err := backend.GetState(signature.UUID) assert.NotNil(t, taskState) assert.NoError(t, err) backend.PurgeState(taskState.TaskUUID) taskState, err = backend.GetState(signature.UUID) assert.Nil(t, taskState) assert.Error(t, err) } ================================================ FILE: v1/backends/dynamodb/dynamodb.go ================================================ package dynamodb import ( "errors" "fmt" "math" "time" "github.com/aws/aws-sdk-go/aws/session" "github.com/RichardKnop/machinery/v1/backends/iface" "github.com/RichardKnop/machinery/v1/common" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/log" "github.com/RichardKnop/machinery/v1/tasks" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/dynamodb" "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute" "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface" ) const ( BatchItemsLimit = 99 MaxFetchAttempts = 3 ) // Backend ... type Backend struct { common.Backend cnf *config.Config client dynamodbiface.DynamoDBAPI } // New creates a Backend instance func New(cnf *config.Config) iface.Backend { backend := &Backend{Backend: common.NewBackend(cnf), cnf: cnf} if cnf.DynamoDB != nil && cnf.DynamoDB.Client != nil { backend.client = cnf.DynamoDB.Client } else { sess := session.Must(session.NewSessionWithOptions(session.Options{ SharedConfigState: session.SharedConfigEnable, })) backend.client = dynamodb.New(sess) } // Check if needed tables exist err := backend.checkRequiredTablesIfExist() if err != nil { log.FATAL.Printf("Failed to prepare tables. Error: %v", err) } return backend } // InitGroup ... func (b *Backend) InitGroup(groupUUID string, taskUUIDs []string) error { meta := tasks.GroupMeta{ GroupUUID: groupUUID, TaskUUIDs: taskUUIDs, CreatedAt: time.Now().UTC(), TTL: b.getExpirationTime(), } av, err := dynamodbattribute.MarshalMap(meta) if err != nil { log.ERROR.Printf("Error when marshaling Dynamodb attributes. Err: %v", err) return err } input := &dynamodb.PutItemInput{ Item: av, TableName: aws.String(b.cnf.DynamoDB.GroupMetasTable), } _, err = b.client.PutItem(input) if err != nil { log.ERROR.Printf("Got error when calling PutItem: %v; Error: %v", input, err) return err } return nil } // GroupCompleted ... func (b *Backend) GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) { groupMeta, err := b.getGroupMeta(groupUUID) if err != nil { return false, err } taskStates, err := b.getStates(groupMeta.TaskUUIDs) if err != nil { return false, err } var countSuccessTasks = 0 for _, taskState := range taskStates { if taskState.IsCompleted() { countSuccessTasks++ } } return countSuccessTasks == groupTaskCount, nil } // GroupTaskStates ... func (b *Backend) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error) { groupMeta, err := b.getGroupMeta(groupUUID) if err != nil { return nil, err } return b.getStates(groupMeta.TaskUUIDs) } // TriggerChord ... func (b *Backend) TriggerChord(groupUUID string) (bool, error) { // Get the group meta data groupMeta, err := b.getGroupMeta(groupUUID) if err != nil { return false, err } // Chord has already been triggered, return false (should not trigger again) if groupMeta.ChordTriggered { return false, nil } // If group meta is locked, wait until it's unlocked for groupMeta.Lock { groupMeta, _ = b.getGroupMeta(groupUUID) log.WARNING.Printf("Group [%s] locked, waiting", groupUUID) time.Sleep(time.Millisecond * 5) } // Acquire lock if err = b.lockGroupMeta(groupUUID); err != nil { return false, err } defer b.unlockGroupMeta(groupUUID) // update group meta data err = b.chordTriggered(groupUUID) if err != nil { return false, err } return true, err } // SetStatePending ... func (b *Backend) SetStatePending(signature *tasks.Signature) error { taskState := tasks.NewPendingTaskState(signature) // taskUUID is the primary key of the table, so a new task need to be created first, instead of using dynamodb.UpdateItemInput directly return b.initTaskState(taskState) } // SetStateReceived ... func (b *Backend) SetStateReceived(signature *tasks.Signature) error { taskState := tasks.NewReceivedTaskState(signature) return b.setTaskState(taskState) } // SetStateStarted ... func (b *Backend) SetStateStarted(signature *tasks.Signature) error { taskState := tasks.NewStartedTaskState(signature) return b.setTaskState(taskState) } // SetStateRetry ... func (b *Backend) SetStateRetry(signature *tasks.Signature) error { taskState := tasks.NewRetryTaskState(signature) return b.setTaskState(taskState) } // SetStateSuccess ... func (b *Backend) SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error { taskState := tasks.NewSuccessTaskState(signature, results) taskState.TTL = b.getExpirationTime() return b.setTaskState(taskState) } // SetStateFailure ... func (b *Backend) SetStateFailure(signature *tasks.Signature, err string) error { taskState := tasks.NewFailureTaskState(signature, err) taskState.TTL = b.getExpirationTime() return b.updateToFailureStateWithError(taskState) } // GetState ... func (b *Backend) GetState(taskUUID string) (*tasks.TaskState, error) { result, err := b.client.GetItem(&dynamodb.GetItemInput{ TableName: aws.String(b.cnf.DynamoDB.TaskStatesTable), Key: map[string]*dynamodb.AttributeValue{ "TaskUUID": { S: aws.String(taskUUID), }, }, ConsistentRead: aws.Bool(true), }) if err != nil { return nil, err } return b.unmarshalTaskStateGetItemResult(result) } // getStates returns the current states for the given list of tasks. // It uses batch fetch API. If any keys fail to fetch, it'll retry with exponential backoff until maxFetchAttempts times. func (b *Backend) getStates(tasksToFetch []string) ([]*tasks.TaskState, error) { fetchedTaskStates := make([]*tasks.TaskState, 0, len(tasksToFetch)) var unfetchedTaskIDs []string // try until all keys are fetched or until we run out of attempts. for attempt := 0; len(tasksToFetch) > 0 && attempt < MaxFetchAttempts; attempt++ { unfetchedTaskIDs = nil for _, batch := range chunkTasks(tasksToFetch, BatchItemsLimit) { fetched, unfetched, err := b.batchFetchTaskStates(batch) if err != nil { return nil, err } fetchedTaskStates = append(fetchedTaskStates, fetched...) unfetchedTaskIDs = append(unfetchedTaskIDs, unfetched...) } tasksToFetch = unfetchedTaskIDs // Check if there were any tasks that were not fetched. If so, retry with exponential backoff. if len(unfetchedTaskIDs) > 0 { backoffDuration := time.Duration(math.Pow(2, float64(attempt))) * time.Second log.DEBUG.Printf("Unable to fetch [%d] keys on attempt [%d]. Sleeping for [%s]", len(unfetchedTaskIDs), attempt+1, backoffDuration) time.Sleep(backoffDuration) } } if len(unfetchedTaskIDs) > 0 { return nil, fmt.Errorf("Failed to fetch [%d] keys even after retries: [%+v]", len(unfetchedTaskIDs), unfetchedTaskIDs) } return fetchedTaskStates, nil } // batchFetchTaskStates returns the current states of the given tasks by fetching them all in a single batched API. // DynamoDB's BatchGetItem() can return partial results. If there are any unfetched keys, they are returned as second // return value so that the caller can retry those keys. // https://docs.aws.amazon.com/sdk-for-go/api/service/dynamodb/#DynamoDB.BatchGetItem func (b *Backend) batchFetchTaskStates(taskUUIDs []string) ([]*tasks.TaskState, []string, error) { tableName := b.cnf.DynamoDB.TaskStatesTable keys := make([]map[string]*dynamodb.AttributeValue, len(taskUUIDs)) for i, tid := range taskUUIDs { keys[i] = map[string]*dynamodb.AttributeValue{ "TaskUUID": { S: aws.String(tid), }, } } input := &dynamodb.BatchGetItemInput{ RequestItems: map[string]*dynamodb.KeysAndAttributes{ tableName: { ConsistentRead: aws.Bool(true), Keys: keys, }, }, } result, err := b.client.BatchGetItem(input) if err != nil { return nil, nil, fmt.Errorf("BatchGetItem failed. Error: [%s]", err) } fetchedKeys, ok := result.Responses[tableName] if !ok { return nil, nil, fmt.Errorf("no keys returned from the table: [%s]", tableName) } states := []*tasks.TaskState{} if err := dynamodbattribute.UnmarshalListOfMaps(fetchedKeys, &states); err != nil { return nil, nil, fmt.Errorf("Got error when unmarshal map. Error: %v", err) } // Look for any unprocessed keys var unfetchedKeys []string if result.UnprocessedKeys[tableName] != nil { unfetchedKeys, err = getUnfetchedKeys(result.UnprocessedKeys[tableName]) if err != nil { return nil, nil, fmt.Errorf("unable to fetch some keys: [%+v]. Error: [%s]", result.UnprocessedKeys, err) } } return states, unfetchedKeys, nil } // PurgeState ... func (b *Backend) PurgeState(taskUUID string) error { input := &dynamodb.DeleteItemInput{ Key: map[string]*dynamodb.AttributeValue{ "TaskUUID": { N: aws.String(taskUUID), }, }, TableName: aws.String(b.cnf.DynamoDB.TaskStatesTable), } _, err := b.client.DeleteItem(input) if err != nil { return err } return nil } // PurgeGroupMeta ... func (b *Backend) PurgeGroupMeta(groupUUID string) error { input := &dynamodb.DeleteItemInput{ Key: map[string]*dynamodb.AttributeValue{ "GroupUUID": { N: aws.String(groupUUID), }, }, TableName: aws.String(b.cnf.DynamoDB.GroupMetasTable), } _, err := b.client.DeleteItem(input) if err != nil { return err } return nil } func (b *Backend) getGroupMeta(groupUUID string) (*tasks.GroupMeta, error) { result, err := b.client.GetItem(&dynamodb.GetItemInput{ TableName: aws.String(b.cnf.DynamoDB.GroupMetasTable), Key: map[string]*dynamodb.AttributeValue{ "GroupUUID": { S: aws.String(groupUUID), }, }, ConsistentRead: aws.Bool(true), }) if err != nil { log.ERROR.Printf("Error when getting group [%s]. Error: [%s]", groupUUID, err) return nil, err } item, err := b.unmarshalGroupMetaGetItemResult(result) if err != nil { log.ERROR.Printf("Failed to unmarshal item. Error: [%s], Result: [%+v]", err, result) return nil, err } return item, nil } func (b *Backend) lockGroupMeta(groupUUID string) error { err := b.updateGroupMetaLock(groupUUID, true) if err != nil { return err } return nil } func (b *Backend) unlockGroupMeta(groupUUID string) error { err := b.updateGroupMetaLock(groupUUID, false) if err != nil { return err } return nil } func (b *Backend) updateGroupMetaLock(groupUUID string, status bool) error { input := &dynamodb.UpdateItemInput{ ExpressionAttributeNames: map[string]*string{ "#L": aws.String("Lock"), }, ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{ ":l": { BOOL: aws.Bool(status), }, }, Key: map[string]*dynamodb.AttributeValue{ "GroupUUID": { S: aws.String(groupUUID), }, }, ReturnValues: aws.String("UPDATED_NEW"), TableName: aws.String(b.cnf.DynamoDB.GroupMetasTable), UpdateExpression: aws.String("SET #L = :l"), } _, err := b.client.UpdateItem(input) if err != nil { return err } return nil } func (b *Backend) chordTriggered(groupUUID string) error { input := &dynamodb.UpdateItemInput{ ExpressionAttributeNames: map[string]*string{ "#CT": aws.String("ChordTriggered"), }, ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{ ":ct": { BOOL: aws.Bool(true), }, }, Key: map[string]*dynamodb.AttributeValue{ "GroupUUID": { S: aws.String(groupUUID), }, }, ReturnValues: aws.String("UPDATED_NEW"), TableName: aws.String(b.cnf.DynamoDB.GroupMetasTable), UpdateExpression: aws.String("SET #CT = :ct"), } _, err := b.client.UpdateItem(input) if err != nil { return err } return nil } func (b *Backend) setTaskState(taskState *tasks.TaskState) error { expAttributeNames := map[string]*string{ "#S": aws.String("State"), } expAttributeValues := map[string]*dynamodb.AttributeValue{ ":s": { S: aws.String(taskState.State), }, } keyAttributeValues := map[string]*dynamodb.AttributeValue{ "TaskUUID": { S: aws.String(taskState.TaskUUID), }, } exp := "SET #S = :s" if !taskState.CreatedAt.IsZero() { expAttributeNames["#C"] = aws.String("CreatedAt") expAttributeValues[":c"] = &dynamodb.AttributeValue{ S: aws.String(taskState.CreatedAt.String()), } exp += ", #C = :c" } if taskState.TTL > 0 { expAttributeNames["#T"] = aws.String("TTL") expAttributeValues[":t"] = &dynamodb.AttributeValue{ N: aws.String(fmt.Sprintf("%d", taskState.TTL)), } exp += ", #T = :t" } if taskState.Results != nil && len(taskState.Results) != 0 { expAttributeNames["#R"] = aws.String("Results") var results []*dynamodb.AttributeValue for _, r := range taskState.Results { avMap := map[string]*dynamodb.AttributeValue{ "Type": { S: aws.String(r.Type), }, "Value": { S: aws.String(fmt.Sprintf("%v", r.Value)), }, } rs := &dynamodb.AttributeValue{ M: avMap, } results = append(results, rs) } expAttributeValues[":r"] = &dynamodb.AttributeValue{ L: results, } exp += ", #R = :r" } input := &dynamodb.UpdateItemInput{ ExpressionAttributeNames: expAttributeNames, ExpressionAttributeValues: expAttributeValues, Key: keyAttributeValues, ReturnValues: aws.String("UPDATED_NEW"), TableName: aws.String(b.cnf.DynamoDB.TaskStatesTable), UpdateExpression: aws.String(exp), } _, err := b.client.UpdateItem(input) if err != nil { return err } return nil } func (b *Backend) initTaskState(taskState *tasks.TaskState) error { av, err := dynamodbattribute.MarshalMap(taskState) input := &dynamodb.PutItemInput{ Item: av, TableName: aws.String(b.cnf.DynamoDB.TaskStatesTable), } if err != nil { return err } _, err = b.client.PutItem(input) if err != nil { return err } return nil } func (b *Backend) updateToFailureStateWithError(taskState *tasks.TaskState) error { input := &dynamodb.UpdateItemInput{ ExpressionAttributeNames: map[string]*string{ "#S": aws.String("State"), "#E": aws.String("Error"), }, ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{ ":s": { S: aws.String(taskState.State), }, ":e": { S: aws.String(taskState.Error), }, }, Key: map[string]*dynamodb.AttributeValue{ "TaskUUID": { S: aws.String(taskState.TaskUUID), }, }, ReturnValues: aws.String("UPDATED_NEW"), TableName: aws.String(b.cnf.DynamoDB.TaskStatesTable), UpdateExpression: aws.String("SET #S = :s, #E = :e"), } if taskState.TTL > 0 { input.ExpressionAttributeNames["#T"] = aws.String("TTL") input.ExpressionAttributeValues[":t"] = &dynamodb.AttributeValue{ N: aws.String(fmt.Sprintf("%d", taskState.TTL)), } input.UpdateExpression = aws.String(aws.StringValue(input.UpdateExpression) + ", #T = :t") } _, err := b.client.UpdateItem(input) if err != nil { return err } return nil } func (b *Backend) unmarshalGroupMetaGetItemResult(result *dynamodb.GetItemOutput) (*tasks.GroupMeta, error) { if result == nil { err := errors.New("task state is nil") log.ERROR.Printf("Got error when unmarshal map. Error: %v", err) return nil, err } item := tasks.GroupMeta{} err := dynamodbattribute.UnmarshalMap(result.Item, &item) if err != nil { log.ERROR.Printf("Got error when unmarshal map. Error: %v", err) return nil, err } return &item, err } func (b *Backend) unmarshalTaskStateGetItemResult(result *dynamodb.GetItemOutput) (*tasks.TaskState, error) { if result == nil { err := errors.New("task state is nil") log.ERROR.Printf("Got error when unmarshal map. Error: %v", err) return nil, err } state := tasks.TaskState{} err := dynamodbattribute.UnmarshalMap(result.Item, &state) if err != nil { log.ERROR.Printf("Got error when unmarshal map. Error: %v", err) return nil, err } return &state, nil } func (b *Backend) checkRequiredTablesIfExist() error { var ( taskTableName = b.cnf.DynamoDB.TaskStatesTable groupTableName = b.cnf.DynamoDB.GroupMetasTable tableNames []*string startFromTable *string ) for { result, err := b.client.ListTables(&dynamodb.ListTablesInput{ ExclusiveStartTableName: startFromTable, }) if err != nil { return err } tableNames = append(tableNames, result.TableNames...) if result.LastEvaluatedTableName == nil { break } startFromTable = result.LastEvaluatedTableName } if !b.tableExists(taskTableName, tableNames) { return errors.New("task table doesn't exist") } if !b.tableExists(groupTableName, tableNames) { return errors.New("group table doesn't exist") } return nil } func (b *Backend) tableExists(tableName string, tableNames []*string) bool { for _, t := range tableNames { if tableName == *t { return true } } return false } func (b *Backend) getExpirationTime() int64 { expiresIn := b.GetConfig().ResultsExpireIn if expiresIn == 0 { // expire results after 1 hour by default expiresIn = config.DefaultResultsExpireIn } return time.Now().Add(time.Second * time.Duration(expiresIn)).Unix() } // getUnfetchedKeys returns keys that were not fetched in a batch request. func getUnfetchedKeys(unprocessed *dynamodb.KeysAndAttributes) ([]string, error) { states := []*tasks.TaskState{} var taskIDs []string if err := dynamodbattribute.UnmarshalListOfMaps(unprocessed.Keys, &states); err != nil { return nil, fmt.Errorf("Got error when unmarshal map. Error: %v", err) } for _, s := range states { taskIDs = append(taskIDs, s.TaskUUID) } return taskIDs, nil } // chunkTasks chunks the list of strings into multiple smaller lists of specified size. func chunkTasks(array []string, chunkSize int) [][]string { var result [][]string for len(array) > 0 { sz := min(len(array), chunkSize) chunk := array[:sz] array = array[sz:] result = append(result, chunk) } return result } func min(a, b int) int { if a < b { return a } return b } ================================================ FILE: v1/backends/dynamodb/dynamodb_export_test.go ================================================ package dynamodb import ( "errors" "os" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/tasks" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/dynamodb" "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface" ) var ( TestDynamoDBBackend *Backend TestErrDynamoDBBackend *Backend TestCnf *config.Config TestDBClient dynamodbiface.DynamoDBAPI TestErrDBClient dynamodbiface.DynamoDBAPI TestGroupMeta *tasks.GroupMeta TestTask1 map[string]*dynamodb.AttributeValue TestTask2 map[string]*dynamodb.AttributeValue TestTask3 map[string]*dynamodb.AttributeValue ) type TestDynamoDBClient struct { dynamodbiface.DynamoDBAPI PutItemOverride func(*dynamodb.PutItemInput) (*dynamodb.PutItemOutput, error) UpdateItemOverride func(*dynamodb.UpdateItemInput) (*dynamodb.UpdateItemOutput, error) GetItemOverride func(input *dynamodb.GetItemInput) (*dynamodb.GetItemOutput, error) BatchGetItemOverride func(*dynamodb.BatchGetItemInput) (*dynamodb.BatchGetItemOutput, error) } func (t *TestDynamoDBClient) ResetOverrides() { t.PutItemOverride = nil t.UpdateItemOverride = nil t.BatchGetItemOverride = nil } func (t *TestDynamoDBClient) PutItem(input *dynamodb.PutItemInput) (*dynamodb.PutItemOutput, error) { if t.PutItemOverride != nil { return t.PutItemOverride(input) } return &dynamodb.PutItemOutput{}, nil } func (t *TestDynamoDBClient) BatchGetItem(input *dynamodb.BatchGetItemInput) (*dynamodb.BatchGetItemOutput, error) { if t.BatchGetItemOverride != nil { return t.BatchGetItemOverride(input) } return &dynamodb.BatchGetItemOutput{}, nil } func (t *TestDynamoDBClient) GetItem(input *dynamodb.GetItemInput) (*dynamodb.GetItemOutput, error) { if t.GetItemOverride != nil { return t.GetItemOverride(input) } var output *dynamodb.GetItemOutput switch *input.TableName { case "group_metas": output = &dynamodb.GetItemOutput{ Item: map[string]*dynamodb.AttributeValue{ "TaskUUIDs": { L: []*dynamodb.AttributeValue{ { S: aws.String("testTaskUUID1"), }, { S: aws.String("testTaskUUID2"), }, { S: aws.String("testTaskUUID3"), }, }, }, "ChordTriggered": { BOOL: aws.Bool(false), }, "GroupUUID": { S: aws.String("testGroupUUID"), }, "Lock": { BOOL: aws.Bool(false), }, }, } case "task_states": if input.Key["TaskUUID"] == nil { output = &dynamodb.GetItemOutput{ Item: map[string]*dynamodb.AttributeValue{ "Error": { NULL: aws.Bool(false), }, "State": { S: aws.String(tasks.StatePending), }, "TaskUUID": { S: aws.String("testTaskUUID1"), }, "Results:": { NULL: aws.Bool(true), }, }, } } else { if *(input.Key["TaskUUID"].S) == "testTaskUUID1" { output = &dynamodb.GetItemOutput{ Item: TestTask1, } } else if *(input.Key["TaskUUID"].S) == "testTaskUUID2" { output = &dynamodb.GetItemOutput{ Item: TestTask2, } } else if *(input.Key["TaskUUID"].S) == "testTaskUUID3" { output = &dynamodb.GetItemOutput{ Item: TestTask3, } } } } return output, nil } func (t *TestDynamoDBClient) DeleteItem(*dynamodb.DeleteItemInput) (*dynamodb.DeleteItemOutput, error) { return &dynamodb.DeleteItemOutput{}, nil } func (t *TestDynamoDBClient) UpdateItem(input *dynamodb.UpdateItemInput) (*dynamodb.UpdateItemOutput, error) { if t.UpdateItemOverride != nil { return t.UpdateItemOverride(input) } return &dynamodb.UpdateItemOutput{}, nil } func (t *TestDynamoDBClient) ListTables(*dynamodb.ListTablesInput) (*dynamodb.ListTablesOutput, error) { return &dynamodb.ListTablesOutput{ TableNames: []*string{ aws.String("group_metas"), aws.String("task_states"), }, }, nil } // Always returns error type TestErrDynamoDBClient struct { dynamodbiface.DynamoDBAPI } func (t *TestErrDynamoDBClient) PutItem(*dynamodb.PutItemInput) (*dynamodb.PutItemOutput, error) { return nil, errors.New("error when putting an item") } func (t *TestErrDynamoDBClient) GetItem(*dynamodb.GetItemInput) (*dynamodb.GetItemOutput, error) { return nil, errors.New("error when getting an item") } func (t *TestErrDynamoDBClient) DeleteItem(*dynamodb.DeleteItemInput) (*dynamodb.DeleteItemOutput, error) { return nil, errors.New("error when deleting an item") } func (t *TestErrDynamoDBClient) Scan(*dynamodb.ScanInput) (*dynamodb.ScanOutput, error) { return nil, errors.New("error when scanning an item") } func (t *TestErrDynamoDBClient) UpdateItem(*dynamodb.UpdateItemInput) (*dynamodb.UpdateItemOutput, error) { return nil, errors.New("error when updating an item") } func (t *TestErrDynamoDBClient) ListTables(*dynamodb.ListTablesInput) (*dynamodb.ListTablesOutput, error) { return nil, errors.New("error when listing tables") } func init() { TestCnf = &config.Config{ ResultBackend: os.Getenv("DYNAMODB_URL"), ResultsExpireIn: 30, DynamoDB: &config.DynamoDBConfig{ TaskStatesTable: "task_states", GroupMetasTable: "group_metas", }, } TestDBClient = new(TestDynamoDBClient) TestDynamoDBBackend = &Backend{cnf: TestCnf, client: TestDBClient} TestErrDBClient = new(TestErrDynamoDBClient) TestErrDynamoDBBackend = &Backend{cnf: TestCnf, client: TestErrDBClient} TestGroupMeta = &tasks.GroupMeta{ GroupUUID: "testGroupUUID", TaskUUIDs: []string{"testTaskUUID1", "testTaskUUID2", "testTaskUUID3"}, } } func (b *Backend) GetConfig() *config.Config { return b.cnf } func (b *Backend) GetClient() dynamodbiface.DynamoDBAPI { return b.client } func (b *Backend) GetGroupMetaForTest(groupUUID string) (*tasks.GroupMeta, error) { return b.getGroupMeta(groupUUID) } func (b *Backend) UnmarshalGroupMetaGetItemResultForTest(result *dynamodb.GetItemOutput) (*tasks.GroupMeta, error) { return b.unmarshalGroupMetaGetItemResult(result) } func (b *Backend) UnmarshalTaskStateGetItemResultForTest(result *dynamodb.GetItemOutput) (*tasks.TaskState, error) { return b.unmarshalTaskStateGetItemResult(result) } func (b *Backend) SetTaskStateForTest(taskState *tasks.TaskState) error { return b.setTaskState(taskState) } func (b *Backend) ChordTriggeredForTest(groupUUID string) error { return b.chordTriggered(groupUUID) } func (b *Backend) UpdateGroupMetaLockForTest(groupUUID string, status bool) error { return b.updateGroupMetaLock(groupUUID, status) } func (b *Backend) UnlockGroupMetaForTest(groupUUID string) error { return b.unlockGroupMeta(groupUUID) } func (b *Backend) LockGroupMetaForTest(groupUUID string) error { return b.lockGroupMeta(groupUUID) } func (b *Backend) GetStatesForTest(taskUUIDs ...string) ([]*tasks.TaskState, error) { return b.getStates(taskUUIDs) } func (b *Backend) UpdateToFailureStateWithErrorForTest(taskState *tasks.TaskState) error { return b.updateToFailureStateWithError(taskState) } func (b *Backend) TableExistsForTest(tableName string, tableNames []*string) bool { return b.tableExists(tableName, tableNames) } func (b *Backend) CheckRequiredTablesIfExistForTest() error { return b.checkRequiredTablesIfExist() } ================================================ FILE: v1/backends/dynamodb/dynamodb_test.go ================================================ package dynamodb_test import ( "fmt" "strconv" "testing" "time" "github.com/RichardKnop/machinery/v1/backends/dynamodb" "github.com/RichardKnop/machinery/v1/log" "github.com/RichardKnop/machinery/v1/tasks" "github.com/aws/aws-sdk-go/aws" "github.com/stretchr/testify/assert" awsdynamodb "github.com/aws/aws-sdk-go/service/dynamodb" ) func TestNew(t *testing.T) { // should call t.Skip if not connected to internet backend := dynamodb.New(dynamodb.TestCnf) assert.IsType(t, new(dynamodb.Backend), backend) } func TestInitGroup(t *testing.T) { groupUUID := "testGroupUUID" taskUUIDs := []string{"testTaskUUID1", "testTaskUUID2", "testTaskUUID3"} log.INFO.Println(dynamodb.TestDynamoDBBackend.GetConfig()) err := dynamodb.TestDynamoDBBackend.InitGroup(groupUUID, taskUUIDs) assert.Nil(t, err) err = dynamodb.TestErrDynamoDBBackend.InitGroup(groupUUID, taskUUIDs) assert.NotNil(t, err) // assert proper TTL value is set in InitGroup() dynamodb.TestDynamoDBBackend.GetConfig().ResultsExpireIn = 3 * 3600 // results should expire after 3 hours client := dynamodb.TestDynamoDBBackend.GetClient().(*dynamodb.TestDynamoDBClient) // Override DynamoDB PutItem() behavior var isPutItemCalled bool client.PutItemOverride = func(input *awsdynamodb.PutItemInput) (*awsdynamodb.PutItemOutput, error) { isPutItemCalled = true assert.NotNil(t, input) actualTTLStr := *input.Item["TTL"].N expectedTTLTime := time.Now().Add(3 * time.Hour) assertTTLValue(t, expectedTTLTime, actualTTLStr) return &awsdynamodb.PutItemOutput{}, nil } err = dynamodb.TestDynamoDBBackend.InitGroup(groupUUID, taskUUIDs) assert.Nil(t, err) assert.True(t, isPutItemCalled) client.ResetOverrides() } func assertTTLValue(t *testing.T, expectedTTLTime time.Time, actualEncodedTTLValue string) { actualTTLTimestamp, err := strconv.ParseInt(actualEncodedTTLValue, 10, 64) assert.Nil(t, err) actualTTLTime := time.Unix(actualTTLTimestamp, 0) assert.WithinDuration(t, expectedTTLTime, actualTTLTime, time.Second) } func TestGroupCompleted(t *testing.T) { client := dynamodb.TestDynamoDBBackend.GetClient().(*dynamodb.TestDynamoDBClient) tableName := dynamodb.TestDynamoDBBackend.GetConfig().DynamoDB.TaskStatesTable // Override DynamoDB BatchGetItem() behavior var isBatchGetItemCalled bool client.BatchGetItemOverride = func(input *awsdynamodb.BatchGetItemInput) (*awsdynamodb.BatchGetItemOutput, error) { isBatchGetItemCalled = true assert.NotNil(t, input) assert.Nil(t, input.Validate()) return &awsdynamodb.BatchGetItemOutput{ Responses: map[string][]map[string]*awsdynamodb.AttributeValue{ tableName: { {"State": {S: aws.String(tasks.StateSuccess)}}, {"State": {S: aws.String(tasks.StateSuccess)}}, {"State": {S: aws.String(tasks.StateFailure)}}, }, }, }, nil } groupUUID := "testGroupUUID" isCompleted, err := dynamodb.TestDynamoDBBackend.GroupCompleted(groupUUID, 3) assert.Nil(t, err) assert.True(t, isCompleted) assert.True(t, isBatchGetItemCalled) client.ResetOverrides() } func TestGroupCompletedReturnsError(t *testing.T) { client := dynamodb.TestDynamoDBBackend.GetClient().(*dynamodb.TestDynamoDBClient) client.BatchGetItemOverride = func(input *awsdynamodb.BatchGetItemInput) (*awsdynamodb.BatchGetItemOutput, error) { return nil, fmt.Errorf("Simulating error from AWS") } isCompleted, err := dynamodb.TestDynamoDBBackend.GroupCompleted("test", 3) assert.NotNil(t, err) assert.False(t, isCompleted) client.ResetOverrides() } // TestGroupCompletedReturnsFalse tests that the GroupCompleted() returns false when some tasks have not yet finished. func TestGroupCompletedReturnsFalse(t *testing.T) { client := dynamodb.TestDynamoDBBackend.GetClient().(*dynamodb.TestDynamoDBClient) tableName := dynamodb.TestDynamoDBBackend.GetConfig().DynamoDB.TaskStatesTable // Override DynamoDB BatchGetItem() behavior client.BatchGetItemOverride = func(_ *awsdynamodb.BatchGetItemInput) (*awsdynamodb.BatchGetItemOutput, error) { return &awsdynamodb.BatchGetItemOutput{ Responses: map[string][]map[string]*awsdynamodb.AttributeValue{ tableName: { {"State": {S: aws.String(tasks.StateSuccess)}}, {"State": {S: aws.String(tasks.StateFailure)}}, {"State": {S: aws.String(tasks.StatePending)}}, }, }, }, nil } isCompleted, err := dynamodb.TestDynamoDBBackend.GroupCompleted("testGroup", 3) assert.Nil(t, err) assert.False(t, isCompleted) client.ResetOverrides() } // TestGroupCompletedReturnsFalse tests that the GroupCompleted() retries the the request until MaxFetchAttempts before returning an error func TestGroupCompletedRetries(t *testing.T) { client := dynamodb.TestDynamoDBBackend.GetClient().(*dynamodb.TestDynamoDBClient) tableName := dynamodb.TestDynamoDBBackend.GetConfig().DynamoDB.TaskStatesTable // Override DynamoDB BatchGetItem() behavior var countBatchGetItemAPICalls int client.BatchGetItemOverride = func(_ *awsdynamodb.BatchGetItemInput) (*awsdynamodb.BatchGetItemOutput, error) { countBatchGetItemAPICalls++ return &awsdynamodb.BatchGetItemOutput{ Responses: map[string][]map[string]*awsdynamodb.AttributeValue{ tableName: { {"State": {S: aws.String(tasks.StateSuccess)}}, }, }, UnprocessedKeys: map[string]*awsdynamodb.KeysAndAttributes{ tableName: { Keys: []map[string]*awsdynamodb.AttributeValue{ {"TaskUUID": {S: aws.String("unfetchedTaskUUID1")}}, {"TaskUUID": {S: aws.String("unfetchedTaskUUID2")}}, }, }, }, }, nil } _, err := dynamodb.TestDynamoDBBackend.GroupCompleted("testGroup", 3) assert.NotNil(t, err) assert.Equal(t, dynamodb.MaxFetchAttempts, countBatchGetItemAPICalls) client.ResetOverrides() } // TestGroupCompletedReturnsFalse tests that the GroupCompleted() retries the the request and returns success if all keys are fetched on retries. func TestGroupCompletedRetrieSuccess(t *testing.T) { client := dynamodb.TestDynamoDBBackend.GetClient().(*dynamodb.TestDynamoDBClient) tableName := dynamodb.TestDynamoDBBackend.GetConfig().DynamoDB.TaskStatesTable // Override DynamoDB BatchGetItem() behavior var countBatchGetItemAPICalls int client.BatchGetItemOverride = func(_ *awsdynamodb.BatchGetItemInput) (*awsdynamodb.BatchGetItemOutput, error) { countBatchGetItemAPICalls++ // simulate unfetched keys on 1st attempt. if countBatchGetItemAPICalls == 1 { return &awsdynamodb.BatchGetItemOutput{ Responses: map[string][]map[string]*awsdynamodb.AttributeValue{ tableName: {}, // no keys returned in this attempt. }, UnprocessedKeys: map[string]*awsdynamodb.KeysAndAttributes{ tableName: { Keys: []map[string]*awsdynamodb.AttributeValue{ {"TaskUUID": {S: aws.String("unfetchedTaskUUID1")}}, {"TaskUUID": {S: aws.String("unfetchedTaskUUID2")}}, {"TaskUUID": {S: aws.String("unfetchedTaskUUID3")}}, }, }, }, }, nil } // Return all keys in subsequent attempts. return &awsdynamodb.BatchGetItemOutput{ Responses: map[string][]map[string]*awsdynamodb.AttributeValue{ tableName: { {"State": {S: aws.String(tasks.StateSuccess)}}, {"State": {S: aws.String(tasks.StateSuccess)}}, {"State": {S: aws.String(tasks.StateSuccess)}}, }, }, }, nil } isCompleted, err := dynamodb.TestDynamoDBBackend.GroupCompleted("testGroup", 3) assert.Nil(t, err) assert.True(t, isCompleted) assert.Equal(t, 2, countBatchGetItemAPICalls) client.ResetOverrides() } func TestPrivateFuncGetGroupMeta(t *testing.T) { groupUUID := "testGroupUUID" meta, err := dynamodb.TestDynamoDBBackend.GetGroupMetaForTest(groupUUID) item := tasks.GroupMeta{ GroupUUID: "testGroupUUID", Lock: false, ChordTriggered: false, TaskUUIDs: []string{ "testTaskUUID1", "testTaskUUID2", "testTaskUUID3", }, } assert.Nil(t, err) assert.EqualValues(t, item, *meta) _, err = dynamodb.TestErrDynamoDBBackend.GetGroupMetaForTest(groupUUID) assert.NotNil(t, err) } func TestPrivateFuncUnmarshalTaskStateGetItemResult(t *testing.T) { result := awsdynamodb.GetItemOutput{ Item: map[string]*awsdynamodb.AttributeValue{ "Error": { NULL: aws.Bool(true), }, "State": { S: aws.String(tasks.StatePending), }, "TaskUUID": { S: aws.String("testTaskUUID1"), }, "Results:": { NULL: aws.Bool(true), }, }, } invalidResult := awsdynamodb.GetItemOutput{ Item: map[string]*awsdynamodb.AttributeValue{ "Error": { BOOL: aws.Bool(true), }, "State": { S: aws.String(tasks.StatePending), }, "TaskUUID": { S: aws.String("testTaskUUID1"), }, "Results:": { BOOL: aws.Bool(true), }, }, } item := tasks.TaskState{ TaskUUID: "testTaskUUID1", Results: nil, State: tasks.StatePending, Error: "", } state, err := dynamodb.TestErrDynamoDBBackend.UnmarshalTaskStateGetItemResultForTest(&result) assert.Nil(t, err) assert.EqualValues(t, item, *state) _, err = dynamodb.TestDynamoDBBackend.UnmarshalTaskStateGetItemResultForTest(nil) assert.NotNil(t, err) _, err = dynamodb.TestDynamoDBBackend.UnmarshalTaskStateGetItemResultForTest(&invalidResult) assert.NotNil(t, err) } func TestPrivateFuncUnmarshalGroupMetaGetItemResult(t *testing.T) { result := awsdynamodb.GetItemOutput{ Item: map[string]*awsdynamodb.AttributeValue{ "TaskUUIDs": { L: []*awsdynamodb.AttributeValue{ { S: aws.String("testTaskUUID1"), }, { S: aws.String("testTaskUUID2"), }, { S: aws.String("testTaskUUID3"), }, }, }, "ChordTriggered": { BOOL: aws.Bool(false), }, "GroupUUID": { S: aws.String("testGroupUUID"), }, "Lock": { BOOL: aws.Bool(false), }, }, } invalidResult := awsdynamodb.GetItemOutput{ Item: map[string]*awsdynamodb.AttributeValue{ "TaskUUIDs": { L: []*awsdynamodb.AttributeValue{ { S: aws.String("testTaskUUID1"), }, { S: aws.String("testTaskUUID2"), }, { S: aws.String("testTaskUUID3"), }, }, }, "ChordTriggered": { S: aws.String("false"), // this attribute is invalid }, "GroupUUID": { S: aws.String("testGroupUUID"), }, "Lock": { BOOL: aws.Bool(false), }, }, } item := tasks.GroupMeta{ GroupUUID: "testGroupUUID", Lock: false, ChordTriggered: false, TaskUUIDs: []string{ "testTaskUUID1", "testTaskUUID2", "testTaskUUID3", }, } meta, err := dynamodb.TestErrDynamoDBBackend.UnmarshalGroupMetaGetItemResultForTest(&result) assert.Nil(t, err) assert.EqualValues(t, item, *meta) _, err = dynamodb.TestErrDynamoDBBackend.UnmarshalGroupMetaGetItemResultForTest(nil) assert.NotNil(t, err) _, err = dynamodb.TestErrDynamoDBBackend.UnmarshalGroupMetaGetItemResultForTest(&invalidResult) assert.NotNil(t, err) } func TestPrivateFuncSetTaskState(t *testing.T) { signature := &tasks.Signature{ Name: "Test", Args: []tasks.Arg{ { Type: "int64", Value: 1, }, }, } state := tasks.NewPendingTaskState(signature) err := dynamodb.TestErrDynamoDBBackend.SetTaskStateForTest(state) assert.NotNil(t, err) err = dynamodb.TestDynamoDBBackend.SetTaskStateForTest(state) assert.Nil(t, err) } // verifyUpdateInput is a helper function to verify valid dynamoDB update input. func verifyUpdateInput(t *testing.T, input *awsdynamodb.UpdateItemInput, expectedTaskID string, expectedState string, expectedTTLTime time.Time) { assert.NotNil(t, input) // verify task ID assert.Equal(t, expectedTaskID, *input.Key["TaskUUID"].S) // verify task state assert.Equal(t, expectedState, *input.ExpressionAttributeValues[":s"].S) // Verify TTL if !expectedTTLTime.IsZero() { actualTTLStr := *input.ExpressionAttributeValues[":t"].N assertTTLValue(t, expectedTTLTime, actualTTLStr) } } func TestSetStateSuccess(t *testing.T) { signature := &tasks.Signature{UUID: "testTaskUUID"} // assert correct task ID, state and TTL value is set in SetStateSuccess() dynamodb.TestDynamoDBBackend.GetConfig().ResultsExpireIn = 3 * 3600 // results should expire after 3 hours client := dynamodb.TestDynamoDBBackend.GetClient().(*dynamodb.TestDynamoDBClient) // Override DynamoDB UpdateItem() behavior var isUpdateItemCalled bool client.UpdateItemOverride = func(input *awsdynamodb.UpdateItemInput) (*awsdynamodb.UpdateItemOutput, error) { isUpdateItemCalled = true verifyUpdateInput(t, input, signature.UUID, tasks.StateSuccess, time.Now().Add(3*time.Hour)) return &awsdynamodb.UpdateItemOutput{}, nil } err := dynamodb.TestDynamoDBBackend.SetStateSuccess(signature, nil) assert.Nil(t, err) assert.True(t, isUpdateItemCalled) client.ResetOverrides() } func TestSetStateFailure(t *testing.T) { signature := &tasks.Signature{UUID: "testTaskUUID"} // assert correct task ID, state and TTL value is set in SetStateFailure() dynamodb.TestDynamoDBBackend.GetConfig().ResultsExpireIn = 2 * 3600 // results should expire after 2 hours client := dynamodb.TestDynamoDBBackend.GetClient().(*dynamodb.TestDynamoDBClient) // Override DynamoDB UpdateItem() behavior var isUpdateItemCalled bool client.UpdateItemOverride = func(input *awsdynamodb.UpdateItemInput) (*awsdynamodb.UpdateItemOutput, error) { isUpdateItemCalled = true verifyUpdateInput(t, input, signature.UUID, tasks.StateFailure, time.Now().Add(2*time.Hour)) return &awsdynamodb.UpdateItemOutput{}, nil } err := dynamodb.TestDynamoDBBackend.SetStateFailure(signature, "Some error occurred") assert.Nil(t, err) assert.True(t, isUpdateItemCalled) client.ResetOverrides() } func TestSetStateReceived(t *testing.T) { signature := &tasks.Signature{UUID: "testTaskUUID"} // assert correct task ID, state and *no* TTL value is set in SetStateReceived() dynamodb.TestDynamoDBBackend.GetConfig().ResultsExpireIn = 2 * 3600 // results should expire after 2 hours (ignored for this state) client := dynamodb.TestDynamoDBBackend.GetClient().(*dynamodb.TestDynamoDBClient) var isUpdateItemCalled bool client.UpdateItemOverride = func(input *awsdynamodb.UpdateItemInput) (*awsdynamodb.UpdateItemOutput, error) { isUpdateItemCalled = true verifyUpdateInput(t, input, signature.UUID, tasks.StateReceived, time.Time{}) return &awsdynamodb.UpdateItemOutput{}, nil } err := dynamodb.TestDynamoDBBackend.SetStateReceived(signature) assert.Nil(t, err) assert.True(t, isUpdateItemCalled) client.ResetOverrides() } func TestSetStateStarted(t *testing.T) { signature := &tasks.Signature{UUID: "testTaskUUID"} // assert correct task ID, state and *no* TTL value is set in SetStateStarted() dynamodb.TestDynamoDBBackend.GetConfig().ResultsExpireIn = 2 * 3600 // results should expire after 2 hours (ignored for this state) client := dynamodb.TestDynamoDBBackend.GetClient().(*dynamodb.TestDynamoDBClient) var isUpdateItemCalled bool client.UpdateItemOverride = func(input *awsdynamodb.UpdateItemInput) (*awsdynamodb.UpdateItemOutput, error) { isUpdateItemCalled = true verifyUpdateInput(t, input, signature.UUID, tasks.StateStarted, time.Time{}) return &awsdynamodb.UpdateItemOutput{}, nil } err := dynamodb.TestDynamoDBBackend.SetStateStarted(signature) assert.Nil(t, err) assert.True(t, isUpdateItemCalled) client.ResetOverrides() } func TestSetStateRetry(t *testing.T) { signature := &tasks.Signature{UUID: "testTaskUUID"} // assert correct task ID, state and *no* TTL value is set in SetStateStarted() dynamodb.TestDynamoDBBackend.GetConfig().ResultsExpireIn = 2 * 3600 // results should expire after 2 hours (ignored for this state) client := dynamodb.TestDynamoDBBackend.GetClient().(*dynamodb.TestDynamoDBClient) var isUpdateItemCalled bool client.UpdateItemOverride = func(input *awsdynamodb.UpdateItemInput) (*awsdynamodb.UpdateItemOutput, error) { isUpdateItemCalled = true verifyUpdateInput(t, input, signature.UUID, tasks.StateRetry, time.Time{}) return &awsdynamodb.UpdateItemOutput{}, nil } err := dynamodb.TestDynamoDBBackend.SetStateRetry(signature) assert.Nil(t, err) assert.True(t, isUpdateItemCalled) client.ResetOverrides() } func TestGroupTaskStates(t *testing.T) { expectedStates := map[string]*tasks.TaskState{ "testTaskUUID1": { TaskUUID: "testTaskUUID1", Results: nil, State: tasks.StatePending, Error: "", }, "testTaskUUID2": { TaskUUID: "testTaskUUID2", Results: nil, State: tasks.StateStarted, Error: "", }, "testTaskUUID3": { TaskUUID: "testTaskUUID3", Results: nil, State: tasks.StateSuccess, Error: "", }, } client := dynamodb.TestDynamoDBBackend.GetClient().(*dynamodb.TestDynamoDBClient) tableName := dynamodb.TestDynamoDBBackend.GetConfig().DynamoDB.TaskStatesTable client.BatchGetItemOverride = func(input *awsdynamodb.BatchGetItemInput) (*awsdynamodb.BatchGetItemOutput, error) { assert.Nil(t, input.Validate()) return &awsdynamodb.BatchGetItemOutput{ Responses: map[string][]map[string]*awsdynamodb.AttributeValue{ tableName: { { "TaskUUID": {S: aws.String("testTaskUUID1")}, "Results:": {NULL: aws.Bool(true)}, "State": {S: aws.String(tasks.StatePending)}, "Error": {NULL: aws.Bool(true)}, }, { "TaskUUID": {S: aws.String("testTaskUUID2")}, "Results:": {NULL: aws.Bool(true)}, "State": {S: aws.String(tasks.StateStarted)}, "Error": {NULL: aws.Bool(true)}, }, { "TaskUUID": {S: aws.String("testTaskUUID3")}, "Results:": {NULL: aws.Bool(true)}, "State": {S: aws.String(tasks.StateSuccess)}, "Error": {NULL: aws.Bool(true)}, }, }, }, }, nil } defer client.ResetOverrides() states, err := dynamodb.TestDynamoDBBackend.GroupTaskStates("testGroupUUID", 3) assert.Nil(t, err) for _, s := range states { assert.EqualValues(t, *s, *expectedStates[s.TaskUUID]) } } func TestTriggerChord(t *testing.T) { groupUUID := "testGroupUUID" triggered, err := dynamodb.TestDynamoDBBackend.TriggerChord(groupUUID) assert.Nil(t, err) assert.True(t, triggered) } func TestGetState(t *testing.T) { taskUUID := "testTaskUUID1" expectedState := &tasks.TaskState{ TaskUUID: "testTaskUUID1", Results: nil, State: tasks.StatePending, Error: "", } client := dynamodb.TestDynamoDBBackend.GetClient().(*dynamodb.TestDynamoDBClient) client.GetItemOverride = func(input *awsdynamodb.GetItemInput) (*awsdynamodb.GetItemOutput, error) { return &awsdynamodb.GetItemOutput{ Item: map[string]*awsdynamodb.AttributeValue{ "TaskUUID": {S: aws.String("testTaskUUID1")}, "Results:": {NULL: aws.Bool(true)}, "State": {S: aws.String(tasks.StatePending)}, "Error": {NULL: aws.Bool(false)}, }, }, nil } defer client.ResetOverrides() state, err := dynamodb.TestDynamoDBBackend.GetState(taskUUID) assert.Nil(t, err) assert.EqualValues(t, expectedState, state) } func TestPurgeState(t *testing.T) { taskUUID := "testTaskUUID1" err := dynamodb.TestDynamoDBBackend.PurgeState(taskUUID) assert.Nil(t, err) err = dynamodb.TestErrDynamoDBBackend.PurgeState(taskUUID) assert.NotNil(t, err) } func TestPurgeGroupMeta(t *testing.T) { groupUUID := "GroupUUID" err := dynamodb.TestDynamoDBBackend.PurgeGroupMeta(groupUUID) assert.Nil(t, err) err = dynamodb.TestErrDynamoDBBackend.PurgeGroupMeta(groupUUID) assert.NotNil(t, err) } func TestPrivateFuncLockGroupMeta(t *testing.T) { groupUUID := "GroupUUID" err := dynamodb.TestDynamoDBBackend.LockGroupMetaForTest(groupUUID) assert.Nil(t, err) err = dynamodb.TestErrDynamoDBBackend.LockGroupMetaForTest(groupUUID) assert.NotNil(t, err) } func TestPrivateFuncUnLockGroupMeta(t *testing.T) { groupUUID := "GroupUUID" err := dynamodb.TestDynamoDBBackend.UnlockGroupMetaForTest(groupUUID) assert.Nil(t, err) err = dynamodb.TestErrDynamoDBBackend.UnlockGroupMetaForTest(groupUUID) assert.NotNil(t, err) } func TestPrivateFuncChordTriggered(t *testing.T) { groupUUID := "GroupUUID" err := dynamodb.TestDynamoDBBackend.ChordTriggeredForTest(groupUUID) assert.Nil(t, err) err = dynamodb.TestErrDynamoDBBackend.ChordTriggeredForTest(groupUUID) assert.NotNil(t, err) } func TestDynamoDBPrivateFuncUpdateGroupMetaLock(t *testing.T) { groupUUID := "GroupUUID" err := dynamodb.TestDynamoDBBackend.UpdateGroupMetaLockForTest(groupUUID, true) assert.Nil(t, err) err = dynamodb.TestErrDynamoDBBackend.UpdateGroupMetaLockForTest(groupUUID, true) assert.NotNil(t, err) } func TestPrivateFuncUpdateToFailureStateWithError(t *testing.T) { signature := &tasks.Signature{ Name: "Test", Args: []tasks.Arg{ { Type: "int64", Value: 1, }, }, } state := tasks.NewFailureTaskState(signature, "This is an error") err := dynamodb.TestDynamoDBBackend.UpdateToFailureStateWithErrorForTest(state) assert.Nil(t, err) } func TestPrivateFuncTableExistsForTest(t *testing.T) { tables := []*string{aws.String("foo")} assert.False(t, dynamodb.TestDynamoDBBackend.TableExistsForTest("bar", tables)) assert.True(t, dynamodb.TestDynamoDBBackend.TableExistsForTest("foo", tables)) } func TestPrivateFuncCheckRequiredTablesIfExistForTest(t *testing.T) { err := dynamodb.TestDynamoDBBackend.CheckRequiredTablesIfExistForTest() assert.Nil(t, err) taskTable := dynamodb.TestDynamoDBBackend.GetConfig().DynamoDB.TaskStatesTable groupTable := dynamodb.TestDynamoDBBackend.GetConfig().DynamoDB.GroupMetasTable err = dynamodb.TestErrDynamoDBBackend.CheckRequiredTablesIfExistForTest() assert.NotNil(t, err) dynamodb.TestDynamoDBBackend.GetConfig().DynamoDB.TaskStatesTable = "foo" err = dynamodb.TestDynamoDBBackend.CheckRequiredTablesIfExistForTest() assert.NotNil(t, err) dynamodb.TestDynamoDBBackend.GetConfig().DynamoDB.TaskStatesTable = taskTable dynamodb.TestDynamoDBBackend.GetConfig().DynamoDB.GroupMetasTable = "foo" err = dynamodb.TestDynamoDBBackend.CheckRequiredTablesIfExistForTest() assert.NotNil(t, err) dynamodb.TestDynamoDBBackend.GetConfig().DynamoDB.GroupMetasTable = groupTable } ================================================ FILE: v1/backends/eager/eager.go ================================================ package eager import ( "bytes" "encoding/json" "fmt" "sync" "github.com/RichardKnop/machinery/v1/backends/iface" "github.com/RichardKnop/machinery/v1/common" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/tasks" ) // ErrGroupNotFound ... type ErrGroupNotFound struct { groupUUID string } // NewErrGroupNotFound returns new instance of ErrGroupNotFound func NewErrGroupNotFound(groupUUID string) ErrGroupNotFound { return ErrGroupNotFound{groupUUID: groupUUID} } // Error implements error interface func (e ErrGroupNotFound) Error() string { return fmt.Sprintf("Group not found: %v", e.groupUUID) } // ErrTasknotFound ... type ErrTasknotFound struct { taskUUID string } // NewErrTasknotFound returns new instance of ErrTasknotFound func NewErrTasknotFound(taskUUID string) ErrTasknotFound { return ErrTasknotFound{taskUUID: taskUUID} } // Error implements error interface func (e ErrTasknotFound) Error() string { return fmt.Sprintf("Task not found: %v", e.taskUUID) } // Backend represents an "eager" in-memory result backend type Backend struct { common.Backend groups map[string][]string tasks map[string][]byte stateMutex sync.Mutex } // New creates EagerBackend instance func New() iface.Backend { return &Backend{ Backend: common.NewBackend(new(config.Config)), groups: make(map[string][]string), tasks: make(map[string][]byte), } } // InitGroup creates and saves a group meta data object func (b *Backend) InitGroup(groupUUID string, taskUUIDs []string) error { tasks := make([]string, 0, len(taskUUIDs)) // copy every task tasks = append(tasks, taskUUIDs...) b.groups[groupUUID] = tasks return nil } // GroupCompleted returns true if all tasks in a group finished func (b *Backend) GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) { tasks, ok := b.groups[groupUUID] if !ok { return false, NewErrGroupNotFound(groupUUID) } var countSuccessTasks = 0 for _, v := range tasks { t, err := b.GetState(v) if err != nil { return false, err } if t.IsCompleted() { countSuccessTasks++ } } return countSuccessTasks == groupTaskCount, nil } // GroupTaskStates returns states of all tasks in the group func (b *Backend) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error) { taskUUIDs, ok := b.groups[groupUUID] if !ok { return nil, NewErrGroupNotFound(groupUUID) } ret := make([]*tasks.TaskState, 0, groupTaskCount) for _, taskUUID := range taskUUIDs { t, err := b.GetState(taskUUID) if err != nil { return nil, err } ret = append(ret, t) } return ret, nil } // TriggerChord flags chord as triggered in the backend storage to make sure // chord is never trigerred multiple times. Returns a boolean flag to indicate // whether the worker should trigger chord (true) or no if it has been triggered // already (false) func (b *Backend) TriggerChord(groupUUID string) (bool, error) { return true, nil } // SetStatePending updates task state to PENDING func (b *Backend) SetStatePending(signature *tasks.Signature) error { state := tasks.NewPendingTaskState(signature) return b.updateState(state) } // SetStateReceived updates task state to RECEIVED func (b *Backend) SetStateReceived(signature *tasks.Signature) error { state := tasks.NewReceivedTaskState(signature) return b.updateState(state) } // SetStateStarted updates task state to STARTED func (b *Backend) SetStateStarted(signature *tasks.Signature) error { state := tasks.NewStartedTaskState(signature) return b.updateState(state) } // SetStateRetry updates task state to RETRY func (b *Backend) SetStateRetry(signature *tasks.Signature) error { state := tasks.NewRetryTaskState(signature) return b.updateState(state) } // SetStateSuccess updates task state to SUCCESS func (b *Backend) SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error { state := tasks.NewSuccessTaskState(signature, results) return b.updateState(state) } // SetStateFailure updates task state to FAILURE func (b *Backend) SetStateFailure(signature *tasks.Signature, err string) error { state := tasks.NewFailureTaskState(signature, err) return b.updateState(state) } // GetState returns the latest task state func (b *Backend) GetState(taskUUID string) (*tasks.TaskState, error) { tasktStateBytes, ok := b.tasks[taskUUID] if !ok { return nil, NewErrTasknotFound(taskUUID) } state := new(tasks.TaskState) decoder := json.NewDecoder(bytes.NewReader(tasktStateBytes)) decoder.UseNumber() if err := decoder.Decode(state); err != nil { return nil, fmt.Errorf("Failed to unmarshal task state %v", b) } return state, nil } // PurgeState deletes stored task state func (b *Backend) PurgeState(taskUUID string) error { _, ok := b.tasks[taskUUID] if !ok { return NewErrTasknotFound(taskUUID) } delete(b.tasks, taskUUID) return nil } // PurgeGroupMeta deletes stored group meta data func (b *Backend) PurgeGroupMeta(groupUUID string) error { _, ok := b.groups[groupUUID] if !ok { return NewErrGroupNotFound(groupUUID) } delete(b.groups, groupUUID) return nil } func (b *Backend) updateState(s *tasks.TaskState) error { // simulate the behavior of json marshal/unmarshal b.stateMutex.Lock() defer b.stateMutex.Unlock() msg, err := json.Marshal(s) if err != nil { return fmt.Errorf("Marshal task state error: %v", err) } b.tasks[s.TaskUUID] = msg return nil } ================================================ FILE: v1/backends/eager/eager_test.go ================================================ package eager_test import ( "encoding/json" "testing" "github.com/RichardKnop/machinery/v1/backends/eager" "github.com/RichardKnop/machinery/v1/backends/iface" "github.com/RichardKnop/machinery/v1/tasks" "github.com/stretchr/testify/suite" ) type EagerBackendTestSuite struct { suite.Suite backend iface.Backend st []*tasks.Signature groups []struct { id string tasks []string } } func (s *EagerBackendTestSuite) SetupSuite() { // prepare common test data s.backend = eager.New() // 2 non-group state s.st = []*tasks.Signature{ {UUID: "1"}, {UUID: "2"}, {UUID: "3"}, {UUID: "4"}, {UUID: "5"}, {UUID: "6"}, } for _, t := range s.st { s.backend.SetStatePending(t) } // groups s.groups = []struct { id string tasks []string }{ {"group1", []string{"1-3", "1-4"}}, {"group2", []string{"2-1", "2-2", "2-3"}}, {"group3", []string(nil)}, {"group4", []string{"4-1", "4-2", "4-3", "4-4"}}, {"group5", []string{"5-1", "5-2"}}, } for _, g := range s.groups { for _, t := range g.tasks { sig := &tasks.Signature{ UUID: t, GroupUUID: g.id, GroupTaskCount: len(g.tasks), } s.st = append(s.st, sig) // default state is pending s.backend.SetStatePending(sig) } s.Nil(s.backend.InitGroup(g.id, g.tasks)) } // prepare for TestInitGroup s.Nil(s.backend.PurgeGroupMeta(s.groups[4].id)) } // // Test Cases // func (s *EagerBackendTestSuite) TestInitGroup() { // group 5 { g := s.groups[4] s.Nil(s.backend.InitGroup(g.id, g.tasks)) } // group3 -- nil as task list { g := s.groups[2] s.Nil(s.backend.InitGroup(g.id, g.tasks)) } } func (s *EagerBackendTestSuite) TestGroupCompleted() { // group 1 { // all tasks are pending g := s.groups[0] completed, err := s.backend.GroupCompleted(g.id, len(g.tasks)) s.False(completed) s.Nil(err) // make these tasks success for _, id := range g.tasks { t := s.getTaskSignature(id) s.NotNil(t) if t == nil { break } s.backend.SetStateSuccess(t, nil) } completed, err = s.backend.GroupCompleted(g.id, len(g.tasks)) s.True(completed) s.Nil(err) } // group 2 { g := s.groups[1] completed, err := s.backend.GroupCompleted(g.id, len(g.tasks)) s.False(completed) s.Nil(err) // make these tasks failure for _, id := range g.tasks { t := s.getTaskSignature(id) s.NotNil(t) if t == nil { break } s.backend.SetStateFailure(t, "just a test") } completed, err = s.backend.GroupCompleted(g.id, len(g.tasks)) s.True(completed) s.Nil(err) } { // call on a not-existed group completed, err := s.backend.GroupCompleted("", 0) s.False(completed) s.NotNil(err) } } func (s *EagerBackendTestSuite) TestGroupTaskStates() { // group 4 { g := s.groups[3] // set failure state with taskUUID as error message for _, id := range g.tasks { t := s.getTaskSignature(id) s.NotNil(t) if t == nil { break } s.backend.SetStateFailure(t, t.UUID) } // get states back ts, err := s.backend.GroupTaskStates(g.id, len(g.tasks)) s.NotNil(ts) s.Nil(err) for _, t := range ts { s.Equal(t.TaskUUID, t.Error) } } { // call on a not-existed group ts, err := s.backend.GroupTaskStates("", 0) s.Nil(ts) s.NotNil(err) } } func (s *EagerBackendTestSuite) TestSetStatePending() { // task 1 { t := s.st[0] // change this state to receiving s.backend.SetStateReceived(t) // change it back to pending s.backend.SetStatePending(t) st, err := s.backend.GetState(t.UUID) s.Nil(err) if st != nil { s.Equal(tasks.StatePending, st.State) } } } func (s *EagerBackendTestSuite) TestSetStateReceived() { // task2 { t := s.st[1] s.backend.SetStateReceived(t) st, err := s.backend.GetState(t.UUID) s.Nil(err) if st != nil { s.Equal(tasks.StateReceived, st.State) } } } func (s *EagerBackendTestSuite) TestSetStateStarted() { // task3 { t := s.st[2] s.backend.SetStateStarted(t) st, err := s.backend.GetState(t.UUID) s.Nil(err) if st != nil { s.Equal(tasks.StateStarted, st.State) } } } func (s *EagerBackendTestSuite) TestSetStateSuccess() { // task4 { t := s.st[3] taskResults := []*tasks.TaskResult{ { Type: "float64", Value: json.Number("300.0"), }, } s.backend.SetStateSuccess(t, taskResults) st, err := s.backend.GetState(t.UUID) s.Nil(err) s.NotNil(st) s.Equal(tasks.StateSuccess, st.State) s.Equal(taskResults, st.Results) } } func (s *EagerBackendTestSuite) TestSetStateFailure() { // task5 { t := s.st[4] s.backend.SetStateFailure(t, "error") st, err := s.backend.GetState(t.UUID) s.Nil(err) if st != nil { s.Equal(tasks.StateFailure, st.State) s.Equal("error", st.Error) } } } func (s *EagerBackendTestSuite) TestSetStateRetry() { // task6 { t := s.st[5] s.backend.SetStateRetry(t) st, err := s.backend.GetState(t.UUID) s.Nil(err) if st != nil { s.Equal(tasks.StateRetry, st.State) } } } func (s *EagerBackendTestSuite) TestGetState() { // get something not existed -- empty string st, err := s.backend.GetState("") s.Nil(st) s.NotNil(err) } func (s *EagerBackendTestSuite) TestPurgeState() { // task6 { t := s.st[5] st, err := s.backend.GetState(t.UUID) s.NotNil(st) s.Nil(err) // purge it s.Nil(s.backend.PurgeState(t.UUID)) // should be not found st, err = s.backend.GetState(t.UUID) s.Nil(st) s.NotNil(err) } { // purge a not-existed state s.NotNil(s.backend.PurgeState("")) } } func (s *EagerBackendTestSuite) TestPurgeGroupMeta() { // group4 { g := s.groups[3] ts, err := s.backend.GroupTaskStates(g.id, len(g.tasks)) s.NotNil(ts) s.Nil(err) // purge group s.Nil(s.backend.PurgeGroupMeta(g.id)) // should be not found ts, err = s.backend.GroupTaskStates(g.id, len(g.tasks)) s.Nil(ts) s.NotNil(err) } { // purge a not-existed group s.NotNil(s.backend.PurgeGroupMeta("")) } } // // internal method // func (s *EagerBackendTestSuite) getTaskSignature(taskUUID string) *tasks.Signature { for _, v := range s.st { if v.UUID == taskUUID { return v } } return nil } func TestEagerBackendMain(t *testing.T) { suite.Run(t, &EagerBackendTestSuite{}) } ================================================ FILE: v1/backends/iface/interfaces.go ================================================ package iface import ( "github.com/RichardKnop/machinery/v1/tasks" ) // Backend - a common interface for all result backends type Backend interface { // Group related functions InitGroup(groupUUID string, taskUUIDs []string) error GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error) TriggerChord(groupUUID string) (bool, error) // Setting / getting task state SetStatePending(signature *tasks.Signature) error SetStateReceived(signature *tasks.Signature) error SetStateStarted(signature *tasks.Signature) error SetStateRetry(signature *tasks.Signature) error SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error SetStateFailure(signature *tasks.Signature, err string) error GetState(taskUUID string) (*tasks.TaskState, error) // Purging stored stored tasks states and group meta data IsAMQP() bool PurgeState(taskUUID string) error PurgeGroupMeta(groupUUID string) error } ================================================ FILE: v1/backends/memcache/memcache.go ================================================ package memcache import ( "bytes" "encoding/json" "time" "github.com/RichardKnop/machinery/v1/backends/iface" "github.com/RichardKnop/machinery/v1/common" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/log" "github.com/RichardKnop/machinery/v1/tasks" gomemcache "github.com/bradfitz/gomemcache/memcache" ) // Backend represents a Memcache result backend type Backend struct { common.Backend servers []string client *gomemcache.Client } // New creates Backend instance func New(cnf *config.Config, servers []string) iface.Backend { return &Backend{ Backend: common.NewBackend(cnf), servers: servers, } } // InitGroup creates and saves a group meta data object func (b *Backend) InitGroup(groupUUID string, taskUUIDs []string) error { groupMeta := &tasks.GroupMeta{ GroupUUID: groupUUID, TaskUUIDs: taskUUIDs, CreatedAt: time.Now().UTC(), } encoded, err := json.Marshal(&groupMeta) if err != nil { return err } return b.getClient().Set(&gomemcache.Item{ Key: groupUUID, Value: encoded, Expiration: b.getExpirationTimestamp(), }) } // GroupCompleted returns true if all tasks in a group finished func (b *Backend) GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) { groupMeta, err := b.getGroupMeta(groupUUID) if err != nil { return false, err } taskStates, err := b.getStates(groupMeta.TaskUUIDs...) if err != nil { return false, err } var countSuccessTasks = 0 for _, taskState := range taskStates { if taskState.IsCompleted() { countSuccessTasks++ } } return countSuccessTasks == groupTaskCount, nil } // GroupTaskStates returns states of all tasks in the group func (b *Backend) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error) { groupMeta, err := b.getGroupMeta(groupUUID) if err != nil { return []*tasks.TaskState{}, err } return b.getStates(groupMeta.TaskUUIDs...) } // TriggerChord flags chord as triggered in the backend storage to make sure // chord is never trigerred multiple times. Returns a boolean flag to indicate // whether the worker should trigger chord (true) or no if it has been triggered // already (false) func (b *Backend) TriggerChord(groupUUID string) (bool, error) { groupMeta, err := b.getGroupMeta(groupUUID) if err != nil { return false, err } // Chord has already been triggered, return false (should not trigger again) if groupMeta.ChordTriggered { return false, nil } // If group meta is locked, wait until it's unlocked for groupMeta.Lock { groupMeta, _ = b.getGroupMeta(groupUUID) log.WARNING.Print("Group meta locked, waiting") time.Sleep(time.Millisecond * 5) } // Acquire lock if err = b.lockGroupMeta(groupMeta); err != nil { return false, err } defer b.unlockGroupMeta(groupMeta) // Update the group meta data groupMeta.ChordTriggered = true encoded, err := json.Marshal(&groupMeta) if err != nil { return false, err } if err = b.getClient().Replace(&gomemcache.Item{ Key: groupUUID, Value: encoded, Expiration: b.getExpirationTimestamp(), }); err != nil { return false, err } return true, nil } // SetStatePending updates task state to PENDING func (b *Backend) SetStatePending(signature *tasks.Signature) error { taskState := tasks.NewPendingTaskState(signature) return b.updateState(taskState) } // SetStateReceived updates task state to RECEIVED func (b *Backend) SetStateReceived(signature *tasks.Signature) error { taskState := tasks.NewReceivedTaskState(signature) return b.updateState(taskState) } // SetStateStarted updates task state to STARTED func (b *Backend) SetStateStarted(signature *tasks.Signature) error { taskState := tasks.NewStartedTaskState(signature) return b.updateState(taskState) } // SetStateRetry updates task state to RETRY func (b *Backend) SetStateRetry(signature *tasks.Signature) error { state := tasks.NewRetryTaskState(signature) return b.updateState(state) } // SetStateSuccess updates task state to SUCCESS func (b *Backend) SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error { taskState := tasks.NewSuccessTaskState(signature, results) return b.updateState(taskState) } // SetStateFailure updates task state to FAILURE func (b *Backend) SetStateFailure(signature *tasks.Signature, err string) error { taskState := tasks.NewFailureTaskState(signature, err) return b.updateState(taskState) } // GetState returns the latest task state func (b *Backend) GetState(taskUUID string) (*tasks.TaskState, error) { item, err := b.getClient().Get(taskUUID) if err != nil { return nil, err } state := new(tasks.TaskState) decoder := json.NewDecoder(bytes.NewReader(item.Value)) decoder.UseNumber() if err := decoder.Decode(state); err != nil { return nil, err } return state, nil } // PurgeState deletes stored task state func (b *Backend) PurgeState(taskUUID string) error { return b.getClient().Delete(taskUUID) } // PurgeGroupMeta deletes stored group meta data func (b *Backend) PurgeGroupMeta(groupUUID string) error { return b.getClient().Delete(groupUUID) } // updateState saves current task state func (b *Backend) updateState(taskState *tasks.TaskState) error { encoded, err := json.Marshal(taskState) if err != nil { return err } return b.getClient().Set(&gomemcache.Item{ Key: taskState.TaskUUID, Value: encoded, Expiration: b.getExpirationTimestamp(), }) } // lockGroupMeta acquires lock on group meta data func (b *Backend) lockGroupMeta(groupMeta *tasks.GroupMeta) error { groupMeta.Lock = true encoded, err := json.Marshal(groupMeta) if err != nil { return err } return b.getClient().Set(&gomemcache.Item{ Key: groupMeta.GroupUUID, Value: encoded, Expiration: b.getExpirationTimestamp(), }) } // unlockGroupMeta releases lock on group meta data func (b *Backend) unlockGroupMeta(groupMeta *tasks.GroupMeta) error { groupMeta.Lock = false encoded, err := json.Marshal(groupMeta) if err != nil { return err } return b.getClient().Set(&gomemcache.Item{ Key: groupMeta.GroupUUID, Value: encoded, Expiration: b.getExpirationTimestamp(), }) } // getGroupMeta retrieves group meta data, convenience function to avoid repetition func (b *Backend) getGroupMeta(groupUUID string) (*tasks.GroupMeta, error) { item, err := b.getClient().Get(groupUUID) if err != nil { return nil, err } groupMeta := new(tasks.GroupMeta) decoder := json.NewDecoder(bytes.NewReader(item.Value)) decoder.UseNumber() if err := decoder.Decode(groupMeta); err != nil { return nil, err } return groupMeta, nil } // getStates returns multiple task states func (b *Backend) getStates(taskUUIDs ...string) ([]*tasks.TaskState, error) { states := make([]*tasks.TaskState, len(taskUUIDs)) for i, taskUUID := range taskUUIDs { item, err := b.getClient().Get(taskUUID) if err != nil { return nil, err } state := new(tasks.TaskState) decoder := json.NewDecoder(bytes.NewReader(item.Value)) decoder.UseNumber() if err := decoder.Decode(state); err != nil { return nil, err } states[i] = state } return states, nil } // getExpirationTimestamp returns expiration timestamp func (b *Backend) getExpirationTimestamp() int32 { expiresIn := b.GetConfig().ResultsExpireIn if expiresIn == 0 { // // expire results after 1 hour by default expiresIn = config.DefaultResultsExpireIn } return int32(time.Now().Unix() + int64(expiresIn)) } // getClient returns or creates instance of Memcache client func (b *Backend) getClient() *gomemcache.Client { if b.client == nil { b.client = gomemcache.New(b.servers...) } return b.client } ================================================ FILE: v1/backends/memcache/memcache_test.go ================================================ package memcache_test import ( "os" "testing" "time" "github.com/RichardKnop/machinery/v1/backends/memcache" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/tasks" "github.com/stretchr/testify/assert" ) func TestGroupCompleted(t *testing.T) { memcacheURL := os.Getenv("MEMCACHE_URL") if memcacheURL == "" { t.Skip("MEMCACHE_URL is not defined") } groupUUID := "testGroupUUID" task1 := &tasks.Signature{ UUID: "testTaskUUID1", GroupUUID: groupUUID, } task2 := &tasks.Signature{ UUID: "testTaskUUID2", GroupUUID: groupUUID, } backend := memcache.New(new(config.Config), []string{memcacheURL}) // Cleanup before the test backend.PurgeState(task1.UUID) backend.PurgeState(task2.UUID) backend.PurgeGroupMeta(groupUUID) groupCompleted, err := backend.GroupCompleted(groupUUID, 2) if assert.Error(t, err) { assert.False(t, groupCompleted) assert.Equal(t, "memcache: cache miss", err.Error()) } backend.InitGroup(groupUUID, []string{task1.UUID, task2.UUID}) groupCompleted, err = backend.GroupCompleted(groupUUID, 2) if assert.Error(t, err) { assert.False(t, groupCompleted) assert.Equal(t, "memcache: cache miss", err.Error()) } backend.SetStatePending(task1) backend.SetStateStarted(task2) groupCompleted, err = backend.GroupCompleted(groupUUID, 2) if assert.NoError(t, err) { assert.False(t, groupCompleted) } taskResults := []*tasks.TaskResult{new(tasks.TaskResult)} backend.SetStateStarted(task1) backend.SetStateSuccess(task2, taskResults) groupCompleted, err = backend.GroupCompleted(groupUUID, 2) if assert.NoError(t, err) { assert.False(t, groupCompleted) } backend.SetStateFailure(task1, "Some error") groupCompleted, err = backend.GroupCompleted(groupUUID, 2) if assert.NoError(t, err) { assert.True(t, groupCompleted) } } func TestGetState(t *testing.T) { memcacheURL := os.Getenv("MEMCACHE_URL") if memcacheURL == "" { t.Skip("MEMCACHE_URL is not defined") } signature := &tasks.Signature{ UUID: "testTaskUUID", GroupUUID: "testGroupUUID", } backend := memcache.New(new(config.Config), []string{memcacheURL}) go func() { backend.SetStatePending(signature) time.Sleep(2 * time.Millisecond) backend.SetStateReceived(signature) time.Sleep(2 * time.Millisecond) backend.SetStateStarted(signature) time.Sleep(2 * time.Millisecond) taskResults := []*tasks.TaskResult{ { Type: "float64", Value: 2, }, } backend.SetStateSuccess(signature, taskResults) }() var ( taskState *tasks.TaskState err error ) for { taskState, err = backend.GetState(signature.UUID) if taskState == nil { assert.Equal(t, "memcache: cache miss", err.Error()) continue } assert.NoError(t, err) if taskState.IsCompleted() { break } } } func TestPurgeState(t *testing.T) { memcacheURL := os.Getenv("MEMCACHE_URL") if memcacheURL == "" { t.Skip("MEMCACHE_URL is not defined") } signature := &tasks.Signature{ UUID: "testTaskUUID", GroupUUID: "testGroupUUID", } backend := memcache.New(new(config.Config), []string{memcacheURL}) backend.SetStatePending(signature) taskState, err := backend.GetState(signature.UUID) assert.NotNil(t, taskState) assert.NoError(t, err) backend.PurgeState(taskState.TaskUUID) taskState, err = backend.GetState(signature.UUID) assert.Nil(t, taskState) assert.Error(t, err) } ================================================ FILE: v1/backends/mongo/mongodb.go ================================================ package mongo import ( "context" "encoding/json" "fmt" "reflect" "strings" "sync" "time" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "github.com/RichardKnop/machinery/v1/backends/iface" "github.com/RichardKnop/machinery/v1/common" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/log" "github.com/RichardKnop/machinery/v1/tasks" ) // Backend represents a MongoDB result backend type Backend struct { common.Backend client *mongo.Client tc *mongo.Collection gmc *mongo.Collection once sync.Once } // New creates Backend instance func New(cnf *config.Config) (iface.Backend, error) { backend := &Backend{ Backend: common.NewBackend(cnf), once: sync.Once{}, } return backend, nil } // InitGroup creates and saves a group meta data object func (b *Backend) InitGroup(groupUUID string, taskUUIDs []string) error { groupMeta := &tasks.GroupMeta{ GroupUUID: groupUUID, TaskUUIDs: taskUUIDs, CreatedAt: time.Now().UTC(), } _, err := b.groupMetasCollection().InsertOne(context.Background(), groupMeta) return err } // GroupCompleted returns true if all tasks in a group finished func (b *Backend) GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) { groupMeta, err := b.getGroupMeta(groupUUID) if err != nil { return false, err } taskStates, err := b.getStates(groupMeta.TaskUUIDs...) if err != nil { return false, err } var countSuccessTasks = 0 for _, taskState := range taskStates { if taskState.IsCompleted() { countSuccessTasks++ } } return countSuccessTasks == groupTaskCount, nil } // GroupTaskStates returns states of all tasks in the group func (b *Backend) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error) { groupMeta, err := b.getGroupMeta(groupUUID) if err != nil { return []*tasks.TaskState{}, err } return b.getStates(groupMeta.TaskUUIDs...) } // TriggerChord flags chord as triggered in the backend storage to make sure // chord is never triggered multiple times. Returns a boolean flag to indicate // whether the worker should trigger chord (true) or no if it has been triggered // already (false) func (b *Backend) TriggerChord(groupUUID string) (bool, error) { query := bson.M{ "_id": groupUUID, "chord_triggered": false, } change := bson.M{ "$set": bson.M{ "chord_triggered": true, }, } _, err := b.groupMetasCollection().UpdateOne(context.Background(), query, change, options.Update()) if err != nil { if err == mongo.ErrNoDocuments { log.WARNING.Printf("Chord already triggered for group %s", groupUUID) return false, nil } return false, err } return true, nil } // SetStatePending updates task state to PENDING func (b *Backend) SetStatePending(signature *tasks.Signature) error { update := bson.M{ "state": tasks.StatePending, "task_name": signature.Name, "created_at": time.Now().UTC(), } return b.updateState(signature, update) } // SetStateReceived updates task state to RECEIVED func (b *Backend) SetStateReceived(signature *tasks.Signature) error { update := bson.M{"state": tasks.StateReceived} return b.updateState(signature, update) } // SetStateStarted updates task state to STARTED func (b *Backend) SetStateStarted(signature *tasks.Signature) error { update := bson.M{"state": tasks.StateStarted} return b.updateState(signature, update) } // SetStateRetry updates task state to RETRY func (b *Backend) SetStateRetry(signature *tasks.Signature) error { update := bson.M{"state": tasks.StateRetry} return b.updateState(signature, update) } // SetStateSuccess updates task state to SUCCESS func (b *Backend) SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error { decodedResults := b.decodeResults(results) update := bson.M{ "state": tasks.StateSuccess, "results": decodedResults, } return b.updateState(signature, update) } // decodeResults detects & decodes json strings in TaskResult.Value and returns a new slice func (b *Backend) decodeResults(results []*tasks.TaskResult) []*tasks.TaskResult { l := len(results) jsonResults := make([]*tasks.TaskResult, l) for i, result := range results { jsonResult := new(bson.M) resultType := reflect.TypeOf(result.Value).Kind() if resultType == reflect.String { err := json.NewDecoder(strings.NewReader(result.Value.(string))).Decode(&jsonResult) if err == nil { jsonResults[i] = &tasks.TaskResult{ Type: "json", Value: jsonResult, } continue } } jsonResults[i] = result } return jsonResults } // SetStateFailure updates task state to FAILURE func (b *Backend) SetStateFailure(signature *tasks.Signature, err string) error { update := bson.M{"state": tasks.StateFailure, "error": err} return b.updateState(signature, update) } // GetState returns the latest task state func (b *Backend) GetState(taskUUID string) (*tasks.TaskState, error) { state := &tasks.TaskState{} err := b.tasksCollection().FindOne(context.Background(), bson.M{"_id": taskUUID}).Decode(state) if err != nil { return nil, err } return state, nil } // PurgeState deletes stored task state func (b *Backend) PurgeState(taskUUID string) error { _, err := b.tasksCollection().DeleteOne(context.Background(), bson.M{"_id": taskUUID}) return err } // PurgeGroupMeta deletes stored group meta data func (b *Backend) PurgeGroupMeta(groupUUID string) error { _, err := b.groupMetasCollection().DeleteOne(context.Background(), bson.M{"_id": groupUUID}) return err } // lockGroupMeta acquires lock on groupUUID document func (b *Backend) lockGroupMeta(groupUUID string) error { query := bson.M{ "_id": groupUUID, "lock": false, } change := bson.M{ "$set": bson.M{ "lock": true, }, } _, err := b.groupMetasCollection().UpdateOne(context.Background(), query, change, options.Update().SetUpsert(true)) return err } // unlockGroupMeta releases lock on groupUUID document func (b *Backend) unlockGroupMeta(groupUUID string) error { update := bson.M{"$set": bson.M{"lock": false}} _, err := b.groupMetasCollection().UpdateOne(context.Background(), bson.M{"_id": groupUUID}, update, options.Update()) return err } // getGroupMeta retrieves group meta data, convenience function to avoid repetition func (b *Backend) getGroupMeta(groupUUID string) (*tasks.GroupMeta, error) { groupMeta := &tasks.GroupMeta{} query := bson.M{"_id": groupUUID} err := b.groupMetasCollection().FindOne(context.Background(), query).Decode(groupMeta) if err != nil { return nil, err } return groupMeta, nil } // getStates returns multiple task states func (b *Backend) getStates(taskUUIDs ...string) ([]*tasks.TaskState, error) { states := make([]*tasks.TaskState, 0, len(taskUUIDs)) cur, err := b.tasksCollection().Find(context.Background(), bson.M{"_id": bson.M{"$in": taskUUIDs}}) if err != nil { return nil, err } defer cur.Close(context.Background()) for cur.Next(context.Background()) { state := &tasks.TaskState{} if err := cur.Decode(state); err != nil { return nil, err } states = append(states, state) } if cur.Err() != nil { return nil, err } return states, nil } // updateState saves current task state func (b *Backend) updateState(signature *tasks.Signature, update bson.M) error { update = bson.M{"$set": update} _, err := b.tasksCollection().UpdateOne(context.Background(), bson.M{"_id": signature.UUID}, update, options.Update().SetUpsert(true)) return err } func (b *Backend) tasksCollection() *mongo.Collection { b.once.Do(func() { b.connect() }) return b.tc } func (b *Backend) groupMetasCollection() *mongo.Collection { b.once.Do(func() { b.connect() }) return b.gmc } // connect creates the underlying mgo connection if it doesn't exist // creates required indexes for our collections func (b *Backend) connect() error { client, err := b.dial() if err != nil { return err } b.client = client database := "machinery" if b.GetConfig().MongoDB != nil { database = b.GetConfig().MongoDB.Database } b.tc = b.client.Database(database).Collection("tasks") b.gmc = b.client.Database(database).Collection("group_metas") err = b.createMongoIndexes(database) if err != nil { return err } return nil } // dial connects to mongo with TLSConfig if provided // else connects via ResultBackend uri func (b *Backend) dial() (*mongo.Client, error) { if b.GetConfig().MongoDB != nil && b.GetConfig().MongoDB.Client != nil { return b.GetConfig().MongoDB.Client, nil } uri := b.GetConfig().ResultBackend if strings.HasPrefix(uri, "mongodb://") == false && strings.HasPrefix(uri, "mongodb+srv://") == false { uri = fmt.Sprintf("mongodb://%s", uri) } client, err := mongo.NewClient(options.Client().ApplyURI(uri)) if err != nil { return nil, err } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() if err := client.Connect(ctx); err != nil { return nil, err } return client, nil } // createMongoIndexes ensures all indexes are in place func (b *Backend) createMongoIndexes(database string) error { tasksCollection := b.client.Database(database).Collection("tasks") expireIn := int32(b.GetConfig().ResultsExpireIn) _, err := tasksCollection.Indexes().CreateMany(context.Background(), []mongo.IndexModel{ { Keys: bson.M{"state": 1}, Options: options.Index().SetBackground(true).SetExpireAfterSeconds(expireIn), }, mongo.IndexModel{ Keys: bson.M{"lock": 1}, Options: options.Index().SetBackground(true).SetExpireAfterSeconds(expireIn), }, }) if err != nil { return err } return err } ================================================ FILE: v1/backends/mongo/mongodb_test.go ================================================ package mongo_test import ( "os" "testing" "github.com/RichardKnop/machinery/v1/backends/iface" "github.com/RichardKnop/machinery/v1/backends/mongo" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/tasks" "github.com/stretchr/testify/assert" ) var ( groupUUID = "123456" taskUUIDs = []string{"1", "2", "3"} ) func newBackend() (iface.Backend, error) { cnf := &config.Config{ ResultBackend: os.Getenv("MONGODB_URL"), ResultsExpireIn: 30, } backend, err := mongo.New(cnf) if err != nil { return nil, err } backend.PurgeGroupMeta(groupUUID) for _, taskUUID := range taskUUIDs { backend.PurgeState(taskUUID) } if err := backend.InitGroup(groupUUID, taskUUIDs); err != nil { return nil, err } return backend, nil } func TestNew(t *testing.T) { if os.Getenv("MONGODB_URL") == "" { t.Skip("MONGODB_URL is not defined") } backend, err := newBackend() if assert.NoError(t, err) { assert.NotNil(t, backend) } } func TestSetStatePending(t *testing.T) { if os.Getenv("MONGODB_URL") == "" { t.Skip("MONGODB_URL is not defined") } backend, err := newBackend() if err != nil { t.Fatal(err) } err = backend.SetStatePending(&tasks.Signature{ UUID: taskUUIDs[0], }) if assert.NoError(t, err) { taskState, err := backend.GetState(taskUUIDs[0]) if assert.NoError(t, err) { assert.Equal(t, tasks.StatePending, taskState.State, "Not StatePending") } } } func TestSetStateReceived(t *testing.T) { if os.Getenv("MONGODB_URL") == "" { t.Skip("MONGODB_URL is not defined") } backend, err := newBackend() if err != nil { t.Fatal(err) } err = backend.SetStateReceived(&tasks.Signature{ UUID: taskUUIDs[0], }) if assert.NoError(t, err) { taskState, err := backend.GetState(taskUUIDs[0]) if assert.NoError(t, err) { assert.Equal(t, tasks.StateReceived, taskState.State, "Not StateReceived") } } } func TestSetStateStarted(t *testing.T) { if os.Getenv("MONGODB_URL") == "" { t.Skip("MONGODB_URL is not defined") } backend, err := newBackend() if err != nil { t.Fatal(err) } err = backend.SetStateStarted(&tasks.Signature{ UUID: taskUUIDs[0], }) if assert.NoError(t, err) { taskState, err := backend.GetState(taskUUIDs[0]) if assert.NoError(t, err) { assert.Equal(t, tasks.StateStarted, taskState.State, "Not StateStarted") } } } func TestSetStateSuccess(t *testing.T) { if os.Getenv("MONGODB_URL") == "" { t.Skip("MONGODB_URL is not defined") } resultType := "float64" resultValue := float64(88.5) backend, err := newBackend() if err != nil { t.Fatal(err) } signature := &tasks.Signature{ UUID: taskUUIDs[0], } taskResults := []*tasks.TaskResult{ { Type: resultType, Value: resultValue, }, } err = backend.SetStateSuccess(signature, taskResults) assert.NoError(t, err) taskState, err := backend.GetState(taskUUIDs[0]) assert.NoError(t, err) assert.Equal(t, tasks.StateSuccess, taskState.State, "Not StateSuccess") assert.Equal(t, resultType, taskState.Results[0].Type, "Wrong result type") assert.Equal(t, float64(resultValue), taskState.Results[0].Value.(float64), "Wrong result value") } func TestSetStateFailure(t *testing.T) { if os.Getenv("MONGODB_URL") == "" { t.Skip("MONGODB_URL is not defined") } failString := "Fail is ok" backend, err := newBackend() if err != nil { t.Fatal(err) } signature := &tasks.Signature{ UUID: taskUUIDs[0], } err = backend.SetStateFailure(signature, failString) assert.NoError(t, err) taskState, err := backend.GetState(taskUUIDs[0]) assert.NoError(t, err) assert.Equal(t, tasks.StateFailure, taskState.State, "Not StateSuccess") assert.Equal(t, failString, taskState.Error, "Wrong fail error") } func TestGroupCompleted(t *testing.T) { if os.Getenv("MONGODB_URL") == "" { t.Skip("MONGODB_URL is not defined") } backend, err := newBackend() if err != nil { t.Fatal(err) } taskResultsState := make(map[string]string) isCompleted, err := backend.GroupCompleted(groupUUID, len(taskUUIDs)) if assert.NoError(t, err) { assert.False(t, isCompleted, "Actually group is not completed") } signature := &tasks.Signature{ UUID: taskUUIDs[0], } err = backend.SetStateFailure(signature, "Fail is ok") assert.NoError(t, err) taskResultsState[taskUUIDs[0]] = tasks.StateFailure signature = &tasks.Signature{ UUID: taskUUIDs[1], } taskResults := []*tasks.TaskResult{ { Type: "string", Value: "Result ok", }, } err = backend.SetStateSuccess(signature, taskResults) assert.NoError(t, err) taskResultsState[taskUUIDs[1]] = tasks.StateSuccess signature = &tasks.Signature{ UUID: taskUUIDs[2], } err = backend.SetStateSuccess(signature, taskResults) assert.NoError(t, err) taskResultsState[taskUUIDs[2]] = tasks.StateSuccess isCompleted, err = backend.GroupCompleted(groupUUID, len(taskUUIDs)) if assert.NoError(t, err) { assert.True(t, isCompleted, "Actually group is completed") } taskStates, err := backend.GroupTaskStates(groupUUID, len(taskUUIDs)) assert.NoError(t, err) assert.Equal(t, len(taskStates), len(taskUUIDs), "Wrong len tasksStates") for _, taskState := range taskStates { assert.Equal( t, taskResultsState[taskState.TaskUUID], taskState.State, "Wrong state on", taskState.TaskUUID, ) } } func TestGroupStates(t *testing.T) { if os.Getenv("MONGODB_URL") == "" { t.Skip("MONGODB_URL is not defined") } backend, err := newBackend() if err != nil { t.Fatal(err) } taskStates, err := backend.GroupTaskStates(groupUUID, len(taskUUIDs)) assert.NoError(t, err) for i, taskState := range taskStates { assert.Equal(t, taskUUIDs[i], taskState.TaskUUID) } } ================================================ FILE: v1/backends/null/null.go ================================================ package null import ( "fmt" "github.com/RichardKnop/machinery/v1/backends/iface" "github.com/RichardKnop/machinery/v1/common" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/tasks" ) // ErrGroupNotFound ... type ErrGroupNotFound struct { groupUUID string } // NewErrGroupNotFound returns new instance of ErrGroupNotFound func NewErrGroupNotFound(groupUUID string) ErrGroupNotFound { return ErrGroupNotFound{groupUUID: groupUUID} } // Error implements error interface func (e ErrGroupNotFound) Error() string { return fmt.Sprintf("Group not found: %v", e.groupUUID) } // ErrTasknotFound ... type ErrTasknotFound struct { taskUUID string } // NewErrTasknotFound returns new instance of ErrTasknotFound func NewErrTasknotFound(taskUUID string) ErrTasknotFound { return ErrTasknotFound{taskUUID: taskUUID} } // Error implements error interface func (e ErrTasknotFound) Error() string { return fmt.Sprintf("Task not found: %v", e.taskUUID) } // Backend represents an "null" result backend type Backend struct { common.Backend groups map[string]struct{} } // New creates NullBackend instance func New() iface.Backend { return &Backend{ Backend: common.NewBackend(new(config.Config)), groups: make(map[string]struct{}), } } // InitGroup creates and saves a group meta data object func (b *Backend) InitGroup(groupUUID string, taskUUIDs []string) error { b.groups[groupUUID] = struct{}{} return nil } // GroupCompleted returns true (always) func (b *Backend) GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) { _, ok := b.groups[groupUUID] if !ok { return false, NewErrGroupNotFound(groupUUID) } return true, nil } // GroupTaskStates returns null states of all tasks in the group func (b *Backend) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error) { _, ok := b.groups[groupUUID] if !ok { return nil, NewErrGroupNotFound(groupUUID) } ret := make([]*tasks.TaskState, 0, groupTaskCount) return ret, nil } // TriggerChord returns true (always) func (b *Backend) TriggerChord(groupUUID string) (bool, error) { return true, nil } // SetStatePending updates task state to PENDING func (b *Backend) SetStatePending(signature *tasks.Signature) error { state := tasks.NewPendingTaskState(signature) return b.updateState(state) } // SetStateReceived updates task state to RECEIVED func (b *Backend) SetStateReceived(signature *tasks.Signature) error { state := tasks.NewReceivedTaskState(signature) return b.updateState(state) } // SetStateStarted updates task state to STARTED func (b *Backend) SetStateStarted(signature *tasks.Signature) error { state := tasks.NewStartedTaskState(signature) return b.updateState(state) } // SetStateRetry updates task state to RETRY func (b *Backend) SetStateRetry(signature *tasks.Signature) error { state := tasks.NewRetryTaskState(signature) return b.updateState(state) } // SetStateSuccess updates task state to SUCCESS func (b *Backend) SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error { state := tasks.NewSuccessTaskState(signature, results) return b.updateState(state) } // SetStateFailure updates task state to FAILURE func (b *Backend) SetStateFailure(signature *tasks.Signature, err string) error { state := tasks.NewFailureTaskState(signature, err) return b.updateState(state) } // GetState returns the latest task state func (b *Backend) GetState(taskUUID string) (*tasks.TaskState, error) { return nil, NewErrTasknotFound(taskUUID) } // PurgeState deletes stored task state func (b *Backend) PurgeState(taskUUID string) error { return NewErrTasknotFound(taskUUID) } // PurgeGroupMeta deletes stored group meta data func (b *Backend) PurgeGroupMeta(groupUUID string) error { _, ok := b.groups[groupUUID] if !ok { return NewErrGroupNotFound(groupUUID) } return nil } func (b *Backend) updateState(s *tasks.TaskState) error { return nil } ================================================ FILE: v1/backends/package.go ================================================ package backends ================================================ FILE: v1/backends/redis/goredis.go ================================================ package redis import ( "bytes" "context" "encoding/json" "strings" "sync" "time" "github.com/go-redsync/redsync/v4" redsyncgoredis "github.com/go-redsync/redsync/v4/redis/goredis/v9" "github.com/redis/go-redis/v9" "github.com/RichardKnop/machinery/v1/backends/iface" "github.com/RichardKnop/machinery/v1/common" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/log" "github.com/RichardKnop/machinery/v1/tasks" ) // BackendGR represents a Redis result backend type BackendGR struct { common.Backend rclient redis.UniversalClient host string password string db int // If set, path to a socket file overrides hostname socketPath string redsync *redsync.Redsync redisOnce sync.Once } // NewGR creates Backend instance func NewGR(cnf *config.Config, addrs []string, db int) iface.Backend { b := &BackendGR{ Backend: common.NewBackend(cnf), } var password string var username string parts := strings.Split(addrs[0], "@") if len(parts) >= 2 { // with password options := strings.SplitN(strings.Join(parts[:len(parts)-1], "@"), ":", 2) if len(options) >= 2 { username = options[0] password = options[1] } else { password = options[0] } addrs[0] = parts[len(parts)-1] // addr is the last one without @ } ropt := &redis.UniversalOptions{ Addrs: addrs, DB: db, Password: password, Username: username, } if cnf.Redis != nil { ropt.MasterName = cnf.Redis.MasterName } if cnf.Redis != nil && cnf.Redis.ClusterMode { b.rclient = redis.NewClusterClient(ropt.Cluster()) } else { b.rclient = redis.NewUniversalClient(ropt) } b.redsync = redsync.New(redsyncgoredis.NewPool(b.rclient)) return b } // InitGroup creates and saves a group meta data object func (b *BackendGR) InitGroup(groupUUID string, taskUUIDs []string) error { groupMeta := &tasks.GroupMeta{ GroupUUID: groupUUID, TaskUUIDs: taskUUIDs, CreatedAt: time.Now().UTC(), } encoded, err := json.Marshal(groupMeta) if err != nil { return err } expiration := b.getExpiration() err = b.rclient.Set(context.Background(), groupUUID, encoded, expiration).Err() if err != nil { return err } return nil } // GroupCompleted returns true if all tasks in a group finished func (b *BackendGR) GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) { groupMeta, err := b.getGroupMeta(groupUUID) if err != nil { return false, err } taskStates, err := b.getStates(groupMeta.TaskUUIDs...) if err != nil { return false, err } var countSuccessTasks = 0 for _, taskState := range taskStates { if taskState.IsCompleted() { countSuccessTasks++ } } return countSuccessTasks == groupTaskCount, nil } // GroupTaskStates returns states of all tasks in the group func (b *BackendGR) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error) { groupMeta, err := b.getGroupMeta(groupUUID) if err != nil { return []*tasks.TaskState{}, err } return b.getStates(groupMeta.TaskUUIDs...) } // TriggerChord flags chord as triggered in the backend storage to make sure // chord is never trigerred multiple times. Returns a boolean flag to indicate // whether the worker should trigger chord (true) or no if it has been triggered // already (false) func (b *BackendGR) TriggerChord(groupUUID string) (bool, error) { m := b.redsync.NewMutex("TriggerChordMutex") if err := m.Lock(); err != nil { return false, err } defer m.Unlock() groupMeta, err := b.getGroupMeta(groupUUID) if err != nil { return false, err } // Chord has already been triggered, return false (should not trigger again) if groupMeta.ChordTriggered { return false, nil } // Set flag to true groupMeta.ChordTriggered = true // Update the group meta encoded, err := json.Marshal(&groupMeta) if err != nil { return false, err } expiration := b.getExpiration() err = b.rclient.Set(context.Background(), groupUUID, encoded, expiration).Err() if err != nil { return false, err } return true, nil } func (b *BackendGR) mergeNewTaskState(newState *tasks.TaskState) { state, err := b.GetState(newState.TaskUUID) if err == nil { newState.CreatedAt = state.CreatedAt newState.TaskName = state.TaskName } } // SetStatePending updates task state to PENDING func (b *BackendGR) SetStatePending(signature *tasks.Signature) error { taskState := tasks.NewPendingTaskState(signature) return b.updateState(taskState) } // SetStateReceived updates task state to RECEIVED func (b *BackendGR) SetStateReceived(signature *tasks.Signature) error { taskState := tasks.NewReceivedTaskState(signature) b.mergeNewTaskState(taskState) return b.updateState(taskState) } // SetStateStarted updates task state to STARTED func (b *BackendGR) SetStateStarted(signature *tasks.Signature) error { taskState := tasks.NewStartedTaskState(signature) b.mergeNewTaskState(taskState) return b.updateState(taskState) } // SetStateRetry updates task state to RETRY func (b *BackendGR) SetStateRetry(signature *tasks.Signature) error { taskState := tasks.NewRetryTaskState(signature) b.mergeNewTaskState(taskState) return b.updateState(taskState) } // SetStateSuccess updates task state to SUCCESS func (b *BackendGR) SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error { taskState := tasks.NewSuccessTaskState(signature, results) b.mergeNewTaskState(taskState) return b.updateState(taskState) } // SetStateFailure updates task state to FAILURE func (b *BackendGR) SetStateFailure(signature *tasks.Signature, err string) error { taskState := tasks.NewFailureTaskState(signature, err) b.mergeNewTaskState(taskState) return b.updateState(taskState) } // GetState returns the latest task state func (b *BackendGR) GetState(taskUUID string) (*tasks.TaskState, error) { item, err := b.rclient.Get(context.Background(), taskUUID).Bytes() if err != nil { return nil, err } state := new(tasks.TaskState) decoder := json.NewDecoder(bytes.NewReader(item)) decoder.UseNumber() if err := decoder.Decode(state); err != nil { return nil, err } return state, nil } // PurgeState deletes stored task state func (b *BackendGR) PurgeState(taskUUID string) error { err := b.rclient.Del(context.Background(), taskUUID).Err() if err != nil { return err } return nil } // PurgeGroupMeta deletes stored group meta data func (b *BackendGR) PurgeGroupMeta(groupUUID string) error { err := b.rclient.Del(context.Background(), groupUUID).Err() if err != nil { return err } return nil } // getGroupMeta retrieves group meta data, convenience function to avoid repetition func (b *BackendGR) getGroupMeta(groupUUID string) (*tasks.GroupMeta, error) { item, err := b.rclient.Get(context.Background(), groupUUID).Bytes() if err != nil { return nil, err } groupMeta := new(tasks.GroupMeta) decoder := json.NewDecoder(bytes.NewReader(item)) decoder.UseNumber() if err := decoder.Decode(groupMeta); err != nil { return nil, err } return groupMeta, nil } // getStates returns multiple task states func (b *BackendGR) getStates(taskUUIDs ...string) ([]*tasks.TaskState, error) { taskStates := make([]*tasks.TaskState, len(taskUUIDs)) // to avoid CROSSSLOT error, use pipeline cmders, err := b.rclient.Pipelined(context.Background(), func(pipeliner redis.Pipeliner) error { for _, uuid := range taskUUIDs { pipeliner.Get(context.Background(), uuid) } return nil }) if err != nil { return taskStates, err } for i, cmder := range cmders { stateBytes, err1 := cmder.(*redis.StringCmd).Bytes() if err1 != nil { return taskStates, err1 } taskState := new(tasks.TaskState) decoder := json.NewDecoder(bytes.NewReader(stateBytes)) decoder.UseNumber() if err1 = decoder.Decode(taskState); err1 != nil { log.ERROR.Print(err1) return taskStates, err1 } taskStates[i] = taskState } return taskStates, nil } // updateState saves current task state func (b *BackendGR) updateState(taskState *tasks.TaskState) error { encoded, err := json.Marshal(taskState) if err != nil { return err } expiration := b.getExpiration() _, err = b.rclient.Set(context.Background(), taskState.TaskUUID, encoded, expiration).Result() if err != nil { return err } return nil } // getExpiration returns expiration for a stored task state func (b *BackendGR) getExpiration() time.Duration { expiresIn := b.GetConfig().ResultsExpireIn if expiresIn == 0 { // expire results after 1 hour by default expiresIn = config.DefaultResultsExpireIn } return time.Duration(expiresIn) * time.Second } ================================================ FILE: v1/backends/redis/goredis_test.go ================================================ package redis_test import ( "github.com/RichardKnop/machinery/v1/backends/iface" "os" "strings" "testing" "github.com/RichardKnop/machinery/v1/backends/redis" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/tasks" "github.com/stretchr/testify/assert" ) func getRedisG() iface.Backend { // host1:port1,host2:port2 redisURL := os.Getenv("REDIS_URL_GR") //redisPassword := os.Getenv("REDIS_PASSWORD") if redisURL == "" { return nil } backend := redis.NewGR(new(config.Config), strings.Split(redisURL, ","), 0) return backend } func TestGroupCompletedGR(t *testing.T) { backend := getRedisG() if backend == nil { t.Skip() } groupUUID := "testGroupUUID" task1 := &tasks.Signature{ UUID: "testTaskUUID1", GroupUUID: groupUUID, } task2 := &tasks.Signature{ UUID: "testTaskUUID2", GroupUUID: groupUUID, } // Cleanup before the test backend.PurgeState(task1.UUID) backend.PurgeState(task2.UUID) backend.PurgeGroupMeta(groupUUID) groupCompleted, err := backend.GroupCompleted(groupUUID, 2) if assert.Error(t, err) { assert.False(t, groupCompleted) assert.Equal(t, "redis: nil", err.Error()) } backend.InitGroup(groupUUID, []string{task1.UUID, task2.UUID}) groupCompleted, err = backend.GroupCompleted(groupUUID, 2) if assert.Error(t, err) { assert.False(t, groupCompleted) assert.Equal(t, "redis: nil", err.Error()) } backend.SetStatePending(task1) backend.SetStateStarted(task2) groupCompleted, err = backend.GroupCompleted(groupUUID, 2) if assert.NoError(t, err) { assert.False(t, groupCompleted) } taskResults := []*tasks.TaskResult{new(tasks.TaskResult)} backend.SetStateStarted(task1) backend.SetStateSuccess(task2, taskResults) groupCompleted, err = backend.GroupCompleted(groupUUID, 2) if assert.NoError(t, err) { assert.False(t, groupCompleted) } backend.SetStateFailure(task1, "Some error") groupCompleted, err = backend.GroupCompleted(groupUUID, 2) if assert.NoError(t, err) { assert.True(t, groupCompleted) } } func TestGetStateGR(t *testing.T) { backend := getRedisG() if backend == nil { t.Skip() } signature := &tasks.Signature{ UUID: "testTaskUUID", GroupUUID: "testGroupUUID", } backend.PurgeState("testTaskUUID") var ( taskState *tasks.TaskState err error ) taskState, err = backend.GetState(signature.UUID) assert.Equal(t, "redis: nil", err.Error()) assert.Nil(t, taskState) //Pending State backend.SetStatePending(signature) taskState, err = backend.GetState(signature.UUID) assert.NoError(t, err) assert.Equal(t, signature.Name, taskState.TaskName) createdAt := taskState.CreatedAt //Received State backend.SetStateReceived(signature) taskState, err = backend.GetState(signature.UUID) assert.NoError(t, err) assert.Equal(t, signature.Name, taskState.TaskName) assert.Equal(t, createdAt, taskState.CreatedAt) //Started State backend.SetStateStarted(signature) taskState, err = backend.GetState(signature.UUID) assert.NoError(t, err) assert.Equal(t, signature.Name, taskState.TaskName) assert.Equal(t, createdAt, taskState.CreatedAt) //Success State taskResults := []*tasks.TaskResult{ { Type: "float64", Value: 2, }, } backend.SetStateSuccess(signature, taskResults) taskState, err = backend.GetState(signature.UUID) assert.NoError(t, err) assert.Equal(t, signature.Name, taskState.TaskName) assert.Equal(t, createdAt, taskState.CreatedAt) assert.NotNil(t, taskState.Results) } func TestPurgeStateGR(t *testing.T) { backend := getRedisG() if backend == nil { t.Skip() } signature := &tasks.Signature{ UUID: "testTaskUUID", GroupUUID: "testGroupUUID", } backend.SetStatePending(signature) taskState, err := backend.GetState(signature.UUID) assert.NotNil(t, taskState) assert.NoError(t, err) backend.PurgeState(taskState.TaskUUID) taskState, err = backend.GetState(signature.UUID) assert.Nil(t, taskState) assert.Error(t, err) } ================================================ FILE: v1/backends/redis/redis.go ================================================ package redis import ( "bytes" "encoding/json" "fmt" "sync" "time" "github.com/go-redsync/redsync/v4" redsyncredis "github.com/go-redsync/redsync/v4/redis/redigo" "github.com/gomodule/redigo/redis" "github.com/RichardKnop/machinery/v1/backends/iface" "github.com/RichardKnop/machinery/v1/common" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/log" "github.com/RichardKnop/machinery/v1/tasks" ) // Backend represents a Redis result backend type Backend struct { common.Backend host string username string password string db int pool *redis.Pool // If set, path to a socket file overrides hostname socketPath string redsync *redsync.Redsync redisOnce sync.Once common.RedisConnector } // New creates Backend instance func New(cnf *config.Config, host, username, password, socketPath string, db int) iface.Backend { return &Backend{ Backend: common.NewBackend(cnf), host: host, db: db, username: username, password: password, socketPath: socketPath, } } // InitGroup creates and saves a group meta data object func (b *Backend) InitGroup(groupUUID string, taskUUIDs []string) error { groupMeta := &tasks.GroupMeta{ GroupUUID: groupUUID, TaskUUIDs: taskUUIDs, CreatedAt: time.Now().UTC(), } encoded, err := json.Marshal(groupMeta) if err != nil { return err } conn := b.open() defer conn.Close() expiration := int64(b.getExpiration().Seconds()) _, err = conn.Do("SET", groupUUID, encoded, "EX", expiration) if err != nil { return err } return nil } // GroupCompleted returns true if all tasks in a group finished func (b *Backend) GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) { conn := b.open() defer conn.Close() groupMeta, err := b.getGroupMeta(conn, groupUUID) if err != nil { return false, err } taskStates, err := b.getStates(conn, groupMeta.TaskUUIDs...) if err != nil { return false, err } var countSuccessTasks = 0 for _, taskState := range taskStates { if taskState.IsCompleted() { countSuccessTasks++ } } return countSuccessTasks == groupTaskCount, nil } // GroupTaskStates returns states of all tasks in the group func (b *Backend) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error) { conn := b.open() defer conn.Close() groupMeta, err := b.getGroupMeta(conn, groupUUID) if err != nil { return []*tasks.TaskState{}, err } return b.getStates(conn, groupMeta.TaskUUIDs...) } // TriggerChord flags chord as triggered in the backend storage to make sure // chord is never trigerred multiple times. Returns a boolean flag to indicate // whether the worker should trigger chord (true) or no if it has been triggered // already (false) func (b *Backend) TriggerChord(groupUUID string) (bool, error) { conn := b.open() defer conn.Close() m := b.redsync.NewMutex("TriggerChordMutex") if err := m.Lock(); err != nil { return false, err } defer m.Unlock() groupMeta, err := b.getGroupMeta(conn, groupUUID) if err != nil { return false, err } // Chord has already been triggered, return false (should not trigger again) if groupMeta.ChordTriggered { return false, nil } // Set flag to true groupMeta.ChordTriggered = true // Update the group meta encoded, err := json.Marshal(&groupMeta) if err != nil { return false, err } expiration := int64(b.getExpiration().Seconds()) _, err = conn.Do("SET", groupUUID, encoded, "EX", expiration) if err != nil { return false, err } return true, nil } func (b *Backend) mergeNewTaskState(conn redis.Conn, newState *tasks.TaskState) { state, err := b.getState(conn, newState.TaskUUID) if err == nil { newState.CreatedAt = state.CreatedAt newState.TaskName = state.TaskName } } // SetStatePending updates task state to PENDING func (b *Backend) SetStatePending(signature *tasks.Signature) error { conn := b.open() defer conn.Close() taskState := tasks.NewPendingTaskState(signature) return b.updateState(conn, taskState) } // SetStateReceived updates task state to RECEIVED func (b *Backend) SetStateReceived(signature *tasks.Signature) error { conn := b.open() defer conn.Close() taskState := tasks.NewReceivedTaskState(signature) b.mergeNewTaskState(conn, taskState) return b.updateState(conn, taskState) } // SetStateStarted updates task state to STARTED func (b *Backend) SetStateStarted(signature *tasks.Signature) error { conn := b.open() defer conn.Close() taskState := tasks.NewStartedTaskState(signature) b.mergeNewTaskState(conn, taskState) return b.updateState(conn, taskState) } // SetStateRetry updates task state to RETRY func (b *Backend) SetStateRetry(signature *tasks.Signature) error { conn := b.open() defer conn.Close() taskState := tasks.NewRetryTaskState(signature) b.mergeNewTaskState(conn, taskState) return b.updateState(conn, taskState) } // SetStateSuccess updates task state to SUCCESS func (b *Backend) SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error { conn := b.open() defer conn.Close() taskState := tasks.NewSuccessTaskState(signature, results) b.mergeNewTaskState(conn, taskState) return b.updateState(conn, taskState) } // SetStateFailure updates task state to FAILURE func (b *Backend) SetStateFailure(signature *tasks.Signature, err string) error { conn := b.open() defer conn.Close() taskState := tasks.NewFailureTaskState(signature, err) b.mergeNewTaskState(conn, taskState) return b.updateState(conn, taskState) } // GetState returns the latest task state func (b *Backend) GetState(taskUUID string) (*tasks.TaskState, error) { conn := b.open() defer conn.Close() return b.getState(conn, taskUUID) } func (b *Backend) getState(conn redis.Conn, taskUUID string) (*tasks.TaskState, error) { item, err := redis.Bytes(conn.Do("GET", taskUUID)) if err != nil { return nil, err } state := new(tasks.TaskState) decoder := json.NewDecoder(bytes.NewReader(item)) decoder.UseNumber() if err := decoder.Decode(state); err != nil { return nil, err } return state, nil } // PurgeState deletes stored task state func (b *Backend) PurgeState(taskUUID string) error { conn := b.open() defer conn.Close() _, err := conn.Do("DEL", taskUUID) if err != nil { return err } return nil } // PurgeGroupMeta deletes stored group meta data func (b *Backend) PurgeGroupMeta(groupUUID string) error { conn := b.open() defer conn.Close() _, err := conn.Do("DEL", groupUUID) if err != nil { return err } return nil } // getGroupMeta retrieves group meta data, convenience function to avoid repetition func (b *Backend) getGroupMeta(conn redis.Conn, groupUUID string) (*tasks.GroupMeta, error) { item, err := redis.Bytes(conn.Do("GET", groupUUID)) if err != nil { return nil, err } groupMeta := new(tasks.GroupMeta) decoder := json.NewDecoder(bytes.NewReader(item)) decoder.UseNumber() if err := decoder.Decode(groupMeta); err != nil { return nil, err } return groupMeta, nil } // getStates returns multiple task states func (b *Backend) getStates(conn redis.Conn, taskUUIDs ...string) ([]*tasks.TaskState, error) { taskStates := make([]*tasks.TaskState, len(taskUUIDs)) // conn.Do requires []interface{}... can't pass []string unfortunately taskUUIDInterfaces := make([]interface{}, len(taskUUIDs)) for i, taskUUID := range taskUUIDs { taskUUIDInterfaces[i] = interface{}(taskUUID) } reply, err := redis.Values(conn.Do("MGET", taskUUIDInterfaces...)) if err != nil { return taskStates, err } for i, value := range reply { stateBytes, ok := value.([]byte) if !ok { return taskStates, fmt.Errorf("Expected byte array, instead got: %v", value) } taskState := new(tasks.TaskState) decoder := json.NewDecoder(bytes.NewReader(stateBytes)) decoder.UseNumber() if err := decoder.Decode(taskState); err != nil { log.ERROR.Print(err) return taskStates, err } taskStates[i] = taskState } return taskStates, nil } // updateState saves current task state func (b *Backend) updateState(conn redis.Conn, taskState *tasks.TaskState) error { encoded, err := json.Marshal(taskState) if err != nil { return err } expiration := int64(b.getExpiration().Seconds()) _, err = conn.Do("SET", taskState.TaskUUID, encoded, "EX", expiration) if err != nil { return err } return nil } // getExpiration returns expiration for a stored task state func (b *Backend) getExpiration() time.Duration { expiresIn := b.GetConfig().ResultsExpireIn if expiresIn == 0 { // expire results after 1 hour by default expiresIn = config.DefaultResultsExpireIn } return time.Duration(expiresIn) * time.Second } // open returns or creates instance of Redis connection func (b *Backend) open() redis.Conn { b.redisOnce.Do(func() { b.pool = b.NewPool(b.socketPath, b.host, b.username, b.password, b.db, b.GetConfig().Redis, b.GetConfig().TLSConfig) b.redsync = redsync.New(redsyncredis.NewPool(b.pool)) }) return b.pool.Get() } ================================================ FILE: v1/backends/redis/redis_test.go ================================================ package redis_test import ( "os" "testing" "github.com/RichardKnop/machinery/v1/backends/redis" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/tasks" "github.com/stretchr/testify/assert" ) func TestGroupCompleted(t *testing.T) { redisURL := os.Getenv("REDIS_URL") redisPassword := os.Getenv("REDIS_PASSWORD") if redisURL == "" { t.Skip("REDIS_URL is not defined") } groupUUID := "testGroupUUID" task1 := &tasks.Signature{ UUID: "testTaskUUID1", GroupUUID: groupUUID, } task2 := &tasks.Signature{ UUID: "testTaskUUID2", GroupUUID: groupUUID, } backend := redis.New(new(config.Config), redisURL, "", redisPassword, "", 0) // Cleanup before the test backend.PurgeState(task1.UUID) backend.PurgeState(task2.UUID) backend.PurgeGroupMeta(groupUUID) groupCompleted, err := backend.GroupCompleted(groupUUID, 2) if assert.Error(t, err) { assert.False(t, groupCompleted) assert.Equal(t, "redigo: nil returned", err.Error()) } backend.InitGroup(groupUUID, []string{task1.UUID, task2.UUID}) groupCompleted, err = backend.GroupCompleted(groupUUID, 2) if assert.Error(t, err) { assert.False(t, groupCompleted) assert.Equal(t, "Expected byte array, instead got: ", err.Error()) } backend.SetStatePending(task1) backend.SetStateStarted(task2) groupCompleted, err = backend.GroupCompleted(groupUUID, 2) if assert.NoError(t, err) { assert.False(t, groupCompleted) } taskResults := []*tasks.TaskResult{new(tasks.TaskResult)} backend.SetStateStarted(task1) backend.SetStateSuccess(task2, taskResults) groupCompleted, err = backend.GroupCompleted(groupUUID, 2) if assert.NoError(t, err) { assert.False(t, groupCompleted) } backend.SetStateFailure(task1, "Some error") groupCompleted, err = backend.GroupCompleted(groupUUID, 2) if assert.NoError(t, err) { assert.True(t, groupCompleted) } } func TestGetState(t *testing.T) { redisURL := os.Getenv("REDIS_URL") redisPassword := os.Getenv("REDIS_PASSWORD") if redisURL == "" { return } signature := &tasks.Signature{ UUID: "testTaskUUID", GroupUUID: "testGroupUUID", } backend := redis.New(new(config.Config), redisURL, "", redisPassword, "", 0) backend.PurgeState("testTaskUUID") var ( taskState *tasks.TaskState err error ) taskState, err = backend.GetState(signature.UUID) assert.Equal(t, "redigo: nil returned", err.Error()) assert.Nil(t, taskState) //Pending State backend.SetStatePending(signature) taskState, err = backend.GetState(signature.UUID) assert.NoError(t, err) assert.Equal(t, signature.Name, taskState.TaskName) createdAt := taskState.CreatedAt //Received State backend.SetStateReceived(signature) taskState, err = backend.GetState(signature.UUID) assert.NoError(t, err) assert.Equal(t, signature.Name, taskState.TaskName) assert.Equal(t, createdAt, taskState.CreatedAt) //Started State backend.SetStateStarted(signature) taskState, err = backend.GetState(signature.UUID) assert.NoError(t, err) assert.Equal(t, signature.Name, taskState.TaskName) assert.Equal(t, createdAt, taskState.CreatedAt) //Success State taskResults := []*tasks.TaskResult{ { Type: "float64", Value: 2, }, } backend.SetStateSuccess(signature, taskResults) taskState, err = backend.GetState(signature.UUID) assert.NoError(t, err) assert.Equal(t, signature.Name, taskState.TaskName) assert.Equal(t, createdAt, taskState.CreatedAt) assert.NotNil(t, taskState.Results) } func TestPurgeState(t *testing.T) { redisURL := os.Getenv("REDIS_URL") redisPassword := os.Getenv("REDIS_PASSWORD") if redisURL == "" { return } signature := &tasks.Signature{ UUID: "testTaskUUID", GroupUUID: "testGroupUUID", } backend := redis.New(new(config.Config), redisURL, "", redisPassword, "", 0) backend.SetStatePending(signature) taskState, err := backend.GetState(signature.UUID) assert.NotNil(t, taskState) assert.NoError(t, err) backend.PurgeState(taskState.TaskUUID) taskState, err = backend.GetState(signature.UUID) assert.Nil(t, taskState) assert.Error(t, err) } ================================================ FILE: v1/backends/result/async_result.go ================================================ package result import ( "errors" "reflect" "time" "github.com/RichardKnop/machinery/v1/backends/iface" "github.com/RichardKnop/machinery/v1/tasks" ) var ( // ErrBackendNotConfigured ... ErrBackendNotConfigured = errors.New("Result backend not configured") // ErrTimeoutReached ... ErrTimeoutReached = errors.New("Timeout reached") ) // AsyncResult represents a task result type AsyncResult struct { Signature *tasks.Signature taskState *tasks.TaskState backend iface.Backend } // ChordAsyncResult represents a result of a chord type ChordAsyncResult struct { groupAsyncResults []*AsyncResult chordAsyncResult *AsyncResult backend iface.Backend } // ChainAsyncResult represents a result of a chain of tasks type ChainAsyncResult struct { asyncResults []*AsyncResult backend iface.Backend } // NewAsyncResult creates AsyncResult instance func NewAsyncResult(signature *tasks.Signature, backend iface.Backend) *AsyncResult { return &AsyncResult{ Signature: signature, taskState: new(tasks.TaskState), backend: backend, } } // NewChordAsyncResult creates ChordAsyncResult instance func NewChordAsyncResult(groupTasks []*tasks.Signature, chordCallback *tasks.Signature, backend iface.Backend) *ChordAsyncResult { asyncResults := make([]*AsyncResult, len(groupTasks)) for i, task := range groupTasks { asyncResults[i] = NewAsyncResult(task, backend) } return &ChordAsyncResult{ groupAsyncResults: asyncResults, chordAsyncResult: NewAsyncResult(chordCallback, backend), backend: backend, } } // NewChainAsyncResult creates ChainAsyncResult instance func NewChainAsyncResult(tasks []*tasks.Signature, backend iface.Backend) *ChainAsyncResult { asyncResults := make([]*AsyncResult, len(tasks)) for i, task := range tasks { asyncResults[i] = NewAsyncResult(task, backend) } return &ChainAsyncResult{ asyncResults: asyncResults, backend: backend, } } // Touch the state and don't wait func (asyncResult *AsyncResult) Touch() ([]reflect.Value, error) { if asyncResult.backend == nil { return nil, ErrBackendNotConfigured } asyncResult.GetState() // Purge state if we are using AMQP backend if asyncResult.backend.IsAMQP() && asyncResult.taskState.IsCompleted() { asyncResult.backend.PurgeState(asyncResult.taskState.TaskUUID) } if asyncResult.taskState.IsFailure() { return nil, errors.New(asyncResult.taskState.Error) } if asyncResult.taskState.IsSuccess() { return tasks.ReflectTaskResults(asyncResult.taskState.Results) } return nil, nil } // Get returns task results (synchronous blocking call) func (asyncResult *AsyncResult) Get(sleepDuration time.Duration) ([]reflect.Value, error) { for { results, err := asyncResult.Touch() if results == nil && err == nil { time.Sleep(sleepDuration) } else { return results, err } } } // GetWithTimeout returns task results with a timeout (synchronous blocking call) func (asyncResult *AsyncResult) GetWithTimeout(timeoutDuration, sleepDuration time.Duration) ([]reflect.Value, error) { timeout := time.NewTimer(timeoutDuration) for { select { case <-timeout.C: return nil, ErrTimeoutReached default: results, err := asyncResult.Touch() if results == nil && err == nil { time.Sleep(sleepDuration) } else { return results, err } } } } // GetState returns latest task state func (asyncResult *AsyncResult) GetState() *tasks.TaskState { if asyncResult.taskState.IsCompleted() { return asyncResult.taskState } taskState, err := asyncResult.backend.GetState(asyncResult.Signature.UUID) if err == nil { asyncResult.taskState = taskState } return asyncResult.taskState } // Get returns results of a chain of tasks (synchronous blocking call) func (chainAsyncResult *ChainAsyncResult) Get(sleepDuration time.Duration) ([]reflect.Value, error) { if chainAsyncResult.backend == nil { return nil, ErrBackendNotConfigured } var ( results []reflect.Value err error ) for _, asyncResult := range chainAsyncResult.asyncResults { results, err = asyncResult.Get(sleepDuration) if err != nil { return nil, err } } return results, err } // Get returns result of a chord (synchronous blocking call) func (chordAsyncResult *ChordAsyncResult) Get(sleepDuration time.Duration) ([]reflect.Value, error) { if chordAsyncResult.backend == nil { return nil, ErrBackendNotConfigured } var err error for _, asyncResult := range chordAsyncResult.groupAsyncResults { _, err = asyncResult.Get(sleepDuration) if err != nil { return nil, err } } return chordAsyncResult.chordAsyncResult.Get(sleepDuration) } // GetWithTimeout returns results of a chain of tasks with timeout (synchronous blocking call) func (chainAsyncResult *ChainAsyncResult) GetWithTimeout(timeoutDuration, sleepDuration time.Duration) ([]reflect.Value, error) { if chainAsyncResult.backend == nil { return nil, ErrBackendNotConfigured } var ( results []reflect.Value err error ) timeout := time.NewTimer(timeoutDuration) ln := len(chainAsyncResult.asyncResults) lastResult := chainAsyncResult.asyncResults[ln-1] for { select { case <-timeout.C: return nil, ErrTimeoutReached default: for _, asyncResult := range chainAsyncResult.asyncResults { _, err = asyncResult.Touch() if err != nil { return nil, err } } results, err = lastResult.Touch() if err != nil { return nil, err } if results != nil { return results, err } time.Sleep(sleepDuration) } } } // GetWithTimeout returns result of a chord with a timeout (synchronous blocking call) func (chordAsyncResult *ChordAsyncResult) GetWithTimeout(timeoutDuration, sleepDuration time.Duration) ([]reflect.Value, error) { if chordAsyncResult.backend == nil { return nil, ErrBackendNotConfigured } var ( results []reflect.Value err error ) timeout := time.NewTimer(timeoutDuration) for { select { case <-timeout.C: return nil, ErrTimeoutReached default: for _, asyncResult := range chordAsyncResult.groupAsyncResults { _, errcur := asyncResult.Touch() if errcur != nil { return nil, err } } results, err = chordAsyncResult.chordAsyncResult.Touch() if err != nil { return nil, nil } if results != nil { return results, err } time.Sleep(sleepDuration) } } } ================================================ FILE: v1/brokers/amqp/amqp.go ================================================ package amqp import ( "bytes" "context" "encoding/json" "fmt" "sync" "time" "github.com/RichardKnop/machinery/v1/brokers/errs" "github.com/RichardKnop/machinery/v1/brokers/iface" "github.com/RichardKnop/machinery/v1/common" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/log" "github.com/RichardKnop/machinery/v1/tasks" "github.com/pkg/errors" amqp "github.com/rabbitmq/amqp091-go" ) type AMQPConnection struct { queueName string connection *amqp.Connection channel *amqp.Channel queue amqp.Queue confirmation <-chan amqp.Confirmation errorchan <-chan *amqp.Error cleanup chan struct{} } // Broker represents an AMQP broker type Broker struct { common.Broker common.AMQPConnector processingWG sync.WaitGroup // use wait group to make sure task processing completes on interrupt signal connections map[string]*AMQPConnection connectionsMutex sync.RWMutex } // New creates new Broker instance func New(cnf *config.Config) iface.Broker { return &Broker{Broker: common.NewBroker(cnf), AMQPConnector: common.AMQPConnector{}, connections: make(map[string]*AMQPConnection)} } // StartConsuming enters a loop and waits for incoming messages func (b *Broker) StartConsuming(consumerTag string, concurrency int, taskProcessor iface.TaskProcessor) (bool, error) { b.Broker.StartConsuming(consumerTag, concurrency, taskProcessor) queueName := taskProcessor.CustomQueue() if queueName == "" { queueName = b.GetConfig().DefaultQueue } conn, channel, queue, _, amqpCloseChan, err := b.Connect( b.GetConfig().Broker, b.GetConfig().MultipleBrokerSeparator, b.GetConfig().TLSConfig, b.GetConfig().AMQP.Exchange, // exchange name b.GetConfig().AMQP.ExchangeType, // exchange type queueName, // queue name true, // queue durable false, // queue delete when unused b.GetConfig().AMQP.BindingKey, // queue binding key nil, // exchange declare args amqp.Table(b.GetConfig().AMQP.QueueDeclareArgs), // queue declare args amqp.Table(b.GetConfig().AMQP.QueueBindingArgs), // queue binding args ) if err != nil { b.GetRetryFunc()(b.GetRetryStopChan()) return b.GetRetry(), err } defer b.Close(channel, conn) if err = channel.Qos( b.GetConfig().AMQP.PrefetchCount, 0, // prefetch size false, // global ); err != nil { return b.GetRetry(), fmt.Errorf("Channel qos error: %s", err) } deliveries, err := channel.Consume( queue.Name, // queue consumerTag, // consumer tag false, // auto-ack false, // exclusive false, // no-local false, // no-wait nil, // arguments ) if err != nil { return b.GetRetry(), fmt.Errorf("Queue consume error: %s", err) } log.INFO.Print("[*] Waiting for messages. To exit press CTRL+C") if err := b.consume(deliveries, concurrency, taskProcessor, amqpCloseChan); err != nil { return b.GetRetry(), err } // Waiting for any tasks being processed to finish b.processingWG.Wait() return b.GetRetry(), nil } // StopConsuming quits the loop func (b *Broker) StopConsuming() { b.Broker.StopConsuming() // Waiting for any tasks being processed to finish b.processingWG.Wait() } // GetOrOpenConnection will return a connection on a particular queue name. Open connections // are saved to avoid having to reopen connection for multiple queues func (b *Broker) GetOrOpenConnection(queueName string, queueBindingKey string, exchangeDeclareArgs, queueDeclareArgs, queueBindingArgs amqp.Table) (*AMQPConnection, error) { var err error b.connectionsMutex.Lock() defer b.connectionsMutex.Unlock() conn, ok := b.connections[queueName] if !ok { conn = &AMQPConnection{ queueName: queueName, cleanup: make(chan struct{}), } conn.connection, conn.channel, conn.queue, conn.confirmation, conn.errorchan, err = b.Connect( b.GetConfig().Broker, b.GetConfig().MultipleBrokerSeparator, b.GetConfig().TLSConfig, b.GetConfig().AMQP.Exchange, // exchange name b.GetConfig().AMQP.ExchangeType, // exchange type queueName, // queue name true, // queue durable false, // queue delete when unused queueBindingKey, // queue binding key exchangeDeclareArgs, // exchange declare args queueDeclareArgs, // queue declare args queueBindingArgs, // queue binding args ) if err != nil { return nil, errors.Wrapf(err, "Failed to connect to queue %s", queueName) } // Reconnect to the channel if it disconnects/errors out go func() { select { case err = <-conn.errorchan: log.INFO.Printf("Error occurred on queue: %s. Reconnecting", queueName) b.connectionsMutex.Lock() delete(b.connections, queueName) b.connectionsMutex.Unlock() _, err := b.GetOrOpenConnection(queueName, queueBindingKey, exchangeDeclareArgs, queueDeclareArgs, queueBindingArgs) if err != nil { log.ERROR.Printf("Failed to reopen queue: %s.", queueName) } case <-conn.cleanup: return } }() b.connections[queueName] = conn } return conn, nil } func (b *Broker) CloseConnections() error { b.connectionsMutex.Lock() defer b.connectionsMutex.Unlock() for key, conn := range b.connections { if err := b.Close(conn.channel, conn.connection); err != nil { log.ERROR.Print("Failed to close channel") return nil } close(conn.cleanup) delete(b.connections, key) } return nil } // Publish places a new message on the default queue func (b *Broker) Publish(ctx context.Context, signature *tasks.Signature) error { // Adjust routing key (this decides which queue the message will be published to) b.AdjustRoutingKey(signature) msg, err := json.Marshal(signature) if err != nil { return fmt.Errorf("JSON marshal error: %s", err) } // Check the ETA signature field, if it is set and it is in the future, // delay the task if signature.ETA != nil { now := time.Now().UTC() if signature.ETA.After(now) { delayMs := int64(signature.ETA.Sub(now) / time.Millisecond) return b.delay(signature, delayMs) } } queue := b.GetConfig().DefaultQueue bindingKey := b.GetConfig().AMQP.BindingKey // queue binding key if b.isDirectExchange() { queue = signature.RoutingKey bindingKey = signature.RoutingKey } connection, err := b.GetOrOpenConnection( queue, bindingKey, // queue binding key nil, // exchange declare args amqp.Table(b.GetConfig().AMQP.QueueDeclareArgs), // queue declare args amqp.Table(b.GetConfig().AMQP.QueueBindingArgs), // queue binding args ) if err != nil { return errors.Wrapf(err, "Failed to get a connection for queue %s", queue) } channel := connection.channel confirmsChan := connection.confirmation if err := channel.Publish( b.GetConfig().AMQP.Exchange, // exchange name signature.RoutingKey, // routing key false, // mandatory false, // immediate amqp.Publishing{ Headers: amqp.Table(signature.Headers), ContentType: "application/json", Body: msg, Priority: signature.Priority, DeliveryMode: amqp.Persistent, }, ); err != nil { return errors.Wrap(err, "Failed to publish task") } confirmed := <-confirmsChan if confirmed.Ack { return nil } return fmt.Errorf("Failed delivery of delivery tag: %v", confirmed.DeliveryTag) } // consume takes delivered messages from the channel and manages a worker pool // to process tasks concurrently func (b *Broker) consume(deliveries <-chan amqp.Delivery, concurrency int, taskProcessor iface.TaskProcessor, amqpCloseChan <-chan *amqp.Error) error { pool := make(chan struct{}, concurrency) // initialize worker pool with maxWorkers workers go func() { for i := 0; i < concurrency; i++ { pool <- struct{}{} } }() // make channel with a capacity makes it become a buffered channel so that a worker which wants to // push an error to `errorsChan` doesn't need to be blocked while the for-loop is blocked waiting // a worker, that is, it avoids a possible deadlock errorsChan := make(chan error, 1) for { select { case amqpErr := <-amqpCloseChan: return amqpErr case err := <-errorsChan: return err case d := <-deliveries: if concurrency > 0 { // get worker from pool (blocks until one is available) <-pool } b.processingWG.Add(1) // Consume the task inside a gotourine so multiple tasks // can be processed concurrently go func() { if err := b.consumeOne(d, taskProcessor, true); err != nil { errorsChan <- err } b.processingWG.Done() if concurrency > 0 { // give worker back to pool pool <- struct{}{} } }() case <-b.GetStopChan(): return nil } } } // consumeOne processes a single message using TaskProcessor func (b *Broker) consumeOne(delivery amqp.Delivery, taskProcessor iface.TaskProcessor, ack bool) error { if len(delivery.Body) == 0 { delivery.Nack(true, false) // multiple, requeue return errors.New("Received an empty message") // RabbitMQ down? } var multiple, requeue = false, false // Unmarshal message body into signature struct signature := new(tasks.Signature) decoder := json.NewDecoder(bytes.NewReader(delivery.Body)) decoder.UseNumber() if err := decoder.Decode(signature); err != nil { delivery.Nack(multiple, requeue) return errs.NewErrCouldNotUnmarshalTaskSignature(delivery.Body, err) } // If the task is not registered, we nack it and requeue, // there might be different workers for processing specific tasks if !b.IsTaskRegistered(signature.Name) { if !signature.IgnoreWhenTaskNotRegistered { requeue = true } log.INFO.Printf("Task not registered with this worker. Requeuing: %t with message: %s", requeue, delivery.Body) delivery.Nack(multiple, requeue) return nil } log.DEBUG.Printf("Received new message: %s", delivery.Body) err := taskProcessor.Process(signature) if ack { delivery.Ack(multiple) } return err } // delay a task by delayDuration miliseconds, the way it works is a new queue // is created without any consumers, the message is then published to this queue // with appropriate ttl expiration headers, after the expiration, it is sent to // the proper queue with consumers func (b *Broker) delay(signature *tasks.Signature, delayMs int64) error { if delayMs <= 0 { return errors.New("Cannot delay task by 0ms") } message, err := json.Marshal(signature) if err != nil { return fmt.Errorf("JSON marshal error: %s", err) } queueName := b.GetConfig().AMQP.DelayedQueue declareQueueArgs := amqp.Table{ // Exchange where to send messages after TTL expiration. "x-dead-letter-exchange": b.GetConfig().AMQP.Exchange, // Routing key which use when resending expired messages. "x-dead-letter-routing-key": signature.RoutingKey, } messageProperties := amqp.Publishing{ Headers: amqp.Table(signature.Headers), ContentType: "application/json", Body: message, DeliveryMode: amqp.Persistent, Expiration: fmt.Sprint(delayMs), } if queueName == "" { // It's necessary to redeclare the queue each time (to zero its TTL timer). queueName = fmt.Sprintf( "delay.%d.%s.%s", delayMs, // delay duration in mileseconds b.GetConfig().AMQP.Exchange, signature.RoutingKey, // routing key ) declareQueueArgs = amqp.Table{ // Exchange where to send messages after TTL expiration. "x-dead-letter-exchange": b.GetConfig().AMQP.Exchange, // Routing key which use when resending expired messages. "x-dead-letter-routing-key": signature.RoutingKey, // Time in milliseconds // after that message will expire and be sent to destination. "x-message-ttl": delayMs, // Time after that the queue will be deleted. "x-expires": delayMs * 2, } messageProperties = amqp.Publishing{ Headers: amqp.Table(signature.Headers), ContentType: "application/json", Body: message, DeliveryMode: amqp.Persistent, } } conn, channel, _, _, _, err := b.Connect( b.GetConfig().Broker, b.GetConfig().MultipleBrokerSeparator, b.GetConfig().TLSConfig, b.GetConfig().AMQP.Exchange, // exchange name b.GetConfig().AMQP.ExchangeType, // exchange type queueName, // queue name true, // queue durable b.GetConfig().AMQP.AutoDelete, // queue delete when unused queueName, // queue binding key nil, // exchange declare args declareQueueArgs, // queue declare args amqp.Table(b.GetConfig().AMQP.QueueBindingArgs), // queue binding args ) if err != nil { return err } defer b.Close(channel, conn) if err := channel.Publish( b.GetConfig().AMQP.Exchange, // exchange queueName, // routing key false, // mandatory false, // immediate messageProperties, ); err != nil { return err } return nil } func (b *Broker) isDirectExchange() bool { return b.GetConfig().AMQP != nil && b.GetConfig().AMQP.ExchangeType == "direct" } // AdjustRoutingKey makes sure the routing key is correct. // If the routing key is an empty string: // a) set it to binding key for direct exchange type // b) set it to default queue name func (b *Broker) AdjustRoutingKey(s *tasks.Signature) { if s.RoutingKey != "" { return } if b.isDirectExchange() { // The routing algorithm behind a direct exchange is simple - a message goes // to the queues whose binding key exactly matches the routing key of the message. s.RoutingKey = b.GetConfig().AMQP.BindingKey return } s.RoutingKey = b.GetConfig().DefaultQueue } // Helper type for GetPendingTasks to accumulate signatures type sigDumper struct { customQueue string Signatures []*tasks.Signature } func (s *sigDumper) Process(sig *tasks.Signature) error { s.Signatures = append(s.Signatures, sig) return nil } func (s *sigDumper) CustomQueue() string { return s.customQueue } func (_ *sigDumper) PreConsumeHandler() bool { return true } func (b *Broker) GetPendingTasks(queue string) ([]*tasks.Signature, error) { if queue == "" { queue = b.GetConfig().DefaultQueue } bindingKey := b.GetConfig().AMQP.BindingKey // queue binding key conn, err := b.GetOrOpenConnection( queue, bindingKey, // queue binding key nil, // exchange declare args amqp.Table(b.GetConfig().AMQP.QueueDeclareArgs), // queue declare args amqp.Table(b.GetConfig().AMQP.QueueBindingArgs), // queue binding args ) if err != nil { return nil, errors.Wrapf(err, "Failed to get a connection for queue %s", queue) } channel := conn.channel queueInfo, err := channel.QueueInspect(queue) if err != nil { return nil, errors.Wrapf(err, "Failed to get info for queue %s", queue) } var tag uint64 defer channel.Nack(tag, true, true) // multiple, requeue dumper := &sigDumper{customQueue: queue} for i := 0; i < queueInfo.Messages; i++ { d, _, err := channel.Get(queue, false) if err != nil { return nil, errors.Wrap(err, "Failed to get from queue") } tag = d.DeliveryTag b.consumeOne(d, dumper, false) } return dumper.Signatures, nil } ================================================ FILE: v1/brokers/amqp/amqp_concurrence_test.go ================================================ package amqp import ( "fmt" "github.com/RichardKnop/machinery/v1/brokers/iface" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/tasks" amqp "github.com/rabbitmq/amqp091-go" "testing" "time" ) type doNothingProcessor struct{} func (_ doNothingProcessor) Process(signature *tasks.Signature) error { return fmt.Errorf("failed") } func (_ doNothingProcessor) CustomQueue() string { return "oops" } func (_ doNothingProcessor) PreConsumeHandler() bool { return true } func TestConsume(t *testing.T) { var ( iBroker iface.Broker deliveries = make(chan amqp.Delivery, 3) closeChan chan *amqp.Error processor doNothingProcessor ) t.Run("with deliveries more than the number of concurrency", func(t *testing.T) { iBroker = New(&config.Config{}) broker, _ := iBroker.(*Broker) errChan := make(chan error) // simulate that there are too much deliveries go func() { for i := 0; i < 3; i++ { deliveries <- amqp.Delivery{} // broker.consumeOne() will complain this error: Received an empty message } }() go func() { err := broker.consume(deliveries, 2, processor, closeChan) if err != nil { errChan <- err } }() select { case <-errChan: case <-time.After(1 * time.Second): t.Error("Maybe deadlock") } }) } ================================================ FILE: v1/brokers/amqp/amqp_test.go ================================================ package amqp_test import ( "testing" "github.com/RichardKnop/machinery/v1/brokers/amqp" "github.com/RichardKnop/machinery/v1/brokers/iface" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/tasks" "github.com/stretchr/testify/assert" ) func TestAdjustRoutingKey(t *testing.T) { t.Parallel() var ( s *tasks.Signature broker iface.Broker ) t.Run("with routing and binding keys", func(t *testing.T) { s := &tasks.Signature{RoutingKey: "routing_key"} broker = amqp.New(&config.Config{ DefaultQueue: "queue", AMQP: &config.AMQPConfig{ ExchangeType: "direct", BindingKey: "binding_key", }, }) broker.AdjustRoutingKey(s) assert.Equal(t, "routing_key", s.RoutingKey) }) t.Run("with binding key", func(t *testing.T) { s = new(tasks.Signature) broker = amqp.New(&config.Config{ DefaultQueue: "queue", AMQP: &config.AMQPConfig{ ExchangeType: "direct", BindingKey: "binding_key", }, }) broker.AdjustRoutingKey(s) assert.Equal(t, "binding_key", s.RoutingKey) }) } ================================================ FILE: v1/brokers/eager/eager.go ================================================ package eager import ( "bytes" "context" "encoding/json" "errors" "fmt" "github.com/RichardKnop/machinery/v1/brokers/iface" "github.com/RichardKnop/machinery/v1/common" "github.com/RichardKnop/machinery/v1/tasks" ) // Broker represents an "eager" in-memory broker type Broker struct { worker iface.TaskProcessor common.Broker } // New creates new Broker instance func New() iface.Broker { return new(Broker) } // Mode interface with methods specific for this broker type Mode interface { AssignWorker(p iface.TaskProcessor) } // StartConsuming enters a loop and waits for incoming messages func (eagerBroker *Broker) StartConsuming(consumerTag string, concurrency int, p iface.TaskProcessor) (bool, error) { return true, nil } // StopConsuming quits the loop func (eagerBroker *Broker) StopConsuming() { // do nothing } // Publish places a new message on the default queue func (eagerBroker *Broker) Publish(ctx context.Context, task *tasks.Signature) error { if eagerBroker.worker == nil { return errors.New("worker is not assigned in eager-mode") } // faking the behavior to marshal input into json // and unmarshal it back message, err := json.Marshal(task) if err != nil { return fmt.Errorf("JSON marshal error: %s", err) } signature := new(tasks.Signature) decoder := json.NewDecoder(bytes.NewReader(message)) decoder.UseNumber() if err := decoder.Decode(signature); err != nil { return fmt.Errorf("JSON unmarshal error: %s", err) } // blocking call to the task directly return eagerBroker.worker.Process(signature) } // AssignWorker assigns a worker to the eager broker func (eagerBroker *Broker) AssignWorker(w iface.TaskProcessor) { eagerBroker.worker = w } ================================================ FILE: v1/brokers/errs/errors.go ================================================ package errs import ( "errors" "fmt" ) // ErrCouldNotUnmarshalTaskSignature ... type ErrCouldNotUnmarshalTaskSignature struct { msg []byte reason string } // Error implements the error interface func (e ErrCouldNotUnmarshalTaskSignature) Error() string { return fmt.Sprintf("Could not unmarshal '%s' into a task signature: %v", e.msg, e.reason) } // NewErrCouldNotUnmarshalTaskSignature returns new ErrCouldNotUnmarshalTaskSignature instance func NewErrCouldNotUnmarshalTaskSignature(msg []byte, err error) ErrCouldNotUnmarshalTaskSignature { return ErrCouldNotUnmarshalTaskSignature{msg: msg, reason: err.Error()} } // ErrConsumerStopped indicates that the operation is now illegal because of the consumer being stopped. var ErrConsumerStopped = errors.New("the server has been stopped") // ErrStopTaskDeletion indicates that the task should not be deleted from source after task failure var ErrStopTaskDeletion = errors.New("task should not be deleted") ================================================ FILE: v1/brokers/gcppubsub/gcp_pubsub.go ================================================ package gcppubsub import ( "bytes" "context" "encoding/json" "fmt" "time" "cloud.google.com/go/pubsub" "github.com/RichardKnop/machinery/v1/brokers/iface" "github.com/RichardKnop/machinery/v1/common" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/log" "github.com/RichardKnop/machinery/v1/tasks" ) // Broker represents an Google Cloud Pub/Sub broker type Broker struct { common.Broker service *pubsub.Client subscriptionName string MaxExtension time.Duration stopDone chan struct{} } // New creates new Broker instance func New(cnf *config.Config, projectID, subscriptionName string) (iface.Broker, error) { b := &Broker{Broker: common.NewBroker(cnf), stopDone: make(chan struct{})} b.subscriptionName = subscriptionName ctx := context.Background() if cnf.GCPPubSub != nil { b.MaxExtension = cnf.GCPPubSub.MaxExtension } if cnf.GCPPubSub != nil && cnf.GCPPubSub.Client != nil { b.service = cnf.GCPPubSub.Client } else { pubsubClient, err := pubsub.NewClient(ctx, projectID) if err != nil { return nil, err } b.service = pubsubClient cnf.GCPPubSub = &config.GCPPubSubConfig{ Client: pubsubClient, } } // Validate topic exists defaultQueue := b.GetConfig().DefaultQueue topic := b.service.Topic(defaultQueue) defer topic.Stop() topicExists, err := topic.Exists(ctx) if err != nil { return nil, err } if !topicExists { return nil, fmt.Errorf("topic does not exist, instead got %s", defaultQueue) } // Validate subscription exists sub := b.service.Subscription(b.subscriptionName) if b.MaxExtension != 0 { sub.ReceiveSettings.MaxExtension = b.MaxExtension } subscriptionExists, err := sub.Exists(ctx) if err != nil { return nil, err } if !subscriptionExists { return nil, fmt.Errorf("subscription does not exist, instead got %s", b.subscriptionName) } return b, nil } // StartConsuming enters a loop and waits for incoming messages func (b *Broker) StartConsuming(consumerTag string, concurrency int, taskProcessor iface.TaskProcessor) (bool, error) { b.Broker.StartConsuming(consumerTag, concurrency, taskProcessor) sub := b.service.Subscription(b.subscriptionName) if b.MaxExtension != 0 { sub.ReceiveSettings.MaxExtension = b.MaxExtension } sub.ReceiveSettings.NumGoroutines = concurrency sub.ReceiveSettings.MaxOutstandingMessages = concurrency log.INFO.Print("[*] Waiting for messages. To exit press CTRL+C") ctx, cancel := context.WithCancel(context.Background()) go func() { <-b.GetStopChan() cancel() }() for { err := sub.Receive(ctx, func(_ctx context.Context, msg *pubsub.Message) { b.consumeOne(msg, taskProcessor) }) if err == nil { break } log.ERROR.Printf("Error when receiving messages. Error: %v", err) continue } close(b.stopDone) return b.GetRetry(), nil } // StopConsuming quits the loop func (b *Broker) StopConsuming() { b.Broker.StopConsuming() // Waiting for any tasks being processed to finish <-b.stopDone } // Publish places a new message on the default queue or the queue pointed to // by the routing key func (b *Broker) Publish(ctx context.Context, signature *tasks.Signature) error { // Adjust routing key (this decides which queue the message will be published to) b.AdjustRoutingKey(signature) msg, err := json.Marshal(signature) if err != nil { return fmt.Errorf("JSON marshal error: %s", err) } topic := b.service.Topic(signature.RoutingKey) defer topic.Stop() // Check the ETA signature field, if it is set and it is in the future, // delay the task if signature.ETA != nil { now := time.Now().UTC() if signature.ETA.After(now) { topic.PublishSettings.DelayThreshold = signature.ETA.Sub(now) } } result := topic.Publish(ctx, &pubsub.Message{ Data: msg, }) id, err := result.Get(ctx) if err != nil { log.ERROR.Printf("Error when sending a message: %v", err) return err } log.INFO.Printf("Sending a message successfully, server-generated message ID %v", id) return nil } // consumeOne processes a single message using TaskProcessor func (b *Broker) consumeOne(delivery *pubsub.Message, taskProcessor iface.TaskProcessor) { if len(delivery.Data) == 0 { delivery.Nack() log.ERROR.Printf("received an empty message, the delivery was %v", delivery) } sig := new(tasks.Signature) decoder := json.NewDecoder(bytes.NewBuffer(delivery.Data)) decoder.UseNumber() if err := decoder.Decode(sig); err != nil { delivery.Nack() log.ERROR.Printf("unmarshal error. the delivery is %v", delivery) } // If the task is not registered return an error // and leave the message in the queue if !b.IsTaskRegistered(sig.Name) { delivery.Nack() log.ERROR.Printf("task %s is not registered", sig.Name) } err := taskProcessor.Process(sig) if err != nil { delivery.Nack() log.ERROR.Printf("Failed process of task", err) } // Call Ack() after successfully consuming and processing the message delivery.Ack() } ================================================ FILE: v1/brokers/iface/interfaces.go ================================================ package iface import ( "context" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/tasks" ) // Broker - a common interface for all brokers type Broker interface { GetConfig() *config.Config SetRegisteredTaskNames(names []string) IsTaskRegistered(name string) bool StartConsuming(consumerTag string, concurrency int, p TaskProcessor) (bool, error) StopConsuming() Publish(ctx context.Context, task *tasks.Signature) error GetPendingTasks(queue string) ([]*tasks.Signature, error) GetDelayedTasks() ([]*tasks.Signature, error) AdjustRoutingKey(s *tasks.Signature) } // TaskProcessor - can process a delivered task // This will probably always be a worker instance type TaskProcessor interface { Process(signature *tasks.Signature) error CustomQueue() string PreConsumeHandler() bool } ================================================ FILE: v1/brokers/package.go ================================================ package brokers ================================================ FILE: v1/brokers/redis/goredis.go ================================================ package redis import ( "bytes" "context" "encoding/json" "fmt" "runtime" "strconv" "strings" "sync" "time" "github.com/go-redsync/redsync/v4" "github.com/redis/go-redis/v9" "github.com/RichardKnop/machinery/v1/brokers/errs" "github.com/RichardKnop/machinery/v1/brokers/iface" "github.com/RichardKnop/machinery/v1/common" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/log" "github.com/RichardKnop/machinery/v1/tasks" ) // BrokerGR represents a Redis broker type BrokerGR struct { common.Broker rclient redis.UniversalClient consumingWG sync.WaitGroup // wait group to make sure whole consumption completes processingWG sync.WaitGroup // use wait group to make sure task processing completes delayedWG sync.WaitGroup // If set, path to a socket file overrides hostname socketPath string redsync *redsync.Redsync redisOnce sync.Once redisDelayedTasksKey string } // NewGR creates new Broker instance func NewGR(cnf *config.Config, addrs []string, db int) iface.Broker { b := &BrokerGR{Broker: common.NewBroker(cnf)} var password string var username string parts := strings.Split(addrs[0], "@") if len(parts) >= 2 { // with password options := strings.SplitN(strings.Join(parts[:len(parts)-1], "@"), ":", 2) if len(options) >= 2 { username = options[0] password = options[1] } else { password = options[0] } addrs[0] = parts[len(parts)-1] // addr is the last one without @ } ropt := &redis.UniversalOptions{ Addrs: addrs, DB: db, Password: password, Username: username, } if cnf.Redis != nil { ropt.MasterName = cnf.Redis.MasterName } if cnf.Redis != nil && cnf.Redis.ClusterMode { b.rclient = redis.NewClusterClient(ropt.Cluster()) } else { b.rclient = redis.NewUniversalClient(ropt) } if cnf.Redis.DelayedTasksKey != "" { b.redisDelayedTasksKey = cnf.Redis.DelayedTasksKey } else { b.redisDelayedTasksKey = defaultRedisDelayedTasksKey } return b } // StartConsuming enters a loop and waits for incoming messages func (b *BrokerGR) StartConsuming(consumerTag string, concurrency int, taskProcessor iface.TaskProcessor) (bool, error) { b.consumingWG.Add(1) defer b.consumingWG.Done() if concurrency < 1 { concurrency = runtime.NumCPU() * 2 } b.Broker.StartConsuming(consumerTag, concurrency, taskProcessor) // Ping the server to make sure connection is live _, err := b.rclient.Ping(context.Background()).Result() if err != nil { b.GetRetryFunc()(b.GetRetryStopChan()) // Return err if retry is still true. // If retry is false, broker.StopConsuming() has been called and // therefore Redis might have been stopped. Return nil exit // StartConsuming() if b.GetRetry() { return b.GetRetry(), err } return b.GetRetry(), errs.ErrConsumerStopped } // Channel to which we will push tasks ready for processing by worker deliveries := make(chan []byte, concurrency) pool := make(chan struct{}, concurrency) // initialize worker pool with maxWorkers workers for i := 0; i < concurrency; i++ { pool <- struct{}{} } // A receiving goroutine keeps popping messages from the queue by BLPOP // If the message is valid and can be unmarshaled into a proper structure // we send it to the deliveries channel go func() { log.INFO.Print("[*] Waiting for messages. To exit press CTRL+C") for { select { // A way to stop this goroutine from b.StopConsuming case <-b.GetStopChan(): close(deliveries) return case <-pool: task, _ := b.nextTask(getQueueGR(b.GetConfig(), taskProcessor)) //TODO: should this error be ignored? if len(task) > 0 { deliveries <- task } pool <- struct{}{} } } }() // A goroutine to watch for delayed tasks and push them to deliveries // channel for consumption by the worker b.delayedWG.Add(1) go func() { defer b.delayedWG.Done() for { select { // A way to stop this goroutine from b.StopConsuming case <-b.GetStopChan(): return default: task, err := b.nextDelayedTask(b.redisDelayedTasksKey) if err != nil { continue } signature := new(tasks.Signature) decoder := json.NewDecoder(bytes.NewReader(task)) decoder.UseNumber() if err := decoder.Decode(signature); err != nil { log.ERROR.Print(errs.NewErrCouldNotUnmarshalTaskSignature(task, err)) } if err := b.Publish(context.Background(), signature); err != nil { log.ERROR.Print(err) } } } }() if err := b.consume(deliveries, concurrency, taskProcessor); err != nil { return b.GetRetry(), err } // Waiting for any tasks being processed to finish b.processingWG.Wait() return b.GetRetry(), nil } // StopConsuming quits the loop func (b *BrokerGR) StopConsuming() { b.Broker.StopConsuming() // Waiting for the delayed tasks goroutine to have stopped b.delayedWG.Wait() // Waiting for consumption to finish b.consumingWG.Wait() b.rclient.Close() } // Publish places a new message on the default queue func (b *BrokerGR) Publish(ctx context.Context, signature *tasks.Signature) error { // Adjust routing key (this decides which queue the message will be published to) b.Broker.AdjustRoutingKey(signature) msg, err := json.Marshal(signature) if err != nil { return fmt.Errorf("JSON marshal error: %s", err) } // Check the ETA signature field, if it is set and it is in the future, // delay the task if signature.ETA != nil { now := time.Now().UTC() if signature.ETA.After(now) { score := signature.ETA.UnixNano() err = b.rclient.ZAdd(context.Background(), b.redisDelayedTasksKey, redis.Z{Score: float64(score), Member: msg}).Err() return err } } err = b.rclient.RPush(context.Background(), signature.RoutingKey, msg).Err() return err } // GetPendingTasks returns a slice of task signatures waiting in the queue func (b *BrokerGR) GetPendingTasks(queue string) ([]*tasks.Signature, error) { if queue == "" { queue = b.GetConfig().DefaultQueue } results, err := b.rclient.LRange(context.Background(), queue, 0, -1).Result() if err != nil { return nil, err } taskSignatures := make([]*tasks.Signature, len(results)) for i, result := range results { signature := new(tasks.Signature) decoder := json.NewDecoder(strings.NewReader(result)) decoder.UseNumber() if err := decoder.Decode(signature); err != nil { return nil, err } taskSignatures[i] = signature } return taskSignatures, nil } // GetDelayedTasks returns a slice of task signatures that are scheduled, but not yet in the queue func (b *BrokerGR) GetDelayedTasks() ([]*tasks.Signature, error) { results, err := b.rclient.ZRange(context.Background(), b.redisDelayedTasksKey, 0, -1).Result() if err != nil { return nil, err } taskSignatures := make([]*tasks.Signature, len(results)) for i, result := range results { signature := new(tasks.Signature) decoder := json.NewDecoder(strings.NewReader(result)) decoder.UseNumber() if err := decoder.Decode(signature); err != nil { return nil, err } taskSignatures[i] = signature } return taskSignatures, nil } // consume takes delivered messages from the channel and manages a worker pool // to process tasks concurrently func (b *BrokerGR) consume(deliveries <-chan []byte, concurrency int, taskProcessor iface.TaskProcessor) error { errorsChan := make(chan error, concurrency*2) pool := make(chan struct{}, concurrency) // init pool for Worker tasks execution, as many slots as Worker concurrency param go func() { for i := 0; i < concurrency; i++ { pool <- struct{}{} } }() for { select { case err := <-errorsChan: return err case d, open := <-deliveries: if !open { return nil } if concurrency > 0 { // get execution slot from pool (blocks until one is available) <-pool } b.processingWG.Add(1) // Consume the task inside a goroutine so multiple tasks // can be processed concurrently go func() { if err := b.consumeOne(d, taskProcessor); err != nil { errorsChan <- err } b.processingWG.Done() if concurrency > 0 { // give slot back to pool pool <- struct{}{} } }() } } } // consumeOne processes a single message using TaskProcessor func (b *BrokerGR) consumeOne(delivery []byte, taskProcessor iface.TaskProcessor) error { signature := new(tasks.Signature) decoder := json.NewDecoder(bytes.NewReader(delivery)) decoder.UseNumber() if err := decoder.Decode(signature); err != nil { return errs.NewErrCouldNotUnmarshalTaskSignature(delivery, err) } // If the task is not registered, we requeue it, // there might be different workers for processing specific tasks if !b.IsTaskRegistered(signature.Name) { if signature.IgnoreWhenTaskNotRegistered { return nil } log.INFO.Printf("Task not registered with this worker. Requeuing message: %s", delivery) b.rclient.RPush(context.Background(), getQueueGR(b.GetConfig(), taskProcessor), delivery) return nil } log.DEBUG.Printf("Received new message: %s", delivery) return taskProcessor.Process(signature) } // nextTask pops next available task from the default queue func (b *BrokerGR) nextTask(queue string) (result []byte, err error) { pollPeriodMilliseconds := 1000 // default poll period for normal tasks if b.GetConfig().Redis != nil { configuredPollPeriod := b.GetConfig().Redis.NormalTasksPollPeriod if configuredPollPeriod > 0 { pollPeriodMilliseconds = configuredPollPeriod } } pollPeriod := time.Duration(pollPeriodMilliseconds) * time.Millisecond items, err := b.rclient.BLPop(context.Background(), pollPeriod, queue).Result() if err != nil { return []byte{}, err } // items[0] - the name of the key where an element was popped // items[1] - the value of the popped element if len(items) != 2 { return []byte{}, redis.Nil } result = []byte(items[1]) return result, nil } // nextDelayedTask pops a value from the ZSET key using WATCH/MULTI/EXEC commands. func (b *BrokerGR) nextDelayedTask(key string) (result []byte, err error) { //pipe := b.rclient.Pipeline() // //defer func() { // // Return connection to normal state on error. // // https://redis.io/commands/discard // if err != nil { // pipe.Discard() // } //}() var ( items []string ) pollPeriod := 500 // default poll period for delayed tasks if b.GetConfig().Redis != nil { configuredPollPeriod := b.GetConfig().Redis.DelayedTasksPollPeriod // the default period is 0, which bombards redis with requests, despite // our intention of doing the opposite if configuredPollPeriod > 0 { pollPeriod = configuredPollPeriod } } for { // Space out queries to ZSET so we don't bombard redis // server with relentless ZRANGEBYSCOREs time.Sleep(time.Duration(pollPeriod) * time.Millisecond) watchFunc := func(tx *redis.Tx) error { now := time.Now().UTC().UnixNano() // https://redis.io/commands/zrangebyscore ctx := context.Background() items, err = tx.ZRevRangeByScore(ctx, key, &redis.ZRangeBy{ Min: "0", Max: strconv.FormatInt(now, 10), Offset: 0, Count: 1, }).Result() if err != nil { return err } if len(items) != 1 { return redis.Nil } // only return the first zrange value if there are no other changes in this key // to make sure a delayed task would only be consumed once _, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error { pipe.ZRem(ctx, key, items[0]) result = []byte(items[0]) return nil }) return err } if err = b.rclient.Watch(context.Background(), watchFunc, key); err != nil { return } else { break } } return } func getQueueGR(config *config.Config, taskProcessor iface.TaskProcessor) string { customQueue := taskProcessor.CustomQueue() if customQueue == "" { return config.DefaultQueue } return customQueue } ================================================ FILE: v1/brokers/redis/redis.go ================================================ package redis import ( "bytes" "context" "encoding/json" "fmt" "math" "runtime" "sync" "time" "github.com/go-redsync/redsync/v4" redsyncredis "github.com/go-redsync/redsync/v4/redis/redigo" "github.com/gomodule/redigo/redis" "github.com/RichardKnop/machinery/v1/brokers/errs" "github.com/RichardKnop/machinery/v1/brokers/iface" "github.com/RichardKnop/machinery/v1/common" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/log" "github.com/RichardKnop/machinery/v1/tasks" ) const defaultRedisDelayedTasksKey = "delayed_tasks" // Broker represents a Redis broker type Broker struct { common.Broker common.RedisConnector username string host string password string db int pool *redis.Pool consumingWG sync.WaitGroup // wait group to make sure whole consumption completes processingWG sync.WaitGroup // use wait group to make sure task processing completes delayedWG sync.WaitGroup // If set, path to a socket file overrides hostname socketPath string redsync *redsync.Redsync redisOnce sync.Once redisDelayedTasksKey string } // New creates new Broker instance func New(cnf *config.Config, host, username, password, socketPath string, db int) iface.Broker { b := &Broker{Broker: common.NewBroker(cnf)} b.host = host b.db = db b.username = username b.password = password b.socketPath = socketPath if cnf.Redis != nil && cnf.Redis.DelayedTasksKey != "" { b.redisDelayedTasksKey = cnf.Redis.DelayedTasksKey } else { b.redisDelayedTasksKey = defaultRedisDelayedTasksKey } return b } // StartConsuming enters a loop and waits for incoming messages func (b *Broker) StartConsuming(consumerTag string, concurrency int, taskProcessor iface.TaskProcessor) (bool, error) { b.consumingWG.Add(1) defer b.consumingWG.Done() if concurrency < 1 { concurrency = runtime.NumCPU() * 2 } b.Broker.StartConsuming(consumerTag, concurrency, taskProcessor) conn := b.open() defer conn.Close() // Ping the server to make sure connection is live _, err := conn.Do("PING") if err != nil { b.GetRetryFunc()(b.GetRetryStopChan()) // Return err if retry is still true. // If retry is false, broker.StopConsuming() has been called and // therefore Redis might have been stopped. Return nil exit // StartConsuming() if b.GetRetry() { return b.GetRetry(), err } return b.GetRetry(), errs.ErrConsumerStopped } // Channel to which we will push tasks ready for processing by worker deliveries := make(chan []byte, concurrency) pool := make(chan struct{}, concurrency) stopConsumer := make(chan struct{}) // initialize worker pool with maxWorkers workers for i := 0; i < concurrency; i++ { pool <- struct{}{} } // A receiving goroutine keeps popping messages from the queue by BLPOP // If the message is valid and can be unmarshaled into a proper structure // we send it to the deliveries channel go func() { log.INFO.Print("[*] Waiting for messages. To exit press CTRL+C") for { select { // A way to stop this goroutine from b.StopConsuming case <-b.GetStopChan(): close(deliveries) return case <-stopConsumer: close(deliveries) return case <-pool: select { case <-b.GetStopChan(): close(deliveries) return default: } if taskProcessor.PreConsumeHandler() { task, _ := b.nextTask(getQueue(b.GetConfig(), taskProcessor)) //TODO: should this error be ignored? if len(task) > 0 { deliveries <- task } } pool <- struct{}{} } } }() // A goroutine to watch for delayed tasks and push them to deliveries // channel for consumption by the worker b.delayedWG.Add(1) go func() { defer b.delayedWG.Done() for { select { // A way to stop this goroutine from b.StopConsuming case <-b.GetStopChan(): return case <-stopConsumer: return default: task, err := b.nextDelayedTask(b.redisDelayedTasksKey) if err != nil { continue } signature := new(tasks.Signature) decoder := json.NewDecoder(bytes.NewReader(task)) decoder.UseNumber() if err := decoder.Decode(signature); err != nil { log.ERROR.Print(errs.NewErrCouldNotUnmarshalTaskSignature(task, err)) } if err := b.Publish(context.Background(), signature); err != nil { log.ERROR.Print(err) } } } }() if err := b.consume(deliveries, concurrency, taskProcessor, stopConsumer); err != nil { return b.GetRetry(), err } // Waiting for any tasks being processed to finish b.processingWG.Wait() return b.GetRetry(), nil } // StopConsuming quits the loop func (b *Broker) StopConsuming() { b.Broker.StopConsuming() // Waiting for the delayed tasks goroutine to have stopped b.delayedWG.Wait() // Waiting for consumption to finish b.consumingWG.Wait() // Wait for currently processing tasks to finish as well. b.processingWG.Wait() if b.pool != nil { b.pool.Close() } } // Publish places a new message on the default queue func (b *Broker) Publish(ctx context.Context, signature *tasks.Signature) error { // Adjust routing key (this decides which queue the message will be published to) b.Broker.AdjustRoutingKey(signature) msg, err := json.Marshal(signature) if err != nil { return fmt.Errorf("JSON marshal error: %s", err) } conn := b.open() defer conn.Close() // Check the ETA signature field, if it is set and it is in the future, // delay the task if signature.ETA != nil { now := time.Now().UTC() if signature.ETA.After(now) { score := signature.ETA.UnixNano() _, err = conn.Do("ZADD", b.redisDelayedTasksKey, score, msg) return err } } _, err = conn.Do("RPUSH", signature.RoutingKey, msg) return err } // GetPendingTasks returns a slice of task signatures waiting in the queue func (b *Broker) GetPendingTasks(queue string) ([]*tasks.Signature, error) { conn := b.open() defer conn.Close() if queue == "" { queue = b.GetConfig().DefaultQueue } dataBytes, err := conn.Do("LRANGE", queue, 0, -1) if err != nil { return nil, err } results, err := redis.ByteSlices(dataBytes, err) if err != nil { return nil, err } taskSignatures := make([]*tasks.Signature, len(results)) for i, result := range results { signature := new(tasks.Signature) decoder := json.NewDecoder(bytes.NewReader(result)) decoder.UseNumber() if err := decoder.Decode(signature); err != nil { return nil, err } taskSignatures[i] = signature } return taskSignatures, nil } // GetDelayedTasks returns a slice of task signatures that are scheduled, but not yet in the queue func (b *Broker) GetDelayedTasks() ([]*tasks.Signature, error) { conn := b.open() defer conn.Close() dataBytes, err := conn.Do("ZRANGE", b.redisDelayedTasksKey, 0, -1) if err != nil { return nil, err } results, err := redis.ByteSlices(dataBytes, err) if err != nil { return nil, err } taskSignatures := make([]*tasks.Signature, len(results)) for i, result := range results { signature := new(tasks.Signature) decoder := json.NewDecoder(bytes.NewReader(result)) decoder.UseNumber() if err := decoder.Decode(signature); err != nil { return nil, err } taskSignatures[i] = signature } return taskSignatures, nil } // consume takes delivered messages from the channel and manages a worker pool // to process tasks concurrently func (b *Broker) consume(deliveries <-chan []byte, concurrency int, taskProcessor iface.TaskProcessor, stopConsumer chan struct{}) error { errorsChan := make(chan error, concurrency*2) pool := make(chan struct{}, concurrency) // init pool for Worker tasks execution, as many slots as Worker concurrency param go func() { for i := 0; i < concurrency; i++ { pool <- struct{}{} } }() for { select { case err := <-errorsChan: close(stopConsumer) for v := range deliveries { b.requeueMessage(v, taskProcessor) } return err case d, open := <-deliveries: if !open { return nil } if concurrency > 0 { // get execution slot from pool (blocks until one is available) select { case <-b.GetStopChan(): b.requeueMessage(d, taskProcessor) continue case <-pool: } } b.processingWG.Add(1) // Consume the task inside a goroutine so multiple tasks // can be processed concurrently go func() { if err := b.consumeOne(d, taskProcessor); err != nil { errorsChan <- err } b.processingWG.Done() if concurrency > 0 { // give slot back to pool pool <- struct{}{} } }() } } } // consumeOne processes a single message using TaskProcessor func (b *Broker) consumeOne(delivery []byte, taskProcessor iface.TaskProcessor) error { signature := new(tasks.Signature) decoder := json.NewDecoder(bytes.NewReader(delivery)) decoder.UseNumber() if err := decoder.Decode(signature); err != nil { return errs.NewErrCouldNotUnmarshalTaskSignature(delivery, err) } // If the task is not registered, we requeue it, // there might be different workers for processing specific tasks if !b.IsTaskRegistered(signature.Name) { if signature.IgnoreWhenTaskNotRegistered { return nil } log.INFO.Printf("Task not registered with this worker. Requeuing message: %s", delivery) b.requeueMessage(delivery, taskProcessor) return nil } log.DEBUG.Printf("Received new message: %s", delivery) return taskProcessor.Process(signature) } // nextTask pops next available task from the default queue func (b *Broker) nextTask(queue string) (result []byte, err error) { conn := b.open() defer conn.Close() pollPeriodMilliseconds := 1000 // default poll period for normal tasks if b.GetConfig().Redis != nil { configuredPollPeriod := b.GetConfig().Redis.NormalTasksPollPeriod if configuredPollPeriod > 0 { pollPeriodMilliseconds = configuredPollPeriod } } pollPeriod := time.Duration(pollPeriodMilliseconds) * time.Millisecond // Issue 548: BLPOP expects an integer timeout expresses in seconds. // The call will if the value is a float. Convert to integer using // math.Ceil(): // math.Ceil(0.0) --> 0 (block indefinitely) // math.Ceil(0.2) --> 1 (timeout after 1 second) pollPeriodSeconds := math.Ceil(pollPeriod.Seconds()) items, err := redis.ByteSlices(conn.Do("BLPOP", queue, pollPeriodSeconds)) if err != nil { return []byte{}, err } // items[0] - the name of the key where an element was popped // items[1] - the value of the popped element if len(items) != 2 { return []byte{}, redis.ErrNil } result = items[1] return result, nil } // nextDelayedTask pops a value from the ZSET key using WATCH/MULTI/EXEC commands. // https://github.com/gomodule/redigo/blob/master/redis/zpop_example_test.go func (b *Broker) nextDelayedTask(key string) (result []byte, err error) { conn := b.open() defer conn.Close() defer func() { // Return connection to normal state on error. // https://redis.io/commands/discard // https://redis.io/commands/unwatch if err == redis.ErrNil { conn.Do("UNWATCH") } else if err != nil { conn.Do("DISCARD") } }() var ( items [][]byte reply interface{} ) pollPeriod := 500 // default poll period for delayed tasks if b.GetConfig().Redis != nil { configuredPollPeriod := b.GetConfig().Redis.DelayedTasksPollPeriod // the default period is 0, which bombards redis with requests, despite // our intention of doing the opposite if configuredPollPeriod > 0 { pollPeriod = configuredPollPeriod } } for { // Space out queries to ZSET so we don't bombard redis // server with relentless ZRANGEBYSCOREs time.Sleep(time.Duration(pollPeriod) * time.Millisecond) if _, err = conn.Do("WATCH", key); err != nil { return } now := time.Now().UTC().UnixNano() // https://redis.io/commands/zrangebyscore items, err = redis.ByteSlices(conn.Do( "ZRANGEBYSCORE", key, 0, now, "LIMIT", 0, 1, )) if err != nil { return } if len(items) != 1 { err = redis.ErrNil return } _ = conn.Send("MULTI") _ = conn.Send("ZREM", key, items[0]) reply, err = conn.Do("EXEC") if err != nil { return } if reply != nil { result = items[0] break } } return } // open returns or creates instance of Redis connection func (b *Broker) open() redis.Conn { b.redisOnce.Do(func() { b.pool = b.NewPool(b.socketPath, b.host, b.username, b.password, b.db, b.GetConfig().Redis, b.GetConfig().TLSConfig) b.redsync = redsync.New(redsyncredis.NewPool(b.pool)) }) return b.pool.Get() } func getQueue(config *config.Config, taskProcessor iface.TaskProcessor) string { customQueue := taskProcessor.CustomQueue() if customQueue == "" { return config.DefaultQueue } return customQueue } func (b *Broker) requeueMessage(delivery []byte, taskProcessor iface.TaskProcessor) { conn := b.open() defer conn.Close() conn.Do("RPUSH", getQueue(b.GetConfig(), taskProcessor), delivery) } ================================================ FILE: v1/brokers/sqs/sqs.go ================================================ package sqs import ( "context" "encoding/json" "errors" "fmt" "strings" "sync" "time" "github.com/RichardKnop/machinery/v1/brokers/errs" "github.com/RichardKnop/machinery/v1/brokers/iface" "github.com/RichardKnop/machinery/v1/common" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/log" "github.com/RichardKnop/machinery/v1/tasks" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/sqs/sqsiface" awssqs "github.com/aws/aws-sdk-go/service/sqs" ) const ( maxAWSSQSDelay = time.Minute * 15 // Max supported SQS delay is 15 min: https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html ) // Broker represents a AWS SQS broker // There are examples on: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/sqs-example-create-queue.html type Broker struct { common.Broker processingWG sync.WaitGroup // use wait group to make sure task processing completes on interrupt signal receivingWG sync.WaitGroup stopReceivingChan chan int sess *session.Session service sqsiface.SQSAPI queueUrl *string } // New creates new Broker instance func New(cnf *config.Config) iface.Broker { b := &Broker{Broker: common.NewBroker(cnf)} if cnf.SQS != nil && cnf.SQS.Client != nil { // Use provided *SQS client b.service = cnf.SQS.Client } else { // Initialize a session that the SDK will use to load credentials from the shared credentials file, ~/.aws/credentials. // See details on: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html // Also, env AWS_REGION is also required b.sess = session.Must(session.NewSessionWithOptions(session.Options{ SharedConfigState: session.SharedConfigEnable, })) b.service = awssqs.New(b.sess) } return b } // StartConsuming enters a loop and waits for incoming messages func (b *Broker) StartConsuming(consumerTag string, concurrency int, taskProcessor iface.TaskProcessor) (bool, error) { b.Broker.StartConsuming(consumerTag, concurrency, taskProcessor) qURL := b.getQueueURL(taskProcessor) //save it so that it can be used later when attempting to delete task b.queueUrl = qURL deliveries := make(chan *awssqs.ReceiveMessageOutput, concurrency) pool := make(chan struct{}, concurrency) // initialize worker pool with maxWorkers workers for i := 0; i < concurrency; i++ { pool <- struct{}{} } b.stopReceivingChan = make(chan int) b.receivingWG.Add(1) go func() { defer b.receivingWG.Done() log.INFO.Printf("[*] Waiting for messages on queue: %s. To exit press CTRL+C\n", *qURL) for { select { // A way to stop this goroutine from b.StopConsuming case <-b.stopReceivingChan: close(deliveries) return case <-pool: output, err := b.receiveMessage(qURL) if err == nil && len(output.Messages) > 0 { deliveries <- output } else { //return back to pool right away pool <- struct{}{} if err != nil { log.ERROR.Printf("Queue consume error: %s", err) } } } } }() if err := b.consume(deliveries, concurrency, taskProcessor, pool); err != nil { return b.GetRetry(), err } return b.GetRetry(), nil } // StopConsuming quits the loop func (b *Broker) StopConsuming() { b.Broker.StopConsuming() b.stopReceiving() // Waiting for any tasks being processed to finish b.processingWG.Wait() // Waiting for the receiving goroutine to have stopped b.receivingWG.Wait() } // Publish places a new message on the default queue func (b *Broker) Publish(ctx context.Context, signature *tasks.Signature) error { msg, err := json.Marshal(signature) if err != nil { return fmt.Errorf("JSON marshal error: %s", err) } // Check that signature.RoutingKey is set, if not switch to DefaultQueue b.AdjustRoutingKey(signature) MsgInput := &awssqs.SendMessageInput{ MessageBody: aws.String(string(msg)), QueueUrl: aws.String(b.GetConfig().Broker + "/" + signature.RoutingKey), } // if this is a fifo queue, there needs to be some additional parameters. if strings.HasSuffix(signature.RoutingKey, ".fifo") { // Use Machinery's signature Task UUID as SQS Message Group ID. MsgDedupID := signature.UUID MsgInput.MessageDeduplicationId = aws.String(MsgDedupID) // Do not Use Machinery's signature Group UUID as SQS Message Group ID, instead use BrokerMessageGroupId MsgGroupID := signature.BrokerMessageGroupId if MsgGroupID == "" { return fmt.Errorf("please specify BrokerMessageGroupId attribute for task Signature when submitting a task to FIFO queue") } MsgInput.MessageGroupId = aws.String(MsgGroupID) } // Check the ETA signature field, if it is set and it is in the future, // and is not a fifo queue, set a delay in seconds for the task. if signature.ETA != nil && !strings.HasSuffix(signature.RoutingKey, ".fifo") { now := time.Now().UTC() delay := signature.ETA.Sub(now) if delay > 0 { if delay > maxAWSSQSDelay { return errors.New("Max AWS SQS delay exceeded") } MsgInput.DelaySeconds = aws.Int64(int64(delay.Seconds())) } } result, err := b.service.SendMessageWithContext(ctx, MsgInput) if err != nil { log.ERROR.Printf("Error when sending a message: %v", err) return err } log.INFO.Printf("Sending a message successfully, the messageId is %v", *result.MessageId) return nil } // consume is a method which keeps consuming deliveries from a channel, until there is an error or a stop signal func (b *Broker) consume(deliveries <-chan *awssqs.ReceiveMessageOutput, concurrency int, taskProcessor iface.TaskProcessor, pool chan struct{}) error { errorsChan := make(chan error) for { whetherContinue, err := b.consumeDeliveries(deliveries, concurrency, taskProcessor, pool, errorsChan) if err != nil { return err } if whetherContinue == false { return nil } } } // consumeOne is a method consumes a delivery. If a delivery was consumed successfully, it will be deleted from AWS SQS func (b *Broker) consumeOne(delivery *awssqs.ReceiveMessageOutput, taskProcessor iface.TaskProcessor) error { if len(delivery.Messages) == 0 { log.ERROR.Printf("received an empty message, the delivery was %v", delivery) return errors.New("received empty message, the delivery is " + delivery.GoString()) } sig := new(tasks.Signature) decoder := json.NewDecoder(strings.NewReader(*delivery.Messages[0].Body)) decoder.UseNumber() if err := decoder.Decode(sig); err != nil { log.ERROR.Printf("unmarshal error. the delivery is %v", delivery) // if the unmarshal fails, remove the delivery from the queue if delErr := b.deleteOne(delivery); delErr != nil { log.ERROR.Printf("error when deleting the delivery. delivery is %v, Error=%s", delivery, delErr) } return err } if delivery.Messages[0].ReceiptHandle != nil { sig.SQSReceiptHandle = *delivery.Messages[0].ReceiptHandle } // If the task is not registered return an error // and leave the message in the queue if !b.IsTaskRegistered(sig.Name) { if sig.IgnoreWhenTaskNotRegistered { b.deleteOne(delivery) } return fmt.Errorf("task %s is not registered", sig.Name) } err := taskProcessor.Process(sig) if err != nil { // stop task deletion in case we want to send messages to dlq in sqs if err == errs.ErrStopTaskDeletion { return nil } return err } // Delete message after successfully consuming and processing the message if err = b.deleteOne(delivery); err != nil { log.ERROR.Printf("error when deleting the delivery. delivery is %v, Error=%s", delivery, err) } return err } // deleteOne is a method delete a delivery from AWS SQS func (b *Broker) deleteOne(delivery *awssqs.ReceiveMessageOutput) error { qURL := b.defaultQueueURL() _, err := b.service.DeleteMessage(&awssqs.DeleteMessageInput{ QueueUrl: qURL, ReceiptHandle: delivery.Messages[0].ReceiptHandle, }) if err != nil { return err } return nil } // defaultQueueURL is a method returns the default queue url func (b *Broker) defaultQueueURL() *string { if b.queueUrl != nil { return b.queueUrl } else { return aws.String(b.GetConfig().Broker + "/" + b.GetConfig().DefaultQueue) } } // receiveMessage is a method receives a message from specified queue url func (b *Broker) receiveMessage(qURL *string) (*awssqs.ReceiveMessageOutput, error) { var waitTimeSeconds int var visibilityTimeout *int if b.GetConfig().SQS != nil { waitTimeSeconds = b.GetConfig().SQS.WaitTimeSeconds visibilityTimeout = b.GetConfig().SQS.VisibilityTimeout } else { waitTimeSeconds = 0 } input := &awssqs.ReceiveMessageInput{ AttributeNames: []*string{ aws.String(awssqs.MessageSystemAttributeNameSentTimestamp), }, MessageAttributeNames: []*string{ aws.String(awssqs.QueueAttributeNameAll), }, QueueUrl: qURL, MaxNumberOfMessages: aws.Int64(1), WaitTimeSeconds: aws.Int64(int64(waitTimeSeconds)), } if visibilityTimeout != nil { input.VisibilityTimeout = aws.Int64(int64(*visibilityTimeout)) } result, err := b.service.ReceiveMessage(input) if err != nil { return nil, err } return result, err } // initializePool is a method which initializes concurrency pool func (b *Broker) initializePool(pool chan struct{}, concurrency int) { for i := 0; i < concurrency; i++ { pool <- struct{}{} } } // consumeDeliveries is a method consuming deliveries from deliveries channel func (b *Broker) consumeDeliveries(deliveries <-chan *awssqs.ReceiveMessageOutput, concurrency int, taskProcessor iface.TaskProcessor, pool chan struct{}, errorsChan chan error) (bool, error) { select { case err := <-errorsChan: return false, err case d := <-deliveries: b.processingWG.Add(1) // Consume the task inside a goroutine so multiple tasks // can be processed concurrently go func() { if err := b.consumeOne(d, taskProcessor); err != nil { errorsChan <- err } b.processingWG.Done() if concurrency > 0 { // give worker back to pool pool <- struct{}{} } }() case <-b.GetStopChan(): return false, nil } return true, nil } // continueReceivingMessages is a method returns a continue signal func (b *Broker) continueReceivingMessages(qURL *string, deliveries chan *awssqs.ReceiveMessageOutput) (bool, error) { select { // A way to stop this goroutine from b.StopConsuming case <-b.stopReceivingChan: return false, nil default: output, err := b.receiveMessage(qURL) if err != nil { return true, err } if len(output.Messages) == 0 { return true, nil } go func() { deliveries <- output }() } return true, nil } // stopReceiving is a method sending a signal to stopReceivingChan func (b *Broker) stopReceiving() { // Stop the receiving goroutine b.stopReceivingChan <- 1 } // getQueueURL is a method returns that returns queueURL first by checking if custom queue was set and usign it // otherwise using default queueName from config func (b *Broker) getQueueURL(taskProcessor iface.TaskProcessor) *string { queueName := b.GetConfig().DefaultQueue if taskProcessor.CustomQueue() != "" { queueName = taskProcessor.CustomQueue() } return aws.String(b.GetConfig().Broker + "/" + queueName) } ================================================ FILE: v1/brokers/sqs/sqs_export_test.go ================================================ package sqs import ( "encoding/json" "errors" "fmt" "os" "sync" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/sqs/sqsiface" "github.com/RichardKnop/machinery/v1/brokers/iface" "github.com/RichardKnop/machinery/v1/common" "github.com/RichardKnop/machinery/v1/config" awssqs "github.com/aws/aws-sdk-go/service/sqs" ) var ( ReceiveMessageOutput *awssqs.ReceiveMessageOutput ) type FakeSQS struct { sqsiface.SQSAPI } func (f *FakeSQS) SendMessage(*awssqs.SendMessageInput) (*awssqs.SendMessageOutput, error) { output := awssqs.SendMessageOutput{ MD5OfMessageAttributes: aws.String("d25a6aea97eb8f585bfa92d314504a92"), MD5OfMessageBody: aws.String("bbdc5fdb8be7251f5c910905db994bab"), MessageId: aws.String("47f8b355-5115-4b45-b33a-439016400411"), } return &output, nil } func (f *FakeSQS) ReceiveMessage(*awssqs.ReceiveMessageInput) (*awssqs.ReceiveMessageOutput, error) { return ReceiveMessageOutput, nil } func (f *FakeSQS) DeleteMessage(*awssqs.DeleteMessageInput) (*awssqs.DeleteMessageOutput, error) { return &awssqs.DeleteMessageOutput{}, nil } type ErrorSQS struct { sqsiface.SQSAPI } func (e *ErrorSQS) SendMessage(*awssqs.SendMessageInput) (*awssqs.SendMessageOutput, error) { err := errors.New("this is an error") return nil, err } func (e *ErrorSQS) ReceiveMessage(*awssqs.ReceiveMessageInput) (*awssqs.ReceiveMessageOutput, error) { err := errors.New("this is an error") return nil, err } func (e *ErrorSQS) DeleteMessage(*awssqs.DeleteMessageInput) (*awssqs.DeleteMessageOutput, error) { err := errors.New("this is an error") return nil, err } func init() { // TODO: chang message body to signature example messageBody, _ := json.Marshal(map[string]int{"apple": 5, "lettuce": 7}) ReceiveMessageOutput = &awssqs.ReceiveMessageOutput{ Messages: []*awssqs.Message{ { Attributes: map[string]*string{ "SentTimestamp": aws.String("1512962021537"), }, Body: aws.String(string(messageBody)), MD5OfBody: aws.String("bbdc5fdb8be7251f5c910905db994bab"), MD5OfMessageAttributes: aws.String("d25a6aea97eb8f585bfa92d314504a92"), MessageAttributes: map[string]*awssqs.MessageAttributeValue{ "Title": { DataType: aws.String("String"), StringValue: aws.String("The Whistler"), }, "Author": { DataType: aws.String("String"), StringValue: aws.String("John Grisham"), }, "WeeksOn": { DataType: aws.String("Number"), StringValue: aws.String("6"), }, }, MessageId: aws.String("47f8b355-5115-4b45-b33a-439016400411"), ReceiptHandle: aws.String("AQEBGhTR/nhq+pDPAunCDgLpwQuCq0JkD2dtv7pAcPF5DA/XaoPAjHfgn/PZ5DeG3YiQdTjCUj+rvFq5b79DTq+hK6r1Niuds02l+jdIk3u2JiL01Dsd203pW1lLUNryd74QAcn462eXzv7/hVDagXTn+KtOzox3X0vmPkCSQkWXWxtc23oa5+5Q7HWDmRm743L0zza1579rQ2R2B0TrdlTMpNsdjQlDmybNu+aDq8bazD/Wew539tIvUyYADuhVyKyS1L2QQuyXll73/DixulPNmvGPRHNoB1GIo+Ex929OHFchXoKonoFJnurX4VNNl1p/Byp2IYBi6nkTRzeJUFCrFq0WMAHKLwuxciezJSlLD7g3bbU8kgEer8+jTz1DBriUlDGsARr0s7mnlsd02cb46K/j+u1oPfA69vIVc0FaRtA="), }, }, } } func NewTestConfig() *config.Config { redisURL := os.Getenv("REDIS_URL") if redisURL == "" { redisURL = "eager" } brokerURL := "https://sqs.foo.amazonaws.com.cn" return &config.Config{ Broker: brokerURL, DefaultQueue: "test_queue", ResultBackend: fmt.Sprintf("redis://%v", redisURL), Lock: fmt.Sprintf("redis://%v", redisURL), } } func NewTestBroker() *Broker { cnf := NewTestConfig() sess := session.Must(session.NewSessionWithOptions(session.Options{ SharedConfigState: session.SharedConfigEnable, })) svc := new(FakeSQS) return &Broker{ Broker: common.NewBroker(cnf), sess: sess, service: svc, processingWG: sync.WaitGroup{}, receivingWG: sync.WaitGroup{}, stopReceivingChan: make(chan int), } } func NewTestErrorBroker() *Broker { cnf := NewTestConfig() sess := session.Must(session.NewSessionWithOptions(session.Options{ SharedConfigState: session.SharedConfigEnable, })) errSvc := new(ErrorSQS) return &Broker{ Broker: common.NewBroker(cnf), sess: sess, service: errSvc, processingWG: sync.WaitGroup{}, receivingWG: sync.WaitGroup{}, stopReceivingChan: make(chan int), } } func (b *Broker) ConsumeForTest(deliveries <-chan *awssqs.ReceiveMessageOutput, concurrency int, taskProcessor iface.TaskProcessor, pool chan struct{}) error { return b.consume(deliveries, concurrency, taskProcessor, pool) } func (b *Broker) ConsumeOneForTest(delivery *awssqs.ReceiveMessageOutput, taskProcessor iface.TaskProcessor) error { return b.consumeOne(delivery, taskProcessor) } func (b *Broker) DeleteOneForTest(delivery *awssqs.ReceiveMessageOutput) error { return b.deleteOne(delivery) } func (b *Broker) DefaultQueueURLForTest() *string { return b.defaultQueueURL() } func (b *Broker) ReceiveMessageForTest(qURL *string) (*awssqs.ReceiveMessageOutput, error) { return b.receiveMessage(qURL) } func (b *Broker) InitializePoolForTest(pool chan struct{}, concurrency int) { b.initializePool(pool, concurrency) } func (b *Broker) ConsumeDeliveriesForTest(deliveries <-chan *awssqs.ReceiveMessageOutput, concurrency int, taskProcessor iface.TaskProcessor, pool chan struct{}, errorsChan chan error) (bool, error) { return b.consumeDeliveries(deliveries, concurrency, taskProcessor, pool, errorsChan) } func (b *Broker) ContinueReceivingMessagesForTest(qURL *string, deliveries chan *awssqs.ReceiveMessageOutput) (bool, error) { return b.continueReceivingMessages(qURL, deliveries) } func (b *Broker) StopReceivingForTest() { b.stopReceiving() } func (b *Broker) GetStopReceivingChanForTest() chan int { return b.stopReceivingChan } func (b *Broker) StartConsumingForTest(consumerTag string, concurrency int, taskProcessor iface.TaskProcessor) { b.Broker.StartConsuming(consumerTag, concurrency, taskProcessor) } func (b *Broker) GetRetryFuncForTest() func(chan int) { return b.GetRetryFunc() } func (b *Broker) GetStopChanForTest() chan int { return b.GetStopChan() } func (b *Broker) GetRetryStopChanForTest() chan int { return b.GetRetryStopChan() } func (b *Broker) GetQueueURLForTest(taskProcessor iface.TaskProcessor) *string { return b.getQueueURL(taskProcessor) } func (b *Broker) GetCustomQueueURL(customQueue string) *string { return aws.String(b.GetConfig().Broker + "/" + customQueue) } ================================================ FILE: v1/brokers/sqs/sqs_test.go ================================================ package sqs_test import ( "context" "errors" "sync" "testing" "time" "github.com/aws/aws-sdk-go/aws" "github.com/stretchr/testify/assert" "github.com/RichardKnop/machinery/v1" "github.com/RichardKnop/machinery/v1/brokers/sqs" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/retry" awssqs "github.com/aws/aws-sdk-go/service/sqs" ) var ( cnf *config.Config receiveMessageOutput *awssqs.ReceiveMessageOutput ) func init() { cnf = sqs.NewTestConfig() receiveMessageOutput = sqs.ReceiveMessageOutput } func TestNewAWSSQSBroker(t *testing.T) { t.Parallel() broker := sqs.NewTestBroker() assert.IsType(t, broker, sqs.New(cnf)) } func TestPrivateFunc_continueReceivingMessages(t *testing.T) { broker := sqs.NewTestBroker() errorBroker := sqs.NewTestErrorBroker() qURL := broker.DefaultQueueURLForTest() deliveries := make(chan *awssqs.ReceiveMessageOutput) firstStep := make(chan int) nextStep := make(chan int) go func() { stopReceivingChan := broker.GetStopReceivingChanForTest() firstStep <- 1 stopReceivingChan <- 1 }() var ( whetherContinue bool err error ) <-firstStep // Test the case that a signal was received from stopReceivingChan go func() { whetherContinue, err = broker.ContinueReceivingMessagesForTest(qURL, deliveries) nextStep <- 1 }() <-nextStep assert.False(t, whetherContinue) assert.Nil(t, err) // Test the default condition whetherContinue, err = broker.ContinueReceivingMessagesForTest(qURL, deliveries) assert.True(t, whetherContinue) assert.Nil(t, err) // Test the error whetherContinue, err = errorBroker.ContinueReceivingMessagesForTest(qURL, deliveries) assert.True(t, whetherContinue) assert.NotNil(t, err) // Test when there is no message outputCopy := *receiveMessageOutput receiveMessageOutput.Messages = []*awssqs.Message{} whetherContinue, err = broker.ContinueReceivingMessagesForTest(qURL, deliveries) assert.True(t, whetherContinue) assert.Nil(t, err) // recover original value *receiveMessageOutput = outputCopy } func TestPrivateFunc_consume(t *testing.T) { server1, err := machinery.NewServer(cnf) if err != nil { t.Fatal(err) } pool := make(chan struct{}) wk := server1.NewWorker("sms_worker", 0) deliveries := make(chan *awssqs.ReceiveMessageOutput) outputCopy := *receiveMessageOutput outputCopy.Messages = []*awssqs.Message{} go func() { deliveries <- &outputCopy }() broker := sqs.NewTestBroker() // an infinite loop will be executed only when there is no error err = broker.ConsumeForTest(deliveries, 0, wk, pool) assert.NotNil(t, err) } func TestPrivateFunc_consumeOne(t *testing.T) { server1, err := machinery.NewServer(cnf) if err != nil { t.Fatal(err) } wk := server1.NewWorker("sms_worker", 0) broker := sqs.NewTestBroker() err = broker.ConsumeOneForTest(receiveMessageOutput, wk) assert.NotNil(t, err) outputCopy := *receiveMessageOutput outputCopy.Messages = []*awssqs.Message{} err = broker.ConsumeOneForTest(&outputCopy, wk) assert.NotNil(t, err) outputCopy.Messages = []*awssqs.Message{ { Body: aws.String("foo message"), }, } err = broker.ConsumeOneForTest(&outputCopy, wk) assert.NotNil(t, err) } func TestPrivateFunc_initializePool(t *testing.T) { broker := sqs.NewTestBroker() concurrency := 9 pool := make(chan struct{}, concurrency) broker.InitializePoolForTest(pool, concurrency) assert.Len(t, pool, concurrency) } func TestPrivateFunc_startConsuming(t *testing.T) { server1, err := machinery.NewServer(cnf) if err != nil { t.Fatal(err) } wk := server1.NewWorker("sms_worker", 0) broker := sqs.NewTestBroker() retryFunc := broker.GetRetryFuncForTest() stopChan := broker.GetStopChanForTest() retryStopChan := broker.GetRetryStopChanForTest() assert.Nil(t, retryFunc) broker.StartConsumingForTest("fooTag", 1, wk) assert.IsType(t, retryFunc, retry.Closure()) assert.Equal(t, len(stopChan), 0) assert.Equal(t, len(retryStopChan), 0) } func TestPrivateFuncDefaultQueueURL(t *testing.T) { broker := sqs.NewTestBroker() qURL := broker.DefaultQueueURLForTest() assert.EqualValues(t, *qURL, "https://sqs.foo.amazonaws.com.cn/test_queue") } func TestPrivateFunc_stopReceiving(t *testing.T) { broker := sqs.NewTestBroker() go broker.StopReceivingForTest() stopReceivingChan := broker.GetStopReceivingChanForTest() assert.NotNil(t, <-stopReceivingChan) } func TestPrivateFunc_receiveMessage(t *testing.T) { broker := sqs.NewTestBroker() qURL := broker.DefaultQueueURLForTest() output, err := broker.ReceiveMessageForTest(qURL) assert.Nil(t, err) assert.Equal(t, receiveMessageOutput, output) } func TestPrivateFunc_consumeDeliveries(t *testing.T) { concurrency := 0 pool := make(chan struct{}, concurrency) errorsChan := make(chan error) deliveries := make(chan *awssqs.ReceiveMessageOutput) server1, err := machinery.NewServer(cnf) if err != nil { t.Fatal(err) } wk := server1.NewWorker("sms_worker", 0) broker := sqs.NewTestBroker() go func() { deliveries <- receiveMessageOutput }() whetherContinue, err := broker.ConsumeDeliveriesForTest(deliveries, concurrency, wk, pool, errorsChan) assert.True(t, whetherContinue) assert.Nil(t, err) go func() { errorsChan <- errors.New("foo error") }() whetherContinue, err = broker.ConsumeDeliveriesForTest(deliveries, concurrency, wk, pool, errorsChan) assert.False(t, whetherContinue) assert.NotNil(t, err) go func() { broker.GetStopChanForTest() <- 1 }() whetherContinue, err = broker.ConsumeDeliveriesForTest(deliveries, concurrency, wk, pool, errorsChan) assert.False(t, whetherContinue) assert.Nil(t, err) outputCopy := *receiveMessageOutput outputCopy.Messages = []*awssqs.Message{} go func() { deliveries <- &outputCopy }() whetherContinue, err = broker.ConsumeDeliveriesForTest(deliveries, concurrency, wk, pool, errorsChan) e := <-errorsChan assert.True(t, whetherContinue) assert.NotNil(t, e) assert.Nil(t, err) // using a wait group and a channel to fix the racing problem var wg sync.WaitGroup wg.Add(1) nextStep := make(chan bool, 1) go func() { defer wg.Done() // nextStep <- true runs after defer wg.Done(), to make sure the next go routine runs after this go routine nextStep <- true deliveries <- receiveMessageOutput }() if <-nextStep { // <-pool will block the routine in the following steps, so pool <- struct{}{} will be executed for sure go func() { wg.Wait(); pool <- struct{}{} }() } whetherContinue, err = broker.ConsumeDeliveriesForTest(deliveries, concurrency, wk, pool, errorsChan) // the pool shouldn't be consumed p := <-pool assert.True(t, whetherContinue) assert.NotNil(t, p) assert.Nil(t, err) } func TestPrivateFunc_deleteOne(t *testing.T) { broker := sqs.NewTestBroker() errorBroker := sqs.NewTestErrorBroker() err := broker.DeleteOneForTest(receiveMessageOutput) assert.Nil(t, err) err = errorBroker.DeleteOneForTest(receiveMessageOutput) assert.NotNil(t, err) } func Test_CustomQueueName(t *testing.T) { server1, err := machinery.NewServer(cnf) if err != nil { t.Fatal(err) } broker := sqs.NewTestBroker() wk := server1.NewWorker("test-worker", 0) qURL := broker.GetQueueURLForTest(wk) assert.Equal(t, qURL, broker.DefaultQueueURLForTest(), "") wk2 := server1.NewCustomQueueWorker("test-worker", 0, "my-custom-queue") qURL2 := broker.GetQueueURLForTest(wk2) assert.Equal(t, qURL2, broker.GetCustomQueueURL("my-custom-queue"), "") } func TestPrivateFunc_consumeWithConcurrency(t *testing.T) { msg := `{ "UUID": "uuid-dummy-task", "Name": "test-task", "RoutingKey": "dummy-routing" } ` testResp := "47f8b355-5115-4b45-b33a-439016400411" output := make(chan string) // The output channel cnf.ResultBackend = "eager" server1, err := machinery.NewServer(cnf) if err != nil { t.Fatal(err) } err = server1.RegisterTask("test-task", func(ctx context.Context) error { output <- testResp return nil }) broker := sqs.NewTestBroker() broker.SetRegisteredTaskNames([]string{"test-task"}) assert.NoError(t, err) pool := make(chan struct{}, 1) pool <- struct{}{} wk := server1.NewWorker("sms_worker", 1) deliveries := make(chan *awssqs.ReceiveMessageOutput) outputCopy := *receiveMessageOutput outputCopy.Messages = []*awssqs.Message{ { MessageId: aws.String("test-sqs-msg1"), Body: aws.String(msg), }, } go func() { deliveries <- &outputCopy }() go func() { err = broker.ConsumeForTest(deliveries, 1, wk, pool) }() select { case resp := <-output: assert.Equal(t, testResp, resp) case <-time.After(10 * time.Second): // call timed out t.Fatal("task not processed in 10 seconds") } } ================================================ FILE: v1/common/amqp.go ================================================ package common import ( "crypto/tls" "fmt" "strings" amqp "github.com/rabbitmq/amqp091-go" ) // AMQPConnector ... type AMQPConnector struct{} // Connect opens a connection to RabbitMQ, declares an exchange, opens a channel, // declares and binds the queue and enables publish notifications func (ac *AMQPConnector) Connect(urls string, urlSeparator string, tlsConfig *tls.Config, exchange, exchangeType, queueName string, queueDurable, queueDelete bool, queueBindingKey string, exchangeDeclareArgs, queueDeclareArgs, queueBindingArgs amqp.Table) (*amqp.Connection, *amqp.Channel, amqp.Queue, <-chan amqp.Confirmation, <-chan *amqp.Error, error) { urlsList := []string{urls} if urlSeparator != "" { urlsList = strings.Split(urls, urlSeparator) } var conn *amqp.Connection var channel *amqp.Channel var err error for _, url := range urlsList { // Connect to server conn, channel, err = ac.Open(url, tlsConfig) if err != nil { continue } else { break } } if err != nil { return nil, nil, amqp.Queue{}, nil, nil, err } if exchange != "" { // Declare an exchange if err = channel.ExchangeDeclare( exchange, // name of the exchange exchangeType, // type true, // durable false, // delete when complete false, // internal false, // noWait exchangeDeclareArgs, // arguments ); err != nil { return conn, channel, amqp.Queue{}, nil, nil, fmt.Errorf("Exchange declare error: %s", err) } } var queue amqp.Queue if queueName != "" { // Declare a queue queue, err = channel.QueueDeclare( queueName, // name queueDurable, // durable queueDelete, // delete when unused false, // exclusive false, // no-wait queueDeclareArgs, // arguments ) if err != nil { return conn, channel, amqp.Queue{}, nil, nil, fmt.Errorf("Queue declare error: %s", err) } // Bind the queue if err = channel.QueueBind( queue.Name, // name of the queue queueBindingKey, // binding key exchange, // source exchange false, // noWait queueBindingArgs, // arguments ); err != nil { return conn, channel, queue, nil, nil, fmt.Errorf("Queue bind error: %s", err) } } // Enable publish confirmations if err = channel.Confirm(false); err != nil { return conn, channel, queue, nil, nil, fmt.Errorf("Channel could not be put into confirm mode: %s", err) } return conn, channel, queue, channel.NotifyPublish(make(chan amqp.Confirmation, 1)), conn.NotifyClose(make(chan *amqp.Error, 1)), nil } // DeleteQueue deletes a queue by name func (ac *AMQPConnector) DeleteQueue(channel *amqp.Channel, queueName string) error { // First return value is number of messages removed _, err := channel.QueueDelete( queueName, // name false, // ifUnused false, // ifEmpty false, // noWait ) return err } // InspectQueue provides information about a specific queue func (*AMQPConnector) InspectQueue(channel *amqp.Channel, queueName string) (*amqp.Queue, error) { queueState, err := channel.QueueInspect(queueName) if err != nil { return nil, fmt.Errorf("Queue inspect error: %s", err) } return &queueState, nil } // Open new RabbitMQ connection func (ac *AMQPConnector) Open(url string, tlsConfig *tls.Config) (*amqp.Connection, *amqp.Channel, error) { // Connect // From amqp docs: DialTLS will use the provided tls.Config when it encounters an amqps:// scheme // and will dial a plain connection when it encounters an amqp:// scheme. conn, err := amqp.DialTLS(url, tlsConfig) if err != nil { return nil, nil, fmt.Errorf("Dial error: %s", err) } // Open a channel channel, err := conn.Channel() if err != nil { return nil, nil, fmt.Errorf("Open channel error: %s", err) } return conn, channel, nil } // Close connection func (ac *AMQPConnector) Close(channel *amqp.Channel, conn *amqp.Connection) error { if channel != nil { if err := channel.Close(); err != nil { return fmt.Errorf("Close channel error: %s", err) } } if conn != nil { if err := conn.Close(); err != nil { return fmt.Errorf("Close connection error: %s", err) } } return nil } ================================================ FILE: v1/common/backend.go ================================================ package common import ( "github.com/RichardKnop/machinery/v1/config" ) // Backend represents a base backend structure type Backend struct { cnf *config.Config } // NewBackend creates new Backend instance func NewBackend(cnf *config.Config) Backend { return Backend{cnf: cnf} } // GetConfig returns config func (b *Backend) GetConfig() *config.Config { return b.cnf } // IsAMQP ... func (b *Backend) IsAMQP() bool { return false } ================================================ FILE: v1/common/broker.go ================================================ package common import ( "errors" "sync" "github.com/RichardKnop/machinery/v1/brokers/iface" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/log" "github.com/RichardKnop/machinery/v1/retry" "github.com/RichardKnop/machinery/v1/tasks" ) type registeredTaskNames struct { sync.RWMutex items []string } // Broker represents a base broker structure type Broker struct { cnf *config.Config registeredTaskNames registeredTaskNames retry bool retryFunc func(chan int) retryStopChan chan int stopChan chan int } // NewBroker creates new Broker instance func NewBroker(cnf *config.Config) Broker { return Broker{ cnf: cnf, retry: true, stopChan: make(chan int), retryStopChan: make(chan int), } } // GetConfig returns config func (b *Broker) GetConfig() *config.Config { return b.cnf } // GetRetry ... func (b *Broker) GetRetry() bool { return b.retry } // GetRetryFunc ... func (b *Broker) GetRetryFunc() func(chan int) { return b.retryFunc } // GetRetryStopChan ... func (b *Broker) GetRetryStopChan() chan int { return b.retryStopChan } // GetStopChan ... func (b *Broker) GetStopChan() chan int { return b.stopChan } // Publish places a new message on the default queue func (b *Broker) Publish(signature *tasks.Signature) error { return errors.New("Not implemented") } // SetRegisteredTaskNames sets registered task names func (b *Broker) SetRegisteredTaskNames(names []string) { b.registeredTaskNames.Lock() defer b.registeredTaskNames.Unlock() b.registeredTaskNames.items = names } // IsTaskRegistered returns true if the task is registered with this broker func (b *Broker) IsTaskRegistered(name string) bool { b.registeredTaskNames.RLock() defer b.registeredTaskNames.RUnlock() for _, registeredTaskName := range b.registeredTaskNames.items { if registeredTaskName == name { return true } } return false } // GetPendingTasks returns a slice of task.Signatures waiting in the queue func (b *Broker) GetPendingTasks(queue string) ([]*tasks.Signature, error) { return nil, errors.New("Not implemented") } // GetDelayedTasks returns a slice of task.Signatures that are scheduled, but not yet in the queue func (b *Broker) GetDelayedTasks() ([]*tasks.Signature, error) { return nil, errors.New("Not implemented") } // StartConsuming is a common part of StartConsuming method func (b *Broker) StartConsuming(consumerTag string, concurrency int, taskProcessor iface.TaskProcessor) { if b.retryFunc == nil { b.retryFunc = retry.Closure() } } // StopConsuming is a common part of StopConsuming func (b *Broker) StopConsuming() { // Do not retry from now on b.retry = false // Stop the retry closure earlier select { case b.retryStopChan <- 1: log.WARNING.Print("Stopping retry closure.") default: } // Notifying the stop channel stops consuming of messages close(b.stopChan) log.WARNING.Print("Stop channel") } // GetRegisteredTaskNames returns registered tasks names func (b *Broker) GetRegisteredTaskNames() []string { b.registeredTaskNames.RLock() defer b.registeredTaskNames.RUnlock() items := b.registeredTaskNames.items return items } // AdjustRoutingKey makes sure the routing key is correct. // If the routing key is an empty string: // a) set it to binding key for direct exchange type // b) set it to default queue name func (b *Broker) AdjustRoutingKey(s *tasks.Signature) { if s.RoutingKey != "" { return } s.RoutingKey = b.GetConfig().DefaultQueue } ================================================ FILE: v1/common/broker_test.go ================================================ package common_test import ( "testing" "github.com/RichardKnop/machinery/v1" "github.com/RichardKnop/machinery/v1/common" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/tasks" "github.com/stretchr/testify/assert" ) func TestIsTaskRegistered(t *testing.T) { t.Parallel() broker := common.NewBroker(new(config.Config)) broker.SetRegisteredTaskNames([]string{"foo", "bar"}) assert.True(t, broker.IsTaskRegistered("foo")) assert.False(t, broker.IsTaskRegistered("bogus")) } func TestAdjustRoutingKey(t *testing.T) { t.Parallel() var ( s *tasks.Signature broker common.Broker ) t.Run("with routing key", func(t *testing.T) { s = &tasks.Signature{RoutingKey: "routing_key"} broker = common.NewBroker(&config.Config{ DefaultQueue: "queue", }) broker.AdjustRoutingKey(s) assert.Equal(t, "routing_key", s.RoutingKey) }) t.Run("without routing key", func(t *testing.T) { s = new(tasks.Signature) broker = common.NewBroker(&config.Config{ DefaultQueue: "queue", }) broker.AdjustRoutingKey(s) assert.Equal(t, "queue", s.RoutingKey) }) } func TestGetRegisteredTaskNames(t *testing.T) { t.Parallel() broker := common.NewBroker(new(config.Config)) fooTasks := []string{"foo", "bar", "baz"} broker.SetRegisteredTaskNames(fooTasks) assert.Equal(t, fooTasks, broker.GetRegisteredTaskNames()) } func TestStopConsuming(t *testing.T) { t.Parallel() t.Run("stop consuming", func(t *testing.T) { broker := common.NewBroker(&config.Config{ DefaultQueue: "queue", }) broker.StartConsuming("", 1, &machinery.Worker{}) broker.StopConsuming() select { case <-broker.GetStopChan(): default: assert.Fail(t, "still blocking") } }) } ================================================ FILE: v1/common/redis.go ================================================ package common import ( "crypto/tls" "time" "github.com/gomodule/redigo/redis" "github.com/RichardKnop/machinery/v1/config" ) var ( defaultConfig = &config.RedisConfig{ MaxIdle: 10, MaxActive: 100, IdleTimeout: 300, Wait: true, ReadTimeout: 15, WriteTimeout: 15, ConnectTimeout: 15, NormalTasksPollPeriod: 1000, DelayedTasksPollPeriod: 20, } ) // RedisConnector ... type RedisConnector struct{} // NewPool returns a new pool of Redis connections func (rc *RedisConnector) NewPool(socketPath, host, username, password string, db int, cnf *config.RedisConfig, tlsConfig *tls.Config) *redis.Pool { if cnf == nil { cnf = defaultConfig } return &redis.Pool{ MaxIdle: cnf.MaxIdle, IdleTimeout: time.Duration(cnf.IdleTimeout) * time.Second, MaxActive: cnf.MaxActive, Wait: cnf.Wait, Dial: func() (redis.Conn, error) { c, err := rc.open(socketPath, host, username, password, db, cnf, tlsConfig) if err != nil { return nil, err } if db != 0 { _, err = c.Do("SELECT", db) if err != nil { return nil, err } } return c, err }, // PINGs connections that have been idle more than 10 seconds TestOnBorrow: func(c redis.Conn, t time.Time) error { if time.Since(t) < time.Duration(10*time.Second) { return nil } _, err := c.Do("PING") return err }, } } // Open a new Redis connection func (rc *RedisConnector) open(socketPath, host, username string, password string, db int, cnf *config.RedisConfig, tlsConfig *tls.Config) (redis.Conn, error) { var opts = []redis.DialOption{ redis.DialDatabase(db), redis.DialReadTimeout(time.Duration(cnf.ReadTimeout) * time.Second), redis.DialWriteTimeout(time.Duration(cnf.WriteTimeout) * time.Second), redis.DialConnectTimeout(time.Duration(cnf.ConnectTimeout) * time.Second), redis.DialClientName(cnf.ClientName), } if tlsConfig != nil { opts = append(opts, redis.DialTLSConfig(tlsConfig), redis.DialUseTLS(true)) } if username != "" { opts = append(opts, redis.DialUsername(username)) } if password != "" { opts = append(opts, redis.DialPassword(password)) } if socketPath != "" { return redis.Dial("unix", socketPath, opts...) } return redis.Dial("tcp", host, opts...) } ================================================ FILE: v1/config/config.go ================================================ package config import ( "crypto/tls" "fmt" "strings" "time" "cloud.google.com/go/pubsub" "github.com/aws/aws-sdk-go/service/dynamodb" "github.com/aws/aws-sdk-go/service/sqs" "go.mongodb.org/mongo-driver/mongo" ) const ( // DefaultResultsExpireIn is a default time used to expire task states and group metadata from the backend DefaultResultsExpireIn = 3600 ) var ( // Start with sensible default values defaultCnf = &Config{ Broker: "amqp://guest:guest@localhost:5672/", DefaultQueue: "machinery_tasks", ResultBackend: "amqp://guest:guest@localhost:5672/", ResultsExpireIn: DefaultResultsExpireIn, AMQP: &AMQPConfig{ Exchange: "machinery_exchange", ExchangeType: "direct", BindingKey: "machinery_task", PrefetchCount: 3, }, DynamoDB: &DynamoDBConfig{ TaskStatesTable: "task_states", GroupMetasTable: "group_metas", }, Redis: &RedisConfig{ MaxIdle: 3, IdleTimeout: 240, ReadTimeout: 15, WriteTimeout: 15, ConnectTimeout: 15, NormalTasksPollPeriod: 1000, DelayedTasksPollPeriod: 500, }, GCPPubSub: &GCPPubSubConfig{ Client: nil, }, } reloadDelay = time.Second * 10 ) // Config holds all configuration for our program type Config struct { Broker string `yaml:"broker" envconfig:"BROKER"` Lock string `yaml:"lock" envconfig:"LOCK"` MultipleBrokerSeparator string `yaml:"multiple_broker_separator" envconfig:"MULTIPLE_BROKEN_SEPARATOR"` DefaultQueue string `yaml:"default_queue" envconfig:"DEFAULT_QUEUE"` ResultBackend string `yaml:"result_backend" envconfig:"RESULT_BACKEND"` ResultsExpireIn int `yaml:"results_expire_in" envconfig:"RESULTS_EXPIRE_IN"` AMQP *AMQPConfig `yaml:"amqp"` SQS *SQSConfig `yaml:"sqs"` Redis *RedisConfig `yaml:"redis"` GCPPubSub *GCPPubSubConfig `yaml:"-" ignored:"true"` MongoDB *MongoDBConfig `yaml:"-" ignored:"true"` TLSConfig *tls.Config // NoUnixSignals - when set disables signal handling in machinery NoUnixSignals bool `yaml:"no_unix_signals" envconfig:"NO_UNIX_SIGNALS"` DynamoDB *DynamoDBConfig `yaml:"dynamodb"` } // QueueBindingArgs arguments which are used when binding to the exchange type QueueBindingArgs map[string]interface{} // QueueDeclareArgs arguments which are used when declaring a queue type QueueDeclareArgs map[string]interface{} // AMQPConfig wraps RabbitMQ related configuration type AMQPConfig struct { Exchange string `yaml:"exchange" envconfig:"AMQP_EXCHANGE"` ExchangeType string `yaml:"exchange_type" envconfig:"AMQP_EXCHANGE_TYPE"` QueueDeclareArgs QueueDeclareArgs `yaml:"queue_declare_args" envconfig:"AMQP_QUEUE_DECLARE_ARGS"` QueueBindingArgs QueueBindingArgs `yaml:"queue_binding_args" envconfig:"AMQP_QUEUE_BINDING_ARGS"` BindingKey string `yaml:"binding_key" envconfig:"AMQP_BINDING_KEY"` PrefetchCount int `yaml:"prefetch_count" envconfig:"AMQP_PREFETCH_COUNT"` AutoDelete bool `yaml:"auto_delete" envconfig:"AMQP_AUTO_DELETE"` DelayedQueue string `yaml:"delayed_queue" envconfig:"AMQP_DELAYED_QUEUE"` } // DynamoDBConfig wraps DynamoDB related configuration type DynamoDBConfig struct { Client *dynamodb.DynamoDB TaskStatesTable string `yaml:"task_states_table" envconfig:"TASK_STATES_TABLE"` GroupMetasTable string `yaml:"group_metas_table" envconfig:"GROUP_METAS_TABLE"` } // SQSConfig wraps SQS related configuration type SQSConfig struct { Client *sqs.SQS WaitTimeSeconds int `yaml:"receive_wait_time_seconds" envconfig:"SQS_WAIT_TIME_SECONDS"` // https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html // visibility timeout should default to nil to use the overall visibility timeout for the queue VisibilityTimeout *int `yaml:"receive_visibility_timeout" envconfig:"SQS_VISIBILITY_TIMEOUT"` } // RedisConfig ... type RedisConfig struct { // Maximum number of idle connections in the pool. // Default: 10 MaxIdle int `yaml:"max_idle" envconfig:"REDIS_MAX_IDLE"` // Maximum number of connections allocated by the pool at a given time. // When zero, there is no limit on the number of connections in the pool. // Default: 100 MaxActive int `yaml:"max_active" envconfig:"REDIS_MAX_ACTIVE"` // Close connections after remaining idle for this duration in seconds. If the value // is zero, then idle connections are not closed. Applications should set // the timeout to a value less than the server's timeout. // Default: 300 IdleTimeout int `yaml:"max_idle_timeout" envconfig:"REDIS_IDLE_TIMEOUT"` // If Wait is true and the pool is at the MaxActive limit, then Get() waits // for a connection to be returned to the pool before returning. // Default: true Wait bool `yaml:"wait" envconfig:"REDIS_WAIT"` // ReadTimeout specifies the timeout in seconds for reading a single command reply. // Default: 15 ReadTimeout int `yaml:"read_timeout" envconfig:"REDIS_READ_TIMEOUT"` // WriteTimeout specifies the timeout in seconds for writing a single command. // Default: 15 WriteTimeout int `yaml:"write_timeout" envconfig:"REDIS_WRITE_TIMEOUT"` // ConnectTimeout specifies the timeout in seconds for connecting to the Redis server when // no DialNetDial option is specified. // Default: 15 ConnectTimeout int `yaml:"connect_timeout" envconfig:"REDIS_CONNECT_TIMEOUT"` // NormalTasksPollPeriod specifies the period in milliseconds when polling redis for normal tasks // Default: 1000 NormalTasksPollPeriod int `yaml:"normal_tasks_poll_period" envconfig:"REDIS_NORMAL_TASKS_POLL_PERIOD"` // DelayedTasksPollPeriod specifies the period in milliseconds when polling redis for delayed tasks // Default: 20 DelayedTasksPollPeriod int `yaml:"delayed_tasks_poll_period" envconfig:"REDIS_DELAYED_TASKS_POLL_PERIOD"` DelayedTasksKey string `yaml:"delayed_tasks_key" envconfig:"REDIS_DELAYED_TASKS_KEY"` // ClientName specifies the redis client name to be set when connecting to the Redis server ClientName string `yaml:"client_name" envconfig:"REDIS_CLIENT_NAME"` // MasterName specifies a redis master name in order to configure a sentinel-backed redis FailoverClient MasterName string `yaml:"master_name" envconfig:"REDIS_MASTER_NAME"` // ClusterMode specifies machinery should use redis cluster client explicitly ClusterMode bool `yaml:"cluster_mode" envconfig:"REDIS_CLUSTER_MODE"` } // GCPPubSubConfig wraps GCP PubSub related configuration type GCPPubSubConfig struct { Client *pubsub.Client MaxExtension time.Duration } // MongoDBConfig ... type MongoDBConfig struct { Client *mongo.Client Database string } // Decode from yaml to map (any field whose type or pointer-to-type implements // envconfig.Decoder can control its own deserialization) func (args *QueueBindingArgs) Decode(value string) error { pairs := strings.Split(value, ",") mp := make(map[string]interface{}, len(pairs)) for _, pair := range pairs { kvpair := strings.Split(pair, ":") if len(kvpair) != 2 { return fmt.Errorf("invalid map item: %q", pair) } mp[kvpair[0]] = kvpair[1] } *args = QueueBindingArgs(mp) return nil } ================================================ FILE: v1/config/env.go ================================================ package config import ( "github.com/kelseyhightower/envconfig" "github.com/RichardKnop/machinery/v1/log" ) // NewFromEnvironment creates a config object from environment variables func NewFromEnvironment() (*Config, error) { cnf, err := fromEnvironment() if err != nil { return nil, err } log.INFO.Print("Successfully loaded config from the environment") return cnf, nil } func fromEnvironment() (*Config, error) { loadedCnf, cnf := new(Config), new(Config) *cnf = *defaultCnf if err := envconfig.Process("", cnf); err != nil { return nil, err } if err := envconfig.Process("", loadedCnf); err != nil { return nil, err } if loadedCnf.AMQP == nil { cnf.AMQP = nil } return cnf, nil } ================================================ FILE: v1/config/env_test.go ================================================ package config_test import ( "bufio" "os" "strings" "testing" "github.com/RichardKnop/machinery/v1/config" "github.com/stretchr/testify/assert" ) func TestNewFromEnvironment(t *testing.T) { t.Parallel() file, err := os.Open("test.env") if err != nil { t.Fatal(err) } reader := bufio.NewReader(file) scanner := bufio.NewScanner(reader) scanner.Split(bufio.ScanLines) for scanner.Scan() { parts := strings.Split(scanner.Text(), "=") if len(parts) != 2 { continue } os.Setenv(parts[0], parts[1]) } cnf, err := config.NewFromEnvironment() if err != nil { t.Fatal(err) } assert.Equal(t, "broker", cnf.Broker) assert.Equal(t, "default_queue", cnf.DefaultQueue) assert.Equal(t, "result_backend", cnf.ResultBackend) assert.Equal(t, 123456, cnf.ResultsExpireIn) assert.Equal(t, "exchange", cnf.AMQP.Exchange) assert.Equal(t, "exchange_type", cnf.AMQP.ExchangeType) assert.Equal(t, "binding_key", cnf.AMQP.BindingKey) assert.Equal(t, "any", cnf.AMQP.QueueBindingArgs["x-match"]) assert.Equal(t, "png", cnf.AMQP.QueueBindingArgs["image-type"]) assert.Equal(t, 123, cnf.AMQP.PrefetchCount) } ================================================ FILE: v1/config/file.go ================================================ package config import ( "fmt" "os" "time" "github.com/RichardKnop/machinery/v1/log" "gopkg.in/yaml.v2" ) // NewFromYaml creates a config object from YAML file func NewFromYaml(cnfPath string, keepReloading bool) (*Config, error) { cnf, err := fromFile(cnfPath) if err != nil { return nil, err } log.INFO.Printf("Successfully loaded config from file %s", cnfPath) if keepReloading { // Open a goroutine to watch remote changes forever go func() { for { // Delay after each request time.Sleep(reloadDelay) // Attempt to reload the config newCnf, newErr := fromFile(cnfPath) if newErr != nil { log.WARNING.Printf("Failed to reload config from file %s: %v", cnfPath, newErr) continue } *cnf = *newCnf } }() } return cnf, nil } // ReadFromFile reads data from a file func ReadFromFile(cnfPath string) ([]byte, error) { file, err := os.Open(cnfPath) // Config file not found if err != nil { return nil, fmt.Errorf("Open file error: %s", err) } defer file.Close() // Config file found, let's try to read it data := make([]byte, 1000) count, err := file.Read(data) if err != nil { return nil, fmt.Errorf("Read from file error: %s", err) } return data[:count], nil } func fromFile(cnfPath string) (*Config, error) { loadedCnf, cnf := new(Config), new(Config) *cnf = *defaultCnf data, err := ReadFromFile(cnfPath) if err != nil { return nil, err } if err := yaml.Unmarshal(data, cnf); err != nil { return nil, fmt.Errorf("Unmarshal YAML error: %s", err) } if err := yaml.Unmarshal(data, loadedCnf); err != nil { return nil, fmt.Errorf("Unmarshal YAML error: %s", err) } if loadedCnf.AMQP == nil { cnf.AMQP = nil } return cnf, nil } ================================================ FILE: v1/config/file_test.go ================================================ package config_test import ( "testing" "github.com/RichardKnop/machinery/v1/config" "github.com/stretchr/testify/assert" ) var configYAMLData = `--- broker: broker default_queue: default_queue result_backend: result_backend results_expire_in: 123456 amqp: binding_key: binding_key exchange: exchange exchange_type: exchange_type prefetch_count: 123 queue_declare_args: x-max-priority: 10 queue_binding_args: image-type: png x-match: any sqs: receive_wait_time_seconds: 123 receive_visibility_timeout: 456 redis: max_idle: 12 max_active: 123 max_idle_timeout: 456 wait: false read_timeout: 17 write_timeout: 19 connect_timeout: 21 normal_tasks_poll_period: 1001 delayed_tasks_poll_period: 23 delayed_tasks_key: delayed_tasks_key master_name: master_name no_unix_signals: true dynamodb: task_states_table: task_states_table group_metas_table: group_metas_table ` func TestReadFromFile(t *testing.T) { t.Parallel() data, err := config.ReadFromFile("testconfig.yml") if err != nil { t.Fatal(err) } assert.Equal(t, configYAMLData, string(data)) } func TestNewFromYaml(t *testing.T) { t.Parallel() cnf, err := config.NewFromYaml("testconfig.yml", false) if err != nil { t.Fatal(err) } assert.Equal(t, "broker", cnf.Broker) assert.Equal(t, "default_queue", cnf.DefaultQueue) assert.Equal(t, "result_backend", cnf.ResultBackend) assert.Equal(t, 123456, cnf.ResultsExpireIn) assert.Equal(t, "exchange", cnf.AMQP.Exchange) assert.Equal(t, "exchange_type", cnf.AMQP.ExchangeType) assert.Equal(t, "binding_key", cnf.AMQP.BindingKey) assert.Equal(t, 10, cnf.AMQP.QueueDeclareArgs["x-max-priority"]) assert.Equal(t, "any", cnf.AMQP.QueueBindingArgs["x-match"]) assert.Equal(t, "png", cnf.AMQP.QueueBindingArgs["image-type"]) assert.Equal(t, 123, cnf.AMQP.PrefetchCount) assert.Equal(t, 123, cnf.SQS.WaitTimeSeconds) assert.Equal(t, 456, *cnf.SQS.VisibilityTimeout) assert.Equal(t, 12, cnf.Redis.MaxIdle) assert.Equal(t, 123, cnf.Redis.MaxActive) assert.Equal(t, 456, cnf.Redis.IdleTimeout) assert.Equal(t, false, cnf.Redis.Wait) assert.Equal(t, 17, cnf.Redis.ReadTimeout) assert.Equal(t, 19, cnf.Redis.WriteTimeout) assert.Equal(t, 21, cnf.Redis.ConnectTimeout) assert.Equal(t, 1001, cnf.Redis.NormalTasksPollPeriod) assert.Equal(t, 23, cnf.Redis.DelayedTasksPollPeriod) assert.Equal(t, "delayed_tasks_key", cnf.Redis.DelayedTasksKey) assert.Equal(t, "master_name", cnf.Redis.MasterName) assert.Equal(t, true, cnf.NoUnixSignals) assert.Equal(t, "task_states_table", cnf.DynamoDB.TaskStatesTable) assert.Equal(t, "group_metas_table", cnf.DynamoDB.GroupMetasTable) } ================================================ FILE: v1/config/test.env ================================================ BROKER=broker DEFAULT_QUEUE=default_queue RESULT_BACKEND=result_backend RESULTS_EXPIRE_IN=123456 AMQP_BINDING_KEY=binding_key AMQP_EXCHANGE=exchange AMQP_EXCHANGE_TYPE=exchange_type AMQP_PREFETCH_COUNT=123 AMQP_QUEUE_BINDING_ARGS=image-type:png,x-match:any ================================================ FILE: v1/config/testconfig.yml ================================================ --- broker: broker default_queue: default_queue result_backend: result_backend results_expire_in: 123456 amqp: binding_key: binding_key exchange: exchange exchange_type: exchange_type prefetch_count: 123 queue_declare_args: x-max-priority: 10 queue_binding_args: image-type: png x-match: any sqs: receive_wait_time_seconds: 123 receive_visibility_timeout: 456 redis: max_idle: 12 max_active: 123 max_idle_timeout: 456 wait: false read_timeout: 17 write_timeout: 19 connect_timeout: 21 normal_tasks_poll_period: 1001 delayed_tasks_poll_period: 23 delayed_tasks_key: delayed_tasks_key master_name: master_name no_unix_signals: true dynamodb: task_states_table: task_states_table group_metas_table: group_metas_table ================================================ FILE: v1/factories.go ================================================ package machinery import ( "errors" "fmt" "os" "strconv" "strings" neturl "net/url" "github.com/RichardKnop/machinery/v1/config" amqpbroker "github.com/RichardKnop/machinery/v1/brokers/amqp" eagerbroker "github.com/RichardKnop/machinery/v1/brokers/eager" gcppubsubbroker "github.com/RichardKnop/machinery/v1/brokers/gcppubsub" brokeriface "github.com/RichardKnop/machinery/v1/brokers/iface" redisbroker "github.com/RichardKnop/machinery/v1/brokers/redis" sqsbroker "github.com/RichardKnop/machinery/v1/brokers/sqs" amqpbackend "github.com/RichardKnop/machinery/v1/backends/amqp" dynamobackend "github.com/RichardKnop/machinery/v1/backends/dynamodb" eagerbackend "github.com/RichardKnop/machinery/v1/backends/eager" backendiface "github.com/RichardKnop/machinery/v1/backends/iface" memcachebackend "github.com/RichardKnop/machinery/v1/backends/memcache" mongobackend "github.com/RichardKnop/machinery/v1/backends/mongo" nullbackend "github.com/RichardKnop/machinery/v1/backends/null" redisbackend "github.com/RichardKnop/machinery/v1/backends/redis" eagerlock "github.com/RichardKnop/machinery/v1/locks/eager" lockiface "github.com/RichardKnop/machinery/v1/locks/iface" redislock "github.com/RichardKnop/machinery/v1/locks/redis" ) // BrokerFactory creates a new object of iface.Broker // Currently only AMQP/S broker is supported func BrokerFactory(cnf *config.Config) (brokeriface.Broker, error) { if strings.HasPrefix(cnf.Broker, "amqp://") { return amqpbroker.New(cnf), nil } if strings.HasPrefix(cnf.Broker, "amqps://") { return amqpbroker.New(cnf), nil } if strings.HasPrefix(cnf.Broker, "redis://") || strings.HasPrefix(cnf.Broker, "rediss://") { var scheme string if strings.HasPrefix(cnf.Broker, "redis://") { scheme = "redis://" } else { scheme = "rediss://" } parts := strings.Split(cnf.Broker, scheme) if len(parts) != 2 { return nil, fmt.Errorf( "Redis broker connection string should be in format %shost:port, instead got %s", scheme, cnf.Broker, ) } brokers := strings.Split(parts[1], ",") if len(brokers) > 1 || (cnf.Redis != nil && cnf.Redis.ClusterMode) { db, brokers := ParseBrokers(brokers) return redisbroker.NewGR(cnf, brokers, db), nil } else { redisHost, redisUsername, redisPassword, redisDB, err := ParseRedisURL(cnf.Broker) if err != nil { return nil, err } return redisbroker.New(cnf, redisHost, redisUsername, redisPassword, "", redisDB), nil } } if strings.HasPrefix(cnf.Broker, "redis+socket://") { redisSocket, redisUsername, redisPassword, redisDB, err := ParseRedisSocketURL(cnf.Broker) if err != nil { return nil, err } return redisbroker.New(cnf, "", redisUsername, redisPassword, redisSocket, redisDB), nil } if strings.HasPrefix(cnf.Broker, "eager") { return eagerbroker.New(), nil } if _, ok := os.LookupEnv("DISABLE_STRICT_SQS_CHECK"); ok { //disable SQS name check, so that users can use this with local simulated SQS //where sql broker url might not start with https://sqs //even when disabling strict SQS naming check, make sure its still a valid http URL if strings.HasPrefix(cnf.Broker, "https://") || strings.HasPrefix(cnf.Broker, "http://") { return sqsbroker.New(cnf), nil } } else { if strings.HasPrefix(cnf.Broker, "https://sqs") { return sqsbroker.New(cnf), nil } } if strings.HasPrefix(cnf.Broker, "gcppubsub://") { projectID, subscriptionName, err := ParseGCPPubSubURL(cnf.Broker) if err != nil { return nil, err } return gcppubsubbroker.New(cnf, projectID, subscriptionName) } return nil, fmt.Errorf("Factory failed with broker URL: %v", cnf.Broker) } // BackendFactory creates a new object of backends.Interface // Currently supported backends are AMQP/S and Memcache func BackendFactory(cnf *config.Config) (backendiface.Backend, error) { if strings.HasPrefix(cnf.ResultBackend, "amqp://") { return amqpbackend.New(cnf), nil } if strings.HasPrefix(cnf.ResultBackend, "amqps://") { return amqpbackend.New(cnf), nil } if strings.HasPrefix(cnf.ResultBackend, "memcache://") { parts := strings.Split(cnf.ResultBackend, "memcache://") if len(parts) != 2 { return nil, fmt.Errorf( "Memcache result backend connection string should be in format memcache://server1:port,server2:port, instead got %s", cnf.ResultBackend, ) } servers := strings.Split(parts[1], ",") return memcachebackend.New(cnf, servers), nil } if strings.HasPrefix(cnf.ResultBackend, "redis://") || strings.HasPrefix(cnf.ResultBackend, "rediss://") { var scheme string if strings.HasPrefix(cnf.ResultBackend, "redis://") { scheme = "redis://" } else { scheme = "rediss://" } parts := strings.Split(cnf.ResultBackend, scheme) addrs := strings.Split(parts[1], ",") if len(addrs) > 1 || (cnf.Redis != nil && cnf.Redis.ClusterMode) { db, addrs := ParseBrokers(addrs) return redisbackend.NewGR(cnf, addrs, db), nil } else { redisHost, redisUsername, redisPassword, redisDB, err := ParseRedisURL(cnf.ResultBackend) if err != nil { return nil, err } return redisbackend.New(cnf, redisHost, redisUsername, redisPassword, "", redisDB), nil } } if strings.HasPrefix(cnf.ResultBackend, "redis+socket://") { redisSocket, redisUsername, redisPassword, redisDB, err := ParseRedisSocketURL(cnf.ResultBackend) if err != nil { return nil, err } return redisbackend.New(cnf, "", redisUsername, redisPassword, redisSocket, redisDB), nil } if strings.HasPrefix(cnf.ResultBackend, "mongodb://") || strings.HasPrefix(cnf.ResultBackend, "mongodb+srv://") { return mongobackend.New(cnf) } if strings.HasPrefix(cnf.ResultBackend, "eager") { return eagerbackend.New(), nil } if strings.HasPrefix(cnf.ResultBackend, "null") { return nullbackend.New(), nil } if strings.HasPrefix(cnf.ResultBackend, "https://dynamodb") { return dynamobackend.New(cnf), nil } return nil, fmt.Errorf("Factory failed with result backend: %v", cnf.ResultBackend) } // ParseRedisURL ... func ParseRedisURL(url string) (host, username, password string, db int, err error) { // redis://pwd@host/db var u *neturl.URL u, err = neturl.Parse(url) if err != nil { return } if u.Scheme != "redis" && u.Scheme != "rediss" { err = errors.New("No redis scheme found") return } if u.User != nil { var exists bool username = u.User.Username() password, exists = u.User.Password() if !exists { password = u.User.Username() username = "" } } host = u.Host parts := strings.Split(u.Path, "/") if len(parts) == 1 { db = 0 //default redis db } else { db, err = strconv.Atoi(parts[1]) if err != nil { db, err = 0, nil //ignore err here } } return } // Parse the database index from the last broker in the list, and remove the /db part from it. func ParseBrokers(brokers []string) (int, []string) { db := 0 last := brokers[len(brokers)-1] if strings.Contains(last, "/") { parts := strings.SplitN(last, "/", 2) brokers[len(brokers)-1] = parts[0] if len(parts) == 2 && parts[1] != "" { if val, err := strconv.Atoi(parts[1]); err == nil { db = val } } } return db, brokers } // LockFactory creates a new object of iface.Lock // Currently supported lock is redis func LockFactory(cnf *config.Config) (lockiface.Lock, error) { if strings.HasPrefix(cnf.Lock, "eager") { return eagerlock.New(), nil } if strings.HasPrefix(cnf.Lock, "redis://") { parts := strings.Split(cnf.Lock, "redis://") if len(parts) != 2 { return nil, fmt.Errorf( "Redis broker connection string should be in format redis://host:port, instead got %s", cnf.Lock, ) } locks := strings.Split(parts[1], ",") return redislock.New(cnf, locks, 0, 3), nil } // Lock is required for periodic tasks to work, therefor return in memory lock in case none is configured return eagerlock.New(), nil } // ParseRedisSocketURL extracts Redis connection options from a URL with the // redis+socket:// scheme. This scheme is not standard (or even de facto) and // is used as a transitional mechanism until the the config package gains the // proper facilities to support socket-based connections. func ParseRedisSocketURL(url string) (path, username, password string, db int, err error) { parts := strings.Split(url, "redis+socket://") if parts[0] != "" { err = errors.New("No redis scheme found") return } // redis+socket://password@/path/to/file.soc:/db if len(parts) != 2 { err = fmt.Errorf("Redis socket connection string should be in format redis+socket://password@/path/to/file.sock:/db, instead got %s", url) return } remainder := parts[1] // Extract password if any parts = strings.SplitN(remainder, "@", 2) if len(parts) == 2 { userinfo := strings.SplitN(parts[0], ":", 2) if len(userinfo) == 2 { username = userinfo[0] password = userinfo[1] } else { password = userinfo[0] } remainder = parts[1] } else { remainder = parts[0] } // Extract path parts = strings.SplitN(remainder, ":", 2) path = parts[0] if path == "" { err = fmt.Errorf("Redis socket connection string should be in format redis+socket://password@/path/to/file.sock:/db, instead got %s", url) return } if len(parts) == 2 { remainder = parts[1] } // Extract DB if any parts = strings.SplitN(remainder, "/", 2) if len(parts) == 2 { db, _ = strconv.Atoi(parts[1]) } return } // ParseGCPPubSubURL Parse GCP Pub/Sub URL // url: gcppubsub://YOUR_GCP_PROJECT_ID/YOUR_PUBSUB_SUBSCRIPTION_NAME func ParseGCPPubSubURL(url string) (string, string, error) { parts := strings.Split(url, "gcppubsub://") if parts[0] != "" { return "", "", errors.New("No gcppubsub scheme found") } if len(parts) != 2 { return "", "", fmt.Errorf("gcppubsub scheme should be in format gcppubsub://YOUR_GCP_PROJECT_ID/YOUR_PUBSUB_SUBSCRIPTION_NAME, instead got %s", url) } remainder := parts[1] parts = strings.Split(remainder, "/") if len(parts) == 2 { if parts[0] == "" { return "", "", fmt.Errorf("gcppubsub scheme should be in format gcppubsub://YOUR_GCP_PROJECT_ID/YOUR_PUBSUB_SUBSCRIPTION_NAME, instead got %s", url) } if parts[1] == "" { return "", "", fmt.Errorf("gcppubsub scheme should be in format gcppubsub://YOUR_GCP_PROJECT_ID/YOUR_PUBSUB_SUBSCRIPTION_NAME, instead got %s", url) } return parts[0], parts[1], nil } return "", "", fmt.Errorf("gcppubsub scheme should be in format gcppubsub://YOUR_GCP_PROJECT_ID/YOUR_PUBSUB_SUBSCRIPTION_NAME, instead got %s", url) } ================================================ FILE: v1/factories_test.go ================================================ package machinery_test import ( "errors" "fmt" "os" "reflect" "testing" "unsafe" machinery "github.com/RichardKnop/machinery/v1" "github.com/RichardKnop/machinery/v1/config" "github.com/stretchr/testify/assert" amqpbroker "github.com/RichardKnop/machinery/v1/brokers/amqp" brokeriface "github.com/RichardKnop/machinery/v1/brokers/iface" redisbroker "github.com/RichardKnop/machinery/v1/brokers/redis" sqsbroker "github.com/RichardKnop/machinery/v1/brokers/sqs" amqpbackend "github.com/RichardKnop/machinery/v1/backends/amqp" memcachebackend "github.com/RichardKnop/machinery/v1/backends/memcache" mongobackend "github.com/RichardKnop/machinery/v1/backends/mongo" redisbackend "github.com/RichardKnop/machinery/v1/backends/redis" ) var ( redisSchemeTestCases = []struct { desc string url string host, user, pwd string db int err error }{ { desc: "invalid redis scheme", url: "non_redis://127.0.0.1:5672", err: errors.New("invalid redis scheme"), }, { desc: "empty redis scheme", url: "redis:/", }, { desc: "redis host", url: "redis://127.0.0.1:5672", host: "127.0.0.1:5672", }, { desc: "redis password and host", url: "redis://pwd@127.0.0.1:5672", host: "127.0.0.1:5672", user: "", pwd: "pwd", }, { desc: "redis password, host and db", url: "redis://pwd@127.0.0.1:5672/2", host: "127.0.0.1:5672", user: "", pwd: "pwd", db: 2, }, { desc: "redis user, password host", url: "redis://user:pwd@127.0.0.1:5672", host: "127.0.0.1:5672", user: "user", pwd: "pwd", }, { desc: "redis user, password with colon host", url: "redis://user:pwd:with:colon@127.0.0.1:5672", host: "127.0.0.1:5672", user: "user", pwd: "pwd:with:colon", }, { desc: "redis user, empty password and host", url: "redis://user:@127.0.0.1:5672", host: "127.0.0.1:5672", user: "user", pwd: "", }, { desc: "redis empty user, password and host", url: "redis://:pwd@127.0.0.1:5672", host: "127.0.0.1:5672", user: "", pwd: "pwd", }, } ) func TestBrokerFactory(t *testing.T) { t.Parallel() var cnf config.Config // 1) AMQP broker test cnf = config.Config{ Broker: "amqp://guest:guest@localhost:5672/", DefaultQueue: "machinery_tasks", AMQP: &config.AMQPConfig{ Exchange: "machinery_exchange", ExchangeType: "direct", BindingKey: "machinery_task", PrefetchCount: 1, }, } actual, err := machinery.BrokerFactory(&cnf) if assert.NoError(t, err) { _, isAMQPBroker := actual.(*amqpbroker.Broker) assert.True( t, isAMQPBroker, "Broker should be instance of *brokers.AMQPBroker", ) expected := amqpbroker.New(&cnf) assert.True( t, brokerEqual(actual, expected), fmt.Sprintf("conn = %v, want %v", actual, expected), ) } // 2) Redis broker test // with password cnf = config.Config{ Broker: "redis://password@localhost:6379", DefaultQueue: "machinery_tasks", } actual, err = machinery.BrokerFactory(&cnf) if assert.NoError(t, err) { _, isRedisBroker := actual.(*redisbroker.Broker) assert.True( t, isRedisBroker, "Broker should be instance of *brokers.RedisBroker", ) expected := redisbroker.New(&cnf, "localhost:6379", "", "password", "", 0) assert.True( t, brokerEqual(actual, expected), fmt.Sprintf("conn = %v, want %v", actual, expected), ) } // without password cnf = config.Config{ Broker: "redis://localhost:6379", DefaultQueue: "machinery_tasks", } actual, err = machinery.BrokerFactory(&cnf) if assert.NoError(t, err) { _, isRedisBroker := actual.(*redisbroker.Broker) assert.True( t, isRedisBroker, "Broker should be instance of *brokers.RedisBroker", ) expected := redisbroker.New(&cnf, "localhost:6379", "", "", "", 0) assert.True( t, brokerEqual(actual, expected), fmt.Sprintf("conn = %v, want %v", actual, expected), ) } // using a socket file cnf = config.Config{ Broker: "redis+socket:///tmp/redis.sock", DefaultQueue: "machinery_tasks", } actual, err = machinery.BrokerFactory(&cnf) if assert.NoError(t, err) { _, isRedisBroker := actual.(*redisbroker.Broker) assert.True( t, isRedisBroker, "Broker should be instance of *brokers.RedisBroker", ) expected := redisbroker.New(&cnf, "", "", "", "/tmp/redis.sock", 0) assert.True( t, brokerEqual(actual, expected), fmt.Sprintf("conn = %v, want %v", actual, expected), ) } // 3) AWS SQS cnf = config.Config{ Broker: "https://sqs.us-east-2.amazonaws.com/123456789012", DefaultQueue: "machinery_tasks", } actual, err = machinery.BrokerFactory(&cnf) if assert.NoError(t, err) { _, isAWSSQSBroker := actual.(*sqsbroker.Broker) assert.True( t, isAWSSQSBroker, "Broker should be instance of *brokers.AWSSQSBroker", ) } // 4) local SQS config should pass with special env variable // AWS SQS Invalid SQS Check cnf = config.Config{ Broker: "http://localhost:5672/some-queue", DefaultQueue: "machinery_tasks", } os.Setenv("DISABLE_STRICT_SQS_CHECK", "yes") actual, err = machinery.BrokerFactory(&cnf) if assert.NoError(t, err) { _, isAWSSQSBroker := actual.(*sqsbroker.Broker) assert.True( t, isAWSSQSBroker, "Broker should be instance of *brokers.AWSSQSBroker", ) } os.Unsetenv("DISABLE_STRICT_SQS_CHECK") } func brokerEqual(x, y brokeriface.Broker) bool { // unset Broker.stopChan and Broker.retryStopChan to nil before using // reflect.DeepEqual() as the objects will have a different address rx := reflect.ValueOf(x).Elem() rxf := rx.FieldByName("stopChan") rxf = reflect.NewAt(rxf.Type(), unsafe.Pointer(rxf.UnsafeAddr())).Elem() rxf.Set(reflect.Zero(rxf.Type())) rxf = rx.FieldByName("retryStopChan") rxf = reflect.NewAt(rxf.Type(), unsafe.Pointer(rxf.UnsafeAddr())).Elem() rxf.Set(reflect.Zero(rxf.Type())) ry := reflect.ValueOf(y).Elem() ryf := ry.FieldByName("stopChan") ryf = reflect.NewAt(ryf.Type(), unsafe.Pointer(ryf.UnsafeAddr())).Elem() ryf.Set(reflect.Zero(ryf.Type())) ryf = ry.FieldByName("retryStopChan") ryf = reflect.NewAt(ryf.Type(), unsafe.Pointer(ryf.UnsafeAddr())).Elem() ryf.Set(reflect.Zero(ryf.Type())) return reflect.DeepEqual(x, y) } func TestBrokerFactoryError(t *testing.T) { t.Parallel() cnf := config.Config{ Broker: "BOGUS", } conn, err := machinery.BrokerFactory(&cnf) if assert.Error(t, err) { assert.Nil(t, conn) assert.Equal(t, "Factory failed with broker URL: BOGUS", err.Error()) } // AWS SQS Invalid SQS Check cnf = config.Config{ Broker: "http://localhost:5672/some-queue", DefaultQueue: "machinery_tasks", } conn, err = machinery.BrokerFactory(&cnf) if assert.Error(t, err) { assert.Nil(t, conn) assert.Equal(t, "Factory failed with broker URL: http://localhost:5672/some-queue", err.Error()) } // Non-AWS SQS URL allowed but not invalid http ones os.Setenv("DISABLE_STRICT_SQS_CHECK", "yes") cnf = config.Config{ Broker: "localhost:5672/some-queue", DefaultQueue: "machinery_tasks", } conn, err = machinery.BrokerFactory(&cnf) if assert.Error(t, err) { assert.Nil(t, conn) assert.Equal(t, "Factory failed with broker URL: localhost:5672/some-queue", err.Error()) } os.Unsetenv("DISABLE_STRICT_SQS_CHECK") } func TestBackendFactory(t *testing.T) { t.Parallel() var cnf config.Config // 1) AMQP backend test cnf = config.Config{ResultBackend: "amqp://guest:guest@localhost:5672/"} actual, err := machinery.BackendFactory(&cnf) if assert.NoError(t, err) { expected := amqpbackend.New(&cnf) assert.True( t, reflect.DeepEqual(actual, expected), fmt.Sprintf("conn = %v, want %v", actual, expected), ) } // 2) Memcache backend test cnf = config.Config{ ResultBackend: "memcache://10.0.0.1:11211,10.0.0.2:11211", } actual, err = machinery.BackendFactory(&cnf) if assert.NoError(t, err) { servers := []string{"10.0.0.1:11211", "10.0.0.2:11211"} expected := memcachebackend.New(&cnf, servers) assert.True( t, reflect.DeepEqual(actual, expected), fmt.Sprintf("conn = %v, want %v", actual, expected), ) } // 2) Redis backend test // with password cnf = config.Config{ ResultBackend: "redis://password@localhost:6379", } actual, err = machinery.BackendFactory(&cnf) if assert.NoError(t, err) { expected := redisbackend.New(&cnf, "localhost:6379", "", "password", "", 0) assert.True( t, reflect.DeepEqual(actual, expected), fmt.Sprintf("conn = %v, want %v", actual, expected), ) } // without password cnf = config.Config{ ResultBackend: "redis://localhost:6379", } actual, err = machinery.BackendFactory(&cnf) if assert.NoError(t, err) { expected := redisbackend.New(&cnf, "localhost:6379", "", "", "", 0) assert.True( t, reflect.DeepEqual(actual, expected), fmt.Sprintf("conn = %v, want %v", actual, expected), ) } // using a socket file cnf = config.Config{ ResultBackend: "redis+socket:///tmp/redis.sock", } actual, err = machinery.BackendFactory(&cnf) if assert.NoError(t, err) { expected := redisbackend.New(&cnf, "", "", "", "/tmp/redis.sock", 0) assert.True( t, reflect.DeepEqual(actual, expected), fmt.Sprintf("conn = %v, want %v", actual, expected), ) } // 4) MongoDB backend test cnf = config.Config{ ResultBackend: "mongodb://mongo:27017", ResultsExpireIn: 30, } actual, err = machinery.BackendFactory(&cnf) if assert.NoError(t, err) { expected, err := mongobackend.New(&cnf) if assert.NoError(t, err) { assert.True( t, reflect.DeepEqual(actual, expected), fmt.Sprintf("conn = %v, want %v", actual, expected), ) } } } func TestBackendFactoryError(t *testing.T) { t.Parallel() cnf := config.Config{ ResultBackend: "BOGUS", } conn, err := machinery.BackendFactory(&cnf) if assert.Error(t, err) { assert.Nil(t, conn) assert.Equal(t, "Factory failed with result backend: BOGUS", err.Error()) } if conn != nil { t.Errorf("conn = %v, should be nil", conn) } } func TestParseRedisURL(t *testing.T) { t.Parallel() for _, tc := range redisSchemeTestCases { tc := tc // capture range variable t.Run(tc.desc, func(t *testing.T) { t.Parallel() host, user, pwd, db, err := machinery.ParseRedisURL(tc.url) if tc.err != nil { assert.Error(t, err, tc.err) return } if assert.NoError(t, err) { assert.Equal(t, tc.host, host) assert.Equal(t, tc.user, user) assert.Equal(t, tc.pwd, pwd) assert.Equal(t, tc.db, db) } }) } } func TestParseRedisSocketURL(t *testing.T) { t.Parallel() var ( path, user, pwd, url string db int err error ) url = "non_redissock:///tmp/redis.sock" _, _, _, _, err = machinery.ParseRedisSocketURL(url) assert.Error(t, err, "invalid redis scheme") url = "redis+socket:/" _, _, _, _, err = machinery.ParseRedisSocketURL(url) assert.Error(t, err, "invalid redis url scheme") url = "redis+socket:///tmp/redis.sock" path, user, pwd, db, err = machinery.ParseRedisSocketURL(url) if assert.NoError(t, err) { assert.Equal(t, "/tmp/redis.sock", path) assert.Equal(t, "", user) assert.Equal(t, "", pwd) assert.Equal(t, 0, db) } url = "redis+socket://user:pwd@/tmp/redis.sock" path, user, pwd, db, _ = machinery.ParseRedisSocketURL(url) if assert.NoError(t, err) { assert.Equal(t, "/tmp/redis.sock", path) assert.Equal(t, "user", user) assert.Equal(t, "pwd", pwd) assert.Equal(t, 0, db) } url = "redis+socket://user:pwd@/tmp/redis.sock:/2" path, user, pwd, db, err = machinery.ParseRedisSocketURL(url) if assert.NoError(t, err) { assert.Equal(t, "/tmp/redis.sock", path) assert.Equal(t, "user", user) assert.Equal(t, "pwd", pwd) assert.Equal(t, 2, db) } } ================================================ FILE: v1/locks/eager/eager.go ================================================ package eager import ( "errors" "sync" "time" ) var ( ErrEagerLockFailed = errors.New("eager lock: failed to acquire lock") ) type Lock struct { retries int interval time.Duration register struct { sync.RWMutex m map[string]int64 } } func New() *Lock { return &Lock{ retries: 3, interval: 5 * time.Second, register: struct { sync.RWMutex m map[string]int64 }{ m: make(map[string]int64), }, } } func (e *Lock) LockWithRetries(key string, value int64) error { for i := 0; i <= e.retries; i++ { err := e.Lock(key, value) if err == nil { //成功拿到锁，返回 return nil } time.Sleep(e.interval) } return ErrEagerLockFailed } func (e *Lock) Lock(key string, value int64) error { e.register.Lock() defer e.register.Unlock() timeout, exist := e.register.m[key] if !exist || time.Now().UnixNano() > timeout { e.register.m[key] = value return nil } return ErrEagerLockFailed } ================================================ FILE: v1/locks/eager/eager_test.go ================================================ package eager import ( lockiface "github.com/RichardKnop/machinery/v1/locks/iface" "github.com/RichardKnop/machinery/v1/utils" "github.com/stretchr/testify/assert" "testing" "time" ) func TestLock_Lock(t *testing.T) { lock := New() keyName := utils.GetPureUUID() go func() { err := lock.Lock(keyName, time.Now().Add(25*time.Second).UnixNano()) assert.NoError(t, err) }() time.Sleep(1 * time.Second) err := lock.Lock(keyName, time.Now().Add(25*time.Second).UnixNano()) assert.Error(t, err) assert.EqualError(t, err, ErrEagerLockFailed.Error()) } func TestLock_LockWithRetries(t *testing.T) { lock := New() keyName := utils.GetPureUUID() go func() { err := lock.LockWithRetries(keyName, time.Now().Add(25*time.Second).UnixNano()) assert.NoError(t, err) }() time.Sleep(1 * time.Second) err := lock.LockWithRetries(keyName, time.Now().Add(25*time.Second).UnixNano()) assert.Error(t, err) assert.EqualError(t, err, ErrEagerLockFailed.Error()) } func TestNew(t *testing.T) { lock := New() assert.Implements(t, (*lockiface.Lock)(nil), lock) } ================================================ FILE: v1/locks/iface/interfaces.go ================================================ package iface type Lock interface { //Acquire the lock with retry //key: the name of the lock, //value: at the nanosecond timestamp that lock needs to be released automatically LockWithRetries(key string, value int64) error //Acquire the lock with once //key: the name of the lock, //value: at the nanosecond timestamp that lock needs to be released automatically Lock(key string, value int64) error } ================================================ FILE: v1/locks/redis/redis.go ================================================ package redis import ( "context" "errors" "strconv" "strings" "time" "github.com/RichardKnop/machinery/v1/config" "github.com/redis/go-redis/v9" ) var ( ErrRedisLockFailed = errors.New("redis lock: failed to acquire lock") ) type Lock struct { rclient redis.UniversalClient retries int interval time.Duration } func New(cnf *config.Config, addrs []string, db, retries int) Lock { if retries <= 0 { return Lock{} } lock := Lock{retries: retries} var password string parts := strings.Split(addrs[0], "@") if len(parts) >= 2 { password = strings.Join(parts[:len(parts)-1], "@") addrs[0] = parts[len(parts)-1] // addr is the last one without @ } ropt := &redis.UniversalOptions{ Addrs: addrs, DB: db, Password: password, } if cnf.Redis != nil { ropt.MasterName = cnf.Redis.MasterName } lock.rclient = redis.NewUniversalClient(ropt) return lock } func (r Lock) LockWithRetries(key string, unixTsToExpireNs int64) error { for i := 0; i <= r.retries; i++ { err := r.Lock(key, unixTsToExpireNs) if err == nil { //成功拿到锁，返回 return nil } time.Sleep(r.interval) } return ErrRedisLockFailed } func (r Lock) Lock(key string, unixTsToExpireNs int64) error { now := time.Now().UnixNano() expiration := time.Duration(unixTsToExpireNs + 1 - now) // ctx := r.rclient.Context() ctx := context.Background() success, err := r.rclient.SetNX(ctx, key, unixTsToExpireNs, expiration).Result() if err != nil { return err } if !success { v, err := r.rclient.Get(ctx, key).Result() if err != nil { return err } timeout, err := strconv.Atoi(v) if err != nil { return err } if timeout != 0 && now > int64(timeout) { newTimeout, err := r.rclient.GetSet(ctx, key, unixTsToExpireNs).Result() if err != nil { return err } curTimeout, err := strconv.Atoi(newTimeout) if err != nil { return err } if now > int64(curTimeout) { // success to acquire lock with get set // set the expiration of redis key r.rclient.Expire(ctx, key, expiration) return nil } return ErrRedisLockFailed } return ErrRedisLockFailed } return nil } ================================================ FILE: v1/log/log.go ================================================ package log import ( "github.com/RichardKnop/logging" ) var ( logger = logging.New(nil, nil, new(logging.ColouredFormatter)) // DEBUG ... DEBUG = logger[logging.DEBUG] // INFO ... INFO = logger[logging.INFO] // WARNING ... WARNING = logger[logging.WARNING] // ERROR ... ERROR = logger[logging.ERROR] // FATAL ... FATAL = logger[logging.FATAL] ) // Set sets a custom logger for all log levels func Set(l logging.LoggerInterface) { DEBUG = l INFO = l WARNING = l ERROR = l FATAL = l } // SetDebug sets a custom logger for DEBUG level logs func SetDebug(l logging.LoggerInterface) { DEBUG = l } // SetInfo sets a custom logger for INFO level logs func SetInfo(l logging.LoggerInterface) { INFO = l } // SetWarning sets a custom logger for WARNING level logs func SetWarning(l logging.LoggerInterface) { WARNING = l } // SetError sets a custom logger for ERROR level logs func SetError(l logging.LoggerInterface) { ERROR = l } // SetFatal sets a custom logger for FATAL level logs func SetFatal(l logging.LoggerInterface) { FATAL = l } ================================================ FILE: v1/log/log_test.go ================================================ package log_test import ( "testing" "github.com/RichardKnop/machinery/v1/log" ) func TestDefaultLogger(t *testing.T) { log.INFO.Print("should not panic") log.WARNING.Print("should not panic") log.ERROR.Print("should not panic") log.FATAL.Print("should not panic") } ================================================ FILE: v1/package.go ================================================ package machinery ================================================ FILE: v1/retry/fibonacci.go ================================================ package retry // Fibonacci returns successive Fibonacci numbers starting from 1 func Fibonacci() func() int { a, b := 0, 1 return func() int { a, b = b, a+b return a } } // FibonacciNext returns next number in Fibonacci sequence greater than start func FibonacciNext(start int) int { fib := Fibonacci() num := fib() for num <= start { num = fib() } return num } ================================================ FILE: v1/retry/fibonacci_test.go ================================================ package retry_test import ( "testing" "github.com/RichardKnop/machinery/v1/retry" "github.com/stretchr/testify/assert" ) func TestFibonacci(t *testing.T) { fibonacci := retry.Fibonacci() sequence := []int{ fibonacci(), fibonacci(), fibonacci(), fibonacci(), fibonacci(), fibonacci(), } assert.EqualValues(t, sequence, []int{1, 1, 2, 3, 5, 8}) } func TestFibonacciNext(t *testing.T) { assert.Equal(t, 1, retry.FibonacciNext(0)) assert.Equal(t, 2, retry.FibonacciNext(1)) assert.Equal(t, 5, retry.FibonacciNext(3)) assert.Equal(t, 5, retry.FibonacciNext(4)) assert.Equal(t, 8, retry.FibonacciNext(5)) assert.Equal(t, 13, retry.FibonacciNext(8)) } ================================================ FILE: v1/retry/retry.go ================================================ package retry import ( "fmt" "time" "github.com/RichardKnop/machinery/v1/log" ) // Closure - a useful closure we can use when there is a problem // connecting to the broker. It uses Fibonacci sequence to space out retry attempts var Closure = func() func(chan int) { retryIn := 0 fibonacci := Fibonacci() return func(stopChan chan int) { if retryIn > 0 { durationString := fmt.Sprintf("%vs", retryIn) duration, _ := time.ParseDuration(durationString) log.WARNING.Printf("Retrying in %v seconds", retryIn) select { case <-stopChan: break case <-time.After(duration): break } } retryIn = fibonacci() } } ================================================ FILE: v1/server.go ================================================ package machinery import ( "context" "errors" "fmt" "sync" "time" "github.com/google/uuid" "github.com/robfig/cron/v3" "github.com/RichardKnop/machinery/v1/backends/result" "github.com/RichardKnop/machinery/v1/brokers/eager" "github.com/RichardKnop/machinery/v1/config" "github.com/RichardKnop/machinery/v1/log" "github.com/RichardKnop/machinery/v1/tasks" "github.com/RichardKnop/machinery/v1/tracing" "github.com/RichardKnop/machinery/v1/utils" backendsiface "github.com/RichardKnop/machinery/v1/backends/iface" brokersiface "github.com/RichardKnop/machinery/v1/brokers/iface" lockiface "github.com/RichardKnop/machinery/v1/locks/iface" opentracing "github.com/opentracing/opentracing-go" ) // Server is the main Machinery object and stores all configuration // All the tasks workers process are registered against the server type Server struct { config *config.Config registeredTasks *sync.Map broker brokersiface.Broker backend backendsiface.Backend lock lockiface.Lock scheduler *cron.Cron prePublishHandler func(*tasks.Signature) } // NewServerWithBrokerBackend ... func NewServerWithBrokerBackendLock(cnf *config.Config, brokerServer brokersiface.Broker, backendServer backendsiface.Backend, lock lockiface.Lock) *Server { srv := &Server{ config: cnf, registeredTasks: new(sync.Map), broker: brokerServer, backend: backendServer, lock: lock, scheduler: cron.New(), } // Run scheduler job go srv.scheduler.Run() return srv } // NewServer creates Server instance func NewServer(cnf *config.Config) (*Server, error) { broker, err := BrokerFactory(cnf) if err != nil { return nil, err } // Backend is optional so we ignore the error backend, _ := BackendFactory(cnf) // Init lock lock, err := LockFactory(cnf) if err != nil { return nil, err } srv := NewServerWithBrokerBackendLock(cnf, broker, backend, lock) // init for eager-mode eager, ok := broker.(eager.Mode) if ok { // we don't have to call worker.Launch in eager mode eager.AssignWorker(srv.NewWorker("eager", 0)) } return srv, nil } // NewWorker creates Worker instance func (server *Server) NewWorker(consumerTag string, concurrency int) *Worker { return &Worker{ server: server, ConsumerTag: consumerTag, Concurrency: concurrency, Queue: "", } } // NewCustomQueueWorker creates Worker instance with Custom Queue func (server *Server) NewCustomQueueWorker(consumerTag string, concurrency int, queue string) *Worker { return &Worker{ server: server, ConsumerTag: consumerTag, Concurrency: concurrency, Queue: queue, } } // GetBroker returns broker func (server *Server) GetBroker() brokersiface.Broker { return server.broker } // SetBroker sets broker func (server *Server) SetBroker(broker brokersiface.Broker) { server.broker = broker } // GetBackend returns backend func (server *Server) GetBackend() backendsiface.Backend { return server.backend } // SetBackend sets backend func (server *Server) SetBackend(backend backendsiface.Backend) { server.backend = backend } // GetConfig returns connection object func (server *Server) GetConfig() *config.Config { return server.config } // SetConfig sets config func (server *Server) SetConfig(cnf *config.Config) { server.config = cnf } // SetPreTaskHandler Sets pre publish handler func (server *Server) SetPreTaskHandler(handler func(*tasks.Signature)) { server.prePublishHandler = handler } // RegisterTasks registers all tasks at once func (server *Server) RegisterTasks(namedTaskFuncs map[string]interface{}) error { for _, task := range namedTaskFuncs { if err := tasks.ValidateTask(task); err != nil { return err } } for k, v := range namedTaskFuncs { server.registeredTasks.Store(k, v) } server.broker.SetRegisteredTaskNames(server.GetRegisteredTaskNames()) return nil } // RegisterTask registers a single task func (server *Server) RegisterTask(name string, taskFunc interface{}) error { if err := tasks.ValidateTask(taskFunc); err != nil { return err } server.registeredTasks.Store(name, taskFunc) server.broker.SetRegisteredTaskNames(server.GetRegisteredTaskNames()) return nil } // IsTaskRegistered returns true if the task name is registered with this broker func (server *Server) IsTaskRegistered(name string) bool { _, ok := server.registeredTasks.Load(name) return ok } // GetRegisteredTask returns registered task by name func (server *Server) GetRegisteredTask(name string) (interface{}, error) { taskFunc, ok := server.registeredTasks.Load(name) if !ok { return nil, fmt.Errorf("Task not registered error: %s", name) } return taskFunc, nil } // SendTaskWithContext will inject the trace context in the signature headers before publishing it func (server *Server) SendTaskWithContext(ctx context.Context, signature *tasks.Signature) (*result.AsyncResult, error) { span, _ := opentracing.StartSpanFromContext(ctx, "SendTask", tracing.ProducerOption(), tracing.MachineryTag) defer span.Finish() // tag the span with some info about the signature signature.Headers = tracing.HeadersWithSpan(signature.Headers, span) // Make sure result backend is defined if server.backend == nil { return nil, errors.New("Result backend required") } // Auto generate a UUID if not set already if signature.UUID == "" { taskID := uuid.New().String() signature.UUID = fmt.Sprintf("task_%v", taskID) } // Set initial task state to PENDING if err := server.backend.SetStatePending(signature); err != nil { return nil, fmt.Errorf("Set state pending error: %s", err) } if server.prePublishHandler != nil { server.prePublishHandler(signature) } if err := server.broker.Publish(ctx, signature); err != nil { return nil, fmt.Errorf("Publish message error: %s", err) } return result.NewAsyncResult(signature, server.backend), nil } // SendTask publishes a task to the default queue func (server *Server) SendTask(signature *tasks.Signature) (*result.AsyncResult, error) { return server.SendTaskWithContext(context.Background(), signature) } // SendChainWithContext will inject the trace context in all the signature headers before publishing it func (server *Server) SendChainWithContext(ctx context.Context, chain *tasks.Chain) (*result.ChainAsyncResult, error) { span, _ := opentracing.StartSpanFromContext(ctx, "SendChain", tracing.ProducerOption(), tracing.MachineryTag, tracing.WorkflowChainTag) defer span.Finish() tracing.AnnotateSpanWithChainInfo(span, chain) return server.SendChain(chain) } // SendChain triggers a chain of tasks func (server *Server) SendChain(chain *tasks.Chain) (*result.ChainAsyncResult, error) { _, err := server.SendTask(chain.Tasks[0]) if err != nil { return nil, err } return result.NewChainAsyncResult(chain.Tasks, server.backend), nil } // SendGroupWithContext will inject the trace context in all the signature headers before publishing it func (server *Server) SendGroupWithContext(ctx context.Context, group *tasks.Group, sendConcurrency int) ([]*result.AsyncResult, error) { span, _ := opentracing.StartSpanFromContext(ctx, "SendGroup", tracing.ProducerOption(), tracing.MachineryTag, tracing.WorkflowGroupTag) defer span.Finish() tracing.AnnotateSpanWithGroupInfo(span, group, sendConcurrency) // Make sure result backend is defined if server.backend == nil { return nil, errors.New("Result backend required") } asyncResults := make([]*result.AsyncResult, len(group.Tasks)) var wg sync.WaitGroup wg.Add(len(group.Tasks)) errorsChan := make(chan error, len(group.Tasks)*2) // Init group server.backend.InitGroup(group.GroupUUID, group.GetUUIDs()) // Init the tasks Pending state first for _, signature := range group.Tasks { if err := server.backend.SetStatePending(signature); err != nil { errorsChan <- err continue } } pool := make(chan struct{}, sendConcurrency) go func() { for i := 0; i < sendConcurrency; i++ { pool <- struct{}{} } }() for i, signature := range group.Tasks { if sendConcurrency > 0 { <-pool } go func(s *tasks.Signature, index int) { defer wg.Done() // Publish task err := server.broker.Publish(ctx, s) if sendConcurrency > 0 { pool <- struct{}{} } if err != nil { errorsChan <- fmt.Errorf("Publish message error: %s", err) return } asyncResults[index] = result.NewAsyncResult(s, server.backend) }(signature, i) } done := make(chan int) go func() { wg.Wait() done <- 1 }() select { case err := <-errorsChan: return asyncResults, err case <-done: return asyncResults, nil } } // SendGroup triggers a group of parallel tasks func (server *Server) SendGroup(group *tasks.Group, sendConcurrency int) ([]*result.AsyncResult, error) { return server.SendGroupWithContext(context.Background(), group, sendConcurrency) } // SendChordWithContext will inject the trace context in all the signature headers before publishing it func (server *Server) SendChordWithContext(ctx context.Context, chord *tasks.Chord, sendConcurrency int) (*result.ChordAsyncResult, error) { span, _ := opentracing.StartSpanFromContext(ctx, "SendChord", tracing.ProducerOption(), tracing.MachineryTag, tracing.WorkflowChordTag) defer span.Finish() tracing.AnnotateSpanWithChordInfo(span, chord, sendConcurrency) _, err := server.SendGroupWithContext(ctx, chord.Group, sendConcurrency) if err != nil { return nil, err } return result.NewChordAsyncResult( chord.Group.Tasks, chord.Callback, server.backend, ), nil } // SendChord triggers a group of parallel tasks with a callback func (server *Server) SendChord(chord *tasks.Chord, sendConcurrency int) (*result.ChordAsyncResult, error) { return server.SendChordWithContext(context.Background(), chord, sendConcurrency) } // GetRegisteredTaskNames returns slice of registered task names func (server *Server) GetRegisteredTaskNames() []string { taskNames := make([]string, 0) server.registeredTasks.Range(func(key, value interface{}) bool { taskNames = append(taskNames, key.(string)) return true }) return taskNames } // RegisterPeriodicTask register a periodic task which will be triggered periodically func (server *Server) RegisterPeriodicTask(spec, name string, signature *tasks.Signature) error { //check spec schedule, err := cron.ParseStandard(spec) if err != nil { return err } f := func() { //get lock err := server.lock.LockWithRetries(utils.GetLockName(name, spec), schedule.Next(time.Now()).UnixNano()-1) if err != nil { return } //send task _, err = server.SendTask(tasks.CopySignature(signature)) if err != nil { log.ERROR.Printf("periodic task failed. task name is: %s. error is %s", name, err.Error()) } } _, err = server.scheduler.AddFunc(spec, f) return err } // RegisterPeriodicChain register a periodic chain which will be triggered periodically func (server *Server) RegisterPeriodicChain(spec, name string, signatures ...*tasks.Signature) error { //check spec schedule, err := cron.ParseStandard(spec) if err != nil { return err } f := func() { // new chain chain, _ := tasks.NewChain(tasks.CopySignatures(signatures...)...) //get lock err := server.lock.LockWithRetries(utils.GetLockName(name, spec), schedule.Next(time.Now()).UnixNano()-1) if err != nil { return } //send task _, err = server.SendChain(chain) if err != nil { log.ERROR.Printf("periodic task failed. task name is: %s. error is %s", name, err.Error()) } } _, err = server.scheduler.AddFunc(spec, f) return err } // RegisterPeriodicGroup register a periodic group which will be triggered periodically func (server *Server) RegisterPeriodicGroup(spec, name string, sendConcurrency int, signatures ...*tasks.Signature) error { //check spec schedule, err := cron.ParseStandard(spec) if err != nil { return err } f := func() { // new group group, _ := tasks.NewGroup(tasks.CopySignatures(signatures...)...) //get lock err := server.lock.LockWithRetries(utils.GetLockName(name, spec), schedule.Next(time.Now()).UnixNano()-1) if err != nil { return } //send task _, err = server.SendGroup(group, sendConcurrency) if err != nil { log.ERROR.Printf("periodic task failed. task name is: %s. error is %s", name, err.Error()) } } _, err = server.scheduler.AddFunc(spec, f) return err } // RegisterPeriodicChord register a periodic chord which will be triggered periodically func (server *Server) RegisterPeriodicChord(spec, name string, sendConcurrency int, callback *tasks.Signature, signatures ...*tasks.Signature) error { //check spec schedule, err := cron.ParseStandard(spec) if err != nil { return err } f := func() { // new chord group, _ := tasks.NewGroup(tasks.CopySignatures(signatures...)...) chord, _ := tasks.NewChord(group, tasks.CopySignature(callback)) //get lock err := server.lock.LockWithRetries(utils.GetLockName(name, spec), schedule.Next(time.Now()).UnixNano()-1) if err != nil { return } //send task _, err = server.SendChord(chord, sendConcurrency) if err != nil { log.ERROR.Printf("periodic task failed. task name is: %s. error is %s", name, err.Error()) } } _, err = server.scheduler.AddFunc(spec, f) return err } ================================================ FILE: v1/server_test.go ================================================ package machinery_test import ( "testing" "github.com/stretchr/testify/assert" "github.com/RichardKnop/machinery/v1" "github.com/RichardKnop/machinery/v1/config" ) func TestRegisterTasks(t *testing.T) { t.Parallel() server := getTestServer(t) err := server.RegisterTasks(map[string]interface{}{ "test_task": func() error { return nil }, }) assert.NoError(t, err) _, err = server.GetRegisteredTask("test_task") assert.NoError(t, err, "test_task is not registered but it should be") } func TestRegisterTask(t *testing.T) { t.Parallel() server := getTestServer(t) err := server.RegisterTask("test_task", func() error { return nil }) assert.NoError(t, err) _, err = server.GetRegisteredTask("test_task") assert.NoError(t, err, "test_task is not registered but it should be") } func TestRegisterTaskInRaceCondition(t *testing.T) { t.Parallel() server := getTestServer(t) for i:=0; i<10; i++ { go func() { err := server.RegisterTask("test_task", func() error { return nil }) assert.NoError(t, err) _, err = server.GetRegisteredTask("test_task") assert.NoError(t, err, "test_task is not registered but it should be") }() } } func TestGetRegisteredTask(t *testing.T) { t.Parallel() server := getTestServer(t) _, err := server.GetRegisteredTask("test_task") assert.Error(t, err, "test_task is registered but it should not be") } func TestGetRegisteredTaskNames(t *testing.T) { t.Parallel() server := getTestServer(t) taskName := "test_task" err := server.RegisterTask(taskName, func() error { return nil }) assert.NoError(t, err) taskNames := server.GetRegisteredTaskNames() assert.Equal(t, 1, len(taskNames)) assert.Equal(t, taskName, taskNames[0]) } func TestNewWorker(t *testing.T) { t.Parallel() server := getTestServer(t) server.NewWorker("test_worker", 1) assert.NoError(t, nil) } func TestNewCustomQueueWorker(t *testing.T) { t.Parallel() server := getTestServer(t) server.NewCustomQueueWorker("test_customqueueworker", 1, "test_queue") assert.NoError(t, nil) } func getTestServer(t *testing.T) *machinery.Server { server, err := machinery.NewServer(&config.Config{ Broker: "amqp://guest:guest@localhost:5672/", DefaultQueue: "machinery_tasks", ResultBackend: "redis://127.0.0.1:6379", Lock: "redis://127.0.0.1:6379", AMQP: &config.AMQPConfig{ Exchange: "machinery_exchange", ExchangeType: "direct", BindingKey: "machinery_task", PrefetchCount: 1, }, }) if err != nil { t.Error(err) } return server } ================================================ FILE: v1/tasks/errors.go ================================================ package tasks import ( "fmt" "time" ) // ErrRetryTaskLater ... type ErrRetryTaskLater struct { name, msg string retryIn time.Duration } // RetryIn returns time.Duration from now when task should be retried func (e ErrRetryTaskLater) RetryIn() time.Duration { return e.retryIn } // Error implements the error interface func (e ErrRetryTaskLater) Error() string { return fmt.Sprintf("Task error: %s Will retry in: %s", e.msg, e.retryIn) } // NewErrRetryTaskLater returns new ErrRetryTaskLater instance func NewErrRetryTaskLater(msg string, retryIn time.Duration) ErrRetryTaskLater { return ErrRetryTaskLater{msg: msg, retryIn: retryIn} } // Retriable is interface that retriable errors should implement type Retriable interface { RetryIn() time.Duration } ================================================ FILE: v1/tasks/reflect.go ================================================ package tasks import ( "context" "encoding/base64" "encoding/json" "fmt" "reflect" "strconv" "strings" ) var ( typesMap = map[string]reflect.Type{ // base types "bool": reflect.TypeOf(true), "int": reflect.TypeOf(int(1)), "int8": reflect.TypeOf(int8(1)), "int16": reflect.TypeOf(int16(1)), "int32": reflect.TypeOf(int32(1)), "int64": reflect.TypeOf(int64(1)), "uint": reflect.TypeOf(uint(1)), "uint8": reflect.TypeOf(uint8(1)), "uint16": reflect.TypeOf(uint16(1)), "uint32": reflect.TypeOf(uint32(1)), "uint64": reflect.TypeOf(uint64(1)), "float32": reflect.TypeOf(float32(0.5)), "float64": reflect.TypeOf(float64(0.5)), "string": reflect.TypeOf(string("")), // slices "[]bool": reflect.TypeOf(make([]bool, 0)), "[]int": reflect.TypeOf(make([]int, 0)), "[]int8": reflect.TypeOf(make([]int8, 0)), "[]int16": reflect.TypeOf(make([]int16, 0)), "[]int32": reflect.TypeOf(make([]int32, 0)), "[]int64": reflect.TypeOf(make([]int64, 0)), "[]uint": reflect.TypeOf(make([]uint, 0)), "[]uint8": reflect.TypeOf(make([]uint8, 0)), "[]uint16": reflect.TypeOf(make([]uint16, 0)), "[]uint32": reflect.TypeOf(make([]uint32, 0)), "[]uint64": reflect.TypeOf(make([]uint64, 0)), "[]float32": reflect.TypeOf(make([]float32, 0)), "[]float64": reflect.TypeOf(make([]float64, 0)), "[]byte": reflect.TypeOf(make([]byte, 0)), "[]string": reflect.TypeOf([]string{""}), } ctxType = reflect.TypeOf((*context.Context)(nil)).Elem() typeConversionError = func(argValue interface{}, argTypeStr string) error { return fmt.Errorf("%v is not %v", argValue, argTypeStr) } ) // ErrUnsupportedType ... type ErrUnsupportedType struct { valueType string } // NewErrUnsupportedType returns new ErrUnsupportedType func NewErrUnsupportedType(valueType string) ErrUnsupportedType { return ErrUnsupportedType{valueType} } // Error method so we implement the error interface func (e ErrUnsupportedType) Error() string { return fmt.Sprintf("%v is not one of supported types", e.valueType) } // ReflectValue converts interface{} to reflect.Value based on string type func ReflectValue(valueType string, value interface{}) (reflect.Value, error) { if strings.HasPrefix(valueType, "[]") { return reflectValues(valueType, value) } return reflectValue(valueType, value) } // reflectValue converts interface{} to reflect.Value based on string type // representing a base type (not a slice) func reflectValue(valueType string, value interface{}) (reflect.Value, error) { theType, ok := typesMap[valueType] if !ok { return reflect.Value{}, NewErrUnsupportedType(valueType) } theValue := reflect.New(theType) // Booleans if theType.String() == "bool" { boolValue, err := getBoolValue(theType.String(), value) if err != nil { return reflect.Value{}, err } theValue.Elem().SetBool(boolValue) return theValue.Elem(), nil } // Integers if strings.HasPrefix(theType.String(), "int") { intValue, err := getIntValue(theType.String(), value) if err != nil { return reflect.Value{}, err } theValue.Elem().SetInt(intValue) return theValue.Elem(), err } // Unsigned integers if strings.HasPrefix(theType.String(), "uint") { uintValue, err := getUintValue(theType.String(), value) if err != nil { return reflect.Value{}, err } theValue.Elem().SetUint(uintValue) return theValue.Elem(), err } // Floating point numbers if strings.HasPrefix(theType.String(), "float") { floatValue, err := getFloatValue(theType.String(), value) if err != nil { return reflect.Value{}, err } theValue.Elem().SetFloat(floatValue) return theValue.Elem(), err } // Strings if theType.String() == "string" { stringValue, err := getStringValue(theType.String(), value) if err != nil { return reflect.Value{}, err } theValue.Elem().SetString(stringValue) return theValue.Elem(), nil } return reflect.Value{}, NewErrUnsupportedType(valueType) } // reflectValues converts interface{} to reflect.Value based on string type // representing a slice of values func reflectValues(valueType string, value interface{}) (reflect.Value, error) { theType, ok := typesMap[valueType] if !ok { return reflect.Value{}, NewErrUnsupportedType(valueType) } // For NULL we return an empty slice if value == nil { return reflect.MakeSlice(theType, 0, 0), nil } var theValue reflect.Value // Booleans if theType.String() == "[]bool" { bools := reflect.ValueOf(value) theValue = reflect.MakeSlice(theType, bools.Len(), bools.Len()) for i := 0; i < bools.Len(); i++ { boolValue, err := getBoolValue(strings.Split(theType.String(), "[]")[1], bools.Index(i).Interface()) if err != nil { return reflect.Value{}, err } theValue.Index(i).SetBool(boolValue) } return theValue, nil } // Integers if strings.HasPrefix(theType.String(), "[]int") { ints := reflect.ValueOf(value) theValue = reflect.MakeSlice(theType, ints.Len(), ints.Len()) for i := 0; i < ints.Len(); i++ { intValue, err := getIntValue(strings.Split(theType.String(), "[]")[1], ints.Index(i).Interface()) if err != nil { return reflect.Value{}, err } theValue.Index(i).SetInt(intValue) } return theValue, nil } // Unsigned integers if strings.HasPrefix(theType.String(), "[]uint") || theType.String() == "[]byte" { // Decode the base64 string if the value type is []uint8 or it's alias []byte // See: https://golang.org/pkg/encoding/json/#Marshal // > Array and slice values encode as JSON arrays, except that []byte encodes as a base64-encoded string if reflect.TypeOf(value).String() == "string" { output, err := base64.StdEncoding.DecodeString(value.(string)) if err != nil { return reflect.Value{}, err } value = output } uints := reflect.ValueOf(value) theValue = reflect.MakeSlice(theType, uints.Len(), uints.Len()) for i := 0; i < uints.Len(); i++ { uintValue, err := getUintValue(strings.Split(theType.String(), "[]")[1], uints.Index(i).Interface()) if err != nil { return reflect.Value{}, err } theValue.Index(i).SetUint(uintValue) } return theValue, nil } // Floating point numbers if strings.HasPrefix(theType.String(), "[]float") { floats := reflect.ValueOf(value) theValue = reflect.MakeSlice(theType, floats.Len(), floats.Len()) for i := 0; i < floats.Len(); i++ { floatValue, err := getFloatValue(strings.Split(theType.String(), "[]")[1], floats.Index(i).Interface()) if err != nil { return reflect.Value{}, err } theValue.Index(i).SetFloat(floatValue) } return theValue, nil } // Strings if theType.String() == "[]string" { strs := reflect.ValueOf(value) theValue = reflect.MakeSlice(theType, strs.Len(), strs.Len()) for i := 0; i < strs.Len(); i++ { strValue, err := getStringValue(strings.Split(theType.String(), "[]")[1], strs.Index(i).Interface()) if err != nil { return reflect.Value{}, err } theValue.Index(i).SetString(strValue) } return theValue, nil } return reflect.Value{}, NewErrUnsupportedType(valueType) } func getBoolValue(theType string, value interface{}) (bool, error) { b, ok := value.(bool) if !ok { return false, typeConversionError(value, typesMap[theType].String()) } return b, nil } func getIntValue(theType string, value interface{}) (int64, error) { // We use https://golang.org/pkg/encoding/json/#Decoder.UseNumber when unmarshaling signatures. // This is because JSON only supports 64-bit floating point numbers and we could lose precision // when converting from float64 to signed integer if strings.HasPrefix(fmt.Sprintf("%T", value), "json.Number") { n, ok := value.(json.Number) if !ok { return 0, typeConversionError(value, typesMap[theType].String()) } return n.Int64() } n, ok := value.(int64) if !ok { return 0, typeConversionError(value, typesMap[theType].String()) } return n, nil } func getUintValue(theType string, value interface{}) (uint64, error) { // Losing precision only happens in receiving a JSON number from a language like js, // and receiving a large uint number from golang or python could cause json.Number.Int64 be turned into a panic. // So we use strconv.ParseUint to correctly parse a uint value. if strings.HasPrefix(fmt.Sprintf("%T", value), "json.Number") { n, ok := value.(json.Number) if !ok { return 0, typeConversionError(value, typesMap[theType].String()) } uintVal, err := strconv.ParseUint(string(n), 10, 64) if err != nil { return 0, err } return uintVal, nil } var n uint64 switch value := value.(type) { case uint64: n = value case uint8: n = uint64(value) default: return 0, typeConversionError(value, typesMap[theType].String()) } return n, nil } func getFloatValue(theType string, value interface{}) (float64, error) { // We use https://golang.org/pkg/encoding/json/#Decoder.UseNumber when unmarshaling signatures. // This is because JSON only supports 64-bit floating point numbers and we could lose precision if strings.HasPrefix(fmt.Sprintf("%T", value), "json.Number") { n, ok := value.(json.Number) if !ok { return 0, typeConversionError(value, typesMap[theType].String()) } return n.Float64() } f, ok := value.(float64) if !ok { return 0, typeConversionError(value, typesMap[theType].String()) } return f, nil } func getStringValue(theType string, value interface{}) (string, error) { s, ok := value.(string) if !ok { return "", typeConversionError(value, typesMap[theType].String()) } return s, nil } // IsContextType checks to see if the type is a context.Context func IsContextType(t reflect.Type) bool { return t == ctxType } ================================================ FILE: v1/tasks/reflect_test.go ================================================ package tasks_test import ( "encoding/json" "reflect" "testing" "github.com/RichardKnop/machinery/v1/tasks" ) var ( reflectValuesTestCases = []struct { name string value interface{} expectedType string expectedValue interface{} }{ // basic types { name: "bool", value: false, expectedType: "bool", }, { name: "int", value: json.Number("123"), expectedType: "int", expectedValue: int(123), }, { name: "int8", value: json.Number("123"), expectedType: "int8", expectedValue: int8(123), }, { name: "int16", value: json.Number("123"), expectedType: "int16", expectedValue: int16(123), }, { name: "int32", value: json.Number("123"), expectedType: "int32", expectedValue: int32(123), }, { name: "int64", value: json.Number("185135722552891243"), expectedType: "int64", expectedValue: int64(185135722552891243), }, { name: "uint", value: json.Number("123"), expectedType: "uint", expectedValue: uint(123), }, { name: "uint8", value: json.Number("123"), expectedType: "uint8", expectedValue: uint8(123), }, { name: "uint16", value: json.Number("123"), expectedType: "uint16", expectedValue: uint16(123), }, { name: "uint32", value: json.Number("123"), expectedType: "uint32", expectedValue: uint32(123), }, { name: "uint64", value: json.Number("185135722552891243"), expectedType: "uint64", expectedValue: uint64(185135722552891243), }, { name: "uint64", value: json.Number("9223372036854775808"), // math.MaxInt64 + 1 expectedType: "uint64", expectedValue: uint64(9223372036854775808), }, { name: "float32", value: json.Number("0.5"), expectedType: "float32", expectedValue: float32(0.5), }, { name: "float64", value: json.Number("0.5"), expectedType: "float64", expectedValue: float64(0.5), }, { name: "string", value: "123", expectedType: "string", expectedValue: "123", }, // slices { name: "[]bool", value: []interface{}{false, true}, expectedType: "[]bool", expectedValue: []bool{false, true}, }, { name: "[]int", value: []interface{}{json.Number("1"), json.Number("2")}, expectedType: "[]int", expectedValue: []int{1, 2}, }, { name: "[]int8", value: []interface{}{json.Number("1"), json.Number("2")}, expectedType: "[]int8", expectedValue: []int8{1, 2}, }, { name: "[]int16", value: []interface{}{json.Number("1"), json.Number("2")}, expectedType: "[]int16", expectedValue: []int16{1, 2}, }, { name: "[]int32", value: []interface{}{json.Number("1"), json.Number("2")}, expectedType: "[]int32", expectedValue: []int32{1, 2}, }, { name: "[]int64", value: []interface{}{json.Number("1"), json.Number("2")}, expectedType: "[]int64", expectedValue: []int64{1, 2}, }, { name: "[]uint", value: []interface{}{json.Number("1"), json.Number("2")}, expectedType: "[]uint", expectedValue: []uint{1, 2}, }, { name: "[]uint8", value: []interface{}{json.Number("1"), json.Number("2")}, expectedType: "[]uint8", expectedValue: []uint8{1, 2}, }, { name: "[]uint16", value: []interface{}{json.Number("1"), json.Number("2")}, expectedType: "[]uint16", expectedValue: []uint16{1, 2}, }, { name: "[]uint32", value: []interface{}{json.Number("1"), json.Number("2")}, expectedType: "[]uint32", expectedValue: []uint32{1, 2}, }, { name: "[]uint64", value: []interface{}{json.Number("1"), json.Number("2")}, expectedType: "[]uint64", expectedValue: []uint64{1, 2}, }, { name: "[]float32", value: []interface{}{json.Number("0.5"), json.Number("1.28")}, expectedType: "[]float32", expectedValue: []float32{0.5, 1.28}, }, { name: "[]float64", value: []interface{}{json.Number("0.5"), json.Number("1.28")}, expectedType: "[]float64", expectedValue: []float64{0.5, 1.28}, }, { name: "[]string", value: []interface{}{"foo", "bar"}, expectedType: "[]string", expectedValue: []string{"foo", "bar"}, }, // empty slices from NULL { name: "[]bool", value: nil, expectedType: "[]bool", expectedValue: []bool{}, }, { name: "[]int64", value: nil, expectedType: "[]int64", expectedValue: []int64{}, }, { name: "[]uint64", value: nil, expectedType: "[]uint64", expectedValue: []uint64{}, }, { name: "[]float64", value: nil, expectedType: "[]float64", expectedValue: []float64{}, }, { name: "[]string", value: nil, expectedType: "[]string", expectedValue: []string{}, }, } ) func TestReflectValue(t *testing.T) { t.Parallel() for _, tc := range reflectValuesTestCases { tc := tc // capture range variable t.Run(tc.name, func(t *testing.T) { t.Parallel() value, err := tasks.ReflectValue(tc.name, tc.value) if err != nil { t.Error(err) } if value.Type().String() != tc.expectedType { t.Errorf("type is %v, want %s", value.Type().String(), tc.expectedType) } if tc.expectedValue != nil { if !reflect.DeepEqual(value.Interface(), tc.expectedValue) { t.Errorf("value is %v, want %v", value.Interface(), tc.expectedValue) } } }) } } ================================================ FILE: v1/tasks/result.go ================================================ package tasks import ( "fmt" "reflect" "strings" ) // TaskResult represents an actual return value of a processed task type TaskResult struct { Type string `bson:"type"` Value interface{} `bson:"value"` } // ReflectTaskResults ... func ReflectTaskResults(taskResults []*TaskResult) ([]reflect.Value, error) { resultValues := make([]reflect.Value, len(taskResults)) for i, taskResult := range taskResults { resultValue, err := ReflectValue(taskResult.Type, taskResult.Value) if err != nil { return nil, err } resultValues[i] = resultValue } return resultValues, nil } // HumanReadableResults ... func HumanReadableResults(results []reflect.Value) string { if len(results) == 1 { return fmt.Sprintf("%v", results[0].Interface()) } readableResults := make([]string, len(results)) for i := 0; i < len(results); i++ { readableResults[i] = fmt.Sprintf("%v", results[i].Interface()) } return fmt.Sprintf("[%s]", strings.Join(readableResults, ", ")) } ================================================ FILE: v1/tasks/result_test.go ================================================ package tasks_test import ( "testing" "github.com/RichardKnop/machinery/v1/tasks" "github.com/stretchr/testify/assert" ) func TestReflectTaskResults(t *testing.T) { t.Parallel() taskResults := []*tasks.TaskResult{ { Type: "[]string", Value: []string{"f", "o", "o"}, }, } results, err := tasks.ReflectTaskResults(taskResults) if assert.NoError(t, err) { assert.Equal(t, 1, len(results)) assert.Equal(t, 3, results[0].Len()) assert.Equal(t, "f", results[0].Index(0).String()) assert.Equal(t, "o", results[0].Index(1).String()) assert.Equal(t, "o", results[0].Index(2).String()) } } ================================================ FILE: v1/tasks/signature.go ================================================ package tasks import ( "fmt" "github.com/RichardKnop/machinery/v1/utils" "time" "github.com/google/uuid" ) // Arg represents a single argument passed to invocation fo a task type Arg struct { Name string `bson:"name"` Type string `bson:"type"` Value interface{} `bson:"value"` } // Headers represents the headers which should be used to direct the task type Headers map[string]interface{} // Set on Headers implements opentracing.TextMapWriter for trace propagation func (h Headers) Set(key, val string) { h[key] = val } // ForeachKey on Headers implements opentracing.TextMapReader for trace propagation. // It is essentially the same as the opentracing.TextMapReader implementation except // for the added casting from interface{} to string. func (h Headers) ForeachKey(handler func(key, val string) error) error { for k, v := range h { // Skip any non string values stringValue, ok := v.(string) if !ok { continue } if err := handler(k, stringValue); err != nil { return err } } return nil } // Signature represents a single task invocation type Signature struct { UUID string Name string RoutingKey string ETA *time.Time GroupUUID string GroupTaskCount int Args []Arg Headers Headers Priority uint8 Immutable bool RetryCount int RetryTimeout int OnSuccess []*Signature OnError []*Signature ChordCallback *Signature //MessageGroupId for Broker, e.g. SQS BrokerMessageGroupId string //ReceiptHandle of SQS Message SQSReceiptHandle string // StopTaskDeletionOnError used with sqs when we want to send failed messages to dlq, // and don't want machinery to delete from source queue StopTaskDeletionOnError bool // IgnoreWhenTaskNotRegistered auto removes the request when there is no handeler available // When this is true a task with no handler will be ignored and not placed back in the queue IgnoreWhenTaskNotRegistered bool } // NewSignature creates a new task signature func NewSignature(name string, args []Arg) (*Signature, error) { signatureID := uuid.New().String() return &Signature{ UUID: fmt.Sprintf("task_%v", signatureID), Name: name, Args: args, }, nil } func CopySignatures(signatures ...*Signature) []*Signature { var sigs = make([]*Signature, len(signatures)) for index, signature := range signatures { sigs[index] = CopySignature(signature) } return sigs } func CopySignature(signature *Signature) *Signature { var sig = new(Signature) _ = utils.DeepCopy(sig, signature) return sig } ================================================ FILE: v1/tasks/state.go ================================================ package tasks import "time" const ( // StatePending - initial state of a task StatePending = "PENDING" // StateReceived - when task is received by a worker StateReceived = "RECEIVED" // StateStarted - when the worker starts processing the task StateStarted = "STARTED" // StateRetry - when failed task has been scheduled for retry StateRetry = "RETRY" // StateSuccess - when the task is processed successfully StateSuccess = "SUCCESS" // StateFailure - when processing of the task fails StateFailure = "FAILURE" ) // TaskState represents a state of a task type TaskState struct { TaskUUID string `bson:"_id"` TaskName string `bson:"task_name"` State string `bson:"state"` Results []*TaskResult `bson:"results"` Error string `bson:"error"` CreatedAt time.Time `bson:"created_at"` TTL int64 `bson:"ttl,omitempty"` } // GroupMeta stores useful metadata about tasks within the same group // E.g. UUIDs of all tasks which are used in order to check if all tasks // completed successfully or not and thus whether to trigger chord callback type GroupMeta struct { GroupUUID string `bson:"_id"` TaskUUIDs []string `bson:"task_uuids"` ChordTriggered bool `bson:"chord_triggered"` Lock bool `bson:"lock"` CreatedAt time.Time `bson:"created_at"` TTL int64 `bson:"ttl,omitempty"` } // NewPendingTaskState ... func NewPendingTaskState(signature *Signature) *TaskState { return &TaskState{ TaskUUID: signature.UUID, TaskName: signature.Name, State: StatePending, CreatedAt: time.Now().UTC(), } } // NewReceivedTaskState ... func NewReceivedTaskState(signature *Signature) *TaskState { return &TaskState{ TaskUUID: signature.UUID, State: StateReceived, } } // NewStartedTaskState ... func NewStartedTaskState(signature *Signature) *TaskState { return &TaskState{ TaskUUID: signature.UUID, State: StateStarted, } } // NewSuccessTaskState ... func NewSuccessTaskState(signature *Signature, results []*TaskResult) *TaskState { return &TaskState{ TaskUUID: signature.UUID, State: StateSuccess, Results: results, } } // NewFailureTaskState ... func NewFailureTaskState(signature *Signature, err string) *TaskState { return &TaskState{ TaskUUID: signature.UUID, State: StateFailure, Error: err, } } // NewRetryTaskState ... func NewRetryTaskState(signature *Signature) *TaskState { return &TaskState{ TaskUUID: signature.UUID, State: StateRetry, } } // IsCompleted returns true if state is SUCCESS or FAILURE, // i.e. the task has finished processing and either succeeded or failed. func (taskState *TaskState) IsCompleted() bool { return taskState.IsSuccess() || taskState.IsFailure() } // IsSuccess returns true if state is SUCCESS func (taskState *TaskState) IsSuccess() bool { return taskState.State == StateSuccess } // IsFailure returns true if state is FAILURE func (taskState *TaskState) IsFailure() bool { return taskState.State == StateFailure } ================================================ FILE: v1/tasks/state_test.go ================================================ package tasks_test import ( "testing" "github.com/RichardKnop/machinery/v1/tasks" "github.com/stretchr/testify/assert" ) func TestTaskStateIsCompleted(t *testing.T) { t.Parallel() taskState := &tasks.TaskState{ TaskUUID: "taskUUID", State: tasks.StatePending, } assert.False(t, taskState.IsCompleted()) taskState.State = tasks.StateReceived assert.False(t, taskState.IsCompleted()) taskState.State = tasks.StateStarted assert.False(t, taskState.IsCompleted()) taskState.State = tasks.StateSuccess assert.True(t, taskState.IsCompleted()) taskState.State = tasks.StateFailure assert.True(t, taskState.IsCompleted()) } ================================================ FILE: v1/tasks/task.go ================================================ package tasks import ( "context" "errors" "fmt" "reflect" "runtime/debug" opentracing "github.com/opentracing/opentracing-go" opentracing_ext "github.com/opentracing/opentracing-go/ext" opentracing_log "github.com/opentracing/opentracing-go/log" "github.com/RichardKnop/machinery/v1/log" ) // ErrTaskPanicked ... var ErrTaskPanicked = errors.New("Invoking task caused a panic") // Task wraps a signature and methods used to reflect task arguments and // return values after invoking the task type Task struct { TaskFunc reflect.Value UseContext bool Context context.Context Args []reflect.Value } type signatureCtxType struct{} var signatureCtx signatureCtxType // SignatureFromContext gets the signature from the context func SignatureFromContext(ctx context.Context) *Signature { if ctx == nil { return nil } v := ctx.Value(signatureCtx) if v == nil { return nil } signature, _ := v.(*Signature) return signature } // NewWithSignature is the same as New but injects the signature func NewWithSignature(taskFunc interface{}, signature *Signature) (*Task, error) { args := signature.Args ctx := context.Background() ctx = context.WithValue(ctx, signatureCtx, signature) task := &Task{ TaskFunc: reflect.ValueOf(taskFunc), Context: ctx, } taskFuncType := reflect.TypeOf(taskFunc) if taskFuncType.NumIn() > 0 { arg0Type := taskFuncType.In(0) if IsContextType(arg0Type) { task.UseContext = true } } if err := task.ReflectArgs(args); err != nil { return nil, fmt.Errorf("Reflect task args error: %s", err) } return task, nil } // New tries to use reflection to convert the function and arguments // into a reflect.Value and prepare it for invocation func New(taskFunc interface{}, args []Arg) (*Task, error) { task := &Task{ TaskFunc: reflect.ValueOf(taskFunc), Context: context.Background(), } taskFuncType := reflect.TypeOf(taskFunc) if taskFuncType.NumIn() > 0 { arg0Type := taskFuncType.In(0) if IsContextType(arg0Type) { task.UseContext = true } } if err := task.ReflectArgs(args); err != nil { return nil, fmt.Errorf("Reflect task args error: %s", err) } return task, nil } // Call attempts to call the task with the supplied arguments. // // `err` is set in the return value in two cases: // 1. The reflected function invocation panics (e.g. due to a mismatched // argument list). // 2. The task func itself returns a non-nil error. func (t *Task) Call() (taskResults []*TaskResult, err error) { // retrieve the span from the task's context and finish it as soon as this function returns if span := opentracing.SpanFromContext(t.Context); span != nil { defer span.Finish() } defer func() { // Recover from panic and set err. if e := recover(); e != nil { switch e := e.(type) { default: err = ErrTaskPanicked case error: err = e case string: err = errors.New(e) } // mark the span as failed and dump the error and stack trace to the span if span := opentracing.SpanFromContext(t.Context); span != nil { opentracing_ext.Error.Set(span, true) span.LogFields( opentracing_log.Error(err), opentracing_log.Object("stack", string(debug.Stack())), ) } // Print stack trace log.ERROR.Printf("%v stack: %s", err, debug.Stack()) } }() args := t.Args if t.UseContext { ctxValue := reflect.ValueOf(t.Context) args = append([]reflect.Value{ctxValue}, args...) } // Invoke the task results := t.TaskFunc.Call(args) // Task must return at least a value if len(results) == 0 { return nil, ErrTaskReturnsNoValue } // Last returned value lastResult := results[len(results)-1] // If the last returned value is not nil, it has to be of error type, if that // is not the case, return error message, otherwise propagate the task error // to the caller if !lastResult.IsNil() { // If the result implements Retriable interface, return instance of Retriable retriableErrorInterface := reflect.TypeOf((*Retriable)(nil)).Elem() if lastResult.Type().Implements(retriableErrorInterface) { return nil, lastResult.Interface().(ErrRetryTaskLater) } // Otherwise, check that the result implements the standard error interface, // if not, return ErrLastReturnValueMustBeError error errorInterface := reflect.TypeOf((*error)(nil)).Elem() if !lastResult.Type().Implements(errorInterface) { return nil, ErrLastReturnValueMustBeError } // Return the standard error return nil, lastResult.Interface().(error) } // Convert reflect values to task results taskResults = make([]*TaskResult, len(results)-1) for i := 0; i < len(results)-1; i++ { val := results[i].Interface() typeStr := reflect.TypeOf(val).String() taskResults[i] = &TaskResult{ Type: typeStr, Value: val, } } return taskResults, err } // ReflectArgs converts []TaskArg to []reflect.Value func (t *Task) ReflectArgs(args []Arg) error { argValues := make([]reflect.Value, len(args)) for i, arg := range args { argValue, err := ReflectValue(arg.Type, arg.Value) if err != nil { return err } argValues[i] = argValue } t.Args = argValues return nil } ================================================ FILE: v1/tasks/task_test.go ================================================ package tasks_test import ( "context" "errors" "math" "testing" "time" "github.com/RichardKnop/machinery/v1/tasks" "github.com/stretchr/testify/assert" ) func TestTaskCallErrorTest(t *testing.T) { t.Parallel() // Create test task that returns tasks.ErrRetryTaskLater error retriable := func() error { return tasks.NewErrRetryTaskLater("some error", 4*time.Hour) } task, err := tasks.New(retriable, []tasks.Arg{}) assert.NoError(t, err) // Invoke TryCall and validate that returned error can be cast to tasks.ErrRetryTaskLater results, err := task.Call() assert.Nil(t, results) assert.NotNil(t, err) _, ok := interface{}(err).(tasks.ErrRetryTaskLater) assert.True(t, ok, "Error should be castable to tasks.ErrRetryTaskLater") // Create test task that returns a standard error standard := func() error { return errors.New("some error") } task, err = tasks.New(standard, []tasks.Arg{}) assert.NoError(t, err) // Invoke TryCall and validate that returned error is standard results, err = task.Call() assert.Nil(t, results) assert.NotNil(t, err) assert.Equal(t, "some error", err.Error()) } func TestTaskReflectArgs(t *testing.T) { t.Parallel() task := new(tasks.Task) args := []tasks.Arg{ { Type: "[]int64", Value: []int64{1, 2}, }, } err := task.ReflectArgs(args) assert.NoError(t, err) assert.Equal(t, 1, len(task.Args)) assert.Equal(t, "[]int64", task.Args[0].Type().String()) } func TestTaskCallInvalidArgRobustnessError(t *testing.T) { t.Parallel() // Create a test task function f := func(x int) error { return nil } // Construct an invalid argument list and reflect it args := []tasks.Arg{ {Type: "bool", Value: true}, } task, err := tasks.New(f, args) assert.NoError(t, err) // Invoke TryCall and validate error handling results, err := task.Call() assert.Equal(t, "reflect: Call using bool as type int", err.Error()) assert.Nil(t, results) } func TestTaskCallInterfaceValuedResult(t *testing.T) { t.Parallel() // Create a test task function f := func() (interface{}, error) { return math.Pi, nil } task, err := tasks.New(f, []tasks.Arg{}) assert.NoError(t, err) taskResults, err := task.Call() assert.NoError(t, err) assert.Equal(t, "float64", taskResults[0].Type) assert.Equal(t, math.Pi, taskResults[0].Value) } func TestTaskCallWithContext(t *testing.T) { t.Parallel() f := func(c context.Context) (interface{}, error) { assert.NotNil(t, c) assert.Nil(t, tasks.SignatureFromContext(c)) return math.Pi, nil } task, err := tasks.New(f, []tasks.Arg{}) assert.NoError(t, err) taskResults, err := task.Call() assert.NoError(t, err) assert.Equal(t, "float64", taskResults[0].Type) assert.Equal(t, math.Pi, taskResults[0].Value) } func TestTaskCallWithSignatureInContext(t *testing.T) { t.Parallel() f := func(c context.Context) (interface{}, error) { assert.NotNil(t, c) signature := tasks.SignatureFromContext(c) assert.NotNil(t, signature) assert.Equal(t, "foo", signature.Name) return math.Pi, nil } signature, err := tasks.NewSignature("foo", []tasks.Arg{}) assert.NoError(t, err) task, err := tasks.NewWithSignature(f, signature) assert.NoError(t, err) taskResults, err := task.Call() assert.NoError(t, err) assert.Equal(t, "float64", taskResults[0].Type) assert.Equal(t, math.Pi, taskResults[0].Value) } ================================================ FILE: v1/tasks/validate.go ================================================ package tasks import ( "errors" "reflect" ) var ( // ErrTaskMustBeFunc ... ErrTaskMustBeFunc = errors.New("Task must be a func type") // ErrTaskReturnsNoValue ... ErrTaskReturnsNoValue = errors.New("Task must return at least a single value") // ErrLastReturnValueMustBeError .. ErrLastReturnValueMustBeError = errors.New("Last return value of a task must be error") ) // ValidateTask validates task function using reflection and makes sure // it has a proper signature. Functions used as tasks must return at least a // single value and the last return type must be error func ValidateTask(task interface{}) error { v := reflect.ValueOf(task) t := v.Type() // Task must be a function if t.Kind() != reflect.Func { return ErrTaskMustBeFunc } // Task must return at least a single value if t.NumOut() < 1 { return ErrTaskReturnsNoValue } // Last return value must be error lastReturnType := t.Out(t.NumOut() - 1) errorInterface := reflect.TypeOf((*error)(nil)).Elem() if !lastReturnType.Implements(errorInterface) { return ErrLastReturnValueMustBeError } return nil } ================================================ FILE: v1/tasks/validate_test.go ================================================ package tasks_test import ( "testing" "github.com/RichardKnop/machinery/v1/tasks" "github.com/stretchr/testify/assert" ) func TestValidateTask(t *testing.T) { t.Parallel() type someStruct struct{} var ( taskOfWrongType = new(someStruct) taskWithoutReturnValue = func() {} taskWithoutErrorAsLastReturnValue = func() int { return 0 } validTask = func(arg string) error { return nil } ) err := tasks.ValidateTask(taskOfWrongType) assert.Equal(t, tasks.ErrTaskMustBeFunc, err) err = tasks.ValidateTask(taskWithoutReturnValue) assert.Equal(t, tasks.ErrTaskReturnsNoValue, err) err = tasks.ValidateTask(taskWithoutErrorAsLastReturnValue) assert.Equal(t, tasks.ErrLastReturnValueMustBeError, err) err = tasks.ValidateTask(validTask) assert.NoError(t, err) } ================================================ FILE: v1/tasks/workflow.go ================================================ package tasks import ( "fmt" "github.com/google/uuid" ) // Chain creates a chain of tasks to be executed one after another type Chain struct { Tasks []*Signature } // Group creates a set of tasks to be executed in parallel type Group struct { GroupUUID string Tasks []*Signature } // Chord adds an optional callback to the group to be executed // after all tasks in the group finished type Chord struct { Group *Group Callback *Signature } // GetUUIDs returns slice of task UUIDS func (group *Group) GetUUIDs() []string { taskUUIDs := make([]string, len(group.Tasks)) for i, signature := range group.Tasks { taskUUIDs[i] = signature.UUID } return taskUUIDs } // NewChain creates a new chain of tasks to be processed one by one, passing // results unless task signatures are set to be immutable func NewChain(signatures ...*Signature) (*Chain, error) { // Auto generate task UUIDs if needed for _, signature := range signatures { if signature.UUID == "" { signatureID := uuid.New().String() signature.UUID = fmt.Sprintf("task_%v", signatureID) } } for i := len(signatures) - 1; i > 0; i-- { if i > 0 { signatures[i-1].OnSuccess = []*Signature{signatures[i]} } } chain := &Chain{Tasks: signatures} return chain, nil } // NewGroup creates a new group of tasks to be processed in parallel func NewGroup(signatures ...*Signature) (*Group, error) { // Generate a group UUID groupUUID := uuid.New().String() groupID := fmt.Sprintf("group_%v", groupUUID) // Auto generate task UUIDs if needed, group tasks by common group UUID for _, signature := range signatures { if signature.UUID == "" { signatureID := uuid.New().String() signature.UUID = fmt.Sprintf("task_%v", signatureID) } signature.GroupUUID = groupID signature.GroupTaskCount = len(signatures) } return &Group{ GroupUUID: groupID, Tasks: signatures, }, nil } // NewChord creates a new chord (a group of tasks with a single callback // to be executed after all tasks in the group has completed) func NewChord(group *Group, callback *Signature) (*Chord, error) { if callback.UUID == "" { // Generate a UUID for the chord callback callbackUUID := uuid.New().String() callback.UUID = fmt.Sprintf("chord_%v", callbackUUID) } // Add a chord callback to all tasks for _, signature := range group.Tasks { signature.ChordCallback = callback } return &Chord{Group: group, Callback: callback}, nil } ================================================ FILE: v1/tasks/workflow_test.go ================================================ package tasks_test import ( "testing" "github.com/RichardKnop/machinery/v1/tasks" "github.com/stretchr/testify/assert" ) func TestNewChain(t *testing.T) { t.Parallel() task1 := tasks.Signature{ Name: "foo", Args: []tasks.Arg{ { Type: "float64", Value: interface{}(1), }, { Type: "float64", Value: interface{}(1), }, }, } task2 := tasks.Signature{ Name: "bar", Args: []tasks.Arg{ { Type: "float64", Value: interface{}(5), }, { Type: "float64", Value: interface{}(6), }, }, } task3 := tasks.Signature{ Name: "qux", Args: []tasks.Arg{ { Type: "float64", Value: interface{}(4), }, }, } chain, err := tasks.NewChain(&task1, &task2, &task3) if err != nil { t.Fatal(err) } firstTask := chain.Tasks[0] assert.Equal(t, "foo", firstTask.Name) assert.Equal(t, "bar", firstTask.OnSuccess[0].Name) assert.Equal(t, "qux", firstTask.OnSuccess[0].OnSuccess[0].Name) } ================================================ FILE: v1/tracing/tracing.go ================================================ package tracing import ( "encoding/json" "github.com/RichardKnop/machinery/v1/tasks" opentracing "github.com/opentracing/opentracing-go" opentracing_ext "github.com/opentracing/opentracing-go/ext" opentracing_log "github.com/opentracing/opentracing-go/log" ) // opentracing tags var ( MachineryTag = opentracing.Tag{Key: string(opentracing_ext.Component), Value: "machinery"} WorkflowGroupTag = opentracing.Tag{Key: "machinery.workflow", Value: "group"} WorkflowChordTag = opentracing.Tag{Key: "machinery.workflow", Value: "chord"} WorkflowChainTag = opentracing.Tag{Key: "machinery.workflow", Value: "chain"} ) // StartSpanFromHeaders will extract a span from the signature headers // and start a new span with the given operation name. func StartSpanFromHeaders(headers tasks.Headers, operationName string) opentracing.Span { // Try to extract the span context from the carrier. spanContext, err := opentracing.GlobalTracer().Extract(opentracing.TextMap, headers) // Create a new span from the span context if found or start a new trace with the function name. // For clarity add the machinery component tag. span := opentracing.StartSpan( operationName, ConsumerOption(spanContext), MachineryTag, ) // Log any error but don't fail if err != nil { span.LogFields(opentracing_log.Error(err)) } return span } // HeadersWithSpan will inject a span into the signature headers func HeadersWithSpan(headers tasks.Headers, span opentracing.Span) tasks.Headers { // check if the headers aren't nil if headers == nil { headers = make(tasks.Headers) } if err := opentracing.GlobalTracer().Inject(span.Context(), opentracing.TextMap, headers); err != nil { span.LogFields(opentracing_log.Error(err)) } return headers } type consumerOption struct { producerContext opentracing.SpanContext } func (c consumerOption) Apply(o *opentracing.StartSpanOptions) { if c.producerContext != nil { opentracing.FollowsFrom(c.producerContext).Apply(o) } opentracing_ext.SpanKindConsumer.Apply(o) } // ConsumerOption ... func ConsumerOption(producer opentracing.SpanContext) opentracing.StartSpanOption { return consumerOption{producer} } type producerOption struct{} func (p producerOption) Apply(o *opentracing.StartSpanOptions) { opentracing_ext.SpanKindProducer.Apply(o) } // ProducerOption ... func ProducerOption() opentracing.StartSpanOption { return producerOption{} } // AnnotateSpanWithSignatureInfo ... func AnnotateSpanWithSignatureInfo(span opentracing.Span, signature *tasks.Signature) { // tag the span with some info about the signature span.SetTag("signature.name", signature.Name) span.SetTag("signature.uuid", signature.UUID) if signature.GroupUUID != "" { span.SetTag("signature.group.uuid", signature.GroupUUID) } if signature.ChordCallback != nil { span.SetTag("signature.chord.callback.uuid", signature.ChordCallback.UUID) span.SetTag("signature.chord.callback.name", signature.ChordCallback.Name) } } // AnnotateSpanWithChainInfo ... func AnnotateSpanWithChainInfo(span opentracing.Span, chain *tasks.Chain) { // tag the span with some info about the chain span.SetTag("chain.tasks.length", len(chain.Tasks)) // inject the tracing span into the tasks signature headers for _, signature := range chain.Tasks { signature.Headers = HeadersWithSpan(signature.Headers, span) } } // AnnotateSpanWithGroupInfo ... func AnnotateSpanWithGroupInfo(span opentracing.Span, group *tasks.Group, sendConcurrency int) { // tag the span with some info about the group span.SetTag("group.uuid", group.GroupUUID) span.SetTag("group.tasks.length", len(group.Tasks)) span.SetTag("group.concurrency", sendConcurrency) // encode the task uuids to json, if that fails just dump it in if taskUUIDs, err := json.Marshal(group.GetUUIDs()); err == nil { span.SetTag("group.tasks", string(taskUUIDs)) } else { span.SetTag("group.tasks", group.GetUUIDs()) } // inject the tracing span into the tasks signature headers for _, signature := range group.Tasks { signature.Headers = HeadersWithSpan(signature.Headers, span) } } // AnnotateSpanWithChordInfo ... func AnnotateSpanWithChordInfo(span opentracing.Span, chord *tasks.Chord, sendConcurrency int) { // tag the span with chord specific info span.SetTag("chord.callback.uuid", chord.Callback.UUID) // inject the tracing span into the callback signature chord.Callback.Headers = HeadersWithSpan(chord.Callback.Headers, span) // tag the span for the group part of the chord AnnotateSpanWithGroupInfo(span, chord.Group, sendConcurrency) } ================================================ FILE: v1/utils/deepcopy.go ================================================ package utils import ( "errors" "reflect" ) var ( ErrNoMatchType = errors.New("no match type") ErrNoPointer = errors.New("must be interface") ErrInvalidArgument = errors.New("invalid arguments") ) func deepCopy(dst, src reflect.Value) { switch src.Kind() { case reflect.Interface: value := src.Elem() if !value.IsValid() { return } newValue := reflect.New(value.Type()).Elem() deepCopy(newValue, value) dst.Set(newValue) case reflect.Ptr: value := src.Elem() if !value.IsValid() { return } dst.Set(reflect.New(value.Type())) deepCopy(dst.Elem(), value) case reflect.Map: dst.Set(reflect.MakeMap(src.Type())) keys := src.MapKeys() for _, key := range keys { value := src.MapIndex(key) newValue := reflect.New(value.Type()).Elem() deepCopy(newValue, value) dst.SetMapIndex(key, newValue) } case reflect.Slice: dst.Set(reflect.MakeSlice(src.Type(), src.Len(), src.Cap())) for i := 0; i < src.Len(); i++ { deepCopy(dst.Index(i), src.Index(i)) } case reflect.Struct: typeSrc := src.Type() for i := 0; i < src.NumField(); i++ { value := src.Field(i) tag := typeSrc.Field(i).Tag if value.CanSet() && tag.Get("deepcopy") != "-" { deepCopy(dst.Field(i), value) } } default: dst.Set(src) } } func DeepCopy(dst, src interface{}) error { typeDst := reflect.TypeOf(dst) typeSrc := reflect.TypeOf(src) if typeDst != typeSrc { return ErrNoMatchType } if typeSrc.Kind() != reflect.Ptr { return ErrNoPointer } valueDst := reflect.ValueOf(dst).Elem() valueSrc := reflect.ValueOf(src).Elem() if !valueDst.IsValid() || !valueSrc.IsValid() { return ErrInvalidArgument } deepCopy(valueDst, valueSrc) return nil } func DeepClone(v interface{}) interface{} { dst := reflect.New(reflect.TypeOf(v)).Elem() deepCopy(dst, reflect.ValueOf(v)) return dst.Interface() } ================================================ FILE: v1/utils/deepcopy_test.go ================================================ package utils import ( "testing" "github.com/stretchr/testify/assert" ) func TestDeepCopy(t *testing.T) { t.Parallel() type s struct { A float64 B int C []int D *int E map[string]int } var d = 3 var dst = new(s) var src = s{1.0, 1, []int{1, 2, 3}, &d, map[string]int{"a": 1}} err := DeepCopy(dst, &src) src.A = 2 assert.NoError(t, err) assert.Equal(t, 1.0, dst.A) assert.Equal(t, 1, dst.B) assert.Equal(t, []int{1, 2, 3}, dst.C) assert.Equal(t, &d, dst.D) assert.Equal(t, map[string]int{"a": 1}, dst.E) } ================================================ FILE: v1/utils/utils.go ================================================ package utils import ( "os" "path/filepath" ) const ( LockKeyPrefix = "machinery_lock_" ) func GetLockName(name, spec string) string { return LockKeyPrefix + filepath.Base(os.Args[0]) + name + spec } ================================================ FILE: v1/utils/utils_test.go ================================================ package utils import ( "testing" "github.com/stretchr/testify/assert" ) func TestGetLockName(t *testing.T) { t.Parallel() lockName := GetLockName("test", "*/3 * * *") assert.Equal(t, "machinery_lock_utils.testtest*/3 * * *", lockName) } ================================================ FILE: v1/utils/uuid.go ================================================ package utils import ( "github.com/google/uuid" "strings" ) func GetPureUUID() string { uid, _ := uuid.NewUUID() return strings.Replace(uid.String(), "-", "", -1) } ================================================ FILE: v1/utils/uuid_test.go ================================================ package utils import ( "testing" "github.com/stretchr/testify/assert" ) func TestGetPureUUID(t *testing.T) { t.Parallel() assert.Len(t, GetPureUUID(), 32) } ================================================ FILE: v1/worker.go ================================================ package machinery import ( "errors" "fmt" "net/url" "os" "os/signal" "sync" "syscall" "time" "github.com/opentracing/opentracing-go" "github.com/RichardKnop/machinery/v1/backends/amqp" "github.com/RichardKnop/machinery/v1/brokers/errs" "github.com/RichardKnop/machinery/v1/log" "github.com/RichardKnop/machinery/v1/retry" "github.com/RichardKnop/machinery/v1/tasks" "github.com/RichardKnop/machinery/v1/tracing" ) // Worker represents a single worker process type Worker struct { server *Server ConsumerTag string Concurrency int Queue string errorHandler func(err error) preTaskHandler func(*tasks.Signature) postTaskHandler func(*tasks.Signature) preConsumeHandler func(*Worker) bool } var ( // ErrWorkerQuitGracefully is returned when worker quit gracefully ErrWorkerQuitGracefully = errors.New("Worker quit gracefully") // ErrWorkerQuitAbruptly is returned when worker quit abruptly ErrWorkerQuitAbruptly = errors.New("Worker quit abruptly") ) // Launch starts a new worker process. The worker subscribes // to the default queue and processes incoming registered tasks func (worker *Worker) Launch() error { errorsChan := make(chan error) worker.LaunchAsync(errorsChan) return <-errorsChan } // LaunchAsync is a non blocking version of Launch func (worker *Worker) LaunchAsync(errorsChan chan<- error) { cnf := worker.server.GetConfig() broker := worker.server.GetBroker() // Log some useful information about worker configuration log.INFO.Printf("Launching a worker with the following settings:") log.INFO.Printf("- Broker: %s", RedactURL(cnf.Broker)) if worker.Queue == "" { log.INFO.Printf("- DefaultQueue: %s", cnf.DefaultQueue) } else { log.INFO.Printf("- CustomQueue: %s", worker.Queue) } log.INFO.Printf("- ResultBackend: %s", RedactURL(cnf.ResultBackend)) if cnf.AMQP != nil { log.INFO.Printf("- AMQP: %s", cnf.AMQP.Exchange) log.INFO.Printf(" - Exchange: %s", cnf.AMQP.Exchange) log.INFO.Printf(" - ExchangeType: %s", cnf.AMQP.ExchangeType) log.INFO.Printf(" - BindingKey: %s", cnf.AMQP.BindingKey) log.INFO.Printf(" - PrefetchCount: %d", cnf.AMQP.PrefetchCount) } var signalWG sync.WaitGroup // Goroutine to start broker consumption and handle retries when broker connection dies go func() { for { retry, err := broker.StartConsuming(worker.ConsumerTag, worker.Concurrency, worker) if retry { if worker.errorHandler != nil { worker.errorHandler(err) } else { log.WARNING.Printf("Broker failed with error: %s", err) } } else { signalWG.Wait() errorsChan <- err // stop the goroutine return } } }() if !cnf.NoUnixSignals { sig := make(chan os.Signal, 1) signal.Notify(sig, os.Interrupt, syscall.SIGTERM) var signalsReceived uint // Goroutine Handle SIGINT and SIGTERM signals go func() { for s := range sig { log.WARNING.Printf("Signal received: %v", s) signalsReceived++ if signalsReceived < 2 { // After first Ctrl+C start quitting the worker gracefully log.WARNING.Print("Waiting for running tasks to finish before shutting down") signalWG.Add(1) go func() { worker.Quit() errorsChan <- ErrWorkerQuitGracefully signalWG.Done() }() } else { // Abort the program when user hits Ctrl+C second time in a row errorsChan <- ErrWorkerQuitAbruptly } } }() } } // CustomQueue returns Custom Queue of the running worker process func (worker *Worker) CustomQueue() string { return worker.Queue } // Quit tears down the running worker process func (worker *Worker) Quit() { worker.server.GetBroker().StopConsuming() } // Process handles received tasks and triggers success/error callbacks func (worker *Worker) Process(signature *tasks.Signature) error { // If the task is not registered with this worker, do not continue // but only return nil as we do not want to restart the worker process if !worker.server.IsTaskRegistered(signature.Name) { return nil } taskFunc, err := worker.server.GetRegisteredTask(signature.Name) if err != nil { return nil } // Update task state to RECEIVED if err = worker.server.GetBackend().SetStateReceived(signature); err != nil { return fmt.Errorf("Set state to 'received' for task %s returned error: %s", signature.UUID, err) } // Prepare task for processing task, err := tasks.NewWithSignature(taskFunc, signature) // if this failed, it means the task is malformed, probably has invalid // signature, go directly to task failed without checking whether to retry if err != nil { worker.taskFailed(signature, err) return err } // try to extract trace span from headers and add it to the function context // so it can be used inside the function if it has context.Context as the first // argument. Start a new span if it isn't found. taskSpan := tracing.StartSpanFromHeaders(signature.Headers, signature.Name) tracing.AnnotateSpanWithSignatureInfo(taskSpan, signature) task.Context = opentracing.ContextWithSpan(task.Context, taskSpan) // Update task state to STARTED if err = worker.server.GetBackend().SetStateStarted(signature); err != nil { return fmt.Errorf("Set state to 'started' for task %s returned error: %s", signature.UUID, err) } //Run handler before the task is called if worker.preTaskHandler != nil { worker.preTaskHandler(signature) } //Defer run handler for the end of the task if worker.postTaskHandler != nil { defer worker.postTaskHandler(signature) } // Call the task results, err := task.Call() if err != nil { // If a tasks.ErrRetryTaskLater was returned from the task, // retry the task after specified duration retriableErr, ok := interface{}(err).(tasks.ErrRetryTaskLater) if ok { return worker.retryTaskIn(signature, retriableErr.RetryIn()) } // Otherwise, execute default retry logic based on signature.RetryCount // and signature.RetryTimeout values if signature.RetryCount > 0 { return worker.taskRetry(signature) } return worker.taskFailed(signature, err) } return worker.taskSucceeded(signature, results) } // retryTask decrements RetryCount counter and republishes the task to the queue func (worker *Worker) taskRetry(signature *tasks.Signature) error { // Update task state to RETRY if err := worker.server.GetBackend().SetStateRetry(signature); err != nil { return fmt.Errorf("Set state to 'retry' for task %s returned error: %s", signature.UUID, err) } // Decrement the retry counter, when it reaches 0, we won't retry again signature.RetryCount-- // Increase retry timeout signature.RetryTimeout = retry.FibonacciNext(signature.RetryTimeout) // Delay task by signature.RetryTimeout seconds eta := time.Now().UTC().Add(time.Second * time.Duration(signature.RetryTimeout)) signature.ETA = &eta log.WARNING.Printf("Task %s failed. Going to retry in %d seconds.", signature.UUID, signature.RetryTimeout) // Send the task back to the queue _, err := worker.server.SendTask(signature) return err } // taskRetryIn republishes the task to the queue with ETA of now + retryIn.Seconds() func (worker *Worker) retryTaskIn(signature *tasks.Signature, retryIn time.Duration) error { // Update task state to RETRY if err := worker.server.GetBackend().SetStateRetry(signature); err != nil { return fmt.Errorf("Set state to 'retry' for task %s returned error: %s", signature.UUID, err) } // Delay task by retryIn duration eta := time.Now().UTC().Add(retryIn) signature.ETA = &eta log.WARNING.Printf("Task %s failed. Going to retry in %.0f seconds.", signature.UUID, retryIn.Seconds()) // Send the task back to the queue _, err := worker.server.SendTask(signature) return err } // taskSucceeded updates the task state and triggers success callbacks or a // chord callback if this was the last task of a group with a chord callback func (worker *Worker) taskSucceeded(signature *tasks.Signature, taskResults []*tasks.TaskResult) error { // Update task state to SUCCESS if err := worker.server.GetBackend().SetStateSuccess(signature, taskResults); err != nil { return fmt.Errorf("Set state to 'success' for task %s returned error: %s", signature.UUID, err) } // Log human readable results of the processed task var debugResults = "[]" results, err := tasks.ReflectTaskResults(taskResults) if err != nil { log.WARNING.Print(err) } else { debugResults = tasks.HumanReadableResults(results) } log.DEBUG.Printf("Processed task %s. Results = %s", signature.UUID, debugResults) // Trigger success callbacks for _, successTask := range signature.OnSuccess { if signature.Immutable == false { // Pass results of the task to success callbacks for _, taskResult := range taskResults { successTask.Args = append(successTask.Args, tasks.Arg{ Type: taskResult.Type, Value: taskResult.Value, }) } } worker.server.SendTask(successTask) } // If the task was not part of a group, just return if signature.GroupUUID == "" { return nil } // There is no chord callback, just return if signature.ChordCallback == nil { return nil } // Check if all task in the group has completed groupCompleted, err := worker.server.GetBackend().GroupCompleted( signature.GroupUUID, signature.GroupTaskCount, ) if err != nil { return fmt.Errorf("Completed check for group %s returned error: %s", signature.GroupUUID, err) } // If the group has not yet completed, just return if !groupCompleted { return nil } // Defer purging of group meta queue if we are using AMQP backend if worker.hasAMQPBackend() { defer worker.server.GetBackend().PurgeGroupMeta(signature.GroupUUID) } // Trigger chord callback shouldTrigger, err := worker.server.GetBackend().TriggerChord(signature.GroupUUID) if err != nil { return fmt.Errorf("Triggering chord for group %s returned error: %s", signature.GroupUUID, err) } // Chord has already been triggered if !shouldTrigger { return nil } // Get task states taskStates, err := worker.server.GetBackend().GroupTaskStates( signature.GroupUUID, signature.GroupTaskCount, ) if err != nil { log.ERROR.Printf( "Failed to get tasks states for group:[%s]. Task count:[%d]. The chord may not be triggered. Error:[%s]", signature.GroupUUID, signature.GroupTaskCount, err, ) return nil } // Append group tasks' return values to chord task if it's not immutable for _, taskState := range taskStates { if !taskState.IsSuccess() { return nil } if signature.ChordCallback.Immutable == false { // Pass results of the task to the chord callback for _, taskResult := range taskState.Results { signature.ChordCallback.Args = append(signature.ChordCallback.Args, tasks.Arg{ Type: taskResult.Type, Value: taskResult.Value, }) } } } // Send the chord task _, err = worker.server.SendTask(signature.ChordCallback) if err != nil { return err } return nil } // taskFailed updates the task state and triggers error callbacks func (worker *Worker) taskFailed(signature *tasks.Signature, taskErr error) error { // Update task state to FAILURE if err := worker.server.GetBackend().SetStateFailure(signature, taskErr.Error()); err != nil { return fmt.Errorf("Set state to 'failure' for task %s returned error: %s", signature.UUID, err) } if worker.errorHandler != nil { worker.errorHandler(taskErr) } else { log.ERROR.Printf("Failed processing task %s. Error = %v", signature.UUID, taskErr) } // Trigger error callbacks for _, errorTask := range signature.OnError { // Pass error as a first argument to error callbacks args := append([]tasks.Arg{{ Type: "string", Value: taskErr.Error(), }}, errorTask.Args...) errorTask.Args = args worker.server.SendTask(errorTask) } if signature.StopTaskDeletionOnError { return errs.ErrStopTaskDeletion } return nil } // Returns true if the worker uses AMQP backend func (worker *Worker) hasAMQPBackend() bool { _, ok := worker.server.GetBackend().(*amqp.Backend) return ok } // SetErrorHandler sets a custom error handler for task errors // A default behavior is just to log the error after all the retry attempts fail func (worker *Worker) SetErrorHandler(handler func(err error)) { worker.errorHandler = handler } //SetPreTaskHandler sets a custom handler func before a job is started func (worker *Worker) SetPreTaskHandler(handler func(*tasks.Signature)) { worker.preTaskHandler = handler } //SetPostTaskHandler sets a custom handler for the end of a job func (worker *Worker) SetPostTaskHandler(handler func(*tasks.Signature)) { worker.postTaskHandler = handler } //SetPreConsumeHandler sets a custom handler for the end of a job func (worker *Worker) SetPreConsumeHandler(handler func(*Worker) bool) { worker.preConsumeHandler = handler } //GetServer returns server func (worker *Worker) GetServer() *Server { return worker.server } // func (worker *Worker) PreConsumeHandler() bool { if worker.preConsumeHandler == nil { return true } return worker.preConsumeHandler(worker) } func RedactURL(urlString string) string { u, err := url.Parse(urlString) if err != nil { return urlString } return fmt.Sprintf("%s://%s", u.Scheme, u.Host) } ================================================ FILE: v1/worker_test.go ================================================ package machinery_test import ( "testing" "github.com/stretchr/testify/assert" "github.com/RichardKnop/machinery/v1" ) func TestRedactURL(t *testing.T) { t.Parallel() broker := "amqp://guest:guest@localhost:5672" redactedURL := machinery.RedactURL(broker) assert.Equal(t, "amqp://localhost:5672", redactedURL) } func TestPreConsumeHandler(t *testing.T) { t.Parallel() worker := &machinery.Worker{} worker.SetPreConsumeHandler(SamplePreConsumeHandler) assert.True(t, worker.PreConsumeHandler()) } func SamplePreConsumeHandler(w *machinery.Worker) bool { return true } ================================================ FILE: v2/Dockerfile.test ================================================ # Start from a Debian image with the latest version of Go installed # and a workspace (GOPATH) configured at /go. FROM golang # Set environment variables ENV PATH /go/bin:$PATH # Cd into the source code directory WORKDIR /go/src/github.com/RichardKnop/machinery/v2 # Copy the local package files to the container's workspace. ADD . /go/src/github.com/RichardKnop/machinery/v2 # Run integration tests as default command CMD /go/src/github.com/RichardKnop/machinery/v2/wait-for-it.sh rabbitmq:5672 -- make test-with-coverage ================================================ FILE: v2/Makefile ================================================ .PHONY: fmt lint golint test test-with-coverage ci # TODO: When Go 1.9 is released vendor folder should be ignored automatically PACKAGES=`go list ./... | grep -v vendor | grep -v mocks` fmt: for pkg in ${PACKAGES}; do \ go fmt $$pkg; \ done; lint: gometalinter --tests --disable-all --deadline=120s -E vet -E gofmt -E misspell -E ineffassign -E goimports -E deadcode ./... golint: for pkg in ${PACKAGES}; do \ golint -set_exit_status $$pkg || GOLINT_FAILED=1; \ done; \ [ -z "$$GOLINT_FAILED" ] test: TEST_FAILED= ; \ for pkg in ${PACKAGES}; do \ go test $$pkg || TEST_FAILED=1; \ done; \ [ -z "$$TEST_FAILED" ] test-with-coverage: echo "" > coverage.out echo "mode: set" > coverage-all.out TEST_FAILED= ; \ for pkg in ${PACKAGES}; do \ go test -coverprofile=coverage.out -covermode=set $$pkg || TEST_FAILED=1; \ tail -n +2 coverage.out >> coverage-all.out; \ done; \ [ -z "$$TEST_FAILED" ] #go tool cover -html=coverage-all.out ci: bash -c 'docker-compose -f docker-compose.test.yml -p machinery_ci up --build --abort-on-container-exit --exit-code-from sut' ================================================ FILE: v2/backends/amqp/amqp.go ================================================ package amqp // NOTE: Using AMQP as a result backend is quite tricky since every time we // read a message from the queue keeping task states, the message is removed // from the queue. This leads to problems with keeping a reliable state of a // group of tasks since concurrent processes updating the group state cause // race conditions and inconsistent state. // // This is avoided by a "clever" hack. A special queue identified by a group // UUID is created and we store serialised TaskState objects of successfully // completed tasks. By inspecting the queue we can then say: // 1) If all group tasks finished (number of unacked messages = group task count) // 2) If all group tasks finished AND succeeded (by consuming the queue) // // It is important to consume the queue exclusively to avoid race conditions. import ( "bytes" "encoding/json" "errors" "fmt" "github.com/RichardKnop/machinery/v2/backends/iface" "github.com/RichardKnop/machinery/v2/common" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/log" "github.com/RichardKnop/machinery/v2/tasks" amqp "github.com/rabbitmq/amqp091-go" ) // Backend represents an AMQP result backend type Backend struct { common.Backend common.AMQPConnector } // New creates Backend instance func New(cnf *config.Config) iface.Backend { return &Backend{Backend: common.NewBackend(cnf), AMQPConnector: common.AMQPConnector{}} } // InitGroup creates and saves a group meta data object func (b *Backend) InitGroup(groupUUID string, taskUUIDs []string) error { return nil } // GroupCompleted returns true if all tasks in a group finished // NOTE: Given AMQP limitation this will only return true if all finished // tasks were successful as we do not keep track of completed failed tasks func (b *Backend) GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) { conn, channel, err := b.Open(b.GetConfig().ResultBackend, b.GetConfig().TLSConfig) if err != nil { return false, err } defer b.Close(channel, conn) queueState, err := b.InspectQueue(channel, groupUUID) if err != nil { return false, nil } return queueState.Messages == groupTaskCount, nil } // GroupTaskStates returns states of all tasks in the group func (b *Backend) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error) { conn, channel, err := b.Open(b.GetConfig().ResultBackend, b.GetConfig().TLSConfig) if err != nil { return nil, err } defer b.Close(channel, conn) queueState, err := b.InspectQueue(channel, groupUUID) if err != nil { return nil, err } if queueState.Messages != groupTaskCount { return nil, fmt.Errorf("Already consumed: %v", err) } deliveries, err := channel.Consume( groupUUID, // queue name "", // consumer tag false, // auto-ack true, // exclusive false, // no-local false, // no-wait nil, // arguments ) if err != nil { return nil, fmt.Errorf("Queue consume error: %s", err) } states := make([]*tasks.TaskState, groupTaskCount) for i := 0; i < groupTaskCount; i++ { d := <-deliveries state := new(tasks.TaskState) decoder := json.NewDecoder(bytes.NewReader([]byte(d.Body))) decoder.UseNumber() if err := decoder.Decode(state); err != nil { d.Nack(false, false) // multiple, requeue return nil, err } d.Ack(false) // multiple states[i] = state } return states, nil } // TriggerChord flags chord as triggered in the backend storage to make sure // chord is never trigerred multiple times. Returns a boolean flag to indicate // whether the worker should trigger chord (true) or no if it has been triggered // already (false) func (b *Backend) TriggerChord(groupUUID string) (bool, error) { conn, channel, err := b.Open(b.GetConfig().ResultBackend, b.GetConfig().TLSConfig) if err != nil { return false, err } defer b.Close(channel, conn) _, err = b.InspectQueue(channel, amqmChordTriggeredQueue(groupUUID)) if err != nil { return true, nil } return false, nil } // SetStatePending updates task state to PENDING func (b *Backend) SetStatePending(signature *tasks.Signature) error { taskState := tasks.NewPendingTaskState(signature) return b.updateState(taskState) } // SetStateReceived updates task state to RECEIVED func (b *Backend) SetStateReceived(signature *tasks.Signature) error { taskState := tasks.NewReceivedTaskState(signature) return b.updateState(taskState) } // SetStateStarted updates task state to STARTED func (b *Backend) SetStateStarted(signature *tasks.Signature) error { taskState := tasks.NewStartedTaskState(signature) return b.updateState(taskState) } // SetStateRetry updates task state to RETRY func (b *Backend) SetStateRetry(signature *tasks.Signature) error { state := tasks.NewRetryTaskState(signature) return b.updateState(state) } // SetStateSuccess updates task state to SUCCESS func (b *Backend) SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error { taskState := tasks.NewSuccessTaskState(signature, results) if err := b.updateState(taskState); err != nil { return err } if signature.GroupUUID == "" { return nil } return b.markTaskCompleted(signature, taskState) } // SetStateFailure updates task state to FAILURE func (b *Backend) SetStateFailure(signature *tasks.Signature, err string) error { taskState := tasks.NewFailureTaskState(signature, err) if err := b.updateState(taskState); err != nil { return err } if signature.GroupUUID == "" { return nil } return b.markTaskCompleted(signature, taskState) } // GetState returns the latest task state. It will only return the status once // as the message will get consumed and removed from the queue. func (b *Backend) GetState(taskUUID string) (*tasks.TaskState, error) { declareQueueArgs := amqp.Table{ // Time in milliseconds // after that message will expire "x-message-ttl": int32(b.getExpiresIn()), // Time after that the queue will be deleted. "x-expires": int32(b.getExpiresIn()), } conn, channel, _, _, _, err := b.Connect( b.GetConfig().ResultBackend, "", b.GetConfig().TLSConfig, b.GetConfig().AMQP.Exchange, // exchange name b.GetConfig().AMQP.ExchangeType, // exchange type taskUUID, // queue name false, // queue durable true, // queue delete when unused taskUUID, // queue binding key nil, // exchange declare args declareQueueArgs, // queue declare args nil, // queue binding args ) if err != nil { return nil, err } defer b.Close(channel, conn) d, ok, err := channel.Get( taskUUID, // queue name false, // multiple ) if err != nil { return nil, err } if !ok { return nil, errors.New("No state ready") } d.Ack(false) state := new(tasks.TaskState) decoder := json.NewDecoder(bytes.NewReader([]byte(d.Body))) decoder.UseNumber() if err := decoder.Decode(state); err != nil { log.ERROR.Printf("Failed to unmarshal task state: %s", string(d.Body)) log.ERROR.Print(err) return nil, err } return state, nil } // PurgeState deletes stored task state func (b *Backend) PurgeState(taskUUID string) error { conn, channel, err := b.Open(b.GetConfig().ResultBackend, b.GetConfig().TLSConfig) if err != nil { return err } defer b.Close(channel, conn) return b.DeleteQueue(channel, taskUUID) } // PurgeGroupMeta deletes stored group meta data func (b *Backend) PurgeGroupMeta(groupUUID string) error { conn, channel, err := b.Open(b.GetConfig().ResultBackend, b.GetConfig().TLSConfig) if err != nil { return err } defer b.Close(channel, conn) b.DeleteQueue(channel, amqmChordTriggeredQueue(groupUUID)) return b.DeleteQueue(channel, groupUUID) } // updateState saves current task state func (b *Backend) updateState(taskState *tasks.TaskState) error { message, err := json.Marshal(taskState) if err != nil { return fmt.Errorf("JSON marshal error: %s", err) } declareQueueArgs := amqp.Table{ // Time in milliseconds // after that message will expire "x-message-ttl": int32(b.getExpiresIn()), // Time after that the queue will be deleted. "x-expires": int32(b.getExpiresIn()), } conn, channel, queue, confirmsChan, _, err := b.Connect( b.GetConfig().ResultBackend, "", b.GetConfig().TLSConfig, b.GetConfig().AMQP.Exchange, // exchange name b.GetConfig().AMQP.ExchangeType, // exchange type taskState.TaskUUID, // queue name false, // queue durable true, // queue delete when unused taskState.TaskUUID, // queue binding key nil, // exchange declare args declareQueueArgs, // queue declare args nil, // queue binding args ) if err != nil { return err } defer b.Close(channel, conn) if err := channel.Publish( b.GetConfig().AMQP.Exchange, // exchange queue.Name, // routing key false, // mandatory false, // immediate amqp.Publishing{ ContentType: "application/json", Body: message, DeliveryMode: amqp.Persistent, // Persistent // Transient }, ); err != nil { return err } confirmed := <-confirmsChan if confirmed.Ack { return nil } return fmt.Errorf("Failed delivery of delivery tag: %d", confirmed.DeliveryTag) } // getExpiresIn returns expiration time func (b *Backend) getExpiresIn() int { resultsExpireIn := b.GetConfig().ResultsExpireIn * 1000 if resultsExpireIn == 0 { // // expire results after 1 hour by default resultsExpireIn = config.DefaultResultsExpireIn * 1000 } return resultsExpireIn } // markTaskCompleted marks task as completed in either groupdUUID_success // or groupUUID_failure queue. This is important for GroupCompleted and // GroupSuccessful methods func (b *Backend) markTaskCompleted(signature *tasks.Signature, taskState *tasks.TaskState) error { if signature.GroupUUID == "" || signature.GroupTaskCount == 0 { return nil } message, err := json.Marshal(taskState) if err != nil { return fmt.Errorf("JSON marshal error: %s", err) } declareQueueArgs := amqp.Table{ // Time in milliseconds // after that message will expire "x-message-ttl": int32(b.getExpiresIn()), // Time after that the queue will be deleted. "x-expires": int32(b.getExpiresIn()), } conn, channel, queue, confirmsChan, _, err := b.Connect( b.GetConfig().ResultBackend, "", b.GetConfig().TLSConfig, b.GetConfig().AMQP.Exchange, // exchange name b.GetConfig().AMQP.ExchangeType, // exchange type signature.GroupUUID, // queue name false, // queue durable true, // queue delete when unused signature.GroupUUID, // queue binding key nil, // exchange declare args declareQueueArgs, // queue declare args nil, // queue binding args ) if err != nil { return err } defer b.Close(channel, conn) if err := channel.Publish( b.GetConfig().AMQP.Exchange, // exchange queue.Name, // routing key false, // mandatory false, // immediate amqp.Publishing{ ContentType: "application/json", Body: message, DeliveryMode: amqp.Persistent, // Persistent // Transient }, ); err != nil { return err } confirmed := <-confirmsChan if !confirmed.Ack { return fmt.Errorf("Failed delivery of delivery tag: %v", confirmed.DeliveryTag) } return nil } func amqmChordTriggeredQueue(groupUUID string) string { return fmt.Sprintf("%s_chord_triggered", groupUUID) } ================================================ FILE: v2/backends/amqp/amqp_test.go ================================================ package amqp_test import ( "os" "testing" "time" "github.com/RichardKnop/machinery/v2/backends/amqp" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/tasks" "github.com/stretchr/testify/assert" ) var ( amqpConfig *config.Config ) func init() { amqpURL := os.Getenv("AMQP_URL") if amqpURL == "" { return } finalAmqpURL := amqpURL var finalSeparator string amqpURLs := os.Getenv("AMQP_URLS") if amqpURLs != "" { separator := os.Getenv("AMQP_URLS_SEPARATOR") if separator == "" { return } finalSeparator = separator finalAmqpURL = amqpURLs } amqp2URL := os.Getenv("AMQP2_URL") if amqp2URL == "" { amqp2URL = amqpURL } amqpConfig = &config.Config{ Broker: finalAmqpURL, MultipleBrokerSeparator: finalSeparator, DefaultQueue: "test_queue", ResultBackend: amqp2URL, AMQP: &config.AMQPConfig{ Exchange: "test_exchange", ExchangeType: "direct", BindingKey: "test_task", PrefetchCount: 1, }, } } func TestGroupCompleted(t *testing.T) { if os.Getenv("AMQP_URL") == "" { t.Skip("AMQP_URL is not defined") } groupUUID := "testGroupUUID" groupTaskCount := 2 task1 := &tasks.Signature{ UUID: "testTaskUUID1", GroupUUID: groupUUID, GroupTaskCount: groupTaskCount, } task2 := &tasks.Signature{ UUID: "testTaskUUID2", GroupUUID: groupUUID, GroupTaskCount: groupTaskCount, } backend := amqp.New(amqpConfig) // Cleanup before the test backend.PurgeState(task1.UUID) backend.PurgeState(task2.UUID) backend.PurgeGroupMeta(groupUUID) groupCompleted, err := backend.GroupCompleted(groupUUID, groupTaskCount) if assert.NoError(t, err) { assert.False(t, groupCompleted) } backend.InitGroup(groupUUID, []string{task1.UUID, task2.UUID}) groupCompleted, err = backend.GroupCompleted(groupUUID, groupTaskCount) if assert.NoError(t, err) { assert.False(t, groupCompleted) } backend.SetStatePending(task1) backend.SetStateStarted(task2) groupCompleted, err = backend.GroupCompleted(groupUUID, groupTaskCount) if assert.NoError(t, err) { assert.False(t, groupCompleted) } taskResults := []*tasks.TaskResult{new(tasks.TaskResult)} backend.SetStateSuccess(task1, taskResults) backend.SetStateSuccess(task2, taskResults) groupCompleted, err = backend.GroupCompleted(groupUUID, groupTaskCount) if assert.NoError(t, err) { assert.True(t, groupCompleted) } } func TestGetState(t *testing.T) { if os.Getenv("AMQP_URL") == "" { t.Skip("AMQP_URL is not defined") } signature := &tasks.Signature{ UUID: "testTaskUUID", GroupUUID: "testGroupUUID", } go func() { backend := amqp.New(amqpConfig) backend.SetStatePending(signature) time.Sleep(2 * time.Millisecond) backend.SetStateReceived(signature) time.Sleep(2 * time.Millisecond) backend.SetStateStarted(signature) time.Sleep(2 * time.Millisecond) taskResults := []*tasks.TaskResult{ { Type: "float64", Value: 2, }, } backend.SetStateSuccess(signature, taskResults) }() backend := amqp.New(amqpConfig) var ( taskState *tasks.TaskState err error ) for { taskState, err = backend.GetState(signature.UUID) if taskState == nil { assert.Equal(t, "No state ready", err.Error()) continue } assert.NoError(t, err) if taskState.IsCompleted() { break } } } func TestPurgeState(t *testing.T) { if os.Getenv("AMQP_URL") == "" { t.Skip("AMQP_URL is not defined") } signature := &tasks.Signature{ UUID: "testTaskUUID", GroupUUID: "testGroupUUID", } backend := amqp.New(amqpConfig) backend.SetStatePending(signature) backend.SetStateReceived(signature) taskState, err := backend.GetState(signature.UUID) assert.NotNil(t, taskState) assert.NoError(t, err) backend.PurgeState(taskState.TaskUUID) taskState, err = backend.GetState(signature.UUID) assert.Nil(t, taskState) assert.Error(t, err) } ================================================ FILE: v2/backends/dynamodb/dynamodb.go ================================================ package dynamodb import ( "context" "errors" "fmt" "math" "time" "github.com/RichardKnop/machinery/v2/backends/iface" dynamodbiface "github.com/RichardKnop/machinery/v2/backends/iface/dynamodb" "github.com/RichardKnop/machinery/v2/common" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/log" "github.com/RichardKnop/machinery/v2/tasks" "github.com/aws/aws-sdk-go-v2/aws" awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue" "github.com/aws/aws-sdk-go-v2/service/dynamodb" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" ) const ( BatchItemsLimit = 99 MaxFetchAttempts = 3 ) // Backend ... type Backend struct { common.Backend cnf *config.Config client dynamodbiface.API } // New creates a Backend instance func New(cnf *config.Config) (iface.Backend, error) { backend := &Backend{Backend: common.NewBackend(cnf), cnf: cnf} if cnf.DynamoDB != nil && cnf.DynamoDB.Client != nil { backend.client = cnf.DynamoDB.Client } else { cfg, err := awsconfig.LoadDefaultConfig(context.TODO()) if err != nil { return nil, fmt.Errorf("%w: unable to load AWS SDK config: ", err) } backend.client = dynamodb.NewFromConfig(cfg) } // Check if needed tables exist err := backend.checkRequiredTablesIfExist() if err != nil { log.FATAL.Printf("Failed to prepare tables. Error: %v", err) } return backend, nil } // InitGroup ... func (b *Backend) InitGroup(groupUUID string, taskUUIDs []string) error { meta := tasks.GroupMeta{ GroupUUID: groupUUID, TaskUUIDs: taskUUIDs, CreatedAt: time.Now().UTC(), TTL: b.getExpirationTime(), } av, err := attributevalue.MarshalMap(meta) if err != nil { log.ERROR.Printf("Error when marshaling Dynamodb attributes. Err: %v", err) return err } input := &dynamodb.PutItemInput{ Item: av, TableName: aws.String(b.cnf.DynamoDB.GroupMetasTable), } _, err = b.client.PutItem(context.TODO(), input) if err != nil { log.ERROR.Printf("Got error when calling PutItem: %v; Error: %v", input, err) return err } return nil } // GroupCompleted ... func (b *Backend) GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) { groupMeta, err := b.getGroupMeta(groupUUID) if err != nil { return false, err } taskStates, err := b.getStates(groupMeta.TaskUUIDs) if err != nil { return false, err } var countSuccessTasks = 0 for _, taskState := range taskStates { if taskState.IsCompleted() { countSuccessTasks++ } } return countSuccessTasks == groupTaskCount, nil } // GroupTaskStates ... func (b *Backend) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error) { groupMeta, err := b.getGroupMeta(groupUUID) if err != nil { return nil, err } return b.getStates(groupMeta.TaskUUIDs) } // TriggerChord ... func (b *Backend) TriggerChord(groupUUID string) (bool, error) { // Get the group meta data groupMeta, err := b.getGroupMeta(groupUUID) if err != nil { return false, err } // Chord has already been triggered, return false (should not trigger again) if groupMeta.ChordTriggered { return false, nil } // If group meta is locked, wait until it's unlocked for groupMeta.Lock { groupMeta, _ = b.getGroupMeta(groupUUID) log.WARNING.Printf("Group [%s] locked, waiting", groupUUID) time.Sleep(time.Millisecond * 5) } // Acquire lock if err = b.lockGroupMeta(groupUUID); err != nil { return false, err } defer b.unlockGroupMeta(groupUUID) // update group meta data err = b.chordTriggered(groupUUID) if err != nil { return false, err } return true, err } // SetStatePending ... func (b *Backend) SetStatePending(signature *tasks.Signature) error { taskState := tasks.NewPendingTaskState(signature) // taskUUID is the primary key of the table, so a new task need to be created first, instead of using dynamodb.UpdateItemInput directly return b.initTaskState(taskState) } // SetStateReceived ... func (b *Backend) SetStateReceived(signature *tasks.Signature) error { taskState := tasks.NewReceivedTaskState(signature) return b.setTaskState(taskState) } // SetStateStarted ... func (b *Backend) SetStateStarted(signature *tasks.Signature) error { taskState := tasks.NewStartedTaskState(signature) return b.setTaskState(taskState) } // SetStateRetry ... func (b *Backend) SetStateRetry(signature *tasks.Signature) error { taskState := tasks.NewRetryTaskState(signature) return b.setTaskState(taskState) } // SetStateSuccess ... func (b *Backend) SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error { taskState := tasks.NewSuccessTaskState(signature, results) taskState.TTL = b.getExpirationTime() return b.setTaskState(taskState) } // SetStateFailure ... func (b *Backend) SetStateFailure(signature *tasks.Signature, err string) error { taskState := tasks.NewFailureTaskState(signature, err) taskState.TTL = b.getExpirationTime() return b.updateToFailureStateWithError(taskState) } // GetState ... func (b *Backend) GetState(taskUUID string) (*tasks.TaskState, error) { result, err := b.client.GetItem(context.TODO(), &dynamodb.GetItemInput{ TableName: aws.String(b.cnf.DynamoDB.TaskStatesTable), Key: map[string]types.AttributeValue{ "TaskUUID": &types.AttributeValueMemberS{ Value: taskUUID, }, }, ConsistentRead: aws.Bool(true), }) if err != nil { return nil, err } return b.unmarshalTaskStateGetItemResult(result) } // getStates returns the current states for the given list of tasks. // It uses batch fetch API. If any keys fail to fetch, it'll retry with exponential backoff until maxFetchAttempts times. func (b *Backend) getStates(tasksToFetch []string) ([]*tasks.TaskState, error) { fetchedTaskStates := make([]*tasks.TaskState, 0, len(tasksToFetch)) var unfetchedTaskIDs []string // try until all keys are fetched or until we run out of attempts. for attempt := 0; len(tasksToFetch) > 0 && attempt < MaxFetchAttempts; attempt++ { unfetchedTaskIDs = nil for _, batch := range chunkTasks(tasksToFetch, BatchItemsLimit) { fetched, unfetched, err := b.batchFetchTaskStates(batch) if err != nil { return nil, err } fetchedTaskStates = append(fetchedTaskStates, fetched...) unfetchedTaskIDs = append(unfetchedTaskIDs, unfetched...) } tasksToFetch = unfetchedTaskIDs // Check if there were any tasks that were not fetched. If so, retry with exponential backoff. if len(unfetchedTaskIDs) > 0 { backoffDuration := time.Duration(math.Pow(2, float64(attempt))) * time.Second log.DEBUG.Printf("Unable to fetch [%d] keys on attempt [%d]. Sleeping for [%s]", len(unfetchedTaskIDs), attempt+1, backoffDuration) time.Sleep(backoffDuration) } } if len(unfetchedTaskIDs) > 0 { return nil, fmt.Errorf("Failed to fetch [%d] keys even after retries: [%+v]", len(unfetchedTaskIDs), unfetchedTaskIDs) } return fetchedTaskStates, nil } // batchFetchTaskStates returns the current states of the given tasks by fetching them all in a single batched API. // DynamoDB's BatchGetItem() can return partial results. If there are any unfetched keys, they are returned as second // return value so that the caller can retry those keys. // https://docs.aws.amazon.com/sdk-for-go/api/service/dynamodb/#DynamoDB.BatchGetItem func (b *Backend) batchFetchTaskStates(taskUUIDs []string) ([]*tasks.TaskState, []string, error) { tableName := b.cnf.DynamoDB.TaskStatesTable keys := make([]map[string]types.AttributeValue, len(taskUUIDs)) for i, tid := range taskUUIDs { keys[i] = map[string]types.AttributeValue{ "TaskUUID": &types.AttributeValueMemberS{ Value: tid, }, } } input := &dynamodb.BatchGetItemInput{ RequestItems: map[string]types.KeysAndAttributes{ tableName: { ConsistentRead: aws.Bool(true), Keys: keys, }, }, } result, err := b.client.BatchGetItem(context.TODO(), input) if err != nil { return nil, nil, fmt.Errorf("BatchGetItem failed. Error: [%s]", err) } fetchedKeys, ok := result.Responses[tableName] if !ok { return nil, nil, fmt.Errorf("no keys returned from the table: [%s]", tableName) } states := []*tasks.TaskState{} if err := attributevalue.UnmarshalListOfMaps(fetchedKeys, &states); err != nil { return nil, nil, fmt.Errorf("Got error when unmarshal map. Error: %v", err) } // Look for any unprocessed keys var unfetchedKeys []string if _, ok = result.UnprocessedKeys[tableName]; !ok { unfetchedKeys, err = getUnfetchedKeys(result.UnprocessedKeys[tableName]) if err != nil { return nil, nil, fmt.Errorf("unable to fetch some keys: [%+v]. Error: [%s]", result.UnprocessedKeys, err) } } return states, unfetchedKeys, nil } // PurgeState ... func (b *Backend) PurgeState(taskUUID string) error { input := &dynamodb.DeleteItemInput{ Key: map[string]types.AttributeValue{ "TaskUUID": &types.AttributeValueMemberS{ Value: taskUUID, }, }, TableName: aws.String(b.cnf.DynamoDB.TaskStatesTable), } _, err := b.client.DeleteItem(context.TODO(), input) if err != nil { return err } return nil } // PurgeGroupMeta ... func (b *Backend) PurgeGroupMeta(groupUUID string) error { input := &dynamodb.DeleteItemInput{ Key: map[string]types.AttributeValue{ "GroupUUID": &types.AttributeValueMemberS{ Value: groupUUID, }, }, TableName: aws.String(b.cnf.DynamoDB.GroupMetasTable), } _, err := b.client.DeleteItem(context.TODO(), input) if err != nil { return err } return nil } func (b *Backend) getGroupMeta(groupUUID string) (*tasks.GroupMeta, error) { result, err := b.client.GetItem(context.TODO(), &dynamodb.GetItemInput{ TableName: aws.String(b.cnf.DynamoDB.GroupMetasTable), Key: map[string]types.AttributeValue{ "GroupUUID": &types.AttributeValueMemberS{ Value: groupUUID, }, }, ConsistentRead: aws.Bool(true), }) if err != nil { log.ERROR.Printf("Error when getting group [%s]. Error: [%s]", groupUUID, err) return nil, err } item, err := b.unmarshalGroupMetaGetItemResult(result) if err != nil { log.ERROR.Printf("Failed to unmarshal item. Error: [%s], Result: [%+v]", err, result) return nil, err } return item, nil } func (b *Backend) lockGroupMeta(groupUUID string) error { err := b.updateGroupMetaLock(groupUUID, true) if err != nil { return err } return nil } func (b *Backend) unlockGroupMeta(groupUUID string) error { err := b.updateGroupMetaLock(groupUUID, false) if err != nil { return err } return nil } func (b *Backend) updateGroupMetaLock(groupUUID string, status bool) error { input := &dynamodb.UpdateItemInput{ ExpressionAttributeNames: map[string]string{ "#L": "Lock", }, ExpressionAttributeValues: map[string]types.AttributeValue{ ":l": &types.AttributeValueMemberBOOL{ Value: status, }, }, Key: map[string]types.AttributeValue{ "GroupUUID": &types.AttributeValueMemberS{ Value: groupUUID, }, }, ReturnValues: types.ReturnValueUpdatedNew, TableName: aws.String(b.cnf.DynamoDB.GroupMetasTable), UpdateExpression: aws.String("SET #L = :l"), } _, err := b.client.UpdateItem(context.TODO(), input) if err != nil { return err } return nil } func (b *Backend) chordTriggered(groupUUID string) error { input := &dynamodb.UpdateItemInput{ ExpressionAttributeNames: map[string]string{ "#CT": "ChordTriggered", }, ExpressionAttributeValues: map[string]types.AttributeValue{ ":ct": &types.AttributeValueMemberBOOL{ Value: true, }, }, Key: map[string]types.AttributeValue{ "GroupUUID": &types.AttributeValueMemberS{ Value: groupUUID, }, }, ReturnValues: types.ReturnValueUpdatedNew, TableName: aws.String(b.cnf.DynamoDB.GroupMetasTable), UpdateExpression: aws.String("SET #CT = :ct"), } _, err := b.client.UpdateItem(context.TODO(), input) if err != nil { return err } return nil } func (b *Backend) setTaskState(taskState *tasks.TaskState) error { expAttributeNames := map[string]string{ "#S": "State", } expAttributeValues := map[string]types.AttributeValue{ ":s": &types.AttributeValueMemberS{ Value: taskState.State, }, } keyAttributeValues := map[string]types.AttributeValue{ "TaskUUID": &types.AttributeValueMemberS{ Value: taskState.TaskUUID, }, } exp := "SET #S = :s" if !taskState.CreatedAt.IsZero() { expAttributeNames["#C"] = "CreatedAt" expAttributeValues[":c"] = &types.AttributeValueMemberS{ Value: taskState.CreatedAt.String(), } exp += ", #C = :c" } if taskState.TTL > 0 { expAttributeNames["#T"] = "TTL" expAttributeValues[":t"] = &types.AttributeValueMemberN{ Value: fmt.Sprintf("%d", taskState.TTL), } exp += ", #T = :t" } if taskState.Results != nil && len(taskState.Results) != 0 { expAttributeNames["#R"] = "Results" var results []types.AttributeValue for _, r := range taskState.Results { avMap := map[string]types.AttributeValue{ "Type": &types.AttributeValueMemberS{ Value: r.Type, }, "Value": &types.AttributeValueMemberS{ Value: fmt.Sprintf("%v", r.Value), }, } rs := &types.AttributeValueMemberM{ Value: avMap, } results = append(results, rs) } expAttributeValues[":r"] = &types.AttributeValueMemberL{ Value: results, } exp += ", #R = :r" } input := &dynamodb.UpdateItemInput{ ExpressionAttributeNames: expAttributeNames, ExpressionAttributeValues: expAttributeValues, Key: keyAttributeValues, ReturnValues: types.ReturnValueUpdatedNew, TableName: aws.String(b.cnf.DynamoDB.TaskStatesTable), UpdateExpression: aws.String(exp), } _, err := b.client.UpdateItem(context.TODO(), input) if err != nil { return err } return nil } func (b *Backend) initTaskState(taskState *tasks.TaskState) error { av, err := attributevalue.MarshalMap(taskState) input := &dynamodb.PutItemInput{ Item: av, TableName: aws.String(b.cnf.DynamoDB.TaskStatesTable), } if err != nil { return err } _, err = b.client.PutItem(context.TODO(), input) if err != nil { return err } return nil } func (b *Backend) updateToFailureStateWithError(taskState *tasks.TaskState) error { input := &dynamodb.UpdateItemInput{ ExpressionAttributeNames: map[string]string{ "#S": "State", "#E": "Error", }, ExpressionAttributeValues: map[string]types.AttributeValue{ ":s": &types.AttributeValueMemberS{ Value: taskState.State, }, ":e": &types.AttributeValueMemberS{ Value: taskState.Error, }, }, Key: map[string]types.AttributeValue{ "TaskUUID": &types.AttributeValueMemberS{ Value: taskState.TaskUUID, }, }, ReturnValues: types.ReturnValueUpdatedNew, TableName: aws.String(b.cnf.DynamoDB.TaskStatesTable), UpdateExpression: aws.String("SET #S = :s, #E = :e"), } if taskState.TTL > 0 { input.ExpressionAttributeNames["#T"] = "TTL" input.ExpressionAttributeValues[":t"] = &types.AttributeValueMemberN{ Value: fmt.Sprintf("%d", taskState.TTL), } input.UpdateExpression = aws.String(*input.UpdateExpression + ", #T = :t") } _, err := b.client.UpdateItem(context.TODO(), input) if err != nil { return err } return nil } func (b *Backend) unmarshalGroupMetaGetItemResult(result *dynamodb.GetItemOutput) (*tasks.GroupMeta, error) { if result == nil { err := errors.New("task state is nil") log.ERROR.Printf("Got error when unmarshal map. Error: %v", err) return nil, err } item := tasks.GroupMeta{} err := attributevalue.UnmarshalMap(result.Item, &item) if err != nil { log.ERROR.Printf("Got error when unmarshal map. Error: %v", err) return nil, err } return &item, err } func (b *Backend) unmarshalTaskStateGetItemResult(result *dynamodb.GetItemOutput) (*tasks.TaskState, error) { if result == nil { err := errors.New("task state is nil") log.ERROR.Printf("Got error when unmarshal map. Error: %v", err) return nil, err } state := tasks.TaskState{} err := attributevalue.UnmarshalMap(result.Item, &state) if err != nil { log.ERROR.Printf("Got error when unmarshal map. Error: %v", err) return nil, err } return &state, nil } func (b *Backend) checkRequiredTablesIfExist() error { var ( taskTableName = b.cnf.DynamoDB.TaskStatesTable groupTableName = b.cnf.DynamoDB.GroupMetasTable tableNames []string startFromTable *string ) for { result, err := b.client.ListTables(context.TODO(), &dynamodb.ListTablesInput{ ExclusiveStartTableName: startFromTable, }) if err != nil { return err } tableNames = append(tableNames, result.TableNames...) if result.LastEvaluatedTableName == nil { break } startFromTable = result.LastEvaluatedTableName } if !b.tableExists(taskTableName, tableNames) { return errors.New("task table doesn't exist") } if !b.tableExists(groupTableName, tableNames) { return errors.New("group table doesn't exist") } return nil } func (b *Backend) tableExists(tableName string, tableNames []string) bool { for _, t := range tableNames { if tableName == t { return true } } return false } func (b *Backend) getExpirationTime() int64 { expiresIn := b.GetConfig().ResultsExpireIn if expiresIn == 0 { // expire results after 1 hour by default expiresIn = config.DefaultResultsExpireIn } return time.Now().Add(time.Second * time.Duration(expiresIn)).Unix() } // getUnfetchedKeys returns keys that were not fetched in a batch request. func getUnfetchedKeys(unprocessed types.KeysAndAttributes) ([]string, error) { states := []*tasks.TaskState{} var taskIDs []string if err := attributevalue.UnmarshalListOfMaps(unprocessed.Keys, &states); err != nil { return nil, fmt.Errorf("Got error when unmarshal map. Error: %v", err) } for _, s := range states { taskIDs = append(taskIDs, s.TaskUUID) } return taskIDs, nil } // chunkTasks chunks the list of strings into multiple smaller lists of specified size. func chunkTasks(array []string, chunkSize int) [][]string { var result [][]string for len(array) > 0 { sz := min(len(array), chunkSize) chunk := array[:sz] array = array[sz:] result = append(result, chunk) } return result } func min(a, b int) int { if a < b { return a } return b } ================================================ FILE: v2/backends/dynamodb/dynamodb_export_test.go ================================================ package dynamodb import ( "context" "errors" "os" dynamodbiface "github.com/RichardKnop/machinery/v2/backends/iface/dynamodb" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/tasks" "github.com/aws/aws-sdk-go-v2/service/dynamodb" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" ) var ( TestDynamoDBBackend *Backend TestErrDynamoDBBackend *Backend TestCnf *config.Config TestDBClient dynamodbiface.API TestErrDBClient dynamodbiface.API TestGroupMeta *tasks.GroupMeta TestTask1 map[string]types.AttributeValue TestTask2 map[string]types.AttributeValue TestTask3 map[string]types.AttributeValue ) type TestDynamoDBClient struct { dynamodbiface.API PutItemOverride func(context.Context, *dynamodb.PutItemInput, ...func(*dynamodb.Options)) (*dynamodb.PutItemOutput, error) UpdateItemOverride func(context.Context, *dynamodb.UpdateItemInput, ...func(*dynamodb.Options)) (*dynamodb.UpdateItemOutput, error) GetItemOverride func(ctx context.Context, input *dynamodb.GetItemInput, ops ...func(*dynamodb.Options)) (*dynamodb.GetItemOutput, error) BatchGetItemOverride func(context.Context, *dynamodb.BatchGetItemInput, ...func(*dynamodb.Options)) (*dynamodb.BatchGetItemOutput, error) } func (t *TestDynamoDBClient) ResetOverrides() { t.PutItemOverride = nil t.UpdateItemOverride = nil t.BatchGetItemOverride = nil } func (t *TestDynamoDBClient) PutItem(ctx context.Context, input *dynamodb.PutItemInput, ops ...func(*dynamodb.Options)) (*dynamodb.PutItemOutput, error) { if t.PutItemOverride != nil { return t.PutItemOverride(ctx, input, ops...) } return &dynamodb.PutItemOutput{}, nil } func (t *TestDynamoDBClient) BatchGetItem(ctx context.Context, input *dynamodb.BatchGetItemInput, ops ...func(*dynamodb.Options)) (*dynamodb.BatchGetItemOutput, error) { if t.BatchGetItemOverride != nil { return t.BatchGetItemOverride(ctx, input, ops...) } return &dynamodb.BatchGetItemOutput{}, nil } func (t *TestDynamoDBClient) GetItem(ctx context.Context, input *dynamodb.GetItemInput, ops ...func(*dynamodb.Options)) (*dynamodb.GetItemOutput, error) { if t.GetItemOverride != nil { return t.GetItemOverride(ctx, input, ops...) } var output *dynamodb.GetItemOutput switch *input.TableName { case "group_metas": output = &dynamodb.GetItemOutput{ Item: map[string]types.AttributeValue{ "TaskUUIDs": &types.AttributeValueMemberL{ Value: []types.AttributeValue{ &types.AttributeValueMemberS{ Value: "testTaskUUID1", }, &types.AttributeValueMemberS{ Value: "testTaskUUID2", }, &types.AttributeValueMemberS{ Value: "testTaskUUID3", }, }, }, "ChordTriggered": &types.AttributeValueMemberBOOL{ Value: false, }, "GroupUUID": &types.AttributeValueMemberS{ Value: "testGroupUUID", }, "Lock": &types.AttributeValueMemberBOOL{ Value: false, }, }, } case "task_states": if input.Key["TaskUUID"] == nil { output = &dynamodb.GetItemOutput{ Item: map[string]types.AttributeValue{ "Error": &types.AttributeValueMemberNULL{ Value: false, }, "State": &types.AttributeValueMemberS{ Value: tasks.StatePending, }, "TaskUUID": &types.AttributeValueMemberS{ Value: "testTaskUUID1", }, "Results:": &types.AttributeValueMemberNULL{ Value: true, }, }, } } else { if input.Key["TaskUUID"].(*types.AttributeValueMemberS).Value == "testTaskUUID1" { output = &dynamodb.GetItemOutput{ Item: TestTask1, } } else if input.Key["TaskUUID"].(*types.AttributeValueMemberS).Value == "testTaskUUID2" { output = &dynamodb.GetItemOutput{ Item: TestTask2, } } else if input.Key["TaskUUID"].(*types.AttributeValueMemberS).Value == "testTaskUUID3" { output = &dynamodb.GetItemOutput{ Item: TestTask3, } } } } return output, nil } func (t *TestDynamoDBClient) DeleteItem(ctx context.Context, input *dynamodb.DeleteItemInput, ops ...func(*dynamodb.Options)) (*dynamodb.DeleteItemOutput, error) { return &dynamodb.DeleteItemOutput{}, nil } func (t *TestDynamoDBClient) UpdateItem(ctx context.Context, input *dynamodb.UpdateItemInput, ops ...func(*dynamodb.Options)) (*dynamodb.UpdateItemOutput, error) { if t.UpdateItemOverride != nil { return t.UpdateItemOverride(ctx, input, ops...) } return &dynamodb.UpdateItemOutput{}, nil } func (t *TestDynamoDBClient) ListTables(ctx context.Context, input *dynamodb.ListTablesInput, ops ...func(*dynamodb.Options)) (*dynamodb.ListTablesOutput, error) { return &dynamodb.ListTablesOutput{ TableNames: []string{ "group_metas", "task_states", }, }, nil } // Always returns error type TestErrDynamoDBClient struct { dynamodbiface.API } func (t *TestErrDynamoDBClient) PutItem(context.Context, *dynamodb.PutItemInput, ...func(*dynamodb.Options)) (*dynamodb.PutItemOutput, error) { return nil, errors.New("error when putting an item") } func (t *TestErrDynamoDBClient) GetItem(context.Context, *dynamodb.GetItemInput, ...func(*dynamodb.Options)) (*dynamodb.GetItemOutput, error) { return nil, errors.New("error when getting an item") } func (t *TestErrDynamoDBClient) DeleteItem(context.Context, *dynamodb.DeleteItemInput, ...func(*dynamodb.Options)) (*dynamodb.DeleteItemOutput, error) { return nil, errors.New("error when deleting an item") } func (t *TestErrDynamoDBClient) Scan(context.Context, *dynamodb.ScanInput, ...func(*dynamodb.Options)) (*dynamodb.ScanOutput, error) { return nil, errors.New("error when scanning an item") } func (t *TestErrDynamoDBClient) UpdateItem(context.Context, *dynamodb.UpdateItemInput, ...func(*dynamodb.Options)) (*dynamodb.UpdateItemOutput, error) { return nil, errors.New("error when updating an item") } func (t *TestErrDynamoDBClient) ListTables(context.Context, *dynamodb.ListTablesInput, ...func(*dynamodb.Options)) (*dynamodb.ListTablesOutput, error) { return nil, errors.New("error when listing tables") } func init() { TestCnf = &config.Config{ ResultBackend: os.Getenv("DYNAMODB_URL"), ResultsExpireIn: 30, DynamoDB: &config.DynamoDBConfig{ TaskStatesTable: "task_states", GroupMetasTable: "group_metas", }, } TestDBClient = new(TestDynamoDBClient) TestDynamoDBBackend = &Backend{cnf: TestCnf, client: TestDBClient} TestErrDBClient = new(TestErrDynamoDBClient) TestErrDynamoDBBackend = &Backend{cnf: TestCnf, client: TestErrDBClient} TestGroupMeta = &tasks.GroupMeta{ GroupUUID: "testGroupUUID", TaskUUIDs: []string{"testTaskUUID1", "testTaskUUID2", "testTaskUUID3"}, } } func (b *Backend) GetConfig() *config.Config { return b.cnf } func (b *Backend) GetClient() dynamodbiface.API { return b.client } func (b *Backend) GetGroupMetaForTest(groupUUID string) (*tasks.GroupMeta, error) { return b.getGroupMeta(groupUUID) } func (b *Backend) UnmarshalGroupMetaGetItemResultForTest(result *dynamodb.GetItemOutput) (*tasks.GroupMeta, error) { return b.unmarshalGroupMetaGetItemResult(result) } func (b *Backend) UnmarshalTaskStateGetItemResultForTest(result *dynamodb.GetItemOutput) (*tasks.TaskState, error) { return b.unmarshalTaskStateGetItemResult(result) } func (b *Backend) SetTaskStateForTest(taskState *tasks.TaskState) error { return b.setTaskState(taskState) } func (b *Backend) ChordTriggeredForTest(groupUUID string) error { return b.chordTriggered(groupUUID) } func (b *Backend) UpdateGroupMetaLockForTest(groupUUID string, status bool) error { return b.updateGroupMetaLock(groupUUID, status) } func (b *Backend) UnlockGroupMetaForTest(groupUUID string) error { return b.unlockGroupMeta(groupUUID) } func (b *Backend) LockGroupMetaForTest(groupUUID string) error { return b.lockGroupMeta(groupUUID) } func (b *Backend) GetStatesForTest(taskUUIDs ...string) ([]*tasks.TaskState, error) { return b.getStates(taskUUIDs) } func (b *Backend) UpdateToFailureStateWithErrorForTest(taskState *tasks.TaskState) error { return b.updateToFailureStateWithError(taskState) } func (b *Backend) TableExistsForTest(tableName string, tableNames []string) bool { return b.tableExists(tableName, tableNames) } func (b *Backend) CheckRequiredTablesIfExistForTest() error { return b.checkRequiredTablesIfExist() } ================================================ FILE: v2/backends/dynamodb/dynamodb_test.go ================================================ package dynamodb_test import ( "context" "fmt" "strconv" "testing" "time" "github.com/RichardKnop/machinery/v2/backends/dynamodb" "github.com/RichardKnop/machinery/v2/log" "github.com/RichardKnop/machinery/v2/tasks" awsdynamodb "github.com/aws/aws-sdk-go-v2/service/dynamodb" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestNew(t *testing.T) { // should call t.Skip if not connected to internet backend, err := dynamodb.New(dynamodb.TestCnf) require.NoError(t, err) assert.IsType(t, new(dynamodb.Backend), backend) } func TestInitGroup(t *testing.T) { groupUUID := "testGroupUUID" taskUUIDs := []string{"testTaskUUID1", "testTaskUUID2", "testTaskUUID3"} log.INFO.Println(dynamodb.TestDynamoDBBackend.GetConfig()) err := dynamodb.TestDynamoDBBackend.InitGroup(groupUUID, taskUUIDs) assert.Nil(t, err) err = dynamodb.TestErrDynamoDBBackend.InitGroup(groupUUID, taskUUIDs) assert.NotNil(t, err) // assert proper TTL value is set in InitGroup() dynamodb.TestDynamoDBBackend.GetConfig().ResultsExpireIn = 3 * 3600 // results should expire after 3 hours client := dynamodb.TestDynamoDBBackend.GetClient().(*dynamodb.TestDynamoDBClient) // Override DynamoDB PutItem() behavior var isPutItemCalled bool client.PutItemOverride = func(ctx context.Context, input *awsdynamodb.PutItemInput, ops ...func(*awsdynamodb.Options)) (*awsdynamodb.PutItemOutput, error) { isPutItemCalled = true assert.NotNil(t, input) actualTTLStr := input.Item["TTL"].(*types.AttributeValueMemberN).Value expectedTTLTime := time.Now().Add(3 * time.Hour) assertTTLValue(t, expectedTTLTime, actualTTLStr) return &awsdynamodb.PutItemOutput{}, nil } err = dynamodb.TestDynamoDBBackend.InitGroup(groupUUID, taskUUIDs) assert.Nil(t, err) assert.True(t, isPutItemCalled) client.ResetOverrides() } func assertTTLValue(t *testing.T, expectedTTLTime time.Time, actualEncodedTTLValue string) { actualTTLTimestamp, err := strconv.ParseInt(actualEncodedTTLValue, 10, 64) assert.Nil(t, err) actualTTLTime := time.Unix(actualTTLTimestamp, 0) assert.WithinDuration(t, expectedTTLTime, actualTTLTime, time.Second) } func TestGroupCompleted(t *testing.T) { client := dynamodb.TestDynamoDBBackend.GetClient().(*dynamodb.TestDynamoDBClient) tableName := dynamodb.TestDynamoDBBackend.GetConfig().DynamoDB.TaskStatesTable // Override DynamoDB BatchGetItem() behavior var isBatchGetItemCalled bool client.BatchGetItemOverride = func(ctx context.Context, input *awsdynamodb.BatchGetItemInput, ops ...func(*awsdynamodb.Options)) (*awsdynamodb.BatchGetItemOutput, error) { isBatchGetItemCalled = true assert.NotNil(t, input) assert.Nil(t, validateBatchGetItemInput(input)) return &awsdynamodb.BatchGetItemOutput{ Responses: map[string][]map[string]types.AttributeValue{ tableName: { {"State": &types.AttributeValueMemberS{Value: tasks.StateSuccess}}, {"State": &types.AttributeValueMemberS{Value: tasks.StateSuccess}}, {"State": &types.AttributeValueMemberS{Value: tasks.StateFailure}}, }, }, }, nil } groupUUID := "testGroupUUID" isCompleted, err := dynamodb.TestDynamoDBBackend.GroupCompleted(groupUUID, 3) assert.Nil(t, err) assert.True(t, isCompleted) assert.True(t, isBatchGetItemCalled) client.ResetOverrides() } func TestGroupCompletedReturnsError(t *testing.T) { client := dynamodb.TestDynamoDBBackend.GetClient().(*dynamodb.TestDynamoDBClient) client.BatchGetItemOverride = func(ctx context.Context, input *awsdynamodb.BatchGetItemInput, ops ...func(options *awsdynamodb.Options)) (*awsdynamodb.BatchGetItemOutput, error) { return nil, fmt.Errorf("Simulating error from AWS") } isCompleted, err := dynamodb.TestDynamoDBBackend.GroupCompleted("test", 3) assert.NotNil(t, err) assert.False(t, isCompleted) client.ResetOverrides() } // TestGroupCompletedReturnsFalse tests that the GroupCompleted() returns false when some tasks have not yet finished. func TestGroupCompletedReturnsFalse(t *testing.T) { client := dynamodb.TestDynamoDBBackend.GetClient().(*dynamodb.TestDynamoDBClient) tableName := dynamodb.TestDynamoDBBackend.GetConfig().DynamoDB.TaskStatesTable // Override DynamoDB BatchGetItem() behavior client.BatchGetItemOverride = func(ctx context.Context, _ *awsdynamodb.BatchGetItemInput, ops ...func(*awsdynamodb.Options)) (*awsdynamodb.BatchGetItemOutput, error) { return &awsdynamodb.BatchGetItemOutput{ Responses: map[string][]map[string]types.AttributeValue{ tableName: { {"State": &types.AttributeValueMemberS{Value: tasks.StateSuccess}}, {"State": &types.AttributeValueMemberS{Value: tasks.StateFailure}}, {"State": &types.AttributeValueMemberS{Value: tasks.StatePending}}, }, }, }, nil } isCompleted, err := dynamodb.TestDynamoDBBackend.GroupCompleted("testGroup", 3) assert.Nil(t, err) assert.False(t, isCompleted) client.ResetOverrides() } // TestGroupCompletedReturnsFalse tests that the GroupCompleted() retries the the request until MaxFetchAttempts before returning an error func TestGroupCompletedRetries(t *testing.T) { client := dynamodb.TestDynamoDBBackend.GetClient().(*dynamodb.TestDynamoDBClient) tableName := dynamodb.TestDynamoDBBackend.GetConfig().DynamoDB.TaskStatesTable // Override DynamoDB BatchGetItem() behavior var countBatchGetItemAPICalls int client.BatchGetItemOverride = func(ctx context.Context, _ *awsdynamodb.BatchGetItemInput, ops ...func(*awsdynamodb.Options)) (*awsdynamodb.BatchGetItemOutput, error) { countBatchGetItemAPICalls++ return &awsdynamodb.BatchGetItemOutput{ Responses: map[string][]map[string]types.AttributeValue{ tableName: { {"State": &types.AttributeValueMemberS{Value: tasks.StateSuccess}}, }, }, UnprocessedKeys: map[string]types.KeysAndAttributes{ tableName: { Keys: []map[string]types.AttributeValue{ {"TaskUUID": &types.AttributeValueMemberS{Value: "unfetchedTaskUUID1"}}, {"TaskUUID": &types.AttributeValueMemberS{Value: "unfetchedTaskUUID2"}}, }, }, }, }, nil } _, err := dynamodb.TestDynamoDBBackend.GroupCompleted("testGroup", 3) assert.NotNil(t, err) assert.Equal(t, dynamodb.MaxFetchAttempts, countBatchGetItemAPICalls) client.ResetOverrides() } // TestGroupCompletedReturnsFalse tests that the GroupCompleted() retries the the request and returns success if all keys are fetched on retries. func TestGroupCompletedRetrieSuccess(t *testing.T) { client := dynamodb.TestDynamoDBBackend.GetClient().(*dynamodb.TestDynamoDBClient) tableName := dynamodb.TestDynamoDBBackend.GetConfig().DynamoDB.TaskStatesTable // Override DynamoDB BatchGetItem() behavior var countBatchGetItemAPICalls int client.BatchGetItemOverride = func(ctx context.Context, _ *awsdynamodb.BatchGetItemInput, ops ...func(*awsdynamodb.Options)) (*awsdynamodb.BatchGetItemOutput, error) { countBatchGetItemAPICalls++ // simulate unfetched keys on 1st attempt. if countBatchGetItemAPICalls == 1 { return &awsdynamodb.BatchGetItemOutput{ Responses: map[string][]map[string]types.AttributeValue{ tableName: {}, // no keys returned in this attempt. }, UnprocessedKeys: map[string]types.KeysAndAttributes{ tableName: { Keys: []map[string]types.AttributeValue{ {"TaskUUID": &types.AttributeValueMemberS{Value: "unfetchedTaskUUID1"}}, {"TaskUUID": &types.AttributeValueMemberS{Value: "unfetchedTaskUUID2"}}, {"TaskUUID": &types.AttributeValueMemberS{Value: "unfetchedTaskUUID3"}}, }, }, }, }, nil } // Return all keys in subsequent attempts. return &awsdynamodb.BatchGetItemOutput{ Responses: map[string][]map[string]types.AttributeValue{ tableName: { {"State": &types.AttributeValueMemberS{Value: tasks.StateSuccess}}, {"State": &types.AttributeValueMemberS{Value: tasks.StateSuccess}}, {"State": &types.AttributeValueMemberS{Value: tasks.StateSuccess}}, }, }, }, nil } isCompleted, err := dynamodb.TestDynamoDBBackend.GroupCompleted("testGroup", 3) assert.Nil(t, err) assert.True(t, isCompleted) assert.Equal(t, 2, countBatchGetItemAPICalls) client.ResetOverrides() } func TestPrivateFuncGetGroupMeta(t *testing.T) { groupUUID := "testGroupUUID" meta, err := dynamodb.TestDynamoDBBackend.GetGroupMetaForTest(groupUUID) item := tasks.GroupMeta{ GroupUUID: "testGroupUUID", Lock: false, ChordTriggered: false, TaskUUIDs: []string{ "testTaskUUID1", "testTaskUUID2", "testTaskUUID3", }, } assert.Nil(t, err) assert.EqualValues(t, item, *meta) _, err = dynamodb.TestErrDynamoDBBackend.GetGroupMetaForTest(groupUUID) assert.NotNil(t, err) } func TestPrivateFuncUnmarshalTaskStateGetItemResult(t *testing.T) { result := awsdynamodb.GetItemOutput{ Item: map[string]types.AttributeValue{ "Error": &types.AttributeValueMemberNULL{ Value: true, }, "State": &types.AttributeValueMemberS{ Value: tasks.StatePending, }, "TaskUUID": &types.AttributeValueMemberS{ Value: "testTaskUUID1", }, "Results:": &types.AttributeValueMemberNULL{ Value: true, }, }, } invalidResult := awsdynamodb.GetItemOutput{ Item: map[string]types.AttributeValue{ "Error": &types.AttributeValueMemberBOOL{ Value: true, }, "State": &types.AttributeValueMemberS{ Value: tasks.StatePending, }, "TaskUUID": &types.AttributeValueMemberS{ Value: "testTaskUUID1", }, "Results:": &types.AttributeValueMemberBOOL{ Value: true, }, }, } item := tasks.TaskState{ TaskUUID: "testTaskUUID1", Results: nil, State: tasks.StatePending, Error: "", } state, err := dynamodb.TestErrDynamoDBBackend.UnmarshalTaskStateGetItemResultForTest(&result) assert.Nil(t, err) assert.EqualValues(t, item, *state) _, err = dynamodb.TestDynamoDBBackend.UnmarshalTaskStateGetItemResultForTest(nil) assert.NotNil(t, err) _, err = dynamodb.TestDynamoDBBackend.UnmarshalTaskStateGetItemResultForTest(&invalidResult) assert.NotNil(t, err) } func TestPrivateFuncUnmarshalGroupMetaGetItemResult(t *testing.T) { result := awsdynamodb.GetItemOutput{ Item: map[string]types.AttributeValue{ "TaskUUIDs": &types.AttributeValueMemberL{ Value: []types.AttributeValue{ &types.AttributeValueMemberS{ Value: "testTaskUUID1", }, &types.AttributeValueMemberS{ Value: "testTaskUUID2", }, &types.AttributeValueMemberS{ Value: "testTaskUUID3", }, }, }, "ChordTriggered": &types.AttributeValueMemberBOOL{ Value: false, }, "GroupUUID": &types.AttributeValueMemberS{ Value: "testGroupUUID", }, "Lock": &types.AttributeValueMemberBOOL{ Value: false, }, }, } invalidResult := awsdynamodb.GetItemOutput{ Item: map[string]types.AttributeValue{ "TaskUUIDs": &types.AttributeValueMemberL{ Value: []types.AttributeValue{ &types.AttributeValueMemberS{ Value: "testTaskUUID1", }, &types.AttributeValueMemberS{ Value: "testTaskUUID2", }, &types.AttributeValueMemberS{ Value: "testTaskUUID3", }, }, }, "ChordTriggered": &types.AttributeValueMemberS{ Value: "false", // this attribute is invalid }, "GroupUUID": &types.AttributeValueMemberS{ Value: "testGroupUUID", }, "Lock": &types.AttributeValueMemberBOOL{ Value: false, }, }, } item := tasks.GroupMeta{ GroupUUID: "testGroupUUID", Lock: false, ChordTriggered: false, TaskUUIDs: []string{ "testTaskUUID1", "testTaskUUID2", "testTaskUUID3", }, } meta, err := dynamodb.TestErrDynamoDBBackend.UnmarshalGroupMetaGetItemResultForTest(&result) assert.Nil(t, err) assert.EqualValues(t, item, *meta) _, err = dynamodb.TestErrDynamoDBBackend.UnmarshalGroupMetaGetItemResultForTest(nil) assert.NotNil(t, err) _, err = dynamodb.TestErrDynamoDBBackend.UnmarshalGroupMetaGetItemResultForTest(&invalidResult) assert.NotNil(t, err) } func TestPrivateFuncSetTaskState(t *testing.T) { signature := &tasks.Signature{ Name: "Test", Args: []tasks.Arg{ { Type: "int64", Value: 1, }, }, } state := tasks.NewPendingTaskState(signature) err := dynamodb.TestErrDynamoDBBackend.SetTaskStateForTest(state) assert.NotNil(t, err) err = dynamodb.TestDynamoDBBackend.SetTaskStateForTest(state) assert.Nil(t, err) } // verifyUpdateInput is a helper function to verify valid dynamoDB update input. func verifyUpdateInput(t *testing.T, input *awsdynamodb.UpdateItemInput, expectedTaskID string, expectedState string, expectedTTLTime time.Time) { assert.NotNil(t, input) // verify task ID assert.Equal(t, expectedTaskID, input.Key["TaskUUID"].(*types.AttributeValueMemberS).Value) // verify task state assert.Equal(t, expectedState, input.ExpressionAttributeValues[":s"].(*types.AttributeValueMemberS).Value) // Verify TTL if !expectedTTLTime.IsZero() { actualTTLStr := input.ExpressionAttributeValues[":t"].(*types.AttributeValueMemberN).Value assertTTLValue(t, expectedTTLTime, actualTTLStr) } } func TestSetStateSuccess(t *testing.T) { signature := &tasks.Signature{UUID: "testTaskUUID"} // assert correct task ID, state and TTL value is set in SetStateSuccess() dynamodb.TestDynamoDBBackend.GetConfig().ResultsExpireIn = 3 * 3600 // results should expire after 3 hours client := dynamodb.TestDynamoDBBackend.GetClient().(*dynamodb.TestDynamoDBClient) // Override DynamoDB UpdateItem() behavior var isUpdateItemCalled bool client.UpdateItemOverride = func(ctx context.Context, input *awsdynamodb.UpdateItemInput, ops ...func(*awsdynamodb.Options)) (*awsdynamodb.UpdateItemOutput, error) { isUpdateItemCalled = true verifyUpdateInput(t, input, signature.UUID, tasks.StateSuccess, time.Now().Add(3*time.Hour)) return &awsdynamodb.UpdateItemOutput{}, nil } err := dynamodb.TestDynamoDBBackend.SetStateSuccess(signature, nil) assert.Nil(t, err) assert.True(t, isUpdateItemCalled) client.ResetOverrides() } func TestSetStateFailure(t *testing.T) { signature := &tasks.Signature{UUID: "testTaskUUID"} // assert correct task ID, state and TTL value is set in SetStateFailure() dynamodb.TestDynamoDBBackend.GetConfig().ResultsExpireIn = 2 * 3600 // results should expire after 2 hours client := dynamodb.TestDynamoDBBackend.GetClient().(*dynamodb.TestDynamoDBClient) // Override DynamoDB UpdateItem() behavior var isUpdateItemCalled bool client.UpdateItemOverride = func(ctx context.Context, input *awsdynamodb.UpdateItemInput, ops ...func(*awsdynamodb.Options)) (*awsdynamodb.UpdateItemOutput, error) { isUpdateItemCalled = true verifyUpdateInput(t, input, signature.UUID, tasks.StateFailure, time.Now().Add(2*time.Hour)) return &awsdynamodb.UpdateItemOutput{}, nil } err := dynamodb.TestDynamoDBBackend.SetStateFailure(signature, "Some error occurred") assert.Nil(t, err) assert.True(t, isUpdateItemCalled) client.ResetOverrides() } func TestSetStateReceived(t *testing.T) { signature := &tasks.Signature{UUID: "testTaskUUID"} // assert correct task ID, state and *no* TTL value is set in SetStateReceived() dynamodb.TestDynamoDBBackend.GetConfig().ResultsExpireIn = 2 * 3600 // results should expire after 2 hours (ignored for this state) client := dynamodb.TestDynamoDBBackend.GetClient().(*dynamodb.TestDynamoDBClient) var isUpdateItemCalled bool client.UpdateItemOverride = func(ctx context.Context, input *awsdynamodb.UpdateItemInput, ops ...func(*awsdynamodb.Options)) (*awsdynamodb.UpdateItemOutput, error) { isUpdateItemCalled = true verifyUpdateInput(t, input, signature.UUID, tasks.StateReceived, time.Time{}) return &awsdynamodb.UpdateItemOutput{}, nil } err := dynamodb.TestDynamoDBBackend.SetStateReceived(signature) assert.Nil(t, err) assert.True(t, isUpdateItemCalled) client.ResetOverrides() } func TestSetStateStarted(t *testing.T) { signature := &tasks.Signature{UUID: "testTaskUUID"} // assert correct task ID, state and *no* TTL value is set in SetStateStarted() dynamodb.TestDynamoDBBackend.GetConfig().ResultsExpireIn = 2 * 3600 // results should expire after 2 hours (ignored for this state) client := dynamodb.TestDynamoDBBackend.GetClient().(*dynamodb.TestDynamoDBClient) var isUpdateItemCalled bool client.UpdateItemOverride = func(ctx context.Context, input *awsdynamodb.UpdateItemInput, ops ...func(*awsdynamodb.Options)) (*awsdynamodb.UpdateItemOutput, error) { isUpdateItemCalled = true verifyUpdateInput(t, input, signature.UUID, tasks.StateStarted, time.Time{}) return &awsdynamodb.UpdateItemOutput{}, nil } err := dynamodb.TestDynamoDBBackend.SetStateStarted(signature) assert.Nil(t, err) assert.True(t, isUpdateItemCalled) client.ResetOverrides() } func TestSetStateRetry(t *testing.T) { signature := &tasks.Signature{UUID: "testTaskUUID"} // assert correct task ID, state and *no* TTL value is set in SetStateStarted() dynamodb.TestDynamoDBBackend.GetConfig().ResultsExpireIn = 2 * 3600 // results should expire after 2 hours (ignored for this state) client := dynamodb.TestDynamoDBBackend.GetClient().(*dynamodb.TestDynamoDBClient) var isUpdateItemCalled bool client.UpdateItemOverride = func(ctx context.Context, input *awsdynamodb.UpdateItemInput, ops ...func(options *awsdynamodb.Options)) (*awsdynamodb.UpdateItemOutput, error) { isUpdateItemCalled = true verifyUpdateInput(t, input, signature.UUID, tasks.StateRetry, time.Time{}) return &awsdynamodb.UpdateItemOutput{}, nil } err := dynamodb.TestDynamoDBBackend.SetStateRetry(signature) assert.Nil(t, err) assert.True(t, isUpdateItemCalled) client.ResetOverrides() } func TestGroupTaskStates(t *testing.T) { expectedStates := map[string]*tasks.TaskState{ "testTaskUUID1": { TaskUUID: "testTaskUUID1", Results: nil, State: tasks.StatePending, Error: "", }, "testTaskUUID2": { TaskUUID: "testTaskUUID2", Results: nil, State: tasks.StateStarted, Error: "", }, "testTaskUUID3": { TaskUUID: "testTaskUUID3", Results: nil, State: tasks.StateSuccess, Error: "", }, } client := dynamodb.TestDynamoDBBackend.GetClient().(*dynamodb.TestDynamoDBClient) tableName := dynamodb.TestDynamoDBBackend.GetConfig().DynamoDB.TaskStatesTable client.BatchGetItemOverride = func(ctx context.Context, input *awsdynamodb.BatchGetItemInput, ops ...func(options *awsdynamodb.Options)) (*awsdynamodb.BatchGetItemOutput, error) { assert.Nil(t, validateBatchGetItemInput(input)) return &awsdynamodb.BatchGetItemOutput{ Responses: map[string][]map[string]types.AttributeValue{ tableName: { { "TaskUUID": &types.AttributeValueMemberS{Value: "testTaskUUID1"}, "Results:": &types.AttributeValueMemberNULL{Value: true}, "State": &types.AttributeValueMemberS{Value: tasks.StatePending}, "Error": &types.AttributeValueMemberNULL{Value: true}, }, { "TaskUUID": &types.AttributeValueMemberS{Value: "testTaskUUID2"}, "Results:": &types.AttributeValueMemberNULL{Value: true}, "State": &types.AttributeValueMemberS{Value: tasks.StateStarted}, "Error": &types.AttributeValueMemberNULL{Value: true}, }, { "TaskUUID": &types.AttributeValueMemberS{Value: "testTaskUUID3"}, "Results:": &types.AttributeValueMemberNULL{Value: true}, "State": &types.AttributeValueMemberS{Value: tasks.StateSuccess}, "Error": &types.AttributeValueMemberNULL{Value: true}, }, }, }, }, nil } defer client.ResetOverrides() states, err := dynamodb.TestDynamoDBBackend.GroupTaskStates("testGroupUUID", 3) assert.Nil(t, err) for _, s := range states { assert.EqualValues(t, *s, *expectedStates[s.TaskUUID]) } } func TestTriggerChord(t *testing.T) { groupUUID := "testGroupUUID" triggered, err := dynamodb.TestDynamoDBBackend.TriggerChord(groupUUID) assert.Nil(t, err) assert.True(t, triggered) } func TestGetState(t *testing.T) { taskUUID := "testTaskUUID1" expectedState := &tasks.TaskState{ TaskUUID: "testTaskUUID1", Results: nil, State: tasks.StatePending, Error: "", } client := dynamodb.TestDynamoDBBackend.GetClient().(*dynamodb.TestDynamoDBClient) client.GetItemOverride = func(ctx context.Context, input *awsdynamodb.GetItemInput, ops ...func(*awsdynamodb.Options)) (*awsdynamodb.GetItemOutput, error) { return &awsdynamodb.GetItemOutput{ Item: map[string]types.AttributeValue{ "TaskUUID": &types.AttributeValueMemberS{Value: "testTaskUUID1"}, "Results:": &types.AttributeValueMemberNULL{Value: true}, "State": &types.AttributeValueMemberS{Value: tasks.StatePending}, "Error": &types.AttributeValueMemberNULL{Value: false}, }, }, nil } defer client.ResetOverrides() state, err := dynamodb.TestDynamoDBBackend.GetState(taskUUID) assert.Nil(t, err) assert.EqualValues(t, expectedState, state) } func TestPurgeState(t *testing.T) { taskUUID := "testTaskUUID1" err := dynamodb.TestDynamoDBBackend.PurgeState(taskUUID) assert.Nil(t, err) err = dynamodb.TestErrDynamoDBBackend.PurgeState(taskUUID) assert.NotNil(t, err) } func TestPurgeGroupMeta(t *testing.T) { groupUUID := "GroupUUID" err := dynamodb.TestDynamoDBBackend.PurgeGroupMeta(groupUUID) assert.Nil(t, err) err = dynamodb.TestErrDynamoDBBackend.PurgeGroupMeta(groupUUID) assert.NotNil(t, err) } func TestPrivateFuncLockGroupMeta(t *testing.T) { groupUUID := "GroupUUID" err := dynamodb.TestDynamoDBBackend.LockGroupMetaForTest(groupUUID) assert.Nil(t, err) err = dynamodb.TestErrDynamoDBBackend.LockGroupMetaForTest(groupUUID) assert.NotNil(t, err) } func TestPrivateFuncUnLockGroupMeta(t *testing.T) { groupUUID := "GroupUUID" err := dynamodb.TestDynamoDBBackend.UnlockGroupMetaForTest(groupUUID) assert.Nil(t, err) err = dynamodb.TestErrDynamoDBBackend.UnlockGroupMetaForTest(groupUUID) assert.NotNil(t, err) } func TestPrivateFuncChordTriggered(t *testing.T) { groupUUID := "GroupUUID" err := dynamodb.TestDynamoDBBackend.ChordTriggeredForTest(groupUUID) assert.Nil(t, err) err = dynamodb.TestErrDynamoDBBackend.ChordTriggeredForTest(groupUUID) assert.NotNil(t, err) } func TestDynamoDBPrivateFuncUpdateGroupMetaLock(t *testing.T) { groupUUID := "GroupUUID" err := dynamodb.TestDynamoDBBackend.UpdateGroupMetaLockForTest(groupUUID, true) assert.Nil(t, err) err = dynamodb.TestErrDynamoDBBackend.UpdateGroupMetaLockForTest(groupUUID, true) assert.NotNil(t, err) } func TestPrivateFuncUpdateToFailureStateWithError(t *testing.T) { signature := &tasks.Signature{ Name: "Test", Args: []tasks.Arg{ { Type: "int64", Value: 1, }, }, } state := tasks.NewFailureTaskState(signature, "This is an error") err := dynamodb.TestDynamoDBBackend.UpdateToFailureStateWithErrorForTest(state) assert.Nil(t, err) } func TestPrivateFuncTableExistsForTest(t *testing.T) { tables := []string{"foo"} assert.False(t, dynamodb.TestDynamoDBBackend.TableExistsForTest("bar", tables)) assert.True(t, dynamodb.TestDynamoDBBackend.TableExistsForTest("foo", tables)) } func TestPrivateFuncCheckRequiredTablesIfExistForTest(t *testing.T) { err := dynamodb.TestDynamoDBBackend.CheckRequiredTablesIfExistForTest() assert.Nil(t, err) taskTable := dynamodb.TestDynamoDBBackend.GetConfig().DynamoDB.TaskStatesTable groupTable := dynamodb.TestDynamoDBBackend.GetConfig().DynamoDB.GroupMetasTable err = dynamodb.TestErrDynamoDBBackend.CheckRequiredTablesIfExistForTest() assert.NotNil(t, err) dynamodb.TestDynamoDBBackend.GetConfig().DynamoDB.TaskStatesTable = "foo" err = dynamodb.TestDynamoDBBackend.CheckRequiredTablesIfExistForTest() assert.NotNil(t, err) dynamodb.TestDynamoDBBackend.GetConfig().DynamoDB.TaskStatesTable = taskTable dynamodb.TestDynamoDBBackend.GetConfig().DynamoDB.GroupMetasTable = "foo" err = dynamodb.TestDynamoDBBackend.CheckRequiredTablesIfExistForTest() assert.NotNil(t, err) dynamodb.TestDynamoDBBackend.GetConfig().DynamoDB.GroupMetasTable = groupTable } func validateBatchGetItemInput(input *awsdynamodb.BatchGetItemInput) error { if input == nil { return fmt.Errorf("input is nil") } if len(input.RequestItems) == 0 { return fmt.Errorf("RequestItems cannot be empty") } for tableName, keysAndAttributes := range input.RequestItems { if tableName == "" { return fmt.Errorf("table name cannot be empty") } if len(keysAndAttributes.Keys) == 0 { return fmt.Errorf("Keys for table %s cannot be empty", tableName) } } return nil } ================================================ FILE: v2/backends/eager/eager.go ================================================ package eager import ( "bytes" "encoding/json" "fmt" "sync" "github.com/RichardKnop/machinery/v2/backends/iface" "github.com/RichardKnop/machinery/v2/common" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/tasks" ) // ErrGroupNotFound ... type ErrGroupNotFound struct { groupUUID string } // NewErrGroupNotFound returns new instance of ErrGroupNotFound func NewErrGroupNotFound(groupUUID string) ErrGroupNotFound { return ErrGroupNotFound{groupUUID: groupUUID} } // Error implements error interface func (e ErrGroupNotFound) Error() string { return fmt.Sprintf("Group not found: %v", e.groupUUID) } // ErrTasknotFound ... type ErrTasknotFound struct { taskUUID string } // NewErrTasknotFound returns new instance of ErrTasknotFound func NewErrTasknotFound(taskUUID string) ErrTasknotFound { return ErrTasknotFound{taskUUID: taskUUID} } // Error implements error interface func (e ErrTasknotFound) Error() string { return fmt.Sprintf("Task not found: %v", e.taskUUID) } // Backend represents an "eager" in-memory result backend type Backend struct { common.Backend groups map[string][]string tasks map[string][]byte stateMutex sync.Mutex } // New creates EagerBackend instance func New() iface.Backend { return &Backend{ Backend: common.NewBackend(new(config.Config)), groups: make(map[string][]string), tasks: make(map[string][]byte), } } // InitGroup creates and saves a group meta data object func (b *Backend) InitGroup(groupUUID string, taskUUIDs []string) error { tasks := make([]string, 0, len(taskUUIDs)) // copy every task tasks = append(tasks, taskUUIDs...) b.groups[groupUUID] = tasks return nil } // GroupCompleted returns true if all tasks in a group finished func (b *Backend) GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) { tasks, ok := b.groups[groupUUID] if !ok { return false, NewErrGroupNotFound(groupUUID) } var countSuccessTasks = 0 for _, v := range tasks { t, err := b.GetState(v) if err != nil { return false, err } if t.IsCompleted() { countSuccessTasks++ } } return countSuccessTasks == groupTaskCount, nil } // GroupTaskStates returns states of all tasks in the group func (b *Backend) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error) { taskUUIDs, ok := b.groups[groupUUID] if !ok { return nil, NewErrGroupNotFound(groupUUID) } ret := make([]*tasks.TaskState, 0, groupTaskCount) for _, taskUUID := range taskUUIDs { t, err := b.GetState(taskUUID) if err != nil { return nil, err } ret = append(ret, t) } return ret, nil } // TriggerChord flags chord as triggered in the backend storage to make sure // chord is never trigerred multiple times. Returns a boolean flag to indicate // whether the worker should trigger chord (true) or no if it has been triggered // already (false) func (b *Backend) TriggerChord(groupUUID string) (bool, error) { return true, nil } // SetStatePending updates task state to PENDING func (b *Backend) SetStatePending(signature *tasks.Signature) error { state := tasks.NewPendingTaskState(signature) return b.updateState(state) } // SetStateReceived updates task state to RECEIVED func (b *Backend) SetStateReceived(signature *tasks.Signature) error { state := tasks.NewReceivedTaskState(signature) return b.updateState(state) } // SetStateStarted updates task state to STARTED func (b *Backend) SetStateStarted(signature *tasks.Signature) error { state := tasks.NewStartedTaskState(signature) return b.updateState(state) } // SetStateRetry updates task state to RETRY func (b *Backend) SetStateRetry(signature *tasks.Signature) error { state := tasks.NewRetryTaskState(signature) return b.updateState(state) } // SetStateSuccess updates task state to SUCCESS func (b *Backend) SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error { state := tasks.NewSuccessTaskState(signature, results) return b.updateState(state) } // SetStateFailure updates task state to FAILURE func (b *Backend) SetStateFailure(signature *tasks.Signature, err string) error { state := tasks.NewFailureTaskState(signature, err) return b.updateState(state) } // GetState returns the latest task state func (b *Backend) GetState(taskUUID string) (*tasks.TaskState, error) { tasktStateBytes, ok := b.tasks[taskUUID] if !ok { return nil, NewErrTasknotFound(taskUUID) } state := new(tasks.TaskState) decoder := json.NewDecoder(bytes.NewReader(tasktStateBytes)) decoder.UseNumber() if err := decoder.Decode(state); err != nil { return nil, fmt.Errorf("Failed to unmarshal task state %v", b) } return state, nil } // PurgeState deletes stored task state func (b *Backend) PurgeState(taskUUID string) error { _, ok := b.tasks[taskUUID] if !ok { return NewErrTasknotFound(taskUUID) } delete(b.tasks, taskUUID) return nil } // PurgeGroupMeta deletes stored group meta data func (b *Backend) PurgeGroupMeta(groupUUID string) error { _, ok := b.groups[groupUUID] if !ok { return NewErrGroupNotFound(groupUUID) } delete(b.groups, groupUUID) return nil } func (b *Backend) updateState(s *tasks.TaskState) error { // simulate the behavior of json marshal/unmarshal b.stateMutex.Lock() defer b.stateMutex.Unlock() msg, err := json.Marshal(s) if err != nil { return fmt.Errorf("Marshal task state error: %v", err) } b.tasks[s.TaskUUID] = msg return nil } ================================================ FILE: v2/backends/eager/eager_test.go ================================================ package eager_test import ( "encoding/json" "testing" "github.com/RichardKnop/machinery/v2/backends/eager" "github.com/RichardKnop/machinery/v2/backends/iface" "github.com/RichardKnop/machinery/v2/tasks" "github.com/stretchr/testify/suite" ) type EagerBackendTestSuite struct { suite.Suite backend iface.Backend st []*tasks.Signature groups []struct { id string tasks []string } } func (s *EagerBackendTestSuite) SetupSuite() { // prepare common test data s.backend = eager.New() // 2 non-group state s.st = []*tasks.Signature{ {UUID: "1"}, {UUID: "2"}, {UUID: "3"}, {UUID: "4"}, {UUID: "5"}, {UUID: "6"}, } for _, t := range s.st { s.backend.SetStatePending(t) } // groups s.groups = []struct { id string tasks []string }{ {"group1", []string{"1-3", "1-4"}}, {"group2", []string{"2-1", "2-2", "2-3"}}, {"group3", []string(nil)}, {"group4", []string{"4-1", "4-2", "4-3", "4-4"}}, {"group5", []string{"5-1", "5-2"}}, } for _, g := range s.groups { for _, t := range g.tasks { sig := &tasks.Signature{ UUID: t, GroupUUID: g.id, GroupTaskCount: len(g.tasks), } s.st = append(s.st, sig) // default state is pending s.backend.SetStatePending(sig) } s.Nil(s.backend.InitGroup(g.id, g.tasks)) } // prepare for TestInitGroup s.Nil(s.backend.PurgeGroupMeta(s.groups[4].id)) } // // Test Cases // func (s *EagerBackendTestSuite) TestInitGroup() { // group 5 { g := s.groups[4] s.Nil(s.backend.InitGroup(g.id, g.tasks)) } // group3 -- nil as task list { g := s.groups[2] s.Nil(s.backend.InitGroup(g.id, g.tasks)) } } func (s *EagerBackendTestSuite) TestGroupCompleted() { // group 1 { // all tasks are pending g := s.groups[0] completed, err := s.backend.GroupCompleted(g.id, len(g.tasks)) s.False(completed) s.Nil(err) // make these tasks success for _, id := range g.tasks { t := s.getTaskSignature(id) s.NotNil(t) if t == nil { break } s.backend.SetStateSuccess(t, nil) } completed, err = s.backend.GroupCompleted(g.id, len(g.tasks)) s.True(completed) s.Nil(err) } // group 2 { g := s.groups[1] completed, err := s.backend.GroupCompleted(g.id, len(g.tasks)) s.False(completed) s.Nil(err) // make these tasks failure for _, id := range g.tasks { t := s.getTaskSignature(id) s.NotNil(t) if t == nil { break } s.backend.SetStateFailure(t, "just a test") } completed, err = s.backend.GroupCompleted(g.id, len(g.tasks)) s.True(completed) s.Nil(err) } { // call on a not-existed group completed, err := s.backend.GroupCompleted("", 0) s.False(completed) s.NotNil(err) } } func (s *EagerBackendTestSuite) TestGroupTaskStates() { // group 4 { g := s.groups[3] // set failure state with taskUUID as error message for _, id := range g.tasks { t := s.getTaskSignature(id) s.NotNil(t) if t == nil { break } s.backend.SetStateFailure(t, t.UUID) } // get states back ts, err := s.backend.GroupTaskStates(g.id, len(g.tasks)) s.NotNil(ts) s.Nil(err) for _, t := range ts { s.Equal(t.TaskUUID, t.Error) } } { // call on a not-existed group ts, err := s.backend.GroupTaskStates("", 0) s.Nil(ts) s.NotNil(err) } } func (s *EagerBackendTestSuite) TestSetStatePending() { // task 1 { t := s.st[0] // change this state to receiving s.backend.SetStateReceived(t) // change it back to pending s.backend.SetStatePending(t) st, err := s.backend.GetState(t.UUID) s.Nil(err) if st != nil { s.Equal(tasks.StatePending, st.State) } } } func (s *EagerBackendTestSuite) TestSetStateReceived() { // task2 { t := s.st[1] s.backend.SetStateReceived(t) st, err := s.backend.GetState(t.UUID) s.Nil(err) if st != nil { s.Equal(tasks.StateReceived, st.State) } } } func (s *EagerBackendTestSuite) TestSetStateStarted() { // task3 { t := s.st[2] s.backend.SetStateStarted(t) st, err := s.backend.GetState(t.UUID) s.Nil(err) if st != nil { s.Equal(tasks.StateStarted, st.State) } } } func (s *EagerBackendTestSuite) TestSetStateSuccess() { // task4 { t := s.st[3] taskResults := []*tasks.TaskResult{ { Type: "float64", Value: json.Number("300.0"), }, } s.backend.SetStateSuccess(t, taskResults) st, err := s.backend.GetState(t.UUID) s.Nil(err) s.NotNil(st) s.Equal(tasks.StateSuccess, st.State) s.Equal(taskResults, st.Results) } } func (s *EagerBackendTestSuite) TestSetStateFailure() { // task5 { t := s.st[4] s.backend.SetStateFailure(t, "error") st, err := s.backend.GetState(t.UUID) s.Nil(err) if st != nil { s.Equal(tasks.StateFailure, st.State) s.Equal("error", st.Error) } } } func (s *EagerBackendTestSuite) TestSetStateRetry() { // task6 { t := s.st[5] s.backend.SetStateRetry(t) st, err := s.backend.GetState(t.UUID) s.Nil(err) if st != nil { s.Equal(tasks.StateRetry, st.State) } } } func (s *EagerBackendTestSuite) TestGetState() { // get something not existed -- empty string st, err := s.backend.GetState("") s.Nil(st) s.NotNil(err) } func (s *EagerBackendTestSuite) TestPurgeState() { // task6 { t := s.st[5] st, err := s.backend.GetState(t.UUID) s.NotNil(st) s.Nil(err) // purge it s.Nil(s.backend.PurgeState(t.UUID)) // should be not found st, err = s.backend.GetState(t.UUID) s.Nil(st) s.NotNil(err) } { // purge a not-existed state s.NotNil(s.backend.PurgeState("")) } } func (s *EagerBackendTestSuite) TestPurgeGroupMeta() { // group4 { g := s.groups[3] ts, err := s.backend.GroupTaskStates(g.id, len(g.tasks)) s.NotNil(ts) s.Nil(err) // purge group s.Nil(s.backend.PurgeGroupMeta(g.id)) // should be not found ts, err = s.backend.GroupTaskStates(g.id, len(g.tasks)) s.Nil(ts) s.NotNil(err) } { // purge a not-existed group s.NotNil(s.backend.PurgeGroupMeta("")) } } // // internal method // func (s *EagerBackendTestSuite) getTaskSignature(taskUUID string) *tasks.Signature { for _, v := range s.st { if v.UUID == taskUUID { return v } } return nil } func TestEagerBackendMain(t *testing.T) { suite.Run(t, &EagerBackendTestSuite{}) } ================================================ FILE: v2/backends/iface/dynamodb/api.go ================================================ package dynamodb import ( "context" "github.com/aws/aws-sdk-go-v2/service/dynamodb" ) // API is an interface for DynamoDB API. type API interface { PutItem(ctx context.Context, params *dynamodb.PutItemInput, optFns ...func(*dynamodb.Options)) (*dynamodb.PutItemOutput, error) GetItem(ctx context.Context, params *dynamodb.GetItemInput, optFns ...func(*dynamodb.Options)) (*dynamodb.GetItemOutput, error) UpdateItem(ctx context.Context, params *dynamodb.UpdateItemInput, optFns ...func(*dynamodb.Options)) (*dynamodb.UpdateItemOutput, error) DeleteItem(ctx context.Context, params *dynamodb.DeleteItemInput, optFns ...func(*dynamodb.Options)) (*dynamodb.DeleteItemOutput, error) BatchGetItem(ctx context.Context, params *dynamodb.BatchGetItemInput, optFns ...func(*dynamodb.Options)) (*dynamodb.BatchGetItemOutput, error) ListTables(ctx context.Context, params *dynamodb.ListTablesInput, optFns ...func(*dynamodb.Options)) (*dynamodb.ListTablesOutput, error) } ================================================ FILE: v2/backends/iface/interfaces.go ================================================ package iface import ( "github.com/RichardKnop/machinery/v2/tasks" ) // Backend - a common interface for all result backends type Backend interface { // Group related functions InitGroup(groupUUID string, taskUUIDs []string) error GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error) TriggerChord(groupUUID string) (bool, error) // Setting / getting task state SetStatePending(signature *tasks.Signature) error SetStateReceived(signature *tasks.Signature) error SetStateStarted(signature *tasks.Signature) error SetStateRetry(signature *tasks.Signature) error SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error SetStateFailure(signature *tasks.Signature, err string) error GetState(taskUUID string) (*tasks.TaskState, error) // Purging stored stored tasks states and group meta data IsAMQP() bool PurgeState(taskUUID string) error PurgeGroupMeta(groupUUID string) error } ================================================ FILE: v2/backends/memcache/memcache.go ================================================ package memcache import ( "bytes" "encoding/json" "time" "github.com/RichardKnop/machinery/v2/backends/iface" "github.com/RichardKnop/machinery/v2/common" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/log" "github.com/RichardKnop/machinery/v2/tasks" gomemcache "github.com/bradfitz/gomemcache/memcache" ) // Backend represents a Memcache result backend type Backend struct { common.Backend servers []string client *gomemcache.Client } // New creates Backend instance func New(cnf *config.Config, servers []string) iface.Backend { return &Backend{ Backend: common.NewBackend(cnf), servers: servers, } } // InitGroup creates and saves a group meta data object func (b *Backend) InitGroup(groupUUID string, taskUUIDs []string) error { groupMeta := &tasks.GroupMeta{ GroupUUID: groupUUID, TaskUUIDs: taskUUIDs, CreatedAt: time.Now().UTC(), } encoded, err := json.Marshal(&groupMeta) if err != nil { return err } return b.getClient().Set(&gomemcache.Item{ Key: groupUUID, Value: encoded, Expiration: b.getExpirationTimestamp(), }) } // GroupCompleted returns true if all tasks in a group finished func (b *Backend) GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) { groupMeta, err := b.getGroupMeta(groupUUID) if err != nil { return false, err } taskStates, err := b.getStates(groupMeta.TaskUUIDs...) if err != nil { return false, err } var countSuccessTasks = 0 for _, taskState := range taskStates { if taskState.IsCompleted() { countSuccessTasks++ } } return countSuccessTasks == groupTaskCount, nil } // GroupTaskStates returns states of all tasks in the group func (b *Backend) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error) { groupMeta, err := b.getGroupMeta(groupUUID) if err != nil { return []*tasks.TaskState{}, err } return b.getStates(groupMeta.TaskUUIDs...) } // TriggerChord flags chord as triggered in the backend storage to make sure // chord is never trigerred multiple times. Returns a boolean flag to indicate // whether the worker should trigger chord (true) or no if it has been triggered // already (false) func (b *Backend) TriggerChord(groupUUID string) (bool, error) { groupMeta, err := b.getGroupMeta(groupUUID) if err != nil { return false, err } // Chord has already been triggered, return false (should not trigger again) if groupMeta.ChordTriggered { return false, nil } // If group meta is locked, wait until it's unlocked for groupMeta.Lock { groupMeta, _ = b.getGroupMeta(groupUUID) log.WARNING.Print("Group meta locked, waiting") time.Sleep(time.Millisecond * 5) } // Acquire lock if err = b.lockGroupMeta(groupMeta); err != nil { return false, err } defer b.unlockGroupMeta(groupMeta) // Update the group meta data groupMeta.ChordTriggered = true encoded, err := json.Marshal(&groupMeta) if err != nil { return false, err } if err = b.getClient().Replace(&gomemcache.Item{ Key: groupUUID, Value: encoded, Expiration: b.getExpirationTimestamp(), }); err != nil { return false, err } return true, nil } // SetStatePending updates task state to PENDING func (b *Backend) SetStatePending(signature *tasks.Signature) error { taskState := tasks.NewPendingTaskState(signature) return b.updateState(taskState) } // SetStateReceived updates task state to RECEIVED func (b *Backend) SetStateReceived(signature *tasks.Signature) error { taskState := tasks.NewReceivedTaskState(signature) return b.updateState(taskState) } // SetStateStarted updates task state to STARTED func (b *Backend) SetStateStarted(signature *tasks.Signature) error { taskState := tasks.NewStartedTaskState(signature) return b.updateState(taskState) } // SetStateRetry updates task state to RETRY func (b *Backend) SetStateRetry(signature *tasks.Signature) error { state := tasks.NewRetryTaskState(signature) return b.updateState(state) } // SetStateSuccess updates task state to SUCCESS func (b *Backend) SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error { taskState := tasks.NewSuccessTaskState(signature, results) return b.updateState(taskState) } // SetStateFailure updates task state to FAILURE func (b *Backend) SetStateFailure(signature *tasks.Signature, err string) error { taskState := tasks.NewFailureTaskState(signature, err) return b.updateState(taskState) } // GetState returns the latest task state func (b *Backend) GetState(taskUUID string) (*tasks.TaskState, error) { item, err := b.getClient().Get(taskUUID) if err != nil { return nil, err } state := new(tasks.TaskState) decoder := json.NewDecoder(bytes.NewReader(item.Value)) decoder.UseNumber() if err := decoder.Decode(state); err != nil { return nil, err } return state, nil } // PurgeState deletes stored task state func (b *Backend) PurgeState(taskUUID string) error { return b.getClient().Delete(taskUUID) } // PurgeGroupMeta deletes stored group meta data func (b *Backend) PurgeGroupMeta(groupUUID string) error { return b.getClient().Delete(groupUUID) } // updateState saves current task state func (b *Backend) updateState(taskState *tasks.TaskState) error { encoded, err := json.Marshal(taskState) if err != nil { return err } return b.getClient().Set(&gomemcache.Item{ Key: taskState.TaskUUID, Value: encoded, Expiration: b.getExpirationTimestamp(), }) } // lockGroupMeta acquires lock on group meta data func (b *Backend) lockGroupMeta(groupMeta *tasks.GroupMeta) error { groupMeta.Lock = true encoded, err := json.Marshal(groupMeta) if err != nil { return err } return b.getClient().Set(&gomemcache.Item{ Key: groupMeta.GroupUUID, Value: encoded, Expiration: b.getExpirationTimestamp(), }) } // unlockGroupMeta releases lock on group meta data func (b *Backend) unlockGroupMeta(groupMeta *tasks.GroupMeta) error { groupMeta.Lock = false encoded, err := json.Marshal(groupMeta) if err != nil { return err } return b.getClient().Set(&gomemcache.Item{ Key: groupMeta.GroupUUID, Value: encoded, Expiration: b.getExpirationTimestamp(), }) } // getGroupMeta retrieves group meta data, convenience function to avoid repetition func (b *Backend) getGroupMeta(groupUUID string) (*tasks.GroupMeta, error) { item, err := b.getClient().Get(groupUUID) if err != nil { return nil, err } groupMeta := new(tasks.GroupMeta) decoder := json.NewDecoder(bytes.NewReader(item.Value)) decoder.UseNumber() if err := decoder.Decode(groupMeta); err != nil { return nil, err } return groupMeta, nil } // getStates returns multiple task states func (b *Backend) getStates(taskUUIDs ...string) ([]*tasks.TaskState, error) { states := make([]*tasks.TaskState, len(taskUUIDs)) for i, taskUUID := range taskUUIDs { item, err := b.getClient().Get(taskUUID) if err != nil { return nil, err } state := new(tasks.TaskState) decoder := json.NewDecoder(bytes.NewReader(item.Value)) decoder.UseNumber() if err := decoder.Decode(state); err != nil { return nil, err } states[i] = state } return states, nil } // getExpirationTimestamp returns expiration timestamp func (b *Backend) getExpirationTimestamp() int32 { expiresIn := b.GetConfig().ResultsExpireIn if expiresIn == 0 { // // expire results after 1 hour by default expiresIn = config.DefaultResultsExpireIn } return int32(time.Now().Unix() + int64(expiresIn)) } // getClient returns or creates instance of Memcache client func (b *Backend) getClient() *gomemcache.Client { if b.client == nil { b.client = gomemcache.New(b.servers...) } return b.client } ================================================ FILE: v2/backends/memcache/memcache_test.go ================================================ package memcache_test import ( "os" "testing" "time" "github.com/RichardKnop/machinery/v2/backends/memcache" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/tasks" "github.com/stretchr/testify/assert" ) func TestGroupCompleted(t *testing.T) { memcacheURL := os.Getenv("MEMCACHE_URL") if memcacheURL == "" { t.Skip("MEMCACHE_URL is not defined") } groupUUID := "testGroupUUID" task1 := &tasks.Signature{ UUID: "testTaskUUID1", GroupUUID: groupUUID, } task2 := &tasks.Signature{ UUID: "testTaskUUID2", GroupUUID: groupUUID, } backend := memcache.New(new(config.Config), []string{memcacheURL}) // Cleanup before the test backend.PurgeState(task1.UUID) backend.PurgeState(task2.UUID) backend.PurgeGroupMeta(groupUUID) groupCompleted, err := backend.GroupCompleted(groupUUID, 2) if assert.Error(t, err) { assert.False(t, groupCompleted) assert.Equal(t, "memcache: cache miss", err.Error()) } backend.InitGroup(groupUUID, []string{task1.UUID, task2.UUID}) groupCompleted, err = backend.GroupCompleted(groupUUID, 2) if assert.Error(t, err) { assert.False(t, groupCompleted) assert.Equal(t, "memcache: cache miss", err.Error()) } backend.SetStatePending(task1) backend.SetStateStarted(task2) groupCompleted, err = backend.GroupCompleted(groupUUID, 2) if assert.NoError(t, err) { assert.False(t, groupCompleted) } taskResults := []*tasks.TaskResult{new(tasks.TaskResult)} backend.SetStateStarted(task1) backend.SetStateSuccess(task2, taskResults) groupCompleted, err = backend.GroupCompleted(groupUUID, 2) if assert.NoError(t, err) { assert.False(t, groupCompleted) } backend.SetStateFailure(task1, "Some error") groupCompleted, err = backend.GroupCompleted(groupUUID, 2) if assert.NoError(t, err) { assert.True(t, groupCompleted) } } func TestGetState(t *testing.T) { memcacheURL := os.Getenv("MEMCACHE_URL") if memcacheURL == "" { t.Skip("MEMCACHE_URL is not defined") } signature := &tasks.Signature{ UUID: "testTaskUUID", GroupUUID: "testGroupUUID", } backend := memcache.New(new(config.Config), []string{memcacheURL}) go func() { backend.SetStatePending(signature) time.Sleep(2 * time.Millisecond) backend.SetStateReceived(signature) time.Sleep(2 * time.Millisecond) backend.SetStateStarted(signature) time.Sleep(2 * time.Millisecond) taskResults := []*tasks.TaskResult{ { Type: "float64", Value: 2, }, } backend.SetStateSuccess(signature, taskResults) }() var ( taskState *tasks.TaskState err error ) for { taskState, err = backend.GetState(signature.UUID) if taskState == nil { assert.Equal(t, "memcache: cache miss", err.Error()) continue } assert.NoError(t, err) if taskState.IsCompleted() { break } } } func TestPurgeState(t *testing.T) { memcacheURL := os.Getenv("MEMCACHE_URL") if memcacheURL == "" { t.Skip("MEMCACHE_URL is not defined") } signature := &tasks.Signature{ UUID: "testTaskUUID", GroupUUID: "testGroupUUID", } backend := memcache.New(new(config.Config), []string{memcacheURL}) backend.SetStatePending(signature) taskState, err := backend.GetState(signature.UUID) assert.NotNil(t, taskState) assert.NoError(t, err) backend.PurgeState(taskState.TaskUUID) taskState, err = backend.GetState(signature.UUID) assert.Nil(t, taskState) assert.Error(t, err) } ================================================ FILE: v2/backends/mongo/mongodb.go ================================================ package mongo import ( "context" "encoding/json" "fmt" "reflect" "strings" "sync" "time" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "github.com/RichardKnop/machinery/v2/backends/iface" "github.com/RichardKnop/machinery/v2/common" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/log" "github.com/RichardKnop/machinery/v2/tasks" ) // Backend represents a MongoDB result backend type Backend struct { common.Backend client *mongo.Client tc *mongo.Collection gmc *mongo.Collection once sync.Once } // New creates Backend instance func New(cnf *config.Config) (iface.Backend, error) { backend := &Backend{ Backend: common.NewBackend(cnf), once: sync.Once{}, } return backend, nil } // InitGroup creates and saves a group meta data object func (b *Backend) InitGroup(groupUUID string, taskUUIDs []string) error { groupMeta := &tasks.GroupMeta{ GroupUUID: groupUUID, TaskUUIDs: taskUUIDs, CreatedAt: time.Now().UTC(), } _, err := b.groupMetasCollection().InsertOne(context.Background(), groupMeta) return err } // GroupCompleted returns true if all tasks in a group finished func (b *Backend) GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) { groupMeta, err := b.getGroupMeta(groupUUID) if err != nil { return false, err } taskStates, err := b.getStates(groupMeta.TaskUUIDs...) if err != nil { return false, err } var countSuccessTasks = 0 for _, taskState := range taskStates { if taskState.IsCompleted() { countSuccessTasks++ } } return countSuccessTasks == groupTaskCount, nil } // GroupTaskStates returns states of all tasks in the group func (b *Backend) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error) { groupMeta, err := b.getGroupMeta(groupUUID) if err != nil { return []*tasks.TaskState{}, err } return b.getStates(groupMeta.TaskUUIDs...) } // TriggerChord flags chord as triggered in the backend storage to make sure // chord is never triggered multiple times. Returns a boolean flag to indicate // whether the worker should trigger chord (true) or no if it has been triggered // already (false) func (b *Backend) TriggerChord(groupUUID string) (bool, error) { query := bson.M{ "_id": groupUUID, "chord_triggered": false, } change := bson.M{ "$set": bson.M{ "chord_triggered": true, }, } _, err := b.groupMetasCollection().UpdateOne(context.Background(), query, change, options.Update()) if err != nil { if err == mongo.ErrNoDocuments { log.WARNING.Printf("Chord already triggered for group %s", groupUUID) return false, nil } return false, err } return true, nil } // SetStatePending updates task state to PENDING func (b *Backend) SetStatePending(signature *tasks.Signature) error { update := bson.M{ "state": tasks.StatePending, "task_name": signature.Name, "created_at": time.Now().UTC(), } return b.updateState(signature, update) } // SetStateReceived updates task state to RECEIVED func (b *Backend) SetStateReceived(signature *tasks.Signature) error { update := bson.M{"state": tasks.StateReceived} return b.updateState(signature, update) } // SetStateStarted updates task state to STARTED func (b *Backend) SetStateStarted(signature *tasks.Signature) error { update := bson.M{"state": tasks.StateStarted} return b.updateState(signature, update) } // SetStateRetry updates task state to RETRY func (b *Backend) SetStateRetry(signature *tasks.Signature) error { update := bson.M{"state": tasks.StateRetry} return b.updateState(signature, update) } // SetStateSuccess updates task state to SUCCESS func (b *Backend) SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error { decodedResults := b.decodeResults(results) update := bson.M{ "state": tasks.StateSuccess, "results": decodedResults, "delete_at": time.Now().Add(time.Duration(b.GetConfig().ResultsExpireIn) * time.Second), } return b.updateState(signature, update) } // decodeResults detects & decodes json strings in TaskResult.Value and returns a new slice func (b *Backend) decodeResults(results []*tasks.TaskResult) []*tasks.TaskResult { l := len(results) jsonResults := make([]*tasks.TaskResult, l) for i, result := range results { jsonResult := new(bson.M) resultType := reflect.TypeOf(result.Value).Kind() if resultType == reflect.String { err := json.NewDecoder(strings.NewReader(result.Value.(string))).Decode(&jsonResult) if err == nil { jsonResults[i] = &tasks.TaskResult{ Type: "json", Value: jsonResult, } continue } } jsonResults[i] = result } return jsonResults } // SetStateFailure updates task state to FAILURE func (b *Backend) SetStateFailure(signature *tasks.Signature, err string) error { update := bson.M{ "state": tasks.StateFailure, "error": err, "delete_at": time.Now().Add(time.Duration(b.GetConfig().ResultsExpireIn) * time.Second), } return b.updateState(signature, update) } // GetState returns the latest task state func (b *Backend) GetState(taskUUID string) (*tasks.TaskState, error) { state := &tasks.TaskState{} err := b.tasksCollection().FindOne(context.Background(), bson.M{"_id": taskUUID}).Decode(state) if err != nil { return nil, err } return state, nil } // PurgeState deletes stored task state func (b *Backend) PurgeState(taskUUID string) error { _, err := b.tasksCollection().DeleteOne(context.Background(), bson.M{"_id": taskUUID}) return err } // PurgeGroupMeta deletes stored group meta data func (b *Backend) PurgeGroupMeta(groupUUID string) error { _, err := b.groupMetasCollection().DeleteOne(context.Background(), bson.M{"_id": groupUUID}) return err } // lockGroupMeta acquires lock on groupUUID document func (b *Backend) lockGroupMeta(groupUUID string) error { query := bson.M{ "_id": groupUUID, "lock": false, } change := bson.M{ "$set": bson.M{ "lock": true, }, } _, err := b.groupMetasCollection().UpdateOne(context.Background(), query, change, options.Update().SetUpsert(true)) return err } // unlockGroupMeta releases lock on groupUUID document func (b *Backend) unlockGroupMeta(groupUUID string) error { update := bson.M{"$set": bson.M{"lock": false}} _, err := b.groupMetasCollection().UpdateOne(context.Background(), bson.M{"_id": groupUUID}, update, options.Update()) return err } // getGroupMeta retrieves group meta data, convenience function to avoid repetition func (b *Backend) getGroupMeta(groupUUID string) (*tasks.GroupMeta, error) { groupMeta := &tasks.GroupMeta{} query := bson.M{"_id": groupUUID} err := b.groupMetasCollection().FindOne(context.Background(), query).Decode(groupMeta) if err != nil { return nil, err } return groupMeta, nil } // getStates returns multiple task states func (b *Backend) getStates(taskUUIDs ...string) ([]*tasks.TaskState, error) { states := make([]*tasks.TaskState, 0, len(taskUUIDs)) cur, err := b.tasksCollection().Find(context.Background(), bson.M{"_id": bson.M{"$in": taskUUIDs}}) if err != nil { return nil, err } defer cur.Close(context.Background()) for cur.Next(context.Background()) { state := &tasks.TaskState{} if err := cur.Decode(state); err != nil { return nil, err } states = append(states, state) } if cur.Err() != nil { return nil, err } return states, nil } // updateState saves current task state func (b *Backend) updateState(signature *tasks.Signature, update bson.M) error { update = bson.M{"$set": update} _, err := b.tasksCollection().UpdateOne(context.Background(), bson.M{"_id": signature.UUID}, update, options.Update().SetUpsert(true)) return err } func (b *Backend) tasksCollection() *mongo.Collection { b.once.Do(func() { b.connect() }) return b.tc } func (b *Backend) groupMetasCollection() *mongo.Collection { b.once.Do(func() { b.connect() }) return b.gmc } // connect creates the underlying mgo connection if it doesn't exist // creates required indexes for our collections func (b *Backend) connect() error { client, err := b.dial() if err != nil { return err } b.client = client database := "machinery" if b.GetConfig().MongoDB != nil { database = b.GetConfig().MongoDB.Database } b.tc = b.client.Database(database).Collection("tasks") b.gmc = b.client.Database(database).Collection("group_metas") err = b.createMongoIndexes(database) if err != nil { return err } return nil } // dial connects to mongo with TLSConfig if provided // else connects via ResultBackend uri func (b *Backend) dial() (*mongo.Client, error) { if b.GetConfig().MongoDB != nil && b.GetConfig().MongoDB.Client != nil { return b.GetConfig().MongoDB.Client, nil } uri := b.GetConfig().ResultBackend if strings.HasPrefix(uri, "mongodb://") == false && strings.HasPrefix(uri, "mongodb+srv://") == false { uri = fmt.Sprintf("mongodb://%s", uri) } client, err := mongo.NewClient(options.Client().ApplyURI(uri)) if err != nil { return nil, err } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() if err := client.Connect(ctx); err != nil { return nil, err } return client, nil } // createMongoIndexes ensures all indexes are in place func (b *Backend) createMongoIndexes(database string) error { tasksCollection := b.client.Database(database).Collection("tasks") _, err := tasksCollection.Indexes().CreateMany( context.Background(), []mongo.IndexModel{ { Keys: bson.M{"delete_at": 1}, Options: options.Index().SetBackground(true).SetExpireAfterSeconds(0), }, }, ) if err != nil { return err } return err } ================================================ FILE: v2/backends/mongo/mongodb_test.go ================================================ package mongo_test import ( "os" "testing" "github.com/RichardKnop/machinery/v2/backends/iface" "github.com/RichardKnop/machinery/v2/backends/mongo" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/tasks" "github.com/stretchr/testify/assert" ) var ( groupUUID = "123456" taskUUIDs = []string{"1", "2", "3"} ) func newBackend() (iface.Backend, error) { cnf := &config.Config{ ResultBackend: os.Getenv("MONGODB_URL"), ResultsExpireIn: 30, } backend, err := mongo.New(cnf) if err != nil { return nil, err } backend.PurgeGroupMeta(groupUUID) for _, taskUUID := range taskUUIDs { backend.PurgeState(taskUUID) } if err := backend.InitGroup(groupUUID, taskUUIDs); err != nil { return nil, err } return backend, nil } func TestNew(t *testing.T) { if os.Getenv("MONGODB_URL") == "" { t.Skip("MONGODB_URL is not defined") } backend, err := newBackend() if assert.NoError(t, err) { assert.NotNil(t, backend) } } func TestSetStatePending(t *testing.T) { if os.Getenv("MONGODB_URL") == "" { t.Skip("MONGODB_URL is not defined") } backend, err := newBackend() if err != nil { t.Fatal(err) } err = backend.SetStatePending(&tasks.Signature{ UUID: taskUUIDs[0], }) if assert.NoError(t, err) { taskState, err := backend.GetState(taskUUIDs[0]) if assert.NoError(t, err) { assert.Equal(t, tasks.StatePending, taskState.State, "Not StatePending") } } } func TestSetStateReceived(t *testing.T) { if os.Getenv("MONGODB_URL") == "" { t.Skip("MONGODB_URL is not defined") } backend, err := newBackend() if err != nil { t.Fatal(err) } err = backend.SetStateReceived(&tasks.Signature{ UUID: taskUUIDs[0], }) if assert.NoError(t, err) { taskState, err := backend.GetState(taskUUIDs[0]) if assert.NoError(t, err) { assert.Equal(t, tasks.StateReceived, taskState.State, "Not StateReceived") } } } func TestSetStateStarted(t *testing.T) { if os.Getenv("MONGODB_URL") == "" { t.Skip("MONGODB_URL is not defined") } backend, err := newBackend() if err != nil { t.Fatal(err) } err = backend.SetStateStarted(&tasks.Signature{ UUID: taskUUIDs[0], }) if assert.NoError(t, err) { taskState, err := backend.GetState(taskUUIDs[0]) if assert.NoError(t, err) { assert.Equal(t, tasks.StateStarted, taskState.State, "Not StateStarted") } } } func TestSetStateSuccess(t *testing.T) { if os.Getenv("MONGODB_URL") == "" { t.Skip("MONGODB_URL is not defined") } resultType := "float64" resultValue := float64(88.5) backend, err := newBackend() if err != nil { t.Fatal(err) } signature := &tasks.Signature{ UUID: taskUUIDs[0], } taskResults := []*tasks.TaskResult{ { Type: resultType, Value: resultValue, }, } err = backend.SetStateSuccess(signature, taskResults) assert.NoError(t, err) taskState, err := backend.GetState(taskUUIDs[0]) assert.NoError(t, err) assert.Equal(t, tasks.StateSuccess, taskState.State, "Not StateSuccess") assert.Equal(t, resultType, taskState.Results[0].Type, "Wrong result type") assert.Equal(t, float64(resultValue), taskState.Results[0].Value.(float64), "Wrong result value") } func TestSetStateFailure(t *testing.T) { if os.Getenv("MONGODB_URL") == "" { t.Skip("MONGODB_URL is not defined") } failString := "Fail is ok" backend, err := newBackend() if err != nil { t.Fatal(err) } signature := &tasks.Signature{ UUID: taskUUIDs[0], } err = backend.SetStateFailure(signature, failString) assert.NoError(t, err) taskState, err := backend.GetState(taskUUIDs[0]) assert.NoError(t, err) assert.Equal(t, tasks.StateFailure, taskState.State, "Not StateSuccess") assert.Equal(t, failString, taskState.Error, "Wrong fail error") } func TestGroupCompleted(t *testing.T) { if os.Getenv("MONGODB_URL") == "" { t.Skip("MONGODB_URL is not defined") } backend, err := newBackend() if err != nil { t.Fatal(err) } taskResultsState := make(map[string]string) isCompleted, err := backend.GroupCompleted(groupUUID, len(taskUUIDs)) if assert.NoError(t, err) { assert.False(t, isCompleted, "Actually group is not completed") } signature := &tasks.Signature{ UUID: taskUUIDs[0], } err = backend.SetStateFailure(signature, "Fail is ok") assert.NoError(t, err) taskResultsState[taskUUIDs[0]] = tasks.StateFailure signature = &tasks.Signature{ UUID: taskUUIDs[1], } taskResults := []*tasks.TaskResult{ { Type: "string", Value: "Result ok", }, } err = backend.SetStateSuccess(signature, taskResults) assert.NoError(t, err) taskResultsState[taskUUIDs[1]] = tasks.StateSuccess signature = &tasks.Signature{ UUID: taskUUIDs[2], } err = backend.SetStateSuccess(signature, taskResults) assert.NoError(t, err) taskResultsState[taskUUIDs[2]] = tasks.StateSuccess isCompleted, err = backend.GroupCompleted(groupUUID, len(taskUUIDs)) if assert.NoError(t, err) { assert.True(t, isCompleted, "Actually group is completed") } taskStates, err := backend.GroupTaskStates(groupUUID, len(taskUUIDs)) assert.NoError(t, err) assert.Equal(t, len(taskStates), len(taskUUIDs), "Wrong len tasksStates") for _, taskState := range taskStates { assert.Equal( t, taskResultsState[taskState.TaskUUID], taskState.State, "Wrong state on", taskState.TaskUUID, ) } } func TestGroupStates(t *testing.T) { if os.Getenv("MONGODB_URL") == "" { t.Skip("MONGODB_URL is not defined") } backend, err := newBackend() if err != nil { t.Fatal(err) } taskStates, err := backend.GroupTaskStates(groupUUID, len(taskUUIDs)) assert.NoError(t, err) for i, taskState := range taskStates { assert.Equal(t, taskUUIDs[i], taskState.TaskUUID) } } ================================================ FILE: v2/backends/null/null.go ================================================ package null import ( "fmt" "github.com/RichardKnop/machinery/v2/backends/iface" "github.com/RichardKnop/machinery/v2/common" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/tasks" ) // ErrGroupNotFound ... type ErrGroupNotFound struct { groupUUID string } // NewErrGroupNotFound returns new instance of ErrGroupNotFound func NewErrGroupNotFound(groupUUID string) ErrGroupNotFound { return ErrGroupNotFound{groupUUID: groupUUID} } // Error implements error interface func (e ErrGroupNotFound) Error() string { return fmt.Sprintf("Group not found: %v", e.groupUUID) } // ErrTasknotFound ... type ErrTasknotFound struct { taskUUID string } // NewErrTasknotFound returns new instance of ErrTasknotFound func NewErrTasknotFound(taskUUID string) ErrTasknotFound { return ErrTasknotFound{taskUUID: taskUUID} } // Error implements error interface func (e ErrTasknotFound) Error() string { return fmt.Sprintf("Task not found: %v", e.taskUUID) } // Backend represents an "null" result backend type Backend struct { common.Backend groups map[string]struct{} } // New creates NullBackend instance func New() iface.Backend { return &Backend{ Backend: common.NewBackend(new(config.Config)), groups: make(map[string]struct{}), } } // InitGroup creates and saves a group meta data object func (b *Backend) InitGroup(groupUUID string, taskUUIDs []string) error { b.groups[groupUUID] = struct{}{} return nil } // GroupCompleted returns true (always) func (b *Backend) GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) { _, ok := b.groups[groupUUID] if !ok { return false, NewErrGroupNotFound(groupUUID) } return true, nil } // GroupTaskStates returns null states of all tasks in the group func (b *Backend) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error) { _, ok := b.groups[groupUUID] if !ok { return nil, NewErrGroupNotFound(groupUUID) } ret := make([]*tasks.TaskState, 0, groupTaskCount) return ret, nil } // TriggerChord returns true (always) func (b *Backend) TriggerChord(groupUUID string) (bool, error) { return true, nil } // SetStatePending updates task state to PENDING func (b *Backend) SetStatePending(signature *tasks.Signature) error { state := tasks.NewPendingTaskState(signature) return b.updateState(state) } // SetStateReceived updates task state to RECEIVED func (b *Backend) SetStateReceived(signature *tasks.Signature) error { state := tasks.NewReceivedTaskState(signature) return b.updateState(state) } // SetStateStarted updates task state to STARTED func (b *Backend) SetStateStarted(signature *tasks.Signature) error { state := tasks.NewStartedTaskState(signature) return b.updateState(state) } // SetStateRetry updates task state to RETRY func (b *Backend) SetStateRetry(signature *tasks.Signature) error { state := tasks.NewRetryTaskState(signature) return b.updateState(state) } // SetStateSuccess updates task state to SUCCESS func (b *Backend) SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error { state := tasks.NewSuccessTaskState(signature, results) return b.updateState(state) } // SetStateFailure updates task state to FAILURE func (b *Backend) SetStateFailure(signature *tasks.Signature, err string) error { state := tasks.NewFailureTaskState(signature, err) return b.updateState(state) } // GetState returns the latest task state func (b *Backend) GetState(taskUUID string) (*tasks.TaskState, error) { return nil, NewErrTasknotFound(taskUUID) } // PurgeState deletes stored task state func (b *Backend) PurgeState(taskUUID string) error { return NewErrTasknotFound(taskUUID) } // PurgeGroupMeta deletes stored group meta data func (b *Backend) PurgeGroupMeta(groupUUID string) error { _, ok := b.groups[groupUUID] if !ok { return NewErrGroupNotFound(groupUUID) } return nil } func (b *Backend) updateState(s *tasks.TaskState) error { return nil } ================================================ FILE: v2/backends/package.go ================================================ package backends ================================================ FILE: v2/backends/redis/goredis.go ================================================ package redis import ( "bytes" "context" "encoding/json" "strings" "sync" "time" "github.com/go-redsync/redsync/v4" redsyncgoredis "github.com/go-redsync/redsync/v4/redis/goredis/v9" "github.com/redis/go-redis/v9" "github.com/RichardKnop/machinery/v2/backends/iface" "github.com/RichardKnop/machinery/v2/common" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/log" "github.com/RichardKnop/machinery/v2/tasks" ) // BackendGR represents a Redis result backend type BackendGR struct { common.Backend rclient redis.UniversalClient host string password string db int // If set, path to a socket file overrides hostname socketPath string redsync *redsync.Redsync redisOnce sync.Once } // NewGR creates Backend instance func NewGR(cnf *config.Config, addrs []string, db int) iface.Backend { b := &BackendGR{ Backend: common.NewBackend(cnf), } var password string var username string parts := strings.Split(addrs[0], "@") if len(parts) >= 2 { // with password options := strings.SplitN(strings.Join(parts[:len(parts)-1], "@"), ":", 2) if len(options) >= 2 { username = options[0] password = options[1] } else { password = options[0] } addrs[0] = parts[len(parts)-1] // addr is the last one without @ } ropt := &redis.UniversalOptions{ Addrs: addrs, DB: db, Username: username, Password: password, } if cnf.Redis != nil { ropt.MasterName = cnf.Redis.MasterName } if cnf.TLSConfig != nil { ropt.TLSConfig = cnf.TLSConfig } if cnf.Redis != nil && cnf.Redis.SentinelPassword != "" { ropt.SentinelPassword = cnf.Redis.SentinelPassword } if cnf.Redis != nil && cnf.Redis.ClusterEnabled { b.rclient = redis.NewClusterClient(ropt.Cluster()) } else { b.rclient = redis.NewUniversalClient(ropt) } b.redsync = redsync.New(redsyncgoredis.NewPool(b.rclient)) return b } // InitGroup creates and saves a group meta data object func (b *BackendGR) InitGroup(groupUUID string, taskUUIDs []string) error { groupMeta := &tasks.GroupMeta{ GroupUUID: groupUUID, TaskUUIDs: taskUUIDs, CreatedAt: time.Now().UTC(), } encoded, err := json.Marshal(groupMeta) if err != nil { return err } expiration := b.getExpiration() err = b.rclient.Set(context.Background(), groupUUID, encoded, expiration).Err() if err != nil { return err } return nil } // GroupCompleted returns true if all tasks in a group finished func (b *BackendGR) GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) { groupMeta, err := b.getGroupMeta(groupUUID) if err != nil { return false, err } taskStates, err := b.getStates(groupMeta.TaskUUIDs...) if err != nil { return false, err } var countSuccessTasks = 0 for _, taskState := range taskStates { if taskState.IsCompleted() { countSuccessTasks++ } } return countSuccessTasks == groupTaskCount, nil } // GroupTaskStates returns states of all tasks in the group func (b *BackendGR) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error) { groupMeta, err := b.getGroupMeta(groupUUID) if err != nil { return []*tasks.TaskState{}, err } return b.getStates(groupMeta.TaskUUIDs...) } // TriggerChord flags chord as triggered in the backend storage to make sure // chord is never trigerred multiple times. Returns a boolean flag to indicate // whether the worker should trigger chord (true) or no if it has been triggered // already (false) func (b *BackendGR) TriggerChord(groupUUID string) (bool, error) { m := b.redsync.NewMutex("TriggerChordMutex") if err := m.Lock(); err != nil { return false, err } defer m.Unlock() groupMeta, err := b.getGroupMeta(groupUUID) if err != nil { return false, err } // Chord has already been triggered, return false (should not trigger again) if groupMeta.ChordTriggered { return false, nil } // Set flag to true groupMeta.ChordTriggered = true // Update the group meta encoded, err := json.Marshal(&groupMeta) if err != nil { return false, err } expiration := b.getExpiration() err = b.rclient.Set(context.Background(), groupUUID, encoded, expiration).Err() if err != nil { return false, err } return true, nil } func (b *BackendGR) mergeNewTaskState(newState *tasks.TaskState) { state, err := b.GetState(newState.TaskUUID) if err == nil { newState.CreatedAt = state.CreatedAt newState.TaskName = state.TaskName } } // SetStatePending updates task state to PENDING func (b *BackendGR) SetStatePending(signature *tasks.Signature) error { taskState := tasks.NewPendingTaskState(signature) return b.updateState(taskState) } // SetStateReceived updates task state to RECEIVED func (b *BackendGR) SetStateReceived(signature *tasks.Signature) error { taskState := tasks.NewReceivedTaskState(signature) b.mergeNewTaskState(taskState) return b.updateState(taskState) } // SetStateStarted updates task state to STARTED func (b *BackendGR) SetStateStarted(signature *tasks.Signature) error { taskState := tasks.NewStartedTaskState(signature) b.mergeNewTaskState(taskState) return b.updateState(taskState) } // SetStateRetry updates task state to RETRY func (b *BackendGR) SetStateRetry(signature *tasks.Signature) error { taskState := tasks.NewRetryTaskState(signature) b.mergeNewTaskState(taskState) return b.updateState(taskState) } // SetStateSuccess updates task state to SUCCESS func (b *BackendGR) SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error { taskState := tasks.NewSuccessTaskState(signature, results) b.mergeNewTaskState(taskState) return b.updateState(taskState) } // SetStateFailure updates task state to FAILURE func (b *BackendGR) SetStateFailure(signature *tasks.Signature, err string) error { taskState := tasks.NewFailureTaskState(signature, err) b.mergeNewTaskState(taskState) return b.updateState(taskState) } // GetState returns the latest task state func (b *BackendGR) GetState(taskUUID string) (*tasks.TaskState, error) { item, err := b.rclient.Get(context.Background(), taskUUID).Bytes() if err != nil { return nil, err } state := new(tasks.TaskState) decoder := json.NewDecoder(bytes.NewReader(item)) decoder.UseNumber() if err := decoder.Decode(state); err != nil { return nil, err } return state, nil } // PurgeState deletes stored task state func (b *BackendGR) PurgeState(taskUUID string) error { err := b.rclient.Del(context.Background(), taskUUID).Err() if err != nil { return err } return nil } // PurgeGroupMeta deletes stored group meta data func (b *BackendGR) PurgeGroupMeta(groupUUID string) error { err := b.rclient.Del(context.Background(), groupUUID).Err() if err != nil { return err } return nil } // getGroupMeta retrieves group meta data, convenience function to avoid repetition func (b *BackendGR) getGroupMeta(groupUUID string) (*tasks.GroupMeta, error) { item, err := b.rclient.Get(context.Background(), groupUUID).Bytes() if err != nil { return nil, err } groupMeta := new(tasks.GroupMeta) decoder := json.NewDecoder(bytes.NewReader(item)) decoder.UseNumber() if err := decoder.Decode(groupMeta); err != nil { return nil, err } return groupMeta, nil } // getStates returns multiple task states func (b *BackendGR) getStates(taskUUIDs ...string) ([]*tasks.TaskState, error) { taskStates := make([]*tasks.TaskState, len(taskUUIDs)) // to avoid CROSSSLOT error, use pipeline cmders, err := b.rclient.Pipelined(context.Background(), func(pipeliner redis.Pipeliner) error { for _, uuid := range taskUUIDs { pipeliner.Get(context.Background(), uuid) } return nil }) if err != nil { return taskStates, err } for i, cmder := range cmders { stateBytes, err1 := cmder.(*redis.StringCmd).Bytes() if err1 != nil { return taskStates, err1 } taskState := new(tasks.TaskState) decoder := json.NewDecoder(bytes.NewReader(stateBytes)) decoder.UseNumber() if err1 = decoder.Decode(taskState); err1 != nil { log.ERROR.Print(err1) return taskStates, err1 } taskStates[i] = taskState } return taskStates, nil } // updateState saves current task state func (b *BackendGR) updateState(taskState *tasks.TaskState) error { encoded, err := json.Marshal(taskState) if err != nil { return err } expiration := b.getExpiration() _, err = b.rclient.Set(context.Background(), taskState.TaskUUID, encoded, expiration).Result() if err != nil { return err } return nil } // getExpiration returns expiration for a stored task state func (b *BackendGR) getExpiration() time.Duration { expiresIn := b.GetConfig().ResultsExpireIn if expiresIn == 0 { // expire results after 1 hour by default expiresIn = config.DefaultResultsExpireIn } return time.Duration(expiresIn) * time.Second } ================================================ FILE: v2/backends/redis/goredis_test.go ================================================ package redis_test import ( "github.com/RichardKnop/machinery/v2/backends/iface" "os" "strings" "testing" "github.com/RichardKnop/machinery/v2/backends/redis" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/tasks" "github.com/stretchr/testify/assert" ) func getRedisG() iface.Backend { // host1:port1,host2:port2 redisURL := os.Getenv("REDIS_URL_GR") //redisPassword := os.Getenv("REDIS_PASSWORD") if redisURL == "" { return nil } backend := redis.NewGR(new(config.Config), strings.Split(redisURL, ","), 0) return backend } func TestGroupCompletedGR(t *testing.T) { backend := getRedisG() if backend == nil { t.Skip() } groupUUID := "testGroupUUID" task1 := &tasks.Signature{ UUID: "testTaskUUID1", GroupUUID: groupUUID, } task2 := &tasks.Signature{ UUID: "testTaskUUID2", GroupUUID: groupUUID, } // Cleanup before the test backend.PurgeState(task1.UUID) backend.PurgeState(task2.UUID) backend.PurgeGroupMeta(groupUUID) groupCompleted, err := backend.GroupCompleted(groupUUID, 2) if assert.Error(t, err) { assert.False(t, groupCompleted) assert.Equal(t, "redis: nil", err.Error()) } backend.InitGroup(groupUUID, []string{task1.UUID, task2.UUID}) groupCompleted, err = backend.GroupCompleted(groupUUID, 2) if assert.Error(t, err) { assert.False(t, groupCompleted) assert.Equal(t, "redis: nil", err.Error()) } backend.SetStatePending(task1) backend.SetStateStarted(task2) groupCompleted, err = backend.GroupCompleted(groupUUID, 2) if assert.NoError(t, err) { assert.False(t, groupCompleted) } taskResults := []*tasks.TaskResult{new(tasks.TaskResult)} backend.SetStateStarted(task1) backend.SetStateSuccess(task2, taskResults) groupCompleted, err = backend.GroupCompleted(groupUUID, 2) if assert.NoError(t, err) { assert.False(t, groupCompleted) } backend.SetStateFailure(task1, "Some error") groupCompleted, err = backend.GroupCompleted(groupUUID, 2) if assert.NoError(t, err) { assert.True(t, groupCompleted) } } func TestGetStateGR(t *testing.T) { backend := getRedisG() if backend == nil { t.Skip() } signature := &tasks.Signature{ UUID: "testTaskUUID", GroupUUID: "testGroupUUID", } backend.PurgeState("testTaskUUID") var ( taskState *tasks.TaskState err error ) taskState, err = backend.GetState(signature.UUID) assert.Equal(t, "redis: nil", err.Error()) assert.Nil(t, taskState) //Pending State backend.SetStatePending(signature) taskState, err = backend.GetState(signature.UUID) assert.NoError(t, err) assert.Equal(t, signature.Name, taskState.TaskName) createdAt := taskState.CreatedAt //Received State backend.SetStateReceived(signature) taskState, err = backend.GetState(signature.UUID) assert.NoError(t, err) assert.Equal(t, signature.Name, taskState.TaskName) assert.Equal(t, createdAt, taskState.CreatedAt) //Started State backend.SetStateStarted(signature) taskState, err = backend.GetState(signature.UUID) assert.NoError(t, err) assert.Equal(t, signature.Name, taskState.TaskName) assert.Equal(t, createdAt, taskState.CreatedAt) //Success State taskResults := []*tasks.TaskResult{ { Type: "float64", Value: 2, }, } backend.SetStateSuccess(signature, taskResults) taskState, err = backend.GetState(signature.UUID) assert.NoError(t, err) assert.Equal(t, signature.Name, taskState.TaskName) assert.Equal(t, createdAt, taskState.CreatedAt) assert.NotNil(t, taskState.Results) } func TestPurgeStateGR(t *testing.T) { backend := getRedisG() if backend == nil { t.Skip() } signature := &tasks.Signature{ UUID: "testTaskUUID", GroupUUID: "testGroupUUID", } backend.SetStatePending(signature) taskState, err := backend.GetState(signature.UUID) assert.NotNil(t, taskState) assert.NoError(t, err) backend.PurgeState(taskState.TaskUUID) taskState, err = backend.GetState(signature.UUID) assert.Nil(t, taskState) assert.Error(t, err) } ================================================ FILE: v2/backends/redis/redis.go ================================================ package redis import ( "bytes" "encoding/json" "fmt" "sync" "time" "github.com/go-redsync/redsync/v4" redsyncredis "github.com/go-redsync/redsync/v4/redis/redigo" "github.com/gomodule/redigo/redis" "github.com/RichardKnop/machinery/v2/backends/iface" "github.com/RichardKnop/machinery/v2/common" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/log" "github.com/RichardKnop/machinery/v2/tasks" ) // Backend represents a Redis result backend type Backend struct { common.Backend host string username string password string db int pool *redis.Pool // If set, path to a socket file overrides hostname socketPath string redsync *redsync.Redsync redisOnce sync.Once common.RedisConnector } // New creates Backend instance func New(cnf *config.Config, host, username, password, socketPath string, db int) iface.Backend { return &Backend{ Backend: common.NewBackend(cnf), host: host, db: db, username: username, password: password, socketPath: socketPath, } } // InitGroup creates and saves a group meta data object func (b *Backend) InitGroup(groupUUID string, taskUUIDs []string) error { groupMeta := &tasks.GroupMeta{ GroupUUID: groupUUID, TaskUUIDs: taskUUIDs, CreatedAt: time.Now().UTC(), } encoded, err := json.Marshal(groupMeta) if err != nil { return err } conn := b.open() defer conn.Close() expiration := int64(b.getExpiration().Seconds()) _, err = conn.Do("SET", groupUUID, encoded, "EX", expiration) if err != nil { return err } return nil } // GroupCompleted returns true if all tasks in a group finished func (b *Backend) GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) { conn := b.open() defer conn.Close() groupMeta, err := b.getGroupMeta(conn, groupUUID) if err != nil { return false, err } taskStates, err := b.getStates(conn, groupMeta.TaskUUIDs...) if err != nil { return false, err } var countSuccessTasks = 0 for _, taskState := range taskStates { if taskState.IsCompleted() { countSuccessTasks++ } } return countSuccessTasks == groupTaskCount, nil } // GroupTaskStates returns states of all tasks in the group func (b *Backend) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error) { conn := b.open() defer conn.Close() groupMeta, err := b.getGroupMeta(conn, groupUUID) if err != nil { return []*tasks.TaskState{}, err } return b.getStates(conn, groupMeta.TaskUUIDs...) } // TriggerChord flags chord as triggered in the backend storage to make sure // chord is never trigerred multiple times. Returns a boolean flag to indicate // whether the worker should trigger chord (true) or no if it has been triggered // already (false) func (b *Backend) TriggerChord(groupUUID string) (bool, error) { conn := b.open() defer conn.Close() m := b.redsync.NewMutex("TriggerChordMutex") if err := m.Lock(); err != nil { return false, err } defer m.Unlock() groupMeta, err := b.getGroupMeta(conn, groupUUID) if err != nil { return false, err } // Chord has already been triggered, return false (should not trigger again) if groupMeta.ChordTriggered { return false, nil } // Set flag to true groupMeta.ChordTriggered = true // Update the group meta encoded, err := json.Marshal(&groupMeta) if err != nil { return false, err } expiration := int64(b.getExpiration().Seconds()) _, err = conn.Do("SET", groupUUID, encoded, "EX", expiration) if err != nil { return false, err } return true, nil } func (b *Backend) mergeNewTaskState(conn redis.Conn, newState *tasks.TaskState) { state, err := b.getState(conn, newState.TaskUUID) if err == nil { newState.CreatedAt = state.CreatedAt newState.TaskName = state.TaskName } } // SetStatePending updates task state to PENDING func (b *Backend) SetStatePending(signature *tasks.Signature) error { conn := b.open() defer conn.Close() taskState := tasks.NewPendingTaskState(signature) return b.updateState(conn, taskState) } // SetStateReceived updates task state to RECEIVED func (b *Backend) SetStateReceived(signature *tasks.Signature) error { conn := b.open() defer conn.Close() taskState := tasks.NewReceivedTaskState(signature) b.mergeNewTaskState(conn, taskState) return b.updateState(conn, taskState) } // SetStateStarted updates task state to STARTED func (b *Backend) SetStateStarted(signature *tasks.Signature) error { conn := b.open() defer conn.Close() taskState := tasks.NewStartedTaskState(signature) b.mergeNewTaskState(conn, taskState) return b.updateState(conn, taskState) } // SetStateRetry updates task state to RETRY func (b *Backend) SetStateRetry(signature *tasks.Signature) error { conn := b.open() defer conn.Close() taskState := tasks.NewRetryTaskState(signature) b.mergeNewTaskState(conn, taskState) return b.updateState(conn, taskState) } // SetStateSuccess updates task state to SUCCESS func (b *Backend) SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error { conn := b.open() defer conn.Close() taskState := tasks.NewSuccessTaskState(signature, results) b.mergeNewTaskState(conn, taskState) return b.updateState(conn, taskState) } // SetStateFailure updates task state to FAILURE func (b *Backend) SetStateFailure(signature *tasks.Signature, err string) error { conn := b.open() defer conn.Close() taskState := tasks.NewFailureTaskState(signature, err) b.mergeNewTaskState(conn, taskState) return b.updateState(conn, taskState) } // GetState returns the latest task state func (b *Backend) GetState(taskUUID string) (*tasks.TaskState, error) { conn := b.open() defer conn.Close() return b.getState(conn, taskUUID) } func (b *Backend) getState(conn redis.Conn, taskUUID string) (*tasks.TaskState, error) { item, err := redis.Bytes(conn.Do("GET", taskUUID)) if err != nil { return nil, err } state := new(tasks.TaskState) decoder := json.NewDecoder(bytes.NewReader(item)) decoder.UseNumber() if err := decoder.Decode(state); err != nil { return nil, err } return state, nil } // PurgeState deletes stored task state func (b *Backend) PurgeState(taskUUID string) error { conn := b.open() defer conn.Close() _, err := conn.Do("DEL", taskUUID) if err != nil { return err } return nil } // PurgeGroupMeta deletes stored group meta data func (b *Backend) PurgeGroupMeta(groupUUID string) error { conn := b.open() defer conn.Close() _, err := conn.Do("DEL", groupUUID) if err != nil { return err } return nil } // getGroupMeta retrieves group meta data, convenience function to avoid repetition func (b *Backend) getGroupMeta(conn redis.Conn, groupUUID string) (*tasks.GroupMeta, error) { item, err := redis.Bytes(conn.Do("GET", groupUUID)) if err != nil { return nil, err } groupMeta := new(tasks.GroupMeta) decoder := json.NewDecoder(bytes.NewReader(item)) decoder.UseNumber() if err := decoder.Decode(groupMeta); err != nil { return nil, err } return groupMeta, nil } // getStates returns multiple task states func (b *Backend) getStates(conn redis.Conn, taskUUIDs ...string) ([]*tasks.TaskState, error) { taskStates := make([]*tasks.TaskState, len(taskUUIDs)) // conn.Do requires []interface{}... can't pass []string unfortunately taskUUIDInterfaces := make([]interface{}, len(taskUUIDs)) for i, taskUUID := range taskUUIDs { taskUUIDInterfaces[i] = interface{}(taskUUID) } reply, err := redis.Values(conn.Do("MGET", taskUUIDInterfaces...)) if err != nil { return taskStates, err } for i, value := range reply { stateBytes, ok := value.([]byte) if !ok { return taskStates, fmt.Errorf("Expected byte array, instead got: %v", value) } taskState := new(tasks.TaskState) decoder := json.NewDecoder(bytes.NewReader(stateBytes)) decoder.UseNumber() if err := decoder.Decode(taskState); err != nil { log.ERROR.Print(err) return taskStates, err } taskStates[i] = taskState } return taskStates, nil } // updateState saves current task state func (b *Backend) updateState(conn redis.Conn, taskState *tasks.TaskState) error { encoded, err := json.Marshal(taskState) if err != nil { return err } expiration := int64(b.getExpiration().Seconds()) _, err = conn.Do("SET", taskState.TaskUUID, encoded, "EX", expiration) if err != nil { return err } return nil } // getExpiration returns expiration for a stored task state func (b *Backend) getExpiration() time.Duration { expiresIn := b.GetConfig().ResultsExpireIn if expiresIn == 0 { // expire results after 1 hour by default expiresIn = config.DefaultResultsExpireIn } return time.Duration(expiresIn) * time.Second } // open returns or creates instance of Redis connection func (b *Backend) open() redis.Conn { b.redisOnce.Do(func() { b.pool = b.NewPool(b.socketPath, b.host, b.username, b.password, b.db, b.GetConfig().Redis, b.GetConfig().TLSConfig) b.redsync = redsync.New(redsyncredis.NewPool(b.pool)) }) return b.pool.Get() } ================================================ FILE: v2/backends/redis/redis_test.go ================================================ package redis_test import ( "os" "testing" "github.com/RichardKnop/machinery/v2/backends/redis" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/tasks" "github.com/stretchr/testify/assert" ) func TestGroupCompleted(t *testing.T) { redisURL := os.Getenv("REDIS_URL") redisUsername := os.Getenv("REDIS_USER") redisPassword := os.Getenv("REDIS_PASSWORD") if redisURL == "" { t.Skip("REDIS_URL is not defined") } groupUUID := "testGroupUUID" task1 := &tasks.Signature{ UUID: "testTaskUUID1", GroupUUID: groupUUID, } task2 := &tasks.Signature{ UUID: "testTaskUUID2", GroupUUID: groupUUID, } backend := redis.New(new(config.Config), redisURL, redisUsername, redisPassword, "", 0) // Cleanup before the test backend.PurgeState(task1.UUID) backend.PurgeState(task2.UUID) backend.PurgeGroupMeta(groupUUID) groupCompleted, err := backend.GroupCompleted(groupUUID, 2) if assert.Error(t, err) { assert.False(t, groupCompleted) assert.Equal(t, "redigo: nil returned", err.Error()) } backend.InitGroup(groupUUID, []string{task1.UUID, task2.UUID}) groupCompleted, err = backend.GroupCompleted(groupUUID, 2) if assert.Error(t, err) { assert.False(t, groupCompleted) assert.Equal(t, "Expected byte array, instead got: ", err.Error()) } backend.SetStatePending(task1) backend.SetStateStarted(task2) groupCompleted, err = backend.GroupCompleted(groupUUID, 2) if assert.NoError(t, err) { assert.False(t, groupCompleted) } taskResults := []*tasks.TaskResult{new(tasks.TaskResult)} backend.SetStateStarted(task1) backend.SetStateSuccess(task2, taskResults) groupCompleted, err = backend.GroupCompleted(groupUUID, 2) if assert.NoError(t, err) { assert.False(t, groupCompleted) } backend.SetStateFailure(task1, "Some error") groupCompleted, err = backend.GroupCompleted(groupUUID, 2) if assert.NoError(t, err) { assert.True(t, groupCompleted) } } func TestGetState(t *testing.T) { redisURL := os.Getenv("REDIS_URL") redisUsername := os.Getenv("REDIS_USER") redisPassword := os.Getenv("REDIS_PASSWORD") if redisURL == "" { return } signature := &tasks.Signature{ UUID: "testTaskUUID", GroupUUID: "testGroupUUID", } backend := redis.New(new(config.Config), redisURL, redisUsername, redisPassword, "", 0) backend.PurgeState("testTaskUUID") var ( taskState *tasks.TaskState err error ) taskState, err = backend.GetState(signature.UUID) assert.Equal(t, "redigo: nil returned", err.Error()) assert.Nil(t, taskState) //Pending State backend.SetStatePending(signature) taskState, err = backend.GetState(signature.UUID) assert.NoError(t, err) assert.Equal(t, signature.Name, taskState.TaskName) createdAt := taskState.CreatedAt //Received State backend.SetStateReceived(signature) taskState, err = backend.GetState(signature.UUID) assert.NoError(t, err) assert.Equal(t, signature.Name, taskState.TaskName) assert.Equal(t, createdAt, taskState.CreatedAt) //Started State backend.SetStateStarted(signature) taskState, err = backend.GetState(signature.UUID) assert.NoError(t, err) assert.Equal(t, signature.Name, taskState.TaskName) assert.Equal(t, createdAt, taskState.CreatedAt) //Success State taskResults := []*tasks.TaskResult{ { Type: "float64", Value: 2, }, } backend.SetStateSuccess(signature, taskResults) taskState, err = backend.GetState(signature.UUID) assert.NoError(t, err) assert.Equal(t, signature.Name, taskState.TaskName) assert.Equal(t, createdAt, taskState.CreatedAt) assert.NotNil(t, taskState.Results) } func TestPurgeState(t *testing.T) { redisURL := os.Getenv("REDIS_URL") redisUsername := os.Getenv("REDIS_USER") redisPassword := os.Getenv("REDIS_PASSWORD") if redisURL == "" { return } signature := &tasks.Signature{ UUID: "testTaskUUID", GroupUUID: "testGroupUUID", } backend := redis.New(new(config.Config), redisURL, redisUsername, redisPassword, "", 0) backend.SetStatePending(signature) taskState, err := backend.GetState(signature.UUID) assert.NotNil(t, taskState) assert.NoError(t, err) backend.PurgeState(taskState.TaskUUID) taskState, err = backend.GetState(signature.UUID) assert.Nil(t, taskState) assert.Error(t, err) } ================================================ FILE: v2/backends/result/async_result.go ================================================ package result import ( "errors" "reflect" "time" "github.com/RichardKnop/machinery/v2/backends/iface" "github.com/RichardKnop/machinery/v2/tasks" ) var ( // ErrBackendNotConfigured ... ErrBackendNotConfigured = errors.New("Result backend not configured") // ErrTimeoutReached ... ErrTimeoutReached = errors.New("Timeout reached") ) // AsyncResult represents a task result type AsyncResult struct { Signature *tasks.Signature taskState *tasks.TaskState backend iface.Backend } // ChordAsyncResult represents a result of a chord type ChordAsyncResult struct { groupAsyncResults []*AsyncResult chordAsyncResult *AsyncResult backend iface.Backend } // ChainAsyncResult represents a result of a chain of tasks type ChainAsyncResult struct { asyncResults []*AsyncResult backend iface.Backend } // NewAsyncResult creates AsyncResult instance func NewAsyncResult(signature *tasks.Signature, backend iface.Backend) *AsyncResult { return &AsyncResult{ Signature: signature, taskState: new(tasks.TaskState), backend: backend, } } // NewChordAsyncResult creates ChordAsyncResult instance func NewChordAsyncResult(groupTasks []*tasks.Signature, chordCallback *tasks.Signature, backend iface.Backend) *ChordAsyncResult { asyncResults := make([]*AsyncResult, len(groupTasks)) for i, task := range groupTasks { asyncResults[i] = NewAsyncResult(task, backend) } return &ChordAsyncResult{ groupAsyncResults: asyncResults, chordAsyncResult: NewAsyncResult(chordCallback, backend), backend: backend, } } // NewChainAsyncResult creates ChainAsyncResult instance func NewChainAsyncResult(tasks []*tasks.Signature, backend iface.Backend) *ChainAsyncResult { asyncResults := make([]*AsyncResult, len(tasks)) for i, task := range tasks { asyncResults[i] = NewAsyncResult(task, backend) } return &ChainAsyncResult{ asyncResults: asyncResults, backend: backend, } } // Touch the state and don't wait func (asyncResult *AsyncResult) Touch() ([]reflect.Value, error) { if asyncResult.backend == nil { return nil, ErrBackendNotConfigured } asyncResult.GetState() // Purge state if we are using AMQP backend if asyncResult.backend.IsAMQP() && asyncResult.taskState.IsCompleted() { asyncResult.backend.PurgeState(asyncResult.taskState.TaskUUID) } if asyncResult.taskState.IsFailure() { return nil, errors.New(asyncResult.taskState.Error) } if asyncResult.taskState.IsSuccess() { return tasks.ReflectTaskResults(asyncResult.taskState.Results) } return nil, nil } // Get returns task results (synchronous blocking call) func (asyncResult *AsyncResult) Get(sleepDuration time.Duration) ([]reflect.Value, error) { for { results, err := asyncResult.Touch() if results == nil && err == nil { time.Sleep(sleepDuration) } else { return results, err } } } // GetWithTimeout returns task results with a timeout (synchronous blocking call) func (asyncResult *AsyncResult) GetWithTimeout(timeoutDuration, sleepDuration time.Duration) ([]reflect.Value, error) { timeout := time.NewTimer(timeoutDuration) for { select { case <-timeout.C: return nil, ErrTimeoutReached default: results, err := asyncResult.Touch() if results == nil && err == nil { time.Sleep(sleepDuration) } else { return results, err } } } } // GetState returns latest task state func (asyncResult *AsyncResult) GetState() *tasks.TaskState { if asyncResult.taskState.IsCompleted() { return asyncResult.taskState } taskState, err := asyncResult.backend.GetState(asyncResult.Signature.UUID) if err == nil { asyncResult.taskState = taskState } return asyncResult.taskState } // Get returns results of a chain of tasks (synchronous blocking call) func (chainAsyncResult *ChainAsyncResult) Get(sleepDuration time.Duration) ([]reflect.Value, error) { if chainAsyncResult.backend == nil { return nil, ErrBackendNotConfigured } var ( results []reflect.Value err error ) for _, asyncResult := range chainAsyncResult.asyncResults { results, err = asyncResult.Get(sleepDuration) if err != nil { return nil, err } } return results, err } // Get returns result of a chord (synchronous blocking call) func (chordAsyncResult *ChordAsyncResult) Get(sleepDuration time.Duration) ([]reflect.Value, error) { if chordAsyncResult.backend == nil { return nil, ErrBackendNotConfigured } var err error for _, asyncResult := range chordAsyncResult.groupAsyncResults { _, err = asyncResult.Get(sleepDuration) if err != nil { return nil, err } } return chordAsyncResult.chordAsyncResult.Get(sleepDuration) } // GetWithTimeout returns results of a chain of tasks with timeout (synchronous blocking call) func (chainAsyncResult *ChainAsyncResult) GetWithTimeout(timeoutDuration, sleepDuration time.Duration) ([]reflect.Value, error) { if chainAsyncResult.backend == nil { return nil, ErrBackendNotConfigured } var ( results []reflect.Value err error ) timeout := time.NewTimer(timeoutDuration) ln := len(chainAsyncResult.asyncResults) lastResult := chainAsyncResult.asyncResults[ln-1] for { select { case <-timeout.C: return nil, ErrTimeoutReached default: for _, asyncResult := range chainAsyncResult.asyncResults { _, err = asyncResult.Touch() if err != nil { return nil, err } } results, err = lastResult.Touch() if err != nil { return nil, err } if results != nil { return results, err } time.Sleep(sleepDuration) } } } // GetWithTimeout returns result of a chord with a timeout (synchronous blocking call) func (chordAsyncResult *ChordAsyncResult) GetWithTimeout(timeoutDuration, sleepDuration time.Duration) ([]reflect.Value, error) { if chordAsyncResult.backend == nil { return nil, ErrBackendNotConfigured } var ( results []reflect.Value err error ) timeout := time.NewTimer(timeoutDuration) for { select { case <-timeout.C: return nil, ErrTimeoutReached default: for _, asyncResult := range chordAsyncResult.groupAsyncResults { _, errcur := asyncResult.Touch() if errcur != nil { return nil, err } } results, err = chordAsyncResult.chordAsyncResult.Touch() if err != nil { return nil, nil } if results != nil { return results, err } time.Sleep(sleepDuration) } } } ================================================ FILE: v2/brokers/amqp/amqp.go ================================================ package amqp import ( "bytes" "context" "encoding/json" "errors" "fmt" "sync" "time" "github.com/RichardKnop/machinery/v2/brokers/errs" "github.com/RichardKnop/machinery/v2/brokers/iface" "github.com/RichardKnop/machinery/v2/common" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/log" "github.com/RichardKnop/machinery/v2/tasks" amqp "github.com/rabbitmq/amqp091-go" ) type AMQPConnection struct { queueName string connection *amqp.Connection channel *amqp.Channel queue amqp.Queue confirmation <-chan amqp.Confirmation errorchan <-chan *amqp.Error cleanup chan struct{} } // Broker represents an AMQP broker type Broker struct { common.Broker common.AMQPConnector processingWG sync.WaitGroup // use wait group to make sure task processing completes on interrupt signal connections map[string]*AMQPConnection connectionsMutex sync.RWMutex } // New creates new Broker instance func New(cnf *config.Config) iface.Broker { return &Broker{Broker: common.NewBroker(cnf), AMQPConnector: common.AMQPConnector{}, connections: make(map[string]*AMQPConnection)} } // StartConsuming enters a loop and waits for incoming messages func (b *Broker) StartConsuming(consumerTag string, concurrency int, taskProcessor iface.TaskProcessor) (bool, error) { b.Broker.StartConsuming(consumerTag, concurrency, taskProcessor) queueName := taskProcessor.CustomQueue() if queueName == "" { queueName = b.GetConfig().DefaultQueue } conn, channel, queue, _, amqpCloseChan, err := b.Connect( b.GetConfig().Broker, b.GetConfig().MultipleBrokerSeparator, b.GetConfig().TLSConfig, b.GetConfig().AMQP.Exchange, // exchange name b.GetConfig().AMQP.ExchangeType, // exchange type queueName, // queue name true, // queue durable false, // queue delete when unused b.GetConfig().AMQP.BindingKey, // queue binding key nil, // exchange declare args amqp.Table(b.GetConfig().AMQP.QueueDeclareArgs), // queue declare args amqp.Table(b.GetConfig().AMQP.QueueBindingArgs), // queue binding args ) if err != nil { b.GetRetryFunc()(b.GetRetryStopChan()) return b.GetRetry(), err } defer b.Close(channel, conn) if err = channel.Qos( b.GetConfig().AMQP.PrefetchCount, 0, // prefetch size false, // global ); err != nil { return b.GetRetry(), fmt.Errorf("Channel qos error: %s", err) } deliveries, err := channel.Consume( queue.Name, // queue consumerTag, // consumer tag false, // auto-ack false, // exclusive false, // no-local false, // no-wait nil, // arguments ) if err != nil { return b.GetRetry(), fmt.Errorf("Queue consume error: %s", err) } log.INFO.Print("[*] Waiting for messages. To exit press CTRL+C") if err := b.consume(deliveries, concurrency, taskProcessor, amqpCloseChan); err != nil { return b.GetRetry(), err } // Waiting for any tasks being processed to finish b.processingWG.Wait() return b.GetRetry(), nil } // StopConsuming quits the loop func (b *Broker) StopConsuming() { b.Broker.StopConsuming() // Waiting for any tasks being processed to finish b.processingWG.Wait() } // GetOrOpenConnection will return a connection on a particular queue name. Open connections // are saved to avoid having to reopen connection for multiple queues func (b *Broker) GetOrOpenConnection(queueName string, queueBindingKey string, exchangeDeclareArgs, queueDeclareArgs, queueBindingArgs amqp.Table) (*AMQPConnection, error) { var err error b.connectionsMutex.Lock() defer b.connectionsMutex.Unlock() conn, ok := b.connections[queueName] if !ok { conn = &AMQPConnection{ queueName: queueName, cleanup: make(chan struct{}), } conn.connection, conn.channel, conn.queue, conn.confirmation, conn.errorchan, err = b.Connect( b.GetConfig().Broker, b.GetConfig().MultipleBrokerSeparator, b.GetConfig().TLSConfig, b.GetConfig().AMQP.Exchange, // exchange name b.GetConfig().AMQP.ExchangeType, // exchange type queueName, // queue name true, // queue durable false, // queue delete when unused queueBindingKey, // queue binding key exchangeDeclareArgs, // exchange declare args queueDeclareArgs, // queue declare args queueBindingArgs, // queue binding args ) if err != nil { return nil, fmt.Errorf("Failed to connect to queue %s: %s", queueName, err) } // Reconnect to the channel if it disconnects/errors out go func() { select { case err = <-conn.errorchan: log.INFO.Printf("Error occurred on queue: %s. Reconnecting", queueName) b.connectionsMutex.Lock() delete(b.connections, queueName) b.connectionsMutex.Unlock() _, err := b.GetOrOpenConnection(queueName, queueBindingKey, exchangeDeclareArgs, queueDeclareArgs, queueBindingArgs) if err != nil { log.ERROR.Printf("Failed to reopen queue: %s.", queueName) } case <-conn.cleanup: return } }() b.connections[queueName] = conn } return conn, nil } func (b *Broker) CloseConnections() error { b.connectionsMutex.Lock() defer b.connectionsMutex.Unlock() for key, conn := range b.connections { if err := b.Close(conn.channel, conn.connection); err != nil { log.ERROR.Print("Failed to close channel") return nil } close(conn.cleanup) delete(b.connections, key) } return nil } // Publish places a new message on the default queue func (b *Broker) Publish(ctx context.Context, signature *tasks.Signature) error { // Adjust routing key (this decides which queue the message will be published to) b.AdjustRoutingKey(signature) msg, err := json.Marshal(signature) if err != nil { return fmt.Errorf("JSON marshal error: %s", err) } // Check the ETA signature field, if it is set and it is in the future, // delay the task if signature.ETA != nil { now := time.Now().UTC() if signature.ETA.After(now) { delayMs := int64(signature.ETA.Sub(now) / time.Millisecond) return b.delay(signature, delayMs) } } queue := b.GetConfig().DefaultQueue bindingKey := b.GetConfig().AMQP.BindingKey // queue binding key if b.isDirectExchange() { queue = signature.RoutingKey bindingKey = signature.RoutingKey } connection, err := b.GetOrOpenConnection( queue, bindingKey, // queue binding key nil, // exchange declare args amqp.Table(b.GetConfig().AMQP.QueueDeclareArgs), // queue declare args amqp.Table(b.GetConfig().AMQP.QueueBindingArgs), // queue binding args ) if err != nil { return fmt.Errorf("Failed to get a connection for queue %s: %w", queue, err) } channel := connection.channel confirmsChan := connection.confirmation if err := channel.Publish( b.GetConfig().AMQP.Exchange, // exchange name signature.RoutingKey, // routing key false, // mandatory false, // immediate amqp.Publishing{ Headers: amqp.Table(signature.Headers), ContentType: "application/json", Body: msg, Priority: signature.Priority, DeliveryMode: amqp.Persistent, }, ); err != nil { return fmt.Errorf("Failed to publish task: %w", err) } confirmed := <-confirmsChan if confirmed.Ack { return nil } return fmt.Errorf("Failed delivery of delivery tag: %v", confirmed.DeliveryTag) } // consume takes delivered messages from the channel and manages a worker pool // to process tasks concurrently func (b *Broker) consume(deliveries <-chan amqp.Delivery, concurrency int, taskProcessor iface.TaskProcessor, amqpCloseChan <-chan *amqp.Error) error { pool := make(chan struct{}, concurrency) // initialize worker pool with maxWorkers workers go func() { for i := 0; i < concurrency; i++ { pool <- struct{}{} } }() // make channel with a capacity makes it become a buffered channel so that a worker which wants to // push an error to `errorsChan` doesn't need to be blocked while the for-loop is blocked waiting // a worker, that is, it avoids a possible deadlock errorsChan := make(chan error, 1) for { select { case amqpErr := <-amqpCloseChan: return amqpErr case err := <-errorsChan: return err case d := <-deliveries: if concurrency > 0 { // get worker from pool (blocks until one is available) <-pool } b.processingWG.Add(1) // Consume the task inside a gotourine so multiple tasks // can be processed concurrently go func() { if err := b.consumeOne(d, taskProcessor, true); err != nil { errorsChan <- err } b.processingWG.Done() if concurrency > 0 { // give worker back to pool pool <- struct{}{} } }() case <-b.GetStopChan(): return nil } } } // consumeOne processes a single message using TaskProcessor func (b *Broker) consumeOne(delivery amqp.Delivery, taskProcessor iface.TaskProcessor, ack bool) error { if len(delivery.Body) == 0 { delivery.Nack(true, false) // multiple, requeue return errors.New("Received an empty message") // RabbitMQ down? } var multiple, requeue = false, false // Unmarshal message body into signature struct signature := new(tasks.Signature) decoder := json.NewDecoder(bytes.NewReader(delivery.Body)) decoder.UseNumber() if err := decoder.Decode(signature); err != nil { delivery.Nack(multiple, requeue) return errs.NewErrCouldNotUnmarshalTaskSignature(delivery.Body, err) } // If the task is not registered, we nack it and requeue, // there might be different workers for processing specific tasks if !b.IsTaskRegistered(signature.Name) { if !signature.IgnoreWhenTaskNotRegistered { requeue = true } log.INFO.Printf("Task not registered with this worker. Requeuing: %t with message: %s", requeue, delivery.Body) delivery.Nack(multiple, requeue) return nil } log.DEBUG.Printf("Received new message: %s", delivery.Body) err := taskProcessor.Process(signature) if ack { delivery.Ack(multiple) } return err } // delay a task by delayDuration miliseconds, the way it works is a new queue // is created without any consumers, the message is then published to this queue // with appropriate ttl expiration headers, after the expiration, it is sent to // the proper queue with consumers func (b *Broker) delay(signature *tasks.Signature, delayMs int64) error { if delayMs <= 0 { return errors.New("Cannot delay task by 0ms") } message, err := json.Marshal(signature) if err != nil { return fmt.Errorf("JSON marshal error: %s", err) } queueName := b.GetConfig().AMQP.DelayedQueue declareQueueArgs := amqp.Table{ // Exchange where to send messages after TTL expiration. "x-dead-letter-exchange": b.GetConfig().AMQP.Exchange, // Routing key which use when resending expired messages. "x-dead-letter-routing-key": signature.RoutingKey, } messageProperties := amqp.Publishing{ Headers: amqp.Table(signature.Headers), ContentType: "application/json", Body: message, DeliveryMode: amqp.Persistent, Expiration: fmt.Sprint(delayMs), } if queueName == "" { // It's necessary to redeclare the queue each time (to zero its TTL timer). queueName = fmt.Sprintf( "delay.%d.%s.%s", delayMs, // delay duration in mileseconds b.GetConfig().AMQP.Exchange, signature.RoutingKey, // routing key ) declareQueueArgs = amqp.Table{ // Exchange where to send messages after TTL expiration. "x-dead-letter-exchange": b.GetConfig().AMQP.Exchange, // Routing key which use when resending expired messages. "x-dead-letter-routing-key": signature.RoutingKey, // Time in milliseconds // after that message will expire and be sent to destination. "x-message-ttl": delayMs, // Time after that the queue will be deleted. "x-expires": delayMs * 2, } messageProperties = amqp.Publishing{ Headers: amqp.Table(signature.Headers), ContentType: "application/json", Body: message, DeliveryMode: amqp.Persistent, } } conn, channel, _, _, _, err := b.Connect( b.GetConfig().Broker, b.GetConfig().MultipleBrokerSeparator, b.GetConfig().TLSConfig, b.GetConfig().AMQP.Exchange, // exchange name b.GetConfig().AMQP.ExchangeType, // exchange type queueName, // queue name true, // queue durable b.GetConfig().AMQP.AutoDelete, // queue delete when unused queueName, // queue binding key nil, // exchange declare args declareQueueArgs, // queue declare args amqp.Table(b.GetConfig().AMQP.QueueBindingArgs), // queue binding args ) if err != nil { return err } defer b.Close(channel, conn) if err := channel.Publish( b.GetConfig().AMQP.Exchange, // exchange queueName, // routing key false, // mandatory false, // immediate messageProperties, ); err != nil { return err } return nil } func (b *Broker) isDirectExchange() bool { return b.GetConfig().AMQP != nil && b.GetConfig().AMQP.ExchangeType == "direct" } // AdjustRoutingKey makes sure the routing key is correct. // If the routing key is an empty string: // a) set it to binding key for direct exchange type // b) set it to default queue name func (b *Broker) AdjustRoutingKey(s *tasks.Signature) { if s.RoutingKey != "" { return } if b.isDirectExchange() { // The routing algorithm behind a direct exchange is simple - a message goes // to the queues whose binding key exactly matches the routing key of the message. s.RoutingKey = b.GetConfig().AMQP.BindingKey return } s.RoutingKey = b.GetConfig().DefaultQueue } // Helper type for GetPendingTasks to accumulate signatures type sigDumper struct { customQueue string Signatures []*tasks.Signature } func (s *sigDumper) Process(sig *tasks.Signature) error { s.Signatures = append(s.Signatures, sig) return nil } func (s *sigDumper) CustomQueue() string { return s.customQueue } func (_ *sigDumper) PreConsumeHandler() bool { return true } func (b *Broker) GetPendingTasks(queue string) ([]*tasks.Signature, error) { if queue == "" { queue = b.GetConfig().DefaultQueue } bindingKey := b.GetConfig().AMQP.BindingKey // queue binding key conn, err := b.GetOrOpenConnection( queue, bindingKey, // queue binding key nil, // exchange declare args amqp.Table(b.GetConfig().AMQP.QueueDeclareArgs), // queue declare args amqp.Table(b.GetConfig().AMQP.QueueBindingArgs), // queue binding args ) if err != nil { return nil, fmt.Errorf("Failed to get a connection for queue %s: %w", queue, err) } channel := conn.channel queueInfo, err := channel.QueueInspect(queue) if err != nil { return nil, fmt.Errorf("Failed to get info for queue %s: %w", queue, err) } var tag uint64 defer channel.Nack(tag, true, true) // multiple, requeue dumper := &sigDumper{customQueue: queue} for i := 0; i < queueInfo.Messages; i++ { d, _, err := channel.Get(queue, false) if err != nil { return nil, fmt.Errorf("Failed to get from queue: %w", err) } tag = d.DeliveryTag b.consumeOne(d, dumper, false) } return dumper.Signatures, nil } ================================================ FILE: v2/brokers/amqp/amqp_concurrence_test.go ================================================ package amqp import ( "fmt" "github.com/RichardKnop/machinery/v2/brokers/iface" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/tasks" amqp "github.com/rabbitmq/amqp091-go" "testing" "time" ) type doNothingProcessor struct{} func (_ doNothingProcessor) Process(signature *tasks.Signature) error { return fmt.Errorf("failed") } func (_ doNothingProcessor) CustomQueue() string { return "oops" } func (_ doNothingProcessor) PreConsumeHandler() bool { return true } func TestConsume(t *testing.T) { var ( iBroker iface.Broker deliveries = make(chan amqp.Delivery, 3) closeChan chan *amqp.Error processor doNothingProcessor ) t.Run("with deliveries more than the number of concurrency", func(t *testing.T) { iBroker = New(&config.Config{}) broker, _ := iBroker.(*Broker) errChan := make(chan error) // simulate that there are too much deliveries go func() { for i := 0; i < 3; i++ { deliveries <- amqp.Delivery{} // broker.consumeOne() will complain this error: Received an empty message } }() go func() { err := broker.consume(deliveries, 2, processor, closeChan) if err != nil { errChan <- err } }() select { case <-errChan: case <-time.After(1 * time.Second): t.Error("Maybe deadlock") } }) } ================================================ FILE: v2/brokers/amqp/amqp_test.go ================================================ package amqp_test import ( "testing" "github.com/RichardKnop/machinery/v2/brokers/amqp" "github.com/RichardKnop/machinery/v2/brokers/iface" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/tasks" "github.com/stretchr/testify/assert" ) func TestAdjustRoutingKey(t *testing.T) { t.Parallel() var ( s *tasks.Signature broker iface.Broker ) t.Run("with routing and binding keys", func(t *testing.T) { s := &tasks.Signature{RoutingKey: "routing_key"} broker = amqp.New(&config.Config{ DefaultQueue: "queue", AMQP: &config.AMQPConfig{ ExchangeType: "direct", BindingKey: "binding_key", }, }) broker.AdjustRoutingKey(s) assert.Equal(t, "routing_key", s.RoutingKey) }) t.Run("with binding key", func(t *testing.T) { s = new(tasks.Signature) broker = amqp.New(&config.Config{ DefaultQueue: "queue", AMQP: &config.AMQPConfig{ ExchangeType: "direct", BindingKey: "binding_key", }, }) broker.AdjustRoutingKey(s) assert.Equal(t, "binding_key", s.RoutingKey) }) } ================================================ FILE: v2/brokers/eager/eager.go ================================================ package eager import ( "bytes" "context" "encoding/json" "errors" "fmt" "github.com/RichardKnop/machinery/v2/brokers/iface" "github.com/RichardKnop/machinery/v2/common" "github.com/RichardKnop/machinery/v2/tasks" ) // Broker represents an "eager" in-memory broker type Broker struct { worker iface.TaskProcessor common.Broker } // New creates new Broker instance func New() iface.Broker { return new(Broker) } // Mode interface with methods specific for this broker type Mode interface { AssignWorker(p iface.TaskProcessor) } // StartConsuming enters a loop and waits for incoming messages func (eagerBroker *Broker) StartConsuming(consumerTag string, concurrency int, p iface.TaskProcessor) (bool, error) { return true, nil } // StopConsuming quits the loop func (eagerBroker *Broker) StopConsuming() { // do nothing } // Publish places a new message on the default queue func (eagerBroker *Broker) Publish(ctx context.Context, task *tasks.Signature) error { if eagerBroker.worker == nil { return errors.New("worker is not assigned in eager-mode") } // faking the behavior to marshal input into json // and unmarshal it back message, err := json.Marshal(task) if err != nil { return fmt.Errorf("JSON marshal error: %s", err) } signature := new(tasks.Signature) decoder := json.NewDecoder(bytes.NewReader(message)) decoder.UseNumber() if err := decoder.Decode(signature); err != nil { return fmt.Errorf("JSON unmarshal error: %s", err) } // blocking call to the task directly return eagerBroker.worker.Process(signature) } // AssignWorker assigns a worker to the eager broker func (eagerBroker *Broker) AssignWorker(w iface.TaskProcessor) { eagerBroker.worker = w } ================================================ FILE: v2/brokers/errs/errors.go ================================================ package errs import ( "errors" "fmt" ) // ErrCouldNotUnmarshalTaskSignature ... type ErrCouldNotUnmarshalTaskSignature struct { msg []byte reason string } // Error implements the error interface func (e ErrCouldNotUnmarshalTaskSignature) Error() string { return fmt.Sprintf("Could not unmarshal '%s' into a task signature: %v", e.msg, e.reason) } // NewErrCouldNotUnmarshalTaskSignature returns new ErrCouldNotUnmarshalTaskSignature instance func NewErrCouldNotUnmarshalTaskSignature(msg []byte, err error) ErrCouldNotUnmarshalTaskSignature { return ErrCouldNotUnmarshalTaskSignature{msg: msg, reason: err.Error()} } // ErrConsumerStopped indicates that the operation is now illegal because of the consumer being stopped. var ErrConsumerStopped = errors.New("the server has been stopped") // ErrStopTaskDeletion indicates that the task should not be deleted from source after task failure var ErrStopTaskDeletion = errors.New("task should not be deleted") ================================================ FILE: v2/brokers/gcppubsub/gcp_pubsub.go ================================================ package gcppubsub import ( "bytes" "context" "encoding/json" "fmt" "time" "cloud.google.com/go/pubsub" "github.com/RichardKnop/machinery/v2/brokers/iface" "github.com/RichardKnop/machinery/v2/common" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/log" "github.com/RichardKnop/machinery/v2/tasks" ) // Broker represents an Google Cloud Pub/Sub broker type Broker struct { common.Broker service *pubsub.Client subscriptionName string MaxExtension time.Duration stopDone chan struct{} } // New creates new Broker instance func New(cnf *config.Config, projectID, subscriptionName string) (iface.Broker, error) { b := &Broker{Broker: common.NewBroker(cnf), stopDone: make(chan struct{})} b.subscriptionName = subscriptionName ctx := context.Background() if cnf.GCPPubSub != nil { b.MaxExtension = cnf.GCPPubSub.MaxExtension } if cnf.GCPPubSub != nil && cnf.GCPPubSub.Client != nil { b.service = cnf.GCPPubSub.Client } else { pubsubClient, err := pubsub.NewClient(ctx, projectID) if err != nil { return nil, err } b.service = pubsubClient cnf.GCPPubSub = &config.GCPPubSubConfig{ Client: pubsubClient, } } // Validate topic exists defaultQueue := b.GetConfig().DefaultQueue topic := b.service.Topic(defaultQueue) defer topic.Stop() topicExists, err := topic.Exists(ctx) if err != nil { return nil, err } if !topicExists { return nil, fmt.Errorf("topic does not exist, instead got %s", defaultQueue) } // Validate subscription exists sub := b.service.Subscription(b.subscriptionName) if b.MaxExtension != 0 { sub.ReceiveSettings.MaxExtension = b.MaxExtension } subscriptionExists, err := sub.Exists(ctx) if err != nil { return nil, err } if !subscriptionExists { return nil, fmt.Errorf("subscription does not exist, instead got %s", b.subscriptionName) } return b, nil } // StartConsuming enters a loop and waits for incoming messages func (b *Broker) StartConsuming(consumerTag string, concurrency int, taskProcessor iface.TaskProcessor) (bool, error) { b.Broker.StartConsuming(consumerTag, concurrency, taskProcessor) sub := b.service.Subscription(b.subscriptionName) if b.MaxExtension != 0 { sub.ReceiveSettings.MaxExtension = b.MaxExtension } sub.ReceiveSettings.NumGoroutines = concurrency log.INFO.Print("[*] Waiting for messages. To exit press CTRL+C") ctx, cancel := context.WithCancel(context.Background()) go func() { <-b.GetStopChan() cancel() }() for { err := sub.Receive(ctx, func(_ctx context.Context, msg *pubsub.Message) { b.consumeOne(msg, taskProcessor) }) if err == nil { break } log.ERROR.Printf("Error when receiving messages. Error: %v", err) continue } close(b.stopDone) return b.GetRetry(), nil } // StopConsuming quits the loop func (b *Broker) StopConsuming() { b.Broker.StopConsuming() // Waiting for any tasks being processed to finish <-b.stopDone } // Publish places a new message on the default queue or the queue pointed to // by the routing key func (b *Broker) Publish(ctx context.Context, signature *tasks.Signature) error { // Adjust routing key (this decides which queue the message will be published to) b.AdjustRoutingKey(signature) msg, err := json.Marshal(signature) if err != nil { return fmt.Errorf("JSON marshal error: %s", err) } topic := b.service.Topic(signature.RoutingKey) defer topic.Stop() // Check the ETA signature field, if it is set and it is in the future, // delay the task if signature.ETA != nil { now := time.Now().UTC() if signature.ETA.After(now) { topic.PublishSettings.DelayThreshold = signature.ETA.Sub(now) } } result := topic.Publish(ctx, &pubsub.Message{ Data: msg, }) id, err := result.Get(ctx) if err != nil { log.ERROR.Printf("Error when sending a message: %v", err) return err } log.INFO.Printf("Sending a message successfully, server-generated message ID %v", id) return nil } // consumeOne processes a single message using TaskProcessor func (b *Broker) consumeOne(delivery *pubsub.Message, taskProcessor iface.TaskProcessor) { if len(delivery.Data) == 0 { delivery.Nack() log.ERROR.Printf("received an empty message, the delivery was %v", delivery) } sig := new(tasks.Signature) decoder := json.NewDecoder(bytes.NewBuffer(delivery.Data)) decoder.UseNumber() if err := decoder.Decode(sig); err != nil { delivery.Nack() log.ERROR.Printf("unmarshal error. the delivery is %v", delivery) } // If the task is not registered return an error // and leave the message in the queue if !b.IsTaskRegistered(sig.Name) { delivery.Nack() log.ERROR.Printf("task %s is not registered", sig.Name) } err := taskProcessor.Process(sig) if err != nil { delivery.Nack() log.ERROR.Printf("Failed process of task", err) } // Call Ack() after successfully consuming and processing the message delivery.Ack() } ================================================ FILE: v2/brokers/iface/interfaces.go ================================================ package iface import ( "context" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/tasks" ) // Broker - a common interface for all brokers type Broker interface { GetConfig() *config.Config SetRegisteredTaskNames(names []string) IsTaskRegistered(name string) bool StartConsuming(consumerTag string, concurrency int, p TaskProcessor) (bool, error) StopConsuming() Publish(ctx context.Context, task *tasks.Signature) error GetPendingTasks(queue string) ([]*tasks.Signature, error) GetDelayedTasks() ([]*tasks.Signature, error) AdjustRoutingKey(s *tasks.Signature) } // TaskProcessor - can process a delivered task // This will probably always be a worker instance type TaskProcessor interface { Process(signature *tasks.Signature) error CustomQueue() string PreConsumeHandler() bool } ================================================ FILE: v2/brokers/iface/sqs/api.go ================================================ package sqs import ( "context" "github.com/aws/aws-sdk-go-v2/service/sqs" ) // API is an interface for SQS API type API interface { SendMessage(ctx context.Context, params *sqs.SendMessageInput, optFns ...func(*sqs.Options)) (*sqs.SendMessageOutput, error) ReceiveMessage(ctx context.Context, params *sqs.ReceiveMessageInput, optFns ...func(*sqs.Options)) (*sqs.ReceiveMessageOutput, error) DeleteMessage(ctx context.Context, params *sqs.DeleteMessageInput, optFns ...func(*sqs.Options)) (*sqs.DeleteMessageOutput, error) ChangeMessageVisibility(ctx context.Context, params *sqs.ChangeMessageVisibilityInput, optFns ...func(*sqs.Options)) (*sqs.ChangeMessageVisibilityOutput, error) } ================================================ FILE: v2/brokers/package.go ================================================ package brokers ================================================ FILE: v2/brokers/redis/goredis.go ================================================ package redis import ( "bytes" "context" "encoding/json" "fmt" "runtime" "strconv" "strings" "sync" "time" "github.com/go-redsync/redsync/v4" "github.com/redis/go-redis/v9" "github.com/RichardKnop/machinery/v2/brokers/errs" "github.com/RichardKnop/machinery/v2/brokers/iface" "github.com/RichardKnop/machinery/v2/common" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/log" "github.com/RichardKnop/machinery/v2/tasks" ) // BrokerGR represents a Redis broker type BrokerGR struct { common.Broker rclient redis.UniversalClient consumingWG sync.WaitGroup // wait group to make sure whole consumption completes processingWG sync.WaitGroup // use wait group to make sure task processing completes delayedWG sync.WaitGroup // If set, path to a socket file overrides hostname socketPath string redsync *redsync.Redsync redisOnce sync.Once redisDelayedTasksKey string } // NewGR creates new Broker instance func NewGR(cnf *config.Config, addrs []string, db int) iface.Broker { b := &BrokerGR{Broker: common.NewBroker(cnf)} var password string var username string parts := strings.Split(addrs[0], "@") if len(parts) >= 2 { // with password options := strings.SplitN(strings.Join(parts[:len(parts)-1], "@"), ":", 2) if len(options) >= 2 { username = options[0] password = options[1] } else { password = options[0] } addrs[0] = parts[len(parts)-1] // addr is the last one without @ } ropt := &redis.UniversalOptions{ Addrs: addrs, DB: db, Password: password, Username: username, } if cnf.Redis != nil { ropt.MasterName = cnf.Redis.MasterName } if cnf.TLSConfig != nil { ropt.TLSConfig = cnf.TLSConfig } if cnf.Redis != nil && cnf.Redis.SentinelPassword != "" { ropt.SentinelPassword = cnf.Redis.SentinelPassword } if cnf.Redis != nil && cnf.Redis.ClusterEnabled { b.rclient = redis.NewClusterClient(ropt.Cluster()) } else { b.rclient = redis.NewUniversalClient(ropt) } if cnf.Redis != nil && cnf.Redis.DelayedTasksKey != "" { b.redisDelayedTasksKey = cnf.Redis.DelayedTasksKey } else { b.redisDelayedTasksKey = defaultRedisDelayedTasksKey } return b } // StartConsuming enters a loop and waits for incoming messages func (b *BrokerGR) StartConsuming(consumerTag string, concurrency int, taskProcessor iface.TaskProcessor) (bool, error) { b.consumingWG.Add(1) defer b.consumingWG.Done() if concurrency < 1 { concurrency = runtime.NumCPU() * 2 } b.Broker.StartConsuming(consumerTag, concurrency, taskProcessor) // Ping the server to make sure connection is live _, err := b.rclient.Ping(context.Background()).Result() if err != nil { b.GetRetryFunc()(b.GetRetryStopChan()) // Return err if retry is still true. // If retry is false, broker.StopConsuming() has been called and // therefore Redis might have been stopped. Return nil exit // StartConsuming() if b.GetRetry() { return b.GetRetry(), err } return b.GetRetry(), errs.ErrConsumerStopped } // Channel to which we will push tasks ready for processing by worker deliveries := make(chan []byte, concurrency) pool := make(chan struct{}, concurrency) // initialize worker pool with maxWorkers workers for i := 0; i < concurrency; i++ { pool <- struct{}{} } // A receiving goroutine keeps popping messages from the queue by BLPOP // If the message is valid and can be unmarshaled into a proper structure // we send it to the deliveries channel go func() { log.INFO.Print("[*] Waiting for messages. To exit press CTRL+C") for { select { // A way to stop this goroutine from b.StopConsuming case <-b.GetStopChan(): close(deliveries) return case <-pool: task, _ := b.nextTask(getQueueGR(b.GetConfig(), taskProcessor)) //TODO: should this error be ignored? if len(task) > 0 { deliveries <- task } pool <- struct{}{} } } }() // A goroutine to watch for delayed tasks and push them to deliveries // channel for consumption by the worker b.delayedWG.Add(1) go func() { defer b.delayedWG.Done() for { select { // A way to stop this goroutine from b.StopConsuming case <-b.GetStopChan(): return default: task, err := b.nextDelayedTask(b.redisDelayedTasksKey) if err != nil { continue } signature := new(tasks.Signature) decoder := json.NewDecoder(bytes.NewReader(task)) decoder.UseNumber() if err := decoder.Decode(signature); err != nil { log.ERROR.Print(errs.NewErrCouldNotUnmarshalTaskSignature(task, err)) } if err := b.Publish(context.Background(), signature); err != nil { log.ERROR.Print(err) } } } }() if err := b.consume(deliveries, concurrency, taskProcessor); err != nil { return b.GetRetry(), err } // Waiting for any tasks being processed to finish b.processingWG.Wait() return b.GetRetry(), nil } // StopConsuming quits the loop func (b *BrokerGR) StopConsuming() { b.Broker.StopConsuming() // Waiting for the delayed tasks goroutine to have stopped b.delayedWG.Wait() // Waiting for consumption to finish b.consumingWG.Wait() b.rclient.Close() } // Publish places a new message on the default queue func (b *BrokerGR) Publish(ctx context.Context, signature *tasks.Signature) error { // Adjust routing key (this decides which queue the message will be published to) b.Broker.AdjustRoutingKey(signature) msg, err := json.Marshal(signature) if err != nil { return fmt.Errorf("JSON marshal error: %s", err) } // Check the ETA signature field, if it is set and it is in the future, // delay the task if signature.ETA != nil { now := time.Now().UTC() if signature.ETA.After(now) { score := signature.ETA.UnixNano() err = b.rclient.ZAdd(context.Background(), b.redisDelayedTasksKey, redis.Z{Score: float64(score), Member: msg}).Err() return err } } err = b.rclient.RPush(context.Background(), signature.RoutingKey, msg).Err() return err } // GetPendingTasks returns a slice of task signatures waiting in the queue func (b *BrokerGR) GetPendingTasks(queue string) ([]*tasks.Signature, error) { if queue == "" { queue = b.GetConfig().DefaultQueue } results, err := b.rclient.LRange(context.Background(), queue, 0, -1).Result() if err != nil { return nil, err } taskSignatures := make([]*tasks.Signature, len(results)) for i, result := range results { signature := new(tasks.Signature) decoder := json.NewDecoder(strings.NewReader(result)) decoder.UseNumber() if err := decoder.Decode(signature); err != nil { return nil, err } taskSignatures[i] = signature } return taskSignatures, nil } // GetDelayedTasks returns a slice of task signatures that are scheduled, but not yet in the queue func (b *BrokerGR) GetDelayedTasks() ([]*tasks.Signature, error) { results, err := b.rclient.ZRange(context.Background(), b.redisDelayedTasksKey, 0, -1).Result() if err != nil { return nil, err } taskSignatures := make([]*tasks.Signature, len(results)) for i, result := range results { signature := new(tasks.Signature) decoder := json.NewDecoder(strings.NewReader(result)) decoder.UseNumber() if err := decoder.Decode(signature); err != nil { return nil, err } taskSignatures[i] = signature } return taskSignatures, nil } // consume takes delivered messages from the channel and manages a worker pool // to process tasks concurrently func (b *BrokerGR) consume(deliveries <-chan []byte, concurrency int, taskProcessor iface.TaskProcessor) error { errorsChan := make(chan error, concurrency*2) pool := make(chan struct{}, concurrency) // init pool for Worker tasks execution, as many slots as Worker concurrency param go func() { for i := 0; i < concurrency; i++ { pool <- struct{}{} } }() for { select { case err := <-errorsChan: return err case d, open := <-deliveries: if !open { return nil } if concurrency > 0 { // get execution slot from pool (blocks until one is available) <-pool } b.processingWG.Add(1) // Consume the task inside a goroutine so multiple tasks // can be processed concurrently go func() { if err := b.consumeOne(d, taskProcessor); err != nil { errorsChan <- err } b.processingWG.Done() if concurrency > 0 { // give slot back to pool pool <- struct{}{} } }() } } } // consumeOne processes a single message using TaskProcessor func (b *BrokerGR) consumeOne(delivery []byte, taskProcessor iface.TaskProcessor) error { signature := new(tasks.Signature) decoder := json.NewDecoder(bytes.NewReader(delivery)) decoder.UseNumber() if err := decoder.Decode(signature); err != nil { return errs.NewErrCouldNotUnmarshalTaskSignature(delivery, err) } // If the task is not registered, we requeue it, // there might be different workers for processing specific tasks if !b.IsTaskRegistered(signature.Name) { if signature.IgnoreWhenTaskNotRegistered { return nil } log.INFO.Printf("Task not registered with this worker. Requeuing message: %s", delivery) b.rclient.RPush(context.Background(), getQueueGR(b.GetConfig(), taskProcessor), delivery) return nil } log.DEBUG.Printf("Received new message: %s", delivery) return taskProcessor.Process(signature) } // nextTask pops next available task from the default queue func (b *BrokerGR) nextTask(queue string) (result []byte, err error) { pollPeriodMilliseconds := 1000 // default poll period for normal tasks if b.GetConfig().Redis != nil { configuredPollPeriod := b.GetConfig().Redis.NormalTasksPollPeriod if configuredPollPeriod > 0 { pollPeriodMilliseconds = configuredPollPeriod } } pollPeriod := time.Duration(pollPeriodMilliseconds) * time.Millisecond items, err := b.rclient.BLPop(context.Background(), pollPeriod, queue).Result() if err != nil { return []byte{}, err } // items[0] - the name of the key where an element was popped // items[1] - the value of the popped element if len(items) != 2 { return []byte{}, redis.Nil } result = []byte(items[1]) return result, nil } // nextDelayedTask pops a value from the ZSET key using WATCH/MULTI/EXEC commands. func (b *BrokerGR) nextDelayedTask(key string) (result []byte, err error) { //pipe := b.rclient.Pipeline() // //defer func() { // // Return connection to normal state on error. // // https://redis.io/commands/discard // if err != nil { // pipe.Discard() // } //}() var ( items []string ) pollPeriod := 500 // default poll period for delayed tasks if b.GetConfig().Redis != nil { configuredPollPeriod := b.GetConfig().Redis.DelayedTasksPollPeriod // the default period is 0, which bombards redis with requests, despite // our intention of doing the opposite if configuredPollPeriod > 0 { pollPeriod = configuredPollPeriod } } for { // Space out queries to ZSET so we don't bombard redis // server with relentless ZRANGEBYSCOREs time.Sleep(time.Duration(pollPeriod) * time.Millisecond) watchFunc := func(tx *redis.Tx) error { now := time.Now().UTC().UnixNano() // https://redis.io/commands/zrangebyscore ctx := context.Background() items, err = tx.ZRevRangeByScore(ctx, key, &redis.ZRangeBy{ Min: "0", Max: strconv.FormatInt(now, 10), Offset: 0, Count: 1, }).Result() if err != nil { return err } if len(items) != 1 { return redis.Nil } // only return the first zrange value if there are no other changes in this key // to make sure a delayed task would only be consumed once _, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error { pipe.ZRem(ctx, key, items[0]) result = []byte(items[0]) return nil }) return err } if err = b.rclient.Watch(context.Background(), watchFunc, key); err != nil { return } else { break } } return } func getQueueGR(config *config.Config, taskProcessor iface.TaskProcessor) string { customQueue := taskProcessor.CustomQueue() if customQueue == "" { return config.DefaultQueue } return customQueue } ================================================ FILE: v2/brokers/redis/redis.go ================================================ package redis import ( "bytes" "context" "encoding/json" "fmt" "math" "runtime" "sync" "time" "github.com/go-redsync/redsync/v4" redsyncredis "github.com/go-redsync/redsync/v4/redis/redigo" "github.com/gomodule/redigo/redis" "github.com/RichardKnop/machinery/v2/brokers/errs" "github.com/RichardKnop/machinery/v2/brokers/iface" "github.com/RichardKnop/machinery/v2/common" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/log" "github.com/RichardKnop/machinery/v2/tasks" ) const defaultRedisDelayedTasksKey = "delayed_tasks" // Broker represents a Redis broker type Broker struct { common.Broker common.RedisConnector host string username string password string db int pool *redis.Pool consumingWG sync.WaitGroup // wait group to make sure whole consumption completes processingWG sync.WaitGroup // use wait group to make sure task processing completes delayedWG sync.WaitGroup // If set, path to a socket file overrides hostname socketPath string redsync *redsync.Redsync redisOnce sync.Once redisDelayedTasksKey string } // New creates new Broker instance func New(cnf *config.Config, host, username, password, socketPath string, db int) iface.Broker { b := &Broker{Broker: common.NewBroker(cnf)} b.host = host b.db = db b.username = username b.password = password b.socketPath = socketPath if cnf.Redis != nil && cnf.Redis.DelayedTasksKey != "" { b.redisDelayedTasksKey = cnf.Redis.DelayedTasksKey } else { b.redisDelayedTasksKey = defaultRedisDelayedTasksKey } return b } // StartConsuming enters a loop and waits for incoming messages func (b *Broker) StartConsuming(consumerTag string, concurrency int, taskProcessor iface.TaskProcessor) (bool, error) { b.consumingWG.Add(1) defer b.consumingWG.Done() if concurrency < 1 { concurrency = runtime.NumCPU() * 2 } b.Broker.StartConsuming(consumerTag, concurrency, taskProcessor) conn := b.open() defer conn.Close() // Ping the server to make sure connection is live _, err := conn.Do("PING") if err != nil { b.GetRetryFunc()(b.GetRetryStopChan()) // Return err if retry is still true. // If retry is false, broker.StopConsuming() has been called and // therefore Redis might have been stopped. Return nil exit // StartConsuming() if b.GetRetry() { return b.GetRetry(), err } return b.GetRetry(), errs.ErrConsumerStopped } // Channel to which we will push tasks ready for processing by worker deliveries := make(chan []byte, concurrency) pool := make(chan struct{}, concurrency) stopConsumer := make(chan struct{}) // initialize worker pool with maxWorkers workers for i := 0; i < concurrency; i++ { pool <- struct{}{} } // A receiving goroutine keeps popping messages from the queue by BLPOP // If the message is valid and can be unmarshaled into a proper structure // we send it to the deliveries channel go func() { log.INFO.Print("[*] Waiting for messages. To exit press CTRL+C") for { select { // A way to stop this goroutine from b.StopConsuming case <-b.GetStopChan(): close(deliveries) return case <-stopConsumer: close(deliveries) return case <-pool: select { case <-b.GetStopChan(): close(deliveries) return default: } if taskProcessor.PreConsumeHandler() { task, _ := b.nextTask(getQueue(b.GetConfig(), taskProcessor)) //TODO: should this error be ignored? if len(task) > 0 { deliveries <- task } } pool <- struct{}{} } } }() // A goroutine to watch for delayed tasks and push them to deliveries // channel for consumption by the worker b.delayedWG.Add(1) go func() { defer b.delayedWG.Done() for { select { // A way to stop this goroutine from b.StopConsuming case <-b.GetStopChan(): return case <-stopConsumer: close(deliveries) return default: task, err := b.nextDelayedTask(b.redisDelayedTasksKey) if err != nil { continue } signature := new(tasks.Signature) decoder := json.NewDecoder(bytes.NewReader(task)) decoder.UseNumber() if err := decoder.Decode(signature); err != nil { log.ERROR.Print(errs.NewErrCouldNotUnmarshalTaskSignature(task, err)) } if err := b.Publish(context.Background(), signature); err != nil { log.ERROR.Print(err) } } } }() if err := b.consume(deliveries, concurrency, taskProcessor, stopConsumer); err != nil { return b.GetRetry(), err } // Waiting for any tasks being processed to finish b.processingWG.Wait() return b.GetRetry(), nil } // StopConsuming quits the loop func (b *Broker) StopConsuming() { b.Broker.StopConsuming() // Waiting for the delayed tasks goroutine to have stopped b.delayedWG.Wait() // Waiting for consumption to finish b.consumingWG.Wait() // Wait for currently processing tasks to finish as well. b.processingWG.Wait() if b.pool != nil { b.pool.Close() } } // Publish places a new message on the default queue func (b *Broker) Publish(ctx context.Context, signature *tasks.Signature) error { // Adjust routing key (this decides which queue the message will be published to) b.Broker.AdjustRoutingKey(signature) msg, err := json.Marshal(signature) if err != nil { return fmt.Errorf("JSON marshal error: %s", err) } conn := b.open() defer conn.Close() // Check the ETA signature field, if it is set and it is in the future, // delay the task if signature.ETA != nil { now := time.Now().UTC() if signature.ETA.After(now) { score := signature.ETA.UnixNano() _, err = conn.Do("ZADD", b.redisDelayedTasksKey, score, msg) return err } } _, err = conn.Do("RPUSH", signature.RoutingKey, msg) return err } // GetPendingTasks returns a slice of task signatures waiting in the queue func (b *Broker) GetPendingTasks(queue string) ([]*tasks.Signature, error) { conn := b.open() defer conn.Close() if queue == "" { queue = b.GetConfig().DefaultQueue } dataBytes, err := conn.Do("LRANGE", queue, 0, -1) if err != nil { return nil, err } results, err := redis.ByteSlices(dataBytes, err) if err != nil { return nil, err } taskSignatures := make([]*tasks.Signature, len(results)) for i, result := range results { signature := new(tasks.Signature) decoder := json.NewDecoder(bytes.NewReader(result)) decoder.UseNumber() if err := decoder.Decode(signature); err != nil { return nil, err } taskSignatures[i] = signature } return taskSignatures, nil } // GetDelayedTasks returns a slice of task signatures that are scheduled, but not yet in the queue func (b *Broker) GetDelayedTasks() ([]*tasks.Signature, error) { conn := b.open() defer conn.Close() dataBytes, err := conn.Do("ZRANGE", b.redisDelayedTasksKey, 0, -1) if err != nil { return nil, err } results, err := redis.ByteSlices(dataBytes, err) if err != nil { return nil, err } taskSignatures := make([]*tasks.Signature, len(results)) for i, result := range results { signature := new(tasks.Signature) decoder := json.NewDecoder(bytes.NewReader(result)) decoder.UseNumber() if err := decoder.Decode(signature); err != nil { return nil, err } taskSignatures[i] = signature } return taskSignatures, nil } // consume takes delivered messages from the channel and manages a worker pool // to process tasks concurrently func (b *Broker) consume(deliveries <-chan []byte, concurrency int, taskProcessor iface.TaskProcessor, stopConsumer chan struct{}) error { errorsChan := make(chan error, concurrency*2) pool := make(chan struct{}, concurrency) // init pool for Worker tasks execution, as many slots as Worker concurrency param go func() { for i := 0; i < concurrency; i++ { pool <- struct{}{} } }() for { select { case err := <-errorsChan: for v := range deliveries { b.requeueMessage(v, taskProcessor) } return err case d, open := <-deliveries: if !open { return nil } if concurrency > 0 { // get execution slot from pool (blocks until one is available) select { case <-b.GetStopChan(): b.requeueMessage(d, taskProcessor) continue case <-pool: } } b.processingWG.Add(1) // Consume the task inside a goroutine so multiple tasks // can be processed concurrently go func() { if err := b.consumeOne(d, taskProcessor); err != nil { errorsChan <- err } b.processingWG.Done() if concurrency > 0 { // give slot back to pool pool <- struct{}{} } }() } } } // consumeOne processes a single message using TaskProcessor func (b *Broker) consumeOne(delivery []byte, taskProcessor iface.TaskProcessor) error { signature := new(tasks.Signature) decoder := json.NewDecoder(bytes.NewReader(delivery)) decoder.UseNumber() if err := decoder.Decode(signature); err != nil { return errs.NewErrCouldNotUnmarshalTaskSignature(delivery, err) } // If the task is not registered, we requeue it, // there might be different workers for processing specific tasks if !b.IsTaskRegistered(signature.Name) { if signature.IgnoreWhenTaskNotRegistered { return nil } log.INFO.Printf("Task not registered with this worker. Requeuing message: %s", delivery) b.requeueMessage(delivery, taskProcessor) return nil } log.DEBUG.Printf("Received new message: %s", delivery) return taskProcessor.Process(signature) } // nextTask pops next available task from the default queue func (b *Broker) nextTask(queue string) (result []byte, err error) { conn := b.open() defer conn.Close() pollPeriodMilliseconds := 1000 // default poll period for normal tasks if b.GetConfig().Redis != nil { configuredPollPeriod := b.GetConfig().Redis.NormalTasksPollPeriod if configuredPollPeriod > 0 { pollPeriodMilliseconds = configuredPollPeriod } } pollPeriod := time.Duration(pollPeriodMilliseconds) * time.Millisecond // Issue 548: BLPOP expects an integer timeout expresses in seconds. // The call will if the value is a float. Convert to integer using // math.Ceil(): // math.Ceil(0.0) --> 0 (block indefinitely) // math.Ceil(0.2) --> 1 (timeout after 1 second) pollPeriodSeconds := math.Ceil(pollPeriod.Seconds()) items, err := redis.ByteSlices(conn.Do("BLPOP", queue, pollPeriodSeconds)) if err != nil { return []byte{}, err } // items[0] - the name of the key where an element was popped // items[1] - the value of the popped element if len(items) != 2 { return []byte{}, redis.ErrNil } result = items[1] return result, nil } // nextDelayedTask pops a value from the ZSET key using WATCH/MULTI/EXEC commands. // https://github.com/gomodule/redigo/blob/master/redis/zpop_example_test.go func (b *Broker) nextDelayedTask(key string) (result []byte, err error) { conn := b.open() defer conn.Close() defer func() { // Return connection to normal state on error. // https://redis.io/commands/discard // https://redis.io/commands/unwatch if err == redis.ErrNil { conn.Do("UNWATCH") } else if err != nil { conn.Do("DISCARD") } }() var ( items [][]byte reply interface{} ) pollPeriod := 500 // default poll period for delayed tasks if b.GetConfig().Redis != nil { configuredPollPeriod := b.GetConfig().Redis.DelayedTasksPollPeriod // the default period is 0, which bombards redis with requests, despite // our intention of doing the opposite if configuredPollPeriod > 0 { pollPeriod = configuredPollPeriod } } for { // Space out queries to ZSET so we don't bombard redis // server with relentless ZRANGEBYSCOREs time.Sleep(time.Duration(pollPeriod) * time.Millisecond) if _, err = conn.Do("WATCH", key); err != nil { return } now := time.Now().UTC().UnixNano() // https://redis.io/commands/zrangebyscore items, err = redis.ByteSlices(conn.Do( "ZRANGEBYSCORE", key, 0, now, "LIMIT", 0, 1, )) if err != nil { return } if len(items) != 1 { err = redis.ErrNil return } _ = conn.Send("MULTI") _ = conn.Send("ZREM", key, items[0]) reply, err = conn.Do("EXEC") if err != nil { return } if reply != nil { result = items[0] break } } return } // open returns or creates instance of Redis connection func (b *Broker) open() redis.Conn { b.redisOnce.Do(func() { b.pool = b.NewPool(b.socketPath, b.host, b.username, b.password, b.db, b.GetConfig().Redis, b.GetConfig().TLSConfig) b.redsync = redsync.New(redsyncredis.NewPool(b.pool)) }) return b.pool.Get() } func getQueue(config *config.Config, taskProcessor iface.TaskProcessor) string { customQueue := taskProcessor.CustomQueue() if customQueue == "" { return config.DefaultQueue } return customQueue } func (b *Broker) requeueMessage(delivery []byte, taskProcessor iface.TaskProcessor) { conn := b.open() defer conn.Close() conn.Do("RPUSH", getQueue(b.GetConfig(), taskProcessor), delivery) } ================================================ FILE: v2/brokers/sqs/sqs.go ================================================ package sqs import ( "context" "encoding/json" "errors" "fmt" "strings" "sync" "time" "github.com/RichardKnop/machinery/v2/brokers/errs" "github.com/RichardKnop/machinery/v2/brokers/iface" sqsiface "github.com/RichardKnop/machinery/v2/brokers/iface/sqs" "github.com/RichardKnop/machinery/v2/common" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/log" "github.com/RichardKnop/machinery/v2/tasks" "github.com/aws/aws-sdk-go-v2/aws" awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/aws/aws-sdk-go-v2/service/sqs/types" ) const ( maxAWSSQSDelay = time.Minute * 15 // Max supported SQS delay is 15 min: https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html ) // Broker represents a AWS SQS broker // There are examples on: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/sqs-example-create-queue.html type Broker struct { common.Broker processingWG sync.WaitGroup // use wait group to make sure task processing completes on interrupt signal receivingWG sync.WaitGroup stopReceivingChan chan int service sqsiface.API queueUrl *string } // New creates new Broker instance func New(cnf *config.Config) (iface.Broker, error) { b := &Broker{Broker: common.NewBroker(cnf)} if cnf.SQS != nil && cnf.SQS.Client != nil { // Use provided *SQS client b.service = cnf.SQS.Client } else { cfg, err := awsconfig.LoadDefaultConfig(context.TODO()) if err != nil { return nil, fmt.Errorf("%w: unable to load AWS SDK config: ", err) } b.service = sqs.NewFromConfig(cfg) } return b, nil } // StartConsuming enters a loop and waits for incoming messages func (b *Broker) StartConsuming(consumerTag string, concurrency int, taskProcessor iface.TaskProcessor) (bool, error) { b.Broker.StartConsuming(consumerTag, concurrency, taskProcessor) qURL := b.getQueueURL(taskProcessor) //save it so that it can be used later when attempting to delete task b.queueUrl = qURL deliveries := make(chan *sqs.ReceiveMessageOutput, concurrency) pool := make(chan struct{}, concurrency) // initialize worker pool with maxWorkers workers for i := 0; i < concurrency; i++ { pool <- struct{}{} } b.stopReceivingChan = make(chan int) b.receivingWG.Add(1) go func() { defer b.receivingWG.Done() log.INFO.Printf("[*] Waiting for messages on queue: %s. To exit press CTRL+C\n", *qURL) for { select { // A way to stop this goroutine from b.StopConsuming case <-b.stopReceivingChan: close(deliveries) return case <-pool: output, err := b.receiveMessage(qURL) if err == nil && len(output.Messages) > 0 { deliveries <- output } else { //return back to pool right away pool <- struct{}{} if err != nil { log.ERROR.Printf("Queue consume error: %s", err) } } } } }() if err := b.consume(deliveries, concurrency, taskProcessor, pool); err != nil { return b.GetRetry(), err } return b.GetRetry(), nil } // StopConsuming quits the loop func (b *Broker) StopConsuming() { b.Broker.StopConsuming() b.stopReceiving() // Waiting for any tasks being processed to finish b.processingWG.Wait() // Waiting for the receiving goroutine to have stopped b.receivingWG.Wait() } // Publish places a new message on the default queue func (b *Broker) Publish(ctx context.Context, signature *tasks.Signature) error { msg, err := json.Marshal(signature) if err != nil { return fmt.Errorf("JSON marshal error: %s", err) } // Check that signature.RoutingKey is set, if not switch to DefaultQueue b.AdjustRoutingKey(signature) MsgInput := &sqs.SendMessageInput{ MessageBody: aws.String(string(msg)), QueueUrl: aws.String(b.GetConfig().Broker + "/" + signature.RoutingKey), } // if this is a fifo queue, there needs to be some additional parameters. if strings.HasSuffix(signature.RoutingKey, ".fifo") { // Use Machinery's signature Task UUID as SQS Message Group ID. MsgDedupID := signature.UUID MsgInput.MessageDeduplicationId = aws.String(MsgDedupID) // Do not Use Machinery's signature Group UUID as SQS Message Group ID, instead use BrokerMessageGroupId MsgGroupID := signature.BrokerMessageGroupId if MsgGroupID == "" { return fmt.Errorf("please specify BrokerMessageGroupId attribute for task Signature when submitting a task to FIFO queue") } MsgInput.MessageGroupId = aws.String(MsgGroupID) } // Check the ETA signature field, if it is set and it is in the future, // and is not a fifo queue, set a delay in seconds for the task. if signature.ETA != nil && !strings.HasSuffix(signature.RoutingKey, ".fifo") { now := time.Now().UTC() delay := signature.ETA.Sub(now) if delay > 0 { if delay > maxAWSSQSDelay { return errors.New("Max AWS SQS delay exceeded") } MsgInput.DelaySeconds = int32(delay.Seconds()) } } result, err := b.service.SendMessage(ctx, MsgInput) if err != nil { log.ERROR.Printf("Error when sending a message: %v", err) return err } log.INFO.Printf("Sending a message successfully, the messageId is %v", *result.MessageId) return nil } // consume is a method which keeps consuming deliveries from a channel, until there is an error or a stop signal func (b *Broker) consume(deliveries <-chan *sqs.ReceiveMessageOutput, concurrency int, taskProcessor iface.TaskProcessor, pool chan struct{}) error { errorsChan := make(chan error) for { whetherContinue, err := b.consumeDeliveries(deliveries, concurrency, taskProcessor, pool, errorsChan) if err != nil { return err } if whetherContinue == false { return nil } } } // consumeOne is a method consumes a delivery. If a delivery was consumed successfully, it will be deleted from AWS SQS func (b *Broker) consumeOne(delivery *sqs.ReceiveMessageOutput, taskProcessor iface.TaskProcessor) error { if len(delivery.Messages) == 0 { log.ERROR.Printf("received an empty message, the delivery was %v", delivery) return fmt.Errorf("received empty message, the delivery is %v", delivery) } if b.GetConfig().SQS.VisibilityHeartBeat { notify := make(chan struct{}) defer close(notify) b.visibilityHeartbeat(delivery, notify) } if b.GetConfig().SQS.VisibilityHeartBeat { notify := make(chan struct{}) defer close(notify) b.visibilityHeartbeat(delivery, notify) } sig := new(tasks.Signature) decoder := json.NewDecoder(strings.NewReader(*delivery.Messages[0].Body)) decoder.UseNumber() if err := decoder.Decode(sig); err != nil { log.ERROR.Printf("unmarshal error. the delivery is %v", delivery) // if the unmarshal fails, remove the delivery from the queue if delErr := b.deleteOne(delivery); delErr != nil { log.ERROR.Printf("error when deleting the delivery. delivery is %v, Error=%s", delivery, delErr) } return err } if delivery.Messages[0].ReceiptHandle != nil { sig.SQSReceiptHandle = *delivery.Messages[0].ReceiptHandle } // If the task is not registered return an error // and leave the message in the queue if !b.IsTaskRegistered(sig.Name) { if sig.IgnoreWhenTaskNotRegistered { if err := b.deleteOne(delivery); err != nil { log.ERROR.Printf("error when deleting the delivery. delivery is %v, Error=%s", delivery, err) } } return fmt.Errorf("task %s is not registered", sig.Name) } err := taskProcessor.Process(sig) if err != nil { // stop task deletion in case we want to send messages to dlq in sqs if errors.Is(err, errs.ErrStopTaskDeletion) { return nil } return err } // Delete message after successfully consuming and processing the message if err = b.deleteOne(delivery); err != nil { log.ERROR.Printf("error when deleting the delivery. delivery is %v, Error=%s", delivery, err) } return err } // deleteOne is a method delete a delivery from AWS SQS func (b *Broker) deleteOne(delivery *sqs.ReceiveMessageOutput) error { qURL := b.defaultQueueURL() _, err := b.service.DeleteMessage(context.TODO(), &sqs.DeleteMessageInput{ QueueUrl: qURL, ReceiptHandle: delivery.Messages[0].ReceiptHandle, }) if err != nil { return err } return nil } // defaultQueueURL is a method returns the default queue url func (b *Broker) defaultQueueURL() *string { if b.queueUrl != nil { return b.queueUrl } else { return aws.String(b.GetConfig().Broker + "/" + b.GetConfig().DefaultQueue) } } // receiveMessage is a method receives a message from specified queue url func (b *Broker) receiveMessage(qURL *string) (*sqs.ReceiveMessageOutput, error) { var waitTimeSeconds int var visibilityTimeout *int if b.GetConfig().SQS != nil { waitTimeSeconds = b.GetConfig().SQS.WaitTimeSeconds visibilityTimeout = b.GetConfig().SQS.VisibilityTimeout } input := &sqs.ReceiveMessageInput{ MessageSystemAttributeNames: []types.MessageSystemAttributeName{ types.MessageSystemAttributeNameSentTimestamp, }, MessageAttributeNames: []string{ string(types.QueueAttributeNameAll), }, QueueUrl: qURL, MaxNumberOfMessages: 1, WaitTimeSeconds: int32(waitTimeSeconds), } if visibilityTimeout != nil { input.VisibilityTimeout = int32(*visibilityTimeout) } result, err := b.service.ReceiveMessage(context.TODO(), input) if err != nil { return nil, err } return result, err } // initializePool is a method which initializes concurrency pool func (b *Broker) initializePool(pool chan struct{}, concurrency int) { for i := 0; i < concurrency; i++ { pool <- struct{}{} } } // consumeDeliveries is a method consuming deliveries from deliveries channel func (b *Broker) consumeDeliveries(deliveries <-chan *sqs.ReceiveMessageOutput, concurrency int, taskProcessor iface.TaskProcessor, pool chan struct{}, errorsChan chan error) (bool, error) { select { case err := <-errorsChan: return false, err case d := <-deliveries: b.processingWG.Add(1) // Consume the task inside a goroutine so multiple tasks // can be processed concurrently go func() { if err := b.consumeOne(d, taskProcessor); err != nil { errorsChan <- err } b.processingWG.Done() if concurrency > 0 { // give worker back to pool pool <- struct{}{} } }() case <-b.GetStopChan(): return false, nil } return true, nil } // continueReceivingMessages is a method returns a continue signal func (b *Broker) continueReceivingMessages(qURL *string, deliveries chan *sqs.ReceiveMessageOutput) (bool, error) { select { // A way to stop this goroutine from b.StopConsuming case <-b.stopReceivingChan: return false, nil default: output, err := b.receiveMessage(qURL) if err != nil { return true, err } if len(output.Messages) == 0 { return true, nil } go func() { deliveries <- output }() } return true, nil } // visibilityHeartbeat is a method that sends a heartbeat signal to AWS SQS to keep a message invisible to other consumers while being processed. func (b *Broker) visibilityHeartbeat(delivery *sqs.ReceiveMessageOutput, notify <-chan struct{}) { if b.GetConfig().SQS.VisibilityTimeout == nil || *b.GetConfig().SQS.VisibilityTimeout == 0 { return } ticker := time.NewTicker(time.Duration(*b.GetConfig().SQS.VisibilityTimeout) * 500 * time.Millisecond) go func() { for { select { case <-notify: ticker.Stop() return case <-b.stopReceivingChan: ticker.Stop() return case <-ticker.C: // Extend the delivery visibility timeout _, err := b.service.ChangeMessageVisibility(context.TODO(), &sqs.ChangeMessageVisibilityInput{ QueueUrl: b.defaultQueueURL(), ReceiptHandle: delivery.Messages[0].ReceiptHandle, VisibilityTimeout: int32(*b.GetConfig().SQS.VisibilityTimeout), }) if err != nil { log.ERROR.Printf("Error when changing delivery visibility: %v", err) } } } }() } // stopReceiving is a method sending a signal to stopReceivingChan func (b *Broker) stopReceiving() { // Stop the receiving goroutine b.stopReceivingChan <- 1 } // getQueueURL is a method returns that returns queueURL first by checking if custom queue was set and usign it // otherwise using default queueName from config func (b *Broker) getQueueURL(taskProcessor iface.TaskProcessor) *string { queueName := b.GetConfig().DefaultQueue if taskProcessor.CustomQueue() != "" { queueName = taskProcessor.CustomQueue() } return aws.String(b.GetConfig().Broker + "/" + queueName) } ================================================ FILE: v2/brokers/sqs/sqs_export_test.go ================================================ package sqs import ( "context" "encoding/json" "errors" "fmt" "os" "sync" sqsiface "github.com/RichardKnop/machinery/v2/brokers/iface/sqs" "github.com/RichardKnop/machinery/v2/brokers/iface" "github.com/RichardKnop/machinery/v2/common" "github.com/RichardKnop/machinery/v2/config" "github.com/aws/aws-sdk-go-v2/aws" awssqs "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/aws/aws-sdk-go-v2/service/sqs/types" ) var ( ReceiveMessageOutput *awssqs.ReceiveMessageOutput ) type FakeSQS struct { sqsiface.API } func (f *FakeSQS) SendMessage(context.Context, *awssqs.SendMessageInput, ...func(*awssqs.Options)) (*awssqs.SendMessageOutput, error) { output := awssqs.SendMessageOutput{ MD5OfMessageAttributes: aws.String("d25a6aea97eb8f585bfa92d314504a92"), MD5OfMessageBody: aws.String("bbdc5fdb8be7251f5c910905db994bab"), MessageId: aws.String("47f8b355-5115-4b45-b33a-439016400411"), } return &output, nil } func (f *FakeSQS) ReceiveMessage(context.Context, *awssqs.ReceiveMessageInput, ...func(*awssqs.Options)) (*awssqs.ReceiveMessageOutput, error) { return ReceiveMessageOutput, nil } func (f *FakeSQS) DeleteMessage(context.Context, *awssqs.DeleteMessageInput, ...func(*awssqs.Options)) (*awssqs.DeleteMessageOutput, error) { return &awssqs.DeleteMessageOutput{}, nil } type ErrorSQS struct { sqsiface.API } func (e *ErrorSQS) SendMessage(context.Context, *awssqs.SendMessageInput, ...func(*awssqs.Options)) (*awssqs.SendMessageOutput, error) { err := errors.New("this is an error") return nil, err } func (e *ErrorSQS) ReceiveMessage(context.Context, *awssqs.ReceiveMessageInput, ...func(*awssqs.Options)) (*awssqs.ReceiveMessageOutput, error) { err := errors.New("this is an error") return nil, err } func (e *ErrorSQS) DeleteMessage(context.Context, *awssqs.DeleteMessageInput, ...func(*awssqs.Options)) (*awssqs.DeleteMessageOutput, error) { err := errors.New("this is an error") return nil, err } func init() { // TODO: chang message body to signature example messageBody, _ := json.Marshal(map[string]int{"apple": 5, "lettuce": 7}) ReceiveMessageOutput = &awssqs.ReceiveMessageOutput{ Messages: []types.Message{ { Attributes: map[string]string{ "SentTimestamp": "1512962021537", }, Body: aws.String(string(messageBody)), MD5OfBody: aws.String("bbdc5fdb8be7251f5c910905db994bab"), MD5OfMessageAttributes: aws.String("d25a6aea97eb8f585bfa92d314504a92"), MessageAttributes: map[string]types.MessageAttributeValue{ "Title": { DataType: aws.String("String"), StringValue: aws.String("The Whistler"), }, "Author": { DataType: aws.String("String"), StringValue: aws.String("John Grisham"), }, "WeeksOn": { DataType: aws.String("Number"), StringValue: aws.String("6"), }, }, MessageId: aws.String("47f8b355-5115-4b45-b33a-439016400411"), ReceiptHandle: aws.String("AQEBGhTR/nhq+pDPAunCDgLpwQuCq0JkD2dtv7pAcPF5DA/XaoPAjHfgn/PZ5DeG3YiQdTjCUj+rvFq5b79DTq+hK6r1Niuds02l+jdIk3u2JiL01Dsd203pW1lLUNryd74QAcn462eXzv7/hVDagXTn+KtOzox3X0vmPkCSQkWXWxtc23oa5+5Q7HWDmRm743L0zza1579rQ2R2B0TrdlTMpNsdjQlDmybNu+aDq8bazD/Wew539tIvUyYADuhVyKyS1L2QQuyXll73/DixulPNmvGPRHNoB1GIo+Ex929OHFchXoKonoFJnurX4VNNl1p/Byp2IYBi6nkTRzeJUFCrFq0WMAHKLwuxciezJSlLD7g3bbU8kgEer8+jTz1DBriUlDGsARr0s7mnlsd02cb46K/j+u1oPfA69vIVc0FaRtA="), }, }, } } func NewTestConfig() *config.Config { redisURL := os.Getenv("REDIS_URL") if redisURL == "" { redisURL = "eager" } brokerURL := "https://sqs.foo.amazonaws.com.cn" return &config.Config{ Broker: brokerURL, DefaultQueue: "test_queue", ResultBackend: fmt.Sprintf("redis://%v", redisURL), Lock: fmt.Sprintf("redis://%v", redisURL), SQS: &config.SQSConfig{ VisibilityTimeout: aws.Int(30), }, } } func NewTestBroker(cnf *config.Config) *Broker { var svc sqsiface.API = new(FakeSQS) if cnf.SQS.Client != nil { svc = cnf.SQS.Client } return &Broker{ Broker: common.NewBroker(cnf), service: svc, processingWG: sync.WaitGroup{}, receivingWG: sync.WaitGroup{}, stopReceivingChan: make(chan int), } } func NewTestErrorBroker() *Broker { cnf := NewTestConfig() errSvc := new(ErrorSQS) return &Broker{ Broker: common.NewBroker(cnf), service: errSvc, processingWG: sync.WaitGroup{}, receivingWG: sync.WaitGroup{}, stopReceivingChan: make(chan int), } } func (b *Broker) ConsumeForTest(deliveries <-chan *awssqs.ReceiveMessageOutput, concurrency int, taskProcessor iface.TaskProcessor, pool chan struct{}) error { return b.consume(deliveries, concurrency, taskProcessor, pool) } func (b *Broker) ConsumeOneForTest(delivery *awssqs.ReceiveMessageOutput, taskProcessor iface.TaskProcessor) error { return b.consumeOne(delivery, taskProcessor) } func (b *Broker) DeleteOneForTest(delivery *awssqs.ReceiveMessageOutput) error { return b.deleteOne(delivery) } func (b *Broker) DefaultQueueURLForTest() *string { return b.defaultQueueURL() } func (b *Broker) ReceiveMessageForTest(qURL *string) (*awssqs.ReceiveMessageOutput, error) { return b.receiveMessage(qURL) } func (b *Broker) InitializePoolForTest(pool chan struct{}, concurrency int) { b.initializePool(pool, concurrency) } func (b *Broker) ConsumeDeliveriesForTest(deliveries <-chan *awssqs.ReceiveMessageOutput, concurrency int, taskProcessor iface.TaskProcessor, pool chan struct{}, errorsChan chan error) (bool, error) { return b.consumeDeliveries(deliveries, concurrency, taskProcessor, pool, errorsChan) } func (b *Broker) ContinueReceivingMessagesForTest(qURL *string, deliveries chan *awssqs.ReceiveMessageOutput) (bool, error) { return b.continueReceivingMessages(qURL, deliveries) } func (b *Broker) StopReceivingForTest() { b.stopReceiving() } func (b *Broker) GetStopReceivingChanForTest() chan int { return b.stopReceivingChan } func (b *Broker) StartConsumingForTest(consumerTag string, concurrency int, taskProcessor iface.TaskProcessor) { b.Broker.StartConsuming(consumerTag, concurrency, taskProcessor) } func (b *Broker) GetRetryFuncForTest() func(chan int) { return b.GetRetryFunc() } func (b *Broker) GetStopChanForTest() chan int { return b.GetStopChan() } func (b *Broker) GetRetryStopChanForTest() chan int { return b.GetRetryStopChan() } func (b *Broker) GetQueueURLForTest(taskProcessor iface.TaskProcessor) *string { return b.getQueueURL(taskProcessor) } func (b *Broker) GetCustomQueueURL(customQueue string) *string { return aws.String(b.GetConfig().Broker + "/" + customQueue) } ================================================ FILE: v2/brokers/sqs/sqs_test.go ================================================ package sqs_test import ( "context" "errors" "sync" "testing" "time" "github.com/RichardKnop/machinery/v2" eagerbck "github.com/RichardKnop/machinery/v2/backends/eager" "github.com/RichardKnop/machinery/v2/brokers/eager" sqsiface "github.com/RichardKnop/machinery/v2/brokers/iface/sqs" "github.com/RichardKnop/machinery/v2/brokers/sqs" "github.com/RichardKnop/machinery/v2/config" eagerlock "github.com/RichardKnop/machinery/v2/locks/eager" "github.com/RichardKnop/machinery/v2/retry" "github.com/aws/aws-sdk-go-v2/aws" awssqs "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/aws/aws-sdk-go-v2/service/sqs/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) var ( cnf *config.Config receiveMessageOutput *awssqs.ReceiveMessageOutput ) func init() { cnf = sqs.NewTestConfig() receiveMessageOutput = sqs.ReceiveMessageOutput } func TestNewAWSSQSBroker(t *testing.T) { t.Parallel() testBroker := sqs.NewTestBroker(cnf) broker, err := sqs.New(cnf) require.NoError(t, err) assert.IsType(t, testBroker, broker) } func TestPrivateFunc_continueReceivingMessages(t *testing.T) { broker := sqs.NewTestBroker(cnf) errorBroker := sqs.NewTestErrorBroker() qURL := broker.DefaultQueueURLForTest() deliveries := make(chan *awssqs.ReceiveMessageOutput) firstStep := make(chan int) nextStep := make(chan int) go func() { stopReceivingChan := broker.GetStopReceivingChanForTest() firstStep <- 1 stopReceivingChan <- 1 }() var ( whetherContinue bool err error ) <-firstStep // Test the case that a signal was received from stopReceivingChan go func() { whetherContinue, err = broker.ContinueReceivingMessagesForTest(qURL, deliveries) nextStep <- 1 }() <-nextStep assert.False(t, whetherContinue) assert.Nil(t, err) // Test the default condition whetherContinue, err = broker.ContinueReceivingMessagesForTest(qURL, deliveries) assert.True(t, whetherContinue) assert.Nil(t, err) // Test the error whetherContinue, err = errorBroker.ContinueReceivingMessagesForTest(qURL, deliveries) assert.True(t, whetherContinue) assert.NotNil(t, err) // Test when there is no message outputCopy := *receiveMessageOutput receiveMessageOutput.Messages = []types.Message{} whetherContinue, err = broker.ContinueReceivingMessagesForTest(qURL, deliveries) assert.True(t, whetherContinue) assert.Nil(t, err) // recover original value *receiveMessageOutput = outputCopy } func TestPrivateFunc_consume(t *testing.T) { server1 := machinery.NewServer(cnf, eager.New(), eagerbck.New(), eagerlock.New()) pool := make(chan struct{}) wk := server1.NewWorker("sms_worker", 0) deliveries := make(chan *awssqs.ReceiveMessageOutput) outputCopy := *receiveMessageOutput outputCopy.Messages = []types.Message{} go func() { deliveries <- &outputCopy }() broker := sqs.NewTestBroker(cnf) // an infinite loop will be executed only when there is no error err := broker.ConsumeForTest(deliveries, 0, wk, pool) assert.NotNil(t, err) } func TestPrivateFunc_consumeOne(t *testing.T) { server1 := machinery.NewServer(cnf, eager.New(), eagerbck.New(), eagerlock.New()) wk := server1.NewWorker("sms_worker", 0) broker := sqs.NewTestBroker(cnf) err := broker.ConsumeOneForTest(receiveMessageOutput, wk) assert.Error(t, err) outputCopy := *receiveMessageOutput outputCopy.Messages = []types.Message{} err = broker.ConsumeOneForTest(&outputCopy, wk) assert.Error(t, err) outputCopy.Messages = []types.Message{ { Body: aws.String("foo message"), }, } err = broker.ConsumeOneForTest(&outputCopy, wk) assert.Error(t, err) } func TestPrivateFunc_consumeOneWithVisibilityHeartBeat(t *testing.T) { cfg := sqs.NewTestConfig() cfg.SQS.VisibilityHeartBeat = true cfg.SQS.VisibilityTimeout = aws.Int(1) // seconds mockClient := new(MockSQSAPI) cfg.SQS.Client = mockClient broker := sqs.NewTestBroker(cfg) server1 := machinery.NewServer(cfg, broker, eagerbck.New(), eagerlock.New()) // Long-running task by two times the visibility timeout. err := server1.RegisterTask("test-task", func(ctx context.Context) error { time.Sleep(time.Duration(*cfg.SQS.VisibilityTimeout) * 2 * time.Second) return nil }) require.NoError(t, err) wk := server1.NewWorker("sms_worker", 0) receiveMessageOutput.Messages = []types.Message{ { Body: aws.String(`{"Name": "test-task"}`), }, } mockClient.On("ChangeMessageVisibility", mock.AnythingOfType("*sqs.ChangeMessageVisibilityInput")).Return(&awssqs.ChangeMessageVisibilityOutput{}, nil) mockClient.On("DeleteMessage", mock.AnythingOfType("*sqs.DeleteMessageInput")).Return(&awssqs.DeleteMessageOutput{}, nil) err = broker.ConsumeOneForTest(receiveMessageOutput, wk) assert.NoError(t, err) time.Sleep(time.Duration(*cfg.SQS.VisibilityTimeout) * time.Second) // Assert that ChangeMessageVisibility was called. mockClient.AssertNumberOfCalls(t, "ChangeMessageVisibility", 4) } func TestPrivateFunc_initializePool(t *testing.T) { broker := sqs.NewTestBroker(cnf) concurrency := 9 pool := make(chan struct{}, concurrency) broker.InitializePoolForTest(pool, concurrency) assert.Len(t, pool, concurrency) } func TestPrivateFunc_startConsuming(t *testing.T) { server1 := machinery.NewServer(cnf, eager.New(), eagerbck.New(), eagerlock.New()) wk := server1.NewWorker("sms_worker", 0) broker := sqs.NewTestBroker(cnf) retryFunc := broker.GetRetryFuncForTest() stopChan := broker.GetStopChanForTest() retryStopChan := broker.GetRetryStopChanForTest() assert.Nil(t, retryFunc) broker.StartConsumingForTest("fooTag", 1, wk) assert.IsType(t, retryFunc, retry.Closure()) assert.Equal(t, len(stopChan), 0) assert.Equal(t, len(retryStopChan), 0) } func TestPrivateFuncDefaultQueueURL(t *testing.T) { broker := sqs.NewTestBroker(cnf) qURL := broker.DefaultQueueURLForTest() assert.EqualValues(t, *qURL, "https://sqs.foo.amazonaws.com.cn/test_queue") } func TestPrivateFunc_stopReceiving(t *testing.T) { broker := sqs.NewTestBroker(cnf) go broker.StopReceivingForTest() stopReceivingChan := broker.GetStopReceivingChanForTest() assert.NotNil(t, <-stopReceivingChan) } func TestPrivateFunc_receiveMessage(t *testing.T) { broker := sqs.NewTestBroker(cnf) qURL := broker.DefaultQueueURLForTest() output, err := broker.ReceiveMessageForTest(qURL) assert.Nil(t, err) assert.Equal(t, receiveMessageOutput, output) } func TestPrivateFunc_consumeDeliveries(t *testing.T) { concurrency := 0 pool := make(chan struct{}, concurrency) errorsChan := make(chan error) deliveries := make(chan *awssqs.ReceiveMessageOutput) server1 := machinery.NewServer(cnf, eager.New(), eagerbck.New(), eagerlock.New()) wk := server1.NewWorker("sms_worker", 0) broker := sqs.NewTestBroker(cnf) go func() { deliveries <- receiveMessageOutput }() whetherContinue, err := broker.ConsumeDeliveriesForTest(deliveries, concurrency, wk, pool, errorsChan) assert.True(t, whetherContinue) assert.Nil(t, err) go func() { errorsChan <- errors.New("foo error") }() whetherContinue, err = broker.ConsumeDeliveriesForTest(deliveries, concurrency, wk, pool, errorsChan) assert.False(t, whetherContinue) assert.NotNil(t, err) go func() { broker.GetStopChanForTest() <- 1 }() whetherContinue, err = broker.ConsumeDeliveriesForTest(deliveries, concurrency, wk, pool, errorsChan) assert.False(t, whetherContinue) assert.Nil(t, err) outputCopy := *receiveMessageOutput outputCopy.Messages = []types.Message{} go func() { deliveries <- &outputCopy }() whetherContinue, err = broker.ConsumeDeliveriesForTest(deliveries, concurrency, wk, pool, errorsChan) e := <-errorsChan assert.True(t, whetherContinue) assert.NotNil(t, e) assert.Nil(t, err) // using a wait group and a channel to fix the racing problem var wg sync.WaitGroup wg.Add(1) nextStep := make(chan bool, 1) go func() { defer wg.Done() // nextStep <- true runs after defer wg.Done(), to make sure the next go routine runs after this go routine nextStep <- true deliveries <- receiveMessageOutput }() if <-nextStep { // <-pool will block the routine in the following steps, so pool <- struct{}{} will be executed for sure go func() { wg.Wait(); pool <- struct{}{} }() } whetherContinue, err = broker.ConsumeDeliveriesForTest(deliveries, concurrency, wk, pool, errorsChan) // the pool shouldn't be consumed p := <-pool assert.True(t, whetherContinue) assert.NotNil(t, p) assert.Nil(t, err) } func TestPrivateFunc_deleteOne(t *testing.T) { broker := sqs.NewTestBroker(cnf) errorBroker := sqs.NewTestErrorBroker() err := broker.DeleteOneForTest(receiveMessageOutput) assert.Nil(t, err) err = errorBroker.DeleteOneForTest(receiveMessageOutput) assert.NotNil(t, err) } func Test_CustomQueueName(t *testing.T) { server1 := machinery.NewServer(cnf, eager.New(), eagerbck.New(), eagerlock.New()) broker := sqs.NewTestBroker(cnf) wk := server1.NewWorker("test-worker", 0) qURL := broker.GetQueueURLForTest(wk) assert.Equal(t, qURL, broker.DefaultQueueURLForTest(), "") wk2 := server1.NewCustomQueueWorker("test-worker", 0, "my-custom-queue") qURL2 := broker.GetQueueURLForTest(wk2) assert.Equal(t, qURL2, broker.GetCustomQueueURL("my-custom-queue"), "") } func TestPrivateFunc_consumeWithConcurrency(t *testing.T) { msg := `{ "UUID": "uuid-dummy-task", "Name": "test-task", "RoutingKey": "dummy-routing" } ` testResp := "47f8b355-5115-4b45-b33a-439016400411" output := make(chan string) // The output channel cnf.ResultBackend = "eager" server1 := machinery.NewServer(cnf, eager.New(), eagerbck.New(), eagerlock.New()) err := server1.RegisterTask("test-task", func(ctx context.Context) error { output <- testResp return nil }) broker := sqs.NewTestBroker(cnf) broker.SetRegisteredTaskNames([]string{"test-task"}) assert.NoError(t, err) pool := make(chan struct{}, 1) pool <- struct{}{} wk := server1.NewWorker("sms_worker", 1) deliveries := make(chan *awssqs.ReceiveMessageOutput) outputCopy := *receiveMessageOutput outputCopy.Messages = []types.Message{ { MessageId: aws.String("test-sqs-msg1"), Body: aws.String(msg), }, } go func() { deliveries <- &outputCopy }() go func() { err = broker.ConsumeForTest(deliveries, 1, wk, pool) }() select { case resp := <-output: assert.Equal(t, testResp, resp) case <-time.After(10 * time.Second): // call timed out t.Fatal("task not processed in 10 seconds") } } // MockSQSAPI is a mock implementation of the sqsiface.SQSAPI interface type MockSQSAPI struct { mock.Mock sqsiface.API } func (m *MockSQSAPI) ReceiveMessage(ctx context.Context, input *awssqs.ReceiveMessageInput, opts ...func(*awssqs.Options)) (*awssqs.ReceiveMessageOutput, error) { args := m.Called(input) return args.Get(0).(*awssqs.ReceiveMessageOutput), args.Error(1) } func (m *MockSQSAPI) DeleteMessage(ctx context.Context, input *awssqs.DeleteMessageInput, opts ...func(*awssqs.Options)) (*awssqs.DeleteMessageOutput, error) { args := m.Called(input) return args.Get(0).(*awssqs.DeleteMessageOutput), args.Error(1) } func (m *MockSQSAPI) ChangeMessageVisibility(ctx context.Context, input *awssqs.ChangeMessageVisibilityInput, opts ...func(*awssqs.Options)) (*awssqs.ChangeMessageVisibilityOutput, error) { args := m.Called(input) return args.Get(0).(*awssqs.ChangeMessageVisibilityOutput), args.Error(1) } ================================================ FILE: v2/common/amqp.go ================================================ package common import ( "crypto/tls" "fmt" "strings" amqp "github.com/rabbitmq/amqp091-go" ) // AMQPConnector ... type AMQPConnector struct{} // Connect opens a connection to RabbitMQ, declares an exchange, opens a channel, // declares and binds the queue and enables publish notifications func (ac *AMQPConnector) Connect(urls string, urlSeparator string, tlsConfig *tls.Config, exchange, exchangeType, queueName string, queueDurable, queueDelete bool, queueBindingKey string, exchangeDeclareArgs, queueDeclareArgs, queueBindingArgs amqp.Table) (*amqp.Connection, *amqp.Channel, amqp.Queue, <-chan amqp.Confirmation, <-chan *amqp.Error, error) { urlsList := []string{urls} if urlSeparator != "" { urlsList = strings.Split(urls, urlSeparator) } var conn *amqp.Connection var channel *amqp.Channel var err error for _, url := range urlsList { // Connect to server conn, channel, err = ac.Open(url, tlsConfig) if err != nil { continue } else { break } } if err != nil { return nil, nil, amqp.Queue{}, nil, nil, err } if exchange != "" { // Declare an exchange if err = channel.ExchangeDeclare( exchange, // name of the exchange exchangeType, // type true, // durable false, // delete when complete false, // internal false, // noWait exchangeDeclareArgs, // arguments ); err != nil { return conn, channel, amqp.Queue{}, nil, nil, fmt.Errorf("Exchange declare error: %s", err) } } var queue amqp.Queue if queueName != "" { // Declare a queue queue, err = channel.QueueDeclare( queueName, // name queueDurable, // durable queueDelete, // delete when unused false, // exclusive false, // no-wait queueDeclareArgs, // arguments ) if err != nil { return conn, channel, amqp.Queue{}, nil, nil, fmt.Errorf("Queue declare error: %s", err) } // Bind the queue if err = channel.QueueBind( queue.Name, // name of the queue queueBindingKey, // binding key exchange, // source exchange false, // noWait queueBindingArgs, // arguments ); err != nil { return conn, channel, queue, nil, nil, fmt.Errorf("Queue bind error: %s", err) } } // Enable publish confirmations if err = channel.Confirm(false); err != nil { return conn, channel, queue, nil, nil, fmt.Errorf("Channel could not be put into confirm mode: %s", err) } return conn, channel, queue, channel.NotifyPublish(make(chan amqp.Confirmation, 1)), conn.NotifyClose(make(chan *amqp.Error, 1)), nil } // DeleteQueue deletes a queue by name func (ac *AMQPConnector) DeleteQueue(channel *amqp.Channel, queueName string) error { // First return value is number of messages removed _, err := channel.QueueDelete( queueName, // name false, // ifUnused false, // ifEmpty false, // noWait ) return err } // InspectQueue provides information about a specific queue func (*AMQPConnector) InspectQueue(channel *amqp.Channel, queueName string) (*amqp.Queue, error) { queueState, err := channel.QueueInspect(queueName) if err != nil { return nil, fmt.Errorf("Queue inspect error: %s", err) } return &queueState, nil } // Open new RabbitMQ connection func (ac *AMQPConnector) Open(url string, tlsConfig *tls.Config) (*amqp.Connection, *amqp.Channel, error) { // Connect // From amqp docs: DialTLS will use the provided tls.Config when it encounters an amqps:// scheme // and will dial a plain connection when it encounters an amqp:// scheme. conn, err := amqp.DialTLS(url, tlsConfig) if err != nil { return nil, nil, fmt.Errorf("Dial error: %s", err) } // Open a channel channel, err := conn.Channel() if err != nil { return nil, nil, fmt.Errorf("Open channel error: %s", err) } return conn, channel, nil } // Close connection func (ac *AMQPConnector) Close(channel *amqp.Channel, conn *amqp.Connection) error { if channel != nil { if err := channel.Close(); err != nil { return fmt.Errorf("Close channel error: %s", err) } } if conn != nil { if err := conn.Close(); err != nil { return fmt.Errorf("Close connection error: %s", err) } } return nil } ================================================ FILE: v2/common/backend.go ================================================ package common import ( "github.com/RichardKnop/machinery/v2/config" ) // Backend represents a base backend structure type Backend struct { cnf *config.Config } // NewBackend creates new Backend instance func NewBackend(cnf *config.Config) Backend { return Backend{cnf: cnf} } // GetConfig returns config func (b *Backend) GetConfig() *config.Config { return b.cnf } // IsAMQP ... func (b *Backend) IsAMQP() bool { return false } ================================================ FILE: v2/common/broker.go ================================================ package common import ( "errors" "sync" "github.com/RichardKnop/machinery/v2/brokers/iface" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/log" "github.com/RichardKnop/machinery/v2/retry" "github.com/RichardKnop/machinery/v2/tasks" ) type registeredTaskNames struct { sync.RWMutex items []string } // Broker represents a base broker structure type Broker struct { cnf *config.Config registeredTaskNames registeredTaskNames retry bool retryFunc func(chan int) retryStopChan chan int stopChan chan int } // NewBroker creates new Broker instance func NewBroker(cnf *config.Config) Broker { return Broker{ cnf: cnf, retry: true, stopChan: make(chan int), retryStopChan: make(chan int), } } // GetConfig returns config func (b *Broker) GetConfig() *config.Config { return b.cnf } // GetRetry ... func (b *Broker) GetRetry() bool { return b.retry } // GetRetryFunc ... func (b *Broker) GetRetryFunc() func(chan int) { return b.retryFunc } // GetRetryStopChan ... func (b *Broker) GetRetryStopChan() chan int { return b.retryStopChan } // GetStopChan ... func (b *Broker) GetStopChan() chan int { return b.stopChan } // Publish places a new message on the default queue func (b *Broker) Publish(signature *tasks.Signature) error { return errors.New("Not implemented") } // SetRegisteredTaskNames sets registered task names func (b *Broker) SetRegisteredTaskNames(names []string) { b.registeredTaskNames.Lock() defer b.registeredTaskNames.Unlock() b.registeredTaskNames.items = names } // IsTaskRegistered returns true if the task is registered with this broker func (b *Broker) IsTaskRegistered(name string) bool { b.registeredTaskNames.RLock() defer b.registeredTaskNames.RUnlock() for _, registeredTaskName := range b.registeredTaskNames.items { if registeredTaskName == name { return true } } return false } // GetPendingTasks returns a slice of task.Signatures waiting in the queue func (b *Broker) GetPendingTasks(queue string) ([]*tasks.Signature, error) { return nil, errors.New("Not implemented") } // GetDelayedTasks returns a slice of task.Signatures that are scheduled, but not yet in the queue func (b *Broker) GetDelayedTasks() ([]*tasks.Signature, error) { return nil, errors.New("Not implemented") } // StartConsuming is a common part of StartConsuming method func (b *Broker) StartConsuming(consumerTag string, concurrency int, taskProcessor iface.TaskProcessor) { if b.retryFunc == nil { b.retryFunc = retry.Closure() } } // StopConsuming is a common part of StopConsuming func (b *Broker) StopConsuming() { // Do not retry from now on b.retry = false // Stop the retry closure earlier select { case b.retryStopChan <- 1: log.WARNING.Print("Stopping retry closure.") default: } // Notifying the stop channel stops consuming of messages close(b.stopChan) log.WARNING.Print("Stop channel") } // GetRegisteredTaskNames returns registered tasks names func (b *Broker) GetRegisteredTaskNames() []string { b.registeredTaskNames.RLock() defer b.registeredTaskNames.RUnlock() items := b.registeredTaskNames.items return items } // AdjustRoutingKey makes sure the routing key is correct. // If the routing key is an empty string: // a) set it to binding key for direct exchange type // b) set it to default queue name func (b *Broker) AdjustRoutingKey(s *tasks.Signature) { if s.RoutingKey != "" { return } s.RoutingKey = b.GetConfig().DefaultQueue } ================================================ FILE: v2/common/broker_test.go ================================================ package common_test import ( "testing" "github.com/RichardKnop/machinery/v2" "github.com/RichardKnop/machinery/v2/common" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/tasks" "github.com/stretchr/testify/assert" ) func TestIsTaskRegistered(t *testing.T) { t.Parallel() broker := common.NewBroker(new(config.Config)) broker.SetRegisteredTaskNames([]string{"foo", "bar"}) assert.True(t, broker.IsTaskRegistered("foo")) assert.False(t, broker.IsTaskRegistered("bogus")) } func TestAdjustRoutingKey(t *testing.T) { t.Parallel() var ( s *tasks.Signature broker common.Broker ) t.Run("with routing key", func(t *testing.T) { s = &tasks.Signature{RoutingKey: "routing_key"} broker = common.NewBroker(&config.Config{ DefaultQueue: "queue", }) broker.AdjustRoutingKey(s) assert.Equal(t, "routing_key", s.RoutingKey) }) t.Run("without routing key", func(t *testing.T) { s = new(tasks.Signature) broker = common.NewBroker(&config.Config{ DefaultQueue: "queue", }) broker.AdjustRoutingKey(s) assert.Equal(t, "queue", s.RoutingKey) }) } func TestGetRegisteredTaskNames(t *testing.T) { t.Parallel() broker := common.NewBroker(new(config.Config)) fooTasks := []string{"foo", "bar", "baz"} broker.SetRegisteredTaskNames(fooTasks) assert.Equal(t, fooTasks, broker.GetRegisteredTaskNames()) } func TestStopConsuming(t *testing.T) { t.Parallel() t.Run("stop consuming", func(t *testing.T) { broker := common.NewBroker(&config.Config{ DefaultQueue: "queue", }) broker.StartConsuming("", 1, &machinery.Worker{}) broker.StopConsuming() select { case <-broker.GetStopChan(): default: assert.Fail(t, "still blocking") } }) } ================================================ FILE: v2/common/redis.go ================================================ package common import ( "crypto/tls" "time" "github.com/gomodule/redigo/redis" "github.com/RichardKnop/machinery/v2/config" ) var ( defaultConfig = &config.RedisConfig{ MaxIdle: 10, MaxActive: 100, IdleTimeout: 300, Wait: true, ReadTimeout: 15, WriteTimeout: 15, ConnectTimeout: 15, NormalTasksPollPeriod: 1000, DelayedTasksPollPeriod: 20, } ) // RedisConnector ... type RedisConnector struct{} // NewPool returns a new pool of Redis connections func (rc *RedisConnector) NewPool(socketPath, host, username, password string, db int, cnf *config.RedisConfig, tlsConfig *tls.Config) *redis.Pool { if cnf == nil { cnf = defaultConfig } return &redis.Pool{ MaxIdle: cnf.MaxIdle, IdleTimeout: time.Duration(cnf.IdleTimeout) * time.Second, MaxActive: cnf.MaxActive, Wait: cnf.Wait, Dial: func() (redis.Conn, error) { c, err := rc.open(socketPath, host, username, password, db, cnf, tlsConfig) if err != nil { return nil, err } if db != 0 { _, err = c.Do("SELECT", db) if err != nil { return nil, err } } return c, err }, // PINGs connections that have been idle more than 10 seconds TestOnBorrow: func(c redis.Conn, t time.Time) error { if time.Since(t) < time.Duration(10*time.Second) { return nil } _, err := c.Do("PING") return err }, } } // Open a new Redis connection func (rc *RedisConnector) open(socketPath, host, username, password string, db int, cnf *config.RedisConfig, tlsConfig *tls.Config) (redis.Conn, error) { var opts = []redis.DialOption{ redis.DialDatabase(db), redis.DialReadTimeout(time.Duration(cnf.ReadTimeout) * time.Second), redis.DialWriteTimeout(time.Duration(cnf.WriteTimeout) * time.Second), redis.DialConnectTimeout(time.Duration(cnf.ConnectTimeout) * time.Second), redis.DialClientName(cnf.ClientName), } if tlsConfig != nil { opts = append(opts, redis.DialTLSConfig(tlsConfig), redis.DialUseTLS(true)) } if username != "" { opts = append(opts, redis.DialUsername(username)) } if password != "" { opts = append(opts, redis.DialPassword(password)) } if socketPath != "" { return redis.Dial("unix", socketPath, opts...) } return redis.Dial("tcp", host, opts...) } ================================================ FILE: v2/config/config.go ================================================ package config import ( "crypto/tls" "fmt" "strings" "time" "cloud.google.com/go/pubsub" dynamodbiface "github.com/RichardKnop/machinery/v2/backends/iface/dynamodb" "github.com/RichardKnop/machinery/v2/brokers/iface/sqs" "go.mongodb.org/mongo-driver/mongo" ) const ( // DefaultResultsExpireIn is a default time used to expire task states and group metadata from the backend DefaultResultsExpireIn = 3600 ) var ( // Start with sensible default values defaultCnf = &Config{ Broker: "amqp://guest:guest@localhost:5672/", DefaultQueue: "machinery_tasks", ResultBackend: "amqp://guest:guest@localhost:5672/", ResultsExpireIn: DefaultResultsExpireIn, AMQP: &AMQPConfig{ Exchange: "machinery_exchange", ExchangeType: "direct", BindingKey: "machinery_task", PrefetchCount: 3, }, DynamoDB: &DynamoDBConfig{ TaskStatesTable: "task_states", GroupMetasTable: "group_metas", }, Redis: &RedisConfig{ MaxIdle: 3, IdleTimeout: 240, ReadTimeout: 15, WriteTimeout: 15, ConnectTimeout: 15, NormalTasksPollPeriod: 1000, DelayedTasksPollPeriod: 500, }, GCPPubSub: &GCPPubSubConfig{ Client: nil, }, } reloadDelay = time.Second * 10 ) // Config holds all configuration for our program type Config struct { Broker string `yaml:"broker" envconfig:"BROKER"` Lock string `yaml:"lock" envconfig:"LOCK"` MultipleBrokerSeparator string `yaml:"multiple_broker_separator" envconfig:"MULTIPLE_BROKEN_SEPARATOR"` DefaultQueue string `yaml:"default_queue" envconfig:"DEFAULT_QUEUE"` ResultBackend string `yaml:"result_backend" envconfig:"RESULT_BACKEND"` ResultsExpireIn int `yaml:"results_expire_in" envconfig:"RESULTS_EXPIRE_IN"` AMQP *AMQPConfig `yaml:"amqp"` SQS *SQSConfig `yaml:"sqs"` Redis *RedisConfig `yaml:"redis"` GCPPubSub *GCPPubSubConfig `yaml:"-" ignored:"true"` MongoDB *MongoDBConfig `yaml:"-" ignored:"true"` TLSConfig *tls.Config // NoUnixSignals - when set disables signal handling in machinery NoUnixSignals bool `yaml:"no_unix_signals" envconfig:"NO_UNIX_SIGNALS"` DynamoDB *DynamoDBConfig `yaml:"dynamodb"` } // QueueBindingArgs arguments which are used when binding to the exchange type QueueBindingArgs map[string]interface{} // QueueDeclareArgs arguments which are used when declaring a queue type QueueDeclareArgs map[string]interface{} // AMQPConfig wraps RabbitMQ related configuration type AMQPConfig struct { Exchange string `yaml:"exchange" envconfig:"AMQP_EXCHANGE"` ExchangeType string `yaml:"exchange_type" envconfig:"AMQP_EXCHANGE_TYPE"` QueueDeclareArgs QueueDeclareArgs `yaml:"queue_declare_args" envconfig:"AMQP_QUEUE_DECLARE_ARGS"` QueueBindingArgs QueueBindingArgs `yaml:"queue_binding_args" envconfig:"AMQP_QUEUE_BINDING_ARGS"` BindingKey string `yaml:"binding_key" envconfig:"AMQP_BINDING_KEY"` PrefetchCount int `yaml:"prefetch_count" envconfig:"AMQP_PREFETCH_COUNT"` AutoDelete bool `yaml:"auto_delete" envconfig:"AMQP_AUTO_DELETE"` DelayedQueue string `yaml:"delayed_queue" envconfig:"AMQP_DELAYED_QUEUE"` } // DynamoDBConfig wraps DynamoDB related configuration type DynamoDBConfig struct { Client dynamodbiface.API TaskStatesTable string `yaml:"task_states_table" envconfig:"TASK_STATES_TABLE"` GroupMetasTable string `yaml:"group_metas_table" envconfig:"GROUP_METAS_TABLE"` } // SQSConfig wraps SQS related configuration type SQSConfig struct { Client sqs.API WaitTimeSeconds int `yaml:"receive_wait_time_seconds" envconfig:"SQS_WAIT_TIME_SECONDS"` // https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html // visibility timeout should default to nil to use the overall visibility timeout for the queue VisibilityTimeout *int `yaml:"receive_visibility_timeout" envconfig:"SQS_VISIBILITY_TIMEOUT"` // https://docs.aws.amazon.com/es_es/AWSSimpleQueueService/latest/SQSDeveloperGuide/best-practices-processing-messages-timely-manner.html // visibility heartbeat should default to true to ensure that the visibility timeout is extended while the task is being processed. VisibilityHeartBeat bool `yaml:"visibility_hearth_beat" envconfig:"SQS_VISIBILITY_HEARTBEAT"` } // RedisConfig ... type RedisConfig struct { // Maximum number of idle connections in the pool. // Default: 10 MaxIdle int `yaml:"max_idle" envconfig:"REDIS_MAX_IDLE"` // Maximum number of connections allocated by the pool at a given time. // When zero, there is no limit on the number of connections in the pool. // Default: 100 MaxActive int `yaml:"max_active" envconfig:"REDIS_MAX_ACTIVE"` // Close connections after remaining idle for this duration in seconds. If the value // is zero, then idle connections are not closed. Applications should set // the timeout to a value less than the server's timeout. // Default: 300 IdleTimeout int `yaml:"max_idle_timeout" envconfig:"REDIS_IDLE_TIMEOUT"` // If Wait is true and the pool is at the MaxActive limit, then Get() waits // for a connection to be returned to the pool before returning. // Default: true Wait bool `yaml:"wait" envconfig:"REDIS_WAIT"` // ReadTimeout specifies the timeout in seconds for reading a single command reply. // Default: 15 ReadTimeout int `yaml:"read_timeout" envconfig:"REDIS_READ_TIMEOUT"` // WriteTimeout specifies the timeout in seconds for writing a single command. // Default: 15 WriteTimeout int `yaml:"write_timeout" envconfig:"REDIS_WRITE_TIMEOUT"` // ConnectTimeout specifies the timeout in seconds for connecting to the Redis server when // no DialNetDial option is specified. // Default: 15 ConnectTimeout int `yaml:"connect_timeout" envconfig:"REDIS_CONNECT_TIMEOUT"` // NormalTasksPollPeriod specifies the period in milliseconds when polling redis for normal tasks // Default: 1000 NormalTasksPollPeriod int `yaml:"normal_tasks_poll_period" envconfig:"REDIS_NORMAL_TASKS_POLL_PERIOD"` // DelayedTasksPollPeriod specifies the period in milliseconds when polling redis for delayed tasks // Default: 20 DelayedTasksPollPeriod int `yaml:"delayed_tasks_poll_period" envconfig:"REDIS_DELAYED_TASKS_POLL_PERIOD"` DelayedTasksKey string `yaml:"delayed_tasks_key" envconfig:"REDIS_DELAYED_TASKS_KEY"` // ClientName specifies the redis client name to be set when connecting to the Redis server ClientName string `yaml:"client_name" envconfig:"REDIS_CLIENT_NAME"` // MasterName specifies a redis master name in order to configure a sentinel-backed redis FailoverClient MasterName string `yaml:"master_name" envconfig:"REDIS_MASTER_NAME"` // ClusterEnabled specifies whether cluster mode is enabled, regardless the number of addresses. // This helps create ClusterClient for Redis servers that enabled cluster mode with 1 node, or using AWS configuration endpoint ClusterEnabled bool `yaml:"cluster_enabled" envconfig:"REDIS_CLUSTER_ENABLED"` // SentinelPassword specifies the password to be used when connecting to a Redis server via Sentinel SentinelPassword string `yaml:"sentinel_password" envconfig:"REDIS_SENTINEL_PASSWORD"` } // GCPPubSubConfig wraps GCP PubSub related configuration type GCPPubSubConfig struct { Client *pubsub.Client MaxExtension time.Duration } // MongoDBConfig ... type MongoDBConfig struct { Client *mongo.Client Database string } // Decode from yaml to map (any field whose type or pointer-to-type implements // envconfig.Decoder can control its own deserialization) func (args *QueueBindingArgs) Decode(value string) error { pairs := strings.Split(value, ",") mp := make(map[string]interface{}, len(pairs)) for _, pair := range pairs { kvpair := strings.Split(pair, ":") if len(kvpair) != 2 { return fmt.Errorf("invalid map item: %q", pair) } mp[kvpair[0]] = kvpair[1] } *args = QueueBindingArgs(mp) return nil } ================================================ FILE: v2/config/env.go ================================================ package config import ( "github.com/kelseyhightower/envconfig" "github.com/RichardKnop/machinery/v2/log" ) // NewFromEnvironment creates a config object from environment variables func NewFromEnvironment() (*Config, error) { cnf, err := fromEnvironment() if err != nil { return nil, err } log.INFO.Print("Successfully loaded config from the environment") return cnf, nil } func fromEnvironment() (*Config, error) { loadedCnf, cnf := new(Config), new(Config) *cnf = *defaultCnf if err := envconfig.Process("", cnf); err != nil { return nil, err } if err := envconfig.Process("", loadedCnf); err != nil { return nil, err } if loadedCnf.AMQP == nil { cnf.AMQP = nil } return cnf, nil } ================================================ FILE: v2/config/env_test.go ================================================ package config_test import ( "bufio" "os" "strings" "testing" "github.com/RichardKnop/machinery/v2/config" "github.com/stretchr/testify/assert" ) func TestNewFromEnvironment(t *testing.T) { t.Parallel() file, err := os.Open("test.env") if err != nil { t.Fatal(err) } reader := bufio.NewReader(file) scanner := bufio.NewScanner(reader) scanner.Split(bufio.ScanLines) for scanner.Scan() { parts := strings.Split(scanner.Text(), "=") if len(parts) != 2 { continue } os.Setenv(parts[0], parts[1]) } cnf, err := config.NewFromEnvironment() if err != nil { t.Fatal(err) } assert.Equal(t, "broker", cnf.Broker) assert.Equal(t, "default_queue", cnf.DefaultQueue) assert.Equal(t, "result_backend", cnf.ResultBackend) assert.Equal(t, 123456, cnf.ResultsExpireIn) assert.Equal(t, "exchange", cnf.AMQP.Exchange) assert.Equal(t, "exchange_type", cnf.AMQP.ExchangeType) assert.Equal(t, "binding_key", cnf.AMQP.BindingKey) assert.Equal(t, "any", cnf.AMQP.QueueBindingArgs["x-match"]) assert.Equal(t, "png", cnf.AMQP.QueueBindingArgs["image-type"]) assert.Equal(t, 123, cnf.AMQP.PrefetchCount) } ================================================ FILE: v2/config/file.go ================================================ package config import ( "fmt" "os" "time" "github.com/RichardKnop/machinery/v2/log" "gopkg.in/yaml.v2" ) // NewFromYaml creates a config object from YAML file func NewFromYaml(cnfPath string, keepReloading bool) (*Config, error) { cnf, err := fromFile(cnfPath) if err != nil { return nil, err } log.INFO.Printf("Successfully loaded config from file %s", cnfPath) if keepReloading { // Open a goroutine to watch remote changes forever go func() { for { // Delay after each request time.Sleep(reloadDelay) // Attempt to reload the config newCnf, newErr := fromFile(cnfPath) if newErr != nil { log.WARNING.Printf("Failed to reload config from file %s: %v", cnfPath, newErr) continue } *cnf = *newCnf } }() } return cnf, nil } // ReadFromFile reads data from a file func ReadFromFile(cnfPath string) ([]byte, error) { file, err := os.Open(cnfPath) // Config file not found if err != nil { return nil, fmt.Errorf("Open file error: %s", err) } defer file.Close() // Config file found, let's try to read it data := make([]byte, 1000) count, err := file.Read(data) if err != nil { return nil, fmt.Errorf("Read from file error: %s", err) } return data[:count], nil } func fromFile(cnfPath string) (*Config, error) { loadedCnf, cnf := new(Config), new(Config) *cnf = *defaultCnf data, err := ReadFromFile(cnfPath) if err != nil { return nil, err } if err := yaml.Unmarshal(data, cnf); err != nil { return nil, fmt.Errorf("Unmarshal YAML error: %s", err) } if err := yaml.Unmarshal(data, loadedCnf); err != nil { return nil, fmt.Errorf("Unmarshal YAML error: %s", err) } if loadedCnf.AMQP == nil { cnf.AMQP = nil } return cnf, nil } ================================================ FILE: v2/config/file_test.go ================================================ package config_test import ( "testing" "github.com/RichardKnop/machinery/v2/config" "github.com/stretchr/testify/assert" ) var configYAMLData = `--- broker: broker default_queue: default_queue result_backend: result_backend results_expire_in: 123456 amqp: binding_key: binding_key exchange: exchange exchange_type: exchange_type prefetch_count: 123 queue_declare_args: x-max-priority: 10 queue_binding_args: image-type: png x-match: any sqs: receive_wait_time_seconds: 123 receive_visibility_timeout: 456 redis: max_idle: 12 max_active: 123 max_idle_timeout: 456 wait: false read_timeout: 17 write_timeout: 19 connect_timeout: 21 normal_tasks_poll_period: 1001 delayed_tasks_poll_period: 23 delayed_tasks_key: delayed_tasks_key master_name: master_name no_unix_signals: true dynamodb: task_states_table: task_states_table group_metas_table: group_metas_table ` func TestReadFromFile(t *testing.T) { t.Parallel() data, err := config.ReadFromFile("testconfig.yml") if err != nil { t.Fatal(err) } assert.Equal(t, configYAMLData, string(data)) } func TestNewFromYaml(t *testing.T) { t.Parallel() cnf, err := config.NewFromYaml("testconfig.yml", false) if err != nil { t.Fatal(err) } assert.Equal(t, "broker", cnf.Broker) assert.Equal(t, "default_queue", cnf.DefaultQueue) assert.Equal(t, "result_backend", cnf.ResultBackend) assert.Equal(t, 123456, cnf.ResultsExpireIn) assert.Equal(t, "exchange", cnf.AMQP.Exchange) assert.Equal(t, "exchange_type", cnf.AMQP.ExchangeType) assert.Equal(t, "binding_key", cnf.AMQP.BindingKey) assert.Equal(t, 10, cnf.AMQP.QueueDeclareArgs["x-max-priority"]) assert.Equal(t, "any", cnf.AMQP.QueueBindingArgs["x-match"]) assert.Equal(t, "png", cnf.AMQP.QueueBindingArgs["image-type"]) assert.Equal(t, 123, cnf.AMQP.PrefetchCount) assert.Equal(t, 123, cnf.SQS.WaitTimeSeconds) assert.Equal(t, 456, *cnf.SQS.VisibilityTimeout) assert.Equal(t, 12, cnf.Redis.MaxIdle) assert.Equal(t, 123, cnf.Redis.MaxActive) assert.Equal(t, 456, cnf.Redis.IdleTimeout) assert.Equal(t, false, cnf.Redis.Wait) assert.Equal(t, 17, cnf.Redis.ReadTimeout) assert.Equal(t, 19, cnf.Redis.WriteTimeout) assert.Equal(t, 21, cnf.Redis.ConnectTimeout) assert.Equal(t, 1001, cnf.Redis.NormalTasksPollPeriod) assert.Equal(t, 23, cnf.Redis.DelayedTasksPollPeriod) assert.Equal(t, "delayed_tasks_key", cnf.Redis.DelayedTasksKey) assert.Equal(t, "master_name", cnf.Redis.MasterName) assert.Equal(t, true, cnf.NoUnixSignals) assert.Equal(t, "task_states_table", cnf.DynamoDB.TaskStatesTable) assert.Equal(t, "group_metas_table", cnf.DynamoDB.GroupMetasTable) } ================================================ FILE: v2/config/test.env ================================================ BROKER=broker DEFAULT_QUEUE=default_queue RESULT_BACKEND=result_backend RESULTS_EXPIRE_IN=123456 AMQP_BINDING_KEY=binding_key AMQP_EXCHANGE=exchange AMQP_EXCHANGE_TYPE=exchange_type AMQP_PREFETCH_COUNT=123 AMQP_QUEUE_BINDING_ARGS=image-type:png,x-match:any ================================================ FILE: v2/config/testconfig.yml ================================================ --- broker: broker default_queue: default_queue result_backend: result_backend results_expire_in: 123456 amqp: binding_key: binding_key exchange: exchange exchange_type: exchange_type prefetch_count: 123 queue_declare_args: x-max-priority: 10 queue_binding_args: image-type: png x-match: any sqs: receive_wait_time_seconds: 123 receive_visibility_timeout: 456 redis: max_idle: 12 max_active: 123 max_idle_timeout: 456 wait: false read_timeout: 17 write_timeout: 19 connect_timeout: 21 normal_tasks_poll_period: 1001 delayed_tasks_poll_period: 23 delayed_tasks_key: delayed_tasks_key master_name: master_name no_unix_signals: true dynamodb: task_states_table: task_states_table group_metas_table: group_metas_table ================================================ FILE: v2/docker-compose.test.yml ================================================ version: "2" services: sut: container_name: machinery_sut image: machinery_sut:latest volumes: - "./:/go/src/github.com/RichardKnop/machinery/v2" depends_on: - rabbitmq - redis links: - rabbitmq - redis build: context: . dockerfile: ./Dockerfile.test environment: AMQP_URLS: 'amqp://guest:guest@dummy:5672/,amqp://guest:guest@rabbitmq:5672/' AMQP_URLS_SEPARATOR: ',' REDIS_URL: 'redis:6379' SQS_URL: ${SQS_URL} AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION} AWS_REGION: 'us-west-2' rabbitmq: container_name: machinery_sut_rabbitmq image: rabbitmq environment: - RABBITMQ_DEFAULT_USER=guest - RABBITMQ_DEFAULT_PASS=guest logging: driver: none redis: container_name: machinery_sut_redis image: redis logging: driver: none ================================================ FILE: v2/example/amqp/main.go ================================================ package main import ( "context" "errors" "fmt" "os" "time" "github.com/google/uuid" "github.com/urfave/cli" "github.com/RichardKnop/machinery/v2" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/log" "github.com/RichardKnop/machinery/v2/tasks" amqpbackend "github.com/RichardKnop/machinery/v2/backends/amqp" amqpbroker "github.com/RichardKnop/machinery/v2/brokers/amqp" exampletasks "github.com/RichardKnop/machinery/v2/example/tasks" "github.com/RichardKnop/machinery/v2/example/tracers" eagerlock "github.com/RichardKnop/machinery/v2/locks/eager" "github.com/opentracing/opentracing-go" opentracinglog "github.com/opentracing/opentracing-go/log" ) var ( app *cli.App ) func init() { // Initialise a CLI app app = cli.NewApp() app.Name = "machinery" app.Usage = "machinery worker and send example tasks with machinery send" app.Version = "0.0.0" } func main() { // Set the CLI app commands app.Commands = []cli.Command{ { Name: "worker", Usage: "launch machinery worker", Action: func(c *cli.Context) error { if err := worker(); err != nil { return cli.NewExitError(err.Error(), 1) } return nil }, }, { Name: "send", Usage: "send example tasks ", Action: func(c *cli.Context) error { if err := send(); err != nil { return cli.NewExitError(err.Error(), 1) } return nil }, }, } // Run the CLI app _ = app.Run(os.Args) } func startServer() (*machinery.Server, error) { cnf := &config.Config{ Broker: "amqp://guest:guest@localhost:5672/", DefaultQueue: "machinery_tasks", ResultBackend: "amqp://guest:guest@localhost:5672/", ResultsExpireIn: 3600, AMQP: &config.AMQPConfig{ Exchange: "machinery_exchange", ExchangeType: "direct", BindingKey: "machinery_task", PrefetchCount: 3, }, } // Create server instance broker := amqpbroker.New(cnf) backend := amqpbackend.New(cnf) lock := eagerlock.New() server := machinery.NewServer(cnf, broker, backend, lock) // Register tasks tasksMap := map[string]interface{}{ "add": exampletasks.Add, "multiply": exampletasks.Multiply, "sum_ints": exampletasks.SumInts, "sum_floats": exampletasks.SumFloats, "concat": exampletasks.Concat, "split": exampletasks.Split, "panic_task": exampletasks.PanicTask, "long_running_task": exampletasks.LongRunningTask, } return server, server.RegisterTasks(tasksMap) } func worker() error { consumerTag := "machinery_worker" cleanup, err := tracers.SetupTracer(consumerTag) if err != nil { log.FATAL.Fatalln("Unable to instantiate a tracer:", err) } defer cleanup() server, err := startServer() if err != nil { return err } // The second argument is a consumer tag // Ideally, each worker should have a unique tag (worker1, worker2 etc) worker := server.NewWorker(consumerTag, 0) // Here we inject some custom code for error handling, // start and end of task hooks, useful for metrics for example. errorHandler := func(err error) { log.ERROR.Println("I am an error handler:", err) } preTaskHandler := func(signature *tasks.Signature) { log.INFO.Println("I am a start of task handler for:", signature.Name) } postTaskHandler := func(signature *tasks.Signature) { log.INFO.Println("I am an end of task handler for:", signature.Name) } worker.SetPostTaskHandler(postTaskHandler) worker.SetErrorHandler(errorHandler) worker.SetPreTaskHandler(preTaskHandler) return worker.Launch() } func send() error { cleanup, err := tracers.SetupTracer("sender") if err != nil { log.FATAL.Fatalln("Unable to instantiate a tracer:", err) } defer cleanup() server, err := startServer() if err != nil { return err } var ( addTask0, addTask1, addTask2 tasks.Signature multiplyTask0, multiplyTask1 tasks.Signature sumIntsTask, sumFloatsTask, concatTask, splitTask tasks.Signature panicTask tasks.Signature longRunningTask tasks.Signature ) var initTasks = func() { addTask0 = tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 1, }, { Type: "int64", Value: 1, }, }, } addTask1 = tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 2, }, { Type: "int64", Value: 2, }, }, } addTask2 = tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 5, }, { Type: "int64", Value: 6, }, }, } multiplyTask0 = tasks.Signature{ Name: "multiply", Args: []tasks.Arg{ { Type: "int64", Value: 4, }, }, } multiplyTask1 = tasks.Signature{ Name: "multiply", } sumIntsTask = tasks.Signature{ Name: "sum_ints", Args: []tasks.Arg{ { Type: "[]int64", Value: []int64{1, 2}, }, }, } sumFloatsTask = tasks.Signature{ Name: "sum_floats", Args: []tasks.Arg{ { Type: "[]float64", Value: []float64{1.5, 2.7}, }, }, } concatTask = tasks.Signature{ Name: "concat", Args: []tasks.Arg{ { Type: "[]string", Value: []string{"foo", "bar"}, }, }, } splitTask = tasks.Signature{ Name: "split", Args: []tasks.Arg{ { Type: "string", Value: "foo", }, }, } panicTask = tasks.Signature{ Name: "panic_task", } longRunningTask = tasks.Signature{ Name: "long_running_task", } } /* * Lets start a span representing this run of the `send` command and * set a batch id as baggage so it can travel all the way into * the worker functions. */ span, ctx := opentracing.StartSpanFromContext(context.Background(), "send") defer span.Finish() batchID := uuid.New().String() span.SetBaggageItem("batch.id", batchID) span.LogFields(opentracinglog.String("batch.id", batchID)) log.INFO.Println("Starting batch:", batchID) /* * First, let's try sending a single task */ initTasks() log.INFO.Println("Single task:") asyncResult, err := server.SendTaskWithContext(ctx, &addTask0) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err := asyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf("1 + 1 = %v\n", tasks.HumanReadableResults(results)) /* * Try couple of tasks with a slice argument and slice return value */ asyncResult, err = server.SendTaskWithContext(ctx, &sumIntsTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err = asyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf("sum([1, 2]) = %v\n", tasks.HumanReadableResults(results)) asyncResult, err = server.SendTaskWithContext(ctx, &sumFloatsTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err = asyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf("sum([1.5, 2.7]) = %v\n", tasks.HumanReadableResults(results)) asyncResult, err = server.SendTaskWithContext(ctx, &concatTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err = asyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf("concat([\"foo\", \"bar\"]) = %v\n", tasks.HumanReadableResults(results)) asyncResult, err = server.SendTaskWithContext(ctx, &splitTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err = asyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf("split([\"foo\"]) = %v\n", tasks.HumanReadableResults(results)) /* * Now let's explore ways of sending multiple tasks */ // Now let's try a parallel execution initTasks() log.INFO.Println("Group of tasks (parallel execution):") group, err := tasks.NewGroup(&addTask0, &addTask1, &addTask2) if err != nil { return fmt.Errorf("Error creating group: %s", err.Error()) } asyncResults, err := server.SendGroupWithContext(ctx, group, 10) if err != nil { return fmt.Errorf("Could not send group: %s", err.Error()) } for _, asyncResult := range asyncResults { results, err = asyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf( "%v + %v = %v\n", asyncResult.Signature.Args[0].Value, asyncResult.Signature.Args[1].Value, tasks.HumanReadableResults(results), ) } // Now let's try a group with a chord initTasks() log.INFO.Println("Group of tasks with a callback (chord):") group, err = tasks.NewGroup(&addTask0, &addTask1, &addTask2) if err != nil { return fmt.Errorf("Error creating group: %s", err.Error()) } chord, err := tasks.NewChord(group, &multiplyTask1) if err != nil { return fmt.Errorf("Error creating chord: %s", err) } chordAsyncResult, err := server.SendChordWithContext(ctx, chord, 10) if err != nil { return fmt.Errorf("Could not send chord: %s", err.Error()) } results, err = chordAsyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting chord result failed with error: %s", err.Error()) } log.INFO.Printf("(1 + 1) * (2 + 2) * (5 + 6) = %v\n", tasks.HumanReadableResults(results)) // Now let's try chaining task results initTasks() log.INFO.Println("Chain of tasks:") chain, err := tasks.NewChain(&addTask0, &addTask1, &addTask2, &multiplyTask0) if err != nil { return fmt.Errorf("Error creating chain: %s", err) } chainAsyncResult, err := server.SendChainWithContext(ctx, chain) if err != nil { return fmt.Errorf("Could not send chain: %s", err.Error()) } results, err = chainAsyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting chain result failed with error: %s", err.Error()) } log.INFO.Printf("(((1 + 1) + (2 + 2)) + (5 + 6)) * 4 = %v\n", tasks.HumanReadableResults(results)) // Let's try a task which throws panic to make sure stack trace is not lost initTasks() asyncResult, err = server.SendTaskWithContext(ctx, &panicTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } _, err = asyncResult.Get(time.Millisecond * 5) if err == nil { return errors.New("Error should not be nil if task panicked") } log.INFO.Printf("Task panicked and returned error = %v\n", err.Error()) // Let's try a long running task initTasks() asyncResult, err = server.SendTaskWithContext(ctx, &longRunningTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err = asyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting long running task result failed with error: %s", err.Error()) } log.INFO.Printf("Long running task returned = %v\n", tasks.HumanReadableResults(results)) return nil } ================================================ FILE: v2/example/go-redis/main.go ================================================ package main import ( "context" "errors" "fmt" "os" "time" "github.com/google/uuid" "github.com/urfave/cli" "github.com/RichardKnop/machinery/v2" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/log" "github.com/RichardKnop/machinery/v2/tasks" redisbackend "github.com/RichardKnop/machinery/v2/backends/redis" redisbroker "github.com/RichardKnop/machinery/v2/brokers/redis" exampletasks "github.com/RichardKnop/machinery/v2/example/tasks" "github.com/RichardKnop/machinery/v2/example/tracers" eagerlock "github.com/RichardKnop/machinery/v2/locks/eager" "github.com/opentracing/opentracing-go" opentracinglog "github.com/opentracing/opentracing-go/log" ) var ( app *cli.App ) func init() { // Initialise a CLI app app = cli.NewApp() app.Name = "machinery" app.Usage = "machinery worker and send example tasks with machinery send" app.Version = "0.0.0" } func main() { // Set the CLI app commands app.Commands = []cli.Command{ { Name: "worker", Usage: "launch machinery worker", Action: func(c *cli.Context) error { if err := worker(); err != nil { return cli.NewExitError(err.Error(), 1) } return nil }, }, { Name: "send", Usage: "send example tasks ", Action: func(c *cli.Context) error { if err := send(); err != nil { return cli.NewExitError(err.Error(), 1) } return nil }, }, } // Run the CLI app _ = app.Run(os.Args) } func startServer() (*machinery.Server, error) { cnf := &config.Config{ DefaultQueue: "machinery_tasks", ResultsExpireIn: 3600, Redis: &config.RedisConfig{ MaxIdle: 3, IdleTimeout: 240, ReadTimeout: 15, WriteTimeout: 15, ConnectTimeout: 15, NormalTasksPollPeriod: 1000, DelayedTasksPollPeriod: 500, }, } // Create server instance broker := redisbroker.NewGR(cnf, []string{"localhost:6379"}, 0) backend := redisbackend.NewGR(cnf, []string{"localhost:6379"}, 0) lock := eagerlock.New() server := machinery.NewServer(cnf, broker, backend, lock) // Register tasks tasksMap := map[string]interface{}{ "add": exampletasks.Add, "multiply": exampletasks.Multiply, "sum_ints": exampletasks.SumInts, "sum_floats": exampletasks.SumFloats, "concat": exampletasks.Concat, "split": exampletasks.Split, "panic_task": exampletasks.PanicTask, "long_running_task": exampletasks.LongRunningTask, } return server, server.RegisterTasks(tasksMap) } func worker() error { consumerTag := "machinery_worker" cleanup, err := tracers.SetupTracer(consumerTag) if err != nil { log.FATAL.Fatalln("Unable to instantiate a tracer:", err) } defer cleanup() server, err := startServer() if err != nil { return err } // The second argument is a consumer tag // Ideally, each worker should have a unique tag (worker1, worker2 etc) worker := server.NewWorker(consumerTag, 0) // Here we inject some custom code for error handling, // start and end of task hooks, useful for metrics for example. errorHandler := func(err error) { log.ERROR.Println("I am an error handler:", err) } preTaskHandler := func(signature *tasks.Signature) { log.INFO.Println("I am a start of task handler for:", signature.Name) } postTaskHandler := func(signature *tasks.Signature) { log.INFO.Println("I am an end of task handler for:", signature.Name) } worker.SetPostTaskHandler(postTaskHandler) worker.SetErrorHandler(errorHandler) worker.SetPreTaskHandler(preTaskHandler) return worker.Launch() } func send() error { cleanup, err := tracers.SetupTracer("sender") if err != nil { log.FATAL.Fatalln("Unable to instantiate a tracer:", err) } defer cleanup() server, err := startServer() if err != nil { return err } var ( addTask0, addTask1, addTask2 tasks.Signature multiplyTask0, multiplyTask1 tasks.Signature sumIntsTask, sumFloatsTask, concatTask, splitTask tasks.Signature panicTask tasks.Signature longRunningTask tasks.Signature ) var initTasks = func() { addTask0 = tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 1, }, { Type: "int64", Value: 1, }, }, } addTask1 = tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 2, }, { Type: "int64", Value: 2, }, }, } addTask2 = tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 5, }, { Type: "int64", Value: 6, }, }, } multiplyTask0 = tasks.Signature{ Name: "multiply", Args: []tasks.Arg{ { Type: "int64", Value: 4, }, }, } multiplyTask1 = tasks.Signature{ Name: "multiply", } sumIntsTask = tasks.Signature{ Name: "sum_ints", Args: []tasks.Arg{ { Type: "[]int64", Value: []int64{1, 2}, }, }, } sumFloatsTask = tasks.Signature{ Name: "sum_floats", Args: []tasks.Arg{ { Type: "[]float64", Value: []float64{1.5, 2.7}, }, }, } concatTask = tasks.Signature{ Name: "concat", Args: []tasks.Arg{ { Type: "[]string", Value: []string{"foo", "bar"}, }, }, } splitTask = tasks.Signature{ Name: "split", Args: []tasks.Arg{ { Type: "string", Value: "foo", }, }, } panicTask = tasks.Signature{ Name: "panic_task", } longRunningTask = tasks.Signature{ Name: "long_running_task", } } /* * Lets start a span representing this run of the `send` command and * set a batch id as baggage so it can travel all the way into * the worker functions. */ span, ctx := opentracing.StartSpanFromContext(context.Background(), "send") defer span.Finish() batchID := uuid.New().String() span.SetBaggageItem("batch.id", batchID) span.LogFields(opentracinglog.String("batch.id", batchID)) log.INFO.Println("Starting batch:", batchID) /* * First, let's try sending a single task */ initTasks() log.INFO.Println("Single task:") asyncResult, err := server.SendTaskWithContext(ctx, &addTask0) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err := asyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf("1 + 1 = %v\n", tasks.HumanReadableResults(results)) /* * Try couple of tasks with a slice argument and slice return value */ asyncResult, err = server.SendTaskWithContext(ctx, &sumIntsTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err = asyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf("sum([1, 2]) = %v\n", tasks.HumanReadableResults(results)) asyncResult, err = server.SendTaskWithContext(ctx, &sumFloatsTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err = asyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf("sum([1.5, 2.7]) = %v\n", tasks.HumanReadableResults(results)) asyncResult, err = server.SendTaskWithContext(ctx, &concatTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err = asyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf("concat([\"foo\", \"bar\"]) = %v\n", tasks.HumanReadableResults(results)) asyncResult, err = server.SendTaskWithContext(ctx, &splitTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err = asyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf("split([\"foo\"]) = %v\n", tasks.HumanReadableResults(results)) /* * Now let's explore ways of sending multiple tasks */ // Now let's try a parallel execution initTasks() log.INFO.Println("Group of tasks (parallel execution):") group, err := tasks.NewGroup(&addTask0, &addTask1, &addTask2) if err != nil { return fmt.Errorf("Error creating group: %s", err.Error()) } asyncResults, err := server.SendGroupWithContext(ctx, group, 10) if err != nil { return fmt.Errorf("Could not send group: %s", err.Error()) } for _, asyncResult := range asyncResults { results, err = asyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf( "%v + %v = %v\n", asyncResult.Signature.Args[0].Value, asyncResult.Signature.Args[1].Value, tasks.HumanReadableResults(results), ) } // Now let's try a group with a chord initTasks() log.INFO.Println("Group of tasks with a callback (chord):") group, err = tasks.NewGroup(&addTask0, &addTask1, &addTask2) if err != nil { return fmt.Errorf("Error creating group: %s", err.Error()) } chord, err := tasks.NewChord(group, &multiplyTask1) if err != nil { return fmt.Errorf("Error creating chord: %s", err) } chordAsyncResult, err := server.SendChordWithContext(ctx, chord, 10) if err != nil { return fmt.Errorf("Could not send chord: %s", err.Error()) } results, err = chordAsyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting chord result failed with error: %s", err.Error()) } log.INFO.Printf("(1 + 1) * (2 + 2) * (5 + 6) = %v\n", tasks.HumanReadableResults(results)) // Now let's try chaining task results initTasks() log.INFO.Println("Chain of tasks:") chain, err := tasks.NewChain(&addTask0, &addTask1, &addTask2, &multiplyTask0) if err != nil { return fmt.Errorf("Error creating chain: %s", err) } chainAsyncResult, err := server.SendChainWithContext(ctx, chain) if err != nil { return fmt.Errorf("Could not send chain: %s", err.Error()) } results, err = chainAsyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting chain result failed with error: %s", err.Error()) } log.INFO.Printf("(((1 + 1) + (2 + 2)) + (5 + 6)) * 4 = %v\n", tasks.HumanReadableResults(results)) // Let's try a task which throws panic to make sure stack trace is not lost initTasks() asyncResult, err = server.SendTaskWithContext(ctx, &panicTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } _, err = asyncResult.Get(time.Millisecond * 5) if err == nil { return errors.New("Error should not be nil if task panicked") } log.INFO.Printf("Task panicked and returned error = %v\n", err.Error()) // Let's try a long running task initTasks() asyncResult, err = server.SendTaskWithContext(ctx, &longRunningTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err = asyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting long running task result failed with error: %s", err.Error()) } log.INFO.Printf("Long running task returned = %v\n", tasks.HumanReadableResults(results)) return nil } ================================================ FILE: v2/example/redigo/main.go ================================================ package main import ( "context" "errors" "fmt" "os" "time" "github.com/google/uuid" "github.com/urfave/cli" "github.com/RichardKnop/machinery/v2" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/log" "github.com/RichardKnop/machinery/v2/tasks" redisbackend "github.com/RichardKnop/machinery/v2/backends/redis" redisbroker "github.com/RichardKnop/machinery/v2/brokers/redis" exampletasks "github.com/RichardKnop/machinery/v2/example/tasks" "github.com/RichardKnop/machinery/v2/example/tracers" eagerlock "github.com/RichardKnop/machinery/v2/locks/eager" "github.com/opentracing/opentracing-go" opentracinglog "github.com/opentracing/opentracing-go/log" ) var ( app *cli.App ) func init() { // Initialise a CLI app app = cli.NewApp() app.Name = "machinery" app.Usage = "machinery worker and send example tasks with machinery send" app.Version = "0.0.0" } func main() { // Set the CLI app commands app.Commands = []cli.Command{ { Name: "worker", Usage: "launch machinery worker", Action: func(c *cli.Context) error { if err := worker(); err != nil { return cli.NewExitError(err.Error(), 1) } return nil }, }, { Name: "send", Usage: "send example tasks ", Action: func(c *cli.Context) error { if err := send(); err != nil { return cli.NewExitError(err.Error(), 1) } return nil }, }, } // Run the CLI app _ = app.Run(os.Args) } func startServer() (*machinery.Server, error) { cnf := &config.Config{ DefaultQueue: "machinery_tasks", ResultsExpireIn: 3600, Redis: &config.RedisConfig{ MaxIdle: 3, IdleTimeout: 240, ReadTimeout: 15, WriteTimeout: 15, ConnectTimeout: 15, NormalTasksPollPeriod: 1000, DelayedTasksPollPeriod: 500, }, } // Create server instance broker := redisbroker.New(cnf, "localhost:6379", "", "", "", 0) backend := redisbackend.New(cnf, "localhost:6379", "", "", "", 0) lock := eagerlock.New() server := machinery.NewServer(cnf, broker, backend, lock) // Register tasks tasksMap := map[string]interface{}{ "add": exampletasks.Add, "multiply": exampletasks.Multiply, "sum_ints": exampletasks.SumInts, "sum_floats": exampletasks.SumFloats, "concat": exampletasks.Concat, "split": exampletasks.Split, "panic_task": exampletasks.PanicTask, "long_running_task": exampletasks.LongRunningTask, } return server, server.RegisterTasks(tasksMap) } func worker() error { consumerTag := "machinery_worker" cleanup, err := tracers.SetupTracer(consumerTag) if err != nil { log.FATAL.Fatalln("Unable to instantiate a tracer:", err) } defer cleanup() server, err := startServer() if err != nil { return err } // The second argument is a consumer tag // Ideally, each worker should have a unique tag (worker1, worker2 etc) worker := server.NewWorker(consumerTag, 0) // Here we inject some custom code for error handling, // start and end of task hooks, useful for metrics for example. errorHandler := func(err error) { log.ERROR.Println("I am an error handler:", err) } preTaskHandler := func(signature *tasks.Signature) { log.INFO.Println("I am a start of task handler for:", signature.Name) } postTaskHandler := func(signature *tasks.Signature) { log.INFO.Println("I am an end of task handler for:", signature.Name) } worker.SetPostTaskHandler(postTaskHandler) worker.SetErrorHandler(errorHandler) worker.SetPreTaskHandler(preTaskHandler) return worker.Launch() } func send() error { cleanup, err := tracers.SetupTracer("sender") if err != nil { log.FATAL.Fatalln("Unable to instantiate a tracer:", err) } defer cleanup() server, err := startServer() if err != nil { return err } var ( addTask0, addTask1, addTask2 tasks.Signature multiplyTask0, multiplyTask1 tasks.Signature sumIntsTask, sumFloatsTask, concatTask, splitTask tasks.Signature panicTask tasks.Signature longRunningTask tasks.Signature ) var initTasks = func() { addTask0 = tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 1, }, { Type: "int64", Value: 1, }, }, } addTask1 = tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 2, }, { Type: "int64", Value: 2, }, }, } addTask2 = tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: 5, }, { Type: "int64", Value: 6, }, }, } multiplyTask0 = tasks.Signature{ Name: "multiply", Args: []tasks.Arg{ { Type: "int64", Value: 4, }, }, } multiplyTask1 = tasks.Signature{ Name: "multiply", } sumIntsTask = tasks.Signature{ Name: "sum_ints", Args: []tasks.Arg{ { Type: "[]int64", Value: []int64{1, 2}, }, }, } sumFloatsTask = tasks.Signature{ Name: "sum_floats", Args: []tasks.Arg{ { Type: "[]float64", Value: []float64{1.5, 2.7}, }, }, } concatTask = tasks.Signature{ Name: "concat", Args: []tasks.Arg{ { Type: "[]string", Value: []string{"foo", "bar"}, }, }, } splitTask = tasks.Signature{ Name: "split", Args: []tasks.Arg{ { Type: "string", Value: "foo", }, }, } panicTask = tasks.Signature{ Name: "panic_task", } longRunningTask = tasks.Signature{ Name: "long_running_task", } } /* * Lets start a span representing this run of the `send` command and * set a batch id as baggage so it can travel all the way into * the worker functions. */ span, ctx := opentracing.StartSpanFromContext(context.Background(), "send") defer span.Finish() batchID := uuid.New().String() span.SetBaggageItem("batch.id", batchID) span.LogFields(opentracinglog.String("batch.id", batchID)) log.INFO.Println("Starting batch:", batchID) /* * First, let's try sending a single task */ initTasks() log.INFO.Println("Single task:") asyncResult, err := server.SendTaskWithContext(ctx, &addTask0) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err := asyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf("1 + 1 = %v\n", tasks.HumanReadableResults(results)) /* * Try couple of tasks with a slice argument and slice return value */ asyncResult, err = server.SendTaskWithContext(ctx, &sumIntsTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err = asyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf("sum([1, 2]) = %v\n", tasks.HumanReadableResults(results)) asyncResult, err = server.SendTaskWithContext(ctx, &sumFloatsTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err = asyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf("sum([1.5, 2.7]) = %v\n", tasks.HumanReadableResults(results)) asyncResult, err = server.SendTaskWithContext(ctx, &concatTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err = asyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf("concat([\"foo\", \"bar\"]) = %v\n", tasks.HumanReadableResults(results)) asyncResult, err = server.SendTaskWithContext(ctx, &splitTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err = asyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf("split([\"foo\"]) = %v\n", tasks.HumanReadableResults(results)) /* * Now let's explore ways of sending multiple tasks */ // Now let's try a parallel execution initTasks() log.INFO.Println("Group of tasks (parallel execution):") group, err := tasks.NewGroup(&addTask0, &addTask1, &addTask2) if err != nil { return fmt.Errorf("Error creating group: %s", err.Error()) } asyncResults, err := server.SendGroupWithContext(ctx, group, 10) if err != nil { return fmt.Errorf("Could not send group: %s", err.Error()) } for _, asyncResult := range asyncResults { results, err = asyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting task result failed with error: %s", err.Error()) } log.INFO.Printf( "%v + %v = %v\n", asyncResult.Signature.Args[0].Value, asyncResult.Signature.Args[1].Value, tasks.HumanReadableResults(results), ) } // Now let's try a group with a chord initTasks() log.INFO.Println("Group of tasks with a callback (chord):") group, err = tasks.NewGroup(&addTask0, &addTask1, &addTask2) if err != nil { return fmt.Errorf("Error creating group: %s", err.Error()) } chord, err := tasks.NewChord(group, &multiplyTask1) if err != nil { return fmt.Errorf("Error creating chord: %s", err) } chordAsyncResult, err := server.SendChordWithContext(ctx, chord, 10) if err != nil { return fmt.Errorf("Could not send chord: %s", err.Error()) } results, err = chordAsyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting chord result failed with error: %s", err.Error()) } log.INFO.Printf("(1 + 1) * (2 + 2) * (5 + 6) = %v\n", tasks.HumanReadableResults(results)) // Now let's try chaining task results initTasks() log.INFO.Println("Chain of tasks:") chain, err := tasks.NewChain(&addTask0, &addTask1, &addTask2, &multiplyTask0) if err != nil { return fmt.Errorf("Error creating chain: %s", err) } chainAsyncResult, err := server.SendChainWithContext(ctx, chain) if err != nil { return fmt.Errorf("Could not send chain: %s", err.Error()) } results, err = chainAsyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting chain result failed with error: %s", err.Error()) } log.INFO.Printf("(((1 + 1) + (2 + 2)) + (5 + 6)) * 4 = %v\n", tasks.HumanReadableResults(results)) // Let's try a task which throws panic to make sure stack trace is not lost initTasks() asyncResult, err = server.SendTaskWithContext(ctx, &panicTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } _, err = asyncResult.Get(time.Millisecond * 5) if err == nil { return errors.New("Error should not be nil if task panicked") } log.INFO.Printf("Task panicked and returned error = %v\n", err.Error()) // Let's try a long running task initTasks() asyncResult, err = server.SendTaskWithContext(ctx, &longRunningTask) if err != nil { return fmt.Errorf("Could not send task: %s", err.Error()) } results, err = asyncResult.Get(time.Millisecond * 5) if err != nil { return fmt.Errorf("Getting long running task result failed with error: %s", err.Error()) } log.INFO.Printf("Long running task returned = %v\n", tasks.HumanReadableResults(results)) return nil } ================================================ FILE: v2/example/tasks/tasks.go ================================================ package exampletasks import ( "errors" "strings" "time" "github.com/RichardKnop/machinery/v2/log" ) // Add ... func Add(args ...int64) (int64, error) { sum := int64(0) for _, arg := range args { sum += arg } return sum, nil } // Multiply ... func Multiply(args ...int64) (int64, error) { sum := int64(1) for _, arg := range args { sum *= arg } return sum, nil } // SumInts ... func SumInts(numbers []int64) (int64, error) { var sum int64 for _, num := range numbers { sum += num } return sum, nil } // SumFloats ... func SumFloats(numbers []float64) (float64, error) { var sum float64 for _, num := range numbers { sum += num } return sum, nil } // Concat ... func Concat(strs []string) (string, error) { var res string for _, s := range strs { res += s } return res, nil } // Split ... func Split(str string) ([]string, error) { return strings.Split(str, ""), nil } // PanicTask ... func PanicTask() (string, error) { panic(errors.New("oops")) } // LongRunningTask ... func LongRunningTask() error { log.INFO.Print("Long running task started") for i := 0; i < 10; i++ { log.INFO.Print(10 - i) time.Sleep(1 * time.Second) } log.INFO.Print("Long running task finished") return nil } ================================================ FILE: v2/example/tracers/jaeger.go ================================================ package tracers // Uncomment the import statement for the jaeger tracer. // make sure you run dep ensure to pull in the jaeger client // // import ( // jaeger "github.com/uber/jaeger-client-go" // jaegercfg "github.com/uber/jaeger-client-go/config" // ) // SetupTracer is the place where you'd setup your specific tracer. // The jaeger tracer is given as an example. // To capture the jaeger traces you should run the jaeger backend. // This can be done using the following docker command: // // `docker run -ti --rm -p6831:6831/udp -p16686:16686 jaegertracing/all-in-one:latest` // // The collector will be listening on localhost:6831 // and the query UI is reachable on localhost:16686. func SetupTracer(serviceName string) (func(), error) { // Jaeger setup code // // config := jaegercfg.Configuration{ // Sampler: &jaegercfg.SamplerConfig{ // Type: jaeger.SamplerTypeConst, // Param: 1, // }, // } // closer, err := config.InitGlobalTracer(serviceName) // if err != nil { // return nil, err // } cleanupFunc := func() { // closer.Close() } return cleanupFunc, nil } ================================================ FILE: v2/go.mod ================================================ module github.com/RichardKnop/machinery/v2 go 1.22 toolchain go1.24.1 require ( cloud.google.com/go/pubsub v1.10.0 github.com/RichardKnop/logging v0.0.0-20190827224416-1a693bdd4fae github.com/aws/aws-sdk-go-v2 v1.38.1 github.com/aws/aws-sdk-go-v2/config v1.29.14 github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.18.12 github.com/aws/aws-sdk-go-v2/service/dynamodb v1.42.4 github.com/aws/aws-sdk-go-v2/service/sqs v1.38.5 github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b github.com/go-redsync/redsync/v4 v4.8.1 github.com/gomodule/redigo v1.9.2 github.com/google/uuid v1.6.0 github.com/kelseyhightower/envconfig v1.4.0 github.com/opentracing/opentracing-go v1.2.0 github.com/rabbitmq/amqp091-go v1.10.0 github.com/redis/go-redis/v9 v9.12.1 github.com/robfig/cron/v3 v3.0.1 github.com/stretchr/testify v1.10.0 github.com/urfave/cli v1.22.5 go.mongodb.org/mongo-driver v1.17.0 gopkg.in/yaml.v2 v2.4.0 ) require ( cloud.google.com/go v0.75.0 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.67 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.25.3 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.15 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect github.com/aws/smithy-go v1.22.5 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/googleapis/gax-go/v2 v2.0.5 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/jstemmer/go-junit-report v0.9.1 // indirect github.com/klauspost/compress v1.13.6 // indirect github.com/montanaflynn/stats v0.7.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/russross/blackfriday/v2 v2.0.1 // indirect github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect go.opencensus.io v0.22.5 // indirect golang.org/x/crypto v0.26.0 // indirect golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/net v0.25.0 // indirect golang.org/x/oauth2 v0.0.0-20210113205817-d3ed898aa8a3 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.23.0 // indirect golang.org/x/text v0.17.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect google.golang.org/api v0.39.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20210207032614-bba0dbe2a9ea // indirect google.golang.org/grpc v1.35.0 // indirect google.golang.org/protobuf v1.26.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) replace git.apache.org/thrift.git => github.com/apache/thrift v0.0.0-20180902110319-2566ecd5d999 ================================================ FILE: v2/go.sum ================================================ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.75.0 h1:XgtDnVJRCPEUG21gjFiRPz4zI1Mjg16R+NYQjfmU4XY= cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/pubsub v1.10.0 h1:JK22g5uNpscGPthjJE/D0siWtA6UlU4Cb6pLcyJkzyQ= cloud.google.com/go/pubsub v1.10.0/go.mod h1:eNpTrkOy7dCpkNyaSNetMa6udbgecJMd0ZsTJS/cuNo= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/RichardKnop/logging v0.0.0-20190827224416-1a693bdd4fae h1:DcFpTQBYQ9Ct2d6sC7ol0/ynxc2pO1cpGUM+f4t5adg= github.com/RichardKnop/logging v0.0.0-20190827224416-1a693bdd4fae/go.mod h1:rJJ84PyA/Wlmw1hO+xTzV2wsSUon6J5ktg0g8BF2PuU= github.com/aws/aws-sdk-go-v2 v1.38.1 h1:j7sc33amE74Rz0M/PoCpsZQ6OunLqys/m5antM0J+Z8= github.com/aws/aws-sdk-go-v2 v1.38.1/go.mod h1:9Q0OoGQoboYIAJyslFyF1f5K1Ryddop8gqMhWx/n4Wg= github.com/aws/aws-sdk-go-v2/config v1.29.14 h1:f+eEi/2cKCg9pqKBoAIwRGzVb70MRKqWX4dg1BDcSJM= github.com/aws/aws-sdk-go-v2/config v1.29.14/go.mod h1:wVPHWcIFv3WO89w0rE10gzf17ZYy+UVS1Geq8Iei34g= github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM= github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ= github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.18.12 h1:mwAIR3fhxhSzXFj530LNCBe0JocYVQx6GuJpQiA+QOs= github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.18.12/go.mod h1:9cWrNL8q7ApFmZzKhnb63ub4zrdMzOGQVn/kxvagfeE= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.42.4 h1:5GjCSGIpndYU/tVABz+4XnAcluU6wrjlPzAAgFUDG98= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.42.4/go.mod h1:yYaWRnVSPyAmexW5t7G3TcuYoalYfT+xQwzWsvtUQ7M= github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.25.3 h1:GHC1WTF3ZBZy+gvz2qtYB6ttALVx35hlwc4IzOIUY7g= github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.25.3/go.mod h1:lUqWdw5/esjPTkITXhN4C66o1ltwDq2qQ12j3SOzhVg= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.15 h1:M1R1rud7HzDrfCdlBQ7NjnRsDNEhXO/vGhuD189Ggmk= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.15/go.mod h1:uvFKBSq9yMPV4LGAi7N4awn4tLY+hKE35f8THes2mzQ= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= github.com/aws/aws-sdk-go-v2/service/sqs v1.38.5 h1:KNgVWw8qbPzjYnIF1gL0EAszy6VKGnmUK6VSm1huYY8= github.com/aws/aws-sdk-go-v2/service/sqs v1.38.5/go.mod h1:Bar4MrRxeqdn6XIh8JGfiXuFRmyrrsZNTJotxEJmWW0= github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8= github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 h1:1XuUZ8mYJw9B6lzAkXhqHlJd/XvaX32evhproijJEZY= github.com/aws/aws-sdk-go-v2/service/sts v1.33.19/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= github.com/aws/smithy-go v1.22.5 h1:P9ATCXPMb2mPjYBgueqJNCA5S9UfktsW0tTxi+a7eqw= github.com/aws/smithy-go v1.22.5/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b h1:L/QXpzIa3pOvUGt1D1lA5KjYhPBAN/3iWdP7xeFS9F0= github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= github.com/bsm/ginkgo/v2 v2.5.0/go.mod h1:AiKlXPm7ItEHNc/2+OkrNG4E0ITzojb9/xWzvQ9XZ9w= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.20.0/go.mod h1:JifAceMQ4crZIWYUKrlGcmbN3bqHogVTADMD2ATsbwk= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-redis/redis/v7 v7.4.0 h1:7obg6wUoj05T0EpY0o8B59S9w5yeMWql7sw2kwNW1x4= github.com/go-redis/redis/v7 v7.4.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg= github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg= github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= github.com/go-redsync/redsync/v4 v4.8.1 h1:rq2RvdTI0obznMdxKUWGdmmulo7lS9yCzb8fgDKOlbM= github.com/go-redsync/redsync/v4 v4.8.1/go.mod h1:LmUAsQuQxhzZAoGY7JS6+dNhNmZyonMZiiEDY9plotM= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/gomodule/redigo v1.9.2 h1:HrutZBLhSIU8abiSfW8pj8mPhOyMYjZT/wcA4/L9L9s= github.com/gomodule/redigo v1.9.2/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= github.com/redis/go-redis/v9 v9.0.2/go.mod h1:/xDTe9EF1LM61hek62Poq2nzQSGj0xSrEtEHbBQevps= github.com/redis/go-redis/v9 v9.12.1 h1:k5iquqv27aBtnTm2tIkROUDp8JBXhXZIVu1InSgvovg= github.com/redis/go-redis/v9 v9.12.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 h1:QVqDTf3h2WHt08YuiTGPZLls0Wq99X9bWd0Q5ZSBesM= github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203/go.mod h1:oqN97ltKNihBbwlX8dLpwxCl3+HnXKV/R0e+sRLd9C8= github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mongodb.org/mongo-driver v1.17.0 h1:Hp4q2MCjvY19ViwimTs00wHi7G4yzxh4/2+nTx8r40k= go.mongodb.org/mongo-driver v1.17.0/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210113205817-d3ed898aa8a3 h1:BaN3BAqnopnKjvl+15DYP6LLrbBHfbfmlFYzmFj/Q9Q= golang.org/x/oauth2 v0.0.0-20210113205817-d3ed898aa8a3/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.39.0 h1:zHCTXf0NeDdKTgcSQpT+ZflWAqHsEp1GmdpxW09f3YM= google.golang.org/api v0.39.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210207032614-bba0dbe2a9ea h1:N98SvVh7Hdle2lgUVFuIkf0B3u29CUakMUQa7Hwz8Wc= google.golang.org/genproto v0.0.0-20210207032614-bba0dbe2a9ea/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= ================================================ FILE: v2/integration-tests/amqp_amqp_test.go ================================================ package integration_test import ( "os" "testing" "github.com/RichardKnop/machinery/v2" "github.com/RichardKnop/machinery/v2/config" amqpbackend "github.com/RichardKnop/machinery/v2/backends/amqp" amqpbroker "github.com/RichardKnop/machinery/v2/brokers/amqp" eagerlock "github.com/RichardKnop/machinery/v2/locks/eager" ) func TestAmqpAmqp(t *testing.T) { amqpURL := os.Getenv("AMQP_URL") if amqpURL == "" { t.Skip("AMQP_URL is not defined") } finalAmqpURL := amqpURL var finalSeparator string amqpURLs := os.Getenv("AMQP_URLS") if amqpURLs != "" { separator := os.Getenv("AMQP_URLS_SEPARATOR") if separator == "" { return } finalSeparator = separator finalAmqpURL = amqpURLs } cnf := &config.Config{ Broker: finalAmqpURL, MultipleBrokerSeparator: finalSeparator, DefaultQueue: "machinery_tasks", ResultBackend: amqpURL, ResultsExpireIn: 3600, AMQP: &config.AMQPConfig{ Exchange: "test_exchange", ExchangeType: "direct", BindingKey: "test_task", PrefetchCount: 1, }, } broker := amqpbroker.New(cnf) backend := amqpbackend.New(cnf) lock := eagerlock.New() server := machinery.NewServer(cnf, broker, backend, lock) registerTestTasks(server) worker := server.NewWorker("test_worker", 0) defer worker.Quit() go worker.Launch() testAll(server, t) } ================================================ FILE: v2/integration-tests/redis_redis_test.go ================================================ package integration_test import ( "os" "testing" "github.com/RichardKnop/machinery/v2" redisbackend "github.com/RichardKnop/machinery/v2/backends/redis" redisbroker "github.com/RichardKnop/machinery/v2/brokers/redis" "github.com/RichardKnop/machinery/v2/config" eagerlock "github.com/RichardKnop/machinery/v2/locks/eager" ) func TestRedisRedis_GoRedis(t *testing.T) { redisURL := os.Getenv("REDIS_URL") if redisURL == "" { t.Skip("REDIS_URL is not defined") } cnf := &config.Config{ DefaultQueue: "machinery_tasks", ResultsExpireIn: 3600, Redis: &config.RedisConfig{ MaxIdle: 3, IdleTimeout: 240, ReadTimeout: 15, WriteTimeout: 15, ConnectTimeout: 15, NormalTasksPollPeriod: 1000, DelayedTasksPollPeriod: 500, }, } broker := redisbroker.NewGR(cnf, []string{redisURL}, 0) backend := redisbackend.NewGR(cnf, []string{redisURL}, 0) lock := eagerlock.New() server := machinery.NewServer(cnf, broker, backend, lock) registerTestTasks(server) worker := server.NewWorker("test_worker", 0) defer worker.Quit() go worker.Launch() testAll(server, t) } ================================================ FILE: v2/integration-tests/suite_test.go ================================================ package integration_test import ( "context" "errors" "reflect" "sort" "testing" "time" "github.com/stretchr/testify/assert" "github.com/RichardKnop/machinery/v2/backends/result" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/tasks" brokersiface "github.com/RichardKnop/machinery/v2/brokers/iface" ) type ascendingInt64s []int64 func (a ascendingInt64s) Len() int { return len(a) } func (a ascendingInt64s) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a ascendingInt64s) Less(i, j int) bool { return a[i] < a[j] } type Server interface { GetBroker() brokersiface.Broker GetConfig() *config.Config RegisterTasks(namedTaskFuncs map[string]interface{}) error SendTaskWithContext(ctx context.Context, signature *tasks.Signature) (*result.AsyncResult, error) SendTask(signature *tasks.Signature) (*result.AsyncResult, error) SendChainWithContext(ctx context.Context, chain *tasks.Chain) (*result.ChainAsyncResult, error) SendChain(chain *tasks.Chain) (*result.ChainAsyncResult, error) SendGroupWithContext(ctx context.Context, group *tasks.Group, sendConcurrency int) ([]*result.AsyncResult, error) SendGroup(group *tasks.Group, sendConcurrency int) ([]*result.AsyncResult, error) SendChordWithContext(ctx context.Context, chord *tasks.Chord, sendConcurrency int) (*result.ChordAsyncResult, error) SendChord(chord *tasks.Chord, sendConcurrency int) (*result.ChordAsyncResult, error) } func testAll(server Server, t *testing.T) { testSendTask(server, t) testSendGroup(server, t, 0) // with unlimited concurrency testSendGroup(server, t, 2) // with limited concurrency (2 parallel tasks at the most) testSendChord(server, t) testSendChain(server, t) testReturnJustError(server, t) testReturnMultipleValues(server, t) testPanic(server, t) testDelay(server, t) } func testSendTask(server Server, t *testing.T) { addTask := newAddTask(1, 1) asyncResult, err := server.SendTask(addTask) if err != nil { t.Error(err) } results, err := asyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { t.Error(err) } if len(results) != 1 { t.Errorf("Number of results returned = %d. Wanted %d", len(results), 1) } if results[0].Interface() != int64(2) { t.Errorf( "result = %v(%v), want int64(2)", results[0].Type().String(), results[0].Interface(), ) } sumTask := newSumTask([]int64{1, 2}) asyncResult, err = server.SendTask(sumTask) if err != nil { t.Error(err) } results, err = asyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { t.Error(err) } if len(results) != 1 { t.Errorf("Number of results returned = %d. Wanted %d", len(results), 1) } if results[0].Interface() != int64(3) { t.Errorf( "result = %v(%v), want int64(3)", results[0].Type().String(), results[0].Interface(), ) } } func testSendGroup(server Server, t *testing.T, sendConcurrency int) { t1, t2, t3 := newAddTask(1, 1), newAddTask(2, 2), newAddTask(5, 6) group, err := tasks.NewGroup(t1, t2, t3) if err != nil { t.Fatal(err) } asyncResults, err := server.SendGroup(group, sendConcurrency) if err != nil { t.Error(err) } expectedResults := []int64{2, 4, 11} actualResults := make([]int64, 3) for i, asyncResult := range asyncResults { results, err := asyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { t.Error(err) } if len(results) != 1 { t.Errorf("Number of results returned = %d. Wanted %d", len(results), 1) } intResult, ok := results[0].Interface().(int64) if !ok { t.Errorf("Could not convert %v to int64", results[0].Interface()) } actualResults[i] = intResult } sort.Sort(ascendingInt64s(actualResults)) if !reflect.DeepEqual(expectedResults, actualResults) { t.Errorf( "expected results = %v, actual results = %v", expectedResults, actualResults, ) } } func testSendChain(server Server, t *testing.T) { t1, t2, t3 := newAddTask(2, 2), newAddTask(5, 6), newMultipleTask(4) chain, err := tasks.NewChain(t1, t2, t3) if err != nil { t.Fatal(err) } chainAsyncResult, err := server.SendChain(chain) if err != nil { t.Error(err) } results, err := chainAsyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { t.Error(err) } if len(results) != 1 { t.Errorf("Number of results returned = %d. Wanted %d", len(results), 1) } if results[0].Interface() != int64(60) { t.Errorf( "result = %v(%v), want int64(60)", results[0].Type().String(), results[0].Interface(), ) } } func testSendChord(server Server, t *testing.T) { t1, t2, t3, t4 := newAddTask(1, 1), newAddTask(2, 2), newAddTask(5, 6), newMultipleTask() group, err := tasks.NewGroup(t1, t2, t3) if err != nil { t.Fatal(err) } chord, err := tasks.NewChord(group, t4) if err != nil { t.Fatal(err) } chordAsyncResult, err := server.SendChord(chord, 10) if err != nil { t.Error(err) } results, err := chordAsyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { t.Error(err) } if len(results) != 1 { t.Errorf("Number of results returned = %d. Wanted %d", len(results), 1) } if results[0].Interface() != int64(88) { t.Errorf( "result = %v(%v), want int64(88)", results[0].Type().String(), results[0].Interface(), ) } } func testReturnJustError(server Server, t *testing.T) { // Fails, returns error as the only value task := newErrorTask("Test error", true) asyncResult, err := server.SendTask(task) if err != nil { t.Error(err) } results, err := asyncResult.Get(time.Duration(time.Millisecond * 5)) if len(results) != 0 { t.Errorf("Number of results returned = %d. Wanted %d", len(results), 0) } assert.Equal(t, "Test error", err.Error()) // Successful, returns nil as the only value task = newErrorTask("", false) asyncResult, err = server.SendTask(task) if err != nil { t.Error(err) } results, err = asyncResult.Get(time.Duration(time.Millisecond * 5)) if len(results) != 0 { t.Errorf("Number of results returned = %d. Wanted %d", len(results), 0) } assert.NoError(t, err) } func testReturnMultipleValues(server Server, t *testing.T) { // Successful task with multiple return values task := newMultipleReturnTask("foo", "bar", false) asyncResult, err := server.SendTask(task) if err != nil { t.Error(err) } results, err := asyncResult.Get(time.Duration(time.Millisecond * 5)) if err != nil { t.Error(err) } if len(results) != 2 { t.Errorf("Number of results returned = %d. Wanted %d", len(results), 2) } if results[0].Interface() != "foo" { t.Errorf( "result = %v(%v), want string(\"foo\":)", results[0].Type().String(), results[0].Interface(), ) } if results[1].Interface() != "bar" { t.Errorf( "result = %v(%v), want string(\"bar\":)", results[1].Type().String(), results[1].Interface(), ) } // Failed task with multiple return values task = newMultipleReturnTask("", "", true) asyncResult, err = server.SendTask(task) if err != nil { t.Error(err) } results, err = asyncResult.Get(time.Duration(time.Millisecond * 5)) if len(results) != 0 { t.Errorf("Number of results returned = %d. Wanted %d", len(results), 0) } assert.Error(t, err) } func testPanic(server Server, t *testing.T) { task := &tasks.Signature{Name: "panic"} asyncResult, err := server.SendTask(task) if err != nil { t.Error(err) } results, err := asyncResult.Get(time.Duration(time.Millisecond * 5)) if len(results) != 0 { t.Errorf("Number of results returned = %d. Wanted %d", len(results), 0) } assert.Equal(t, "oops", err.Error()) } func testDelay(server Server, t *testing.T) { now := time.Now().UTC() eta := now.Add(100 * time.Millisecond) task := newDelayTask(eta) asyncResult, err := server.SendTask(task) if err != nil { t.Error(err) } results, err := asyncResult.Get(time.Duration(5 * time.Millisecond)) if err != nil { t.Error(err) } if len(results) != 1 { t.Errorf("Number of results returned = %d. Wanted %d", len(results), 1) } tm, ok := results[0].Interface().(int64) if !ok { t.Errorf( "Could not type assert = %v(%v) to int64", results[0].Type().String(), results[0].Interface(), ) } if tm < eta.UnixNano() { t.Errorf( "result = %v(%v), want >= int64(%d)", results[0].Type().String(), results[0].Interface(), eta.UnixNano(), ) } } func registerTestTasks(server Server) { tasks := map[string]interface{}{ "add": func(args ...int64) (int64, error) { sum := int64(0) for _, arg := range args { sum += arg } return sum, nil }, "multiply": func(args ...int64) (int64, error) { sum := int64(1) for _, arg := range args { sum *= arg } return sum, nil }, "sum": func(numbers []int64) (int64, error) { var sum int64 for _, num := range numbers { sum += num } return sum, nil }, "return_just_error": func(msg string, fail bool) (err error) { if fail { err = errors.New(msg) } return err }, "return_multiple_values": func(arg1, arg2 string, fail bool) (r1 string, r2 string, err error) { if fail { err = errors.New("some error") } else { r1 = arg1 r2 = arg2 } return r1, r2, err }, "panic": func() (string, error) { panic(errors.New("oops")) }, "delay_test": func() (int64, error) { return time.Now().UTC().UnixNano(), nil }, } server.RegisterTasks(tasks) } func newAddTask(a, b int) *tasks.Signature { return &tasks.Signature{ Name: "add", Args: []tasks.Arg{ { Type: "int64", Value: a, }, { Type: "int64", Value: b, }, }, } } func newMultipleTask(nums ...int) *tasks.Signature { args := make([]tasks.Arg, len(nums)) for i, n := range nums { args[i] = tasks.Arg{ Type: "int64", Value: n, } } return &tasks.Signature{ Name: "multiply", Args: args, } } func newSumTask(nums []int64) *tasks.Signature { return &tasks.Signature{ Name: "sum", Args: []tasks.Arg{ { Type: "[]int64", Value: nums, }, }, } } func newErrorTask(msg string, fail bool) *tasks.Signature { return &tasks.Signature{ Name: "return_just_error", Args: []tasks.Arg{ { Type: "string", Value: msg, }, { Type: "bool", Value: fail, }, }, } } func newMultipleReturnTask(arg1, arg2 string, fail bool) *tasks.Signature { return &tasks.Signature{ Name: "return_multiple_values", Args: []tasks.Arg{ { Type: "string", Value: arg1, }, { Type: "string", Value: arg2, }, { Type: "bool", Value: fail, }, }, } } func newDelayTask(eta time.Time) *tasks.Signature { return &tasks.Signature{ Name: "delay_test", ETA: &eta, } } ================================================ FILE: v2/locks/eager/eager.go ================================================ package eager import ( "errors" "sync" "time" ) var ( ErrEagerLockFailed = errors.New("eager lock: failed to acquire lock") ) type Lock struct { retries int interval time.Duration register struct { sync.RWMutex m map[string]int64 } } func New() *Lock { return &Lock{ retries: 3, interval: 5 * time.Second, register: struct { sync.RWMutex m map[string]int64 }{m: make(map[string]int64)}, } } func (e *Lock) LockWithRetries(key string, value int64) error { for i := 0; i <= e.retries; i++ { err := e.Lock(key, value) if err == nil { //成功拿到锁，返回 return nil } time.Sleep(e.interval) } return ErrEagerLockFailed } func (e *Lock) Lock(key string, value int64) error { e.register.Lock() defer e.register.Unlock() timeout, exist := e.register.m[key] if !exist || time.Now().UnixNano() > timeout { e.register.m[key] = value return nil } return ErrEagerLockFailed } ================================================ FILE: v2/locks/eager/eager_test.go ================================================ package eager import ( lockiface "github.com/RichardKnop/machinery/v2/locks/iface" "github.com/RichardKnop/machinery/v2/utils" "github.com/stretchr/testify/assert" "testing" "time" ) func TestLock_Lock(t *testing.T) { lock := New() keyName := utils.GetPureUUID() go func() { err := lock.Lock(keyName, time.Now().Add(25*time.Second).UnixNano()) assert.NoError(t, err) }() time.Sleep(1 * time.Second) err := lock.Lock(keyName, time.Now().Add(25*time.Second).UnixNano()) assert.Error(t, err) assert.EqualError(t, err, ErrEagerLockFailed.Error()) } func TestLock_LockWithRetries(t *testing.T) { lock := New() keyName := utils.GetPureUUID() go func() { err := lock.LockWithRetries(keyName, time.Now().Add(25*time.Second).UnixNano()) assert.NoError(t, err) }() time.Sleep(1 * time.Second) err := lock.LockWithRetries(keyName, time.Now().Add(25*time.Second).UnixNano()) assert.Error(t, err) assert.EqualError(t, err, ErrEagerLockFailed.Error()) } func TestNew(t *testing.T) { lock := New() assert.Implements(t, (*lockiface.Lock)(nil), lock) } ================================================ FILE: v2/locks/iface/interfaces.go ================================================ package iface type Lock interface { //Acquire the lock with retry //key: the name of the lock, //value: at the nanosecond timestamp that lock needs to be released automatically LockWithRetries(key string, value int64) error //Acquire the lock with once //key: the name of the lock, //value: at the nanosecond timestamp that lock needs to be released automatically Lock(key string, value int64) error } ================================================ FILE: v2/locks/redis/redis.go ================================================ package redis import ( "context" "errors" "strconv" "strings" "time" "github.com/redis/go-redis/v9" "github.com/RichardKnop/machinery/v2/config" ) var ( ErrRedisLockFailed = errors.New("redis lock: failed to acquire lock") ) type Lock struct { rclient redis.UniversalClient retries int interval time.Duration } func New(cnf *config.Config, addrs []string, db, retries int) Lock { if retries <= 0 { return Lock{} } lock := Lock{retries: retries} var password string parts := strings.Split(addrs[0], "@") if len(parts) >= 2 { password = strings.Join(parts[:len(parts)-1], "@") addrs[0] = parts[len(parts)-1] // addr is the last one without @ } ropt := &redis.UniversalOptions{ Addrs: addrs, DB: db, Password: password, } if cnf.Redis != nil { ropt.MasterName = cnf.Redis.MasterName } if cnf.Redis != nil && cnf.Redis.SentinelPassword != "" { ropt.SentinelPassword = cnf.Redis.SentinelPassword } if cnf.Redis != nil && cnf.Redis.ClusterEnabled { lock.rclient = redis.NewClusterClient(ropt.Cluster()) } else { lock.rclient = redis.NewUniversalClient(ropt) } return lock } func (r Lock) LockWithRetries(key string, unixTsToExpireNs int64) error { for i := 0; i <= r.retries; i++ { err := r.Lock(key, unixTsToExpireNs) if err == nil { // 成功拿到锁，返回 return nil } time.Sleep(r.interval) } return ErrRedisLockFailed } func (r Lock) Lock(key string, unixTsToExpireNs int64) error { now := time.Now().UnixNano() expiration := time.Duration(unixTsToExpireNs + 1 - now) ctx := context.Background() success, err := r.rclient.SetNX(ctx, key, unixTsToExpireNs, expiration).Result() if err != nil { return err } if !success { v, err := r.rclient.Get(ctx, key).Result() if err != nil { return err } timeout, err := strconv.Atoi(v) if err != nil { return err } if timeout != 0 && now > int64(timeout) { newTimeout, err := r.rclient.GetSet(ctx, key, unixTsToExpireNs).Result() if err != nil { return err } curTimeout, err := strconv.Atoi(newTimeout) if err != nil { return err } if now > int64(curTimeout) { // success to acquire lock with get set // set the expiration of redis key r.rclient.Expire(ctx, key, expiration) return nil } return ErrRedisLockFailed } return ErrRedisLockFailed } return nil } ================================================ FILE: v2/log/log.go ================================================ package log import ( "github.com/RichardKnop/logging" ) var ( logger = logging.New(nil, nil, new(logging.ColouredFormatter)) // DEBUG ... DEBUG = logger[logging.DEBUG] // INFO ... INFO = logger[logging.INFO] // WARNING ... WARNING = logger[logging.WARNING] // ERROR ... ERROR = logger[logging.ERROR] // FATAL ... FATAL = logger[logging.FATAL] ) // Set sets a custom logger for all log levels func Set(l logging.LoggerInterface) { DEBUG = l INFO = l WARNING = l ERROR = l FATAL = l } // SetDebug sets a custom logger for DEBUG level logs func SetDebug(l logging.LoggerInterface) { DEBUG = l } // SetInfo sets a custom logger for INFO level logs func SetInfo(l logging.LoggerInterface) { INFO = l } // SetWarning sets a custom logger for WARNING level logs func SetWarning(l logging.LoggerInterface) { WARNING = l } // SetError sets a custom logger for ERROR level logs func SetError(l logging.LoggerInterface) { ERROR = l } // SetFatal sets a custom logger for FATAL level logs func SetFatal(l logging.LoggerInterface) { FATAL = l } ================================================ FILE: v2/log/log_test.go ================================================ package log_test import ( "testing" "github.com/RichardKnop/machinery/v2/log" ) func TestDefaultLogger(t *testing.T) { log.INFO.Print("should not panic") log.WARNING.Print("should not panic") log.ERROR.Print("should not panic") log.FATAL.Print("should not panic") } ================================================ FILE: v2/package.go ================================================ package machinery ================================================ FILE: v2/retry/fibonacci.go ================================================ package retry // Fibonacci returns successive Fibonacci numbers starting from 1 func Fibonacci() func() int { a, b := 0, 1 return func() int { a, b = b, a+b return a } } // FibonacciNext returns next number in Fibonacci sequence greater than start func FibonacciNext(start int) int { fib := Fibonacci() num := fib() for num <= start { num = fib() } return num } ================================================ FILE: v2/retry/fibonacci_test.go ================================================ package retry_test import ( "testing" "github.com/RichardKnop/machinery/v2/retry" "github.com/stretchr/testify/assert" ) func TestFibonacci(t *testing.T) { fibonacci := retry.Fibonacci() sequence := []int{ fibonacci(), fibonacci(), fibonacci(), fibonacci(), fibonacci(), fibonacci(), } assert.EqualValues(t, sequence, []int{1, 1, 2, 3, 5, 8}) } func TestFibonacciNext(t *testing.T) { assert.Equal(t, 1, retry.FibonacciNext(0)) assert.Equal(t, 2, retry.FibonacciNext(1)) assert.Equal(t, 5, retry.FibonacciNext(3)) assert.Equal(t, 5, retry.FibonacciNext(4)) assert.Equal(t, 8, retry.FibonacciNext(5)) assert.Equal(t, 13, retry.FibonacciNext(8)) } ================================================ FILE: v2/retry/retry.go ================================================ package retry import ( "fmt" "time" "github.com/RichardKnop/machinery/v2/log" ) // Closure - a useful closure we can use when there is a problem // connecting to the broker. It uses Fibonacci sequence to space out retry attempts var Closure = func() func(chan int) { retryIn := 0 fibonacci := Fibonacci() return func(stopChan chan int) { if retryIn > 0 { durationString := fmt.Sprintf("%vs", retryIn) duration, _ := time.ParseDuration(durationString) log.WARNING.Printf("Retrying in %v seconds", retryIn) select { case <-stopChan: break case <-time.After(duration): break } } retryIn = fibonacci() } } ================================================ FILE: v2/server.go ================================================ package machinery import ( "context" "errors" "fmt" "sync" "time" "github.com/google/uuid" "github.com/robfig/cron/v3" "github.com/RichardKnop/machinery/v2/backends/result" "github.com/RichardKnop/machinery/v2/config" "github.com/RichardKnop/machinery/v2/log" "github.com/RichardKnop/machinery/v2/tasks" "github.com/RichardKnop/machinery/v2/tracing" "github.com/RichardKnop/machinery/v2/utils" backendsiface "github.com/RichardKnop/machinery/v2/backends/iface" brokersiface "github.com/RichardKnop/machinery/v2/brokers/iface" lockiface "github.com/RichardKnop/machinery/v2/locks/iface" opentracing "github.com/opentracing/opentracing-go" ) // Server is the main Machinery object and stores all configuration // All the tasks workers process are registered against the server type Server struct { config *config.Config registeredTasks *sync.Map broker brokersiface.Broker backend backendsiface.Backend lock lockiface.Lock scheduler *cron.Cron prePublishHandler func(*tasks.Signature) } // NewServer creates Server instance func NewServer(cnf *config.Config, brokerServer brokersiface.Broker, backendServer backendsiface.Backend, lock lockiface.Lock) *Server { srv := &Server{ config: cnf, registeredTasks: new(sync.Map), broker: brokerServer, backend: backendServer, lock: lock, scheduler: cron.New(), } // Run scheduler job go srv.scheduler.Run() return srv } // NewWorker creates Worker instance func (server *Server) NewWorker(consumerTag string, concurrency int) *Worker { return &Worker{ server: server, ConsumerTag: consumerTag, Concurrency: concurrency, Queue: "", } } // NewCustomQueueWorker creates Worker instance with Custom Queue func (server *Server) NewCustomQueueWorker(consumerTag string, concurrency int, queue string) *Worker { return &Worker{ server: server, ConsumerTag: consumerTag, Concurrency: concurrency, Queue: queue, } } // GetBroker returns broker func (server *Server) GetBroker() brokersiface.Broker { return server.broker } // SetBroker sets broker func (server *Server) SetBroker(broker brokersiface.Broker) { server.broker = broker } // GetBackend returns backend func (server *Server) GetBackend() backendsiface.Backend { return server.backend } // SetBackend sets backend func (server *Server) SetBackend(backend backendsiface.Backend) { server.backend = backend } // GetConfig returns connection object func (server *Server) GetConfig() *config.Config { return server.config } // SetConfig sets config func (server *Server) SetConfig(cnf *config.Config) { server.config = cnf } // SetPreTaskHandler Sets pre publish handler func (server *Server) SetPreTaskHandler(handler func(*tasks.Signature)) { server.prePublishHandler = handler } // RegisterTasks registers all tasks at once func (server *Server) RegisterTasks(namedTaskFuncs map[string]interface{}) error { for _, task := range namedTaskFuncs { if err := tasks.ValidateTask(task); err != nil { return err } } for k, v := range namedTaskFuncs { server.registeredTasks.Store(k, v) } server.broker.SetRegisteredTaskNames(server.GetRegisteredTaskNames()) return nil } // RegisterTask registers a single task func (server *Server) RegisterTask(name string, taskFunc interface{}) error { if err := tasks.ValidateTask(taskFunc); err != nil { return err } server.registeredTasks.Store(name, taskFunc) server.broker.SetRegisteredTaskNames(server.GetRegisteredTaskNames()) return nil } // IsTaskRegistered returns true if the task name is registered with this broker func (server *Server) IsTaskRegistered(name string) bool { _, ok := server.registeredTasks.Load(name) return ok } // GetRegisteredTask returns registered task by name func (server *Server) GetRegisteredTask(name string) (interface{}, error) { taskFunc, ok := server.registeredTasks.Load(name) if !ok { return nil, fmt.Errorf("Task not registered error: %s", name) } return taskFunc, nil } // SendTaskWithContext will inject the trace context in the signature headers before publishing it func (server *Server) SendTaskWithContext(ctx context.Context, signature *tasks.Signature) (*result.AsyncResult, error) { span, _ := opentracing.StartSpanFromContext(ctx, "SendTask", tracing.ProducerOption(), tracing.MachineryTag) defer span.Finish() // tag the span with some info about the signature signature.Headers = tracing.HeadersWithSpan(signature.Headers, span) // Make sure result backend is defined if server.backend == nil { return nil, errors.New("Result backend required") } // Auto generate a UUID if not set already if signature.UUID == "" { taskID := uuid.New().String() signature.UUID = fmt.Sprintf("task_%v", taskID) } // Set initial task state to PENDING if err := server.backend.SetStatePending(signature); err != nil { return nil, fmt.Errorf("Set state pending error: %s", err) } if server.prePublishHandler != nil { server.prePublishHandler(signature) } if err := server.broker.Publish(ctx, signature); err != nil { return nil, fmt.Errorf("Publish message error: %s", err) } return result.NewAsyncResult(signature, server.backend), nil } // SendTask publishes a task to the default queue func (server *Server) SendTask(signature *tasks.Signature) (*result.AsyncResult, error) { return server.SendTaskWithContext(context.Background(), signature) } // SendChainWithContext will inject the trace context in all the signature headers before publishing it func (server *Server) SendChainWithContext(ctx context.Context, chain *tasks.Chain) (*result.ChainAsyncResult, error) { span, _ := opentracing.StartSpanFromContext(ctx, "SendChain", tracing.ProducerOption(), tracing.MachineryTag, tracing.WorkflowChainTag) defer span.Finish() tracing.AnnotateSpanWithChainInfo(span, chain) return server.SendChain(chain) } // SendChain triggers a chain of tasks func (server *Server) SendChain(chain *tasks.Chain) (*result.ChainAsyncResult, error) { _, err := server.SendTask(chain.Tasks[0]) if err != nil { return nil, err } return result.NewChainAsyncResult(chain.Tasks, server.backend), nil } // SendGroupWithContext will inject the trace context in all the signature headers before publishing it func (server *Server) SendGroupWithContext(ctx context.Context, group *tasks.Group, sendConcurrency int) ([]*result.AsyncResult, error) { span, _ := opentracing.StartSpanFromContext(ctx, "SendGroup", tracing.ProducerOption(), tracing.MachineryTag, tracing.WorkflowGroupTag) defer span.Finish() tracing.AnnotateSpanWithGroupInfo(span, group, sendConcurrency) // Make sure result backend is defined if server.backend == nil { return nil, errors.New("Result backend required") } asyncResults := make([]*result.AsyncResult, len(group.Tasks)) var wg sync.WaitGroup wg.Add(len(group.Tasks)) errorsChan := make(chan error, len(group.Tasks)*2) // Init group server.backend.InitGroup(group.GroupUUID, group.GetUUIDs()) // Init the tasks Pending state first for _, signature := range group.Tasks { if err := server.backend.SetStatePending(signature); err != nil { errorsChan <- err continue } } pool := make(chan struct{}, sendConcurrency) go func() { for i := 0; i < sendConcurrency; i++ { pool <- struct{}{} } }() for i, signature := range group.Tasks { if sendConcurrency > 0 { <-pool } go func(s *tasks.Signature, index int) { defer wg.Done() // Publish task err := server.broker.Publish(ctx, s) if sendConcurrency > 0 { pool <- struct{}{} } if err != nil { errorsChan <- fmt.Errorf("Publish message error: %s", err) return } asyncResults[index] = result.NewAsyncResult(s, server.backend) }(signature, i) } done := make(chan int) go func() { wg.Wait() done <- 1 }() select { case err := <-errorsChan: return asyncResults, err case <-done: return asyncResults, nil } } // SendGroup triggers a group of parallel tasks func (server *Server) SendGroup(group *tasks.Group, sendConcurrency int) ([]*result.AsyncResult, error) { return server.SendGroupWithContext(context.Background(), group, sendConcurrency) } // SendChordWithContext will inject the trace context in all the signature headers before publishing it func (server *Server) SendChordWithContext(ctx context.Context, chord *tasks.Chord, sendConcurrency int) (*result.ChordAsyncResult, error) { span, _ := opentracing.StartSpanFromContext(ctx, "SendChord", tracing.ProducerOption(), tracing.MachineryTag, tracing.WorkflowChordTag) defer span.Finish() tracing.AnnotateSpanWithChordInfo(span, chord, sendConcurrency) _, err := server.SendGroupWithContext(ctx, chord.Group, sendConcurrency) if err != nil { return nil, err } return result.NewChordAsyncResult( chord.Group.Tasks, chord.Callback, server.backend, ), nil } // SendChord triggers a group of parallel tasks with a callback func (server *Server) SendChord(chord *tasks.Chord, sendConcurrency int) (*result.ChordAsyncResult, error) { return server.SendChordWithContext(context.Background(), chord, sendConcurrency) } // GetRegisteredTaskNames returns slice of registered task names func (server *Server) GetRegisteredTaskNames() []string { taskNames := make([]string, 0) server.registeredTasks.Range(func(key, value interface{}) bool { taskNames = append(taskNames, key.(string)) return true }) return taskNames } // RegisterPeriodicTask register a periodic task which will be triggered periodically func (server *Server) RegisterPeriodicTask(spec, name string, signature *tasks.Signature) error { //check spec schedule, err := cron.ParseStandard(spec) if err != nil { return err } f := func() { //get lock err := server.lock.LockWithRetries(utils.GetLockName(name, spec), schedule.Next(time.Now()).UnixNano()-1) if err != nil { return } //send task _, err = server.SendTask(tasks.CopySignature(signature)) if err != nil { log.ERROR.Printf("periodic task failed. task name is: %s. error is %s", name, err.Error()) } } _, err = server.scheduler.AddFunc(spec, f) return err } // RegisterPeriodicChain register a periodic chain which will be triggered periodically func (server *Server) RegisterPeriodicChain(spec, name string, signatures ...*tasks.Signature) error { //check spec schedule, err := cron.ParseStandard(spec) if err != nil { return err } f := func() { // new chain chain, _ := tasks.NewChain(tasks.CopySignatures(signatures...)...) //get lock err := server.lock.LockWithRetries(utils.GetLockName(name, spec), schedule.Next(time.Now()).UnixNano()-1) if err != nil { return } //send task _, err = server.SendChain(chain) if err != nil { log.ERROR.Printf("periodic task failed. task name is: %s. error is %s", name, err.Error()) } } _, err = server.scheduler.AddFunc(spec, f) return err } // RegisterPeriodicGroup register a periodic group which will be triggered periodically func (server *Server) RegisterPeriodicGroup(spec, name string, sendConcurrency int, signatures ...*tasks.Signature) error { //check spec schedule, err := cron.ParseStandard(spec) if err != nil { return err } f := func() { // new group group, _ := tasks.NewGroup(tasks.CopySignatures(signatures...)...) //get lock err := server.lock.LockWithRetries(utils.GetLockName(name, spec), schedule.Next(time.Now()).UnixNano()-1) if err != nil { return } //send task _, err = server.SendGroup(group, sendConcurrency) if err != nil { log.ERROR.Printf("periodic task failed. task name is: %s. error is %s", name, err.Error()) } } _, err = server.scheduler.AddFunc(spec, f) return err } // RegisterPeriodicChord register a periodic chord which will be triggered periodically func (server *Server) RegisterPeriodicChord(spec, name string, sendConcurrency int, callback *tasks.Signature, signatures ...*tasks.Signature) error { //check spec schedule, err := cron.ParseStandard(spec) if err != nil { return err } f := func() { // new chord group, _ := tasks.NewGroup(tasks.CopySignatures(signatures...)...) chord, _ := tasks.NewChord(group, tasks.CopySignature(callback)) //get lock err := server.lock.LockWithRetries(utils.GetLockName(name, spec), schedule.Next(time.Now()).UnixNano()-1) if err != nil { return } //send task _, err = server.SendChord(chord, sendConcurrency) if err != nil { log.ERROR.Printf("periodic task failed. task name is: %s. error is %s", name, err.Error()) } } _, err = server.scheduler.AddFunc(spec, f) return err } ================================================ FILE: v2/server_test.go ================================================ package machinery_test import ( "testing" "github.com/stretchr/testify/assert" "github.com/RichardKnop/machinery/v2" "github.com/RichardKnop/machinery/v2/config" backend "github.com/RichardKnop/machinery/v2/backends/eager" broker "github.com/RichardKnop/machinery/v2/brokers/eager" lock "github.com/RichardKnop/machinery/v2/locks/eager" ) func TestRegisterTasks(t *testing.T) { t.Parallel() server := getTestServer(t) err := server.RegisterTasks(map[string]interface{}{ "test_task": func() error { return nil }, }) assert.NoError(t, err) _, err = server.GetRegisteredTask("test_task") assert.NoError(t, err, "test_task is not registered but it should be") } func TestRegisterTask(t *testing.T) { t.Parallel() server := getTestServer(t) err := server.RegisterTask("test_task", func() error { return nil }) assert.NoError(t, err) _, err = server.GetRegisteredTask("test_task") assert.NoError(t, err, "test_task is not registered but it should be") } func TestGetRegisteredTask(t *testing.T) { t.Parallel() server := getTestServer(t) _, err := server.GetRegisteredTask("test_task") assert.Error(t, err, "test_task is registered but it should not be") } func TestGetRegisteredTaskNames(t *testing.T) { t.Parallel() server := getTestServer(t) taskName := "test_task" err := server.RegisterTask(taskName, func() error { return nil }) assert.NoError(t, err) taskNames := server.GetRegisteredTaskNames() assert.Equal(t, 1, len(taskNames)) assert.Equal(t, taskName, taskNames[0]) } func TestNewWorker(t *testing.T) { t.Parallel() server := getTestServer(t) server.NewWorker("test_worker", 1) assert.NoError(t, nil) } func TestNewCustomQueueWorker(t *testing.T) { t.Parallel() server := getTestServer(t) server.NewCustomQueueWorker("test_customqueueworker", 1, "test_queue") assert.NoError(t, nil) } func getTestServer(t *testing.T) *machinery.Server { return machinery.NewServer(&config.Config{}, broker.New(), backend.New(), lock.New()) } ================================================ FILE: v2/tasks/errors.go ================================================ package tasks import ( "fmt" "time" ) // ErrRetryTaskLater ... type ErrRetryTaskLater struct { name, msg string retryIn time.Duration } // RetryIn returns time.Duration from now when task should be retried func (e ErrRetryTaskLater) RetryIn() time.Duration { return e.retryIn } // Error implements the error interface func (e ErrRetryTaskLater) Error() string { return fmt.Sprintf("Task error: %s Will retry in: %s", e.msg, e.retryIn) } // NewErrRetryTaskLater returns new ErrRetryTaskLater instance func NewErrRetryTaskLater(msg string, retryIn time.Duration) ErrRetryTaskLater { return ErrRetryTaskLater{msg: msg, retryIn: retryIn} } // Retriable is interface that retriable errors should implement type Retriable interface { RetryIn() time.Duration } ================================================ FILE: v2/tasks/reflect.go ================================================ package tasks import ( "context" "encoding/base64" "encoding/json" "fmt" "reflect" "strconv" "strings" ) var ( typesMap = map[string]reflect.Type{ // base types "bool": reflect.TypeOf(true), "int": reflect.TypeOf(int(1)), "int8": reflect.TypeOf(int8(1)), "int16": reflect.TypeOf(int16(1)), "int32": reflect.TypeOf(int32(1)), "int64": reflect.TypeOf(int64(1)), "uint": reflect.TypeOf(uint(1)), "uint8": reflect.TypeOf(uint8(1)), "uint16": reflect.TypeOf(uint16(1)), "uint32": reflect.TypeOf(uint32(1)), "uint64": reflect.TypeOf(uint64(1)), "float32": reflect.TypeOf(float32(0.5)), "float64": reflect.TypeOf(float64(0.5)), "string": reflect.TypeOf(string("")), // slices "[]bool": reflect.TypeOf(make([]bool, 0)), "[]int": reflect.TypeOf(make([]int, 0)), "[]int8": reflect.TypeOf(make([]int8, 0)), "[]int16": reflect.TypeOf(make([]int16, 0)), "[]int32": reflect.TypeOf(make([]int32, 0)), "[]int64": reflect.TypeOf(make([]int64, 0)), "[]uint": reflect.TypeOf(make([]uint, 0)), "[]uint8": reflect.TypeOf(make([]uint8, 0)), "[]uint16": reflect.TypeOf(make([]uint16, 0)), "[]uint32": reflect.TypeOf(make([]uint32, 0)), "[]uint64": reflect.TypeOf(make([]uint64, 0)), "[]float32": reflect.TypeOf(make([]float32, 0)), "[]float64": reflect.TypeOf(make([]float64, 0)), "[]byte": reflect.TypeOf(make([]byte, 0)), "[]string": reflect.TypeOf([]string{""}), } ctxType = reflect.TypeOf((*context.Context)(nil)).Elem() typeConversionError = func(argValue interface{}, argTypeStr string) error { return fmt.Errorf("%v is not %v", argValue, argTypeStr) } ) // ErrUnsupportedType ... type ErrUnsupportedType struct { valueType string } // NewErrUnsupportedType returns new ErrUnsupportedType func NewErrUnsupportedType(valueType string) ErrUnsupportedType { return ErrUnsupportedType{valueType} } // Error method so we implement the error interface func (e ErrUnsupportedType) Error() string { return fmt.Sprintf("%v is not one of supported types", e.valueType) } // ReflectValue converts interface{} to reflect.Value based on string type func ReflectValue(valueType string, value interface{}) (reflect.Value, error) { if strings.HasPrefix(valueType, "[]") { return reflectValues(valueType, value) } return reflectValue(valueType, value) } // reflectValue converts interface{} to reflect.Value based on string type // representing a base type (not a slice) func reflectValue(valueType string, value interface{}) (reflect.Value, error) { theType, ok := typesMap[valueType] if !ok { return reflect.Value{}, NewErrUnsupportedType(valueType) } theValue := reflect.New(theType) // Booleans if theType.String() == "bool" { boolValue, err := getBoolValue(theType.String(), value) if err != nil { return reflect.Value{}, err } theValue.Elem().SetBool(boolValue) return theValue.Elem(), nil } // Integers if strings.HasPrefix(theType.String(), "int") { intValue, err := getIntValue(theType.String(), value) if err != nil { return reflect.Value{}, err } theValue.Elem().SetInt(intValue) return theValue.Elem(), err } // Unsigned integers if strings.HasPrefix(theType.String(), "uint") { uintValue, err := getUintValue(theType.String(), value) if err != nil { return reflect.Value{}, err } theValue.Elem().SetUint(uintValue) return theValue.Elem(), err } // Floating point numbers if strings.HasPrefix(theType.String(), "float") { floatValue, err := getFloatValue(theType.String(), value) if err != nil { return reflect.Value{}, err } theValue.Elem().SetFloat(floatValue) return theValue.Elem(), err } // Strings if theType.String() == "string" { stringValue, err := getStringValue(theType.String(), value) if err != nil { return reflect.Value{}, err } theValue.Elem().SetString(stringValue) return theValue.Elem(), nil } return reflect.Value{}, NewErrUnsupportedType(valueType) } // reflectValues converts interface{} to reflect.Value based on string type // representing a slice of values func reflectValues(valueType string, value interface{}) (reflect.Value, error) { theType, ok := typesMap[valueType] if !ok { return reflect.Value{}, NewErrUnsupportedType(valueType) } // For NULL we return an empty slice if value == nil { return reflect.MakeSlice(theType, 0, 0), nil } var theValue reflect.Value // Booleans if theType.String() == "[]bool" { bools := reflect.ValueOf(value) theValue = reflect.MakeSlice(theType, bools.Len(), bools.Len()) for i := 0; i < bools.Len(); i++ { boolValue, err := getBoolValue(strings.Split(theType.String(), "[]")[1], bools.Index(i).Interface()) if err != nil { return reflect.Value{}, err } theValue.Index(i).SetBool(boolValue) } return theValue, nil } // Integers if strings.HasPrefix(theType.String(), "[]int") { ints := reflect.ValueOf(value) theValue = reflect.MakeSlice(theType, ints.Len(), ints.Len()) for i := 0; i < ints.Len(); i++ { intValue, err := getIntValue(strings.Split(theType.String(), "[]")[1], ints.Index(i).Interface()) if err != nil { return reflect.Value{}, err } theValue.Index(i).SetInt(intValue) } return theValue, nil } // Unsigned integers if strings.HasPrefix(theType.String(), "[]uint") || theType.String() == "[]byte" { // Decode the base64 string if the value type is []uint8 or it's alias []byte // See: https://golang.org/pkg/encoding/json/#Marshal // > Array and slice values encode as JSON arrays, except that []byte encodes as a base64-encoded string if reflect.TypeOf(value).String() == "string" { output, err := base64.StdEncoding.DecodeString(value.(string)) if err != nil { return reflect.Value{}, err } value = output } uints := reflect.ValueOf(value) theValue = reflect.MakeSlice(theType, uints.Len(), uints.Len()) for i := 0; i < uints.Len(); i++ { uintValue, err := getUintValue(strings.Split(theType.String(), "[]")[1], uints.Index(i).Interface()) if err != nil { return reflect.Value{}, err } theValue.Index(i).SetUint(uintValue) } return theValue, nil } // Floating point numbers if strings.HasPrefix(theType.String(), "[]float") { floats := reflect.ValueOf(value) theValue = reflect.MakeSlice(theType, floats.Len(), floats.Len()) for i := 0; i < floats.Len(); i++ { floatValue, err := getFloatValue(strings.Split(theType.String(), "[]")[1], floats.Index(i).Interface()) if err != nil { return reflect.Value{}, err } theValue.Index(i).SetFloat(floatValue) } return theValue, nil } // Strings if theType.String() == "[]string" { strs := reflect.ValueOf(value) theValue = reflect.MakeSlice(theType, strs.Len(), strs.Len()) for i := 0; i < strs.Len(); i++ { strValue, err := getStringValue(strings.Split(theType.String(), "[]")[1], strs.Index(i).Interface()) if err != nil { return reflect.Value{}, err } theValue.Index(i).SetString(strValue) } return theValue, nil } return reflect.Value{}, NewErrUnsupportedType(valueType) } func getBoolValue(theType string, value interface{}) (bool, error) { b, ok := value.(bool) if !ok { return false, typeConversionError(value, typesMap[theType].String()) } return b, nil } func getIntValue(theType string, value interface{}) (int64, error) { // We use https://golang.org/pkg/encoding/json/#Decoder.UseNumber when unmarshaling signatures. // This is because JSON only supports 64-bit floating point numbers and we could lose precision // when converting from float64 to signed integer if strings.HasPrefix(fmt.Sprintf("%T", value), "json.Number") { n, ok := value.(json.Number) if !ok { return 0, typeConversionError(value, typesMap[theType].String()) } return n.Int64() } n, ok := value.(int64) if !ok { return 0, typeConversionError(value, typesMap[theType].String()) } return n, nil } func getUintValue(theType string, value interface{}) (uint64, error) { // Losing precision only happens in receiving a JSON number from a language like js, // and receiving a large uint number from golang or python could cause json.Number.Int64 be turned into a panic. // So we use strconv.ParseUint to correctly parse a uint value. if strings.HasPrefix(fmt.Sprintf("%T", value), "json.Number") { n, ok := value.(json.Number) if !ok { return 0, typeConversionError(value, typesMap[theType].String()) } uintVal, err := strconv.ParseUint(string(n), 10, 64) if err != nil { return 0, err } return uintVal, nil } var n uint64 switch value := value.(type) { case uint64: n = value case uint8: n = uint64(value) default: return 0, typeConversionError(value, typesMap[theType].String()) } return n, nil } func getFloatValue(theType string, value interface{}) (float64, error) { // We use https://golang.org/pkg/encoding/json/#Decoder.UseNumber when unmarshaling signatures. // This is because JSON only supports 64-bit floating point numbers and we could lose precision if strings.HasPrefix(fmt.Sprintf("%T", value), "json.Number") { n, ok := value.(json.Number) if !ok { return 0, typeConversionError(value, typesMap[theType].String()) } return n.Float64() } f, ok := value.(float64) if !ok { return 0, typeConversionError(value, typesMap[theType].String()) } return f, nil } func getStringValue(theType string, value interface{}) (string, error) { s, ok := value.(string) if !ok { return "", typeConversionError(value, typesMap[theType].String()) } return s, nil } // IsContextType checks to see if the type is a context.Context func IsContextType(t reflect.Type) bool { return t == ctxType } ================================================ FILE: v2/tasks/reflect_test.go ================================================ package tasks_test import ( "encoding/json" "reflect" "testing" "github.com/RichardKnop/machinery/v2/tasks" ) var ( reflectValuesTestCases = []struct { name string value interface{} expectedType string expectedValue interface{} }{ // basic types { name: "bool", value: false, expectedType: "bool", }, { name: "int", value: json.Number("123"), expectedType: "int", expectedValue: int(123), }, { name: "int8", value: json.Number("123"), expectedType: "int8", expectedValue: int8(123), }, { name: "int16", value: json.Number("123"), expectedType: "int16", expectedValue: int16(123), }, { name: "int32", value: json.Number("123"), expectedType: "int32", expectedValue: int32(123), }, { name: "int64", value: json.Number("185135722552891243"), expectedType: "int64", expectedValue: int64(185135722552891243), }, { name: "uint", value: json.Number("123"), expectedType: "uint", expectedValue: uint(123), }, { name: "uint8", value: json.Number("123"), expectedType: "uint8", expectedValue: uint8(123), }, { name: "uint16", value: json.Number("123"), expectedType: "uint16", expectedValue: uint16(123), }, { name: "uint32", value: json.Number("123"), expectedType: "uint32", expectedValue: uint32(123), }, { name: "uint64", value: json.Number("185135722552891243"), expectedType: "uint64", expectedValue: uint64(185135722552891243), }, { name: "uint64", value: json.Number("9223372036854775808"), // math.MaxInt64 + 1 expectedType: "uint64", expectedValue: uint64(9223372036854775808), }, { name: "float32", value: json.Number("0.5"), expectedType: "float32", expectedValue: float32(0.5), }, { name: "float64", value: json.Number("0.5"), expectedType: "float64", expectedValue: float64(0.5), }, { name: "string", value: "123", expectedType: "string", expectedValue: "123", }, // slices { name: "[]bool", value: []interface{}{false, true}, expectedType: "[]bool", expectedValue: []bool{false, true}, }, { name: "[]int", value: []interface{}{json.Number("1"), json.Number("2")}, expectedType: "[]int", expectedValue: []int{1, 2}, }, { name: "[]int8", value: []interface{}{json.Number("1"), json.Number("2")}, expectedType: "[]int8", expectedValue: []int8{1, 2}, }, { name: "[]int16", value: []interface{}{json.Number("1"), json.Number("2")}, expectedType: "[]int16", expectedValue: []int16{1, 2}, }, { name: "[]int32", value: []interface{}{json.Number("1"), json.Number("2")}, expectedType: "[]int32", expectedValue: []int32{1, 2}, }, { name: "[]int64", value: []interface{}{json.Number("1"), json.Number("2")}, expectedType: "[]int64", expectedValue: []int64{1, 2}, }, { name: "[]uint", value: []interface{}{json.Number("1"), json.Number("2")}, expectedType: "[]uint", expectedValue: []uint{1, 2}, }, { name: "[]uint8", value: []interface{}{json.Number("1"), json.Number("2")}, expectedType: "[]uint8", expectedValue: []uint8{1, 2}, }, { name: "[]uint16", value: []interface{}{json.Number("1"), json.Number("2")}, expectedType: "[]uint16", expectedValue: []uint16{1, 2}, }, { name: "[]uint32", value: []interface{}{json.Number("1"), json.Number("2")}, expectedType: "[]uint32", expectedValue: []uint32{1, 2}, }, { name: "[]uint64", value: []interface{}{json.Number("1"), json.Number("2")}, expectedType: "[]uint64", expectedValue: []uint64{1, 2}, }, { name: "[]float32", value: []interface{}{json.Number("0.5"), json.Number("1.28")}, expectedType: "[]float32", expectedValue: []float32{0.5, 1.28}, }, { name: "[]float64", value: []interface{}{json.Number("0.5"), json.Number("1.28")}, expectedType: "[]float64", expectedValue: []float64{0.5, 1.28}, }, { name: "[]string", value: []interface{}{"foo", "bar"}, expectedType: "[]string", expectedValue: []string{"foo", "bar"}, }, // empty slices from NULL { name: "[]bool", value: nil, expectedType: "[]bool", expectedValue: []bool{}, }, { name: "[]int64", value: nil, expectedType: "[]int64", expectedValue: []int64{}, }, { name: "[]uint64", value: nil, expectedType: "[]uint64", expectedValue: []uint64{}, }, { name: "[]float64", value: nil, expectedType: "[]float64", expectedValue: []float64{}, }, { name: "[]string", value: nil, expectedType: "[]string", expectedValue: []string{}, }, } ) func TestReflectValue(t *testing.T) { t.Parallel() for _, tc := range reflectValuesTestCases { tc := tc // capture range variable t.Run(tc.name, func(t *testing.T) { t.Parallel() value, err := tasks.ReflectValue(tc.name, tc.value) if err != nil { t.Error(err) } if value.Type().String() != tc.expectedType { t.Errorf("type is %v, want %s", value.Type().String(), tc.expectedType) } if tc.expectedValue != nil { if !reflect.DeepEqual(value.Interface(), tc.expectedValue) { t.Errorf("value is %v, want %v", value.Interface(), tc.expectedValue) } } }) } } ================================================ FILE: v2/tasks/result.go ================================================ package tasks import ( "fmt" "reflect" "strings" ) // TaskResult represents an actual return value of a processed task type TaskResult struct { Type string `bson:"type"` Value interface{} `bson:"value"` } // ReflectTaskResults ... func ReflectTaskResults(taskResults []*TaskResult) ([]reflect.Value, error) { resultValues := make([]reflect.Value, len(taskResults)) for i, taskResult := range taskResults { resultValue, err := ReflectValue(taskResult.Type, taskResult.Value) if err != nil { return nil, err } resultValues[i] = resultValue } return resultValues, nil } // HumanReadableResults ... func HumanReadableResults(results []reflect.Value) string { if len(results) == 1 { return fmt.Sprintf("%v", results[0].Interface()) } readableResults := make([]string, len(results)) for i := 0; i < len(results); i++ { readableResults[i] = fmt.Sprintf("%v", results[i].Interface()) } return fmt.Sprintf("[%s]", strings.Join(readableResults, ", ")) } ================================================ FILE: v2/tasks/result_test.go ================================================ package tasks_test import ( "testing" "github.com/RichardKnop/machinery/v2/tasks" "github.com/stretchr/testify/assert" ) func TestReflectTaskResults(t *testing.T) { t.Parallel() taskResults := []*tasks.TaskResult{ { Type: "[]string", Value: []string{"f", "o", "o"}, }, } results, err := tasks.ReflectTaskResults(taskResults) if assert.NoError(t, err) { assert.Equal(t, 1, len(results)) assert.Equal(t, 3, results[0].Len()) assert.Equal(t, "f", results[0].Index(0).String()) assert.Equal(t, "o", results[0].Index(1).String()) assert.Equal(t, "o", results[0].Index(2).String()) } } ================================================ FILE: v2/tasks/signature.go ================================================ package tasks import ( "fmt" "github.com/RichardKnop/machinery/v2/utils" "time" "github.com/google/uuid" ) // Arg represents a single argument passed to invocation fo a task type Arg struct { Name string `bson:"name"` Type string `bson:"type"` Value interface{} `bson:"value"` } // Headers represents the headers which should be used to direct the task type Headers map[string]interface{} // Set on Headers implements opentracing.TextMapWriter for trace propagation func (h Headers) Set(key, val string) { h[key] = val } // ForeachKey on Headers implements opentracing.TextMapReader for trace propagation. // It is essentially the same as the opentracing.TextMapReader implementation except // for the added casting from interface{} to string. func (h Headers) ForeachKey(handler func(key, val string) error) error { for k, v := range h { // Skip any non string values stringValue, ok := v.(string) if !ok { continue } if err := handler(k, stringValue); err != nil { return err } } return nil } // Signature represents a single task invocation type Signature struct { UUID string Name string RoutingKey string ETA *time.Time GroupUUID string GroupTaskCount int Args []Arg Headers Headers Priority uint8 Immutable bool RetryCount int RetryTimeout int OnSuccess []*Signature OnError []*Signature ChordCallback *Signature //MessageGroupId for Broker, e.g. SQS BrokerMessageGroupId string //ReceiptHandle of SQS Message SQSReceiptHandle string // StopTaskDeletionOnError used with sqs when we want to send failed messages to dlq, // and don't want machinery to delete from source queue StopTaskDeletionOnError bool // IgnoreWhenTaskNotRegistered auto removes the request when there is no handeler available // When this is true a task with no handler will be ignored and not placed back in the queue IgnoreWhenTaskNotRegistered bool } // NewSignature creates a new task signature func NewSignature(name string, args []Arg) (*Signature, error) { signatureID := uuid.New().String() return &Signature{ UUID: fmt.Sprintf("task_%v", signatureID), Name: name, Args: args, }, nil } func CopySignatures(signatures ...*Signature) []*Signature { var sigs = make([]*Signature, len(signatures)) for index, signature := range signatures { sigs[index] = CopySignature(signature) } return sigs } func CopySignature(signature *Signature) *Signature { var sig = new(Signature) _ = utils.DeepCopy(sig, signature) return sig } ================================================ FILE: v2/tasks/state.go ================================================ package tasks import "time" const ( // StatePending - initial state of a task StatePending = "PENDING" // StateReceived - when task is received by a worker StateReceived = "RECEIVED" // StateStarted - when the worker starts processing the task StateStarted = "STARTED" // StateRetry - when failed task has been scheduled for retry StateRetry = "RETRY" // StateSuccess - when the task is processed successfully StateSuccess = "SUCCESS" // StateFailure - when processing of the task fails StateFailure = "FAILURE" ) // TaskState represents a state of a task type TaskState struct { TaskUUID string `bson:"_id"` TaskName string `bson:"task_name"` State string `bson:"state"` Results []*TaskResult `bson:"results"` Error string `bson:"error"` CreatedAt time.Time `bson:"created_at"` TTL int64 `bson:"ttl,omitempty"` } // GroupMeta stores useful metadata about tasks within the same group // E.g. UUIDs of all tasks which are used in order to check if all tasks // completed successfully or not and thus whether to trigger chord callback type GroupMeta struct { GroupUUID string `bson:"_id"` TaskUUIDs []string `bson:"task_uuids"` ChordTriggered bool `bson:"chord_triggered"` Lock bool `bson:"lock"` CreatedAt time.Time `bson:"created_at"` TTL int64 `bson:"ttl,omitempty"` } // NewPendingTaskState ... func NewPendingTaskState(signature *Signature) *TaskState { return &TaskState{ TaskUUID: signature.UUID, TaskName: signature.Name, State: StatePending, CreatedAt: time.Now().UTC(), } } // NewReceivedTaskState ... func NewReceivedTaskState(signature *Signature) *TaskState { return &TaskState{ TaskUUID: signature.UUID, State: StateReceived, } } // NewStartedTaskState ... func NewStartedTaskState(signature *Signature) *TaskState { return &TaskState{ TaskUUID: signature.UUID, State: StateStarted, } } // NewSuccessTaskState ... func NewSuccessTaskState(signature *Signature, results []*TaskResult) *TaskState { return &TaskState{ TaskUUID: signature.UUID, State: StateSuccess, Results: results, } } // NewFailureTaskState ... func NewFailureTaskState(signature *Signature, err string) *TaskState { return &TaskState{ TaskUUID: signature.UUID, State: StateFailure, Error: err, } } // NewRetryTaskState ... func NewRetryTaskState(signature *Signature) *TaskState { return &TaskState{ TaskUUID: signature.UUID, State: StateRetry, } } // IsCompleted returns true if state is SUCCESS or FAILURE, // i.e. the task has finished processing and either succeeded or failed. func (taskState *TaskState) IsCompleted() bool { return taskState.IsSuccess() || taskState.IsFailure() } // IsSuccess returns true if state is SUCCESS func (taskState *TaskState) IsSuccess() bool { return taskState.State == StateSuccess } // IsFailure returns true if state is FAILURE func (taskState *TaskState) IsFailure() bool { return taskState.State == StateFailure } ================================================ FILE: v2/tasks/state_test.go ================================================ package tasks_test import ( "testing" "github.com/RichardKnop/machinery/v2/tasks" "github.com/stretchr/testify/assert" ) func TestTaskStateIsCompleted(t *testing.T) { t.Parallel() taskState := &tasks.TaskState{ TaskUUID: "taskUUID", State: tasks.StatePending, } assert.False(t, taskState.IsCompleted()) taskState.State = tasks.StateReceived assert.False(t, taskState.IsCompleted()) taskState.State = tasks.StateStarted assert.False(t, taskState.IsCompleted()) taskState.State = tasks.StateSuccess assert.True(t, taskState.IsCompleted()) taskState.State = tasks.StateFailure assert.True(t, taskState.IsCompleted()) } ================================================ FILE: v2/tasks/task.go ================================================ package tasks import ( "context" "errors" "fmt" "reflect" "runtime/debug" opentracing "github.com/opentracing/opentracing-go" opentracing_ext "github.com/opentracing/opentracing-go/ext" opentracing_log "github.com/opentracing/opentracing-go/log" "github.com/RichardKnop/machinery/v2/log" ) // ErrTaskPanicked ... var ErrTaskPanicked = errors.New("Invoking task caused a panic") // Task wraps a signature and methods used to reflect task arguments and // return values after invoking the task type Task struct { TaskFunc reflect.Value UseContext bool Context context.Context Args []reflect.Value } type signatureCtxType struct{} var signatureCtx signatureCtxType // SignatureFromContext gets the signature from the context func SignatureFromContext(ctx context.Context) *Signature { if ctx == nil { return nil } v := ctx.Value(signatureCtx) if v == nil { return nil } signature, _ := v.(*Signature) return signature } // NewWithSignature is the same as New but injects the signature func NewWithSignature(taskFunc interface{}, signature *Signature) (*Task, error) { args := signature.Args ctx := context.Background() ctx = context.WithValue(ctx, signatureCtx, signature) task := &Task{ TaskFunc: reflect.ValueOf(taskFunc), Context: ctx, } taskFuncType := reflect.TypeOf(taskFunc) if taskFuncType.NumIn() > 0 { arg0Type := taskFuncType.In(0) if IsContextType(arg0Type) { task.UseContext = true } } if err := task.ReflectArgs(args); err != nil { return nil, fmt.Errorf("Reflect task args error: %s", err) } return task, nil } // New tries to use reflection to convert the function and arguments // into a reflect.Value and prepare it for invocation func New(taskFunc interface{}, args []Arg) (*Task, error) { task := &Task{ TaskFunc: reflect.ValueOf(taskFunc), Context: context.Background(), } taskFuncType := reflect.TypeOf(taskFunc) if taskFuncType.NumIn() > 0 { arg0Type := taskFuncType.In(0) if IsContextType(arg0Type) { task.UseContext = true } } if err := task.ReflectArgs(args); err != nil { return nil, fmt.Errorf("Reflect task args error: %s", err) } return task, nil } // Call attempts to call the task with the supplied arguments. // // `err` is set in the return value in two cases: // 1. The reflected function invocation panics (e.g. due to a mismatched // argument list). // 2. The task func itself returns a non-nil error. func (t *Task) Call() (taskResults []*TaskResult, err error) { // retrieve the span from the task's context and finish it as soon as this function returns if span := opentracing.SpanFromContext(t.Context); span != nil { defer span.Finish() } defer func() { // Recover from panic and set err. if e := recover(); e != nil { switch e := e.(type) { default: err = ErrTaskPanicked case error: err = e case string: err = errors.New(e) } // mark the span as failed and dump the error and stack trace to the span if span := opentracing.SpanFromContext(t.Context); span != nil { opentracing_ext.Error.Set(span, true) span.LogFields( opentracing_log.Error(err), opentracing_log.Object("stack", string(debug.Stack())), ) } // Print stack trace log.ERROR.Printf("%s", debug.Stack()) } }() args := t.Args if t.UseContext { ctxValue := reflect.ValueOf(t.Context) args = append([]reflect.Value{ctxValue}, args...) } // Invoke the task results := t.TaskFunc.Call(args) // Task must return at least a value if len(results) == 0 { return nil, ErrTaskReturnsNoValue } // Last returned value lastResult := results[len(results)-1] // If the last returned value is not nil, it has to be of error type, if that // is not the case, return error message, otherwise propagate the task error // to the caller if !lastResult.IsNil() { // If the result implements Retriable interface, return instance of Retriable retriableErrorInterface := reflect.TypeOf((*Retriable)(nil)).Elem() if lastResult.Type().Implements(retriableErrorInterface) { return nil, lastResult.Interface().(ErrRetryTaskLater) } // Otherwise, check that the result implements the standard error interface, // if not, return ErrLastReturnValueMustBeError error errorInterface := reflect.TypeOf((*error)(nil)).Elem() if !lastResult.Type().Implements(errorInterface) { return nil, ErrLastReturnValueMustBeError } // Return the standard error return nil, lastResult.Interface().(error) } // Convert reflect values to task results taskResults = make([]*TaskResult, len(results)-1) for i := 0; i < len(results)-1; i++ { val := results[i].Interface() typeStr := reflect.TypeOf(val).String() taskResults[i] = &TaskResult{ Type: typeStr, Value: val, } } return taskResults, err } // ReflectArgs converts []TaskArg to []reflect.Value func (t *Task) ReflectArgs(args []Arg) error { argValues := make([]reflect.Value, len(args)) for i, arg := range args { argValue, err := ReflectValue(arg.Type, arg.Value) if err != nil { return err } argValues[i] = argValue } t.Args = argValues return nil } ================================================ FILE: v2/tasks/task_test.go ================================================ package tasks_test import ( "context" "errors" "math" "testing" "time" "github.com/RichardKnop/machinery/v2/tasks" "github.com/stretchr/testify/assert" ) func TestTaskCallErrorTest(t *testing.T) { t.Parallel() // Create test task that returns tasks.ErrRetryTaskLater error retriable := func() error { return tasks.NewErrRetryTaskLater("some error", 4*time.Hour) } task, err := tasks.New(retriable, []tasks.Arg{}) assert.NoError(t, err) // Invoke TryCall and validate that returned error can be cast to tasks.ErrRetryTaskLater results, err := task.Call() assert.Nil(t, results) assert.NotNil(t, err) _, ok := interface{}(err).(tasks.ErrRetryTaskLater) assert.True(t, ok, "Error should be castable to tasks.ErrRetryTaskLater") // Create test task that returns a standard error standard := func() error { return errors.New("some error") } task, err = tasks.New(standard, []tasks.Arg{}) assert.NoError(t, err) // Invoke TryCall and validate that returned error is standard results, err = task.Call() assert.Nil(t, results) assert.NotNil(t, err) assert.Equal(t, "some error", err.Error()) } func TestTaskReflectArgs(t *testing.T) { t.Parallel() task := new(tasks.Task) args := []tasks.Arg{ { Type: "[]int64", Value: []int64{1, 2}, }, } err := task.ReflectArgs(args) assert.NoError(t, err) assert.Equal(t, 1, len(task.Args)) assert.Equal(t, "[]int64", task.Args[0].Type().String()) } func TestTaskCallInvalidArgRobustnessError(t *testing.T) { t.Parallel() // Create a test task function f := func(x int) error { return nil } // Construct an invalid argument list and reflect it args := []tasks.Arg{ {Type: "bool", Value: true}, } task, err := tasks.New(f, args) assert.NoError(t, err) // Invoke TryCall and validate error handling results, err := task.Call() assert.Equal(t, "reflect: Call using bool as type int", err.Error()) assert.Nil(t, results) } func TestTaskCallInterfaceValuedResult(t *testing.T) { t.Parallel() // Create a test task function f := func() (interface{}, error) { return math.Pi, nil } task, err := tasks.New(f, []tasks.Arg{}) assert.NoError(t, err) taskResults, err := task.Call() assert.NoError(t, err) assert.Equal(t, "float64", taskResults[0].Type) assert.Equal(t, math.Pi, taskResults[0].Value) } func TestTaskCallWithContext(t *testing.T) { t.Parallel() f := func(c context.Context) (interface{}, error) { assert.NotNil(t, c) assert.Nil(t, tasks.SignatureFromContext(c)) return math.Pi, nil } task, err := tasks.New(f, []tasks.Arg{}) assert.NoError(t, err) taskResults, err := task.Call() assert.NoError(t, err) assert.Equal(t, "float64", taskResults[0].Type) assert.Equal(t, math.Pi, taskResults[0].Value) } func TestTaskCallWithSignatureInContext(t *testing.T) { t.Parallel() f := func(c context.Context) (interface{}, error) { assert.NotNil(t, c) signature := tasks.SignatureFromContext(c) assert.NotNil(t, signature) assert.Equal(t, "foo", signature.Name) return math.Pi, nil } signature, err := tasks.NewSignature("foo", []tasks.Arg{}) assert.NoError(t, err) task, err := tasks.NewWithSignature(f, signature) assert.NoError(t, err) taskResults, err := task.Call() assert.NoError(t, err) assert.Equal(t, "float64", taskResults[0].Type) assert.Equal(t, math.Pi, taskResults[0].Value) } ================================================ FILE: v2/tasks/validate.go ================================================ package tasks import ( "errors" "reflect" ) var ( // ErrTaskMustBeFunc ... ErrTaskMustBeFunc = errors.New("Task must be a func type") // ErrTaskReturnsNoValue ... ErrTaskReturnsNoValue = errors.New("Task must return at least a single value") // ErrLastReturnValueMustBeError .. ErrLastReturnValueMustBeError = errors.New("Last return value of a task must be error") ) // ValidateTask validates task function using reflection and makes sure // it has a proper signature. Functions used as tasks must return at least a // single value and the last return type must be error func ValidateTask(task interface{}) error { v := reflect.ValueOf(task) t := v.Type() // Task must be a function if t.Kind() != reflect.Func { return ErrTaskMustBeFunc } // Task must return at least a single value if t.NumOut() < 1 { return ErrTaskReturnsNoValue } // Last return value must be error lastReturnType := t.Out(t.NumOut() - 1) errorInterface := reflect.TypeOf((*error)(nil)).Elem() if !lastReturnType.Implements(errorInterface) { return ErrLastReturnValueMustBeError } return nil } ================================================ FILE: v2/tasks/validate_test.go ================================================ package tasks_test import ( "testing" "github.com/RichardKnop/machinery/v2/tasks" "github.com/stretchr/testify/assert" ) func TestValidateTask(t *testing.T) { t.Parallel() type someStruct struct{} var ( taskOfWrongType = new(someStruct) taskWithoutReturnValue = func() {} taskWithoutErrorAsLastReturnValue = func() int { return 0 } validTask = func(arg string) error { return nil } ) err := tasks.ValidateTask(taskOfWrongType) assert.Equal(t, tasks.ErrTaskMustBeFunc, err) err = tasks.ValidateTask(taskWithoutReturnValue) assert.Equal(t, tasks.ErrTaskReturnsNoValue, err) err = tasks.ValidateTask(taskWithoutErrorAsLastReturnValue) assert.Equal(t, tasks.ErrLastReturnValueMustBeError, err) err = tasks.ValidateTask(validTask) assert.NoError(t, err) } ================================================ FILE: v2/tasks/workflow.go ================================================ package tasks import ( "fmt" "github.com/google/uuid" ) // Chain creates a chain of tasks to be executed one after another type Chain struct { Tasks []*Signature } // Group creates a set of tasks to be executed in parallel type Group struct { GroupUUID string Tasks []*Signature } // Chord adds an optional callback to the group to be executed // after all tasks in the group finished type Chord struct { Group *Group Callback *Signature } // GetUUIDs returns slice of task UUIDS func (group *Group) GetUUIDs() []string { taskUUIDs := make([]string, len(group.Tasks)) for i, signature := range group.Tasks { taskUUIDs[i] = signature.UUID } return taskUUIDs } // NewChain creates a new chain of tasks to be processed one by one, passing // results unless task signatures are set to be immutable func NewChain(signatures ...*Signature) (*Chain, error) { // Auto generate task UUIDs if needed for _, signature := range signatures { if signature.UUID == "" { signatureID := uuid.New().String() signature.UUID = fmt.Sprintf("task_%v", signatureID) } } for i := len(signatures) - 1; i > 0; i-- { if i > 0 { signatures[i-1].OnSuccess = []*Signature{signatures[i]} } } chain := &Chain{Tasks: signatures} return chain, nil } // NewGroup creates a new group of tasks to be processed in parallel func NewGroup(signatures ...*Signature) (*Group, error) { // Generate a group UUID groupUUID := uuid.New().String() groupID := fmt.Sprintf("group_%v", groupUUID) // Auto generate task UUIDs if needed, group tasks by common group UUID for _, signature := range signatures { if signature.UUID == "" { signatureID := uuid.New().String() signature.UUID = fmt.Sprintf("task_%v", signatureID) } signature.GroupUUID = groupID signature.GroupTaskCount = len(signatures) } return &Group{ GroupUUID: groupID, Tasks: signatures, }, nil } // NewChord creates a new chord (a group of tasks with a single callback // to be executed after all tasks in the group has completed) func NewChord(group *Group, callback *Signature) (*Chord, error) { if callback.UUID == "" { // Generate a UUID for the chord callback callbackUUID := uuid.New().String() callback.UUID = fmt.Sprintf("chord_%v", callbackUUID) } // Add a chord callback to all tasks for _, signature := range group.Tasks { signature.ChordCallback = callback } return &Chord{Group: group, Callback: callback}, nil } ================================================ FILE: v2/tasks/workflow_test.go ================================================ package tasks_test import ( "testing" "github.com/RichardKnop/machinery/v2/tasks" "github.com/stretchr/testify/assert" ) func TestNewChain(t *testing.T) { t.Parallel() task1 := tasks.Signature{ Name: "foo", Args: []tasks.Arg{ { Type: "float64", Value: interface{}(1), }, { Type: "float64", Value: interface{}(1), }, }, } task2 := tasks.Signature{ Name: "bar", Args: []tasks.Arg{ { Type: "float64", Value: interface{}(5), }, { Type: "float64", Value: interface{}(6), }, }, } task3 := tasks.Signature{ Name: "qux", Args: []tasks.Arg{ { Type: "float64", Value: interface{}(4), }, }, } chain, err := tasks.NewChain(&task1, &task2, &task3) if err != nil { t.Fatal(err) } firstTask := chain.Tasks[0] assert.Equal(t, "foo", firstTask.Name) assert.Equal(t, "bar", firstTask.OnSuccess[0].Name) assert.Equal(t, "qux", firstTask.OnSuccess[0].OnSuccess[0].Name) } ================================================ FILE: v2/tracing/tracing.go ================================================ package tracing import ( "encoding/json" "github.com/RichardKnop/machinery/v2/tasks" opentracing "github.com/opentracing/opentracing-go" opentracing_ext "github.com/opentracing/opentracing-go/ext" opentracing_log "github.com/opentracing/opentracing-go/log" ) // opentracing tags var ( MachineryTag = opentracing.Tag{Key: string(opentracing_ext.Component), Value: "machinery"} WorkflowGroupTag = opentracing.Tag{Key: "machinery.workflow", Value: "group"} WorkflowChordTag = opentracing.Tag{Key: "machinery.workflow", Value: "chord"} WorkflowChainTag = opentracing.Tag{Key: "machinery.workflow", Value: "chain"} ) // StartSpanFromHeaders will extract a span from the signature headers // and start a new span with the given operation name. func StartSpanFromHeaders(headers tasks.Headers, operationName string) opentracing.Span { // Try to extract the span context from the carrier. spanContext, err := opentracing.GlobalTracer().Extract(opentracing.TextMap, headers) // Create a new span from the span context if found or start a new trace with the function name. // For clarity add the machinery component tag. span := opentracing.StartSpan( operationName, ConsumerOption(spanContext), MachineryTag, ) // Log any error but don't fail if err != nil { span.LogFields(opentracing_log.Error(err)) } return span } // HeadersWithSpan will inject a span into the signature headers func HeadersWithSpan(headers tasks.Headers, span opentracing.Span) tasks.Headers { // check if the headers aren't nil if headers == nil { headers = make(tasks.Headers) } if err := opentracing.GlobalTracer().Inject(span.Context(), opentracing.TextMap, headers); err != nil { span.LogFields(opentracing_log.Error(err)) } return headers } type consumerOption struct { producerContext opentracing.SpanContext } func (c consumerOption) Apply(o *opentracing.StartSpanOptions) { if c.producerContext != nil { opentracing.FollowsFrom(c.producerContext).Apply(o) } opentracing_ext.SpanKindConsumer.Apply(o) } // ConsumerOption ... func ConsumerOption(producer opentracing.SpanContext) opentracing.StartSpanOption { return consumerOption{producer} } type producerOption struct{} func (p producerOption) Apply(o *opentracing.StartSpanOptions) { opentracing_ext.SpanKindProducer.Apply(o) } // ProducerOption ... func ProducerOption() opentracing.StartSpanOption { return producerOption{} } // AnnotateSpanWithSignatureInfo ... func AnnotateSpanWithSignatureInfo(span opentracing.Span, signature *tasks.Signature) { // tag the span with some info about the signature span.SetTag("signature.name", signature.Name) span.SetTag("signature.uuid", signature.UUID) if signature.GroupUUID != "" { span.SetTag("signature.group.uuid", signature.GroupUUID) } if signature.ChordCallback != nil { span.SetTag("signature.chord.callback.uuid", signature.ChordCallback.UUID) span.SetTag("signature.chord.callback.name", signature.ChordCallback.Name) } } // AnnotateSpanWithChainInfo ... func AnnotateSpanWithChainInfo(span opentracing.Span, chain *tasks.Chain) { // tag the span with some info about the chain span.SetTag("chain.tasks.length", len(chain.Tasks)) // inject the tracing span into the tasks signature headers for _, signature := range chain.Tasks { signature.Headers = HeadersWithSpan(signature.Headers, span) } } // AnnotateSpanWithGroupInfo ... func AnnotateSpanWithGroupInfo(span opentracing.Span, group *tasks.Group, sendConcurrency int) { // tag the span with some info about the group span.SetTag("group.uuid", group.GroupUUID) span.SetTag("group.tasks.length", len(group.Tasks)) span.SetTag("group.concurrency", sendConcurrency) // encode the task uuids to json, if that fails just dump it in if taskUUIDs, err := json.Marshal(group.GetUUIDs()); err == nil { span.SetTag("group.tasks", string(taskUUIDs)) } else { span.SetTag("group.tasks", group.GetUUIDs()) } // inject the tracing span into the tasks signature headers for _, signature := range group.Tasks { signature.Headers = HeadersWithSpan(signature.Headers, span) } } // AnnotateSpanWithChordInfo ... func AnnotateSpanWithChordInfo(span opentracing.Span, chord *tasks.Chord, sendConcurrency int) { // tag the span with chord specific info span.SetTag("chord.callback.uuid", chord.Callback.UUID) // inject the tracing span into the callback signature chord.Callback.Headers = HeadersWithSpan(chord.Callback.Headers, span) // tag the span for the group part of the chord AnnotateSpanWithGroupInfo(span, chord.Group, sendConcurrency) } ================================================ FILE: v2/utils/deepcopy.go ================================================ package utils import ( "errors" "reflect" ) var ( ErrNoMatchType = errors.New("no match type") ErrNoPointer = errors.New("must be interface") ErrInvalidArgument = errors.New("invalid arguments") ) func deepCopy(dst, src reflect.Value) { switch src.Kind() { case reflect.Interface: value := src.Elem() if !value.IsValid() { return } newValue := reflect.New(value.Type()).Elem() deepCopy(newValue, value) dst.Set(newValue) case reflect.Ptr: value := src.Elem() if !value.IsValid() { return } dst.Set(reflect.New(value.Type())) deepCopy(dst.Elem(), value) case reflect.Map: dst.Set(reflect.MakeMap(src.Type())) keys := src.MapKeys() for _, key := range keys { value := src.MapIndex(key) newValue := reflect.New(value.Type()).Elem() deepCopy(newValue, value) dst.SetMapIndex(key, newValue) } case reflect.Slice: dst.Set(reflect.MakeSlice(src.Type(), src.Len(), src.Cap())) for i := 0; i < src.Len(); i++ { deepCopy(dst.Index(i), src.Index(i)) } case reflect.Struct: typeSrc := src.Type() for i := 0; i < src.NumField(); i++ { value := src.Field(i) tag := typeSrc.Field(i).Tag if value.CanSet() && tag.Get("deepcopy") != "-" { deepCopy(dst.Field(i), value) } } default: dst.Set(src) } } func DeepCopy(dst, src interface{}) error { typeDst := reflect.TypeOf(dst) typeSrc := reflect.TypeOf(src) if typeDst != typeSrc { return ErrNoMatchType } if typeSrc.Kind() != reflect.Ptr { return ErrNoPointer } valueDst := reflect.ValueOf(dst).Elem() valueSrc := reflect.ValueOf(src).Elem() if !valueDst.IsValid() || !valueSrc.IsValid() { return ErrInvalidArgument } deepCopy(valueDst, valueSrc) return nil } func DeepClone(v interface{}) interface{} { dst := reflect.New(reflect.TypeOf(v)).Elem() deepCopy(dst, reflect.ValueOf(v)) return dst.Interface() } ================================================ FILE: v2/utils/deepcopy_test.go ================================================ package utils import ( "testing" "github.com/stretchr/testify/assert" ) func TestDeepCopy(t *testing.T) { t.Parallel() type s struct { A float64 B int C []int D *int E map[string]int } var d = 3 var dst = new(s) var src = s{1.0, 1, []int{1, 2, 3}, &d, map[string]int{"a": 1}} err := DeepCopy(dst, &src) src.A = 2 assert.NoError(t, err) assert.Equal(t, 1.0, dst.A) assert.Equal(t, 1, dst.B) assert.Equal(t, []int{1, 2, 3}, dst.C) assert.Equal(t, &d, dst.D) assert.Equal(t, map[string]int{"a": 1}, dst.E) } ================================================ FILE: v2/utils/utils.go ================================================ package utils import ( "os" "path/filepath" ) const ( LockKeyPrefix = "machinery_lock_" ) func GetLockName(name, spec string) string { return LockKeyPrefix + filepath.Base(os.Args[0]) + name + spec } ================================================ FILE: v2/utils/utils_test.go ================================================ package utils import ( "testing" "github.com/stretchr/testify/assert" ) func TestGetLockName(t *testing.T) { t.Parallel() lockName := GetLockName("test", "*/3 * * *") assert.Equal(t, "machinery_lock_utils.testtest*/3 * * *", lockName) } ================================================ FILE: v2/utils/uuid.go ================================================ package utils import ( "github.com/google/uuid" "strings" ) func GetPureUUID() string { uid, _ := uuid.NewUUID() return strings.Replace(uid.String(), "-", "", -1) } ================================================ FILE: v2/utils/uuid_test.go ================================================ package utils import ( "testing" "github.com/stretchr/testify/assert" ) func TestGetPureUUID(t *testing.T) { t.Parallel() assert.Len(t, GetPureUUID(), 32) } ================================================ FILE: v2/wait-for-it.sh ================================================ #!/usr/bin/env bash # https://github.com/vishnubob/wait-for-it/blob/master/wait-for-it.sh # Use this script to test if a given TCP host/port are available cmdname=$(basename $0) echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } usage() { cat << USAGE >&2 Usage: $cmdname host:port [-s] [-t timeout] [-- command args] -h HOST | --host=HOST Host or IP under test -p PORT | --port=PORT TCP port under test Alternatively, you specify the host and port as host:port -s | --strict Only execute subcommand if the test succeeds -q | --quiet Don't output any status messages -t TIMEOUT | --timeout=TIMEOUT Timeout in seconds, zero for no timeout -- COMMAND ARGS Execute command with args after the test finishes USAGE exit 1 } wait_for() { if [[ $TIMEOUT -gt 0 ]]; then echoerr "$cmdname: waiting $TIMEOUT seconds for $HOST:$PORT" else echoerr "$cmdname: waiting for $HOST:$PORT without a timeout" fi start_ts=$(date +%s) while : do if [[ $ISBUSY -eq 1 ]]; then nc -z $HOST $PORT result=$? else (echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1 result=$? fi if [[ $result -eq 0 ]]; then end_ts=$(date +%s) echoerr "$cmdname: $HOST:$PORT is available after $((end_ts - start_ts)) seconds" break fi sleep 1 done return $result } wait_for_wrapper() { # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 if [[ $QUIET -eq 1 ]]; then timeout $BUSYTIMEFLAG $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & else timeout $BUSYTIMEFLAG $TIMEOUT $0 --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & fi PID=$! trap "kill -INT -$PID" INT wait $PID RESULT=$? if [[ $RESULT -ne 0 ]]; then echoerr "$cmdname: timeout occurred after waiting $TIMEOUT seconds for $HOST:$PORT" fi return $RESULT } # process arguments while [[ $# -gt 0 ]] do case "$1" in *:* ) hostport=(${1//:/ }) HOST=${hostport[0]} PORT=${hostport[1]} shift 1 ;; --child) CHILD=1 shift 1 ;; -q | --quiet) QUIET=1 shift 1 ;; -s | --strict) STRICT=1 shift 1 ;; -h) HOST="$2" if [[ $HOST == "" ]]; then break; fi shift 2 ;; --host=*) HOST="${1#*=}" shift 1 ;; -p) PORT="$2" if [[ $PORT == "" ]]; then break; fi shift 2 ;; --port=*) PORT="${1#*=}" shift 1 ;; -t) TIMEOUT="$2" if [[ $TIMEOUT == "" ]]; then break; fi shift 2 ;; --timeout=*) TIMEOUT="${1#*=}" shift 1 ;; --) shift CLI=("$@") break ;; --help) usage ;; *) echoerr "Unknown argument: $1" usage ;; esac done if [[ "$HOST" == "" || "$PORT" == "" ]]; then echoerr "Error: you need to provide a host and port to test." usage fi TIMEOUT=${TIMEOUT:-15} STRICT=${STRICT:-0} CHILD=${CHILD:-0} QUIET=${QUIET:-0} # check to see if timeout is from busybox? TIMEOUT_PATH=$(realpath $(which timeout)) if [[ $TIMEOUT_PATH =~ "busybox" ]]; then ISBUSY=1 BUSYTIMEFLAG="-t" else ISBUSY=0 BUSYTIMEFLAG="" fi if [[ $CHILD -gt 0 ]]; then wait_for RESULT=$? exit $RESULT else if [[ $TIMEOUT -gt 0 ]]; then wait_for_wrapper RESULT=$? else wait_for RESULT=$? fi fi if [[ $CLI != "" ]]; then if [[ $RESULT -ne 0 && $STRICT -eq 1 ]]; then echoerr "$cmdname: strict mode, refusing to execute subprocess" exit $RESULT fi exec "${CLI[@]}" else exit $RESULT fi ================================================ FILE: v2/worker.go ================================================ package machinery import ( "errors" "fmt" "net/url" "os" "os/signal" "sync" "syscall" "time" "github.com/opentracing/opentracing-go" "github.com/RichardKnop/machinery/v2/backends/amqp" "github.com/RichardKnop/machinery/v2/brokers/errs" "github.com/RichardKnop/machinery/v2/log" "github.com/RichardKnop/machinery/v2/retry" "github.com/RichardKnop/machinery/v2/tasks" "github.com/RichardKnop/machinery/v2/tracing" ) // Worker represents a single worker process type Worker struct { server *Server ConsumerTag string Concurrency int Queue string errorHandler func(err error) preTaskHandler func(*tasks.Signature) postTaskHandler func(*tasks.Signature) preConsumeHandler func(*Worker) bool } var ( // ErrWorkerQuitGracefully is return when worker quit gracefully ErrWorkerQuitGracefully = errors.New("Worker quit gracefully") // ErrWorkerQuitGracefully is return when worker quit abruptly ErrWorkerQuitAbruptly = errors.New("Worker quit abruptly") ) // Launch starts a new worker process. The worker subscribes // to the default queue and processes incoming registered tasks func (worker *Worker) Launch() error { errorsChan := make(chan error) worker.LaunchAsync(errorsChan) return <-errorsChan } // LaunchAsync is a non blocking version of Launch func (worker *Worker) LaunchAsync(errorsChan chan<- error) { cnf := worker.server.GetConfig() broker := worker.server.GetBroker() // Log some useful information about worker configuration log.INFO.Printf("Launching a worker with the following settings:") log.INFO.Printf("- Broker: %s", RedactURL(cnf.Broker)) if worker.Queue == "" { log.INFO.Printf("- DefaultQueue: %s", cnf.DefaultQueue) } else { log.INFO.Printf("- CustomQueue: %s", worker.Queue) } log.INFO.Printf("- ResultBackend: %s", RedactURL(cnf.ResultBackend)) if cnf.AMQP != nil { log.INFO.Printf("- AMQP: %s", cnf.AMQP.Exchange) log.INFO.Printf(" - Exchange: %s", cnf.AMQP.Exchange) log.INFO.Printf(" - ExchangeType: %s", cnf.AMQP.ExchangeType) log.INFO.Printf(" - BindingKey: %s", cnf.AMQP.BindingKey) log.INFO.Printf(" - PrefetchCount: %d", cnf.AMQP.PrefetchCount) } var signalWG sync.WaitGroup // Goroutine to start broker consumption and handle retries when broker connection dies go func() { for { retry, err := broker.StartConsuming(worker.ConsumerTag, worker.Concurrency, worker) if retry { if worker.errorHandler != nil { worker.errorHandler(err) } else { log.WARNING.Printf("Broker failed with error: %s", err) } } else { signalWG.Wait() errorsChan <- err // stop the goroutine return } } }() if !cnf.NoUnixSignals { sig := make(chan os.Signal, 1) signal.Notify(sig, os.Interrupt, syscall.SIGTERM) var signalsReceived uint // Goroutine Handle SIGINT and SIGTERM signals go func() { for s := range sig { log.WARNING.Printf("Signal received: %v", s) signalsReceived++ if signalsReceived < 2 { // After first Ctrl+C start quitting the worker gracefully log.WARNING.Print("Waiting for running tasks to finish before shutting down") signalWG.Add(1) go func() { worker.Quit() errorsChan <- ErrWorkerQuitGracefully signalWG.Done() }() } else { // Abort the program when user hits Ctrl+C second time in a row errorsChan <- ErrWorkerQuitAbruptly } } }() } } // CustomQueue returns Custom Queue of the running worker process func (worker *Worker) CustomQueue() string { return worker.Queue } // Quit tears down the running worker process func (worker *Worker) Quit() { worker.server.GetBroker().StopConsuming() } // Process handles received tasks and triggers success/error callbacks func (worker *Worker) Process(signature *tasks.Signature) error { // If the task is not registered with this worker, do not continue // but only return nil as we do not want to restart the worker process if !worker.server.IsTaskRegistered(signature.Name) { return nil } taskFunc, err := worker.server.GetRegisteredTask(signature.Name) if err != nil { return nil } // Update task state to RECEIVED if err = worker.server.GetBackend().SetStateReceived(signature); err != nil { return fmt.Errorf("Set state to 'received' for task %s returned error: %s", signature.UUID, err) } // Prepare task for processing task, err := tasks.NewWithSignature(taskFunc, signature) // if this failed, it means the task is malformed, probably has invalid // signature, go directly to task failed without checking whether to retry if err != nil { worker.taskFailed(signature, err) return err } // try to extract trace span from headers and add it to the function context // so it can be used inside the function if it has context.Context as the first // argument. Start a new span if it isn't found. taskSpan := tracing.StartSpanFromHeaders(signature.Headers, signature.Name) tracing.AnnotateSpanWithSignatureInfo(taskSpan, signature) task.Context = opentracing.ContextWithSpan(task.Context, taskSpan) // Update task state to STARTED if err = worker.server.GetBackend().SetStateStarted(signature); err != nil { return fmt.Errorf("Set state to 'started' for task %s returned error: %s", signature.UUID, err) } //Run handler before the task is called if worker.preTaskHandler != nil { worker.preTaskHandler(signature) } //Defer run handler for the end of the task if worker.postTaskHandler != nil { defer worker.postTaskHandler(signature) } // Call the task results, err := task.Call() if err != nil { // If a tasks.ErrRetryTaskLater was returned from the task, // retry the task after specified duration retriableErr, ok := interface{}(err).(tasks.ErrRetryTaskLater) if ok { return worker.retryTaskIn(signature, retriableErr.RetryIn()) } // Otherwise, execute default retry logic based on signature.RetryCount // and signature.RetryTimeout values if signature.RetryCount > 0 { return worker.taskRetry(signature) } return worker.taskFailed(signature, err) } return worker.taskSucceeded(signature, results) } // retryTask decrements RetryCount counter and republishes the task to the queue func (worker *Worker) taskRetry(signature *tasks.Signature) error { // Update task state to RETRY if err := worker.server.GetBackend().SetStateRetry(signature); err != nil { return fmt.Errorf("Set state to 'retry' for task %s returned error: %s", signature.UUID, err) } // Decrement the retry counter, when it reaches 0, we won't retry again signature.RetryCount-- // Increase retry timeout signature.RetryTimeout = retry.FibonacciNext(signature.RetryTimeout) // Delay task by signature.RetryTimeout seconds eta := time.Now().UTC().Add(time.Second * time.Duration(signature.RetryTimeout)) signature.ETA = &eta log.WARNING.Printf("Task %s failed. Going to retry in %d seconds.", signature.UUID, signature.RetryTimeout) // Send the task back to the queue _, err := worker.server.SendTask(signature) return err } // taskRetryIn republishes the task to the queue with ETA of now + retryIn.Seconds() func (worker *Worker) retryTaskIn(signature *tasks.Signature, retryIn time.Duration) error { // Update task state to RETRY if err := worker.server.GetBackend().SetStateRetry(signature); err != nil { return fmt.Errorf("Set state to 'retry' for task %s returned error: %s", signature.UUID, err) } // Delay task by retryIn duration eta := time.Now().UTC().Add(retryIn) signature.ETA = &eta log.WARNING.Printf("Task %s failed. Going to retry in %.0f seconds.", signature.UUID, retryIn.Seconds()) // Send the task back to the queue _, err := worker.server.SendTask(signature) return err } // taskSucceeded updates the task state and triggers success callbacks or a // chord callback if this was the last task of a group with a chord callback func (worker *Worker) taskSucceeded(signature *tasks.Signature, taskResults []*tasks.TaskResult) error { // Update task state to SUCCESS if err := worker.server.GetBackend().SetStateSuccess(signature, taskResults); err != nil { return fmt.Errorf("Set state to 'success' for task %s returned error: %s", signature.UUID, err) } // Log human readable results of the processed task var debugResults = "[]" results, err := tasks.ReflectTaskResults(taskResults) if err != nil { log.WARNING.Print(err) } else { debugResults = tasks.HumanReadableResults(results) } log.DEBUG.Printf("Processed task %s. Results = %s", signature.UUID, debugResults) // Trigger success callbacks for _, successTask := range signature.OnSuccess { if signature.Immutable == false { // Pass results of the task to success callbacks for _, taskResult := range taskResults { successTask.Args = append(successTask.Args, tasks.Arg{ Type: taskResult.Type, Value: taskResult.Value, }) } } worker.server.SendTask(successTask) } // If the task was not part of a group, just return if signature.GroupUUID == "" { return nil } // There is no chord callback, just return if signature.ChordCallback == nil { return nil } // Check if all task in the group has completed groupCompleted, err := worker.server.GetBackend().GroupCompleted( signature.GroupUUID, signature.GroupTaskCount, ) if err != nil { return fmt.Errorf("Completed check for group %s returned error: %s", signature.GroupUUID, err) } // If the group has not yet completed, just return if !groupCompleted { return nil } // Defer purging of group meta queue if we are using AMQP backend if worker.hasAMQPBackend() { defer worker.server.GetBackend().PurgeGroupMeta(signature.GroupUUID) } // Trigger chord callback shouldTrigger, err := worker.server.GetBackend().TriggerChord(signature.GroupUUID) if err != nil { return fmt.Errorf("Triggering chord for group %s returned error: %s", signature.GroupUUID, err) } // Chord has already been triggered if !shouldTrigger { return nil } // Get task states taskStates, err := worker.server.GetBackend().GroupTaskStates( signature.GroupUUID, signature.GroupTaskCount, ) if err != nil { log.ERROR.Printf( "Failed to get tasks states for group:[%s]. Task count:[%d]. The chord may not be triggered. Error:[%s]", signature.GroupUUID, signature.GroupTaskCount, err, ) return nil } // Append group tasks' return values to chord task if it's not immutable for _, taskState := range taskStates { if !taskState.IsSuccess() { return nil } if signature.ChordCallback.Immutable == false { // Pass results of the task to the chord callback for _, taskResult := range taskState.Results { signature.ChordCallback.Args = append(signature.ChordCallback.Args, tasks.Arg{ Type: taskResult.Type, Value: taskResult.Value, }) } } } // Send the chord task _, err = worker.server.SendTask(signature.ChordCallback) if err != nil { return err } return nil } // taskFailed updates the task state and triggers error callbacks func (worker *Worker) taskFailed(signature *tasks.Signature, taskErr error) error { // Update task state to FAILURE if err := worker.server.GetBackend().SetStateFailure(signature, taskErr.Error()); err != nil { return fmt.Errorf("Set state to 'failure' for task %s returned error: %s", signature.UUID, err) } if worker.errorHandler != nil { worker.errorHandler(taskErr) } else { log.ERROR.Printf("Failed processing task %s. Error = %v", signature.UUID, taskErr) } // Trigger error callbacks for _, errorTask := range signature.OnError { // Pass error as a first argument to error callbacks args := append([]tasks.Arg{{ Type: "string", Value: taskErr.Error(), }}, errorTask.Args...) errorTask.Args = args worker.server.SendTask(errorTask) } if signature.StopTaskDeletionOnError { return errs.ErrStopTaskDeletion } return nil } // Returns true if the worker uses AMQP backend func (worker *Worker) hasAMQPBackend() bool { _, ok := worker.server.GetBackend().(*amqp.Backend) return ok } // SetErrorHandler sets a custom error handler for task errors // A default behavior is just to log the error after all the retry attempts fail func (worker *Worker) SetErrorHandler(handler func(err error)) { worker.errorHandler = handler } //SetPreTaskHandler sets a custom handler func before a job is started func (worker *Worker) SetPreTaskHandler(handler func(*tasks.Signature)) { worker.preTaskHandler = handler } //SetPostTaskHandler sets a custom handler for the end of a job func (worker *Worker) SetPostTaskHandler(handler func(*tasks.Signature)) { worker.postTaskHandler = handler } //SetPreConsumeHandler sets a custom handler for the end of a job func (worker *Worker) SetPreConsumeHandler(handler func(*Worker) bool) { worker.preConsumeHandler = handler } //GetServer returns server func (worker *Worker) GetServer() *Server { return worker.server } // func (worker *Worker) PreConsumeHandler() bool { if worker.preConsumeHandler == nil { return true } return worker.preConsumeHandler(worker) } func RedactURL(urlString string) string { u, err := url.Parse(urlString) if err != nil { return urlString } return fmt.Sprintf("%s://%s", u.Scheme, u.Host) } ================================================ FILE: v2/worker_test.go ================================================ package machinery_test import ( "testing" "github.com/stretchr/testify/assert" "github.com/RichardKnop/machinery/v2" ) func TestRedactURL(t *testing.T) { t.Parallel() broker := "amqp://guest:guest@localhost:5672" redactedURL := machinery.RedactURL(broker) assert.Equal(t, "amqp://localhost:5672", redactedURL) } func TestPreConsumeHandler(t *testing.T) { t.Parallel() worker := &machinery.Worker{} worker.SetPreConsumeHandler(SamplePreConsumeHandler) assert.True(t, worker.PreConsumeHandler()) } func SamplePreConsumeHandler(w *machinery.Worker) bool { return true } ================================================ FILE: wait-for-it.sh ================================================ #!/usr/bin/env bash # https://github.com/vishnubob/wait-for-it/blob/master/wait-for-it.sh # Use this script to test if a given TCP host/port are available cmdname=$(basename $0) echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } usage() { cat << USAGE >&2 Usage: $cmdname host:port [-s] [-t timeout] [-- command args] -h HOST | --host=HOST Host or IP under test -p PORT | --port=PORT TCP port under test Alternatively, you specify the host and port as host:port -s | --strict Only execute subcommand if the test succeeds -q | --quiet Don't output any status messages -t TIMEOUT | --timeout=TIMEOUT Timeout in seconds, zero for no timeout -- COMMAND ARGS Execute command with args after the test finishes USAGE exit 1 } wait_for() { if [[ $TIMEOUT -gt 0 ]]; then echoerr "$cmdname: waiting $TIMEOUT seconds for $HOST:$PORT" else echoerr "$cmdname: waiting for $HOST:$PORT without a timeout" fi start_ts=$(date +%s) while : do if [[ $ISBUSY -eq 1 ]]; then nc -z $HOST $PORT result=$? else (echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1 result=$? fi if [[ $result -eq 0 ]]; then end_ts=$(date +%s) echoerr "$cmdname: $HOST:$PORT is available after $((end_ts - start_ts)) seconds" break fi sleep 1 done return $result } wait_for_wrapper() { # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 if [[ $QUIET -eq 1 ]]; then timeout $BUSYTIMEFLAG $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & else timeout $BUSYTIMEFLAG $TIMEOUT $0 --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & fi PID=$! trap "kill -INT -$PID" INT wait $PID RESULT=$? if [[ $RESULT -ne 0 ]]; then echoerr "$cmdname: timeout occurred after waiting $TIMEOUT seconds for $HOST:$PORT" fi return $RESULT } # process arguments while [[ $# -gt 0 ]] do case "$1" in *:* ) hostport=(${1//:/ }) HOST=${hostport[0]} PORT=${hostport[1]} shift 1 ;; --child) CHILD=1 shift 1 ;; -q | --quiet) QUIET=1 shift 1 ;; -s | --strict) STRICT=1 shift 1 ;; -h) HOST="$2" if [[ $HOST == "" ]]; then break; fi shift 2 ;; --host=*) HOST="${1#*=}" shift 1 ;; -p) PORT="$2" if [[ $PORT == "" ]]; then break; fi shift 2 ;; --port=*) PORT="${1#*=}" shift 1 ;; -t) TIMEOUT="$2" if [[ $TIMEOUT == "" ]]; then break; fi shift 2 ;; --timeout=*) TIMEOUT="${1#*=}" shift 1 ;; --) shift CLI=("$@") break ;; --help) usage ;; *) echoerr "Unknown argument: $1" usage ;; esac done if [[ "$HOST" == "" || "$PORT" == "" ]]; then echoerr "Error: you need to provide a host and port to test." usage fi TIMEOUT=${TIMEOUT:-15} STRICT=${STRICT:-0} CHILD=${CHILD:-0} QUIET=${QUIET:-0} # check to see if timeout is from busybox? TIMEOUT_PATH=$(realpath $(which timeout)) if [[ $TIMEOUT_PATH =~ "busybox" ]]; then ISBUSY=1 BUSYTIMEFLAG="-t" else ISBUSY=0 BUSYTIMEFLAG="" fi if [[ $CHILD -gt 0 ]]; then wait_for RESULT=$? exit $RESULT else if [[ $TIMEOUT -gt 0 ]]; then wait_for_wrapper RESULT=$? else wait_for RESULT=$? fi fi if [[ $CLI != "" ]]; then if [[ $RESULT -ne 0 && $STRICT -eq 1 ]]; then echoerr "$cmdname: strict mode, refusing to execute subprocess" exit $RESULT fi exec "${CLI[@]}" else exit $RESULT fi