Repository: didil/goblero Branch: master Commit: 9973f856b0fd Files: 16 Total size: 40.6 KB Directory structure: gitextract_iosxn5t5/ ├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── go.mod ├── go.sum └── pkg/ └── blero/ ├── backend.go ├── backend_test.go ├── dispatch.go ├── dispatch_test.go ├── jobstatus_string.go ├── jobstatus_string_test.go ├── processing.go ├── queue.go └── queue_test.go ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ .idea .vscode db/* # code coverage *.out cover.html coverage.txt # profiling *.test profile_cpu.pdf ================================================ FILE: .travis.yml ================================================ language: go sudo: false dist: trusty branches: only: - master go: - 1.15.x before_install: - make deps-ci script: - make test-ci after_success: - bash <(curl -s "https://codecov.io/bash") - $HOME/gopath/bin/goveralls -coverprofile=coverage.txt -service=travis-ci -repotoken $COVERALLS_TOKEN ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) 2019 Adil H Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: Makefile ================================================ test: go test -race ./pkg/... test-cover: go test -race -coverprofile cover.out -covermode=atomic ./pkg/... go tool cover -html=cover.out -o cover.html open cover.html test-ci: go test -race -coverprofile=coverage.txt -covermode=atomic ./pkg/... bench: go test -run=XXX -bench=. -benchtime=5s ./pkg/blero/ deps: go mod download deps-ci: deps go get golang.org/x/tools/cmd/cover go get github.com/mattn/goveralls ================================================ FILE: README.md ================================================ # Goblero Pure Go, Simple, Embedded, Persistent Job Queue, backed by [BadgerDB](https://github.com/dgraph-io/badger) [![Build Status](https://travis-ci.org/didil/goblero.svg?branch=master)](https://travis-ci.org/didil/goblero) [![Coverage Status](https://coveralls.io/repos/github/didil/goblero/badge.svg?branch=master)](https://coveralls.io/github/didil/goblero?branch=master) [![goreportcard](https://goreportcard.com/badge/github.com/didil/goblero)](https://goreportcard.com/report/github.com/didil/goblero) [![codebeat badge](https://codebeat.co/badges/1d261e4f-36ff-42b5-b015-e31eb7aa7e7d)](https://codebeat.co/projects/github-com-didil-goblero-master) [![Documentation](https://godoc.org/github.com/didil/goblero/pkg/blero?status.svg)](https://godoc.org/github.com/didil/goblero/pkg/blero) **DO NOT USE IN PRODUCTION** This library is still in Alpha / Work In Progress ## About Goblero Intro article: [Introducing Goblero: a Go Embedded Job Queue](https://medium.com/@didil/introducing-goblero-a-go-embedded-job-queue-6dfec8e24d4c) - Pure Go library, no cgo - Simple, embedded, persistent job queue - Provides in-process job processing to any Go app - The jobs/status changes are persisted to disk after each operation and pending jobs can continue processing after an app restart or a crash - Allows multiple "processors", each processor/worker processes one job at a time then is assigned a new job, etc - The storage engine used is [BadgerDB](https://github.com/dgraph-io/badger) *P.S: Why is the library named Goblero ? Go for the Go programming language obviously, and Badger in french is "Blaireau", but blero is easier to pronounce :)* ## Usage The full API is documented on [godoc.org](https://godoc.org/github.com/didil/goblero/pkg/blero). There is also a demo repo [goblero-demo](https://github.com/didil/goblero-demo/tree/master) Get package ```` go get -u github.com/didil/goblero/pkg/blero ```` API ```` // Create a new Blero backend bl := blero.New("db/") // Start Blero bl.Start() // defer Stopping Blero defer bl.Stop() // register a processor bl.RegisterProcessorFunc(func(j *blero.Job) error { // Do some processing, access job name with j.Name, job data with j.Data }) // enqueue a job bl.EnqueueJob("MyJob", []byte("My Job Data")) ```` ## Benchmarks ```` # Core i5 laptop / 8GB Ram / SSD make bench BenchmarkEnqueue/EnqueueJob-4 50000 159942 ns/op (~ 6250 ops/s) BenchmarkEnqueue/dequeueJob-4 5000 2767260 ns/op (~ 361 ops/s) ```` ## Todo: - Restart interrupted jobs after app restart/crashes - Sweep completed jobs from the "complete" queue - Failed Jobs retry options - Allow batch enqueuing - Add support for Go contexts - Test in real conditions under high load - Expose Prometheus Metrics in an Http handler - Optimize performance / Locking ## Contributing All contributions (PR, feedback, bug reports, ideas, etc.) are welcome ! [![](https://sourcerer.io/fame/didil/didil/goblero/images/0)](https://sourcerer.io/fame/didil/didil/goblero/links/0)[![](https://sourcerer.io/fame/didil/didil/goblero/images/1)](https://sourcerer.io/fame/didil/didil/goblero/links/1)[![](https://sourcerer.io/fame/didil/didil/goblero/images/2)](https://sourcerer.io/fame/didil/didil/goblero/links/2)[![](https://sourcerer.io/fame/didil/didil/goblero/images/3)](https://sourcerer.io/fame/didil/didil/goblero/links/3)[![](https://sourcerer.io/fame/didil/didil/goblero/images/4)](https://sourcerer.io/fame/didil/didil/goblero/links/4)[![](https://sourcerer.io/fame/didil/didil/goblero/images/5)](https://sourcerer.io/fame/didil/didil/goblero/links/5)[![](https://sourcerer.io/fame/didil/didil/goblero/images/6)](https://sourcerer.io/fame/didil/didil/goblero/links/6)[![](https://sourcerer.io/fame/didil/didil/goblero/images/7)](https://sourcerer.io/fame/didil/didil/goblero/links/7) ================================================ FILE: go.mod ================================================ module github.com/didil/goblero go 1.26 require ( github.com/dgraph-io/badger v1.6.2 github.com/stretchr/testify v1.11.1 ) require ( github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgraph-io/ristretto v0.2.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/stretchr/objx v0.5.3 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/sys v0.42.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) ================================================ FILE: go.sum ================================================ github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.2.0 h1:XAfl+7cmoUDWW/2Lx8TGZQjjxIQ2Ley9DSf52dru4WE= github.com/dgraph-io/ristretto v0.2.0/go.mod h1:8uBHCU/PBV4Ag0CJrP47b9Ofby5dqWNh4FicAdoqFNU= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= ================================================ FILE: pkg/blero/backend.go ================================================ package blero import "fmt" // Blero struct type Blero struct { dispatcher *dispatcher queue *queue } // New creates new Blero Backend func New(dbPath string) *Blero { bl := &Blero{} pStore := newProcessorsStore() bl.dispatcher = newDispatcher(pStore) bl.queue = newQueue(queueOpts{DBPath: dbPath}) return bl } // Start Blero func (bl *Blero) Start() error { fmt.Println("Starting Blero ...") err := bl.queue.start() if err != nil { return err } bl.dispatcher.startLoop(bl.queue) return nil } // Stop Blero and Release resources func (bl *Blero) Stop() error { fmt.Println("Stopping Blero ...") bl.dispatcher.stopLoop() return bl.queue.stop() } // EnqueueJob enqueues a new Job and returns the job id func (bl *Blero) EnqueueJob(name string, data []byte) (uint64, error) { jID, err := bl.queue.enqueueJob(name, data) if err != nil { return 0, err } // signal that a new job was enqueued bl.dispatcher.signalLoop() return jID, nil } // EnqueueJobs enqueues new Jobs /*func (bl *Blero) EnqueueJobs(names string) (uint64, error) { }*/ // RegisterProcessor registers a new processor and returns the processor id func (bl *Blero) RegisterProcessor(p Processor) int { return bl.dispatcher.registerProcessor(p) } // RegisterProcessorFunc registers a new ProcessorFunc and returns the processor id func (bl *Blero) RegisterProcessorFunc(f func(j *Job) error) int { return bl.dispatcher.registerProcessor(ProcessorFunc(f)) } // UnregisterProcessor unregisters a processor // No more jobs will be assigned but if will not cancel a job that already started processing func (bl *Blero) UnregisterProcessor(pID int) { bl.dispatcher.unregisterProcessor(pID) } ================================================ FILE: pkg/blero/backend_test.go ================================================ package blero import ( "testing" "github.com/stretchr/testify/assert" ) func TestBlero_StartNoDBPath(t *testing.T) { bl := New("") err := bl.Start() assert.EqualError(t, err, "DBPath is required") } func TestBlero_StartDBPathDoesntExist(t *testing.T) { bl := New("/tmp/12354/56547/45459/blero/dbx/") err := bl.Start() assert.EqualError(t, err, "Error Creating Dir: \"/tmp/12354/56547/45459/blero/dbx/\": mkdir /tmp/12354/56547/45459/blero/dbx/: no such file or directory") } func BenchmarkEnqueue(b *testing.B) { bl := New(testDBPath) err := bl.Start() if err != nil { b.Error(err) } // stop gracefully defer deleteDBFolder(testDBPath) defer bl.Stop() jobName := "MyJob" jobData := []byte("MyJobData") b.ResetTimer() b.Run("EnqueueJob", func(b *testing.B) { for n := 0; n < b.N; n++ { _, err := bl.EnqueueJob(jobName, jobData) if err != nil { b.Error(err) } } }) b.ResetTimer() b.Run("dequeueJob", func(b *testing.B) { for n := 0; n < b.N; n++ { _, err := bl.queue.dequeueJob() if err != nil { b.Error(err) } } }) } ================================================ FILE: pkg/blero/dispatch.go ================================================ package blero import ( "fmt" "io" "os" "sync" ) // dispatcher struct type dispatcher struct { dispatchL sync.Mutex ch chan int quitCh chan struct{} pStore *processorsStore } // newDispatcher creates new Dispatcher func newDispatcher(pStore *processorsStore) *dispatcher { d := &dispatcher{} d.ch = make(chan int, 100) d.quitCh = make(chan struct{}) d.pStore = pStore return d } // testable stderr var stdErr io.ReadWriter = os.Stderr // startLoop starts the dispatcher assignment loop func (d *dispatcher) startLoop(q *queue) { go func() { for { select { case <-d.ch: err := d.assignJobs(q) if err != nil { fmt.Fprintf(stdErr, "Cannot assign jobs: %v", err) } case <-d.quitCh: // loop was stopped return } } }() } // signalLoop signals to the dispatcher loop that an assignment check might need to run func (d *dispatcher) signalLoop() { go func() { select { case <-d.quitCh: // loop was stopped return case d.ch <- 1: return } }() } // stopLoop stops the dispatcher assignment loop func (d *dispatcher) stopLoop() { close(d.quitCh) } // registerProcessor registers a new processor func (d *dispatcher) registerProcessor(p Processor) int { d.dispatchL.Lock() defer d.dispatchL.Unlock() pID := d.pStore.registerProcessor(p) // signal that the processor is now available d.signalLoop() return pID } // unregisterProcessor unregisters a processor // No more jobs will be assigned but if will not cancel a job that already started processing func (d *dispatcher) unregisterProcessor(pID int) { d.dispatchL.Lock() defer d.dispatchL.Unlock() d.pStore.unregisterProcessor(pID) } // assignJobs assigns pending jobs from the queue to free processors func (d *dispatcher) assignJobs(q *queue) error { d.dispatchL.Lock() defer d.dispatchL.Unlock() pIDs := d.pStore.getAvailableProcessorsIDs() for _, pID := range pIDs { err := d.assignJob(q, pID) if err != nil { return err } } return nil } // assignJob assigns a pending job processor #pID and starts the run // NOT THREAD SAFE !! only call from assignJobs func (d *dispatcher) assignJob(q *queue, pID int) error { p := d.pStore.getProcessor(pID) if p == nil { return fmt.Errorf("Processor %v not found", pID) } j, err := q.dequeueJob() if err != nil { return err } // no jobs to assign if j == nil { return nil } if d.pStore.isProcessorBusy(pID) { return fmt.Errorf("Cannot assign job %v to Processor %v. Processor busy", j.ID, pID) } d.pStore.setProcessing(pID, j.ID) go d.runJob(q, pID, p, j) return nil } // unassignJob unmarks a job as assigned to #pID func (d *dispatcher) unassignJob(pID int) { d.dispatchL.Lock() defer d.dispatchL.Unlock() d.pStore.unsetProcessing(pID) } // runJob runs a job on the corresponding processor and moves it to the right queue depending on results func (d *dispatcher) runJob(q *queue, pID int, p Processor, j *Job) { defer d.processorDone(pID) err := p.Run(j) if err != nil { fmt.Printf("Processor: %v. Job %v failed with err: %v\n", pID, j.ID, err) err := q.markJobDone(j.ID, jobFailed) if err != nil { fmt.Printf("markJobDone -> %v jobFailed failed: %v\n", j.ID, err) } return } err = q.markJobDone(j.ID, jobComplete) if err != nil { fmt.Printf("markJobDone -> %v jobComplete failed: %v\n", j.ID, err) } } func (d *dispatcher) processorDone(pID int) { d.unassignJob(pID) // signal that the processor might now be available d.signalLoop() } ================================================ FILE: pkg/blero/dispatch_test.go ================================================ package blero import ( "bytes" "fmt" "io/ioutil" "runtime" "sync" "testing" "time" "github.com/dgraph-io/badger" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) type noOpProcessor struct{} func (p *noOpProcessor) Run(j *Job) error { return nil } func TestBlero_RegisterUnregisterProcessor(t *testing.T) { bl := New(testDBPath) p1 := &noOpProcessor{} p2 := &noOpProcessor{} // test usage of ProcessorFunc p3 := func(j *Job) error { return nil } d := bl.dispatcher pStore := d.pStore pID1 := bl.RegisterProcessor(p1) pID2 := bl.RegisterProcessor(p2) pID3 := bl.RegisterProcessorFunc(p3) assert.Len(t, pStore.processors, 3) assert.Equal(t, 3, pStore.maxProcessorID) bl.UnregisterProcessor(pID2) assert.Len(t, pStore.processors, 2) assert.Equal(t, p1, pStore.processors[pID1]) assert.Equal(t, nil, pStore.processors[pID2]) assert.NotNil(t, pStore.processors[pID3]) bl.UnregisterProcessor(pID3) assert.Len(t, pStore.processors, 1) assert.Equal(t, p1, pStore.processors[pID1]) assert.Equal(t, nil, pStore.processors[pID2]) assert.Equal(t, nil, pStore.processors[pID3]) } type testProcessor struct { mock.Mock } func (m *testProcessor) Run(j *Job) error { _ = m.Called(j) var err error if j.Name == "MyOtherOtherJob" { err = fmt.Errorf("%v errors out", j.Name) } return err } func TestBlero_assignJobs(t *testing.T) { bl := New(testDBPath) // only start the queue and not the dispatch loop to allow manual jobs assignment err := bl.queue.start() assert.NoError(t, err) // stop gracefully defer deleteDBFolder(testDBPath) defer bl.Stop() d := bl.dispatcher q := bl.queue j1Name := "MyJob" j2Name := "MyOtherJob" j3Name := "MyOtherOtherJob" p1 := new(testProcessor) p2 := new(testProcessor) p3 := new(testProcessor) for _, p := range []*testProcessor{p1, p2, p3} { p.On("Run", mock.AnythingOfType("*blero.Job")) bl.RegisterProcessor(p) } // test assign without jobs err = d.assignJobs(q) assert.NoError(t, err) // enqueue jobs j1ID, err := bl.EnqueueJob(j1Name, nil) assert.NoError(t, err) p1.AssertNumberOfCalls(t, "Run", 0) p2.AssertNumberOfCalls(t, "Run", 0) p3.AssertNumberOfCalls(t, "Run", 0) j2ID, err := bl.EnqueueJob(j2Name, nil) assert.NoError(t, err) p1.AssertNumberOfCalls(t, "Run", 0) p2.AssertNumberOfCalls(t, "Run", 0) p3.AssertNumberOfCalls(t, "Run", 0) j3ID, err := bl.EnqueueJob(j3Name, nil) assert.NoError(t, err) p1.AssertNumberOfCalls(t, "Run", 0) p2.AssertNumberOfCalls(t, "Run", 0) p3.AssertNumberOfCalls(t, "Run", 0) err = d.assignJobs(q) assert.NoError(t, err) // wait for jobs to be processed time.Sleep(50 * time.Millisecond) p1.AssertNumberOfCalls(t, "Run", 1) p2.AssertNumberOfCalls(t, "Run", 1) p3.AssertNumberOfCalls(t, "Run", 1) err = q.db.View(func(txn *badger.Txn) error { // check that job 1 is in the complete queue _, err := txn.Get([]byte("q:complete:" + jIDString(j1ID))) assert.NoError(t, err) // check that job 2 is in the complete queue _, err = txn.Get([]byte("q:complete:" + jIDString(j2ID))) assert.NoError(t, err) // check that job 3 is in the failed queue _, err = txn.Get([]byte("q:failed:" + jIDString(j3ID))) assert.NoError(t, err) return nil }) assert.NoError(t, err) p1.AssertExpectations(t) p2.AssertExpectations(t) p3.AssertExpectations(t) } func TestBlero_AutoProcessing_ProcessorFirst(t *testing.T) { bl := New(testDBPath) err := bl.Start() assert.NoError(t, err) // stop gracefully defer deleteDBFolder(testDBPath) defer bl.Stop() var m sync.Mutex var calls []string bl.RegisterProcessor(ProcessorFunc(func(j *Job) error { m.Lock() calls = append(calls, j.Name) m.Unlock() return nil })) // simulate wait period time.Sleep(50 * time.Millisecond) j1Name := "MyJob" j2Name := "MyOtherJob" bl.EnqueueJob(j1Name, nil) bl.EnqueueJob(j2Name, nil) // wait for jobs to be processed time.Sleep(50 * time.Millisecond) m.Lock() assert.Len(t, calls, 2) assert.ElementsMatch(t, []string{j1Name, j2Name}, calls) m.Unlock() } func TestBlero_AutoProcessing_JobsFirst(t *testing.T) { bl := New(testDBPath) err := bl.Start() assert.NoError(t, err) // stop gracefully defer deleteDBFolder(testDBPath) defer bl.Stop() var m sync.Mutex var calls []string j1Name := "MyJob" j2Name := "MyOtherJob" bl.EnqueueJob(j1Name, nil) bl.EnqueueJob(j2Name, nil) // simulate wait period time.Sleep(50 * time.Millisecond) bl.RegisterProcessor(ProcessorFunc(func(j *Job) error { m.Lock() calls = append(calls, j.Name) m.Unlock() return nil })) // wait for jobs to be processed time.Sleep(50 * time.Millisecond) m.Lock() assert.Len(t, calls, 2) assert.ElementsMatch(t, []string{j1Name, j2Name}, calls) m.Unlock() } func TestBlero_AutoProcessing_GoRoutinesHanging(t *testing.T) { bl := New(testDBPath) // only start the queue and not the dispatch loop to not run assign and show goroutines hanging problem err := bl.queue.start() assert.NoError(t, err) // stop gracefully defer deleteDBFolder(testDBPath) for index := 0; index < 300; index++ { bl.EnqueueJob("FakeJob", nil) } bl.Stop() // simulate wait period time.Sleep(50 * time.Millisecond) assert.True(t, runtime.NumGoroutine() < 20) } type safeBuffer struct { bytes.Buffer m sync.Mutex } func (b *safeBuffer) Read(p []byte) (n int, err error) { b.m.Lock() defer b.m.Unlock() return b.Buffer.Read(p) } func (b *safeBuffer) Write(p []byte) (n int, err error) { b.m.Lock() defer b.m.Unlock() return b.Buffer.Write(p) } func Test_dispatcherAssignFails(t *testing.T) { stdErr = new(safeBuffer) pStore := newProcessorsStore() // introduce error by registering nil processor pStore.registerProcessor(nil) d := newDispatcher(pStore) d.startLoop(nil) // send assign signal d.signalLoop() time.Sleep(50 * time.Millisecond) errText, err := ioutil.ReadAll(stdErr) assert.NoError(t, err) assert.Equal(t, "Cannot assign jobs: Processor 1 not found", string(errText)) } ================================================ FILE: pkg/blero/jobstatus_string.go ================================================ // Code generated by "stringer -type jobStatus ."; DO NOT EDIT. package blero import "strconv" const _jobStatus_name = "pendinginprogresscompletefailed" var _jobStatus_index = [...]uint8{0, 7, 17, 25, 31} func (i jobStatus) String() string { if i >= jobStatus(len(_jobStatus_index)-1) { return "jobStatus(" + strconv.FormatInt(int64(i), 10) + ")" } return _jobStatus_name[_jobStatus_index[i]:_jobStatus_index[i+1]] } ================================================ FILE: pkg/blero/jobstatus_string_test.go ================================================ package blero import ( "testing" "github.com/stretchr/testify/assert" ) func TestJobStatus_String(t *testing.T) { assert.Equal(t, "pending", jobPending.String()) assert.Equal(t, "inprogress", jobInProgress.String()) assert.Equal(t, "complete", jobComplete.String()) assert.Equal(t, "failed", jobFailed.String()) // unknown assert.Equal(t, "jobStatus(50)", jobStatus(50).String()) } ================================================ FILE: pkg/blero/processing.go ================================================ package blero // Job represents a Goblero job definition type Job struct { ID uint64 Name string Data []byte } // Processor interface type Processor interface { Run(j *Job) error } // ProcessorFunc is a processor function type ProcessorFunc func(j *Job) error // Run allows using ProcessorFunc as a Processor func (pf ProcessorFunc) Run(j *Job) error { return pf(j) } // processorsStore struct type processorsStore struct { maxProcessorID int processors map[int]Processor processing map[int]uint64 } // newProcessorsStore creates a new ProcessorsStore func newProcessorsStore() *processorsStore { pStore := &processorsStore{} pStore.processors = make(map[int]Processor) pStore.processing = make(map[int]uint64) return pStore } // registerProcessor registers a new processor func (pStore *processorsStore) registerProcessor(p Processor) int { pStore.maxProcessorID++ pStore.processors[pStore.maxProcessorID] = p return pStore.maxProcessorID } // unregisterProcessor unregisters a processor func (pStore *processorsStore) unregisterProcessor(pID int) { delete(pStore.processors, pID) } // getAvailableProcessorsIDs returns the currently free processors func (pStore *processorsStore) getAvailableProcessorsIDs() []int { var pIDs []int for pID := range pStore.processors { if _, ok := pStore.processing[pID]; !ok { pIDs = append(pIDs, pID) } } return pIDs } // getProcessor fetches processors by ID func (pStore *processorsStore) getProcessor(pID int) Processor { return pStore.processors[pID] } // isProcessorBusy checks if a processor is already working on a job func (pStore *processorsStore) isProcessorBusy(pID int) bool { _, ok := pStore.processing[pID] return ok } // setProcessing sets a processor as working on a job func (pStore *processorsStore) setProcessing(pID int, jID uint64) { pStore.processing[pID] = jID } // unsetProcessing unsets a processor as working on a job func (pStore *processorsStore) unsetProcessing(pID int) { delete(pStore.processing, pID) } ================================================ FILE: pkg/blero/queue.go ================================================ package blero import ( "bytes" "encoding/gob" "errors" "fmt" "os" "sync" "github.com/dgraph-io/badger" ) // queueOpts struct type queueOpts struct { DBPath string Logger badger.Logger } // queue struct type queue struct { opts queueOpts db *badger.DB seq *badger.Sequence dbL sync.Mutex } // newQueue creates new ueue func newQueue(opts queueOpts) *queue { q := &queue{opts: opts} return q } type badgerLogger struct{} func (l *badgerLogger) Infof(format string, a ...interface{}) { // ignore badger info logs } func (l *badgerLogger) Errorf(format string, a ...interface{}) { fmt.Fprintf(os.Stderr, format, a...) } func (l *badgerLogger) Warningf(format string, a ...interface{}) { fmt.Fprintf(os.Stderr, format, a...) } func (l *badgerLogger) Debugf(format string, a ...interface{}) { // Leave blank for the time being, // track it in the future. } // start Queue func (q *queue) start() error { // validate opts if q.opts.DBPath == "" { return errors.New("DBPath is required") } x := badger.Options{} x.Dir = "ds" // open db badgerOpts := badger.DefaultOptions(q.opts.DBPath) badgerOpts.Logger = &badgerLogger{} badgerOpts.SyncWrites = true db, err := badger.Open(badgerOpts) if err != nil { return err } q.db = db // init sequence q.seq, err = db.GetSequence([]byte("standard"), 1000) return err } // stop Queue and Release resources func (q *queue) stop() error { // release sequence err := q.seq.Release() if err != nil { return err } // close db err = q.db.Close() if err != nil { return err } return nil } func getNextSeq(seq *badger.Sequence) (num uint64, err error) { defer func() { r := recover() if r != nil { // recover from panic and send err instead err = r.(error) } }() num, err = seq.Next() return num, err } // enqueueJob enqueues a new Job to the Pending queue func (q *queue) enqueueJob(name string, data []byte) (uint64, error) { num, err := getNextSeq(q.seq) if err != nil { return 0, err } j := &Job{ID: num + 1, Name: name, Data: data} jKey := getJobKey(jobPending, j.ID) err = q.db.Update(func(txn *badger.Txn) error { b, err := encodeJob(j) if err != nil { return err } err = txn.Set([]byte(jKey), b) return err }) if err != nil { return 0, err } return j.ID, nil } func encodeJob(j *Job) ([]byte, error) { var b bytes.Buffer err := gob.NewEncoder(&b).Encode(j) return b.Bytes(), err } // jobStatus Enum Type type jobStatus uint8 const ( // jobPending : waiting to be processed jobPending jobStatus = iota // jobInProgress : processing in progress jobInProgress // jobComplete : processing complete jobComplete // jobFailed : processing errored out jobFailed ) func getQueueKeyPrefix(status jobStatus) string { return fmt.Sprintf("q:%v:", status) } func getJobKey(status jobStatus, jID uint64) string { return getQueueKeyPrefix(status) + jIDString(jID) } func jIDString(jID uint64) string { return fmt.Sprintf("%020d", jID) } // dequeueJob moves the next pending job from the pending status to inprogress func (q *queue) dequeueJob() (*Job, error) { var j *Job q.dbL.Lock() defer q.dbL.Unlock() err := q.db.Update(func(txn *badger.Txn) error { prefix := []byte(getQueueKeyPrefix(jobPending)) k, v, err := getFirstKVForPrefix(txn, prefix) if err != nil { return err } // iteration is done, no job was found if k == nil { return nil } j, err = decodeJob(v) if err != nil { return err } // Move from from Pending queue to InProgress queue err = moveItem(txn, k, []byte(getJobKey(jobInProgress, j.ID)), v) return err }) return j, err } func getFirstKVForPrefix(txn *badger.Txn, prefix []byte) ([]byte, []byte, error) { itOpts := badger.DefaultIteratorOptions itOpts.PrefetchValues = true itOpts.PrefetchSize = 1 it := txn.NewIterator(itOpts) // go to smallest key after prefix it.Seek(prefix) defer it.Close() // iteration done, no item found if !it.ValidForPrefix(prefix) { return nil, nil, nil } item := it.Item() k := item.KeyCopy(nil) v, err := item.ValueCopy(nil) return k, v, err } // markJobDone moves a job from the inprogress status to complete/failed func (q *queue) markJobDone(id uint64, status jobStatus) error { if status != jobComplete && status != jobFailed { return errors.New("Can only move to Complete or Failed Status") } key := []byte(getJobKey(jobInProgress, id)) q.dbL.Lock() defer q.dbL.Unlock() err := q.db.Update(func(txn *badger.Txn) error { b, err := getBytesForKey(txn, key) if err != nil { return err } // Move from from InProgress queue to dest queue err = moveItem(txn, key, []byte(getJobKey(status, id)), b) return err }) return err } func decodeJob(b []byte) (*Job, error) { var j *Job err := gob.NewDecoder(bytes.NewBuffer(b)).Decode(&j) if err != nil { return nil, err } return j, nil } func getJobForKey(txn *badger.Txn, key []byte) (*Job, error) { b, err := getBytesForKey(txn, key) if err != nil { return nil, err } j, err := decodeJob(b) if err != nil { return nil, err } return j, nil } func getBytesForKey(txn *badger.Txn, key []byte) ([]byte, error) { item, err := txn.Get(key) if err != nil { return nil, err } b, err := item.ValueCopy(nil) if err != nil { return nil, err } return b, nil } func moveItem(txn *badger.Txn, oldKey []byte, newKey []byte, b []byte) error { // remove from Source queue err := txn.Delete(oldKey) if err != nil { return err } // create in Dest queue err = txn.Set(newKey, b) return err } ================================================ FILE: pkg/blero/queue_test.go ================================================ package blero import ( "os" "path/filepath" "sort" "testing" "github.com/dgraph-io/badger" "github.com/stretchr/testify/assert" ) const testDBPath = "../../db/test" func deleteDBFolder(dbPath string) { // prevent accidental deletion of non badgerdb folder if _, err := os.Stat(filepath.Join(dbPath, "MANIFEST")); os.IsNotExist(err) { panic("Attempted to delete non badgerdb folder " + dbPath) } err := os.RemoveAll(dbPath) if err != nil { panic(err) } } /* func TestBlero_StopQueueAlreadyStopped(t *testing.T) { bl := New(testDBPath) err := bl.Start() assert.NoError(t, err) // delete folder defer deleteDBFolder(testDBPath) bl.Stop() err = bl.queue.stop() assert.EqualError(t, err, "Writes are blocked, possibly due to DropAll or Close") }*/ func TestBlero_BadgerLogger(t *testing.T) { logger := &badgerLogger{} // test logger logger.Infof("[badgerLogger]TEST Infof") logger.Warningf("[badgerLogger]TEST Warningf") logger.Errorf("[badgerLogger]TEST Errorf") } func TestBlero_EnqueueJob(t *testing.T) { bl := New(testDBPath) err := bl.Start() assert.NoError(t, err) q := bl.queue // stop gracefully defer deleteDBFolder(testDBPath) defer bl.Stop() jName := "TestJob" jData := []byte("TestJob Args") jID, err := bl.EnqueueJob(jName, jData) assert.NoError(t, err) assert.Equal(t, uint64(1), jID) var j *Job err = q.db.View(func(txn *badger.Txn) error { j, err = getJobForKey(txn, []byte("q:pending:"+jIDString(jID))) assert.NoError(t, err) return nil }) assert.NoError(t, err) assert.Equal(t, jID, j.ID) assert.Equal(t, jName, j.Name) assert.Equal(t, jData, j.Data) } func TestBlero_EnqueueJob_Concurrent(t *testing.T) { bl := New(testDBPath) err := bl.Start() assert.NoError(t, err) // stop gracefully defer deleteDBFolder(testDBPath) defer bl.Stop() ch := make(chan uint64) go func() { id, err := bl.EnqueueJob("TestJob", nil) assert.NoError(t, err) ch <- id }() go func() { id, err := bl.EnqueueJob("TestJob", nil) assert.NoError(t, err) ch <- id }() id1 := <-ch id2 := <-ch assert.ElementsMatch(t, []uint64{1, 2}, []uint64{id1, id2}) } func TestBlero_EnqueueJobQueueStopped(t *testing.T) { bl := New(testDBPath) err := bl.Start() assert.NoError(t, err) // delete folder defer deleteDBFolder(testDBPath) bl.Stop() _, err = bl.EnqueueJob("TestJob", nil) assert.EqualError(t, err, "runtime error: invalid memory address or nil pointer dereference") } func TestBlero_DequeueJob(t *testing.T) { bl := New(testDBPath) err := bl.Start() assert.NoError(t, err) // stop gracefully defer deleteDBFolder(testDBPath) defer bl.Stop() q := bl.queue j1Name := "TestJob" j1ID, err := bl.EnqueueJob(j1Name, nil) assert.NoError(t, err) j2Name := "TestJob" j2ID, err := bl.EnqueueJob(j2Name, nil) assert.NoError(t, err) j, err := q.dequeueJob() assert.NoError(t, err) assert.Equal(t, j1ID, j.ID) assert.Equal(t, j1Name, j.Name) err = q.db.View(func(txn *badger.Txn) error { // check that job 1 is not in the pending queue anymore _, err := txn.Get([]byte("q:pending:" + jIDString(j1ID))) assert.EqualError(t, err, badger.ErrKeyNotFound.Error()) // check that job 2 is still in the pending queue _, err = txn.Get([]byte("q:pending:" + jIDString(j2ID))) assert.NoError(t, err) // get job 1 from inprogress queue completeJob, err := getJobForKey(txn, []byte("q:inprogress:"+jIDString(j1ID))) assert.NoError(t, err) assert.Equal(t, j1ID, completeJob.ID) assert.Equal(t, j1Name, completeJob.Name) return nil }) assert.NoError(t, err) } func TestBlero_DequeueJob_Concurrent(t *testing.T) { bl := New(testDBPath) err := bl.Start() assert.NoError(t, err) // stop gracefully defer deleteDBFolder(testDBPath) defer bl.Stop() q := bl.queue j1Name := "TestJob" j1ID, err := bl.EnqueueJob(j1Name, nil) assert.NoError(t, err) j2Name := "TestJob" j2ID, err := bl.EnqueueJob(j2Name, nil) assert.NoError(t, err) ch := make(chan *Job) go func() { j, err := q.dequeueJob() assert.NoError(t, err) ch <- j }() go func() { j, err := q.dequeueJob() assert.NoError(t, err) ch <- j }() j1 := <-ch j2 := <-ch jobs := []*Job{j1, j2} sort.Slice(jobs, func(i, j int) bool { return jobs[i].ID < jobs[j].ID }) assert.Equal(t, jobs[0].ID, j1ID) assert.Equal(t, jobs[0].Name, j1Name) assert.Equal(t, jobs[1].ID, j2ID) assert.Equal(t, jobs[1].Name, j2Name) } func TestBlero_MarkJobDone(t *testing.T) { bl := New(testDBPath) err := bl.Start() assert.NoError(t, err) // stop gracefully defer deleteDBFolder(testDBPath) defer bl.Stop() q := bl.queue j1Name := "TestJob" j1ID, err := bl.EnqueueJob(j1Name, nil) assert.NoError(t, err) j2Name := "TestJob" j2ID, err := bl.EnqueueJob(j2Name, nil) assert.NoError(t, err) // move job 1 to inprogress _, err = q.dequeueJob() assert.NoError(t, err) // move job 2 to inprogress _, err = q.dequeueJob() assert.NoError(t, err) err = q.markJobDone(j1ID, jobComplete) assert.NoError(t, err) err = q.markJobDone(j2ID, jobFailed) assert.NoError(t, err) err = q.db.View(func(txn *badger.Txn) error { // check that job 1 is not in the inprogress queue anymore _, err := txn.Get([]byte("q:inprogress:" + jIDString(j1ID))) assert.EqualError(t, err, badger.ErrKeyNotFound.Error()) // check that job 2 is not in the inprogress queue anymore _, err = txn.Get([]byte("q:inprogress:" + jIDString(j2ID))) assert.EqualError(t, err, badger.ErrKeyNotFound.Error()) // check that job 1 is now in the complete queue completeJob, err := getJobForKey(txn, []byte("q:complete:"+jIDString(j1ID))) assert.NoError(t, err) assert.Equal(t, j1ID, completeJob.ID) assert.Equal(t, j1Name, completeJob.Name) failedJob, err := getJobForKey(txn, []byte("q:failed:"+jIDString(j2ID))) assert.NoError(t, err) assert.Equal(t, j2ID, failedJob.ID) assert.Equal(t, j2Name, failedJob.Name) return nil }) assert.NoError(t, err) // check random job id is not in queue error err = q.markJobDone(uint64(4151231), jobComplete) assert.EqualError(t, err, "Key not found") // check moving job to pending error err = q.markJobDone(j2ID, jobPending) assert.EqualError(t, err, "Can only move to Complete or Failed Status") } func TestBlero_moveItemErr(t *testing.T) { txn := &badger.Txn{} err := moveItem(txn, nil, nil, nil) assert.EqualError(t, err, "No sets or deletes are allowed in a read-only transaction") } func TestBlero_decodeJobErr(t *testing.T) { _, err := decodeJob(nil) assert.EqualError(t, err, "EOF") }