Repository: pote/disc Branch: master Commit: ee5f7b5531a2 Files: 26 Total size: 37.2 KB Directory structure: gitextract_y33c6rsn/ ├── .gems ├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── README.md ├── bin/ │ └── disc ├── disc.gemspec ├── examples/ │ ├── disc_init.rb │ ├── echoer.rb │ ├── failer.rb │ ├── greeter.rb │ ├── identifier.rb │ └── returner.rb ├── lib/ │ ├── active_job/ │ │ └── queue_adapters/ │ │ └── disc_adapter.rb │ ├── disc/ │ │ ├── errors.rb │ │ ├── job.rb │ │ ├── testing.rb │ │ ├── version.rb │ │ └── worker.rb │ └── disc.rb └── test/ ├── disc_test.rb ├── job_test.rb ├── process_test.rb └── testing_test.rb ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gems ================================================ cutest:1.2.2 byebug:5.0.0 disque:0.0.6 celluloid:0.17.0 ================================================ FILE: .gitignore ================================================ *.gem nodes.conf .deps ================================================ FILE: .travis.yml ================================================ language: ruby rvm: - 2.2 sudo: false cache: bundler: false directories: - disque - gems before_install: - test -d disque/.git || git clone -n https://github.com/antirez/disque.git - cd disque && git fetch origin && git checkout -f origin/master && make && cd .. install: - export GEM_HOME=$PWD/gems/$RUBY_VERSION - export GEM_PATH=$GEM_HOME:$GEM_PATH - export PATH=$GEM_HOME/bin:$PWD/disque/src:$PATH - cat .gems* | xargs gem install before_script: disque-server --daemonize yes script: make test ================================================ FILE: CONTRIBUTING.md ================================================ ### The Soveran Contribution Guidelines. (taken from some project at https://github.com/soveran) This code tries to solve a particular problem with a very simple implementation. We try to keep the code to a minimum while making it as clear as possible. The design is very likely finished, and if some feature is missing it is possible that it was left out on purpose. That said, new usage patterns may arise, and when that happens we are ready to adapt if necessary. A good first step for contributing is to meet us on IRC and discuss ideas. We spend a lot of time on #lesscode at freenode, always ready to talk about code and simplicity. If connecting to IRC is not an option, you can create an issue explaining the proposed change and a use case. We pay a lot of attention to use cases, because our goal is to keep the code base simple. Usually the result of a conversation is the creation of a different tool. Please don't start the conversation with a pull request. The code should come at last, and even though it may help to convey an idea, more often than not it draws the attention to a particular implementation. ================================================ FILE: LICENSE ================================================ The MIT License Copyright (c) 2015 Pablo Astigarraga | pote Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: Makefile ================================================ ifndef GEM_HOME $(error GEM_HOME not set.) endif PACKAGES := disc VERSION_FILE := lib/disc/version.rb DEPS := ${GEM_HOME}/installed VERSION := $(shell sed -ne '/.*VERSION *= *"\(.*\)".*/s//\1/p' <$(VERSION_FILE)) GEMS := $(addprefix pkg/, $(addsuffix -$(VERSION).gem, $(PACKAGES))) export RUBYLIB := lib:test:$(RUBYLIB) all: test $(GEMS) console: $(DEPS) irb -r disc test: $(DEPS) cutest ./test/**/*_test.rb clean: rm pkg/*.gem release: $(GEMS) git tag v$(VERSION) git push --tags for gem in $^; do gem push $$gem; done pkg/%-$(VERSION).gem: %.gemspec $(VERSION_FILE) | pkg gem build $< mv $(@F) pkg/ $(DEPS): $(GEM_HOME) .gems cat .gems | xargs gem install && touch $(GEM_HOME)/installed pkg $(GEM_HOME): mkdir -p $@ .PHONY: all test release clean ================================================ FILE: README.md ================================================ # Disc [![Build Status](https://travis-ci.org/pote/disc.svg?branch=master)](https://travis-ci.org/pote/disc) Disc fills the gap between your Ruby service objects and [antirez](http://antirez.com/)'s wonderful [Disque](https://github.com/antirez/disque) backend. ![Disc Wars!](https://cloud.githubusercontent.com/assets/437/8634016/b63ee0f8-27e6-11e5-9a78-51921bd32c88.jpg) ## Basic Usage 1. Install the gem ```bash $ gem install disc ``` 2. Write your jobs ```ruby require 'disc' class CreateGameGrid include Disc::Job disc queue: 'urgent' def perform(type) # perform rather lengthy operations here. end end ``` 3. Enqueue them to perform them asynchronously ```ruby CreateGameGrid.enqueue('light_cycle') ``` 4. Create a file that requires anything needed for your jobs to run ```ruby # disc_init.rb # Require here anything that your application needs to run, # like ORMs and your models, database configuration, etc. Dir['./jobs/**/*.rb'].each { |job| require job } ``` 5. Run as many Disc Worker processes as you wish, requiring your `disc_init.rb` file ```bash $ QUEUES=urgent,default disc -r ./disc_init.rb ``` 4. Or enqueue them to be performed at some time in the future, or on a queue other than it's default. ```ruby CreateGameGrid.enqueue( 'disc_arena', at: DateTime.new(2020, 12, 31), queue: 'not_so_important' ) ``` ## Disc Configuration Disc takes its configuration from environment variables. | ENV Variable | Default Value | Description |:------------------:|:-----------------|:------------| | `QUEUES` | 'default' | The list of queues that `Disc::Worker` will listen to, it can be a single queue name or a list of comma-separated queues | | `DISC_CONCURRENCY` | '25' | Amount of threads to spawn when Celluloid is available. | | `DISQUE_NODES` | 'localhost:7711' | This is the list of Disque servers to connect to, it can be a single node or a list of comma-separated nodes | | `DISQUE_AUTH` | '' | Authorization credentials for Disque. | | `DISQUE_TIMEOUT` | '100' | Time in milliseconds that the client will wait for the Disque server to acknowledge and replicate a job | | `DISQUE_CYCLE` | '1000' | The client keeps track of which nodes are providing more jobs, after the amount of operations specified in cycle it tries to connect to the preferred node. | ## Disc Jobs `Disc::Job` is a module you can include in your Ruby classes, this allows a Disc worker process to execute the code in them by adding a class method (`#enqueue`) with the following signature: ```Ruby def enqueue(arguments, at: nil, queue: nil, **options) end ``` Signature documentation follows: ```ruby ## Disc's `#enqueue` is the main user-facing method of a Disc job, it # enqueues a job with a given set of arguments in Disque, so it can be # picked up by a Disc worker process. # ## Parameters: # ## `arguments` - an optional array of arguments with which to execute # the job's `perform` method. # # `at` - an optional named parameter specifying a moment in the # future in which to run the job, must respond to # `#to_time`. # ## `queue` - an optional named parameter specifying the name of the # queue in which to store the job, defaults to the class # Disc queue or to 'default' if no Disc queue is specified # in the class. # ## `**options` - an optional hash of options to forward internally to # [disque-rb](https://github.com/soveran/disque-rb)'s # `#push` method, valid options are: # ## `replicate: ` - specifies the number of nodes the job should # be replicated to. # ### `delay: ` - specifies a delay time in seconds for the job # to be delivered to a Disc worker, it is ignored # if using the `at` parameter. # ### `ttl: ` - specifies the job's time to live in seconds: # after this time, the job is deleted even if # it was not successfully delivered. If not # specified, the default TTL is one day. # ### `maxlen: ` - specifies that if there are already # messages queued for the specified queue name, # the message is refused. # ### `async: true` - asks the server to let the command return ASAP # and replicate the job to other nodes in the background. # # ### CAVEATS # ## For convenience, any object can be passed as the `arguments` parameter, # `Array()` will be used internally to preserve the array structure. # ## The `arguments` parameter is serialized for storage using `Disc.serialize` # and Disc workers picking it up use `Disc.deserialize` on it, both methods # use standard library json but can be overriden by the user # ``` You can see [Disque's ADDJOB documentation](https://github.com/antirez/disque#addjob-queue_name-job-ms-timeout-replicate-count-delay-sec-retry-sec-ttl-sec-maxlen-count-async) for more details When a Disc worker process is assigned a job, it will create a new intance of the job's class and execute the `perform` method with whatever arguments were previously passed to `#enqueue`. Example: ```ruby class ComplexJob include Disc::Job disc queue: 'urgent' def perform(first_parameter, second_parameter) # do things... end end ComplexJob.enqueue(['first argument', { second: 'argument' }]) ``` ### Managing Jobs Disc jobs can be managed by knowing their disque ID, this id is returned by the `#enqueue` method so you can control the job or query it's state from your application code. ```ruby Echoer.enqueue('test') #=> "DIa18101491133639148a574eb30cd2e12f25dcf8805a0SQ" ``` The disque ID is also available from within the context of an executing job, you can access it via `self.disque_id` if you wish to do things like notify Disque that a long-running job is still being executed. ```ruby class LongJob include Disc::Job def perform(first_parameter, second_parameter) # Do things that take a while. Disc.disque.call('WORKING', self.disque_id) # Do more things that take a while. end end ``` #### Job Status After a job is enqueued, you can check it's current status like so: ```ruby Echoer.enqueue('test') #=> "DIa18101491133639148a574eb30cd2e12f25dcf8805a0SQ" Disc["DIa18101491133639148a574eb30cd2e12f25dcf8805a0SQ"] #=> { "arguments"=>["test"], "class"=>"Echoer", "id"=>"DIa18101491133639148a574eb30cd2e12f25dcf8805a0SQ", "queue"=>"test", "state"=>"queued", "repl"=>1, "ttl"=>86391, "ctime"=>1462488116652000000, "delay"=>0, "retry"=>8640, "nacks"=>0, "additional-deliveries"=>0, "nodes-delivered"=>["a18101496d562e412a459c6b114561efe95c57cc"], "nodes-confirmed"=>[], "next-requeue-within"=>8630995, "next-awake-within"=>8630495, "body"=>"{\"class\":\"Echoer\",\"arguments\":[\"test\"]}" } ``` This information might vary, as it's retreived from Disque via the [`SHOW`](https://github.com/antirez/disque#show-job-id) command, only `arguments` and `class` are filled in by Disc, which are added by using `Disc.deserialize` on the `body` value. #### Do everything Disque can Access to the disque ID allows us to leverage the Disque API to manage the job, you can execute Disque commands via the `Disc.disque.call()` method, see [the Disque API](https://github.com/antirez/disque#main-api) to see all the commands available. ### Job Serialization Job information (their arguments, and class) need to be serialized in order to be stored in Disque, to this end Disc uses the `Disc.serialize` and `Disc.deserialize` methods. By default, these methods use the Ruby standard library json implementation in order to serialize and deserialize job data, this has a few implications: 1. Arguments passed to a job's `#enqueue` method need to be serializable by `Disc.serialize` and parsed back by `Disc.deserialize`, so by default you can't pass complex Ruby objects like a `user` model, instead, pass `user.id`, and use that from your job code. 2. You can override `Disc.serialize` and `Disc.deserialize` to use a different JSON implementation, or MessagePack, or whatever else you wish. ## Error handling When a job raises an exception, `Disc.on_error` is invoked with the error and the job data. By default, this method prints the error to standard error, but you can override it to report the error to your favorite error aggregator. ``` ruby # On disc_init.rb def Disc.on_error(exception, job) # ... report the error end Dir["./jobs/**/*.rb"].each { |job| require job } ``` ### Job Definition The error handler function gets the data of the current job as a Hash, that has the following schema. | | | |:-------------:|:------------------------------------------------------| | `'class'` | (String) The Job class. | | `'arguments'` | (Array) The arguments passed to perform. | | `'queue'` | (String) The queue from which this job was picked up. | | `'disque_id'` | (String) Disque's job ID. | ## Testing modes Disc includes a testing mode, so you can run your test suite without a need to run a Disque server. ### Enqueue mode By default, Disc places your jobs in an in-memory hash, with each queue being a key in the hash and values being an array. ```ruby require 'disc' require 'disc/testing' require_relative 'examples/returner' Disc.enqueue! #=> This is the default mode for disc/testing so you don't need to specify it, # you can use this method to go back to the enqueue mode if you switch it. Returner.enqueue('test argument') Disc.queues #=> {"default"=>[{:arguments=>["test argument"], :class=>"Returner", :options=>{}}]} Returner.enqueue('another test') #=> => {"default"=>[{:arguments=>["test argument"], :class=>"Returner", :options=>{}}, {:arguments=>["another test"], :class=>"Returner", :options=>{}}]} ``` You can still flush the queues just as you would running on regular mode. ```ruby Disc.flush Disc.queues #=> {} ``` ### Inline mode You also have the option for Disc to execute jobs immediately when `#enqueue` is called. ```ruby require 'disc' require 'disc/testing' require_relative 'examples/returner' Disc.inline! Returner.enqueue('test argument') #=> 'test argument' ``` ## [Optional] Celluloid integration Disc workers run just fine on their own, but if you happen to be using [Celluloid](https://github.com/celluloid/celluloid) you might want Disc to take advantage of it and spawn multiple worker threads per process, doing this is trivial! Just require Celluloid before your init file: ```bash $ QUEUES=urgent,default disc -r celluloid/current -r ./disc_init.rb ``` Whenever Disc detects that Celluloid is available it will use it to spawn a number of threads equal to the `DISC_CONCURRENCY` environment variable, or 25 by default. ## [Optional] Rails and ActiveJob integration You can use Disc easily in Rails without any more hassle, but if you'd like to use it via [ActiveJob](http://edgeguides.rubyonrails.org/active_job_basics.html) you can use the adapter included in this gem. ```ruby # Gemfile gem 'disc' # config/application.rb module YourApp class Application < Rails::Application require 'active_job/queue_adapters/disc_adapter' config.active_job.queue_adapter = :disc end end # app/jobs/clu_job.rb class CluJob < ActiveJob::Base queue_as :urgent def perform(*args) # Try to take over The Grid here... end end # disc_init.rb require ::File.expand_path('../config/environment', __FILE__) # Wherever you want CluJob.perform_later(a_bunch_of_arguments) ``` Disc is run in the exact same way, for this example it'd be: ```bash $ QUEUES=urgent disc -r ./disc_init.rb ``` ## Similar Projects If you want to use Disque but Disc isn't cutting it for you then you should take a look at [Havanna](https://github.com/djanowski/havanna), a project by my friend [@djanowski](https://twitter.com/djanowski). ## License The code is released under an MIT license. See the [LICENSE](./LICENSE) file for more information. ## Acknowledgements * To [@foca](https://github.com/foca) for helping me ship a quality thing and putting up with my constant whining. * To [@antirez](https://github.com/antirez) for Redis, Disque, and his refreshing way of programming wonderful tools. * To [@soveran](https://github.com/soveran) for pushing me to work on this and publishing gems that keep me enjoying ruby. * To [all contributors](https://github.com/pote/disc/graphs/contributors) ## Sponsorship This open source tool is proudly sponsored by [13Floor](http://13Floor.org) ![13Floor](./13Floor-circulo-1.png) ================================================ FILE: bin/disc ================================================ #!/usr/bin/env ruby if ARGV.empty? $stdout.puts('Usage: disc -r FILE') exit(1) end stop = proc do if defined?(Disc) Disc::Worker.stop else exit 0 end end trap(:INT, &stop) trap(:TERM, &stop) require 'clap' require_relative '../lib/disc' require_relative '../lib/disc/worker' Clap.run ARGV, "-r" => lambda { |file| require file } if defined?(Celluloid) $stdout.puts( "[Notice] Disc running in celluloid mode! Current DISC_CONCURRENCY is\ #{ Integer(ENV.fetch('DISC_CONCURRENCY', '25')) }." ) Disc::Worker.send(:include, Celluloid) if defined?(Celluloid::SupervisionGroup) # Deprecated as of Celluloid 0.17, but still supported via "backported mode" class Disc::WorkerGroup < Celluloid::SupervisionGroup pool Disc::Worker, size: Integer(ENV.fetch('DISC_CONCURRENCY', '25')), as: :worker_pool, args: [{ run: true }] end Disc::WorkerGroup.run else Disc::Worker.pool( size: Integer(ENV.fetch('DISC_CONCURRENCY', '25')), args: [{ run: true }] ) end else $stdout.puts("[Notice] Disc running in non-threaded mode") Disc::Worker.run end ================================================ FILE: disc.gemspec ================================================ require_relative "lib/disc/version" Gem::Specification.new do |s| s.name = 'disc' s.version = Disc::VERSION s.summary = 'A simple and powerful Disque job implementation' s.description = 'Easily define and run background jobs using Disque' s.authors = ['pote'] s.email = ['pote@tardis.com.uy'] s.homepage = 'https://github.com/pote/disc' s.license = 'MIT' s.files = `git ls-files`.split("\n") s.executables.push('disc') s.add_dependency('disque', '~> 0.0.6') s.add_dependency('clap', '~> 1.0') end ================================================ FILE: examples/disc_init.rb ================================================ $:.unshift('lib') Dir.glob("./examples/**/*.rb") { |f| require f } ================================================ FILE: examples/echoer.rb ================================================ require 'disc' class Echoer include Disc::Job disc queue: 'test' def perform(first, second, third) puts "First: #{ first }, Second: #{ second }, Third: #{ third }" end end ================================================ FILE: examples/failer.rb ================================================ require 'disc' def Disc.on_error(exception, job) $stdout.puts('') $stdout.puts(exception.message) $stdout.puts(job) end class Failer include Disc::Job disc queue: 'test' def perform(string) raise string end end ================================================ FILE: examples/greeter.rb ================================================ require 'disc' class Greeter include Disc::Job disc queue: 'test_medium' def perform(string) $stdout.puts(string) end end ================================================ FILE: examples/identifier.rb ================================================ require 'disc' class Identifier include Disc::Job disc queue: 'test' def perform $stdout.puts("Working with Disque ID: #{ self.disque_id }") end end ================================================ FILE: examples/returner.rb ================================================ class Returner include Disc::Job def perform(argument) return argument end end ================================================ FILE: lib/active_job/queue_adapters/disc_adapter.rb ================================================ require 'date' require 'msgpack' require 'disc/worker' module ActiveJob module QueueAdapters class DiscAdapter def self.enqueue(job) enqueue_at(job, nil) end def self.enqueue_at(job, timestamp) Disc.disque.push( job.queue_name, Disc.serialize({ class: job.class.name, arguments: job.arguments }), Disc.disque_timeout, delay: timestamp.nil? ? nil : (timestamp.to_time.to_i - DateTime.now.to_time.to_i) ) end end end end ================================================ FILE: lib/disc/errors.rb ================================================ class Disc class Error < StandardError; end class UnknownJobClassError < Error; end class NonParsableJobError < Error; end class NonJobClassError < Error; end end ================================================ FILE: lib/disc/job.rb ================================================ module Disc::Job attr_accessor :disque_id def self.included(base) base.extend(ClassMethods) end module ClassMethods def disque defined?(@disque) ? @disque : Disc.disque end def disque=(disque) @disque = disque end def disc(queue: nil, **options) @queue = queue @disc_options = options end def disc_options @disc_options ||= {} end def queue @queue || Disc.default_queue end def perform(arguments) self.new.perform(*arguments) end ## Disc's `#enqueue` is the main user-facing method of a Disc job, it # enqueues a job with a given set of arguments in Disque, so it can be # picked up by a Disc worker process. # ## Parameters: # ## `arguments` - an optional array of arguments with which to execute # the job's #perform method. # ## `at` - an optional named parameter specifying a moment in the # future in which to run the job, must respond to # `#to_time`. # ## `queue` - an optional named parameter specifying the name of the # queue in which to store the job, defaults to the class # Disc queue or to 'default' if no Disc queue is specified # in the class. # ## `**options` - an optional hash of options to forward internally to # [disque-rb](https://github.com/soveran/disque-rb)'s # `#push` method, valid options are: # ## `replicate: ` - specifies the number of nodes the job should # be replicated to. # ### `delay: ` - specifies a delay time in seconds for the job # to be delivered to a Disc worker, it is ignored # if using the `at` parameter. # ### `ttl: ` - specifies the job's time to live in seconds: # after this time, the job is deleted even if # it was not successfully delivered. If not # specified, the default TTL is one day. # ### `maxlen: ` - specifies that if there are already # messages queued for the specified queue name, # the message is refused. # ### `async: true` - asks the server to let the command return ASAP # and replicate the job to other nodes in the background. # # ### CAVEATS # ## For convenience, any object can be passed as the `arguments` parameter, # `Array()` will be used internally to preserve the array structure. # ## The `arguments` parameter is serialized for storage using `Disc.serialize` # and Disc workers picking it up use `Disc.deserialize` on it, both methods # use standard library json but can be overriden by the user # def enqueue(args = [], at: nil, queue: nil, **options) options = disc_options.merge(options).tap do |opt| opt[:delay] = at.to_time.to_i - DateTime.now.to_time.to_i unless at.nil? end disque.push( queue || self.queue, Disc.serialize({ class: self.name, arguments: Array(args) }), Disc.disque_timeout, options ) end end end ================================================ FILE: lib/disc/testing.rb ================================================ class Disc def self.queues @queues ||= {} end def self.testing_mode @testing_mode ||= 'enqueue' end def self.enqueue! @testing_mode = 'enqueue' end def self.inline! @testing_mode = 'inline' end def self.flush @queues = {} end def self.qlen(queue) return 0 if Disc.queues[queue].nil? Disc.queues[queue].length end def self.enqueue(klass, arguments, at: nil, queue: nil, **options) job_attrs = { arguments: arguments, class: klass, options: options } if queues[queue].nil? queues[queue] = [job_attrs] else queues[queue] << job_attrs end job_attrs end end module Disc::Job::ClassMethods def enqueue(args = [], at: nil, queue: nil, **options) case Disc.testing_mode when 'enqueue' Disc.enqueue( self.name, Array(args), queue: queue || self.queue, at: at, **options) when 'inline' self.perform(*args) else raise "Unknown Disc testing mode, this shouldn't happen" end end end ================================================ FILE: lib/disc/version.rb ================================================ class Disc VERSION = "0.2.0" end ================================================ FILE: lib/disc/worker.rb ================================================ require 'disc' class Disc::Worker attr_reader :disque, :queues, :timeout, :count def self.current @current ||= new end def self.run current.run end def self.stop current.stop end def initialize(options = {}) @disque = options.fetch(:disque, Disc.disque) @queues = options.fetch( :queues, ENV.fetch('QUEUES', Disc.default_queue) ).split(',') @count = Integer( options.fetch( :count, ENV.fetch('DISQUE_COUNT', '1') ) ) @timeout = Integer( options.fetch( :timeout, ENV.fetch('DISQUE_TIMEOUT', '2000') ) ) self.run if options[:run] self end def stop @stop = true end def run $stdout.puts("Disc::Worker listening in #{queues}") loop do jobs = disque.fetch(from: queues, timeout: timeout, count: count) Array(jobs).each do |queue, msgid, serialized_job| begin job_instance, arguments = Disc.load_job(serialized_job, msgid) job_instance.perform(*arguments) disque.call('ACKJOB', msgid) $stdout.puts("Completed #{ job_instance.class.name } id #{ msgid }") rescue => err Disc.on_error(err, { disque_id: msgid, queue: queue, class: defined?(job_instance) ? job_instance.class.name : '', arguments: defined?(arguments) ? arguments : [] }) end end break if @stop end ensure disque.quit end end ================================================ FILE: lib/disc.rb ================================================ require 'date' require 'disque' require 'json' class Disc def self.disque @disque ||= Disque.new( ENV.fetch('DISQUE_NODES', 'localhost:7711'), auth: ENV.fetch('DISQUE_AUTH', nil), cycle: Integer(ENV.fetch('DISQUE_CYCLE', '1000')) ) end def self.disque=(disque) @disque = disque end def self.disque_timeout @disque_timeout ||= 100 end def self.disque_timeout=(timeout) @disque_timeout = timeout end def self.default_queue @default_queue ||= 'default' end def self.default_queue=(queue) @default_queue = queue end def self.qlen(queue) disque.call('QLEN', queue) end def self.flush Disc.disque.call('DEBUG', 'FLUSHALL') end def self.on_error(exception, job) $stderr.puts exception end def self.serialize(args) JSON.dump(args) end def self.deserialize(data) JSON.parse(data) end def self.[](disque_id) job_data = disque.call("SHOW", disque_id) return nil if job_data.nil? job_data = Hash[*job_data] job_data['arguments'] = Disc.deserialize(job_data['body'])['arguments'] job_data['class'] = Disc.deserialize(job_data['body'])['class'] job_data end ## Receives: # # A string containing data serialized by `Disc.serialize` # ## Returns: # # An array containing: # # * An instance of the given job class # * An array of arguments to pass to the job's `#perorm` class. # def self.load_job(serialized_job, disque_id = nil) begin job_data = Disc.deserialize(serialized_job) rescue => err raise Disc::NonParsableJobError.new(err) end begin job_class = Object.const_get(job_data['class']) rescue => err raise Disc::UnknownJobClassError.new(err) end begin job_instance = job_class.new job_instance.disque_id = disque_id rescue => err raise Disc::NonJobClassError.new(err) end return [job_instance, job_data['arguments']] end end require_relative 'disc/errors' require_relative 'disc/job' require_relative 'disc/version' ================================================ FILE: test/disc_test.rb ================================================ require 'cutest' require 'disc' require_relative '../examples/echoer' prepare do Disc.disque_timeout = 1 # 1ms so we don't wait at all. Disc.flush end scope do test 'Disc should be able to communicate with Disque' do assert !Disc.disque.nil? assert_equal 'PONG', Disc.disque.call('PING') end test 'we get easy access to the job via the job id with Disc[job_id]' do job_id = Echoer.enqueue(['one argument', { random: 'data' }, 3]) job_data = Disc[job_id] assert_equal 'Echoer', job_data['class'] assert_equal 'queued', job_data['state'] assert_equal 3, job_data['arguments'].count assert_equal 'one argument', job_data['arguments'].first end test 'we can query the length of a given queue with Disc.qlen' do Echoer.enqueue(['one argument', { random: 'data' }, 3]) assert_equal 1, Disc.qlen(Echoer.queue) end test 'Disc.flush deletes everything in the queue' do Echoer.enqueue(['one argument', { random: 'data' }, 3]) Disc.flush assert_equal 0, Disc.qlen(Echoer.queue) end test 'Disc.load_job returns a job instance and arguments' do serialized_job = Disc.serialize( { class: 'Echoer', arguments: ['one argument', { random: 'data' }, 3] } ) job_instance, arguments = Disc.load_job(serialized_job) assert_equal Echoer, job_instance.class assert arguments.is_a?(Array) assert_equal 3, arguments.count assert_equal 'one argument', arguments.first end test 'Disc.load_job raises appropriate errors ' do begin job_instance, arguments = Disc.load_job('gibberish') assert_equal 'Should not reach this point', false rescue => err assert err.is_a?(Disc::Error) assert err.is_a?(Disc::NonParsableJobError) end serialized_job = Disc.serialize( { class: 'NonExistantDiscJobClass', arguments: [] } ) begin job_instance, arguments = Disc.load_job(serialized_job) assert_equal 'Should not reach this point', false rescue => err assert err.is_a?(Disc::Error) assert err.is_a?(Disc::UnknownJobClassError) end end end ================================================ FILE: test/job_test.rb ================================================ require 'cutest' require 'disc' require_relative '../examples/echoer' prepare do Disc.disque_timeout = 1 # 1ms so we don't wait at all. Disc.flush end scope do test 'jobs are enqueued to the correct Disque queue with appropriate parameters and class' do job_id = Echoer.enqueue(['one argument', { random: 'data' }, 3]) jobs = Array(Disc.disque.fetch(from: ['test'], timeout: Disc.disque_timeout, count: 1)) assert jobs.any? assert_equal 1, jobs.count jobs.first.tap do |queue, id, serialized_job| job = Disc.deserialize(serialized_job) assert job.has_key?('class') assert job.has_key?('arguments') assert_equal 'Echoer', job['class'] assert_equal job_id, id args = job['arguments'] assert_equal 3, args.size assert_equal 'one argument', args[0] assert_equal({ 'random' => 'data' }, args[1]) assert_equal(3, args[2]) end end test 'enqueue at timestamp behaves properly' do job_id = Echoer.enqueue(['one argument', { random: 'data' }, 3], at: Time.now + 1) jobs = Array(Disc.disque.fetch(from: ['test'], timeout: Disc.disque_timeout, count: 1)) assert jobs.empty? sleep 0.5 jobs = Array(Disc.disque.fetch(from: ['test'], timeout: Disc.disque_timeout, count: 1)) assert jobs.empty? sleep 0.5 jobs = Array(Disc.disque.fetch(from: ['test'], timeout: Disc.disque_timeout, count: 1)) assert jobs.any? assert_equal 1, jobs.size jobs.first.tap do |queue, id, serialized_job| assert_equal 'test', queue assert_equal job_id, id job = Disc.deserialize(serialized_job) assert job.has_key?('class') assert job.has_key?('arguments') assert_equal 'Echoer', job['class'] assert_equal 3, job['arguments'].size end end test 'enqueue supports replicate' do error = Echoer.enqueue(['one argument', { random: 'data' }, 3], replicate: 100) rescue $! assert_equal RuntimeError, error.class assert_equal "NOREPL Not enough reachable nodes for the requested replication level", error.message end test 'enqueue supports delay' do job_instance = Echoer.enqueue(['one argument', { random: 'data' }, 3], delay: 2) jobs = Array(Disc.disque.fetch(from: ['test'], timeout: Disc.disque_timeout, count: 1)) assert jobs.empty? sleep 1 jobs = Array(Disc.disque.fetch(from: ['test'], timeout: Disc.disque_timeout, count: 1)) assert jobs.empty? sleep 2 jobs = Array(Disc.disque.fetch(from: ['test'], timeout: Disc.disque_timeout, count: 1)) assert jobs.any? assert_equal 1, jobs.size end test 'enqueue supports retry' do job_instance = Echoer.enqueue(['one argument', { random: 'data' }, 3], retry: 1) jobs = Array(Disc.disque.fetch(from: ['test'], timeout: Disc.disque_timeout, count: 1)) assert jobs.any? assert_equal 1, jobs.size sleep 1.5 jobs = Array(Disc.disque.fetch(from: ['test'], timeout: Disc.disque_timeout, count: 1)) assert jobs.any? assert_equal 1, jobs.size end test 'enqueue supports ttl' do job_instance = Echoer.enqueue(['one argument', { random: 'data' }, 3], ttl: 1) sleep 1.5 jobs = Array(Disc.disque.fetch(from: ['test'], timeout: Disc.disque_timeout, count: 1)) assert jobs.empty? end test 'enqueue supports maxlen' do Echoer.enqueue(['one argument', { random: 'data' }, 3], maxlen: 1) error = Echoer.enqueue(['one argument', { random: 'data' }, 3], maxlen: 1) rescue $! assert_equal RuntimeError, error.class assert_equal "MAXLEN Queue is already longer than the specified MAXLEN count", error.message end test 'enqueue supports async' do job_instance = Echoer.enqueue(['one argument', { random: 'data' }, 3], async: true) sleep 1 # async is too fast to reliably assert an empty queue, let's wait instead jobs = Array(Disc.disque.fetch(from: ['test'], timeout: Disc.disque_timeout, count: 1)) assert jobs.any? assert_equal 1, jobs.size end end ================================================ FILE: test/process_test.rb ================================================ require 'cutest' require 'disc' require 'pty' require 'timeout' require_relative '../examples/echoer' require_relative '../examples/failer' require_relative '../examples/identifier' prepare do Disc.disque_timeout = 1 # 1ms so we don't wait at all. Disc.flush end scope do # Runs a given command, yielding the stdout (as an IO) and the PID (a String). # Makes sure the process finishes after the block runs. def run(command) out, _, pid = PTY.spawn(command) yield out, pid ensure Process.kill("KILL", pid) sleep 0.1 # Make sure we give it time to finish. end # Checks whether a process is running. def is_running?(pid) Process.getpgid(pid) true rescue Errno::ESRCH false end test 'Disc.on_error will catch unhandled exceptions and keep disc alive' do failer = Failer.enqueue('this can only end positively') run('QUEUES=test ruby -Ilib bin/disc -r ./examples/failer') do |cout, pid| output = Timeout.timeout(1) { cout.take(5) } assert output.grep(//).any? assert output.grep(/this can only end positively/).any? assert output.grep(/Failer/).any? assert is_running?(pid) assert_equal 0, Disc.qlen(Failer.queue) end end test 'jobs are executed' do Echoer.enqueue(['one argument', { random: 'data' }, 3]) run('QUEUES=test ruby -Ilib bin/disc -r ./examples/echoer') do |cout, pid| output = Timeout.timeout(1) { cout.take(3) } assert output.grep(/First: one argument, Second: {"random"=>"data"}, Third: 3/).any? assert_equal 0, Disc.qlen(Echoer.queue) end end test 'running jobs have access to their Disque job ID' do disque_id = Identifier.enqueue run('QUEUES=test ruby -Ilib bin/disc -r ./examples/identifier') do |cout, pid| output = Timeout.timeout(1) { cout.take(3) } assert output.grep(/Working with Disque ID: #{ disque_id }/).any? assert_equal 0, Disc.qlen(Identifier.queue) end end end ================================================ FILE: test/testing_test.rb ================================================ # Yo dawg I put some testing in your testing so you can test while you test. require 'disc' require 'disc/testing' require_relative '../examples/echoer' require_relative '../examples/returner' prepare do Disc.disque_timeout = 1 # 1ms so we don't wait at all. Disc.enqueue! Disc.flush end scope do test "testing mode should not enqueue jobs into Disque" do Echoer.enqueue(['one argument', { random: 'data' }, 3]) assert_equal 0, Disc.disque.call('QLEN', 'test') assert_equal 1, Disc.qlen('test') # Flush should still work though Disc.flush assert_equal 0, Disc.qlen('test') end test "simple enqueuing should work " do Echoer.enqueue('one thing') assert_equal 1, Disc.queues['test'].count assert Disc.queues['test'].first.has_key?(:arguments) assert_equal 1, Disc.queues['test'].first[:arguments].count assert_equal 'one thing', Disc.queues['test'].first[:arguments].first end test "testing mode enqueue jobs into an in-memory list by default" do Echoer.enqueue(['one argument', { random: 'data' }, 3]) assert_equal 1, Disc.queues['test'].count assert Disc.queues['test'].first.has_key?(:arguments) assert_equal 3, Disc.queues['test'].first[:arguments].count assert_equal 'one argument', Disc.queues['test'].first[:arguments].first assert_equal 'Echoer', Disc.queues['test'].first[:class] end test "testing mode enqueue jobs into an in-memory list by default" do Echoer.enqueue(['one argument', { random: 'data' }, 3]) assert_equal 'one argument', Disc.queues['test'].first[:arguments].first end test "ability to run jobs inline" do Disc.inline! assert_equal 'this is an argument', Returner.enqueue('this is an argument') end end