Set up Delayed Job as the backend for Active Job

This persists jobs into the database, and uses locking to ensure that
workers from multiple machines avoid treading on each other.

Jobs can be run by using `bundle exec rake jobs:work`

Fixes #2015

diff --git a/Gemfile b/Gemfile
index 9ba2703134d5d9b9bf8e5c47904c188396a762e7..d60f1ba947bde44a591a0036042a2e493c82be8a 100644 (file)
--- a/Gemfile
+++ b/Gemfile
@@ -46,6 +46,7 @@ gem "image_optim_rails"
 # Load rails plugins
 gem "actionpack-page_caching"
 gem "composite_primary_keys", "~> 11.0.0"
+gem "delayed_job_active_record"
 gem "dynamic_form"
 gem "http_accept_language", "~> 2.0.0"
 gem "i18n-js", ">= 3.0.0"

diff --git a/Gemfile.lock b/Gemfile.lock
index 76a31e169e59a8c4aacd58ef738f2de056957302..9080f009e1e2929b1b833cdb5fe343114d0f9ce0 100644 (file)
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -99,6 +99,11 @@ GEM
     crass (1.0.4)
     dalli (2.7.8)
     debug_inspector (0.0.3)
+    delayed_job (4.1.5)
+      activesupport (>= 3.0, < 5.3)
+    delayed_job_active_record (4.1.3)
+      activerecord (>= 3.0, < 5.3)
+      delayed_job (>= 3.0, < 5)
     docile (1.3.1)
     dynamic_form (1.1.4)
     erubi (1.7.1)
@@ -393,6 +398,7 @@ DEPENDENCIES
   composite_primary_keys (~> 11.0.0)
   coveralls
   dalli
+  delayed_job_active_record
   dynamic_form
   factory_bot_rails
   faraday

diff --git a/config/application.rb b/config/application.rb
index 5139111cc8460dce6bb65d296ff8b84b2c82c37e..a5431f09f09bb94ea8eb32f46e0a2dfd1d02af00 100644 (file)
--- a/config/application.rb
+++ b/config/application.rb
@@ -51,5 +51,8 @@ module OpenStreetMap
       config.logstasher.logger_path = LOGSTASH_PATH
       config.logstasher.log_controller_parameters = true
     end
+
+    # Use DelayedJob as the persistent backend for ActiveJob
+    config.active_job.queue_adapter = :delayed_job
   end
 end

diff --git a/db/migrate/20181031113522_create_delayed_jobs.rb b/db/migrate/20181031113522_create_delayed_jobs.rb
new file mode 100644 (file)
index 0000000..69c6f1b
--- /dev/null
+++ b/db/migrate/20181031113522_create_delayed_jobs.rb
@@ -0,0 +1,22 @@
+class CreateDelayedJobs < ActiveRecord::Migration[5.2]
+  def self.up
+    create_table :delayed_jobs, :force => true do |table|
+      table.integer :priority, :default => 0, :null => false # Allows some jobs to jump to the front of the queue
+      table.integer :attempts, :default => 0, :null => false # Provides for retries, but still fail eventually.
+      table.text :handler,                    :null => false # YAML-encoded string of the object that will do work
+      table.text :last_error                                 # reason for last failure (See Note below)
+      table.datetime :run_at                                 # When to run. Could be Time.zone.now for immediately, or sometime in the future.
+      table.datetime :locked_at                              # Set when a client is working on this object
+      table.datetime :failed_at                              # Set when all retries have failed (actually, by default, the record is deleted instead)
+      table.string :locked_by                                # Who is working on this object (if locked)
+      table.string :queue                                    # The name of the queue this job is in
+      table.timestamps :null => true
+    end
+
+    add_index :delayed_jobs, [:priority, :run_at], :name => "delayed_jobs_priority"
+  end
+
+  def self.down
+    drop_table :delayed_jobs
+  end
+end

diff --git a/db/structure.sql b/db/structure.sql
index 48fedeba56b65d1247762f7a35eb29f9a5a1b98b..dcab302bb1c05f943ebe14319e2d7ada68ef7cb0 100644 (file)
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -494,6 +494,45 @@ CREATE SEQUENCE current_ways_id_seq
 ALTER SEQUENCE current_ways_id_seq OWNED BY current_ways.id;
 
 
+--
+-- Name: delayed_jobs; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE delayed_jobs (
+    id bigint NOT NULL,
+    priority integer DEFAULT 0 NOT NULL,
+    attempts integer DEFAULT 0 NOT NULL,
+    handler text NOT NULL,
+    last_error text,
+    run_at timestamp without time zone,
+    locked_at timestamp without time zone,
+    failed_at timestamp without time zone,
+    locked_by character varying,
+    queue character varying,
+    created_at timestamp without time zone,
+    updated_at timestamp without time zone
+);
+
+
+--
+-- Name: delayed_jobs_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE delayed_jobs_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: delayed_jobs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE delayed_jobs_id_seq OWNED BY delayed_jobs.id;
+
+
 --
 -- Name: diary_comments; Type: TABLE; Schema: public; Owner: -
 --
@@ -1365,6 +1404,13 @@ ALTER TABLE ONLY current_relations ALTER COLUMN id SET DEFAULT nextval('current_
 ALTER TABLE ONLY current_ways ALTER COLUMN id SET DEFAULT nextval('current_ways_id_seq'::regclass);
 
 
+--
+-- Name: delayed_jobs id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY delayed_jobs ALTER COLUMN id SET DEFAULT nextval('delayed_jobs_id_seq'::regclass);
+
+
 --
 -- Name: diary_comments id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -1595,6 +1641,14 @@ ALTER TABLE ONLY current_ways
     ADD CONSTRAINT current_ways_pkey PRIMARY KEY (id);
 
 
+--
+-- Name: delayed_jobs delayed_jobs_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY delayed_jobs
+    ADD CONSTRAINT delayed_jobs_pkey PRIMARY KEY (id);
+
+
 --
 -- Name: diary_comments diary_comments_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -1918,6 +1972,13 @@ CREATE INDEX current_way_nodes_node_idx ON current_way_nodes USING btree (node_i
 CREATE INDEX current_ways_timestamp_idx ON current_ways USING btree ("timestamp");
 
 
+--
+-- Name: delayed_jobs_priority; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX delayed_jobs_priority ON delayed_jobs USING btree (priority, run_at);
+
+
 --
 -- Name: diary_comment_user_id_created_at_index; Type: INDEX; Schema: public; Owner: -
 --
@@ -2870,6 +2931,7 @@ INSERT INTO "schema_migrations" (version) VALUES
 ('20161011010929'),
 ('20170222134109'),
 ('20180204153242'),
+('20181031113522'),
 ('21'),
 ('22'),
 ('23'),