From 8704671395d7a4984cdf1e9cdc300c15681f8c5a Mon Sep 17 00:00:00 2001 From: Andrew Tomaka Date: Sun, 8 Sep 2024 21:10:53 -0400 Subject: [PATCH 1/2] Upgrade to 0.6.0 --- Gemfile | 2 +- Gemfile.lock | 12 ++++++------ ...9011042_add_name_to_processes.solid_queue.rb | 6 ++++++ ...0909011043_make_name_not_null.solid_queue.rb | 17 +++++++++++++++++ ...ring_tasks_static_to_not_null.solid_queue.rb | 6 ++++++ db/schema.rb | 6 ++++-- 6 files changed, 40 insertions(+), 9 deletions(-) create mode 100644 db/migrate/20240909011042_add_name_to_processes.solid_queue.rb create mode 100644 db/migrate/20240909011043_make_name_not_null.solid_queue.rb create mode 100644 db/migrate/20240909011044_change_solid_queue_recurring_tasks_static_to_not_null.solid_queue.rb diff --git a/Gemfile b/Gemfile index 84f9cd1..dfd24e0 100644 --- a/Gemfile +++ b/Gemfile @@ -18,7 +18,7 @@ gem "jbuilder" # Use Redis adapter to run Action Cable in production # gem "redis", ">= 4.0.1" -gem "solid_queue" +gem "solid_queue", "= 0.6.0" # Use Kredis to get higher-level data types in Redis [https://github.com/rails/kredis] # gem "kredis" diff --git a/Gemfile.lock b/Gemfile.lock index b76edaf..ddae94d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -161,7 +161,7 @@ GEM activesupport (>= 5.0.0) json (2.7.2) language_server-protocol (3.17.0.3) - logger (1.6.0) + logger (1.6.1) loofah (2.22.0) crass (~> 1.0.2) nokogiri (>= 1.12.0) @@ -222,7 +222,7 @@ GEM rdoc (6.7.0) psych (>= 4.0.0) regexp_parser (2.9.2) - reline (0.5.9) + reline (0.5.10) io-console (~> 0.5) rexml (3.3.5) strscan @@ -264,7 +264,7 @@ GEM rexml (~> 3.2, >= 3.2.5) rubyzip (>= 1.2.2, < 3.0) websocket (~> 1.0) - solid_queue (0.5.0) + solid_queue (0.6.0) activejob (>= 7.1) activerecord (>= 7.1) concurrent-ruby (>= 1.3.1) @@ -283,7 +283,7 @@ GEM railties (>= 7.0.0) tailwindcss-rails (2.7.3-x86_64-linux) railties (>= 7.0.0) - thor (1.3.1) + thor (1.3.2) timeout (0.4.1) turbo-rails (2.0.6) actionpack (>= 6.0.0) @@ -305,7 +305,7 @@ GEM websocket-extensions (0.1.5) xpath (3.2.0) nokogiri (~> 1.8) - zeitwerk (2.6.17) + zeitwerk (2.6.18) PLATFORMS aarch64-linux @@ -325,7 +325,7 @@ DEPENDENCIES rails! rubocop-rails-omakase selenium-webdriver - solid_queue + solid_queue (= 0.6.0) sqlite3 (>= 1.4) stimulus-rails tailwindcss-rails (~> 2.0) diff --git a/db/migrate/20240909011042_add_name_to_processes.solid_queue.rb b/db/migrate/20240909011042_add_name_to_processes.solid_queue.rb new file mode 100644 index 0000000..d5c14cd --- /dev/null +++ b/db/migrate/20240909011042_add_name_to_processes.solid_queue.rb @@ -0,0 +1,6 @@ +# This migration comes from solid_queue (originally 20240811173327) +class AddNameToProcesses < ActiveRecord::Migration[7.1] + def change + add_column :solid_queue_processes, :name, :string + end +end diff --git a/db/migrate/20240909011043_make_name_not_null.solid_queue.rb b/db/migrate/20240909011043_make_name_not_null.solid_queue.rb new file mode 100644 index 0000000..d1dcc6b --- /dev/null +++ b/db/migrate/20240909011043_make_name_not_null.solid_queue.rb @@ -0,0 +1,17 @@ +# This migration comes from solid_queue (originally 20240813160053) +class MakeNameNotNull < ActiveRecord::Migration[7.1] + def up + SolidQueue::Process.where(name: nil).find_each do |process| + process.name ||= [ process.kind.downcase, SecureRandom.hex(10) ].join("-") + process.save! + end + + change_column :solid_queue_processes, :name, :string, null: false + add_index :solid_queue_processes, [ :name, :supervisor_id ], unique: true + end + + def down + remove_index :solid_queue_processes, [ :name, :supervisor_id ] + change_column :solid_queue_processes, :name, :string, null: false + end +end diff --git a/db/migrate/20240909011044_change_solid_queue_recurring_tasks_static_to_not_null.solid_queue.rb b/db/migrate/20240909011044_change_solid_queue_recurring_tasks_static_to_not_null.solid_queue.rb new file mode 100644 index 0000000..325320a --- /dev/null +++ b/db/migrate/20240909011044_change_solid_queue_recurring_tasks_static_to_not_null.solid_queue.rb @@ -0,0 +1,6 @@ +# This migration comes from solid_queue (originally 20240819165045) +class ChangeSolidQueueRecurringTasksStaticToNotNull < ActiveRecord::Migration[7.1] + def change + change_column_null :solid_queue_recurring_tasks, :static, false, true + end +end diff --git a/db/schema.rb b/db/schema.rb index f03ca23..1637c09 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2024_08_17_015831) do +ActiveRecord::Schema[8.0].define(version: 2024_09_09_011044) do create_table "credit_card_bills", force: :cascade do |t| t.string "description" t.decimal "amount" @@ -104,7 +104,9 @@ ActiveRecord::Schema[8.0].define(version: 2024_08_17_015831) do t.string "hostname" t.text "metadata" t.datetime "created_at", null: false + t.string "name", null: false t.index ["last_heartbeat_at"], name: "index_solid_queue_processes_on_last_heartbeat_at" + t.index ["name", "supervisor_id"], name: "index_solid_queue_processes_on_name_and_supervisor_id", unique: true t.index ["supervisor_id"], name: "index_solid_queue_processes_on_supervisor_id" end @@ -135,7 +137,7 @@ ActiveRecord::Schema[8.0].define(version: 2024_08_17_015831) do t.text "arguments" t.string "queue_name" t.integer "priority", default: 0 - t.boolean "static", default: true + t.boolean "static", default: true, null: false t.text "description" t.datetime "created_at", null: false t.datetime "updated_at", null: false -- 2.45.2 From 90ebe18524f4459a4df870949c5e02b6bf3dab6e Mon Sep 17 00:00:00 2001 From: Andrew Tomaka Date: Sun, 8 Sep 2024 21:11:30 -0400 Subject: [PATCH 2/2] Upgrade solid_queue the rest of the way --- Gemfile | 2 +- Gemfile.lock | 5 +- bin/jobs | 6 ++ config/environments/production.rb | 2 + config/solid_queue.yml | 36 ++++----- db/queue_schema.rb | 129 ++++++++++++++++++++++++++++++ 6 files changed, 159 insertions(+), 21 deletions(-) create mode 100755 bin/jobs create mode 100644 db/queue_schema.rb diff --git a/Gemfile b/Gemfile index dfd24e0..84f9cd1 100644 --- a/Gemfile +++ b/Gemfile @@ -18,7 +18,7 @@ gem "jbuilder" # Use Redis adapter to run Action Cable in production # gem "redis", ">= 4.0.1" -gem "solid_queue", "= 0.6.0" +gem "solid_queue" # Use Kredis to get higher-level data types in Redis [https://github.com/rails/kredis] # gem "kredis" diff --git a/Gemfile.lock b/Gemfile.lock index ddae94d..1c675a5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -264,12 +264,13 @@ GEM rexml (~> 3.2, >= 3.2.5) rubyzip (>= 1.2.2, < 3.0) websocket (~> 1.0) - solid_queue (0.6.0) + solid_queue (0.8.2) activejob (>= 7.1) activerecord (>= 7.1) concurrent-ruby (>= 1.3.1) fugit (~> 1.11.0) railties (>= 7.1) + thor (~> 1.3.1) sqlite3 (2.0.4-aarch64-linux-gnu) sqlite3 (2.0.4-arm64-darwin) sqlite3 (2.0.4-x86_64-linux-gnu) @@ -325,7 +326,7 @@ DEPENDENCIES rails! rubocop-rails-omakase selenium-webdriver - solid_queue (= 0.6.0) + solid_queue sqlite3 (>= 1.4) stimulus-rails tailwindcss-rails (~> 2.0) diff --git a/bin/jobs b/bin/jobs new file mode 100755 index 0000000..dcf59f3 --- /dev/null +++ b/bin/jobs @@ -0,0 +1,6 @@ +#!/usr/bin/env ruby + +require_relative "../config/environment" +require "solid_queue/cli" + +SolidQueue::Cli.start(ARGV) diff --git a/config/environments/production.rb b/config/environments/production.rb index 1443b35..08d3893 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -70,6 +70,8 @@ Rails.application.configure do # Use a real queuing backend for Active Job (and separate queues per environment). config.active_job.queue_adapter = :solid_queue + config.solid_queue.connects_to = { database: { writing: :queue } } + # config.active_job.queue_name_prefix = "family_budget_production" # Disable caching for Action Mailer templates even if Action Controller diff --git a/config/solid_queue.yml b/config/solid_queue.yml index 2d8d922..9eace59 100644 --- a/config/solid_queue.yml +++ b/config/solid_queue.yml @@ -1,18 +1,18 @@ -# default: &default -# dispatchers: -# - polling_interval: 1 -# batch_size: 500 -# workers: -# - queues: "*" -# threads: 3 -# processes: 1 -# polling_interval: 0.1 -# -# development: -# <<: *default -# -# test: -# <<: *default -# -# production: -# <<: *default +default: &default + dispatchers: + - polling_interval: 1 + batch_size: 500 + workers: + - queues: "*" + threads: 3 + processes: <%= ENV.fetch("JOB_CONCURRENCY", 1) %> + polling_interval: 0.1 + +development: + <<: *default + +test: + <<: *default + +production: + <<: *default diff --git a/db/queue_schema.rb b/db/queue_schema.rb new file mode 100644 index 0000000..4b49a12 --- /dev/null +++ b/db/queue_schema.rb @@ -0,0 +1,129 @@ +ActiveRecord::Schema[7.1].define(version: 2024_09_04_193154) do + create_table "solid_queue_blocked_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.string "concurrency_key", null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.index [ "concurrency_key", "priority", "job_id" ], name: "index_solid_queue_blocked_executions_for_release" + t.index [ "expires_at", "concurrency_key" ], name: "index_solid_queue_blocked_executions_for_maintenance" + t.index [ "job_id" ], name: "index_solid_queue_blocked_executions_on_job_id", unique: true + end + + create_table "solid_queue_claimed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.bigint "process_id" + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_claimed_executions_on_job_id", unique: true + t.index [ "process_id", "job_id" ], name: "index_solid_queue_claimed_executions_on_process_id_and_job_id" + end + + create_table "solid_queue_failed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.text "error" + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_failed_executions_on_job_id", unique: true + end + + create_table "solid_queue_jobs", force: :cascade do |t| + t.string "queue_name", null: false + t.string "class_name", null: false + t.text "arguments" + t.integer "priority", default: 0, null: false + t.string "active_job_id" + t.datetime "scheduled_at" + t.datetime "finished_at" + t.string "concurrency_key" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index [ "active_job_id" ], name: "index_solid_queue_jobs_on_active_job_id" + t.index [ "class_name" ], name: "index_solid_queue_jobs_on_class_name" + t.index [ "finished_at" ], name: "index_solid_queue_jobs_on_finished_at" + t.index [ "queue_name", "finished_at" ], name: "index_solid_queue_jobs_for_filtering" + t.index [ "scheduled_at", "finished_at" ], name: "index_solid_queue_jobs_for_alerting" + end + + create_table "solid_queue_pauses", force: :cascade do |t| + t.string "queue_name", null: false + t.datetime "created_at", null: false + t.index [ "queue_name" ], name: "index_solid_queue_pauses_on_queue_name", unique: true + end + + create_table "solid_queue_processes", force: :cascade do |t| + t.string "kind", null: false + t.datetime "last_heartbeat_at", null: false + t.bigint "supervisor_id" + t.integer "pid", null: false + t.string "hostname" + t.text "metadata" + t.datetime "created_at", null: false + t.string "name", null: false + t.index [ "last_heartbeat_at" ], name: "index_solid_queue_processes_on_last_heartbeat_at" + t.index [ "name", "supervisor_id" ], name: "index_solid_queue_processes_on_name_and_supervisor_id", unique: true + t.index [ "supervisor_id" ], name: "index_solid_queue_processes_on_supervisor_id" + end + + create_table "solid_queue_ready_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_ready_executions_on_job_id", unique: true + t.index [ "priority", "job_id" ], name: "index_solid_queue_poll_all" + t.index [ "queue_name", "priority", "job_id" ], name: "index_solid_queue_poll_by_queue" + end + + create_table "solid_queue_recurring_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "task_key", null: false + t.datetime "run_at", null: false + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_recurring_executions_on_job_id", unique: true + t.index [ "task_key", "run_at" ], name: "index_solid_queue_recurring_executions_on_task_key_and_run_at", unique: true + end + + create_table "solid_queue_recurring_tasks", force: :cascade do |t| + t.string "key", null: false + t.string "schedule", null: false + t.string "command", limit: 2048 + t.string "class_name" + t.text "arguments" + t.string "queue_name" + t.integer "priority", default: 0 + t.boolean "static", default: true, null: false + t.text "description" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index [ "key" ], name: "index_solid_queue_recurring_tasks_on_key", unique: true + t.index [ "static" ], name: "index_solid_queue_recurring_tasks_on_static" + end + + create_table "solid_queue_scheduled_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "scheduled_at", null: false + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_scheduled_executions_on_job_id", unique: true + t.index [ "scheduled_at", "priority", "job_id" ], name: "index_solid_queue_dispatch_all" + end + + create_table "solid_queue_semaphores", force: :cascade do |t| + t.string "key", null: false + t.integer "value", default: 1, null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index [ "expires_at" ], name: "index_solid_queue_semaphores_on_expires_at" + t.index [ "key", "value" ], name: "index_solid_queue_semaphores_on_key_and_value" + t.index [ "key" ], name: "index_solid_queue_semaphores_on_key", unique: true + end + + add_foreign_key "solid_queue_blocked_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_claimed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_failed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_ready_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_recurring_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_scheduled_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade +end -- 2.45.2