diff --git a/Gemfile b/Gemfile index 0460ebe08..b57f8f3a2 100644 --- a/Gemfile +++ b/Gemfile @@ -186,3 +186,7 @@ group :test do # Adds ability to stub out methods in tests easier gem "mocha" end + +gem "solid_cable", "~> 3.0" + +gem "solid_queue", "~> 1.1" diff --git a/Gemfile.lock b/Gemfile.lock index a730519b0..341ee3a63 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -185,6 +185,8 @@ GEM eventmachine (>= 0.12.9) http_parser.rb (~> 0) erubi (1.13.0) + et-orbi (1.2.11) + tzinfo eventmachine (1.2.7) evil_systems (1.1.5) capybara @@ -234,6 +236,9 @@ GEM figaro (1.2.0) thor (>= 0.14.0, < 2) formatador (1.1.0) + fugit (1.11.1) + et-orbi (~> 1, >= 1.2.11) + raabro (~> 1.4) globalid (1.2.1) activesupport (>= 6.1) google-protobuf (4.29.1) @@ -382,6 +387,7 @@ GEM public_suffix (6.0.1) puma (6.5.0) nio4r (~> 2.0) + raabro (1.4.0) racc (1.8.1) rack (3.1.8) rack-session (2.0.0) @@ -490,6 +496,18 @@ GEM simplecov_json_formatter (~> 0.1) simplecov-html (0.13.1) simplecov_json_formatter (0.1.4) + solid_cable (3.0.4) + actioncable (>= 7.2) + activejob (>= 7.2) + activerecord (>= 7.2) + railties (>= 7.2) + solid_queue (1.1.0) + activejob (>= 7.1) + activerecord (>= 7.1) + concurrent-ruby (>= 1.3.1) + fugit (~> 1.11.0) + railties (>= 7.1) + thor (~> 1.3.1) sshkit (1.23.2) base64 net-scp (>= 1.1.2) @@ -603,6 +621,8 @@ DEPENDENCIES shoulda (~> 4.0) shoulda-matchers simplecov + solid_cable (~> 3.0) + solid_queue (~> 1.1) standard stimulus-rails strong_migrations (~> 2.1) diff --git a/Procfile.dev b/Procfile.dev index 852e6c710..70ce128c7 100644 --- a/Procfile.dev +++ b/Procfile.dev @@ -1,2 +1,3 @@ web: bin/rails server -p 3000 css: bin/rails dartsass:watch +jobs: bin/jobs diff --git a/bin/jobs b/bin/jobs new file mode 100755 index 000000000..dcf59f309 --- /dev/null +++ b/bin/jobs @@ -0,0 +1,6 @@ +#!/usr/bin/env ruby + +require_relative "../config/environment" +require "solid_queue/cli" + +SolidQueue::Cli.start(ARGV) diff --git a/config/cable.yml b/config/cable.yml index 9f4c2c46d..7866c8d15 100644 --- a/config/cable.yml +++ b/config/cable.yml @@ -1,10 +1,22 @@ +# Async adapter only works within the same process, so for manually triggering cable updates from a console, +# and seeing results in the browser, you must do so from the web console (running inside the dev process), +# not a terminal started via bin/rails console! Add "console" to any action or any ERB template view +# to make the web console appear. development: - adapter: async + adapter: solid_cable + connects_to: + database: + writing: cable + polling_interval: 0.1.seconds + message_retention: 1.day test: adapter: test production: - adapter: redis - url: <%= ENV.fetch("REDIS_URL") { "redis://localhost:6379/1" } %> - channel_prefix: pet_rescue_production + adapter: solid_cable + connects_to: + database: + writing: cable + polling_interval: 0.1.seconds + message_retention: 1.day diff --git a/config/database.yml b/config/database.yml index 3b280a1f5..3be5568eb 100644 --- a/config/database.yml +++ b/config/database.yml @@ -33,11 +33,24 @@ default: &default pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> development: - <<: *default - database: pet_rescue_development - # The following line comes from: - # https://stackoverflow.com/questions/59089991/ruby-2-6-5-and-postgresql-pg-gem-segmentation-fault - gssencmode: disable + primary: + <<: *default + database: pet_rescue_development + # The following line comes from: + # https://stackoverflow.com/questions/59089991/ruby-2-6-5-and-postgresql-pg-gem-segmentation-fault + gssencmode: disable + + cable: + <<: *default + database: cable + migrations_paths: db/cable_migrate + gssencmode: disable + + queue: + <<: *default + database: queue + migrations_paths: db/queue_migrate + gssencmode: disable #username: <%= ENV.fetch('DATABASE_USERNAME') { 'postgres' } %> #password: <%= ENV.fetch('DATABASE_PASSWORD') { 'password' } %> @@ -91,10 +104,18 @@ test: # for a full overview on how database connection configuration can be specified. # production: - <<: *default - host: <%= ENV["DB_HOST"] %> - pool: 5 - username: <%= ENV["DB_USER"] %> - password: <%= ENV["DB_PASSWORD"] %> - database: postgres - timeout: 5000 + primary: &primary_production + <<: *default + host: <%= ENV["DB_HOST"] %> + username: <%= ENV["DB_USER"] %> + password: <%= ENV["DB_PASSWORD"] %> + database: postgres + timeout: 5000 + cable: + <<: *primary_production + database: cable + migrations_paths: db/cable_migrate + queue: + <<: *primary_production + database: queue + migrations_paths: db/queue_migrate diff --git a/config/deploy.yml b/config/deploy.yml index 825c07ab8..7d09c35c6 100644 --- a/config/deploy.yml +++ b/config/deploy.yml @@ -48,13 +48,20 @@ env: - DB_HOST - DB_USER - DB_PASSWORD + clear: + # Run the Solid Queue Supervisor inside the web server's Puma process to do jobs. + # When you start using multiple servers, you should split out job processing to a dedicated machine. + SOLID_QUEUE_IN_PUMA: true + + # Log everything from Rails + # RAILS_LOG_LEVEL: debug # Aliases are triggered with "bin/kamal ". You can overwrite arguments on invocation: # "bin/kamal logs -r job" will tail logs from the first server in the job section. # aliases: shell: app exec --interactive --reuse "bash" - logs: app exec -i --reuse "tail -f log/production.log" + logs: app logs -f console: app exec -i --reuse "bin/rails console" # Use a different ssh user than root diff --git a/config/environments/development.rb b/config/environments/development.rb index 1ae2f56bc..68f6cbeec 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -85,4 +85,7 @@ config.app_url = "http://localhost:3000/" # Apply autocorrection by RuboCop to files generated by `bin/rails generate`. # config.generators.apply_rubocop_autocorrect_after_generate! + + config.active_job.queue_adapter = :solid_queue + config.solid_queue.connects_to = {database: {writing: :queue}} end diff --git a/config/environments/production.rb b/config/environments/production.rb index f621bc524..ad7e81a8f 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -55,9 +55,6 @@ config.assume_ssl = true config.force_ssl = true - # Include generic and useful information about system operation, but avoid logging too much - # information to avoid inadvertent exposure of personally identifiable information (PII). - config.log_level = :info # Skip http-to-https redirect for the default health check endpoint. # config.ssl_options = { redirect: { exclude: ->(request) { request.path == "/up" } } } @@ -67,7 +64,6 @@ # Change to "debug" to log everything (including potentially personally-identifiable information!) config.log_level = ENV.fetch("RAILS_LOG_LEVEL", "info") - # config.active_job.queue_name_prefix = "baja_pet_rescue_production" # Prevent health checks from clogging up the logs. config.silence_healthcheck_path = "/up" @@ -83,7 +79,8 @@ # config.cache_store = :mem_cache_store # Replace the default in-process and non-durable queuing backend for Active Job. - # config.active_job.queue_adapter = :resque + config.active_job.queue_adapter = :solid_queue + config.solid_queue.connects_to = {database: {writing: :queue}} # Ignore bad email addresses and do not raise email delivery errors. # Set this to true and configure the email server for immediate delivery to raise delivery errors. # config.action_mailer.raise_delivery_errors = false diff --git a/config/queue.yml b/config/queue.yml new file mode 100644 index 000000000..9eace59c4 --- /dev/null +++ b/config/queue.yml @@ -0,0 +1,18 @@ +default: &default + dispatchers: + - polling_interval: 1 + batch_size: 500 + workers: + - queues: "*" + threads: 3 + processes: <%= ENV.fetch("JOB_CONCURRENCY", 1) %> + polling_interval: 0.1 + +development: + <<: *default + +test: + <<: *default + +production: + <<: *default diff --git a/config/recurring.yml b/config/recurring.yml new file mode 100644 index 000000000..d045b1914 --- /dev/null +++ b/config/recurring.yml @@ -0,0 +1,10 @@ +# production: +# periodic_cleanup: +# class: CleanSoftDeletedRecordsJob +# queue: background +# args: [ 1000, { batch_size: 500 } ] +# schedule: every hour +# periodic_command: +# command: "SoftDeletedRecord.due.delete_all" +# priority: 2 +# schedule: at 5am every day diff --git a/db/cable_migrate/.keep b/db/cable_migrate/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/db/cable_schema.rb b/db/cable_schema.rb new file mode 100644 index 000000000..90beff611 --- /dev/null +++ b/db/cable_schema.rb @@ -0,0 +1,26 @@ +# This file is auto-generated from the current state of the database. Instead +# of editing this file, please use the migrations feature of Active Record to +# incrementally modify your database, and then regenerate this schema definition. +# +# This file is the source Rails uses to define your schema when running `bin/rails +# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to +# be faster and is potentially less error prone than running all of your +# migrations from scratch. Old migrations may fail to apply correctly if those +# migrations use external dependencies or application code. +# +# It's strongly recommended that you check this file into your version control system. + +ActiveRecord::Schema[8.0].define(version: 1) do + # These are extensions that must be enabled in order to support this database + enable_extension "pg_catalog.plpgsql" + + create_table "solid_cable_messages", force: :cascade do |t| + t.binary "channel", null: false + t.binary "payload", null: false + t.datetime "created_at", null: false + t.bigint "channel_hash", null: false + t.index ["channel"], name: "index_solid_cable_messages_on_channel" + t.index ["channel_hash"], name: "index_solid_cable_messages_on_channel_hash" + t.index ["created_at"], name: "index_solid_cable_messages_on_created_at" + end +end diff --git a/db/queue_migrate/.keep b/db/queue_migrate/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/db/queue_schema.rb b/db/queue_schema.rb new file mode 100644 index 000000000..089e93801 --- /dev/null +++ b/db/queue_schema.rb @@ -0,0 +1,144 @@ +# This file is auto-generated from the current state of the database. Instead +# of editing this file, please use the migrations feature of Active Record to +# incrementally modify your database, and then regenerate this schema definition. +# +# This file is the source Rails uses to define your schema when running `bin/rails +# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to +# be faster and is potentially less error prone than running all of your +# migrations from scratch. Old migrations may fail to apply correctly if those +# migrations use external dependencies or application code. +# +# It's strongly recommended that you check this file into your version control system. + +ActiveRecord::Schema[8.0].define(version: 1) do + # These are extensions that must be enabled in order to support this database + enable_extension "pg_catalog.plpgsql" + + create_table "solid_queue_blocked_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.string "concurrency_key", null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.index ["concurrency_key", "priority", "job_id"], name: "index_solid_queue_blocked_executions_for_release" + t.index ["expires_at", "concurrency_key"], name: "index_solid_queue_blocked_executions_for_maintenance" + t.index ["job_id"], name: "index_solid_queue_blocked_executions_on_job_id", unique: true + end + + create_table "solid_queue_claimed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.bigint "process_id" + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_claimed_executions_on_job_id", unique: true + t.index ["process_id", "job_id"], name: "index_solid_queue_claimed_executions_on_process_id_and_job_id" + end + + create_table "solid_queue_failed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.text "error" + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_failed_executions_on_job_id", unique: true + end + + create_table "solid_queue_jobs", force: :cascade do |t| + t.string "queue_name", null: false + t.string "class_name", null: false + t.text "arguments" + t.integer "priority", default: 0, null: false + t.string "active_job_id" + t.datetime "scheduled_at" + t.datetime "finished_at" + t.string "concurrency_key" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["active_job_id"], name: "index_solid_queue_jobs_on_active_job_id" + t.index ["class_name"], name: "index_solid_queue_jobs_on_class_name" + t.index ["finished_at"], name: "index_solid_queue_jobs_on_finished_at" + t.index ["queue_name", "finished_at"], name: "index_solid_queue_jobs_for_filtering" + t.index ["scheduled_at", "finished_at"], name: "index_solid_queue_jobs_for_alerting" + end + + create_table "solid_queue_pauses", force: :cascade do |t| + t.string "queue_name", null: false + t.datetime "created_at", null: false + t.index ["queue_name"], name: "index_solid_queue_pauses_on_queue_name", unique: true + end + + create_table "solid_queue_processes", force: :cascade do |t| + t.string "kind", null: false + t.datetime "last_heartbeat_at", null: false + t.bigint "supervisor_id" + t.integer "pid", null: false + t.string "hostname" + t.text "metadata" + t.datetime "created_at", null: false + t.string "name", null: false + t.index ["last_heartbeat_at"], name: "index_solid_queue_processes_on_last_heartbeat_at" + t.index ["name", "supervisor_id"], name: "index_solid_queue_processes_on_name_and_supervisor_id", unique: true + t.index ["supervisor_id"], name: "index_solid_queue_processes_on_supervisor_id" + end + + create_table "solid_queue_ready_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_ready_executions_on_job_id", unique: true + t.index ["priority", "job_id"], name: "index_solid_queue_poll_all" + t.index ["queue_name", "priority", "job_id"], name: "index_solid_queue_poll_by_queue" + end + + create_table "solid_queue_recurring_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "task_key", null: false + t.datetime "run_at", null: false + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_recurring_executions_on_job_id", unique: true + t.index ["task_key", "run_at"], name: "index_solid_queue_recurring_executions_on_task_key_and_run_at", unique: true + end + + create_table "solid_queue_recurring_tasks", force: :cascade do |t| + t.string "key", null: false + t.string "schedule", null: false + t.string "command", limit: 2048 + t.string "class_name" + t.text "arguments" + t.string "queue_name" + t.integer "priority", default: 0 + t.boolean "static", default: true, null: false + t.text "description" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["key"], name: "index_solid_queue_recurring_tasks_on_key", unique: true + t.index ["static"], name: "index_solid_queue_recurring_tasks_on_static" + end + + create_table "solid_queue_scheduled_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "scheduled_at", null: false + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_scheduled_executions_on_job_id", unique: true + t.index ["scheduled_at", "priority", "job_id"], name: "index_solid_queue_dispatch_all" + end + + create_table "solid_queue_semaphores", force: :cascade do |t| + t.string "key", null: false + t.integer "value", default: 1, null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["expires_at"], name: "index_solid_queue_semaphores_on_expires_at" + t.index ["key", "value"], name: "index_solid_queue_semaphores_on_key_and_value" + t.index ["key"], name: "index_solid_queue_semaphores_on_key", unique: true + end + + add_foreign_key "solid_queue_blocked_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_claimed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_failed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_ready_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_recurring_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_scheduled_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade +end