From 5a1bd6be53f6532c261f9ec26af81e36697eaff8 Mon Sep 17 00:00:00 2001 From: smudge Date: Mon, 2 Feb 2026 16:13:44 -0500 Subject: [PATCH] refactor: De-complexify 'count' metric Given that `live_count_grouped` is already necessary for computing all the other live queue metrics, we can reduce the work on MySQL by running `failed_count_grouped` (and filtering the scan down to fewer rows) instead of an entire count against the table. stack-info: PR: https://github.com/Betterment/delayed/pull/97, branch: smudge/stack/8 --- lib/delayed/monitor.rb | 6 +- .../__snapshots__/monitor_spec.rb.snap | 56 +++++-------------- 2 files changed, 15 insertions(+), 47 deletions(-) diff --git a/lib/delayed/monitor.rb b/lib/delayed/monitor.rb index 80b8b31..1245ea9 100644 --- a/lib/delayed/monitor.rb +++ b/lib/delayed/monitor.rb @@ -119,11 +119,7 @@ def as_expression(aggregate_function, aggregate_expression, column_name) end def count_grouped - if Job.connection.supports_partial_index? - failed_count_grouped.merge(live_count_grouped) { |_, l, f| l + f } - else - grouped_query(jobs, count: [:count, '*']).transform_values(&:count) - end + failed_count_grouped.merge(live_count_grouped) { |_, l, f| l + f } end def live_count_grouped diff --git a/spec/delayed/__snapshots__/monitor_spec.rb.snap b/spec/delayed/__snapshots__/monitor_spec.rb.snap index 6d670a7..0a9b802 100644 --- a/spec/delayed/__snapshots__/monitor_spec.rb.snap +++ b/spec/delayed/__snapshots__/monitor_spec.rb.snap @@ -433,6 +433,7 @@ SELECT SUM(count) AS count, queue AS queue FROM (SELECT `delayed_jobs`.`priority`, `delayed_jobs`.`queue`, COUNT(*) AS count FROM `delayed_jobs` + WHERE `delayed_jobs`.`failed_at` IS NOT NULL GROUP BY `delayed_jobs`.`priority`, `delayed_jobs`.`queue`) subquery GROUP BY CASE WHEN priority < 10 THEN 0 WHEN priority < 20 THEN 10 WHEN priority < 30 THEN 20 WHEN priority >= 30 THEN 30 END, `queue` @@ -442,10 +443,9 @@ SELECT SUM(count) AS count, -> Materialize (cost=...) -> Table scan on -> Aggregate using temporary table - -> Covering index scan on delayed_jobs using idx_delayed_jobs_live (cost=...) + -> Filter: (delayed_jobs.failed_at is not null) (cost=...) + -> Covering index range scan on delayed_jobs using idx_delayed_jobs_live over (NULL < failed_at) (cost=...) --- --- QUERIES FOR `future_count`: ---------------------------------- SELECT SUM(count) AS count, SUM(future_count) AS future_count, SUM(erroring_count) AS erroring_count, @@ -471,6 +471,9 @@ SELECT SUM(count) AS count, -> Filter: (delayed_jobs.failed_at is null) (cost=...) -> Covering index lookup on delayed_jobs using idx_delayed_jobs_live (failed_at = NULL) (cost=...) --- +-- QUERIES FOR `future_count`: +--------------------------------- +-- (no new queries) -- QUERIES FOR `locked_count`: --------------------------------- SELECT SUM(claimed_count) AS claimed_count, @@ -503,24 +506,7 @@ SELECT SUM(claimed_count) AS claimed_count, -- (no new queries) -- QUERIES FOR `failed_count`: --------------------------------- -SELECT SUM(count) AS count, - CASE WHEN priority < 10 THEN 0 WHEN priority < 20 THEN 10 WHEN priority < 30 THEN 20 WHEN priority >= 30 THEN 30 END AS priority, - queue AS queue - FROM (SELECT `delayed_jobs`.`priority`, `delayed_jobs`.`queue`, COUNT(*) AS count - FROM `delayed_jobs` - WHERE `delayed_jobs`.`failed_at` IS NOT NULL - GROUP BY `delayed_jobs`.`priority`, `delayed_jobs`.`queue`) subquery - GROUP BY CASE WHEN priority < 10 THEN 0 WHEN priority < 20 THEN 10 WHEN priority < 30 THEN 20 WHEN priority >= 30 THEN 30 END, `queue` - --> Table scan on - -> Aggregate using temporary table - -> Table scan on subquery (cost=...) - -> Materialize (cost=...) - -> Table scan on - -> Aggregate using temporary table - -> Filter: (delayed_jobs.failed_at is not null) (cost=...) - -> Covering index range scan on delayed_jobs using idx_delayed_jobs_live over (NULL < failed_at) (cost=...) ---- +-- (no new queries) -- QUERIES FOR `max_lock_age`: --------------------------------- -- (no new queries) @@ -546,6 +532,7 @@ SELECT SUM(count) AS count, queue AS queue FROM (SELECT `delayed_jobs`.`priority`, `delayed_jobs`.`queue`, COUNT(*) AS count FROM `delayed_jobs` + WHERE `delayed_jobs`.`failed_at` IS NOT NULL GROUP BY `delayed_jobs`.`priority`, `delayed_jobs`.`queue`) subquery GROUP BY CASE WHEN priority < 10 THEN 0 WHEN priority < 20 THEN 10 WHEN priority < 30 THEN 20 WHEN priority >= 30 THEN 30 END, `queue` @@ -555,10 +542,9 @@ SELECT SUM(count) AS count, -> Materialize (cost=...) -> Table scan on -> Aggregate using temporary table - -> Table scan on delayed_jobs (cost=...) + -> Filter: (delayed_jobs.failed_at is not null) (cost=...) + -> Table scan on delayed_jobs (cost=...) --- --- QUERIES FOR `future_count`: ---------------------------------- SELECT SUM(count) AS count, SUM(future_count) AS future_count, SUM(erroring_count) AS erroring_count, @@ -584,6 +570,9 @@ SELECT SUM(count) AS count, -> Filter: (delayed_jobs.failed_at is null) (cost=...) -> Table scan on delayed_jobs (cost=...) --- +-- QUERIES FOR `future_count`: +--------------------------------- +-- (no new queries) -- QUERIES FOR `locked_count`: --------------------------------- SELECT SUM(claimed_count) AS claimed_count, @@ -616,24 +605,7 @@ SELECT SUM(claimed_count) AS claimed_count, -- (no new queries) -- QUERIES FOR `failed_count`: --------------------------------- -SELECT SUM(count) AS count, - CASE WHEN priority < 10 THEN 0 WHEN priority < 20 THEN 10 WHEN priority < 30 THEN 20 WHEN priority >= 30 THEN 30 END AS priority, - queue AS queue - FROM (SELECT `delayed_jobs`.`priority`, `delayed_jobs`.`queue`, COUNT(*) AS count - FROM `delayed_jobs` - WHERE `delayed_jobs`.`failed_at` IS NOT NULL - GROUP BY `delayed_jobs`.`priority`, `delayed_jobs`.`queue`) subquery - GROUP BY CASE WHEN priority < 10 THEN 0 WHEN priority < 20 THEN 10 WHEN priority < 30 THEN 20 WHEN priority >= 30 THEN 30 END, `queue` - --> Table scan on - -> Aggregate using temporary table - -> Table scan on subquery (cost=...) - -> Materialize (cost=...) - -> Table scan on - -> Aggregate using temporary table - -> Filter: (delayed_jobs.failed_at is not null) (cost=...) - -> Table scan on delayed_jobs (cost=...) ---- +-- (no new queries) -- QUERIES FOR `max_lock_age`: --------------------------------- -- (no new queries)