From 4b1dfee65a46affb3bf8e84c98d4a385019d7f46 Mon Sep 17 00:00:00 2001 From: bubiche Date: Mon, 6 Apr 2026 02:39:37 +0800 Subject: [PATCH] Improve DISTINCT queries on PostgreSQL via recursive CTE --- README.md | 2 +- app/models/solid_queue/queue.rb | 4 +- app/models/solid_queue/queue_selector.rb | 4 +- app/models/solid_queue/record.rb | 2 + .../solid_queue/record/distinct_values.rb | 48 +++++++++++++++++++ .../solid_queue/ready_execution_test.rb | 34 +++++++++++++ 6 files changed, 88 insertions(+), 6 deletions(-) create mode 100644 app/models/solid_queue/record/distinct_values.rb diff --git a/README.md b/README.md index e37db581a..a815185db 100644 --- a/README.md +++ b/README.md @@ -339,7 +339,7 @@ FROM solid_queue_ready_executions WHERE queue_name LIKE 'beta%'; ``` -This type of `DISTINCT` query on a column that's the leftmost column in an index can be performed very fast in MySQL thanks to a technique called [Loose Index Scan](https://dev.mysql.com/doc/refman/8.0/en/group-by-optimization.html#loose-index-scan). PostgreSQL and SQLite, however, don't implement this technique, which means that if your `solid_queue_ready_executions` table is very big because your queues get very deep, this query will get slow. Normally your `solid_queue_ready_executions` table will be small, but it can happen. +This type of `DISTINCT` query on a column that's the leftmost column in an index can be performed very fast in MySQL thanks to a technique called [Loose Index Scan](https://dev.mysql.com/doc/refman/8.0/en/group-by-optimization.html#loose-index-scan). PostgreSQL doesn't implement this technique natively, so Solid Queue uses a [recursive CTE](https://www.postgresql.org/docs/current/queries-with.html#QUERIES-WITH-RECURSIVE) to emulate it, achieving similar performance by walking the B-tree index and jumping between distinct values. SQLite doesn't implement loose index scan either, but this is unlikely to be a problem since SQLite is typically used in development with small datasets. Similarly to using prefixes, the same will happen if you have paused queues, because we need to get a list of all queues with a query like ```sql diff --git a/app/models/solid_queue/queue.rb b/app/models/solid_queue/queue.rb index 7968d3957..60074a878 100644 --- a/app/models/solid_queue/queue.rb +++ b/app/models/solid_queue/queue.rb @@ -6,9 +6,7 @@ class Queue class << self def all - Job.select(:queue_name).distinct.collect do |job| - new(job.queue_name) - end + Job.distinct_values_of(:queue_name).map { |name| new(name) } end def find_by_name(name) diff --git a/app/models/solid_queue/queue_selector.rb b/app/models/solid_queue/queue_selector.rb index 24f6a6ad2..d0d61dd14 100644 --- a/app/models/solid_queue/queue_selector.rb +++ b/app/models/solid_queue/queue_selector.rb @@ -43,7 +43,7 @@ def include_all_queues? end def all_queues - relation.distinct(:queue_name).pluck(:queue_name) + relation.distinct_values_of(:queue_name) end def exact_names @@ -53,7 +53,7 @@ def exact_names def prefixed_names if prefixes.empty? then [] else - relation.where(([ "queue_name LIKE ?" ] * prefixes.count).join(" OR "), *prefixes).distinct(:queue_name).pluck(:queue_name) + relation.where(([ "queue_name LIKE ?" ] * prefixes.count).join(" OR "), *prefixes).distinct_values_of(:queue_name) end end diff --git a/app/models/solid_queue/record.rb b/app/models/solid_queue/record.rb index 8c7000bfe..be4c3c620 100644 --- a/app/models/solid_queue/record.rb +++ b/app/models/solid_queue/record.rb @@ -4,6 +4,8 @@ module SolidQueue class Record < ActiveRecord::Base self.abstract_class = true + include DistinctValues + connects_to(**SolidQueue.connects_to) if SolidQueue.connects_to class << self diff --git a/app/models/solid_queue/record/distinct_values.rb b/app/models/solid_queue/record/distinct_values.rb new file mode 100644 index 000000000..59b2545c3 --- /dev/null +++ b/app/models/solid_queue/record/distinct_values.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module SolidQueue + class Record + module DistinctValues + extend ActiveSupport::Concern + + # PostgreSQL has no native loose index scan, so a plain DISTINCT on a leading + # index column degrades to a full index scan on large tables. We emulate one + # with a recursive CTE that walks the index jumping between distinct values. + class_methods do + def distinct_values_of(column) + if loose_index_scan_emulation_needed? + loose_distinct_via_recursive_cte(column) + else + distinct.pluck(column) + end + end + + private + def loose_index_scan_emulation_needed? + connection.adapter_name == "PostgreSQL" + end + + # Emulates a loose index scan, honoring the current scope (e.g. LIKE prefixes) + # by building the anchor and the recursive step as scoped relations, whose + # #to_sql inlines any bind parameters so they can be embedded in the raw CTE. + def loose_distinct_via_recursive_cte(column) + col = connection.quote_column_name(column) + + connection.select_values(<<~SQL.squish) + WITH RECURSIVE t AS ( + (#{next_distinct_value(col, "#{col} IS NOT NULL")}) + UNION ALL + SELECT (#{next_distinct_value(col, "#{col} > t.#{col}")}) FROM t WHERE t.#{col} IS NOT NULL + ) + SELECT #{col} FROM t WHERE #{col} IS NOT NULL + SQL + end + + # Smallest value of `col` within the current scope that matches `condition`. + def next_distinct_value(col, condition) + all.where(Arel.sql(condition)).reorder(Arel.sql(col)).limit(1).select(Arel.sql(col)).to_sql + end + end + end + end +end diff --git a/test/models/solid_queue/ready_execution_test.rb b/test/models/solid_queue/ready_execution_test.rb index dd9269ca1..a5f40659a 100644 --- a/test/models/solid_queue/ready_execution_test.rb +++ b/test/models/solid_queue/ready_execution_test.rb @@ -168,6 +168,40 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase claimed_jobs.map(&:queue_name) end + test "distinct_values_of returns all distinct queue names" do + AddToBufferJob.perform_later("hey") # goes to background queue + + names = SolidQueue::ReadyExecution.distinct_values_of(:queue_name) + assert_includes names, "backend" + assert_includes names, "background" + assert_equal 2, names.size + end + + test "distinct_values_of honors a chained where" do + AddToBufferJob.perform_later("hey") # background queue + + names = SolidQueue::ReadyExecution.where("queue_name LIKE ?", "back%").distinct_values_of(:queue_name) + assert_includes names, "backend" + assert_includes names, "background" + assert_equal 2, names.size + + names = SolidQueue::ReadyExecution.where("queue_name LIKE ?", "backe%").distinct_values_of(:queue_name) + assert_equal [ "backend" ], names + end + + test "distinct_values_of returns empty array for no matches" do + names = SolidQueue::ReadyExecution.where("queue_name LIKE ?", "nonexistent%").distinct_values_of(:queue_name) + assert_equal [], names + end + + test "distinct_values_of returns empty array on empty table" do + SolidQueue::ReadyExecution.delete_all + SolidQueue::Job.delete_all + + names = SolidQueue::ReadyExecution.distinct_values_of(:queue_name) + assert_equal [], names + end + test "discard all" do 3.times { |i| AddToBufferJob.perform_later(i) }