diff --git a/.changeset/task-concurrency.md b/.changeset/task-concurrency.md index dea1c3bf1e7..d9da5712416 100644 --- a/.changeset/task-concurrency.md +++ b/.changeset/task-concurrency.md @@ -21,3 +21,5 @@ export const generateSummary = task({ `perKey` caps each `concurrencyKey` pool and `total` caps across everything, keys or not. The queue-level `concurrencyLimit` option keeps working unchanged and is deprecated in favor of `concurrency`. Enforcement happens server-side; servers without support accept the option but do not enforce it yet. Manage limits at runtime with the new `concurrencyLimits` namespace: `list()` and `retrieve(name)` report each limit's bounds plus its live `running` and `queued` counts, `override(name, { perKey, total })` changes only the given bounds (overriding `total` to `0` pauses the limit), and `reset(name)` restores the declared values. + +Queue reads (`queues.list()` and `queues.retrieve()`) now report a `version` that discriminates the shape: `V1` queues keep today's fields (their own `concurrencyLimit` and its override state), while `V2` queues (tasks declared with `concurrency`) carry no queue-level concurrency, since their limits are read and overridden through `concurrencyLimits` (a task's inline limit under its derived `task/` name). Existing reads keep compiling: a `V2` queue reports `concurrencyLimit` as null and `concurrency` as undefined. diff --git a/apps/webapp/app/components/billing/OrgBanner.tsx b/apps/webapp/app/components/billing/OrgBanner.tsx index acf10f2469d..a10cced1cf5 100644 --- a/apps/webapp/app/components/billing/OrgBanner.tsx +++ b/apps/webapp/app/components/billing/OrgBanner.tsx @@ -14,7 +14,7 @@ import { import { useOptionalProject, useProject } from "~/hooks/useProject"; import { useShowSelfServe } from "~/hooks/useShowSelfServe"; import { useCurrentPlan } from "~/routes/_app.orgs.$organizationSlug/route"; -import { v3BillingLimitsPath, v3BillingPath, v3QueuesPath } from "~/utils/pathBuilder"; +import { v3BillingLimitsPath, v3BillingPath, concurrencyPath } from "~/utils/pathBuilder"; import { ENVIRONMENT_PAUSE_SOURCE_BILLING_LIMIT } from "~/utils/environmentPauseSource"; function getUpgradeResetDate(): Date { @@ -54,7 +54,7 @@ export function OrgBanner() { showSelfServe, }); - const hideQueuesButton = location.pathname.endsWith("/queues"); + const hideConcurrencyButton = location.pathname.endsWith("/concurrency"); const hideBillingLimitBanner = location.pathname.endsWith("/settings/billing-limits"); switch (bannerKind) { @@ -70,7 +70,7 @@ export function OrgBanner() { return isArchived ? ( ) : ( - + ); default: return null; @@ -209,7 +209,7 @@ function PausedEnvironmentBanner({ hideButton }: { hideButton: boolean }) { hideButton ? undefined : ( Manage diff --git a/apps/webapp/app/components/dashboard-agent/DashboardAgentPanel.tsx b/apps/webapp/app/components/dashboard-agent/DashboardAgentPanel.tsx index 986c3f9bdf7..6bcc738869c 100644 --- a/apps/webapp/app/components/dashboard-agent/DashboardAgentPanel.tsx +++ b/apps/webapp/app/components/dashboard-agent/DashboardAgentPanel.tsx @@ -50,7 +50,7 @@ import { } from "./unread-counts"; import { AgentPanelColumn } from "./panel-layout"; import { markerAfterActiveChat, markerAfterActivity } from "./thinking-marker"; -import { concurrencyPath } from "~/utils/pathBuilder"; +import { concurrencyLimitsPath } from "~/utils/pathBuilder"; function serializePageContext(pageContext: AgentPageContext): string | undefined { try { @@ -131,7 +131,7 @@ export function DashboardAgentPanel({ const currentPage = agentPageLabel(pageContext, location.pathname); const pagePaths = useMemo>( - () => ({ raise_env_limit: concurrencyPath(organization, project, environment) }), + () => ({ raise_env_limit: concurrencyLimitsPath(organization, project, environment) }), [organization, project, environment] ); diff --git a/apps/webapp/app/components/dashboard-agent/page-label.ts b/apps/webapp/app/components/dashboard-agent/page-label.ts index c5e949117fa..70126e0f4c4 100644 --- a/apps/webapp/app/components/dashboard-agent/page-label.ts +++ b/apps/webapp/app/components/dashboard-agent/page-label.ts @@ -24,7 +24,7 @@ const KIND_LABELS: Record, string> = { alerts: "Alerts", apikeys: "API keys", envvars: "Environment variables", - concurrency: "Concurrency", + concurrency: "Concurrency limits", regions: "Regions", settings: "Settings", waitpoints: "Waitpoints", @@ -50,6 +50,7 @@ const SECTION_LABELS: Record = { "bulk-actions": "Bulk actions", branches: "Branches", concurrency: "Concurrency", + "concurrency-limits": "Concurrency limits", dashboards: "Dashboards", deployments: "Deployments", "dev-branches": "Branches", diff --git a/apps/webapp/app/components/navigation/favoritePages.tsx b/apps/webapp/app/components/navigation/favoritePages.tsx index 50668bae889..0b9742dcca2 100644 --- a/apps/webapp/app/components/navigation/favoritePages.tsx +++ b/apps/webapp/app/components/navigation/favoritePages.tsx @@ -223,7 +223,8 @@ const ENV_PAGE_META: Record = { logs: { icon: "logs", name: "Logs" }, errors: { icon: "errors", name: "Errors", singular: "Error" }, query: { icon: "query", name: "Query" }, - queues: { icon: "queues", name: "Queues", singular: "Queue" }, + queues: { icon: "queues", name: "Concurrency" }, + concurrency: { icon: "queues", name: "Concurrency" }, dashboards: { icon: "dashboards", name: "Dashboards", singular: "Dashboard" }, deployments: { icon: "deployments", name: "Deploys", singular: "Deploy" }, "environment-variables": { icon: "environment-variables", name: "Environment variables" }, @@ -234,7 +235,7 @@ const ENV_PAGE_META: Record = { "bulk-actions": { icon: "bulk-actions", name: "Bulk actions", singular: "Bulk action" }, apikeys: { icon: "apikeys", name: "API keys" }, alerts: { icon: "alerts", name: "Alerts", singular: "Alert" }, - concurrency: { icon: "concurrency", name: "Concurrency" }, + "concurrency-limits": { icon: "concurrency", name: "Concurrency limits" }, limits: { icon: "limits", name: "Limits" }, schedules: { icon: "schedules", name: "Schedules", singular: "Schedule" }, test: { icon: "test", name: "Test", singular: "Test" }, diff --git a/apps/webapp/app/components/navigation/sideMenuSections.tsx b/apps/webapp/app/components/navigation/sideMenuSections.tsx index 87d5da61195..6a9c0f7145a 100644 --- a/apps/webapp/app/components/navigation/sideMenuSections.tsx +++ b/apps/webapp/app/components/navigation/sideMenuSections.tsx @@ -23,7 +23,7 @@ import { type OrgForPath, type ProjectForPath, branchesPath, - concurrencyPath, + concurrencyLimitsPath, limitsPath, queryPath, regionsPath, @@ -39,7 +39,7 @@ import { v3ProjectAlertsPath, v3ProjectSettingsIntegrationsPath, v3PromptsPath, - v3QueuesPath, + concurrencyPath, v3WaitpointTokensPath, } from "~/utils/pathBuilder"; import { AlphaBadge, NewBadge } from "../FeatureBadges"; @@ -160,10 +160,10 @@ export function buildSideMenuSections({ } satisfies SideMenuItemConfig, { id: "queues", - name: "Queues", + name: "Concurrency", icon: QueuesIcon, activeIconColor: "text-queues", - to: v3QueuesPath(organization, project, environment), + to: concurrencyPath(organization, project, environment), dataAction: "queues", } satisfies SideMenuItemConfig, { @@ -269,10 +269,10 @@ export function buildSideMenuSections({ ? [ { id: "concurrency", - name: "Concurrency", + name: "Concurrency limits", icon: ConcurrencyIcon, activeIconColor: "text-text-bright", - to: concurrencyPath(organization, project, environment), + to: concurrencyLimitsPath(organization, project, environment), dataAction: "concurrency", } satisfies SideMenuItemConfig, ] diff --git a/apps/webapp/app/components/queues/QueueControls.tsx b/apps/webapp/app/components/queues/QueueControls.tsx index b2499bb6d0e..e27facda859 100644 --- a/apps/webapp/app/components/queues/QueueControls.tsx +++ b/apps/webapp/app/components/queues/QueueControls.tsx @@ -1,9 +1,9 @@ import { AdjustmentsHorizontalIcon, PauseIcon, PlayIcon } from "@heroicons/react/20/solid"; import { DialogClose } from "@radix-ui/react-dialog"; import { Form, useNavigation } from "@remix-run/react"; -import type { QueueItem } from "@trigger.dev/core/v3/schemas"; import { useEffect, useState } from "react"; import { cn } from "~/utils/cn"; +import type { QueueLimits } from "~/components/queues/queue-limits"; import { Button, type ButtonVariant } from "~/components/primitives/Buttons"; import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "~/components/primitives/Dialog"; import { FormButtons } from "~/components/primitives/FormButtons"; @@ -159,7 +159,12 @@ export function QueueOverrideConcurrencyButton({ environmentConcurrencyLimit, trigger, }: { - queue: QueueItem & { concurrencyLimitOverridePercent: number | null }; + queue: { + id: string; + name: string; + limits: QueueLimits; + concurrencyLimitOverridePercent: number | null; + }; environmentConcurrencyLimit: number; /** How to render the dialog trigger. "menu-item" (default) is a PopoverMenuItem for row menus; * "button" is a standalone labeled button; "icon" is an icon-only button with the label in a @@ -172,14 +177,14 @@ export function QueueOverrideConcurrencyButton({ queue.concurrencyLimitOverridePercent !== null ? "percent" : "absolute" ); const [concurrencyLimit, setConcurrencyLimit] = useState( - queue.concurrencyLimit?.toString() ?? environmentConcurrencyLimit.toString() + queue.limits.perKey.current?.toString() ?? environmentConcurrencyLimit.toString() ); const [percent, setPercent] = useState( queue.concurrencyLimitOverridePercent?.toString() ?? "100" ); - const isOverridden = !!queue.concurrency?.overriddenAt; - const currentLimit = queue.concurrencyLimit ?? environmentConcurrencyLimit; + const isOverridden = !!queue.limits.perKey.overriddenAt; + const currentLimit = queue.limits.perKey.current ?? environmentConcurrencyLimit; useEffect(() => { if (navigation.state === "loading" || navigation.state === "idle") { @@ -277,10 +282,10 @@ export function QueueOverrideConcurrencyButton({ {isOverridden ? ( This queue's concurrency limit is currently overridden to {currentLimit}. - {typeof queue.concurrency?.base === "number" && - ` The original limit set in code was ${queue.concurrency.base}.`}{" "} + {typeof queue.limits.perKey.base === "number" && + ` The original limit set in code was ${queue.limits.perKey.base}.`}{" "} You can update the override or remove it to restore the{" "} - {typeof queue.concurrency?.base === "number" + {typeof queue.limits.perKey.base === "number" ? "limit set in code" : "environment concurrency limit"} . @@ -288,7 +293,7 @@ export function QueueOverrideConcurrencyButton({ ) : ( Override this queue's concurrency limit. The current limit is {currentLimit}, which is - set {queue.concurrencyLimit !== null ? "in code" : "by the environment"}. + set {queue.limits.perKey.current !== null ? "in code" : "by the environment"}. )}
setIsOpen(false)} className="space-y-3"> diff --git a/apps/webapp/app/components/queues/QueueMetricCards.tsx b/apps/webapp/app/components/queues/QueueMetricCards.tsx index 7c87a1e2d02..8a9fa8f5549 100644 --- a/apps/webapp/app/components/queues/QueueMetricCards.tsx +++ b/apps/webapp/app/components/queues/QueueMetricCards.tsx @@ -102,6 +102,12 @@ type QueueMetricChartProps = { * are config values that existed all along, so carry the first value backward instead. */ carryBackfill?: string[]; + /** + * Column that marks a bucket as genuinely sampled. When set, carryBackfill only + * overwrites buckets where this column is absent or zero, so history from before + * a config value existed keeps its truthful gap instead of inheriting the value. + */ + carryBackfillGuard?: string; /** Show the series legend below the chart (use for multi-series charts). */ showLegend?: boolean; /** @@ -141,6 +147,7 @@ export function QueueMetricChart({ defaultPeriod, warningOverlay, carryBackfill, + carryBackfillGuard, thresholdStroke, onHasDataChange, minBucketSeconds, @@ -163,6 +170,7 @@ export function QueueMetricChart({ }; const hasSamples = sampleCountColumn ? toNumber(r[sampleCountColumn]) > 0 : true; for (const s of series) point[s.key] = hasSamples ? toNumber(r[s.key]) : null; + if (carryBackfillGuard) point[carryBackfillGuard] = toNumber(r[carryBackfillGuard]); return point; }) .filter((p) => Number.isFinite(p.bucket)); @@ -174,12 +182,15 @@ export function QueueMetricChart({ const first = points.findIndex((p) => toNumber(p[key]) > 0); if (first > 0) { const value = points[first]![key]!; - for (let i = 0; i < first; i++) points[i]![key] = value; + for (let i = 0; i < first; i++) { + if (carryBackfillGuard && toNumber(points[i]![carryBackfillGuard]) > 0) continue; + points[i]![key] = value; + } } } } return points; - }, [rows, series, carryBackfill, sampleCountColumn]); + }, [rows, series, carryBackfill, carryBackfillGuard, sampleCountColumn]); const chartConfig = useMemo(() => { const cfg: ChartConfig = {}; @@ -340,7 +351,7 @@ export function QueueSidebarStats({ }; const { rows, showLoading } = useQueueMetric( - `SELECT max(max_queued) AS peak_queued,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS worst_p95\nFROM queue_metrics`, + `SELECT max(max_queued) AS peak_queued,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS worst_p95\nFROM concurrency_metrics`, { ids, timeRange, queueName, defaultPeriod } ); const row = rows[0]; diff --git a/apps/webapp/app/components/queues/queue-limits.ts b/apps/webapp/app/components/queues/queue-limits.ts new file mode 100644 index 00000000000..457103ff8e4 --- /dev/null +++ b/apps/webapp/app/components/queues/queue-limits.ts @@ -0,0 +1,32 @@ +/** + * Dashboard-only view of a queue row's configured bounds. The public QueueItem + * shape is version-discriminated (V2 queues carry no queue-level concurrency), + * but the dashboard shows configured limits for every row, so presenters attach + * this alongside the public fields. + */ +type QueueLimitBound = { + /** The enforced value right now (declared, or the override when one is active) */ + current: number | null; + /** The declared value an override reverts to */ + base: number | null; + /** The overridden value, when an override is active */ + override: number | null; + overriddenAt: Date | null; + /** Display name of who applied the override (null when via the API) */ + overriddenBy: string | null; +}; + +type QueueTotalBound = { + current: number; + base: number | null; + override: number | null; + overriddenAt: Date | null; + /** Runs in flight across every pool of the row (keyed and keyless) */ + running: number | null; +}; + +export type QueueLimits = { + perKey: QueueLimitBound; + /** Null when the row declares no total bound */ + total: QueueTotalBound | null; +}; diff --git a/apps/webapp/app/components/runs/v3/QueueName.tsx b/apps/webapp/app/components/runs/v3/QueueName.tsx index e65b86a220e..056434652bb 100644 --- a/apps/webapp/app/components/runs/v3/QueueName.tsx +++ b/apps/webapp/app/components/runs/v3/QueueName.tsx @@ -1,19 +1,49 @@ import { TasksIcon } from "~/assets/icons/TasksIcon"; +import { ConcurrencyIcon } from "~/assets/icons/ConcurrencyIcon"; import { SimpleTooltip } from "~/components/primitives/Tooltip"; import { cn } from "~/utils/cn"; import { RectangleStackIcon } from "@heroicons/react/20/solid"; +const LIMIT_PREFIX = "limit/"; +const TASK_PREFIX = "task/"; + export function QueueName({ name, type, + kind, paused, className, }: { name: string; type: "task" | "custom"; + /** "limit" rows are named concurrency limits rather than queues. */ + kind?: "queue" | "limit"; paused?: boolean; className?: string; }) { + if (kind === "limit") { + const displayName = name.startsWith(LIMIT_PREFIX) ? name.slice(LIMIT_PREFIX.length) : name; + return ( + + + } + content={ + displayName.startsWith(TASK_PREFIX) + ? `This is the inline concurrency limit of your "${displayName.slice( + TASK_PREFIX.length + )}" task` + : "This is a named concurrency limit declared in your code." + } + /> + {displayName} + + ); + } + return ( {type === "task" ? ( diff --git a/apps/webapp/app/presenters/v3/BuiltInDashboards.server.ts b/apps/webapp/app/presenters/v3/BuiltInDashboards.server.ts index d831568248d..7c609322ab2 100644 --- a/apps/webapp/app/presenters/v3/BuiltInDashboards.server.ts +++ b/apps/webapp/app/presenters/v3/BuiltInDashboards.server.ts @@ -634,7 +634,7 @@ const queuesDashboard: BuiltInDashboard = { "t-pressure": { title: "Queue pressure", query: "", display: { type: "title" } }, pressure: { title: "Queue pressure", - query: `SELECT queue,\n argMax(max_running, bucket_start) AS running,\n argMax(max_queued, bucket_start) AS queued,\n argMax(max_limit, bucket_start) AS limit,\n running + queued AS demand,\n max(max_queued) AS peak_queued,\n sum(throttled_count) AS throttled,\n multiIf(running >= limit AND queued > 0, 'queue-limited', queued > 0, 'backlogged', 'healthy') AS status\nFROM queue_metrics\nGROUP BY queue\nORDER BY peak_queued DESC`, + query: `SELECT queue,\n argMax(max_running, bucket_start) AS running,\n argMax(max_queued, bucket_start) AS queued,\n argMax(max_limit, bucket_start) AS limit,\n running + queued AS demand,\n max(max_queued) AS peak_queued,\n sum(throttled_count) AS throttled,\n multiIf(running >= limit AND queued > 0, 'queue-limited', queued > 0, 'backlogged', 'healthy') AS status\nFROM concurrency_metrics\nGROUP BY queue\nORDER BY peak_queued DESC`, display: { type: "table", prettyFormatting: true, @@ -644,7 +644,7 @@ const queuesDashboard: BuiltInDashboard = { "t-trends": { title: "Per-queue trends", query: "", display: { type: "title" } }, "running-q": { title: "Running by queue", - query: `SELECT timeBucket() AS t, queue, max(max_running) AS running\nFROM queue_metrics\nGROUP BY t, queue\nORDER BY t`, + query: `SELECT timeBucket() AS t, queue, max(max_running) AS running\nFROM concurrency_metrics\nGROUP BY t, queue\nORDER BY t`, // Grouped gauge: carry each queue's running across idle buckets (per-group LOCF). fillGaps: true, display: { @@ -661,7 +661,7 @@ const queuesDashboard: BuiltInDashboard = { }, "queued-q": { title: "Queue depth (backlog) by queue", - query: `SELECT timeBucket() AS t, queue, max(max_queued) AS queued\nFROM queue_metrics\nGROUP BY t, queue\nORDER BY t`, + query: `SELECT timeBucket() AS t, queue, max(max_queued) AS queued\nFROM concurrency_metrics\nGROUP BY t, queue\nORDER BY t`, // Grouped gauge: carry each queue's backlog across idle buckets (per-group LOCF). fillGaps: true, display: { @@ -678,7 +678,7 @@ const queuesDashboard: BuiltInDashboard = { }, "throttled-q": { title: "Throttled buckets by queue", - query: `SELECT timeBucket() AS t, queue, sum(throttled_count) AS throttled\nFROM queue_metrics\nGROUP BY t, queue\nORDER BY t`, + query: `SELECT timeBucket() AS t, queue, sum(throttled_count) AS throttled\nFROM concurrency_metrics\nGROUP BY t, queue\nORDER BY t`, // Grouped counter: per-group zero-fill so idle buckets read 0, not a gap. fillGaps: true, display: { @@ -697,7 +697,7 @@ const queuesDashboard: BuiltInDashboard = { title: "Enqueued vs started", // Counter states merge per queue, then sum outside: a single merge across queues // mixes unrelated odometers and returns wrong totals. - query: `SELECT t, sum(enq) AS enqueued, sum(st) AS started\nFROM (\n SELECT timeBucket() AS t, queue,\n deltaSumTimestampMerge(enqueue_delta) AS enq,\n deltaSumTimestampMerge(started_delta) AS st\n FROM queue_metrics\n GROUP BY t, queue\n)\nGROUP BY t\nORDER BY t`, + query: `SELECT t, sum(enq) AS enqueued, sum(st) AS started\nFROM (\n SELECT timeBucket() AS t, queue,\n deltaSumTimestampMerge(enqueue_delta) AS enq,\n deltaSumTimestampMerge(started_delta) AS st\n FROM concurrency_metrics\n GROUP BY t, queue\n)\nGROUP BY t\nORDER BY t`, display: { type: "chart", chartType: "line", diff --git a/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts b/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts index 7db2a6d2e39..05353f2ccdb 100644 --- a/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts @@ -7,9 +7,16 @@ import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstan import { logger } from "~/services/logger.server"; import { engine } from "~/v3/runEngine.server"; import { BasePresenter } from "./basePresenter.server"; -import { toQueueItem } from "./QueueRetrievePresenter.server"; +import { toQueueItem, toQueueLimits } from "./QueueRetrievePresenter.server"; +import type { QueueLimits } from "~/components/queues/queue-limits"; -type QueueListEngine = Pick; +type QueueListEngine = Pick< + RunEngine, + | "lengthOfQueues" + | "currentConcurrencyOfQueues" + | "totalConcurrencyOfQueues" + | "gateQueuedCountOfQueues" +>; export const QUEUE_LIST_DEFAULT_ITEMS_PER_PAGE = 25; const MAX_ITEMS_PER_PAGE = 100; @@ -34,8 +41,13 @@ const queueListSelect = { concurrencyLimitOverriddenAt: true, concurrencyLimitOverriddenBy: true, concurrencyLimitOverridePercent: true, + totalConcurrencyLimit: true, + totalConcurrencyLimitBase: true, + totalConcurrencyLimitOverriddenAt: true, type: true, paused: true, + role: true, + concurrencyVersion: true, } satisfies Prisma.TaskQueueSelect; type QueueListRow = Prisma.TaskQueueGetPayload<{ select: typeof queueListSelect }>; @@ -44,6 +56,12 @@ type QueueListRow = Prisma.TaskQueueGetPayload<{ select: typeof queueListSelect // schema (that's a public contract), so we surface it as an extra field on the list item. type QueueListItem = ReturnType & { concurrencyLimitOverridePercent: number | null; + /** "queue" rows wait and order runs; "limit" rows are named concurrency limits. */ + kind: "queue" | "limit"; + /** V2 rows hold the new perKey/total vocabulary in their limit columns. */ + concurrencyVersion: "V1" | "V2"; + /** The row's configured bounds, dashboard-only (the public shape hides them on V2). */ + limits: QueueLimits; }; type QueueListPagination = @@ -64,22 +82,49 @@ function formatClickhouseDateTime(date: Date): string { function buildQueueListWhere( environmentId: string, query: string | undefined, - type: "task" | "custom" | undefined + type: "task" | "custom" | undefined, + includeLimits: boolean ): Prisma.TaskQueueWhereInput { const trimmedQuery = query?.trim(); - return { + const common = { runtimeEnvironmentId: environmentId, - role: "QUEUE" as const, - version: "V2", + version: "V2" as const, name: trimmedQuery ? { contains: trimmedQuery, - mode: "insensitive", + mode: "insensitive" as const, } : undefined, type: type ? typeToDBQueueType[type] : undefined, }; + + /** Only the dashboard interleaves named limits, and the type filter names queue + * shapes, so either condition scopes the list to queue rows; the public queues + * API always stays queue-only. Boundless rows in the anonymous limit/task/ + * namespace are retired (their inline limit moved onto the task's own queue) + * and stay hidden; boundless NAMED limits are real uncapped rows and show. */ + if (includeLimits && !type) { + return { + ...common, + OR: [ + { role: "QUEUE" as const }, + { + role: "LIMIT" as const, + OR: [ + { name: { not: { startsWith: "limit/task/" } } }, + { concurrencyLimit: { not: null } }, + { totalConcurrencyLimit: { not: null } }, + ], + }, + ], + }; + } + + return { + ...common, + role: "QUEUE" as const, + }; } export class QueueListPresenter extends BasePresenter { @@ -103,6 +148,7 @@ export class QueueListPresenter extends BasePresenter { page, type, sort = "name", + includeLimits = false, }: { environment: AuthenticatedEnvironment; query?: string; @@ -110,13 +156,21 @@ export class QueueListPresenter extends BasePresenter { perPage?: number; type?: "task" | "custom"; sort?: QueueListSort; + includeLimits?: boolean; }): Promise { const hasFilters = Boolean(query?.trim()) || type !== undefined; if (sort !== "name") { // Ranking is additive: any failure or unsupported input falls back to name order. try { - const ranked = await this.getRankedQueues(environment, query, page, type, sort); + const ranked = await this.getRankedQueues( + environment, + query, + page, + type, + sort, + includeLimits + ); if (ranked) { return ranked; } @@ -126,7 +180,13 @@ export class QueueListPresenter extends BasePresenter { } if (hasFilters) { - const { queues, hasMore } = await this.getFilteredQueues(environment, query, page, type); + const { queues, hasMore } = await this.getFilteredQueues( + environment, + query, + page, + type, + includeLimits + ); return { queues, @@ -140,11 +200,11 @@ export class QueueListPresenter extends BasePresenter { } const totalQueues = await this._replica.taskQueue.count({ - where: buildQueueListWhere(environment.id, query, type), + where: buildQueueListWhere(environment.id, query, type, includeLimits), }); return { - queues: await this.getUnfilteredQueues(environment, page, type), + queues: await this.getUnfilteredQueues(environment, page, type, includeLimits), pagination: { mode: "unfiltered" as const, currentPage: page, @@ -165,7 +225,8 @@ export class QueueListPresenter extends BasePresenter { query: string | undefined, page: number, type: "task" | "custom" | undefined, - sort: Exclude + sort: Exclude, + includeLimits: boolean ) { if (type !== undefined) { return null; @@ -214,7 +275,7 @@ export class QueueListPresenter extends BasePresenter { return null; } - const where = buildQueueListWhere(environment.id, query, type); + const where = buildQueueListWhere(environment.id, query, type, includeLimits); const totalQueues = await this._replica.taskQueue.count({ where }); let rankedPageQueues: QueueListRow[] = []; @@ -285,10 +346,11 @@ export class QueueListPresenter extends BasePresenter { environment: AuthenticatedEnvironment, query: string | undefined, page: number, - type: "task" | "custom" | undefined + type: "task" | "custom" | undefined, + includeLimits: boolean ) { const queues = await this._replica.taskQueue.findMany({ - where: buildQueueListWhere(environment.id, query, type), + where: buildQueueListWhere(environment.id, query, type, includeLimits), select: queueListSelect, orderBy: { orderableName: "asc", @@ -308,10 +370,11 @@ export class QueueListPresenter extends BasePresenter { private async getUnfilteredQueues( environment: AuthenticatedEnvironment, page: number, - type: "task" | "custom" | undefined + type: "task" | "custom" | undefined, + includeLimits: boolean ) { const queues = await this._replica.taskQueue.findMany({ - where: buildQueueListWhere(environment.id, undefined, type), + where: buildQueueListWhere(environment.id, undefined, type, includeLimits), select: queueListSelect, orderBy: { orderableName: "asc", @@ -334,20 +397,49 @@ export class QueueListPresenter extends BasePresenter { concurrencyLimitOverriddenAt: Date | null; concurrencyLimitOverriddenBy: string | null; concurrencyLimitOverridePercent: Prisma.Decimal | null; + totalConcurrencyLimit: number | null; + totalConcurrencyLimitBase: number | null; + totalConcurrencyLimitOverriddenAt: Date | null; type: TaskQueueType; paused: boolean; + role: "QUEUE" | "LIMIT"; + concurrencyVersion: "V1" | "V2"; }[] ): Promise { - const [queuedByQueue, runningByQueue] = await Promise.all([ - this.engineClient.lengthOfQueues( - environment, - queues.map((q) => q.name) - ), - this.engineClient.currentConcurrencyOfQueues( - environment, - queues.map((q) => q.name) - ), - ]); + const queueRows = queues.filter((q) => q.role === "QUEUE"); + const limitRows = queues.filter((q) => q.role === "LIMIT"); + /** + * Queue rows read their zset length and home concurrency; limit rows read the + * group set (every holder, keyed or keyless) as running and the per-gate queued + * counter as queued. The group read also serves queue rows with a total cap. + */ + const rowsWithGroupRead = [ + ...queueRows.filter((q) => q.totalConcurrencyLimit !== null), + ...limitRows, + ]; + const [queuedByQueue, runningByQueue, totalRunningByQueue, gateQueuedByQueue] = + await Promise.all([ + this.engineClient.lengthOfQueues( + environment, + queueRows.map((q) => q.name) + ), + this.engineClient.currentConcurrencyOfQueues( + environment, + queueRows.map((q) => q.name) + ), + rowsWithGroupRead.length > 0 + ? this.engineClient.totalConcurrencyOfQueues( + environment, + rowsWithGroupRead.map((q) => q.name) + ) + : Promise.resolve({} as Record), + limitRows.length > 0 + ? this.engineClient.gateQueuedCountOfQueues( + environment, + limitRows.map((q) => q.name) + ) + : Promise.resolve({} as Record), + ]); // Manually "join" the overridden users because there is no way to implement the relationship // in prisma without adding a foreign key constraint @@ -360,26 +452,45 @@ export class QueueListPresenter extends BasePresenter { const overriddenByMap = new Map(overriddenByUsers.map((u) => [u.id, u])); - return queues.map((queue) => ({ - ...toQueueItem({ - friendlyId: queue.friendlyId, - name: queue.name, - type: queue.type, - running: runningByQueue[queue.name] ?? 0, - queued: queuedByQueue[queue.name] ?? 0, - concurrencyLimit: queue.concurrencyLimit ?? null, - concurrencyLimitBase: queue.concurrencyLimitBase ?? null, - concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt ?? null, - concurrencyLimitOverriddenBy: queue.concurrencyLimitOverriddenBy - ? (overriddenByMap.get(queue.concurrencyLimitOverriddenBy) ?? null) - : null, - paused: queue.paused, - }), - // Prisma returns Decimal; the client only needs a plain number (null for absolute overrides). - concurrencyLimitOverridePercent: - queue.concurrencyLimitOverridePercent !== null - ? Number(queue.concurrencyLimitOverridePercent) - : null, - })); + return queues.map((queue) => { + const overriddenByUser = queue.concurrencyLimitOverriddenBy + ? (overriddenByMap.get(queue.concurrencyLimitOverriddenBy) ?? null) + : null; + return { + ...toQueueItem({ + friendlyId: queue.friendlyId, + name: queue.name, + type: queue.type, + version: queue.concurrencyVersion, + running: + queue.role === "LIMIT" + ? (totalRunningByQueue[queue.name] ?? 0) + : (runningByQueue[queue.name] ?? 0), + queued: + queue.role === "LIMIT" + ? (gateQueuedByQueue[queue.name] ?? 0) + : (queuedByQueue[queue.name] ?? 0), + concurrencyLimit: queue.concurrencyLimit ?? null, + concurrencyLimitBase: queue.concurrencyLimitBase ?? null, + concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt ?? null, + concurrencyLimitOverriddenBy: overriddenByUser, + paused: queue.paused, + }), + // Prisma returns Decimal; the client only needs a plain number (null for absolute overrides). + concurrencyLimitOverridePercent: + queue.concurrencyLimitOverridePercent !== null + ? Number(queue.concurrencyLimitOverridePercent) + : null, + kind: queue.role === "LIMIT" ? ("limit" as const) : ("queue" as const), + concurrencyVersion: queue.concurrencyVersion, + limits: toQueueLimits(queue, { + totalRunning: + queue.totalConcurrencyLimit !== null ? (totalRunningByQueue[queue.name] ?? 0) : null, + overriddenByName: overriddenByUser + ? (overriddenByUser.displayName ?? overriddenByUser.name ?? null) + : null, + }), + }; + }); } } diff --git a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts index 6385b388d1f..b05d3ed5082 100644 --- a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts +++ b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts @@ -1,5 +1,11 @@ import { assertExhaustive } from "@trigger.dev/core"; -import { type Prettify, type QueueItem, type RetrieveQueueParam } from "@trigger.dev/core/v3"; +import { + QueueItem as QueueItemSchema, + type Prettify, + type QueueItem, + type RetrieveQueueParam, +} from "@trigger.dev/core/v3"; +import type { QueueLimits } from "~/components/queues/queue-limits"; import { type PrismaClientOrTransaction, type TaskQueue, @@ -24,6 +30,8 @@ export async function getQueue( environment: AuthenticatedEnvironment, queue: RetrieveQueueParam ) { + const role = "QUEUE" as const; + if (typeof queue === "string") { return joinQueueWithUser( prismaClient, @@ -31,7 +39,7 @@ export async function getQueue( where: { friendlyId: queue, runtimeEnvironmentId: environment.id, - role: "QUEUE", + role, }, }) ); @@ -45,7 +53,7 @@ export async function getQueue( where: { name: queueName, runtimeEnvironmentId: environment.id, - role: "QUEUE", + role, }, }) ); @@ -92,9 +100,15 @@ export class QueueRetrievePresenter extends BasePresenter { const results = await Promise.all([ engine.lengthOfQueues(environment, [queue.name]), engine.currentConcurrencyOfQueues(environment, [queue.name]), + queue.totalConcurrencyLimit != null + ? engine.totalConcurrencyOfQueues(environment, [queue.name]) + : undefined, ]); - // Transform queues to include running and queued counts + /** The returned queue = the public QueueItem fields plus dashboard extras + * (percent override source, configured bounds); the public API routes strip + * the extras via `toPublicQueueItem`. Prisma returns Decimal for the + * percent; the client only needs a plain number (null for absolute). */ return { success: true as const, queue: { @@ -102,6 +116,7 @@ export class QueueRetrievePresenter extends BasePresenter { friendlyId: queue.friendlyId, name: queue.name, type: queue.type, + version: queue.concurrencyVersion, running: results[1]?.[queue.name] ?? 0, queued: results[0]?.[queue.name] ?? 0, concurrencyLimit: queue.concurrencyLimit ?? null, @@ -110,19 +125,71 @@ export class QueueRetrievePresenter extends BasePresenter { concurrencyLimitOverriddenBy: queue.concurrencyLimitOverriddenBy ?? null, paused: queue.paused, }), - // The percent source-of-truth for percent-based overrides isn't part of the shared - // `QueueItem` schema (that's a public contract), so we surface it as an extra field on - // the returned queue — mirroring QueueListPresenter. Prisma returns Decimal; the client - // only needs a plain number (null for absolute overrides). concurrencyLimitOverridePercent: queue.concurrencyLimitOverridePercent !== null ? Number(queue.concurrencyLimitOverridePercent) : null, + limits: toQueueLimits(queue, { + totalRunning: + queue.totalConcurrencyLimit != null ? (results[2]?.[queue.name] ?? 0) : null, + overriddenByName: toQueueConcurrencyOverriddenBy( + queue.concurrencyLimitOverriddenBy ?? null + ), + }), }, }; } } +/** + * The dashboard-only configured bounds of a row, independent of the public + * shape's version discrimination (V2 queues expose no queue-level concurrency + * publicly, but the dashboard still shows what's configured). + */ +export function toQueueLimits( + row: { + concurrencyLimit: number | null; + concurrencyLimitBase: number | null; + concurrencyLimitOverriddenAt: Date | null; + totalConcurrencyLimit: number | null; + totalConcurrencyLimitBase: number | null; + totalConcurrencyLimitOverriddenAt: Date | null; + }, + extras: { totalRunning: number | null; overriddenByName: string | null } +): QueueLimits { + return { + perKey: { + current: row.concurrencyLimit, + base: row.concurrencyLimitOverriddenAt ? row.concurrencyLimitBase : row.concurrencyLimit, + override: row.concurrencyLimitOverriddenAt ? row.concurrencyLimit : null, + overriddenAt: row.concurrencyLimitOverriddenAt, + overriddenBy: extras.overriddenByName, + }, + total: + row.totalConcurrencyLimit !== null + ? { + current: row.totalConcurrencyLimit, + base: row.totalConcurrencyLimitOverriddenAt + ? row.totalConcurrencyLimitBase + : row.totalConcurrencyLimit, + override: row.totalConcurrencyLimitOverriddenAt ? row.totalConcurrencyLimit : null, + overriddenAt: row.totalConcurrencyLimitOverriddenAt, + running: extras.totalRunning, + } + : null, + }; +} + +/** + * The public API wire shape: the `QueueItem` contract exactly (dashboard extras + * stripped by the schema parse) plus the legacy field older clients require. + */ +export function toPublicQueueItem(item: QueueItem): QueueItem & { + releaseConcurrencyOnWaitpoint: boolean; +} { + return { ...QueueItemSchema.parse(item), releaseConcurrencyOnWaitpoint: true }; +} + export function queueTypeFromType(type: TaskQueueType) { switch (type) { case "NAMED": @@ -135,14 +202,17 @@ export function queueTypeFromType(type: TaskQueueType) { } /** - * Converts raw queue data into a standardized QueueItem format - * @param data Raw queue data containing required queue properties - * @returns A validated QueueItem object + * Converts raw queue data into the public QueueItem shape. The queue's version + * discriminates it: V1 carries the queue's own limit and override state; V2 + * carries neither (a V2 queue is only the line — concurrency lives on the task + * `concurrency` option and the `concurrencyLimits` surface), with + * `concurrencyLimit` kept as null so older clients keep parsing. */ export function toQueueItem(data: { friendlyId: string; name: string; type: TaskQueueType; + version: "V1" | "V2"; running: number; queued: number; concurrencyLimit: number | null; @@ -151,7 +221,7 @@ export function toQueueItem(data: { concurrencyLimitOverriddenBy: User | null; paused: boolean; }): QueueItem & { releaseConcurrencyOnWaitpoint: boolean } { - return { + const common = { id: data.friendlyId, //remove the task/ prefix if it exists name: data.name.replace(/^task\//, ""), @@ -159,6 +229,21 @@ export function toQueueItem(data: { running: data.running, queued: data.queued, paused: data.paused, + // TODO: This needs to be removed but keeping this here for now to avoid breaking existing clients + releaseConcurrencyOnWaitpoint: true, + }; + + if (data.version === "V2") { + return { + ...common, + version: "V2" as const, + concurrencyLimit: null, + }; + } + + return { + ...common, + version: "V1" as const, concurrencyLimit: data.concurrencyLimit, concurrency: { current: data.concurrencyLimit, @@ -167,8 +252,6 @@ export function toQueueItem(data: { overriddenBy: toQueueConcurrencyOverriddenBy(data.concurrencyLimitOverriddenBy), overriddenAt: data.concurrencyLimitOverriddenAt, }, - // TODO: This needs to be removed but keeping this here for now to avoid breaking existing clients - releaseConcurrencyOnWaitpoint: true, }; } diff --git a/apps/webapp/app/presenters/v3/reports/health/flow.ts b/apps/webapp/app/presenters/v3/reports/health/flow.ts index ce6ba724527..e1d2e77654f 100644 --- a/apps/webapp/app/presenters/v3/reports/health/flow.ts +++ b/apps/webapp/app/presenters/v3/reports/health/flow.ts @@ -116,7 +116,7 @@ export function interpretFlow(metrics: Metric[], input: HealthInput): Finding { exclusions: [], observations: finishedPerMin > 0 ? [{ code: "not_workers_platform", evidence: { finishedPerMin } }] : [], - recommendation: { code: "raise_env_limit", link: "concurrency" }, + recommendation: { code: "raise_env_limit", link: "concurrency-limits" }, usesAttribution: true, }; } else if (ev.throttledShare >= t.throttledShare && !pinned) { diff --git a/apps/webapp/app/presenters/v3/reports/health/health-data.ts b/apps/webapp/app/presenters/v3/reports/health/health-data.ts index 3c14b498e57..de2cc01b4cd 100644 --- a/apps/webapp/app/presenters/v3/reports/health/health-data.ts +++ b/apps/webapp/app/presenters/v3/reports/health/health-data.ts @@ -212,7 +212,7 @@ function queueWorstQuery(): string { return `SELECT queue AS name, argMax(max_queued, bucket_start) AS latest_queued -FROM queue_metrics +FROM concurrency_metrics GROUP BY queue ORDER BY latest_queued DESC LIMIT 20`; @@ -228,7 +228,7 @@ FROM ( SELECT deltaSumTimestampMerge(dlq_delta) AS dlq, argMax(max_queued, bucket_start) AS latest_queued - FROM queue_metrics + FROM concurrency_metrics GROUP BY queue )`; } diff --git a/apps/webapp/app/presenters/v3/reports/report-registry.ts b/apps/webapp/app/presenters/v3/reports/report-registry.ts index 23cc178befb..d8b3bc4e798 100644 --- a/apps/webapp/app/presenters/v3/reports/report-registry.ts +++ b/apps/webapp/app/presenters/v3/reports/report-registry.ts @@ -4,7 +4,7 @@ import { loadHealthInput } from "./health/health-data"; import { type ReportViewModel } from "./report-view-model"; /** A query table a report may read. Same table names the query API authorizes against. */ -export type ReportQueryTable = "runs" | "env_metrics" | "queue_metrics"; +export type ReportQueryTable = "runs" | "env_metrics" | "concurrency_metrics"; export type ReportLoader = { /** Authorization metadata: the route derives its per-table JWT scope check from this. */ @@ -19,7 +19,7 @@ function defineReport(loader: ReportLoader): ReportLoader> = { health: defineReport({ - tables: ["runs", "env_metrics", "queue_metrics"], + tables: ["runs", "env_metrics", "concurrency_metrics"], load: (env, period) => loadHealthInput(env, period), interpret: interpretHealth, }), diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency-limits/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency-limits/route.tsx new file mode 100644 index 00000000000..1df5a2f6c1a --- /dev/null +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency-limits/route.tsx @@ -0,0 +1,886 @@ +import { getFormProps, getInputProps, useForm } from "@conform-to/react"; +import { parseWithZod } from "@conform-to/zod"; +import { + ArrowDownIcon, + EnvelopeIcon, + ExclamationTriangleIcon, + InformationCircleIcon, +} from "@heroicons/react/20/solid"; +import { DialogClose } from "@radix-ui/react-dialog"; +import { Form, useActionData, useNavigation, useSearchParams } from "@remix-run/react"; +import { json, type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { tryCatch } from "@trigger.dev/core"; +import { useEffect, useState } from "react"; +import { typedjson, useTypedLoaderData } from "remix-typedjson"; +import simplur from "simplur"; +import { z } from "zod"; +import { AdminDebugTooltip } from "~/components/admin/debugTooltip"; +import { CopyableText } from "~/components/primitives/CopyableText"; +import { EnvironmentCombo } from "~/components/environments/EnvironmentLabel"; +import { Feedback } from "~/components/Feedback"; +import { + MainHorizontallyCenteredContainer, + PageBody, + PageContainer, +} from "~/components/layout/AppLayout"; +import { Button, LinkButton } from "~/components/primitives/Buttons"; +import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "~/components/primitives/Dialog"; +import { Fieldset } from "~/components/primitives/Fieldset"; +import { FormButtons } from "~/components/primitives/FormButtons"; +import { FormError } from "~/components/primitives/FormError"; +import { Header2, Header3 } from "~/components/primitives/Headers"; +import { Input } from "~/components/primitives/Input"; +import { InputGroup } from "~/components/primitives/InputGroup"; +import { InputNumberStepper } from "~/components/primitives/InputNumberStepper"; +import { Label } from "~/components/primitives/Label"; +import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import * as Property from "~/components/primitives/PropertyTable"; +import { SpinnerWhite } from "~/components/primitives/Spinner"; +import { + Table, + TableBody, + TableCell, + TableHeader, + TableHeaderCell, + TableRow, +} from "~/components/primitives/Table"; +import { InfoIconTooltip } from "~/components/primitives/Tooltip"; +import { useFeatures } from "~/hooks/useFeatures"; +import { useOrganization } from "~/hooks/useOrganizations"; +import { useShowSelfServe } from "~/hooks/useShowSelfServe"; +import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server"; +import { findProjectBySlug } from "~/models/project.server"; +import { + ManageConcurrencyPresenter, + type ConcurrencyResult, + type EnvironmentWithConcurrency, +} from "~/presenters/v3/ManageConcurrencyPresenter.server"; +import { + getCurrentPlan, + getPlans, + getSelfServePurchaseBlockReason, +} from "~/services/platform.v3.server"; +import { textLinkClassName } from "~/components/primitives/TextLink"; +import { requireUserId } from "~/services/session.server"; +import { cn } from "~/utils/cn"; +import { formatCurrency, formatNumber } from "~/utils/numberFormatter"; +import { concurrencyLimitsPath, EnvironmentParamSchema, v3BillingPath } from "~/utils/pathBuilder"; +import { AllocateConcurrencyService } from "~/v3/services/allocateConcurrency.server"; +import { SetConcurrencyAddOnService } from "~/v3/services/setConcurrencyAddOn.server"; +import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route"; +import { sectionAgentPageContext } from "~/components/dashboard-agent/suggested-prompts"; +import type { Handle } from "~/utils/handle"; + +export const handle: Handle = { + agentPageContext: () => sectionAgentPageContext("concurrency"), +}; +import { pageMeta } from "~/utils/pageTitle"; + +export const meta = pageMeta("Manage concurrency"); + +export const loader = async ({ request, params }: LoaderFunctionArgs) => { + const userId = await requireUserId(request); + const { + organizationSlug, + projectParam, + envParam: _envParam, + } = EnvironmentParamSchema.parse(params); + + const project = await findProjectBySlug(organizationSlug, projectParam, userId); + if (!project) { + throw new Response(undefined, { + status: 404, + statusText: "Project not found", + }); + } + + const presenter = new ManageConcurrencyPresenter(); + const [error, result] = await tryCatch( + presenter.call({ + userId: userId, + projectId: project.id, + organizationId: project.organizationId, + }) + ); + + if (error) { + throw new Response(undefined, { + status: 400, + statusText: error.message, + }); + } + + const plans = await tryCatch(getPlans()); + if (!plans) { + throw new Response(null, { status: 404, statusText: "Plans not found" }); + } + + return typedjson(result); +}; + +const FormSchema = z.discriminatedUnion("action", [ + z.object({ + action: z.enum(["purchase"]), + amount: z.coerce.number().min(0, "Amount must be 0 or more"), + }), + z.object({ + action: z.enum(["quota-increase"]), + amount: z.coerce.number().min(1, "Amount must be greater than 0"), + }), + z.object({ + action: z.enum(["allocate"]), + // It will only update environments that are passed in + environments: z.array( + z.object({ + id: z.string(), + amount: z.coerce.number().min(0, "Amount must be 0 or more"), + }) + ), + }), +]); + +export const action = async ({ request, params }: ActionFunctionArgs) => { + const userId = await requireUserId(request); + const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); + + const project = await findProjectBySlug(organizationSlug, projectParam, userId); + const redirectPath = concurrencyLimitsPath( + { slug: organizationSlug }, + { slug: projectParam }, + { slug: envParam } + ); + + if (!project) { + throw await redirectWithErrorMessage(redirectPath, request, "Project not found"); + } + + const formData = await request.formData(); + const submission = parseWithZod(formData, { schema: FormSchema }); + + if (submission.status !== "success") { + return json(submission.reply()); + } + + if (submission.value.action === "allocate") { + const allocate = new AllocateConcurrencyService(); + const [error, result] = await tryCatch( + allocate.call({ + userId, + projectId: project.id, + organizationId: project.organizationId, + environments: submission.value.environments, + }) + ); + + if (error) { + return json( + submission.reply({ + fieldErrors: { + environments: [error instanceof Error ? error.message : "Unknown error"], + }, + }) + ); + } + + if (!result.success) { + return json(submission.reply({ fieldErrors: { environments: [result.error] } })); + } + + return redirectWithSuccessMessage( + `${redirectPath}?success=true`, + request, + "Concurrency allocated successfully" + ); + } + + const currentPlan = await getCurrentPlan(project.organizationId); + const purchaseBlockReason = getSelfServePurchaseBlockReason(currentPlan); + if (purchaseBlockReason === "plan_unavailable") { + return json( + submission.reply({ + fieldErrors: { amount: ["Unable to verify billing status. Please try again."] }, + }), + { status: 503 } + ); + } + if (purchaseBlockReason === "managed_billing") { + return json( + submission.reply({ fieldErrors: { amount: ["Contact us to request more concurrency."] } }), + { status: 403 } + ); + } + + const service = new SetConcurrencyAddOnService(); + const [error, result] = await tryCatch( + service.call({ + userId, + projectId: project.id, + organizationId: project.organizationId, + action: submission.value.action, + amount: submission.value.amount, + }) + ); + + if (error) { + return json( + submission.reply({ + fieldErrors: { amount: [error instanceof Error ? error.message : "Unknown error"] }, + }) + ); + } + + if (!result.success) { + return json(submission.reply({ fieldErrors: { amount: [result.error] } })); + } + + return redirectWithSuccessMessage( + `${redirectPath}?success=true`, + request, + submission.value.action === "purchase" + ? "Concurrency updated successfully" + : "Requested extra concurrency, we'll get back to you soon." + ); +}; + +export default function Page() { + const { + canAddConcurrency, + extraConcurrency, + extraAllocatedConcurrency, + extraUnallocatedConcurrency, + environments, + concurrencyPricing, + maxQuota, + } = useTypedLoaderData(); + + return ( + + + + + + + {environments.map((environment) => ( + + + {environment.type}{" "} + {environment.branchName ? ` (${environment.branchName})` : ""} + + + + + + ))} + + + + + + + {canAddConcurrency ? ( + + ) : ( + + )} + + + + ); +} + +function initialAllocation(environments: ConcurrencyResult["environments"]) { + return new Map( + environments + .filter((e) => e.type !== "DEVELOPMENT") + .map((e) => [e.id, Math.max(0, e.maximumConcurrencyLimit - e.planConcurrencyLimit)]) + ); +} + +function allocationTotal(environments: ConcurrencyResult["environments"]) { + const allocation = initialAllocation(environments); + return Array.from(allocation.values()).reduce((e, acc) => e + acc, 0); +} + +function Upgradable({ + extraConcurrency, + extraAllocatedConcurrency, + extraUnallocatedConcurrency, + environments, + concurrencyPricing, + maxQuota, +}: ConcurrencyResult) { + const lastSubmission = useActionData(); + const [form, fields] = useForm({ + id: "allocate-concurrency", + // TODO: type this + lastResult: lastSubmission as any, + onValidate({ formData }) { + return parseWithZod(formData, { schema: FormSchema }); + }, + shouldRevalidate: "onSubmit", + }); + const { environments: formEnvironments } = fields; + + const navigation = useNavigation(); + const isLoading = navigation.state !== "idle" && navigation.formMethod === "POST"; + + const [allocation, setAllocation] = useState(initialAllocation(environments)); + + const allocatedInProject = Array.from(allocation.values()).reduce((e, acc) => e + acc, 0); + const initialAllocationInProject = allocationTotal(environments); + const changeInAllocation = allocatedInProject - initialAllocationInProject; + const unallocated = extraUnallocatedConcurrency - changeInAllocation; + const allocationModified = changeInAllocation !== 0; + + return ( +
+
+ Manage your concurrency +
+ + Concurrency limits determine how many runs you can execute at the same time. You can add + extra concurrency to your organization which you can allocate to environments in your + projects. + +
+
+
+ Extra concurrency + +
+ + + + Extra concurrency purchased + + {extraConcurrency} + + + + Allocated concurrency + + {allocationModified ? ( + <> + + {extraAllocatedConcurrency} + {" "} + {extraAllocatedConcurrency + changeInAllocation} + + ) : ( + extraAllocatedConcurrency + )} + + + + Unallocated concurrency + 0 + ? "text-success" + : unallocated < 0 + ? "text-error" + : "text-text-bright" + )} + > + {allocationModified ? ( + <> + + {extraUnallocatedConcurrency} + {" "} + {extraUnallocatedConcurrency - changeInAllocation} + + ) : ( + extraUnallocatedConcurrency + )} + + + 0 ? undefined : "after:bg-transparent" + } + > + 0 || allocationModified) && "pr-0")} + > +
+ {allocationModified ? ( + unallocated < 0 ? ( +
+ + + You're trying to allocate more concurrency than your total purchased + amount. + +
+ ) : ( +
+
+ + + Save your changes or{" "} + + . + +
+ +
+ ) + ) : unallocated > 0 ? ( +
+
+ + + You have {unallocated} extra concurrency available to allocate below. + +
+ +
+ ) : null} +
+
+
+
+
+ {formEnvironments.errors} +
+ + +
+ Concurrency allocation +
+ + + + Environment + + + Included{" "} + + + + Extra concurrency + Total + + + + {environments.map((environment, index) => ( + + + + + {environment.planConcurrencyLimit} + +
+ {environment.type === "DEVELOPMENT" ? ( + Math.max( + 0, + environment.maximumConcurrencyLimit - environment.planConcurrencyLimit + ) + ) : ( + <> + + { + const value = e.target.value === "" ? 0 : Number(e.target.value); + setAllocation(new Map(allocation).set(environment.id, value)); + }} + min={0} + /> + + )} +
+
+ + {environment.type === "DEVELOPMENT" + ? environment.maximumConcurrencyLimit + : environment.planConcurrencyLimit + (allocation.get(environment.id) ?? 0)} + +
+ ))} +
+
+ +
+
+ ); +} + +function NotUpgradable({ environments }: { environments: EnvironmentWithConcurrency[] }) { + const { isManagedCloud } = useFeatures(); + const plan = useCurrentPlan(); + const organization = useOrganization(); + const showSelfServe = useShowSelfServe(); + + return ( +
+
+ Your concurrency +
+ {isManagedCloud ? ( + <> + + Concurrency limits determine how many runs you can execute at the same time. You can + upgrade your plan to get more concurrency. You are currently on the{" "} + {plan?.v3Subscription?.plan?.title ?? "Free"} plan. + + {showSelfServe ? ( + + Upgrade for more concurrency + + ) : ( + Contact us} + /> + )} + + ) : null} +
+ + + + Environment + Concurrency limit + + + + {environments.map((environment) => ( + + + + + {environment.maximumConcurrencyLimit} + + ))} + +
+
+
+ ); +} + +function PurchaseConcurrencyModal({ + concurrencyPricing, + extraConcurrency, + extraUnallocatedConcurrency, + maxQuota, + disabled, +}: { + concurrencyPricing: { + stepSize: number; + centsPerStep: number; + }; + extraConcurrency: number; + extraUnallocatedConcurrency: number; + maxQuota: number; + disabled: boolean; +}) { + const showSelfServe = useShowSelfServe(); + const lastSubmission = useActionData(); + const [form, fields] = useForm({ + id: "purchase-concurrency", + // TODO: type this + lastResult: lastSubmission as any, + onValidate({ formData }) { + return parseWithZod(formData, { schema: FormSchema }); + }, + shouldRevalidate: "onSubmit", + }); + const { amount } = fields; + + const [amountValue, setAmountValue] = useState(extraConcurrency); + const navigation = useNavigation(); + const isLoading = navigation.state !== "idle" && navigation.formMethod === "POST"; + + // Close the panel, when we've succeeded + // This is required because a redirect to the same path doesn't clear state + const [searchParams, setSearchParams] = useSearchParams(); + const purchaseSucceeded = Boolean(searchParams.get("success")); + const [open, setOpen] = useState(false); + useEffect(() => { + if (purchaseSucceeded) { + // oxlint-disable-next-line react/set-state-in-effect -- This effect intentionally synchronizes route state after an external or lifecycle change. + setOpen(false); + setSearchParams((s) => { + s.delete("success"); + return s; + }); + } + }, [purchaseSucceeded, setSearchParams]); + + const state = updateState({ + value: amountValue, + existingValue: extraConcurrency, + quota: maxQuota, + extraUnallocatedConcurrency, + }); + const changeClassName = + state === "decrease" ? "text-error" : state === "increase" ? "text-success" : undefined; + + const title = extraConcurrency === 0 ? "Purchase extra concurrency" : "Add/remove concurrency"; + + if (!showSelfServe) { + return ( + Request more} + /> + ); + } + + return ( + + + + + + {title} +
+
+ + You can purchase bundles of {concurrencyPricing.stepSize} concurrency for{" "} + {formatCurrency(concurrencyPricing.centsPerStep / 100, false)}/month. Or you can + remove any extra concurrency after you have unallocated it from your environments + first. + +
+ + + setAmountValue(Number(e.target.value))} + disabled={isLoading} + /> + {amount.errors} + {form.errors} + +
+ {state === "need_to_increase_unallocated" ? ( +
+ + You need to unallocate{" "} + {formatNumber(extraConcurrency - amountValue - extraUnallocatedConcurrency)} more + concurrency from your environments in order to remove{" "} + {formatNumber(extraConcurrency - amountValue)} concurrency from your account. + +
+ ) : state === "above_quota" ? ( +
+ + Currently you can only have up to {maxQuota} extra concurrency. Send a request + below to lift your current limit. We'll get back to you soon. + +
+ ) : ( +
+
+ Summary + Total +
+
+ + {formatNumber(extraConcurrency)}{" "} + current total + + + {formatCurrency( + (extraConcurrency * concurrencyPricing.centsPerStep) / + concurrencyPricing.stepSize / + 100, + true + )} + +
+
+ + ({simplur`${extraConcurrency / concurrencyPricing.stepSize} bundle[|s]`}) + + /mth +
+
+ + {state === "increase" ? "+" : null} + {formatNumber(amountValue - extraConcurrency)} + + + {state === "increase" ? "+" : null} + {formatCurrency( + ((amountValue - extraConcurrency) * concurrencyPricing.centsPerStep) / + concurrencyPricing.stepSize / + 100, + true + )} + +
+
+ + ( + {simplur`${ + (amountValue - extraConcurrency) / concurrencyPricing.stepSize + } bundle[|s]`}{" "} + @ {formatCurrency(concurrencyPricing.centsPerStep / 100, true)}/mth) + + /mth +
+
+ + {formatNumber(amountValue)} new total + + + {formatCurrency( + (amountValue * concurrencyPricing.centsPerStep) / + concurrencyPricing.stepSize / + 100, + true + )} + +
+
+ + ({simplur`${amountValue / concurrencyPricing.stepSize} bundle[|s]`}) + + /mth +
+
+ )} +
+ + + + + ) : state === "decrease" || state === "need_to_increase_unallocated" ? ( + <> + + + + ) : ( + <> + + + + ) + } + cancelButton={ + + + + } + /> + +
+
+ ); +} + +function updateState({ + value, + existingValue, + quota, + extraUnallocatedConcurrency, +}: { + value: number; + existingValue: number; + quota: number; + extraUnallocatedConcurrency: number; +}): "no_change" | "increase" | "decrease" | "above_quota" | "need_to_increase_unallocated" { + if (value === existingValue) return "no_change"; + if (value < existingValue) { + const difference = existingValue - value; + if (difference > extraUnallocatedConcurrency) { + return "need_to_increase_unallocated"; + } + return "decrease"; + } + if (value > quota) return "above_quota"; + return "increase"; +} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency/route.tsx index 7b5b8b0ac63..79160814891 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency/route.tsx @@ -1,91 +1,157 @@ -import { getFormProps, getInputProps, useForm } from "@conform-to/react"; -import { parseWithZod } from "@conform-to/zod"; import { - ArrowDownIcon, - EnvelopeIcon, + ArrowUpCircleIcon, + BookOpenIcon, ExclamationTriangleIcon, - InformationCircleIcon, + PauseIcon, + PlayIcon, + RectangleStackIcon, } from "@heroicons/react/20/solid"; import { DialogClose } from "@radix-ui/react-dialog"; -import { Form, useActionData, useNavigation, useSearchParams } from "@remix-run/react"; -import { json, type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime"; -import { tryCatch } from "@trigger.dev/core"; -import { useEffect, useState } from "react"; +import { Form, useNavigation } from "@remix-run/react"; +import { type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime"; +import type { RuntimeEnvironmentType } from "@trigger.dev/database"; +import { useEffect, useMemo, useState, type ReactNode } from "react"; +import { QueuesIcon } from "~/assets/icons/QueuesIcon"; import { typedjson, useTypedLoaderData } from "remix-typedjson"; -import simplur from "simplur"; import { z } from "zod"; +import { ConcurrencyIcon } from "~/assets/icons/ConcurrencyIcon"; +import { RunsIcon } from "~/assets/icons/RunsIcon"; import { AdminDebugTooltip } from "~/components/admin/debugTooltip"; -import { CopyableText } from "~/components/primitives/CopyableText"; -import { EnvironmentCombo } from "~/components/environments/EnvironmentLabel"; -import { Feedback } from "~/components/Feedback"; -import { - MainHorizontallyCenteredContainer, - PageBody, - PageContainer, -} from "~/components/layout/AppLayout"; +import { environmentFullTitle } from "~/components/environments/EnvironmentLabel"; +import { PageBody, PageContainer } from "~/components/layout/AppLayout"; +import { MetricsLayout } from "~/components/layout/MetricsLayout"; +import { Badge } from "~/components/primitives/Badge"; import { Button, LinkButton } from "~/components/primitives/Buttons"; import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "~/components/primitives/Dialog"; -import { Fieldset } from "~/components/primitives/Fieldset"; import { FormButtons } from "~/components/primitives/FormButtons"; -import { FormError } from "~/components/primitives/FormError"; -import { Header2, Header3 } from "~/components/primitives/Headers"; -import { Input } from "~/components/primitives/Input"; -import { InputGroup } from "~/components/primitives/InputGroup"; -import { InputNumberStepper } from "~/components/primitives/InputNumberStepper"; -import { Label } from "~/components/primitives/Label"; +import { Header3 } from "~/components/primitives/Headers"; import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader"; +import { PaginationControls } from "~/components/primitives/Pagination"; import { Paragraph } from "~/components/primitives/Paragraph"; -import * as Property from "~/components/primitives/PropertyTable"; -import { SpinnerWhite } from "~/components/primitives/Spinner"; +import { PopoverMenuItem } from "~/components/primitives/Popover"; +import { SearchInput } from "~/components/primitives/SearchInput"; +import { Spinner } from "~/components/primitives/Spinner"; import { Table, TableBody, TableCell, + TableCellMenu, TableHeader, TableHeaderCell, TableRow, } from "~/components/primitives/Table"; -import { InfoIconTooltip } from "~/components/primitives/Tooltip"; -import { useFeatures } from "~/hooks/useFeatures"; +import { + InfoIconTooltip, + SimpleTooltip, + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger, +} from "~/components/primitives/Tooltip"; +import { TasksIcon } from "~/assets/icons/TasksIcon"; +import { QueueName } from "~/components/runs/v3/QueueName"; +import { env } from "~/env.server"; +import { useAutoRevalidate } from "~/hooks/useAutoRevalidate"; +import { useEnvironment } from "~/hooks/useEnvironment"; import { useOrganization } from "~/hooks/useOrganizations"; -import { useShowSelfServe } from "~/hooks/useShowSelfServe"; +import { useProject } from "~/hooks/useProject"; import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server"; import { findProjectBySlug } from "~/models/project.server"; +import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; +import { EnvironmentQueuePresenter } from "~/presenters/v3/EnvironmentQueuePresenter.server"; +import { QueueListPresenter } from "~/presenters/v3/QueueListPresenter.server"; import { - ManageConcurrencyPresenter, - type ConcurrencyResult, - type EnvironmentWithConcurrency, -} from "~/presenters/v3/ManageConcurrencyPresenter.server"; + QueueMetricsPresenter, + type QueueListMetric, +} from "~/presenters/v3/QueueMetricsPresenter.server"; +import { TimeFilter, timeFilterFromTo } from "~/components/runs/v3/SharedFilters"; +import { useSearchParams } from "~/hooks/useSearchParam"; +import { parseFiniteInt } from "~/utils/searchParams"; +import { MiniLineChart } from "~/components/metrics/MiniLineChart"; +import { buildActivityTimeAxis } from "~/components/primitives/charts/activityTimeAxis"; +import { Chart, type ChartConfig } from "~/components/primitives/charts/ChartCompound"; +import { ChartCard } from "~/components/primitives/charts/ChartCard"; +import { ChartSyncProvider } from "~/components/primitives/charts/ChartSyncContext"; +import { useZoomToTimeFilter } from "~/hooks/useZoomToTimeFilter"; import { - getCurrentPlan, - getPlans, - getSelfServePurchaseBlockReason, -} from "~/services/platform.v3.server"; -import { textLinkClassName } from "~/components/primitives/TextLink"; + useIsMetricResponseFresh, + useMetricResourceQuery, + type MetricResourceTimeRange, +} from "~/hooks/useMetricResourceQuery"; +import { logger } from "~/services/logger.server"; import { requireUserId } from "~/services/session.server"; import { cn } from "~/utils/cn"; -import { formatCurrency, formatNumber } from "~/utils/numberFormatter"; -import { concurrencyPath, EnvironmentParamSchema, v3BillingPath } from "~/utils/pathBuilder"; -import { AllocateConcurrencyService } from "~/v3/services/allocateConcurrency.server"; -import { SetConcurrencyAddOnService } from "~/v3/services/setConcurrencyAddOn.server"; -import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route"; -import { sectionAgentPageContext } from "~/components/dashboard-agent/suggested-prompts"; +import { ENVIRONMENT_PAUSE_SOURCE_BILLING_LIMIT } from "~/utils/environmentPauseSource"; +import { + concurrencyLimitsPath, + docsPath, + EnvironmentParamSchema, + v3BillingPath, + concurrencyQueuePath, + v3RunsPath, +} from "~/utils/pathBuilder"; import type { Handle } from "~/utils/handle"; +import { queuesAgentPageContext } from "~/components/dashboard-agent/suggested-prompts"; +import { WhenAgentUnavailable } from "~/components/dashboard-agent/WhenAgentUnavailable"; +import { PauseEnvironmentService } from "~/v3/services/pauseEnvironment.server"; +import { handleQueueMutationAction } from "~/models/queueMutation.server"; +import { + QueueOverrideConcurrencyButton, + QueuePauseResumeButton, +} from "~/components/queues/QueueControls"; +import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route"; +import { BigNumber } from "~/components/metrics/BigNumber"; +import { canAccessQueueMetricsUi } from "~/v3/canAccessQueueMetricsUi.server"; +import { QueueAllocationPresenter } from "~/presenters/v3/QueueAllocationPresenter.server"; +import { + QUEUE_METRICS_DEFAULT_PERIOD, + QUEUE_METRICS_RETENTION_DAYS, + clampQueueMetricsPeriod, + clipQueueMetricsWindow, + queueMetricsPeriodFromRequest, + resolveQueueMetricsPeriod, + useRememberQueueMetricsPeriod, +} from "~/components/queues/queueMetricsPeriod"; +import { queueMetricsMaxPeriodDays } from "~/components/queues/queueMetricsPeriod.server"; +import { isQueueAtCapacity } from "~/components/queues/queue-thresholds"; +import { pageMeta } from "~/utils/pageTitle"; +import { InlineCode } from "~/components/code/InlineCode"; + +const SearchParamsSchema = z.object({ + query: z.string().optional(), + page: z.coerce.number().min(1).default(1), + period: z.string().optional(), + from: z.string().optional(), + to: z.string().optional(), + sort: z.enum(["busiest", "queued", "name"]).optional(), +}); + +// The live "Queued" / "Running" header blocks poll ClickHouse on a short cadence so they stay +// current after first paint. They read the env-wide gauges from env_metrics (the env-level rollup +// of concurrency_metrics, cheapest for a dimension-free query), always over a fixed 15m window regardless +// of the chart/table period, and are NOT scoped to the visible queue set (the blocks are env-wide). +const QUEUE_LIVE_BLOCKS_PERIOD = "15m"; +const QUEUE_LIVE_BLOCKS_QUERY = + "SELECT timeBucket() AS t, max(max_env_queued) AS env_queued, max(max_env_running) AS env_running FROM env_metrics GROUP BY t ORDER BY t"; +// Trust the ClickHouse gauge only while its newest bucket is this recent; otherwise fall back to +// the loader's Redis-exact live values (matches LIVE_GAUGE_FRESH_MS on the queue detail page / run +// inspector). +const LIVE_GAUGE_FRESH_MS = 90_000; export const handle: Handle = { - agentPageContext: () => sectionAgentPageContext("concurrency"), + agentPageContext: (data) => queuesAgentPageContext(data), }; -import { pageMeta } from "~/utils/pageTitle"; -export const meta = pageMeta("Manage concurrency"); +export const meta = pageMeta("Queues"); export const loader = async ({ request, params }: LoaderFunctionArgs) => { const userId = await requireUserId(request); - const { - organizationSlug, - projectParam, - envParam: _envParam, - } = EnvironmentParamSchema.parse(params); + const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); + + const url = new URL(request.url); + const { page, query, period, from, to, sort } = SearchParamsSchema.parse( + Object.fromEntries(url.searchParams) + ); const project = await findProjectBySlug(organizationSlug, projectParam, userId); if (!project) { @@ -95,792 +161,2081 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { }); } - const presenter = new ManageConcurrencyPresenter(); - const [error, result] = await tryCatch( - presenter.call({ - userId: userId, - projectId: project.id, - organizationId: project.organizationId, - }) - ); - - if (error) { + const environment = await findEnvironmentBySlug(project.id, envParam, userId); + if (!environment) { throw new Response(undefined, { - status: 400, - statusText: error.message, + status: 404, + statusText: "Environment not found", }); } - const plans = await tryCatch(getPlans()); - if (!plans) { - throw new Response(null, { status: 404, statusText: "Plans not found" }); - } - - return typedjson(result); -}; - -const FormSchema = z.discriminatedUnion("action", [ - z.object({ - action: z.enum(["purchase"]), - amount: z.coerce.number().min(0, "Amount must be 0 or more"), - }), - z.object({ - action: z.enum(["quota-increase"]), - amount: z.coerce.number().min(1, "Amount must be greater than 0"), - }), - z.object({ - action: z.enum(["allocate"]), - // It will only update environments that are passed in - environments: z.array( - z.object({ - id: z.string(), - amount: z.coerce.number().min(0, "Amount must be 0 or more"), - }) - ), - }), -]); - -export const action = async ({ request, params }: ActionFunctionArgs) => { - const userId = await requireUserId(request); - const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); + // Per-org gate for the metrics UI. When off, this org gets the classic Queues page and + // no metrics query fires. + const queueMetricsUiEnabled = await canAccessQueueMetricsUi({ + request, + userId, + organizationSlug, + }); - const project = await findProjectBySlug(organizationSlug, projectParam, userId); - const redirectPath = concurrencyPath( - { slug: organizationSlug }, - { slug: projectParam }, - { slug: envParam } + const maxPeriodDays = queueMetricsUiEnabled + ? await queueMetricsMaxPeriodDays(environment.organizationId) + : QUEUE_METRICS_RETENTION_DAYS; + const defaultPeriod = clampQueueMetricsPeriod( + queueMetricsPeriodFromRequest(request), + maxPeriodDays ); - if (!project) { - throw await redirectWithErrorMessage(redirectPath, request, "Project not found"); - } + try { + const queueListPresenter = new QueueListPresenter(); + const queues = await queueListPresenter.call({ + environment, + query, + page, + includeLimits: true, + // Relevance ordering rides the metrics pipeline, so it is part of the gated UI. + sort: queueMetricsUiEnabled ? (sort ?? "busiest") : "name", + }); - const formData = await request.formData(); - const submission = parseWithZod(formData, { schema: FormSchema }); + const environmentQueuePresenter = new EnvironmentQueuePresenter(); - if (submission.status !== "success") { - return json(submission.reply()); - } + const autoReloadPollIntervalMs = env.QUEUES_AUTORELOAD_POLL_INTERVAL_MS; - if (submission.value.action === "allocate") { - const allocate = new AllocateConcurrencyService(); - const [error, result] = await tryCatch( - allocate.call({ - userId, - projectId: project.id, - organizationId: project.organizationId, - environments: submission.value.environments, - }) - ); + // Per-queue list metrics (Delay p95 + backlog sparkline columns) are SSR'd with the table. + // The environment header tiles are fetched client-side per card (see QueueEnvMetricChart) so a + // slow ClickHouse query never blocks the queues list from rendering. + let metrics: { + bucketStartMs: number; + bucketIntervalMs: number; + byQueue: Record; + } | null = null; - if (error) { - return json( - submission.reply({ - fieldErrors: { - environments: [error instanceof Error ? error.message : "Unknown error"], - }, - }) - ); + if (queueMetricsUiEnabled) { + // Metrics are additive observability; a ClickHouse hiccup must not take down queue + // management. Fail open to metrics: null instead of bubbling to the page-level 400. + try { + const presenter = new QueueMetricsPresenter(); + const queueNames = queues.queues.map((q) => + q.type === "task" ? `task/${q.name}` : q.name + ); + const timeRange = clipQueueMetricsWindow( + timeFilterFromTo({ + period: + resolveQueueMetricsPeriod({ + period, + from, + to, + defaultPeriod, + maxPeriodDays, + }) ?? undefined, + from: parseFiniteInt(from), + to: parseFiniteInt(to), + defaultPeriod, + }), + maxPeriodDays + ); + const queueMetrics = + queueNames.length > 0 + ? await presenter.getQueueListMetrics({ + environment, + queueNames, + from: timeRange.from, + to: timeRange.to, + }) + : null; + if (queueMetrics) { + metrics = { + bucketStartMs: queueMetrics.bucketStartMs, + bucketIntervalMs: queueMetrics.bucketIntervalMs, + byQueue: Object.fromEntries(queueMetrics.byQueue), + }; + } + } catch (error) { + logger.warn("Queue list metrics unavailable, rendering without them", { + error, + }); + } } - if (!result.success) { - return json(submission.reply({ fieldErrors: { environments: [result.error] } })); + // Allocation summary (Environment limit + Allocated tiles) is additive; a presenter + // failure must not 400 the page, so fail open to null like the metrics block above. + let allocation: Awaited> | null = null; + if (queueMetricsUiEnabled) { + try { + allocation = await new QueueAllocationPresenter().call({ environment }); + } catch (error) { + logger.warn("Queue allocation summary unavailable, rendering without it", { error }); + } } - return redirectWithSuccessMessage( - `${redirectPath}?success=true`, + return typedjson({ + ...queues, + environment: await environmentQueuePresenter.call(environment), + autoReloadPollIntervalMs, + metrics, + allocation, + queueMetricsUiEnabled, + defaultPeriod, + maxPeriodDays, + }); + } catch (error) { + console.error(error); + throw new Response(undefined, { + status: 400, + statusText: "Something went wrong, if this problem persists please contact support.", + }); + } +}; + +export const action = async ({ request, params }: ActionFunctionArgs) => { + const userId = await requireUserId(request); + if (request.method.toLowerCase() !== "post") { + return redirectWithErrorMessage( + `/orgs/${params.organizationSlug}/projects/${params.projectParam}/env/${params.envParam}/concurrency`, request, - "Concurrency allocated successfully" + "Wrong method" ); } - const currentPlan = await getCurrentPlan(project.organizationId); - const purchaseBlockReason = getSelfServePurchaseBlockReason(currentPlan); - if (purchaseBlockReason === "plan_unavailable") { - return json( - submission.reply({ - fieldErrors: { amount: ["Unable to verify billing status. Please try again."] }, - }), - { status: 503 } - ); + const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); + + const project = await findProjectBySlug(organizationSlug, projectParam, userId); + if (!project) { + throw new Response(undefined, { + status: 404, + statusText: "Project not found", + }); } - if (purchaseBlockReason === "managed_billing") { - return json( - submission.reply({ fieldErrors: { amount: ["Contact us to request more concurrency."] } }), - { status: 403 } - ); + + const environment = await findEnvironmentBySlug(project.id, envParam, userId); + if (!environment) { + throw new Response(undefined, { + status: 404, + statusText: "Environment not found", + }); } - const service = new SetConcurrencyAddOnService(); - const [error, result] = await tryCatch( - service.call({ - userId, - projectId: project.id, - organizationId: project.organizationId, - action: submission.value.action, - amount: submission.value.amount, - }) - ); + const formData = await request.formData(); + const action = formData.get("action"); - if (error) { - return json( - submission.reply({ - fieldErrors: { amount: [error instanceof Error ? error.message : "Unknown error"] }, - }) - ); - } + const url = new URL(request.url); + const redirectPath = `/orgs/${organizationSlug}/projects/${projectParam}/env/${envParam}/concurrency${url.search}`; - if (!result.success) { - return json(submission.reply({ fieldErrors: { amount: [result.error] } })); + if (environment.archivedAt) { + return redirectWithErrorMessage(redirectPath, request, "This branch is archived"); } - return redirectWithSuccessMessage( - `${redirectPath}?success=true`, + // Per-queue actions (pause/resume/override/remove-override) are shared with the queue detail + // route, so they live in a helper that both routes call. + const queueMutation = await handleQueueMutationAction({ request, - submission.value.action === "purchase" - ? "Concurrency updated successfully" - : "Requested extra concurrency, we'll get back to you soon." - ); + environment, + userId, + formData, + redirectPath, + }); + if (queueMutation) { + return queueMutation; + } + + switch (action) { + case "environment-pause": { + const pauseService = new PauseEnvironmentService(); + const result = await pauseService.call(environment, "paused"); + if (!result.success) { + return redirectWithErrorMessage(redirectPath, request, result.error); + } + return redirectWithSuccessMessage(redirectPath, request, "Environment paused"); + } + case "environment-resume": { + const resumeService = new PauseEnvironmentService(); + const result = await resumeService.call(environment, "resumed"); + if (!result.success) { + return redirectWithErrorMessage(redirectPath, request, result.error); + } + return redirectWithSuccessMessage(redirectPath, request, "Environment resumed"); + } + default: + return redirectWithErrorMessage(redirectPath, request, "Something went wrong"); + } }; +// Derives the environment concurrency status ("limit" | "burst" | "within") and the matching +// text color from the current running count vs. the env limit and burst factor. Shared by both +// the classic and metrics views so the "Running" tile styling stays in sync. +function getEnvConcurrencyLimitStatus(environment: { + running: number; + concurrencyLimit: number; + burstFactor: number; +}) { + const limitStatus = + environment.running === environment.concurrencyLimit * environment.burstFactor + ? "limit" + : environment.running > environment.concurrencyLimit + ? "burst" + : "within"; + + const limitClassName = + limitStatus === "burst" ? "text-warning" : limitStatus === "limit" ? "text-error" : undefined; + + return { limitStatus, limitClassName }; +} + export default function Page() { + // Per-org flag decides which whole page renders. Off => the classic Queues page, + // byte-for-byte the pre-metrics UI. Each branch is its own component (own hooks). + const { queueMetricsUiEnabled } = useTypedLoaderData(); + return queueMetricsUiEnabled ? : ; +} + +function QueuesWithMetricsView() { const { - canAddConcurrency, - extraConcurrency, - extraAllocatedConcurrency, - extraUnallocatedConcurrency, - environments, - concurrencyPricing, - maxQuota, + environment, + queues, + pagination, + totalQueues, + hasFilters, + autoReloadPollIntervalMs, + metrics, + allocation, + defaultPeriod, + maxPeriodDays, } = useTypedLoaderData(); + const metricsByQueue = metrics?.byQueue ?? {}; + + const organization = useOrganization(); + const project = useProject(); + const env = useEnvironment(); + const plan = useCurrentPlan(); + + // The header tiles fetch client-side with the same period/from/to the TimeFilter writes. + const { value } = useSearchParams(); + const timeRange = { + period: resolveQueueMetricsPeriod({ + period: value("period"), + from: value("from"), + to: value("to"), + defaultPeriod, + maxPeriodDays, + }), + from: value("from") ?? null, + to: value("to") ?? null, + }; + useRememberQueueMetricsPeriod(value("period")); + + useAutoRevalidate({ interval: autoReloadPollIntervalMs, onFocus: true }); + + // Drag-to-zoom on either chart narrows the page's from/to search params, which the + // TimeFilter and the client-side metric queries both read (same wiring as the Agent page). + const zoomToTimeFilter = useZoomToTimeFilter(); + + // Live env-wide Queued/Running blocks. First paint uses the loader's Redis-exact values; from the + // first poll on we prefer ClickHouse so the blocks stay current without a full page revalidate. + // Empty rows (quiet env, or the very first fetch still in flight) fall back to the loader values, + // so we never flash a stale 0. Fixed 15m window, env-wide (no queue filter), CH-only recurring + // load; pauses while the tab is hidden (handled inside the hook). + const { rows: liveBlockRows, responseReceivedAt } = useMetricResourceQuery( + QUEUE_LIVE_BLOCKS_QUERY, + { + organizationId: organization.id, + projectId: project.id, + environmentId: env.id, + timeRange: { period: QUEUE_LIVE_BLOCKS_PERIOD, from: null, to: null }, + defaultPeriod: QUEUE_LIVE_BLOCKS_PERIOD, + fillGaps: false, + refreshIntervalMs: 15_000, + } + ); + const lastLiveBlockRow = + liveBlockRows.length > 0 ? liveBlockRows[liveBlockRows.length - 1] : null; + // Only trust the gauge while its newest bucket is fresh. A row painted from the hook's cache on + // client-side nav-back (responseCache), or a quiet env whose latest bucket is minutes old, must + // not override the loader's Redis-exact live values with a stale count. + const lastLiveBucketMs = lastLiveBlockRow ? tileTimeToMs(lastLiveBlockRow.t) : NaN; + const liveBlockIsFresh = useIsMetricResponseFresh( + responseReceivedAt, + lastLiveBucketMs, + LIVE_GAUGE_FRESH_MS + ); + const freshLiveBlockRow = lastLiveBlockRow && liveBlockIsFresh ? lastLiveBlockRow : null; + const envQueuedLive = freshLiveBlockRow + ? tileNumber(freshLiveBlockRow.env_queued) + : environment.queued; + const envRunningLive = freshLiveBlockRow + ? tileNumber(freshLiveBlockRow.env_running) + : environment.running; + + // Allocation summary tiles. The presenter computes the env-wide allocated total (sum of + // each queue's explicit limit clamped to the env limit) in a single aggregate query. + const envLimit = environment.concurrencyLimit; + const burstLimit = Math.round(envLimit * environment.burstFactor); + const allocated = allocation?.allocated ?? 0; + const allocationPct = envLimit > 0 ? Math.round((allocated / envLimit) * 100) : 0; + + // Running-block tinting (burst/limit) tracks the live running value, not the loader snapshot. + const { limitStatus, limitClassName } = getEnvConcurrencyLimitStatus({ + running: envRunningLive, + concurrencyLimit: environment.concurrencyLimit, + burstFactor: environment.burstFactor, + }); + + // Client-side, header-click sorting over the current page's rows. Server pagination and the + // default busiest order are unchanged; clearing a sort returns to that server order. + const queueRows = queues ?? []; + return ( - - - {environments.map((environment) => ( - - - {environment.type}{" "} - {environment.branchName ? ` (${environment.branchName})` : ""} - - - - - - ))} - - + + + + Queues docs + + - - - {canAddConcurrency ? ( - + {/* Filters — pinned bar directly under the NavBar. This row is page-wide only: Period is + the one control that changes the tiles and charts below, so it leads the row. Search + and pagination scope the table alone and live in that table's own bar instead. */} + +
+ - ) : ( - - )} - - - - ); -} - -function initialAllocation(environments: ConcurrencyResult["environments"]) { - return new Map( - environments - .filter((e) => e.type !== "DEVELOPMENT") - .map((e) => [e.id, Math.max(0, e.maximumConcurrencyLimit - e.planConcurrencyLimit)]) - ); -} - -function allocationTotal(environments: ConcurrencyResult["environments"]) { - const allocation = initialAllocation(environments); - return Array.from(allocation.values()).reduce((e, acc) => e + acc, 0); -} - -function Upgradable({ - extraConcurrency, - extraAllocatedConcurrency, - extraUnallocatedConcurrency, - environments, - concurrencyPricing, - maxQuota, -}: ConcurrencyResult) { - const lastSubmission = useActionData(); - const [form, fields] = useForm({ - id: "allocate-concurrency", - // TODO: type this - lastResult: lastSubmission as any, - onValidate({ formData }) { - return parseWithZod(formData, { schema: FormSchema }); - }, - shouldRevalidate: "onSubmit", - }); - const { environments: formEnvironments } = fields; - - const navigation = useNavigation(); - const isLoading = navigation.state !== "idle" && navigation.formMethod === "POST"; +
+
+ {environment.runsEnabled && + env.pauseSource !== ENVIRONMENT_PAUSE_SOURCE_BILLING_LIMIT ? ( + + ) : null} +
+
- const [allocation, setAllocation] = useState(initialAllocation(environments)); + {/* Queued + Running + Allocated + Environment limit summary. Four stat tiles: the grid + derives its columns from the tile count (two-up, four-up from lg). The allocation + presenter fails open to null (a ClickHouse/PG hiccup mustn't take down the tiles), so + only the Allocated tile depends on it — the other three + controls always render, and + Allocated shows a "–" placeholder to keep the 4-tile grid shape stable. */} + + paused
: undefined} + animate + accessory={ + + + + } + valueClassName={env.paused ? "text-warning tabular-nums" : "tabular-nums"} + compactThreshold={1000000} + /> + + Including {envRunningLive - environment.concurrencyLimit} burst runs{" "} + + + ) : limitStatus === "limit" ? ( + "At concurrency limit" + ) : undefined + } + accessory={ + + + + } + compactThreshold={1000000} + /> + + Allocated + {allocation ? ( + + ) : null} + + } + value={allocation ? allocated : undefined} + formattedValue={allocation ? undefined : "–"} + suffix={allocation ? `${allocationPct}% of the environment limit` : undefined} + suffixClassName="text-text-dimmed" + /> + 1 ? `bursts up to ${burstLimit}` : undefined} + suffixClassName="text-text-dimmed" + accessory={ + plan ? ( + plan?.v3Subscription?.plan?.limits.concurrentRuns.canExceed ? ( + + Increase limit + + ) : ( + + Increase limit + + ) + ) : undefined + } + /> + - const allocatedInProject = Array.from(allocation.values()).reduce((e, acc) => e + acc, 0); - const initialAllocationInProject = allocationTotal(environments); - const changeInAllocation = allocatedInProject - initialAllocationInProject; - const unallocated = extraUnallocatedConcurrency - changeInAllocation; - const allocationModified = changeInAllocation !== 0; + {hasFilters || totalQueues !== 0 ? ( + + + {QUEUE_HEADER_TILES.map((tile) => ( + 1 + ? [ + { + y: Math.round(environment.burstFactor * 100), + label: `Burst ${Math.round( + environment.concurrencyLimit * environment.burstFactor + )}`, + labelPlacement: "outside" as const, + }, + ] + : []), + ] + : undefined + } + // Saturation recolours the line above its 100% limit with a gradient split, so + // only the portion over the line is orange (the offset is derived from the line's + // own value range, so the split lands exactly at 100% regardless of domain + // padding). p95 and throttled use a per-bucket overlay: it retraces only the + // over-threshold stretches, so under-threshold buckets stay blue. + thresholdStroke={ + tile.id === "saturation" + ? { value: 100, aboveColor: "var(--color-warning)" } + : undefined + } + warningOverlay={ + tile.id === "p95" + ? { threshold: 60_000 } + : tile.id === "throttled" + ? // Integer counts: threshold 0 warns once a bucket has ≥1 throttle. + { threshold: 0 } + : undefined + } + /> + ))} + + + ) : null} - return ( -
-
- Manage your concurrency -
- - Concurrency limits determine how many runs you can execute at the same time. You can add - extra concurrency to your organization which you can allocate to environments in your - projects. - -
-
-
- Extra concurrency - -
- - - - Extra concurrency purchased - - {extraConcurrency} - - - - Allocated concurrency - - {allocationModified ? ( - <> - - {extraAllocatedConcurrency} - {" "} - {extraAllocatedConcurrency + changeInAllocation} - - ) : ( - extraAllocatedConcurrency - )} - - + + + + + } + > + {/* Default overflow-x-auto container so wide tables still scroll horizontally on + narrow viewports; the page (not this region) owns vertical scrolling. */} +
+ - Unallocated concurrency - Name + Queued + Running + 0 - ? "text-success" - : unallocated < 0 - ? "text-error" - : "text-text-bright" - )} + disableTooltipHoverableContent + tooltip={limitTooltip} + tooltipContentClassName="max-w-xs" + > + Limit + + +

+ Environment: uses the environment + limit of {environment.concurrencyLimit}. +

+

+ User: a limit you set in your + code. +

+

+ Override: a limit you set here or + via the API. +

+ + } > - {allocationModified ? ( + Limited by +
+ Health + + Delay p95 + + - - {extraUnallocatedConcurrency} - {" "} - {extraUnallocatedConcurrency - changeInAllocation} + How many runs were waiting, over the selected time. marks + where the queue was throttled. - ) : ( - extraUnallocatedConcurrency - )} -
-
- 0 ? undefined : "after:bg-transparent" - } - > - 0 || allocationModified) && "pr-0")} + } > -
- {allocationModified ? ( - unallocated < 0 ? ( -
- - - You're trying to allocate more concurrency than your total purchased - amount. - -
- ) : ( -
-
- - - Save your changes or{" "} - - . - -
- -
- ) - ) : unallocated > 0 ? ( -
-
- - - You have {unallocated} extra concurrency available to allocate below. - -
- -
- ) : null} -
-
-
- -
- {formEnvironments.errors} -
-
- -
- Concurrency allocation -
- - - - Environment - - - Included{" "} - - + Backlog + + + Pause/resume - Extra concurrency - Total - {environments.map((environment, index) => ( - - - - - {environment.planConcurrencyLimit} - -
- {environment.type === "DEVELOPMENT" ? ( - Math.max( - 0, - environment.maximumConcurrencyLimit - environment.planConcurrencyLimit - ) - ) : ( - <> - - { - const value = e.target.value === "" ? 0 : Number(e.target.value); - setAllocation(new Map(allocation).set(environment.id, value)); - }} - min={0} + {queueRows.length > 0 ? ( + queueRows.map((queue) => { + const limit = queue.limits.perKey.current ?? environment.concurrencyLimit; + const isLimit = queue.kind === "limit"; + /** A limit row's `running` counts holders across every key, so only its + * total bound compares against it; a perKey-only limit has no aggregate + * threshold and never reads as at-limit here. */ + const atLimitThreshold = isLimit + ? queue.limits.total?.current != null + ? Math.min(queue.limits.total.current, environment.concurrencyLimit) + : null + : limit; + /** A zero threshold is a pause (nothing may run), not saturation — + * without the guard `running >= 0` holds for every row. */ + const isAtConcurrencyLimit = + atLimitThreshold !== null && + atLimitThreshold > 0 && + queue.running >= atLimitThreshold; + const isAtQueueLimit = + environment.queueSizeLimit !== null && + queue.queued >= environment.queueSizeLimit; + const queueFilterableName = queueMetricsKey(queue); + const queueMetric = metricsByQueue[queueFilterableName]; + /** The detail page is queue observability (queue metrics, run filters, pause + * and override actions); a limit's activity lives on each holder's home queue, + * so limit rows don't link anywhere. */ + const queueDetailPath = isLimit + ? undefined + : concurrencyQueuePath(organization, project, env, { + friendlyId: queue.id, + }); + const displayName = isLimit ? queue.name.replace(/^limit\//, "") : queue.name; + return ( + + s, so + // they render beside the link (leading/trailing), never inside it — + // otherwise the cell is invalid
- -
-
+ + + ); } -function NotUpgradable({ environments }: { environments: EnvironmentWithConcurrency[] }) { - const { isManagedCloud } = useFeatures(); - const plan = useCurrentPlan(); - const organization = useOrganization(); - const showSelfServe = useShowSelfServe(); +function EnvironmentPauseResumeButton({ + env, +}: { + env: { type: RuntimeEnvironmentType; paused: boolean }; +}) { + const navigation = useNavigation(); + const [isOpen, setIsOpen] = useState(false); + + useEffect(() => { + if (navigation.state === "loading" || navigation.state === "idle") { + // oxlint-disable-next-line react/set-state-in-effect -- This effect intentionally synchronizes route state after an external or lifecycle change. + setIsOpen(false); + } + }, [navigation.state]); + + const isLoading = Boolean( + navigation.formData?.get("action") === (env.paused ? "environment-resume" : "environment-pause") + ); return ( -
-
- Your concurrency + +
+ + + +
+ + + +
+
+ + {env.paused + ? `Resumes ${environmentFullTitle(env)} so its runs can be dequeued again.` + : `Pauses all runs from being dequeued in ${environmentFullTitle(env)}. Any executing runs will continue to run.`} + +
+
- {isManagedCloud ? ( - <> - - Concurrency limits determine how many runs you can execute at the same time. You can - upgrade your plan to get more concurrency. You are currently on the{" "} - {plan?.v3Subscription?.plan?.title ?? "Free"} plan. + + {env.paused ? "Resume environment?" : "Pause environment?"} +
+ + {env.paused + ? `This will allow runs to be dequeued in ${environmentFullTitle(env)} again.` + : `This will pause all runs from being dequeued in ${environmentFullTitle( + env + )}. Any executing runs will continue to run.`} - {showSelfServe ? ( - - Upgrade for more concurrency - - ) : ( - Contact us} +
setIsOpen(false)}> + - )} - - ) : null} -
- - - - Environment - Concurrency limit - - - - {environments.map((environment) => ( - - - - - {environment.maximumConcurrencyLimit} - - ))} - -
-
-
+ : env.paused ? PlayIcon : PauseIcon + } + shortcut={{ modifiers: ["mod"], key: "enter" }} + > + {env.paused ? "Resume environment" : "Pause environment"} + + } + cancelButton={ + + + + } + /> + +
+ + ); } -function PurchaseConcurrencyModal({ - concurrencyPricing, - extraConcurrency, - extraUnallocatedConcurrency, - maxQuota, - disabled, -}: { - concurrencyPricing: { - stepSize: number; - centsPerStep: number; +export function isEnvironmentPauseResumeFormSubmission( + formMethod: string | undefined, + formData: FormData | undefined +) { + if (!formMethod || !formData) { + return false; + } + + return ( + formMethod.toLowerCase() === "post" && + (formData.get("action") === "environment-pause" || + formData.get("action") === "environment-resume") + ); +} + +export function QueueFilters() { + return ; +} + +type MetricTileRow = Record; + +type TilePoint = { bucket: number; value: number | null }; + +// Inline colour swatch matching the chart's warning ("yellow") line — used in tooltip copy that +// refers to that colour instead of naming it, so the swatch always matches the chart. +function WarningSwatch() { + return ( + + ); +} + +type QueueHeaderTile = { + id: string; + label: string; + /** Info-icon copy explaining what the chart shows, rendered next to the card title. */ + description: ReactNode; + color: string; + /** Optional inline legend rendered below the card title: a fixed set of {colored square, label} + * entries, for charts where a colour (e.g. the orange warning line) needs explaining. */ + legend?: Array<{ color: string; label: string }>; + query: string; + /** Formats a single bucket's value in the chart tooltip. */ + formatValue?: (value: number) => string; + /** Formats the y-axis tick labels. Without it the axis shows raw numbers (bad for durations + * in ms or percent scales). Passed through to Chart.Line's yAxisProps.tickFormatter. */ + formatAxis?: (value: number) => string; + /** Hover tooltip explaining the headline readout next to the title (e.g. what "9% of current + * period" means). Without it the readout has no tooltip. */ + totalTooltip?: string; + /** Turns one row per bucket into the per-bucket points the chart draws. A null value is a + * bucket the metric has nothing to say about, and the line breaks there rather than reading 0. */ + derive: (rows: MetricTileRow[]) => { + points: TilePoint[]; + total: number; + formatTotal?: (total: number) => string; + totalClassName?: string; }; - extraConcurrency: number; - extraUnallocatedConcurrency: number; - maxQuota: number; - disabled: boolean; -}) { - const showSelfServe = useShowSelfServe(); - const lastSubmission = useActionData(); - const [form, fields] = useForm({ - id: "purchase-concurrency", - // TODO: type this - lastResult: lastSubmission as any, - onValidate({ formData }) { - return parseWithZod(formData, { schema: FormSchema }); - }, - shouldRevalidate: "onSubmit", - }); - const { amount } = fields; + /** + * Optional second query, run at the range's natural bucket width, that owns the headline readout. + * For a headline that is not invariant to bucket width — a share of buckets, or a percentile, + * as opposed to a max over gauges — deriving it from the plotted rows would move the number + * whenever this tile's floor widens them. Only requested while the floor is actually widening + * anything; on ranges whose natural width is already at or above the floor the chart's own rows + * are identical and are reused. + */ + readout?: { + query: string; + derive: (rows: MetricTileRow[]) => { + total: number; + formatTotal?: (total: number) => string; + totalClassName?: string; + }; + }; +}; - const [amountValue, setAmountValue] = useState(extraConcurrency); - const navigation = useNavigation(); - const isLoading = navigation.state !== "idle" && navigation.formMethod === "POST"; +function tileNumber(value: number | string | null): number { + const n = typeof value === "number" ? value : Number(value); + return Number.isFinite(n) ? n : 0; +} - // Close the panel, when we've succeeded - // This is required because a redirect to the same path doesn't clear state - const [searchParams, setSearchParams] = useSearchParams(); - const purchaseSucceeded = Boolean(searchParams.get("success")); - const [open, setOpen] = useState(false); - useEffect(() => { - if (purchaseSucceeded) { - // oxlint-disable-next-line react/set-state-in-effect -- This effect intentionally synchronizes route state after an external or lifecycle change. - setOpen(false); - setSearchParams((s) => { - s.delete("success"); - return s; +function tileTimeToMs(value: number | string | null): number { + const s = String(value).replace(" ", "T"); + return Date.parse(s.endsWith("Z") ? s : `${s}Z`); +} + +/** Peak of a series, ignoring the buckets it has nothing to say about. */ +function peakOf(points: TilePoint[]): number { + return points.reduce((max, p) => (p.value === null ? max : Math.max(max, p.value)), 0); +} + +const SCHEDULING_DELAY_QUERY = `SELECT timeBucket() AS t,\n round(quantilesTDigestMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95,\n sum(wait_ms_count) AS samples\nFROM env_metrics\nGROUP BY t\nORDER BY t`; + +const THROTTLED_QUERY = `SELECT timeBucket() AS t,\n sum(throttled_count) AS throttled\nFROM env_metrics\nGROUP BY t\nORDER BY t`; + +const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ + { + id: "saturation", + label: "Env saturation", + description: ( + <> + How much of the environment's concurrency is in use. Turns above 100%, + when it's into burst capacity. + + ), + color: "var(--color-queues-chart)", + legend: [ + { color: "var(--color-queues-chart)", label: "Saturation" }, + { color: "var(--color-warning)", label: "Over limit" }, + ], + query: `SELECT timeBucket() AS t,\n max(max_env_running) AS running,\n max(max_env_limit) AS env_limit\nFROM env_metrics\nGROUP BY t\nORDER BY t`, + formatValue: (v) => (v > 100 ? `${v}% — over the environment limit` : `${v}%`), + formatAxis: (v) => `${v}%`, + derive: (rows) => { + const points = rows.map((r) => { + const limit = tileNumber(r.env_limit); + return { + bucket: tileTimeToMs(r.t), + value: limit > 0 ? Math.round((tileNumber(r.running) / limit) * 100) : 0, + }; }); - } - }, [purchaseSucceeded, setSearchParams]); + return { + points, + total: peakOf(points), + formatTotal: (v) => `${v}% peak`, + }; + }, + }, + { + id: "backlog", + label: "Backlog", + description: "How many runs are waiting across the environment, over time.", + color: "var(--color-queues-chart)", + query: `SELECT timeBucket() AS t,\n max(max_env_queued) AS queued\nFROM env_metrics\nGROUP BY t\nORDER BY t`, + derive: (rows) => { + const points = rows.map((r) => ({ + bucket: tileTimeToMs(r.t), + value: tileNumber(r.queued), + })); + return { + points, + total: peakOf(points), + formatTotal: (v) => `${v.toLocaleString()} peak`, + }; + }, + }, + { + id: "p95", + label: "Scheduling delay p95", + description: ( + <> + How long runs wait before they start (95% start faster than this). Turns {" "} + above 1 minute. + + ), + totalTooltip: "The worst p95 in the selected window.", + color: "var(--color-queues-chart)", + legend: [ + { color: "var(--color-queues-chart)", label: "p95" }, + { color: "var(--color-warning)", label: "Over 1 min" }, + ], + query: SCHEDULING_DELAY_QUERY, + formatValue: formatWaitMs, + formatAxis: formatWaitMs, + derive: (rows) => { + const points = rows.map((r) => ({ + bucket: tileTimeToMs(r.t), + value: tileNumber(r.samples) > 0 ? tileNumber(r.p95) : null, + })); + return { points, total: peakOf(points) }; + }, + readout: { + query: SCHEDULING_DELAY_QUERY, + /** + * Merging quantile states over a wider bucket yields a p95 between the sub-buckets' own, so + * the worst p95 has to be read at the range's natural width or a burst of slow starts shorter + * than the plotted bucket is averaged away. Unlike the gauges, whose max of maxes is the same + * at any width. + */ + derive: (rows) => { + const worst = rows.reduce( + (max, r) => (tileNumber(r.samples) > 0 ? Math.max(max, tileNumber(r.p95)) : max), + 0 + ); + return { + total: worst, + formatTotal: (v) => (v > 0 ? formatWaitMs(v) : "–"), + totalClassName: worst >= 60_000 ? "text-warning" : undefined, + }; + }, + }, + }, + { + id: "throttled", + label: "Throttled", + description: "How often runs were held back by a limit.", + totalTooltip: "The share of the selected window with at least one blocked dequeue.", + color: "var(--color-queues-chart)", + legend: [{ color: "var(--color-warning)", label: "Throttled" }], + query: THROTTLED_QUERY, + derive: (rows) => { + const points = rows.map((r) => ({ + bucket: tileTimeToMs(r.t), + value: tileNumber(r.throttled), + })); + return { points, total: peakOf(points) }; + }, + readout: { + query: THROTTLED_QUERY, + /** + * Share of the window that saw any throttling. A raw event sum isn't interpretable (it + * scales with poll rate and window length); the fraction of buckets with a throttle is. + * Gap fill zero-fills this counter, so every bucket in the window is present and the row + * count is the honest denominator. + */ + derive: (rows) => { + const nonzero = rows.filter((r) => tileNumber(r.throttled) > 0).length; + const pct = rows.length > 0 ? Math.round((nonzero / rows.length) * 100) : 0; + return { + total: pct, + formatTotal: (v) => `${v}% of current period`, + totalClassName: pct > 0 ? "text-warning" : undefined, + }; + }, + }, + }, +]; + +/** + * Bucket floor shared by every hero tile. Scheduling delay and throttling are event-driven, so at + * the 10-second width a short range would otherwise pick, most buckets hold no samples at all. One + * floor for all four keeps their x-axes identical, which the shared hover crosshair relies on. + */ +const HERO_CHART_MIN_BUCKET_SECONDS = 60; - const state = updateState({ - value: amountValue, - existingValue: extraConcurrency, - quota: maxQuota, - extraUnallocatedConcurrency, +type TileTimeRange = MetricResourceTimeRange; + +// Full-size env metric chart rendered inside a ChartCard. Same data path as before +// (client-side TRQL via useMetricResourceQuery with fillGaps), drawn as a line that +// participates in the shared hover + drag-to-zoom of the enclosing ChartSyncProvider. +function QueueEnvMetricChart({ + tile, + timeRange, + referenceLines, + thresholdStroke, + warningOverlay, + solidWarning = false, +}: { + tile: QueueHeaderTile; + timeRange: TileTimeRange; + referenceLines?: Array<{ + y: number; + label?: string; + labelPlacement?: "inside" | "outside"; + }>; + thresholdStroke?: { value: number; aboveColor: string }; + warningOverlay?: { threshold: number }; + /** When set, the ENTIRE line turns warning-coloured if the series is ever non-zero (used for + * throttling: any throttle in the window colours the whole line). Mutually exclusive with the + * per-bucket warningOverlay. */ + solidWarning?: boolean; +}) { + const organization = useOrganization(); + const project = useProject(); + const environment = useEnvironment(); + + const sharedOptions = { + organizationId: organization.id, + projectId: project.id, + environmentId: environment.id, + timeRange, + defaultPeriod: QUEUE_METRICS_DEFAULT_PERIOD, + fillGaps: true, + }; + + const { rows, showLoading, failed } = useMetricResourceQuery(tile.query, { + ...sharedOptions, + minBucketSeconds: HERO_CHART_MIN_BUCKET_SECONDS, }); - const changeClassName = - state === "decrease" ? "text-error" : state === "increase" ? "text-success" : undefined; - const title = extraConcurrency === 0 ? "Purchase extra concurrency" : "Add/remove concurrency"; + const derived = tile.derive(rows); + const points = derived.points; - if (!showSelfServe) { - return ( - Request more} - /> - ); - } + const plottedBucketMs = points.length > 1 ? points[1]!.bucket - points[0]!.bucket : 0; + const floorWidenedBuckets = + plottedBucketMs > 0 && plottedBucketMs <= HERO_CHART_MIN_BUCKET_SECONDS * 1000; + const readoutQuery = tile.readout && floorWidenedBuckets ? tile.readout.query : ""; + const readoutResult = useMetricResourceQuery(readoutQuery, sharedOptions); + + const { total, formatTotal, totalClassName } = tile.readout + ? tile.readout.derive(readoutQuery ? readoutResult.rows : rows) + : derived; + + // Same point shape the shared axis/tooltip helpers expect. + const data = points + .map((p) => ({ bucket: p.bucket, [tile.id]: p.value })) + .filter((p) => Number.isFinite(p.bucket)); + + // Whole-line warning colour when the series was ever non-zero (throttling: one throttle in the + // window colours the entire line). Otherwise the tile's normal colour. + const wholeLineWarning = solidWarning && total > 0; + const lineColor = wholeLineWarning ? "var(--color-warning)" : tile.color; + + const chartConfig = useMemo( + () => ({ [tile.id]: { label: tile.label, color: lineColor } }), + [tile.id, tile.label, lineColor] + ); + + const { tickFormatter, tooltipLabelFormatter } = buildActivityTimeAxis(data); + const hasData = data.length > 0 && data.some((p) => Number(p[tile.id] ?? 0) > 0); + + // Peak readout lives in the card title (ChartCard has no dedicated value slot). A zero/empty + // total renders no readout at all (skipping "0% peak", "0 peak", "0" and the p95 "–" placeholder) + // so the card title stands alone until there's a non-zero value to show. + const readoutLoading = tile.readout ? readoutResult.showLoading : showLoading; + const readoutFailed = tile.readout ? readoutResult.failed : failed; + const peak = readoutLoading ? ( + + ) : readoutFailed || total === 0 ? null : formatTotal ? ( + formatTotal(total) + ) : ( + total.toLocaleString() + ); return ( - - - - - - {title} -
-
- - You can purchase bundles of {concurrencyPricing.stepSize} concurrency for{" "} - {formatCurrency(concurrencyPricing.centsPerStep / 100, false)}/month. Or you can - remove any extra concurrency after you have unallocated it from your environments - first. - -
- - - setAmountValue(Number(e.target.value))} - disabled={isLoading} + + + + {tile.label} + + + {peak != null ? ( + tile.totalTooltip && !readoutLoading ? ( + + {peak} + + } + content={tile.totalTooltip} + className="max-w-[230px]" + disableHoverableContent /> - {amount.errors} - {form.errors} - -
- {state === "need_to_increase_unallocated" ? ( -
- - You need to unallocate{" "} - {formatNumber(extraConcurrency - amountValue - extraUnallocatedConcurrency)} more - concurrency from your environments in order to remove{" "} - {formatNumber(extraConcurrency - amountValue)} concurrency from your account. - -
- ) : state === "above_quota" ? ( -
- - Currently you can only have up to {maxQuota} extra concurrency. Send a request - below to lift your current limit. We'll get back to you soon. - -
- ) : ( -
-
- Summary - Total -
-
- - {formatNumber(extraConcurrency)}{" "} - current total - - - {formatCurrency( - (extraConcurrency * concurrencyPricing.centsPerStep) / - concurrencyPricing.stepSize / - 100, - true - )} - -
-
- - ({simplur`${extraConcurrency / concurrencyPricing.stepSize} bundle[|s]`}) - - /mth + ) : ( + + {peak} + + ) + ) : null} + + {tile.legend && (showLoading || hasData) ? ( + + {tile.legend.map((item) => ( + + + {item.label} + + ))} + + ) : null} + + } + > + {showLoading ? ( + + ) : failed ? ( +
+ Unable to load metrics +
+ ) : hasData ? ( + + + + ) : ( +
+ No activity +
+ )} + + ); +} + +function QueueMetricChartSkeleton() { + return ( +
+ {Array.from({ length: 42 }).map((_, i) => ( +
+ ))} +
+ ); +} + +/** Health as a stock Badge: color carries the state, w-fit keeps it content-width. */ +type QueueHealth = { + paused: boolean; + running: number; + queued: number; + limit: number; +}; + +type QueueHealthLabel = "Paused" | "At capacity" | "Backlogged" | "Active" | "Idle"; + +// Single source of truth for the queue health decision, shared by the badge and the table's +// health-column sort so the sorted order always matches the labels shown. +function queueHealthLabel({ paused, running, queued, limit }: QueueHealth): QueueHealthLabel { + if (paused) return "Paused"; + if (isQueueAtCapacity({ running, queued, limit })) return "At capacity"; + if (queued > 0) return "Backlogged"; + if (running > 0) return "Active"; + return "Idle"; +} + +// Tint + colored text, sized like the error status chips (see ErrorStatusBadge). +const QUEUE_HEALTH_STYLES: Record = { + Paused: "bg-warning/10 text-warning system:bg-warning system:text-white", + "At capacity": "bg-warning/10 text-warning system:bg-warning system:text-white", + Backlogged: "bg-blue-500/10 text-blue-500 system:bg-blue-500 system:text-white", + Active: "bg-success/10 text-success system:bg-success system:text-white", + Idle: "bg-charcoal-500/10 text-text-dimmed system:bg-charcoal-500 system:text-white", +}; + +function QueueHealthBadge(health: QueueHealth) { + const label = queueHealthLabel(health); + return ( + + {label} + + ); +} + +// The metrics-row key a queue is stored under (task queues are prefixed `task/`). +function queueMetricsKey(queue: { type: string; name: string }): string { + return `${queue.type === "task" ? "task/" : ""}${queue.name}`; +} + +function formatWaitMs(ms: number): string { + if (ms < 1000) return `${Math.round(ms)}ms`; + if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s`; + if (ms < 3_600_000) return `${(ms / 60_000).toFixed(1)}m`; + return `${(ms / 3_600_000).toFixed(1)}h`; +} + +// Drop a trailing ".00" from whole percentages so "50.00" reads as "50" but "12.50" is preserved. +function formatOverridePercent(percent: number): string { + return Number.isInteger(percent) ? percent.toString() : percent.toFixed(2).replace(/\.?0+$/, ""); +} + +// Classic Queues page, restored verbatim from before the Queue Metrics feature. Rendered +// when queueMetricsUiEnabled is off so a gated org sees exactly the pre-metrics UI. +function ClassicQueuesView() { + const { environment, queues, pagination, hasFilters, autoReloadPollIntervalMs } = + useTypedLoaderData(); + + const organization = useOrganization(); + const project = useProject(); + const env = useEnvironment(); + const plan = useCurrentPlan(); + + useAutoRevalidate({ interval: autoReloadPollIntervalMs, onFocus: true }); + + const { limitStatus, limitClassName } = getEnvConcurrencyLimitStatus(environment); + + return ( + + + + + + + + Queues docs + + + + + +
+
+ paused : undefined} + animate + accessory={ +
+ {environment.runsEnabled && + env.pauseSource !== ENVIRONMENT_PAUSE_SOURCE_BILLING_LIMIT ? ( + + ) : null} +
-
- - {state === "increase" ? "+" : null} - {formatNumber(amountValue - extraConcurrency)} - - - {state === "increase" ? "+" : null} - {formatCurrency( - ((amountValue - extraConcurrency) * concurrencyPricing.centsPerStep) / - concurrencyPricing.stepSize / - 100, - true + } + valueClassName={env.paused ? "text-warning tabular-nums" : "tabular-nums"} + compactThreshold={1000000} + /> + -
-
- - ( - {simplur`${ - (amountValue - extraConcurrency) / concurrencyPricing.stepSize - } bundle[|s]`}{" "} - @ {formatCurrency(concurrencyPricing.centsPerStep / 100, true)}/mth) + > + Including {environment.running - environment.concurrencyLimit} burst runs{" "} + - /mth -
-
- - {formatNumber(amountValue)} new total - - - {formatCurrency( - (amountValue * concurrencyPricing.centsPerStep) / - concurrencyPricing.stepSize / - 100, - true + ) : limitStatus === "limit" ? ( + "At concurrency limit" + ) : undefined + } + accessory={ + + } + compactThreshold={1000000} + /> + 1 ? ( + -
-
- - ({simplur`${amountValue / concurrencyPricing.stepSize} bundle[|s]`}) + > + Burst limit {environment.burstFactor * environment.concurrencyLimit}{" "} + - /mth -
-
- )} + ) : undefined + } + accessory={ + plan ? ( + plan?.v3Subscription?.plan?.limits.concurrentRuns.canExceed ? ( + + Increase limit + + ) : ( + + Increase limit + + ) + ) : null + } + />
- - - - - ) : state === "decrease" || state === "need_to_increase_unallocated" ? ( - <> - -
+
+ + ); } -function updateState({ - value, - existingValue, - quota, - extraUnallocatedConcurrency, +function BurstFactorTooltip({ + environment, }: { - value: number; - existingValue: number; - quota: number; - extraUnallocatedConcurrency: number; -}): "no_change" | "increase" | "decrease" | "above_quota" | "need_to_increase_unallocated" { - if (value === existingValue) return "no_change"; - if (value < existingValue) { - const difference = existingValue - value; - if (difference > extraUnallocatedConcurrency) { - return "need_to_increase_unallocated"; - } - return "decrease"; - } - if (value > quota) return "above_quota"; - return "increase"; + environment: { burstFactor: number; concurrencyLimit: number }; +}) { + return ( + + ); } + +const limitTooltip = ( + <> + + How many runs can execute at once.{" "} + + + 1 (20) means 1 run per concurrency key, + but at most 20 runs across all keys. Set using{" "} + combinedConcurrencyLimit in your code. + + +); diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency_.$queueParam/route.tsx similarity index 93% rename from apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx rename to apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency_.$queueParam/route.tsx index 7ddcdb1d607..7ce627b810c 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency_.$queueParam/route.tsx @@ -1,6 +1,7 @@ import { type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime"; import { useCallback, useEffect, useMemo, useRef, useState, type ReactNode } from "react"; import type { QueueItem } from "@trigger.dev/core/v3/schemas"; +import type { QueueLimits } from "~/components/queues/queue-limits"; import { typedjson, useTypedLoaderData } from "remix-typedjson"; import { z } from "zod"; import { MainCenteredContainer, PageContainer } from "~/components/layout/AppLayout"; @@ -95,7 +96,7 @@ export const handle: Handle = { export const meta = pageMeta(({ data, params }) => [ data?.queue?.name ?? params.queueParam ?? "Queue", - "Queues", + "Concurrency", ]); const ParamsSchema = EnvironmentParamSchema.extend({ queueParam: z.string() }); @@ -119,7 +120,10 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { if (!environment) throw new Response(undefined, { status: 404, statusText: "Environment not found" }); - const retrieve = await new QueueRetrievePresenter().call({ environment, queueInput: queueParam }); + const retrieve = await new QueueRetrievePresenter().call({ + environment, + queueInput: queueParam, + }); if (!retrieve.success) { throw new Response(undefined, { status: 404, statusText: "Queue not found" }); } @@ -172,7 +176,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { const { organizationSlug, projectParam, envParam, queueParam } = ParamsSchema.parse(params); const url = new URL(request.url); - const redirectPath = `/orgs/${organizationSlug}/projects/${projectParam}/env/${envParam}/queues/${queueParam}${url.search}`; + const redirectPath = `/orgs/${organizationSlug}/projects/${projectParam}/env/${envParam}/concurrency/${queueParam}${url.search}`; if (request.method.toLowerCase() !== "post") { return redirectWithErrorMessage(redirectPath, request, "Wrong method"); @@ -266,7 +270,7 @@ export default function Page() { // The Concurrency keys tab exists only for queues with key activity: live keys in the // ckIndex, or nonzero CK history in the selected range (one cached scalar query decides). const { rows: gateRows, showLoading: gateLoading } = useQueueMetric( - `SELECT max(max_ck_backlogged) AS peak_keys, max(max_ck_wait_ms) AS peak_wait\nFROM queue_metrics`, + `SELECT max(max_ck_backlogged) AS peak_keys, max(max_ck_wait_ms) AS peak_wait\nFROM concurrency_metrics`, { ids, timeRange, queueName: fullName } ); const gateRow = gateRows[0]; @@ -285,14 +289,14 @@ export default function Page() { paused: queue.paused, running: queue.running, queued: queue.queued, - limit: queue.concurrencyLimit ?? environmentConcurrencyLimit, + limit: queue.limits.perKey.current ?? environmentConcurrencyLimit, oldestWaitMs, }); return ( - + {/* Paused-queue banner — mirrors the environment-paused banner (OrgBanner) at the top of the page when this individual queue is paused. */} @@ -392,7 +396,12 @@ export default function Page() { ) ) : ( - + )} @@ -436,10 +445,12 @@ function OverviewCharts({ ids, timeRange, queueName, + hasTotalLimit, }: { ids: Ids; timeRange: TimeRangeParams; queueName: string; + hasTotalLimit: boolean; }) { const zoomToTimeFilter = useZoomToTimeFilter(); return ( @@ -456,7 +467,7 @@ function OverviewCharts({ } showLegend className="aspect-[2/1]" - query={`SELECT timeBucket() AS t, max(max_running) AS running, max(max_limit) AS limit\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, max(max_running) AS running, max(max_limit) AS limit\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} @@ -479,11 +490,43 @@ function OverviewCharts({ // leading zeros so the reference line doesn't start with a false 0→limit step. carryBackfill={["limit"]} /> + {hasTotalLimit ? ( + + Runs in flight across ALL concurrency keys ( + ) versus the queue's combined limit ( + + ). + + } + showLegend + className="aspect-[2/1]" + query={`SELECT timeBucket() AS t, max(max_total_running) AS running, least(nullIf(max(max_total_limit), 0), max(max_env_limit)) AS cap, max(max_env_limit) AS sampled\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} + fillGaps + minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} + ids={ids} + timeRange={timeRange} + queueName={queueName} + series={[ + { key: "cap", label: "Total limit", color: COLORS.limit }, + { key: "running", label: "Running", color: COLORS.running }, + ]} + thresholdStroke={{ + series: "running", + valueFromSeries: "cap", + aboveColor: "var(--color-warning)", + }} + carryBackfill={["cap"]} + carryBackfillGuard="sampled" + /> + ) : null} } className="aspect-[2/1] sm:col-span-2 sm:aspect-[4/1]" - query={`SELECT timeBucket() AS t, sum(throttled_count) AS throttled\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, sum(throttled_count) AS throttled\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} @@ -660,7 +703,7 @@ function ConcurrencyKeyCharts({ title="Keys with backlog" info="Keys with runs waiting at once." className="aspect-[2/1]" - query={`SELECT timeBucket() AS t, max(max_ck_backlogged) AS keys\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, max(max_ck_backlogged) AS keys\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} fillGaps ids={ids} timeRange={timeRange} @@ -682,7 +725,7 @@ function ConcurrencyKeyCharts({ ) : null } className="aspect-[2/1]" - query={`SELECT timeBucket() AS t, max(max_ck_wait_ms) AS wait\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, max(max_ck_wait_ms) AS wait\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`} fillGaps ids={ids} timeRange={timeRange} @@ -744,7 +787,7 @@ type GroupedKeyChartProps = { // search can match keys outside the top 8; then filter by the search and keep the top 8 of those. function GroupedKeyChartCard(props: GroupedKeyChartProps) { const { rows, showLoading, failed } = useQueueMetric( - `SELECT concurrency_key, ${props.rankExpr} AS peak\nFROM queue_metrics_by_key\nGROUP BY concurrency_key\nORDER BY peak DESC\nLIMIT 50`, + `SELECT concurrency_key, ${props.rankExpr} AS peak\nFROM concurrency_metrics_by_key\nGROUP BY concurrency_key\nORDER BY peak DESC\nLIMIT 50`, { ids: props.ids, timeRange: props.timeRange, queueName: props.queueName } ); const keyFilter = props.keyFilter; @@ -772,7 +815,7 @@ function GroupedKeySeries({ }: GroupedKeyChartProps & { keys: string[] }) { const inList = keys.map((k) => `'${trqlString(k)}'`).join(", "); const { rows, showLoading, failed } = useQueueMetric( - `SELECT timeBucket() AS t, concurrency_key, ${seriesExpr} AS v\nFROM queue_metrics_by_key\nWHERE concurrency_key IN (${inList})\nGROUP BY t, concurrency_key\nORDER BY t`, + `SELECT timeBucket() AS t, concurrency_key, ${seriesExpr} AS v\nFROM concurrency_metrics_by_key\nWHERE concurrency_key IN (${inList})\nGROUP BY t, concurrency_key\nORDER BY t`, { ids, timeRange, queueName, fillGaps } ); @@ -1036,7 +1079,7 @@ function KeyDrilldown({ } className="aspect-[2/1]" - query={`SELECT timeBucket() AS t, max(max_queued) AS queued, max(max_running) AS running\nFROM queue_metrics_by_key\nWHERE ${pin}\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, max(max_queued) AS queued, max(max_running) AS running\nFROM concurrency_metrics_by_key\nWHERE ${pin}\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} @@ -1051,7 +1094,7 @@ function KeyDrilldown({ 0, round(sum(wait_ms_sum) / sum(wait_ms_count)), 0) AS wait, sum(wait_ms_count) AS samples\nFROM queue_metrics_by_key\nWHERE ${pin}\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, if(sum(wait_ms_count) > 0, round(sum(wait_ms_sum) / sum(wait_ms_count)), 0) AS wait, sum(wait_ms_count) AS samples\nFROM concurrency_metrics_by_key\nWHERE ${pin}\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} sampleCountColumn="samples" @@ -1099,7 +1142,7 @@ function QueueStats({ }: { // Carries the percent override source-of-truth (not part of the shared QueueItem contract) so the // override dialog reopens in percent mode for percent-based overrides. - queue: QueueItem & { concurrencyLimitOverridePercent: number | null }; + queue: QueueItem & { concurrencyLimitOverridePercent: number | null; limits: QueueLimits }; environmentConcurrencyLimit: number; queuedRunsPath: string; oldestWaitMs: number | null; @@ -1107,18 +1150,21 @@ function QueueStats({ timeRange: TimeRangeParams; queueName: string; }) { - const { rows } = useQueueMetric(`SELECT max(max_queued) AS peak_queued\nFROM queue_metrics`, { - ids, - timeRange, - queueName, - }); + const { rows } = useQueueMetric( + `SELECT max(max_queued) AS peak_queued\nFROM concurrency_metrics`, + { + ids, + timeRange, + queueName, + } + ); const peakQueued = rows[0] ? toNumber(rows[0].peak_queued) : 0; // Latest gauges from ClickHouse, polled every 15s so the live blocks keep ticking after first // paint. Read the newest bucket (largest t); until the first poll lands liveRows is empty and the // *Live values stay null, so the blocks show the loader values instead of flashing 0. const { rows: liveRows, responseReceivedAt } = useQueueMetric( - `SELECT timeBucket() AS t, max(max_running) AS running, max(max_queued) AS queued, max(max_limit) AS q_limit, max(max_ck_wait_ms) AS ck_wait FROM queue_metrics GROUP BY t ORDER BY t`, + `SELECT timeBucket() AS t, max(max_running) AS running, max(max_queued) AS queued, max(max_limit) AS q_limit, max(max_ck_wait_ms) AS ck_wait FROM concurrency_metrics GROUP BY t ORDER BY t`, { ids, timeRange: { period: "15m", from: null, to: null }, @@ -1149,7 +1195,7 @@ function QueueStats({ const queuedDisplay = queuedLive ?? queue.queued; // Limit is queue config, not a live signal: keep the loader's value. Only if the loader had none // do we fall back to the CH gauge for display. - const limitDisplay = queue.concurrencyLimit ?? (limitLive || null); + const limitDisplay = queue.limits.perKey.current ?? (limitLive || null); // Keyed queues report head-of-line wait via CH (max_ck_wait_ms); use it as the live headline when // present. Non-keyed queues have no CH signal, so they stay on the loader value. const oldestWaitDisplayMs = ckWaitLive !== null && ckWaitLive > 0 ? ckWaitLive : oldestWaitMs; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.limits/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.limits/route.tsx index e8fb74da399..045c67e592b 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.limits/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.limits/route.tsx @@ -43,7 +43,7 @@ import { requireUserId } from "~/services/session.server"; import { cn } from "~/utils/cn"; import { formatNumber } from "~/utils/numberFormatter"; import { - concurrencyPath, + concurrencyLimitsPath, docsPath, EnvironmentParamSchema, organizationBillingPath, @@ -153,7 +153,7 @@ export default function Page() { {/* Concurrency Section */} {/* Rate Limits Section */} @@ -231,7 +231,7 @@ function CurrentPlanSection({ ); } -function ConcurrencySection({ concurrencyPath }: { concurrencyPath: string }) { +function ConcurrencySection({ concurrencyLimitsPath }: { concurrencyLimitsPath: string }) { return (
@@ -247,7 +247,7 @@ function ConcurrencySection({ concurrencyPath }: { concurrencyPath: string }) { Concurrency - + Manage concurrency diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues.$.ts b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues.$.ts new file mode 100644 index 00000000000..cc950eb6786 --- /dev/null +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues.$.ts @@ -0,0 +1,10 @@ +import { redirect, type LoaderFunctionArgs } from "@remix-run/server-runtime"; + +/** The queues page became the Concurrency page; old URLs (bookmarks, agent deep + * links, the queue detail path) redirect with their sub-path and query intact. */ +export const loader = async ({ params, request }: LoaderFunctionArgs) => { + const url = new URL(request.url); + const subPath = params["*"] ? `/${params["*"]}` : ""; + const base = url.pathname.replace(/\/queues(\/.*)?$/, "/concurrency"); + return redirect(`${base}${subPath}${url.search}`, 301); +}; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues.ts b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues.ts new file mode 100644 index 00000000000..69646e8c781 --- /dev/null +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues.ts @@ -0,0 +1,8 @@ +import { redirect, type LoaderFunctionArgs } from "@remix-run/server-runtime"; + +/** The queues page became the Concurrency page; the bare old URL redirects with + * its query intact. */ +export const loader = async ({ request }: LoaderFunctionArgs) => { + const url = new URL(request.url); + return redirect(`${url.pathname.replace(/\/queues$/, "/concurrency")}${url.search}`, 301); +}; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx deleted file mode 100644 index 475f25362dc..00000000000 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ /dev/null @@ -1,2024 +0,0 @@ -import { - ArrowUpCircleIcon, - BookOpenIcon, - ExclamationTriangleIcon, - PauseIcon, - PlayIcon, - RectangleStackIcon, -} from "@heroicons/react/20/solid"; -import { DialogClose } from "@radix-ui/react-dialog"; -import { Form, useNavigation } from "@remix-run/react"; -import { type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime"; -import type { RuntimeEnvironmentType } from "@trigger.dev/database"; -import { useEffect, useMemo, useState, type ReactNode } from "react"; -import { QueuesIcon } from "~/assets/icons/QueuesIcon"; -import { typedjson, useTypedLoaderData } from "remix-typedjson"; -import { z } from "zod"; -import { ConcurrencyIcon } from "~/assets/icons/ConcurrencyIcon"; -import { RunsIcon } from "~/assets/icons/RunsIcon"; -import { AdminDebugTooltip } from "~/components/admin/debugTooltip"; -import { environmentFullTitle } from "~/components/environments/EnvironmentLabel"; -import { PageBody, PageContainer } from "~/components/layout/AppLayout"; -import { MetricsLayout } from "~/components/layout/MetricsLayout"; -import { Badge } from "~/components/primitives/Badge"; -import { Button, LinkButton } from "~/components/primitives/Buttons"; -import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "~/components/primitives/Dialog"; -import { FormButtons } from "~/components/primitives/FormButtons"; -import { Header3 } from "~/components/primitives/Headers"; -import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader"; -import { PaginationControls } from "~/components/primitives/Pagination"; -import { Paragraph } from "~/components/primitives/Paragraph"; -import { PopoverMenuItem } from "~/components/primitives/Popover"; -import { SearchInput } from "~/components/primitives/SearchInput"; -import { Spinner } from "~/components/primitives/Spinner"; -import { - Table, - TableBody, - TableCell, - TableCellMenu, - TableHeader, - TableHeaderCell, - TableRow, -} from "~/components/primitives/Table"; -import { - InfoIconTooltip, - SimpleTooltip, - Tooltip, - TooltipContent, - TooltipProvider, - TooltipTrigger, -} from "~/components/primitives/Tooltip"; -import { TasksIcon } from "~/assets/icons/TasksIcon"; -import { QueueName } from "~/components/runs/v3/QueueName"; -import { env } from "~/env.server"; -import { useAutoRevalidate } from "~/hooks/useAutoRevalidate"; -import { useEnvironment } from "~/hooks/useEnvironment"; -import { useOrganization } from "~/hooks/useOrganizations"; -import { useProject } from "~/hooks/useProject"; -import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server"; -import { findProjectBySlug } from "~/models/project.server"; -import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; -import { EnvironmentQueuePresenter } from "~/presenters/v3/EnvironmentQueuePresenter.server"; -import { QueueListPresenter } from "~/presenters/v3/QueueListPresenter.server"; -import { - QueueMetricsPresenter, - type QueueListMetric, -} from "~/presenters/v3/QueueMetricsPresenter.server"; -import { TimeFilter, timeFilterFromTo } from "~/components/runs/v3/SharedFilters"; -import { useSearchParams } from "~/hooks/useSearchParam"; -import { parseFiniteInt } from "~/utils/searchParams"; -import { MiniLineChart } from "~/components/metrics/MiniLineChart"; -import { buildActivityTimeAxis } from "~/components/primitives/charts/activityTimeAxis"; -import { Chart, type ChartConfig } from "~/components/primitives/charts/ChartCompound"; -import { ChartCard } from "~/components/primitives/charts/ChartCard"; -import { ChartSyncProvider } from "~/components/primitives/charts/ChartSyncContext"; -import { useZoomToTimeFilter } from "~/hooks/useZoomToTimeFilter"; -import { - useIsMetricResponseFresh, - useMetricResourceQuery, - type MetricResourceTimeRange, -} from "~/hooks/useMetricResourceQuery"; -import { logger } from "~/services/logger.server"; -import { requireUserId } from "~/services/session.server"; -import { cn } from "~/utils/cn"; -import { ENVIRONMENT_PAUSE_SOURCE_BILLING_LIMIT } from "~/utils/environmentPauseSource"; -import { - concurrencyPath, - docsPath, - EnvironmentParamSchema, - v3BillingPath, - v3QueuePath, - v3RunsPath, -} from "~/utils/pathBuilder"; -import type { Handle } from "~/utils/handle"; -import { queuesAgentPageContext } from "~/components/dashboard-agent/suggested-prompts"; -import { WhenAgentUnavailable } from "~/components/dashboard-agent/WhenAgentUnavailable"; -import { PauseEnvironmentService } from "~/v3/services/pauseEnvironment.server"; -import { handleQueueMutationAction } from "~/models/queueMutation.server"; -import { - QueueOverrideConcurrencyButton, - QueuePauseResumeButton, -} from "~/components/queues/QueueControls"; -import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route"; -import { BigNumber } from "~/components/metrics/BigNumber"; -import { canAccessQueueMetricsUi } from "~/v3/canAccessQueueMetricsUi.server"; -import { QueueAllocationPresenter } from "~/presenters/v3/QueueAllocationPresenter.server"; -import { - QUEUE_METRICS_DEFAULT_PERIOD, - QUEUE_METRICS_RETENTION_DAYS, - clampQueueMetricsPeriod, - clipQueueMetricsWindow, - queueMetricsPeriodFromRequest, - resolveQueueMetricsPeriod, - useRememberQueueMetricsPeriod, -} from "~/components/queues/queueMetricsPeriod"; -import { queueMetricsMaxPeriodDays } from "~/components/queues/queueMetricsPeriod.server"; -import { isQueueAtCapacity } from "~/components/queues/queue-thresholds"; -import { pageMeta } from "~/utils/pageTitle"; - -const SearchParamsSchema = z.object({ - query: z.string().optional(), - page: z.coerce.number().min(1).default(1), - period: z.string().optional(), - from: z.string().optional(), - to: z.string().optional(), - sort: z.enum(["busiest", "queued", "name"]).optional(), -}); - -// The live "Queued" / "Running" header blocks poll ClickHouse on a short cadence so they stay -// current after first paint. They read the env-wide gauges from env_metrics (the env-level rollup -// of queue_metrics, cheapest for a dimension-free query), always over a fixed 15m window regardless -// of the chart/table period, and are NOT scoped to the visible queue set (the blocks are env-wide). -const QUEUE_LIVE_BLOCKS_PERIOD = "15m"; -const QUEUE_LIVE_BLOCKS_QUERY = - "SELECT timeBucket() AS t, max(max_env_queued) AS env_queued, max(max_env_running) AS env_running FROM env_metrics GROUP BY t ORDER BY t"; -// Trust the ClickHouse gauge only while its newest bucket is this recent; otherwise fall back to -// the loader's Redis-exact live values (matches LIVE_GAUGE_FRESH_MS on the queue detail page / run -// inspector). -const LIVE_GAUGE_FRESH_MS = 90_000; - -export const handle: Handle = { - agentPageContext: (data) => queuesAgentPageContext(data), -}; - -export const meta = pageMeta("Queues"); - -export const loader = async ({ request, params }: LoaderFunctionArgs) => { - const userId = await requireUserId(request); - const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); - - const url = new URL(request.url); - const { page, query, period, from, to, sort } = SearchParamsSchema.parse( - Object.fromEntries(url.searchParams) - ); - - const project = await findProjectBySlug(organizationSlug, projectParam, userId); - if (!project) { - throw new Response(undefined, { - status: 404, - statusText: "Project not found", - }); - } - - const environment = await findEnvironmentBySlug(project.id, envParam, userId); - if (!environment) { - throw new Response(undefined, { - status: 404, - statusText: "Environment not found", - }); - } - - // Per-org gate for the metrics UI. When off, this org gets the classic Queues page and - // no metrics query fires. - const queueMetricsUiEnabled = await canAccessQueueMetricsUi({ - request, - userId, - organizationSlug, - }); - - const maxPeriodDays = queueMetricsUiEnabled - ? await queueMetricsMaxPeriodDays(environment.organizationId) - : QUEUE_METRICS_RETENTION_DAYS; - const defaultPeriod = clampQueueMetricsPeriod( - queueMetricsPeriodFromRequest(request), - maxPeriodDays - ); - - try { - const queueListPresenter = new QueueListPresenter(); - const queues = await queueListPresenter.call({ - environment, - query, - page, - // Relevance ordering rides the metrics pipeline, so it is part of the gated UI. - sort: queueMetricsUiEnabled ? (sort ?? "busiest") : "name", - }); - - const environmentQueuePresenter = new EnvironmentQueuePresenter(); - - const autoReloadPollIntervalMs = env.QUEUES_AUTORELOAD_POLL_INTERVAL_MS; - - // Per-queue list metrics (Delay p95 + backlog sparkline columns) are SSR'd with the table. - // The environment header tiles are fetched client-side per card (see QueueEnvMetricChart) so a - // slow ClickHouse query never blocks the queues list from rendering. - let metrics: { - bucketStartMs: number; - bucketIntervalMs: number; - byQueue: Record; - } | null = null; - - if (queueMetricsUiEnabled) { - // Metrics are additive observability; a ClickHouse hiccup must not take down queue - // management. Fail open to metrics: null instead of bubbling to the page-level 400. - try { - const presenter = new QueueMetricsPresenter(); - const queueNames = queues.queues.map((q) => - q.type === "task" ? `task/${q.name}` : q.name - ); - const timeRange = clipQueueMetricsWindow( - timeFilterFromTo({ - period: - resolveQueueMetricsPeriod({ period, from, to, defaultPeriod, maxPeriodDays }) ?? - undefined, - from: parseFiniteInt(from), - to: parseFiniteInt(to), - defaultPeriod, - }), - maxPeriodDays - ); - const queueMetrics = - queueNames.length > 0 - ? await presenter.getQueueListMetrics({ - environment, - queueNames, - from: timeRange.from, - to: timeRange.to, - }) - : null; - if (queueMetrics) { - metrics = { - bucketStartMs: queueMetrics.bucketStartMs, - bucketIntervalMs: queueMetrics.bucketIntervalMs, - byQueue: Object.fromEntries(queueMetrics.byQueue), - }; - } - } catch (error) { - logger.warn("Queue list metrics unavailable, rendering without them", { error }); - } - } - - // Allocation summary (Environment limit + Allocated tiles) is additive; a presenter - // failure must not 400 the page, so fail open to null like the metrics block above. - let allocation: Awaited> | null = null; - if (queueMetricsUiEnabled) { - try { - allocation = await new QueueAllocationPresenter().call({ environment }); - } catch (error) { - logger.warn("Queue allocation summary unavailable, rendering without it", { error }); - } - } - - return typedjson({ - ...queues, - environment: await environmentQueuePresenter.call(environment), - autoReloadPollIntervalMs, - metrics, - allocation, - queueMetricsUiEnabled, - defaultPeriod, - maxPeriodDays, - }); - } catch (error) { - console.error(error); - throw new Response(undefined, { - status: 400, - statusText: "Something went wrong, if this problem persists please contact support.", - }); - } -}; - -export const action = async ({ request, params }: ActionFunctionArgs) => { - const userId = await requireUserId(request); - if (request.method.toLowerCase() !== "post") { - return redirectWithErrorMessage( - `/orgs/${params.organizationSlug}/projects/${params.projectParam}/env/${params.envParam}/queues`, - request, - "Wrong method" - ); - } - - const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); - - const project = await findProjectBySlug(organizationSlug, projectParam, userId); - if (!project) { - throw new Response(undefined, { - status: 404, - statusText: "Project not found", - }); - } - - const environment = await findEnvironmentBySlug(project.id, envParam, userId); - if (!environment) { - throw new Response(undefined, { - status: 404, - statusText: "Environment not found", - }); - } - - const formData = await request.formData(); - const action = formData.get("action"); - - const url = new URL(request.url); - const redirectPath = `/orgs/${organizationSlug}/projects/${projectParam}/env/${envParam}/queues${url.search}`; - - if (environment.archivedAt) { - return redirectWithErrorMessage(redirectPath, request, "This branch is archived"); - } - - // Per-queue actions (pause/resume/override/remove-override) are shared with the queue detail - // route, so they live in a helper that both routes call. - const queueMutation = await handleQueueMutationAction({ - request, - environment, - userId, - formData, - redirectPath, - }); - if (queueMutation) { - return queueMutation; - } - - switch (action) { - case "environment-pause": { - const pauseService = new PauseEnvironmentService(); - const result = await pauseService.call(environment, "paused"); - if (!result.success) { - return redirectWithErrorMessage(redirectPath, request, result.error); - } - return redirectWithSuccessMessage(redirectPath, request, "Environment paused"); - } - case "environment-resume": { - const resumeService = new PauseEnvironmentService(); - const result = await resumeService.call(environment, "resumed"); - if (!result.success) { - return redirectWithErrorMessage(redirectPath, request, result.error); - } - return redirectWithSuccessMessage(redirectPath, request, "Environment resumed"); - } - default: - return redirectWithErrorMessage(redirectPath, request, "Something went wrong"); - } -}; - -// Derives the environment concurrency status ("limit" | "burst" | "within") and the matching -// text color from the current running count vs. the env limit and burst factor. Shared by both -// the classic and metrics views so the "Running" tile styling stays in sync. -function getEnvConcurrencyLimitStatus(environment: { - running: number; - concurrencyLimit: number; - burstFactor: number; -}) { - const limitStatus = - environment.running === environment.concurrencyLimit * environment.burstFactor - ? "limit" - : environment.running > environment.concurrencyLimit - ? "burst" - : "within"; - - const limitClassName = - limitStatus === "burst" ? "text-warning" : limitStatus === "limit" ? "text-error" : undefined; - - return { limitStatus, limitClassName }; -} - -export default function Page() { - // Per-org flag decides which whole page renders. Off => the classic Queues page, - // byte-for-byte the pre-metrics UI. Each branch is its own component (own hooks). - const { queueMetricsUiEnabled } = useTypedLoaderData(); - return queueMetricsUiEnabled ? : ; -} - -function QueuesWithMetricsView() { - const { - environment, - queues, - pagination, - totalQueues, - hasFilters, - autoReloadPollIntervalMs, - metrics, - allocation, - defaultPeriod, - maxPeriodDays, - } = useTypedLoaderData(); - - const metricsByQueue = metrics?.byQueue ?? {}; - - const organization = useOrganization(); - const project = useProject(); - const env = useEnvironment(); - const plan = useCurrentPlan(); - - // The header tiles fetch client-side with the same period/from/to the TimeFilter writes. - const { value } = useSearchParams(); - const timeRange = { - period: resolveQueueMetricsPeriod({ - period: value("period"), - from: value("from"), - to: value("to"), - defaultPeriod, - maxPeriodDays, - }), - from: value("from") ?? null, - to: value("to") ?? null, - }; - useRememberQueueMetricsPeriod(value("period")); - - useAutoRevalidate({ interval: autoReloadPollIntervalMs, onFocus: true }); - - // Drag-to-zoom on either chart narrows the page's from/to search params, which the - // TimeFilter and the client-side metric queries both read (same wiring as the Agent page). - const zoomToTimeFilter = useZoomToTimeFilter(); - - // Live env-wide Queued/Running blocks. First paint uses the loader's Redis-exact values; from the - // first poll on we prefer ClickHouse so the blocks stay current without a full page revalidate. - // Empty rows (quiet env, or the very first fetch still in flight) fall back to the loader values, - // so we never flash a stale 0. Fixed 15m window, env-wide (no queue filter), CH-only recurring - // load; pauses while the tab is hidden (handled inside the hook). - const { rows: liveBlockRows, responseReceivedAt } = useMetricResourceQuery( - QUEUE_LIVE_BLOCKS_QUERY, - { - organizationId: organization.id, - projectId: project.id, - environmentId: env.id, - timeRange: { period: QUEUE_LIVE_BLOCKS_PERIOD, from: null, to: null }, - defaultPeriod: QUEUE_LIVE_BLOCKS_PERIOD, - fillGaps: false, - refreshIntervalMs: 15_000, - } - ); - const lastLiveBlockRow = - liveBlockRows.length > 0 ? liveBlockRows[liveBlockRows.length - 1] : null; - // Only trust the gauge while its newest bucket is fresh. A row painted from the hook's cache on - // client-side nav-back (responseCache), or a quiet env whose latest bucket is minutes old, must - // not override the loader's Redis-exact live values with a stale count. - const lastLiveBucketMs = lastLiveBlockRow ? tileTimeToMs(lastLiveBlockRow.t) : NaN; - const liveBlockIsFresh = useIsMetricResponseFresh( - responseReceivedAt, - lastLiveBucketMs, - LIVE_GAUGE_FRESH_MS - ); - const freshLiveBlockRow = lastLiveBlockRow && liveBlockIsFresh ? lastLiveBlockRow : null; - const envQueuedLive = freshLiveBlockRow - ? tileNumber(freshLiveBlockRow.env_queued) - : environment.queued; - const envRunningLive = freshLiveBlockRow - ? tileNumber(freshLiveBlockRow.env_running) - : environment.running; - - // Allocation summary tiles. The presenter computes the env-wide allocated total (sum of - // each queue's explicit limit clamped to the env limit) in a single aggregate query. - const envLimit = environment.concurrencyLimit; - const burstLimit = Math.round(envLimit * environment.burstFactor); - const allocated = allocation?.allocated ?? 0; - const allocationPct = envLimit > 0 ? Math.round((allocated / envLimit) * 100) : 0; - - // Running-block tinting (burst/limit) tracks the live running value, not the loader snapshot. - const { limitStatus, limitClassName } = getEnvConcurrencyLimitStatus({ - running: envRunningLive, - concurrencyLimit: environment.concurrencyLimit, - burstFactor: environment.burstFactor, - }); - - // Client-side, header-click sorting over the current page's rows. Server pagination and the - // default busiest order are unchanged; clearing a sort returns to that server order. - const queueRows = queues ?? []; - - return ( - - - - - - - - Queues docs - - - - - - {/* Filters — pinned bar directly under the NavBar. This row is page-wide only: Period is - the one control that changes the tiles and charts below, so it leads the row. Search - and pagination scope the table alone and live in that table's own bar instead. */} - -
- -
-
- {environment.runsEnabled && - env.pauseSource !== ENVIRONMENT_PAUSE_SOURCE_BILLING_LIMIT ? ( - - ) : null} -
-
- - {/* Queued + Running + Allocated + Environment limit summary. Four stat tiles: the grid - derives its columns from the tile count (two-up, four-up from lg). The allocation - presenter fails open to null (a ClickHouse/PG hiccup mustn't take down the tiles), so - only the Allocated tile depends on it — the other three + controls always render, and - Allocated shows a "–" placeholder to keep the 4-tile grid shape stable. */} - - paused : undefined} - animate - accessory={ - - - - } - valueClassName={env.paused ? "text-warning tabular-nums" : "tabular-nums"} - compactThreshold={1000000} - /> - - Including {envRunningLive - environment.concurrencyLimit} burst runs{" "} - - - ) : limitStatus === "limit" ? ( - "At concurrency limit" - ) : undefined - } - accessory={ - - - - } - compactThreshold={1000000} - /> - - Allocated - {allocation ? ( - - ) : null} - - } - value={allocation ? allocated : undefined} - formattedValue={allocation ? undefined : "–"} - suffix={allocation ? `${allocationPct}% of the environment limit` : undefined} - suffixClassName="text-text-dimmed" - /> - 1 ? `bursts up to ${burstLimit}` : undefined} - suffixClassName="text-text-dimmed" - accessory={ - plan ? ( - plan?.v3Subscription?.plan?.limits.concurrentRuns.canExceed ? ( - - Increase limit - - ) : ( - - Increase limit - - ) - ) : undefined - } - /> - - - {hasFilters || totalQueues !== 0 ? ( - - - {QUEUE_HEADER_TILES.map((tile) => ( - 1 - ? [ - { - y: Math.round(environment.burstFactor * 100), - label: `Burst ${Math.round( - environment.concurrencyLimit * environment.burstFactor - )}`, - labelPlacement: "outside" as const, - }, - ] - : []), - ] - : undefined - } - // Saturation recolours the line above its 100% limit with a gradient split, so - // only the portion over the line is orange (the offset is derived from the line's - // own value range, so the split lands exactly at 100% regardless of domain - // padding). p95 and throttled use a per-bucket overlay: it retraces only the - // over-threshold stretches, so under-threshold buckets stay blue. - thresholdStroke={ - tile.id === "saturation" - ? { value: 100, aboveColor: "var(--color-warning)" } - : undefined - } - warningOverlay={ - tile.id === "p95" - ? { threshold: 60_000 } - : tile.id === "throttled" - ? // Integer counts: threshold 0 warns once a bucket has ≥1 throttle. - { threshold: 0 } - : undefined - } - /> - ))} - - - ) : null} - - - - - - } - > - {/* Default overflow-x-auto container so wide tables still scroll horizontally on - narrow viewports; the page (not this region) owns vertical scrolling. */} - - - - Name - Queued - Running - Limit - -

- Environment: uses the environment - limit of {environment.concurrencyLimit}. -

-

- User: a limit you set in your - code. -

-

- Override: a limit you set here or - via the API. -

- - } - > - Limited by -
- Health - - Delay p95 - - - How many runs were waiting, over the selected time. marks - where the queue was throttled. - - } - > - Backlog - - - Pause/resume - -
-
- - {queueRows.length > 0 ? ( - queueRows.map((queue) => { - const limit = queue.concurrencyLimit ?? environment.concurrencyLimit; - const isAtConcurrencyLimit = queue.running >= limit; - const isAtQueueLimit = - environment.queueSizeLimit !== null && - queue.queued >= environment.queueSizeLimit; - const queueFilterableName = queueMetricsKey(queue); - const queueMetric = metricsByQueue[queueFilterableName]; - const queueDetailPath = v3QueuePath(organization, project, env, { - friendlyId: queue.id, - }); - return ( - - s, so - // they render beside the link (leading/trailing), never inside it — - // otherwise the cell is invalid
-
-
-
- ); -} - -function EnvironmentPauseResumeButton({ - env, -}: { - env: { type: RuntimeEnvironmentType; paused: boolean }; -}) { - const navigation = useNavigation(); - const [isOpen, setIsOpen] = useState(false); - - useEffect(() => { - if (navigation.state === "loading" || navigation.state === "idle") { - // oxlint-disable-next-line react/set-state-in-effect -- This effect intentionally synchronizes route state after an external or lifecycle change. - setIsOpen(false); - } - }, [navigation.state]); - - const isLoading = Boolean( - navigation.formData?.get("action") === (env.paused ? "environment-resume" : "environment-pause") - ); - - return ( - -
- - - -
- - - -
-
- - {env.paused - ? `Resumes ${environmentFullTitle(env)} so its runs can be dequeued again.` - : `Pauses all runs from being dequeued in ${environmentFullTitle(env)}. Any executing runs will continue to run.`} - -
-
-
- - {env.paused ? "Resume environment?" : "Pause environment?"} -
- - {env.paused - ? `This will allow runs to be dequeued in ${environmentFullTitle(env)} again.` - : `This will pause all runs from being dequeued in ${environmentFullTitle( - env - )}. Any executing runs will continue to run.`} - -
setIsOpen(false)}> - - : env.paused ? PlayIcon : PauseIcon - } - shortcut={{ modifiers: ["mod"], key: "enter" }} - > - {env.paused ? "Resume environment" : "Pause environment"} - - } - cancelButton={ - - - - } - /> - -
-
-
- ); -} - -export function isEnvironmentPauseResumeFormSubmission( - formMethod: string | undefined, - formData: FormData | undefined -) { - if (!formMethod || !formData) { - return false; - } - - return ( - formMethod.toLowerCase() === "post" && - (formData.get("action") === "environment-pause" || - formData.get("action") === "environment-resume") - ); -} - -export function QueueFilters() { - return ; -} - -type MetricTileRow = Record; - -type TilePoint = { bucket: number; value: number | null }; - -// Inline colour swatch matching the chart's warning ("yellow") line — used in tooltip copy that -// refers to that colour instead of naming it, so the swatch always matches the chart. -function WarningSwatch() { - return ( - - ); -} - -type QueueHeaderTile = { - id: string; - label: string; - /** Info-icon copy explaining what the chart shows, rendered next to the card title. */ - description: ReactNode; - color: string; - /** Optional inline legend rendered below the card title: a fixed set of {colored square, label} - * entries, for charts where a colour (e.g. the orange warning line) needs explaining. */ - legend?: Array<{ color: string; label: string }>; - query: string; - /** Formats a single bucket's value in the chart tooltip. */ - formatValue?: (value: number) => string; - /** Formats the y-axis tick labels. Without it the axis shows raw numbers (bad for durations - * in ms or percent scales). Passed through to Chart.Line's yAxisProps.tickFormatter. */ - formatAxis?: (value: number) => string; - /** Hover tooltip explaining the headline readout next to the title (e.g. what "9% of current - * period" means). Without it the readout has no tooltip. */ - totalTooltip?: string; - /** Turns one row per bucket into the per-bucket points the chart draws. A null value is a - * bucket the metric has nothing to say about, and the line breaks there rather than reading 0. */ - derive: (rows: MetricTileRow[]) => { - points: TilePoint[]; - total: number; - formatTotal?: (total: number) => string; - totalClassName?: string; - }; - /** - * Optional second query, run at the range's natural bucket width, that owns the headline readout. - * For a headline that is not invariant to bucket width — a share of buckets, or a percentile, - * as opposed to a max over gauges — deriving it from the plotted rows would move the number - * whenever this tile's floor widens them. Only requested while the floor is actually widening - * anything; on ranges whose natural width is already at or above the floor the chart's own rows - * are identical and are reused. - */ - readout?: { - query: string; - derive: (rows: MetricTileRow[]) => { - total: number; - formatTotal?: (total: number) => string; - totalClassName?: string; - }; - }; -}; - -function tileNumber(value: number | string | null): number { - const n = typeof value === "number" ? value : Number(value); - return Number.isFinite(n) ? n : 0; -} - -function tileTimeToMs(value: number | string | null): number { - const s = String(value).replace(" ", "T"); - return Date.parse(s.endsWith("Z") ? s : `${s}Z`); -} - -/** Peak of a series, ignoring the buckets it has nothing to say about. */ -function peakOf(points: TilePoint[]): number { - return points.reduce((max, p) => (p.value === null ? max : Math.max(max, p.value)), 0); -} - -const SCHEDULING_DELAY_QUERY = `SELECT timeBucket() AS t,\n round(quantilesTDigestMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95,\n sum(wait_ms_count) AS samples\nFROM env_metrics\nGROUP BY t\nORDER BY t`; - -const THROTTLED_QUERY = `SELECT timeBucket() AS t,\n sum(throttled_count) AS throttled\nFROM env_metrics\nGROUP BY t\nORDER BY t`; - -const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ - { - id: "saturation", - label: "Env saturation", - description: ( - <> - How much of the environment's concurrency is in use. Turns above 100%, - when it's into burst capacity. - - ), - color: "var(--color-queues-chart)", - legend: [ - { color: "var(--color-queues-chart)", label: "Saturation" }, - { color: "var(--color-warning)", label: "Over limit" }, - ], - query: `SELECT timeBucket() AS t,\n max(max_env_running) AS running,\n max(max_env_limit) AS env_limit\nFROM env_metrics\nGROUP BY t\nORDER BY t`, - formatValue: (v) => (v > 100 ? `${v}% — over the environment limit` : `${v}%`), - formatAxis: (v) => `${v}%`, - derive: (rows) => { - const points = rows.map((r) => { - const limit = tileNumber(r.env_limit); - return { - bucket: tileTimeToMs(r.t), - value: limit > 0 ? Math.round((tileNumber(r.running) / limit) * 100) : 0, - }; - }); - return { points, total: peakOf(points), formatTotal: (v) => `${v}% peak` }; - }, - }, - { - id: "backlog", - label: "Backlog", - description: "How many runs are waiting across the environment, over time.", - color: "var(--color-queues-chart)", - query: `SELECT timeBucket() AS t,\n max(max_env_queued) AS queued\nFROM env_metrics\nGROUP BY t\nORDER BY t`, - derive: (rows) => { - const points = rows.map((r) => ({ - bucket: tileTimeToMs(r.t), - value: tileNumber(r.queued), - })); - return { points, total: peakOf(points), formatTotal: (v) => `${v.toLocaleString()} peak` }; - }, - }, - { - id: "p95", - label: "Scheduling delay p95", - description: ( - <> - How long runs wait before they start (95% start faster than this). Turns {" "} - above 1 minute. - - ), - totalTooltip: "The worst p95 in the selected window.", - color: "var(--color-queues-chart)", - legend: [ - { color: "var(--color-queues-chart)", label: "p95" }, - { color: "var(--color-warning)", label: "Over 1 min" }, - ], - query: SCHEDULING_DELAY_QUERY, - formatValue: formatWaitMs, - formatAxis: formatWaitMs, - derive: (rows) => { - const points = rows.map((r) => ({ - bucket: tileTimeToMs(r.t), - value: tileNumber(r.samples) > 0 ? tileNumber(r.p95) : null, - })); - return { points, total: peakOf(points) }; - }, - readout: { - query: SCHEDULING_DELAY_QUERY, - /** - * Merging quantile states over a wider bucket yields a p95 between the sub-buckets' own, so - * the worst p95 has to be read at the range's natural width or a burst of slow starts shorter - * than the plotted bucket is averaged away. Unlike the gauges, whose max of maxes is the same - * at any width. - */ - derive: (rows) => { - const worst = rows.reduce( - (max, r) => (tileNumber(r.samples) > 0 ? Math.max(max, tileNumber(r.p95)) : max), - 0 - ); - return { - total: worst, - formatTotal: (v) => (v > 0 ? formatWaitMs(v) : "–"), - totalClassName: worst >= 60_000 ? "text-warning" : undefined, - }; - }, - }, - }, - { - id: "throttled", - label: "Throttled", - description: "How often runs were held back by a limit.", - totalTooltip: "The share of the selected window with at least one blocked dequeue.", - color: "var(--color-queues-chart)", - legend: [{ color: "var(--color-warning)", label: "Throttled" }], - query: THROTTLED_QUERY, - derive: (rows) => { - const points = rows.map((r) => ({ - bucket: tileTimeToMs(r.t), - value: tileNumber(r.throttled), - })); - return { points, total: peakOf(points) }; - }, - readout: { - query: THROTTLED_QUERY, - /** - * Share of the window that saw any throttling. A raw event sum isn't interpretable (it - * scales with poll rate and window length); the fraction of buckets with a throttle is. - * Gap fill zero-fills this counter, so every bucket in the window is present and the row - * count is the honest denominator. - */ - derive: (rows) => { - const nonzero = rows.filter((r) => tileNumber(r.throttled) > 0).length; - const pct = rows.length > 0 ? Math.round((nonzero / rows.length) * 100) : 0; - return { - total: pct, - formatTotal: (v) => `${v}% of current period`, - totalClassName: pct > 0 ? "text-warning" : undefined, - }; - }, - }, - }, -]; - -/** - * Bucket floor shared by every hero tile. Scheduling delay and throttling are event-driven, so at - * the 10-second width a short range would otherwise pick, most buckets hold no samples at all. One - * floor for all four keeps their x-axes identical, which the shared hover crosshair relies on. - */ -const HERO_CHART_MIN_BUCKET_SECONDS = 60; - -type TileTimeRange = MetricResourceTimeRange; - -// Full-size env metric chart rendered inside a ChartCard. Same data path as before -// (client-side TRQL via useMetricResourceQuery with fillGaps), drawn as a line that -// participates in the shared hover + drag-to-zoom of the enclosing ChartSyncProvider. -function QueueEnvMetricChart({ - tile, - timeRange, - referenceLines, - thresholdStroke, - warningOverlay, - solidWarning = false, -}: { - tile: QueueHeaderTile; - timeRange: TileTimeRange; - referenceLines?: Array<{ - y: number; - label?: string; - labelPlacement?: "inside" | "outside"; - }>; - thresholdStroke?: { value: number; aboveColor: string }; - warningOverlay?: { threshold: number }; - /** When set, the ENTIRE line turns warning-coloured if the series is ever non-zero (used for - * throttling: any throttle in the window colours the whole line). Mutually exclusive with the - * per-bucket warningOverlay. */ - solidWarning?: boolean; -}) { - const organization = useOrganization(); - const project = useProject(); - const environment = useEnvironment(); - - const sharedOptions = { - organizationId: organization.id, - projectId: project.id, - environmentId: environment.id, - timeRange, - defaultPeriod: QUEUE_METRICS_DEFAULT_PERIOD, - fillGaps: true, - }; - - const { rows, showLoading, failed } = useMetricResourceQuery(tile.query, { - ...sharedOptions, - minBucketSeconds: HERO_CHART_MIN_BUCKET_SECONDS, - }); - - const derived = tile.derive(rows); - const points = derived.points; - - const plottedBucketMs = points.length > 1 ? points[1]!.bucket - points[0]!.bucket : 0; - const floorWidenedBuckets = - plottedBucketMs > 0 && plottedBucketMs <= HERO_CHART_MIN_BUCKET_SECONDS * 1000; - const readoutQuery = tile.readout && floorWidenedBuckets ? tile.readout.query : ""; - const readoutResult = useMetricResourceQuery(readoutQuery, sharedOptions); - - const { total, formatTotal, totalClassName } = tile.readout - ? tile.readout.derive(readoutQuery ? readoutResult.rows : rows) - : derived; - - // Same point shape the shared axis/tooltip helpers expect. - const data = points - .map((p) => ({ bucket: p.bucket, [tile.id]: p.value })) - .filter((p) => Number.isFinite(p.bucket)); - - // Whole-line warning colour when the series was ever non-zero (throttling: one throttle in the - // window colours the entire line). Otherwise the tile's normal colour. - const wholeLineWarning = solidWarning && total > 0; - const lineColor = wholeLineWarning ? "var(--color-warning)" : tile.color; - - const chartConfig = useMemo( - () => ({ [tile.id]: { label: tile.label, color: lineColor } }), - [tile.id, tile.label, lineColor] - ); - - const { tickFormatter, tooltipLabelFormatter } = buildActivityTimeAxis(data); - const hasData = data.length > 0 && data.some((p) => Number(p[tile.id] ?? 0) > 0); - - // Peak readout lives in the card title (ChartCard has no dedicated value slot). A zero/empty - // total renders no readout at all (skipping "0% peak", "0 peak", "0" and the p95 "–" placeholder) - // so the card title stands alone until there's a non-zero value to show. - const readoutLoading = tile.readout ? readoutResult.showLoading : showLoading; - const readoutFailed = tile.readout ? readoutResult.failed : failed; - const peak = readoutLoading ? ( - - ) : readoutFailed || total === 0 ? null : formatTotal ? ( - formatTotal(total) - ) : ( - total.toLocaleString() - ); - - return ( - - - - {tile.label} - - - {peak != null ? ( - tile.totalTooltip && !readoutLoading ? ( - - {peak} - - } - content={tile.totalTooltip} - className="max-w-[230px]" - disableHoverableContent - /> - ) : ( - - {peak} - - ) - ) : null} - - {tile.legend && (showLoading || hasData) ? ( - - {tile.legend.map((item) => ( - - - {item.label} - - ))} - - ) : null} - - } - > - {showLoading ? ( - - ) : failed ? ( -
- Unable to load metrics -
- ) : hasData ? ( - - - - ) : ( -
- No activity -
- )} - - ); -} - -function QueueMetricChartSkeleton() { - return ( -
- {Array.from({ length: 42 }).map((_, i) => ( -
- ))} -
- ); -} - -/** Health as a stock Badge: color carries the state, w-fit keeps it content-width. */ -type QueueHealth = { - paused: boolean; - running: number; - queued: number; - limit: number; -}; - -type QueueHealthLabel = "Paused" | "At capacity" | "Backlogged" | "Active" | "Idle"; - -// Single source of truth for the queue health decision, shared by the badge and the table's -// health-column sort so the sorted order always matches the labels shown. -function queueHealthLabel({ paused, running, queued, limit }: QueueHealth): QueueHealthLabel { - if (paused) return "Paused"; - if (isQueueAtCapacity({ running, queued, limit })) return "At capacity"; - if (queued > 0) return "Backlogged"; - if (running > 0) return "Active"; - return "Idle"; -} - -// Tint + colored text, sized like the error status chips (see ErrorStatusBadge). -const QUEUE_HEALTH_STYLES: Record = { - Paused: "bg-warning/10 text-warning system:bg-warning system:text-white", - "At capacity": "bg-warning/10 text-warning system:bg-warning system:text-white", - Backlogged: "bg-blue-500/10 text-blue-500 system:bg-blue-500 system:text-white", - Active: "bg-success/10 text-success system:bg-success system:text-white", - Idle: "bg-charcoal-500/10 text-text-dimmed system:bg-charcoal-500 system:text-white", -}; - -function QueueHealthBadge(health: QueueHealth) { - const label = queueHealthLabel(health); - return ( - - {label} - - ); -} - -// The `queue_metrics`-prefixed key a queue is stored under (task queues are prefixed `task/`). -function queueMetricsKey(queue: { type: string; name: string }): string { - return `${queue.type === "task" ? "task/" : ""}${queue.name}`; -} - -function formatWaitMs(ms: number): string { - if (ms < 1000) return `${Math.round(ms)}ms`; - if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s`; - if (ms < 3_600_000) return `${(ms / 60_000).toFixed(1)}m`; - return `${(ms / 3_600_000).toFixed(1)}h`; -} - -// Drop a trailing ".00" from whole percentages so "50.00" reads as "50" but "12.50" is preserved. -function formatOverridePercent(percent: number): string { - return Number.isInteger(percent) ? percent.toString() : percent.toFixed(2).replace(/\.?0+$/, ""); -} - -// Classic Queues page, restored verbatim from before the Queue Metrics feature. Rendered -// when queueMetricsUiEnabled is off so a gated org sees exactly the pre-metrics UI. -function ClassicQueuesView() { - const { environment, queues, pagination, hasFilters, autoReloadPollIntervalMs } = - useTypedLoaderData(); - - const organization = useOrganization(); - const project = useProject(); - const env = useEnvironment(); - const plan = useCurrentPlan(); - - useAutoRevalidate({ interval: autoReloadPollIntervalMs, onFocus: true }); - - const { limitStatus, limitClassName } = getEnvConcurrencyLimitStatus(environment); - - return ( - - - - - - - - Queues docs - - - - - -
-
- paused : undefined} - animate - accessory={ -
- {environment.runsEnabled && - env.pauseSource !== ENVIRONMENT_PAUSE_SOURCE_BILLING_LIMIT ? ( - - ) : null} - -
- } - valueClassName={env.paused ? "text-warning tabular-nums" : "tabular-nums"} - compactThreshold={1000000} - /> - - Including {environment.running - environment.concurrencyLimit} burst runs{" "} - - - ) : limitStatus === "limit" ? ( - "At concurrency limit" - ) : undefined - } - accessory={ - - } - compactThreshold={1000000} - /> - 1 ? ( - - Burst limit {environment.burstFactor * environment.concurrencyLimit}{" "} - - - ) : undefined - } - accessory={ - plan ? ( - plan?.v3Subscription?.plan?.limits.concurrentRuns.canExceed ? ( - - Increase limit - - ) : ( - - Increase limit - - ) - ) : null - } - /> -
- -
-
- - -
- - - - Name - Queued - Running - Limit - -
- Environment - - This queue is limited by your environment's concurrency limit of{" "} - {environment.concurrencyLimit}. - -
-
- User - - This queue is limited by a concurrency limit set in your code. - -
-
- Override - - This queue's concurrency limit has been manually overridden from the - dashboard or API. - -
- - } - > - Limited by -
- - Pause/resume - -
-
- - {queues.length > 0 ? ( - queues.map((queue) => { - const limit = queue.concurrencyLimit ?? environment.concurrencyLimit; - const isAtConcurrencyLimit = queue.running >= limit; - const isAtQueueLimit = - environment.queueSizeLimit !== null && - queue.queued >= environment.queueSizeLimit; - const queueFilterableName = `${queue.type === "task" ? "task/" : ""}${ - queue.name - }`; - return ( - - - - - {queue.concurrency?.overriddenAt ? ( - - Concurrency limit overridden - - } - content="This queue's concurrency limit has been manually overridden from the dashboard or API." - className="max-w-[230px]" - disableHoverableContent - /> - ) : null} - {queue.paused ? ( - - Paused - - ) : null} - {isAtQueueLimit ? ( - - At queue limit - - ) : null} - {isAtConcurrencyLimit ? ( - - At concurrency limit - - ) : null} - - - - {queue.queued} - - 0 && "text-text-bright", - isAtConcurrencyLimit && "text-warning" - )} - > - {queue.running} - - - {limit} - - - {queue.concurrency?.overriddenAt ? ( - Override - ) : queue.concurrencyLimit ? ( - "User" - ) : ( - "Environment" - )} - - } - hiddenButtons={!queue.paused && } - popoverContent={ - <> - {queue.paused ? ( - - ) : ( - - )} - - - - - - - } - /> - - ); - }) - ) : ( - - -
- - {hasFilters ? "No queues found matching your filters" : "No queues found"} - -
-
-
- )} -
-
-
-
-
-
- ); -} - -function BurstFactorTooltip({ - environment, -}: { - environment: { burstFactor: number; concurrencyLimit: number }; -}) { - return ( - - ); -} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx index 75822975307..ad292e51526 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx @@ -97,7 +97,7 @@ import { v3EditSchedulePath, v3EnvironmentPath, v3NewSchedulePath, - v3QueuePath, + concurrencyQueuePath, v3RunsPath, v3SchedulePath, v3SchedulesAddOnPath, @@ -258,7 +258,9 @@ export default function Page() { taskIdentifier: task.slug, }); const queuePath = task.queue - ? v3QueuePath(organization, project, environment, { friendlyId: task.queue.friendlyId }) + ? concurrencyQueuePath(organization, project, environment, { + friendlyId: task.queue.friendlyId, + }) : undefined; const filters: TaskRunListSearchFilters = useMemo(() => ({ tasks: [task.slug] }), [task.slug]); diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx index 56f553547b1..6615a2fc8d6 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx @@ -63,8 +63,8 @@ import { requireUser } from "~/services/session.server"; import { EnvironmentParamSchema, v3EnvironmentPath, - v3QueuePath, - v3QueuesPath, + concurrencyQueuePath, + concurrencyPath, v3TestTaskPath, } from "~/utils/pathBuilder"; import { parseFiniteInt } from "~/utils/searchParams"; @@ -198,9 +198,11 @@ export default function Page() { const testPath = v3TestTaskPath(organization, project, environment, { taskIdentifier: task.slug, }); - const queuesPath = v3QueuesPath(organization, project, environment); + const queuesPath = concurrencyPath(organization, project, environment); const queuePath = task.queue - ? v3QueuePath(organization, project, environment, { friendlyId: task.queue.friendlyId }) + ? concurrencyQueuePath(organization, project, environment, { + friendlyId: task.queue.friendlyId, + }) : undefined; const { value } = useSearchParams(); @@ -527,7 +529,7 @@ function TaskActivityCard({ > {view === "queue" ? ( ({ type: "queues" }), + }, + corsStrategy: "all", + }, + async ({ params, body, authentication }) => { + return concurrencyLimitsSystem.limits + .override(authentication.environment, params.name, body) + .match( + (limit) => json(limit), + (error) => { + switch (error.type) { + case "limit_not_found": + return json({ error: "Concurrency limit not found" }, { status: 404 }); + case "conflict": + return json( + { error: "The limit changed concurrently; retry the request" }, + { status: 409 } + ); + case "invalid_override": + return json({ error: error.message }, { status: 400 }); + default: + return json({ error: "Failed to override concurrency limit" }, { status: 500 }); + } + } + ); + } +); + +export const action = route.action; +export const loader = route.loader; diff --git a/apps/webapp/app/routes/api.v1.concurrency-limits.$name.reset.ts b/apps/webapp/app/routes/api.v1.concurrency-limits.$name.reset.ts new file mode 100644 index 00000000000..5e0af9192b1 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.concurrency-limits.$name.reset.ts @@ -0,0 +1,42 @@ +import { json } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +import { concurrencyLimitsSystem } from "~/v3/services/concurrencyLimitsSystemInstance.server"; + +const ParamsSchema = z.object({ + name: z.string(), +}); + +const route = createActionApiRoute( + { + params: ParamsSchema, + authorization: { + action: "write", + resource: () => ({ type: "queues" }), + }, + corsStrategy: "all", + }, + async ({ params, authentication }) => { + return concurrencyLimitsSystem.limits.reset(authentication.environment, params.name).match( + (limit) => json(limit), + (error) => { + switch (error.type) { + case "limit_not_found": + return json({ error: "Concurrency limit not found" }, { status: 404 }); + case "conflict": + return json( + { error: "The limit changed concurrently; retry the request" }, + { status: 409 } + ); + case "limit_not_overridden": + return json({ error: "Concurrency limit has no override to reset" }, { status: 400 }); + default: + return json({ error: "Failed to reset concurrency limit" }, { status: 500 }); + } + } + ); + } +); + +export const action = route.action; +export const loader = route.loader; diff --git a/apps/webapp/app/routes/api.v1.concurrency-limits.$name.ts b/apps/webapp/app/routes/api.v1.concurrency-limits.$name.ts new file mode 100644 index 00000000000..3035de12baf --- /dev/null +++ b/apps/webapp/app/routes/api.v1.concurrency-limits.$name.ts @@ -0,0 +1,31 @@ +import { json } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +import { concurrencyLimitsSystem } from "~/v3/services/concurrencyLimitsSystemInstance.server"; + +const ParamsSchema = z.object({ + name: z.string(), +}); + +export const loader = createLoaderApiRoute( + { + params: ParamsSchema, + findResource: async () => 1, + authorization: { + action: "read", + resource: () => ({ type: "queues" }), + }, + corsStrategy: "all", + }, + async ({ params, authentication }) => { + return concurrencyLimitsSystem.limits.retrieve(authentication.environment, params.name).match( + (limit) => json(limit), + (error) => { + if (error.type === "limit_not_found") { + return json({ error: "Concurrency limit not found" }, { status: 404 }); + } + return json({ error: "Failed to retrieve concurrency limit" }, { status: 500 }); + } + ); + } +); diff --git a/apps/webapp/app/routes/api.v1.concurrency-limits.ts b/apps/webapp/app/routes/api.v1.concurrency-limits.ts new file mode 100644 index 00000000000..0b9c936019d --- /dev/null +++ b/apps/webapp/app/routes/api.v1.concurrency-limits.ts @@ -0,0 +1,43 @@ +import { json } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +import { concurrencyLimitsSystem } from "~/v3/services/concurrencyLimitsSystemInstance.server"; + +const SearchParamsSchema = z.object({ + page: z.coerce.number().int().positive().default(1), + perPage: z.coerce.number().int().positive().max(100).default(25), +}); + +export const loader = createLoaderApiRoute( + { + searchParams: SearchParamsSchema, + findResource: async () => 1, + authorization: { + action: "read", + resource: () => ({ type: "queues" }), + }, + corsStrategy: "all", + }, + async ({ searchParams, authentication }) => { + const [data, count] = await Promise.all([ + concurrencyLimitsSystem.limits.list(authentication.environment, { + page: searchParams.page, + perPage: searchParams.perPage, + }), + concurrencyLimitsSystem.limits.totalCount(authentication.environment), + ]); + + if (data.isErr() || count.isErr()) { + return json({ error: "Failed to list concurrency limits" }, { status: 500 }); + } + + return json({ + data: data.value, + pagination: { + currentPage: searchParams.page, + totalPages: Math.ceil(count.value / searchParams.perPage), + count: count.value, + }, + }); + } +); diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.override.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.override.ts deleted file mode 100644 index c643b77965a..00000000000 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.override.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { json } from "@remix-run/server-runtime"; -import { type RetrieveQueueParam, RetrieveQueueType } from "@trigger.dev/core/v3"; -import { z } from "zod"; -import { toQueueItem } from "~/presenters/v3/QueueRetrievePresenter.server"; -import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; -import { concurrencySystem } from "~/v3/services/concurrencySystemInstance.server"; - -const BodySchema = z.object({ - type: RetrieveQueueType.default("id"), - concurrencyLimit: z.number().int().min(0).max(100000), -}); - -const route = createActionApiRoute( - { - body: BodySchema, - params: z.object({ - queueParam: z.string().transform((val) => val.replace(/%2F/g, "/")), - }), - authorization: { - action: "write", - resource: () => ({ type: "queues" }), - }, - }, - async ({ params, body, authentication }) => { - const input: RetrieveQueueParam = - body.type === "id" - ? params.queueParam - : { - type: body.type, - name: decodeURIComponent(params.queueParam).replace(/%2F/g, "/"), - }; - - return concurrencySystem.queues - .overrideTotalConcurrencyLimit(authentication.environment, input, body.concurrencyLimit) - .match( - (queue) => { - return json( - toQueueItem({ - friendlyId: queue.friendlyId, - name: queue.name, - type: queue.type, - running: queue.running, - queued: queue.queued, - concurrencyLimit: queue.concurrencyLimit, - concurrencyLimitBase: queue.concurrencyLimitBase, - concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt, - concurrencyLimitOverriddenBy: null, - paused: queue.paused, - }), - { status: 200 } - ); - }, - (error) => { - switch (error.type) { - case "queue_not_found": { - return json({ error: "Queue not found" }, { status: 404 }); - } - case "invalid_override": - case "concurrency_limit_exceeds_maximum": { - return json({ error: error.message }, { status: 400 }); - } - case "queue_update_failed": { - return json( - { error: "Failed to update queue total concurrency limit" }, - { status: 500 } - ); - } - case "sync_queue_concurrency_to_engine_failed": { - return json({ error: "Failed to sync the total concurrency limit" }, { status: 500 }); - } - case "get_queue_stats_failed": { - return json({ error: "Failed to read queue stats" }, { status: 500 }); - } - case "other": { - return json( - { error: "Failed to update queue total concurrency limit" }, - { - status: 500, - } - ); - } - default: { - return json( - { error: "Failed to update queue total concurrency limit" }, - { - status: 500, - } - ); - } - } - } - ); - } -); - -export const action = route.action; -/** The builder's loader answers non-POST methods with a 405. */ -export const loader = route.loader; diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.reset.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.reset.ts deleted file mode 100644 index b2841f1efe6..00000000000 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.reset.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { json } from "@remix-run/server-runtime"; -import { type RetrieveQueueParam, RetrieveQueueType } from "@trigger.dev/core/v3"; -import { z } from "zod"; -import { toQueueItem } from "~/presenters/v3/QueueRetrievePresenter.server"; -import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; -import { concurrencySystem } from "~/v3/services/concurrencySystemInstance.server"; - -const BodySchema = z.object({ - type: RetrieveQueueType.default("id"), -}); - -const route = createActionApiRoute( - { - body: BodySchema, - params: z.object({ - queueParam: z.string().transform((val) => val.replace(/%2F/g, "/")), - }), - authorization: { - action: "write", - resource: () => ({ type: "queues" }), - }, - }, - async ({ params, body, authentication }) => { - const input: RetrieveQueueParam = - body.type === "id" - ? params.queueParam - : { - type: body.type, - name: decodeURIComponent(params.queueParam).replace(/%2F/g, "/"), - }; - - return concurrencySystem.queues - .resetTotalConcurrencyLimit(authentication.environment, input) - .match( - (queue) => { - return json( - toQueueItem({ - friendlyId: queue.friendlyId, - name: queue.name, - type: queue.type, - running: queue.running, - queued: queue.queued, - concurrencyLimit: queue.concurrencyLimit, - concurrencyLimitBase: queue.concurrencyLimitBase, - concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt, - concurrencyLimitOverriddenBy: null, - paused: queue.paused, - }), - { status: 200 } - ); - }, - (error) => { - switch (error.type) { - case "queue_not_found": { - return json({ error: "Queue not found" }, { status: 404 }); - } - case "queue_not_overridden": { - return json( - { error: "The queue total concurrency limit is not overridden" }, - { status: 400 } - ); - } - case "queue_update_failed": { - return json( - { error: "Failed to reset the queue total concurrency limit" }, - { status: 500 } - ); - } - case "sync_queue_concurrency_to_engine_failed": { - return json({ error: "Failed to sync the total concurrency limit" }, { status: 500 }); - } - case "get_queue_stats_failed": { - return json({ error: "Failed to read queue stats" }, { status: 500 }); - } - case "other": { - return json( - { error: "Failed to reset the queue total concurrency limit" }, - { - status: 500, - } - ); - } - default: { - return json( - { error: "Failed to reset the queue total concurrency limit" }, - { - status: 500, - } - ); - } - } - } - ); - } -); - -export const action = route.action; -/** The builder's loader answers non-POST methods with a 405. */ -export const loader = route.loader; diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts index 90f5772c5d3..f5537876887 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts @@ -1,7 +1,7 @@ import { json } from "@remix-run/server-runtime"; import { type RetrieveQueueParam, RetrieveQueueType } from "@trigger.dev/core/v3"; import { z } from "zod"; -import { toQueueItem } from "~/presenters/v3/QueueRetrievePresenter.server"; +import { toPublicQueueItem, toQueueItem } from "~/presenters/v3/QueueRetrievePresenter.server"; import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; import { concurrencySystem } from "~/v3/services/concurrencySystemInstance.server"; import { @@ -46,27 +46,41 @@ const route = createActionApiRoute( body.percent !== undefined ? { percent: body.percent } : { limit: body.concurrencyLimit! }; return concurrencySystem.queues - .overrideQueueConcurrencyLimit(authentication.environment, input, override) + .overrideQueueConcurrencyLimit(authentication.environment, input, override, undefined, { + v1Only: true, + }) .match( (queue) => { return json( - toQueueItem({ - friendlyId: queue.friendlyId, - name: queue.name, - type: queue.type, - running: queue.running, - queued: queue.queued, - concurrencyLimit: queue.concurrencyLimit, - concurrencyLimitBase: queue.concurrencyLimitBase, - concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt, - concurrencyLimitOverriddenBy: null, - paused: queue.paused, - }), + toPublicQueueItem( + toQueueItem({ + friendlyId: queue.friendlyId, + name: queue.name, + type: queue.type, + version: queue.concurrencyVersion, + running: queue.running, + queued: queue.queued, + concurrencyLimit: queue.concurrencyLimit, + concurrencyLimitBase: queue.concurrencyLimitBase, + concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt, + concurrencyLimitOverriddenBy: null, + paused: queue.paused, + }) + ), { status: 200 } ); }, (error) => { switch (error.type) { + case "queue_version_unsupported": { + return json( + { + error: + "This queue's concurrency is declared with the task `concurrency` option; manage it through the concurrency-limits endpoints instead (a task's inline limit lives under its derived `task/` name)", + }, + { status: 400 } + ); + } case "queue_not_found": { return json({ error: "Queue not found" }, { status: 404 }); } diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.reset.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.reset.ts index 503d875e471..34eb04973f8 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.reset.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.reset.ts @@ -1,7 +1,7 @@ import { json } from "@remix-run/server-runtime"; import { type RetrieveQueueParam, RetrieveQueueType } from "@trigger.dev/core/v3"; import { z } from "zod"; -import { toQueueItem } from "~/presenters/v3/QueueRetrievePresenter.server"; +import { toPublicQueueItem, toQueueItem } from "~/presenters/v3/QueueRetrievePresenter.server"; import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; import { concurrencySystem } from "~/v3/services/concurrencySystemInstance.server"; @@ -29,52 +29,66 @@ const route = createActionApiRoute( name: decodeURIComponent(params.queueParam).replace(/%2F/g, "/"), }; - return concurrencySystem.queues.resetConcurrencyLimit(authentication.environment, input).match( - (queue) => { - return json( - toQueueItem({ - friendlyId: queue.friendlyId, - name: queue.name, - type: queue.type, - running: queue.running, - queued: queue.queued, - concurrencyLimit: queue.concurrencyLimit, - concurrencyLimitBase: queue.concurrencyLimitBase, - concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt, - concurrencyLimitOverriddenBy: null, - paused: queue.paused, - }), - { status: 200 } - ); - }, - (error) => { - switch (error.type) { - case "queue_not_found": { - return json({ error: "Queue not found" }, { status: 404 }); - } - case "queue_not_overridden": { - return json({ error: "Queue is not overridden" }, { status: 400 }); - } - case "queue_update_failed": { - return json({ error: "Failed to update queue concurrency limit" }, { status: 500 }); - } - case "sync_queue_concurrency_to_engine_failed": { - return json( - { error: "Failed to sync queue concurrency limit to engine" }, - { status: 500 } - ); - } - case "get_queue_stats_failed": { - return json({ error: "Failed to get queue stats" }, { status: 500 }); - } - case "other": - default: { - error.type satisfies "other"; - return json({ error: "Internal server error" }, { status: 500 }); + return concurrencySystem.queues + .resetConcurrencyLimit(authentication.environment, input, { v1Only: true }) + .match( + (queue) => { + return json( + toPublicQueueItem( + toQueueItem({ + friendlyId: queue.friendlyId, + name: queue.name, + type: queue.type, + version: queue.concurrencyVersion, + running: queue.running, + queued: queue.queued, + concurrencyLimit: queue.concurrencyLimit, + concurrencyLimitBase: queue.concurrencyLimitBase, + concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt, + concurrencyLimitOverriddenBy: null, + paused: queue.paused, + }) + ), + { status: 200 } + ); + }, + (error) => { + switch (error.type) { + case "queue_version_unsupported": { + return json( + { + error: + "This queue's concurrency is declared with the task `concurrency` option; manage it through the concurrency-limits endpoints instead (a task's inline limit lives under its derived `task/` name)", + }, + { status: 400 } + ); + } + case "queue_not_found": { + return json({ error: "Queue not found" }, { status: 404 }); + } + case "queue_not_overridden": { + return json({ error: "Queue is not overridden" }, { status: 400 }); + } + case "queue_update_failed": { + return json({ error: "Failed to update queue concurrency limit" }, { status: 500 }); + } + case "sync_queue_concurrency_to_engine_failed": { + return json( + { error: "Failed to sync queue concurrency limit to engine" }, + { status: 500 } + ); + } + case "get_queue_stats_failed": { + return json({ error: "Failed to get queue stats" }, { status: 500 }); + } + case "other": + default: { + error.type satisfies "other"; + return json({ error: "Internal server error" }, { status: 500 }); + } } } - } - ); + ); } ); diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.metrics.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.metrics.ts index f4d1e8cf8ae..838b2b904d2 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.metrics.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.metrics.ts @@ -51,7 +51,7 @@ export const loader = createLoaderApiRoute( findResource: async () => 1, // dummy — the queue name isn't resolved against Postgres authorization: { action: "read", - resource: () => ({ type: "query", id: "queue_metrics" }), + resource: () => ({ type: "query", id: "concurrency_metrics" }), }, }, async ({ params, searchParams, authentication }) => { diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.pause.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.pause.ts index dd5f43da5d1..a51777fefb2 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.pause.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.pause.ts @@ -1,5 +1,6 @@ import { json } from "@remix-run/server-runtime"; -import { type QueueItem, type RetrieveQueueParam, RetrieveQueueType } from "@trigger.dev/core/v3"; +import { type RetrieveQueueParam, RetrieveQueueType } from "@trigger.dev/core/v3"; +import { toPublicQueueItem } from "~/presenters/v3/QueueRetrievePresenter.server"; import { z } from "zod"; import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; import { PauseQueueService } from "~/v3/services/pauseQueue.server"; @@ -44,8 +45,7 @@ const route = createActionApiRoute( return json({ error: result.code }, { status: 400 }); } - const q: QueueItem = result.queue; - return json(q); + return json(toPublicQueueItem(result.queue)); } ); diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.ts index 434046d961d..82503142903 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.ts @@ -1,7 +1,10 @@ import { json } from "@remix-run/server-runtime"; -import { type QueueItem, type RetrieveQueueParam, RetrieveQueueType } from "@trigger.dev/core/v3"; +import { type RetrieveQueueParam, RetrieveQueueType } from "@trigger.dev/core/v3"; import { z } from "zod"; -import { QueueRetrievePresenter } from "~/presenters/v3/QueueRetrievePresenter.server"; +import { + QueueRetrievePresenter, + toPublicQueueItem, +} from "~/presenters/v3/QueueRetrievePresenter.server"; import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; import { determineEngineVersion } from "~/v3/engineVersion.server"; @@ -52,7 +55,6 @@ export const loader = createLoaderApiRoute( return json({ error: result.code }, { status: 404 }); } - const q: QueueItem = result.queue; - return json(q); + return json(toPublicQueueItem(result.queue)); } ); diff --git a/apps/webapp/app/routes/api.v1.queues.ts b/apps/webapp/app/routes/api.v1.queues.ts index 463aacc38ba..51cf90c72ba 100644 --- a/apps/webapp/app/routes/api.v1.queues.ts +++ b/apps/webapp/app/routes/api.v1.queues.ts @@ -1,10 +1,10 @@ import { json } from "@remix-run/server-runtime"; -import { type QueueItem } from "@trigger.dev/core/v3"; import { z } from "zod"; import { QUEUE_LIST_DEFAULT_ITEMS_PER_PAGE, QueueListPresenter, } from "~/presenters/v3/QueueListPresenter.server"; +import { toPublicQueueItem } from "~/presenters/v3/QueueRetrievePresenter.server"; import { toOffsetLimitQueueListPagination } from "~/presenters/v3/queueListPagination.server"; import { logger } from "~/services/logger.server"; import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; @@ -45,7 +45,7 @@ export const loader = createLoaderApiRoute( page: searchParams.page ?? 1, }); - const queues: QueueItem[] = result.queues; + const queues = result.queues.map(toPublicQueueItem); return json( { data: queues, diff --git a/apps/webapp/app/routes/orgs.$organizationSlug.projects.$projectParam.concurrency.ts b/apps/webapp/app/routes/orgs.$organizationSlug.projects.$projectParam.concurrency.ts index 4bde0ccaef3..9459382e62b 100644 --- a/apps/webapp/app/routes/orgs.$organizationSlug.projects.$projectParam.concurrency.ts +++ b/apps/webapp/app/routes/orgs.$organizationSlug.projects.$projectParam.concurrency.ts @@ -2,7 +2,7 @@ import { redirect, type LoaderFunctionArgs } from "@remix-run/server-runtime"; import { prisma } from "~/db.server"; import { SelectBestEnvironmentPresenter } from "~/presenters/SelectBestEnvironmentPresenter.server"; import { requireUser } from "~/services/session.server"; -import { ProjectParamSchema, v3QueuesPath } from "~/utils/pathBuilder"; +import { ProjectParamSchema, concurrencyPath } from "~/utils/pathBuilder"; export const loader = async ({ request, params }: LoaderFunctionArgs) => { const user = await requireUser(request); @@ -41,5 +41,5 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const selector = new SelectBestEnvironmentPresenter(); const environment = await selector.selectBestEnvironment(project.id, user, project.environments); - return redirect(v3QueuesPath({ slug: organizationSlug }, project, environment)); + return redirect(concurrencyPath({ slug: organizationSlug }, project, environment)); }; diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx index 02571272fec..0558d7445a4 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx @@ -116,7 +116,7 @@ import { docsPath, v3BatchPath, v3DeploymentVersionPath, - v3QueuePath, + concurrencyQueuePath, v3RunDownloadLogsPath, v3RunIdempotencyKeyResetPath, v3RunPath, @@ -424,7 +424,7 @@ function RunBody({ const resetFetcher = useTypedFetcher(); const queuePath = queueMetrics?.queueFriendlyId - ? v3QueuePath(organization, project, environment, { + ? concurrencyQueuePath(organization, project, environment, { friendlyId: queueMetrics.queueFriendlyId, }) : undefined; @@ -1342,7 +1342,7 @@ function WaitingInQueueBlock({ responseReceivedAt, lastSuccessfulResponseAt, } = useQueueMetric( - `SELECT timeBucket() AS t, max(max_running) AS running, max(max_queued) AS queued, max(max_limit) AS q_limit\nFROM queue_metrics\nGROUP BY t\nORDER BY t`, + `SELECT timeBucket() AS t, max(max_running) AS running, max(max_queued) AS queued, max(max_limit) AS q_limit\nFROM concurrency_metrics\nGROUP BY t\nORDER BY t`, { ids: waiting.ids, timeRange: { period: "15m", from: null, to: null }, @@ -1441,7 +1441,7 @@ function WaitingInQueueBlock({
entry.startsWith(ENV_ROUTE_PREFIX) && isRouteModule(entry)) .map((entry) => @@ -281,9 +281,9 @@ describe("the route Remix compiles from the filename", () => { expect(compiledUrl("routes/login.magic")).toBe("/login/magic"); expect( compiledUrl( - "routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam" + "routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.concurrency_.$queueParam" ) - ).toBe("/orgs/:organizationSlug/projects/:projectParam/env/:envParam/queues/:queueParam"); + ).toBe("/orgs/:organizationSlug/projects/:projectParam/env/:envParam/concurrency/:queueParam"); }); }); diff --git a/apps/webapp/app/utils/deeplinkPages.ts b/apps/webapp/app/utils/deeplinkPages.ts index 1ba60b354f9..bea7bbc919c 100644 --- a/apps/webapp/app/utils/deeplinkPages.ts +++ b/apps/webapp/app/utils/deeplinkPages.ts @@ -15,6 +15,7 @@ export const ENV_PAGE_TARGETS: ReadonlyMap = new Map([ ["branches", page("branches")], ["bulk-actions", page("bulk-actions")], ["concurrency", page("concurrency")], + ["concurrency-limits", page("concurrency-limits")], ["dashboards", page("dashboards")], ["deployments", page("deployments")], ["dev-branches", page("dev-branches")], @@ -26,7 +27,7 @@ export const ENV_PAGE_TARGETS: ReadonlyMap = new Map([ ["playground", page("playground")], ["prompts", page("prompts")], ["query", page("query")], - ["queues", page("queues")], + ["queues", page("concurrency")], ["regions", page("regions")], ["runs", page("runs")], ["schedules", page("schedules")], diff --git a/apps/webapp/app/utils/pageSwitching.test.ts b/apps/webapp/app/utils/pageSwitching.test.ts index 5b8f8e69881..bdefb6de9ba 100644 --- a/apps/webapp/app/utils/pageSwitching.test.ts +++ b/apps/webapp/app/utils/pageSwitching.test.ts @@ -393,7 +393,7 @@ describe("pages named after a resource", () => { it("truncate to the list they were reached from", () => { expect(projectPortablePage("runs/run_123")).toBe("runs"); expect(projectPortablePage("batches/batch_123")).toBe("batches"); - expect(projectPortablePage("queues/my-queue")).toBe("queues"); + expect(projectPortablePage("concurrency/my-queue")).toBe("concurrency"); expect(projectPortablePage("schedules/sched_123")).toBe("schedules"); expect(projectPortablePage("schedules/edit/sched_123")).toBe("schedules"); expect(projectPortablePage("deployments/deploy_123")).toBe("deployments"); @@ -670,11 +670,11 @@ describe("pathForEnvironmentSwitch", () => { expect( pathForEnvironmentSwitch({ - location: locationOn("queues/my-queue", "?page=2"), + location: locationOn("concurrency/my-queue", "?page=2"), environmentPathname: environmentLocation.pathname, environmentSlug: "prod", }) - ).toBe("/orgs/acme/projects/api/env/prod/queues"); + ).toBe("/orgs/acme/projects/api/env/prod/concurrency"); }); it("only swaps the environment slug when it cannot tell where the environment path ends", () => { diff --git a/apps/webapp/app/utils/pathBuilder.ts b/apps/webapp/app/utils/pathBuilder.ts index 95727d6c76a..a700b225997 100644 --- a/apps/webapp/app/utils/pathBuilder.ts +++ b/apps/webapp/app/utils/pathBuilder.ts @@ -587,21 +587,21 @@ export function v3SchedulesAddOnPath(organization: OrgForPath) { return `/resources/orgs/${organizationParam(organization)}/schedules-addon`; } -export function v3QueuesPath( +export function concurrencyPath( organization: OrgForPath, project: ProjectForPath, environment: EnvironmentForPath ) { - return `${v3EnvironmentPath(organization, project, environment)}/queues`; + return `${v3EnvironmentPath(organization, project, environment)}/concurrency`; } -export function v3QueuePath( +export function concurrencyQueuePath( organization: OrgForPath, project: ProjectForPath, environment: EnvironmentForPath, queue: { friendlyId: string } ) { - return `${v3QueuesPath(organization, project, environment)}/${queue.friendlyId}`; + return `${concurrencyPath(organization, project, environment)}/${queue.friendlyId}`; } export function v3WaitpointTokensPath( @@ -806,12 +806,12 @@ export function branchesDevPath( return `${v3EnvironmentPath(organization, project, environment)}/dev-branches`; } -export function concurrencyPath( +export function concurrencyLimitsPath( organization: OrgForPath, project: ProjectForPath, environment: EnvironmentForPath ) { - return `${v3EnvironmentPath(organization, project, environment)}/concurrency`; + return `${v3EnvironmentPath(organization, project, environment)}/concurrency-limits`; } export function limitsPath( diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts index 690bbaf5396..419785a6e1e 100644 --- a/apps/webapp/app/v3/querySchemas.ts +++ b/apps/webapp/app/v3/querySchemas.ts @@ -610,12 +610,12 @@ const metricsSchema: TableSchema = { }; /** - * Schema definition for the queue_metrics table (trigger_dev.queue_metrics_v1). + * Schema definition for the concurrency_metrics table (trigger_dev.queue_metrics_v1). * Pre-aggregated into 10-second buckets. Counter columns re-aggregate with sum(), * gauges with max(), and wait_quantiles with quantilesMerge() — never FINAL. */ const queueMetricsSchema: TableSchema = { - name: "queue_metrics", + name: "concurrency_metrics", clickhouseName: "trigger_dev.queue_metrics_v1", description: "Per-queue depth, concurrency, throttling, and scheduling-delay metrics", timeConstraint: "bucket_start", @@ -770,6 +770,22 @@ const queueMetricsSchema: TableSchema = { fillMode: "carry", }), }, + max_total_running: { + name: "max_total_running", + ...column("UInt32", { + description: + "Peak in-flight runs across ALL concurrency keys of the queue in the bucket (only emitted for keyed queues). Aggregate with max().", + fillMode: "carry", + }), + }, + max_total_limit: { + name: "max_total_limit", + ...column("UInt32", { + description: + "The queue's combined concurrency limit across all keys, as stored (0 = no cap; clamp against max_env_limit). Aggregate with max().", + fillMode: "carry", + }), + }, max_ck_backlogged: { name: "max_ck_backlogged", ...column("UInt32", { @@ -828,7 +844,7 @@ const queueMetricsSchema: TableSchema = { /** * Schema definition for the env_metrics table (trigger_dev.env_metrics_v1). - * Environment-level rollup of queue_metrics with the queue dimension dropped, so + * Environment-level rollup of concurrency_metrics with the queue dimension dropped, so * header tiles and saturation charts cost the same regardless of how many queues * the environment has. Keeps the full 10-second granularity: row count is * queue-independent, so even 30-day ranges stay small. @@ -1304,7 +1320,7 @@ const llmModelsSchema: TableSchema = { * only when that key had events, so key cardinality cannot inflate the table. */ const queueMetricsByKeySchema: TableSchema = { - name: "queue_metrics_by_key", + name: "concurrency_metrics_by_key", clickhouseName: "trigger_dev.queue_metrics_ck_v1", description: "Per-concurrency-key queue metrics: backlog, throughput, and wait by key", hidden: true, @@ -1406,6 +1422,14 @@ const queueMetricsByKeySchema: TableSchema = { fillMode: "carry", }), }, + max_limit: { + name: "max_limit", + ...column("UInt32", { + description: + "The queue concurrency limit that applied to this key in the bucket (1000000 = no explicit limit). Aggregate with max().", + fillMode: "carry", + }), + }, wait_ms_sum: { name: "wait_ms_sum", ...column("UInt64", { diff --git a/apps/webapp/app/v3/queueMetricsMapping.ts b/apps/webapp/app/v3/queueMetricsMapping.ts index 9433b361a88..d341f4a63cd 100644 --- a/apps/webapp/app/v3/queueMetricsMapping.ts +++ b/apps/webapp/app/v3/queueMetricsMapping.ts @@ -131,6 +131,8 @@ export function mapEntryToRows( throttled: num(f.thr), ck_backlogged: num(f.ckq), ck_max_wait_ms: num(f.ckw), + total_running: num(f.tcc), + total_limit: num(f.tlim), }, ]; } diff --git a/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts b/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts new file mode 100644 index 00000000000..b6a6b0b51dc --- /dev/null +++ b/apps/webapp/app/v3/services/concurrencyLimitsSystem.server.ts @@ -0,0 +1,558 @@ +import type { TaskQueue, User } from "@trigger.dev/database"; +import { errAsync, fromPromise, okAsync, type ResultAsync } from "neverthrow"; +import { Prisma, type PrismaClientOrTransaction } from "~/db.server"; +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { + removeQueueConcurrencyLimits, + removeQueueTotalConcurrencyLimits, + updateQueueConcurrencyLimits, + updateQueueTotalConcurrencyLimits, +} from "../runQueue.server"; +import { engine } from "../runEngine.server"; + +export type ConcurrencyLimitsSystemOptions = { + db: PrismaClientOrTransaction; + reader: PrismaClientOrTransaction; +}; + +/** Queue rows that back named concurrency limits live under this reserved prefix. */ +const LIMIT_QUEUE_PREFIX = "limit/"; + +const LIMIT_NAME_PATTERN = /^[a-zA-Z0-9_/-]{1,122}$/; + +type ConcurrencyLimitBoundValue = { + current: number | null; + base: number | null; + override: number | null; + overriddenAt: Date | null; +}; + +type ConcurrencyLimitItem = { + id: string; + name: string; + perKey: ConcurrencyLimitBoundValue; + total: ConcurrencyLimitBoundValue; + running: number; + queued: number; +}; + +/** + * An override changes only the given bounds; each bound is a non-negative integer + * (zero blocks every run holding the limit, which is how a limit is paused). + */ +type ConcurrencyLimitOverrideInput = { + perKey?: number; + total?: number; +}; + +export class ConcurrencyLimitsSystem { + constructor(private readonly options: ConcurrencyLimitsSystemOptions) {} + + private get db() { + return this.options.db; + } + + private get reader() { + return this.options.reader; + } + + get limits() { + return { + list: (environment: AuthenticatedEnvironment, page: { page: number; perPage: number }) => { + return fromPromise( + this.reader.taskQueue.findMany({ + where: limitRowsWhere(environment), + orderBy: { name: "asc" }, + skip: (page.page - 1) * page.perPage, + take: page.perPage, + }), + (error) => ({ type: "other" as const, cause: error }) + ).andThen((rows) => + fromPromise(toLimitItems(environment, rows), (error) => ({ + type: "other" as const, + cause: error, + })) + ); + }, + totalCount: (environment: AuthenticatedEnvironment) => { + return fromPromise( + this.reader.taskQueue.count({ where: limitRowsWhere(environment) }), + (error) => ({ type: "other" as const, cause: error }) + ); + }, + retrieve: (environment: AuthenticatedEnvironment, name: string) => { + return findLimitByName(this.db, environment, name).andThen((row) => + fromPromise(toLimitItems(environment, [row]), (error) => ({ + type: "other" as const, + cause: error, + })).map((items) => items[0]) + ); + }, + override: ( + environment: AuthenticatedEnvironment, + name: string, + override: ConcurrencyLimitOverrideInput, + overriddenBy?: User + ) => { + if (override.perKey === undefined && override.total === undefined) { + return errAsync({ + type: "invalid_override" as const, + message: "Provide at least one of `perKey` or `total`", + }); + } + + for (const [field, value] of Object.entries(override)) { + if (value === undefined) continue; + if (!Number.isInteger(value) || value < 0 || value > 100000) { + return errAsync({ + type: "invalid_override" as const, + message: `\`${field}\` must be an integer between 0 and 100000`, + }); + } + if (value > environment.maximumConcurrencyLimit) { + return errAsync({ + type: "invalid_override" as const, + message: `\`${field}\` (${value}) cannot exceed the environment limit (${environment.maximumConcurrencyLimit})`, + }); + } + } + + return findLimitByName(this.db, environment, name) + .andThen((row) => applyLimitOverride(this.db, row, override, overriddenBy)) + .andThen((row) => + syncLimitToEngine(environment, row) + .andThen(() => + compensateEngineFromFreshRow(this.db, environment, row.id, { + alreadySynced: { + perKey: row.concurrencyLimit, + total: row.totalConcurrencyLimit, + paused: row.paused, + }, + }) + .orElse(() => okAsync(undefined)) + .map(() => row) + ) + .orElse((error) => + compensateEngineFromFreshRow(this.db, environment, row.id) + .orElse(() => okAsync(undefined)) + .andThen(() => errAsync(error)) + ) + ) + .andThen((row) => + fromPromise(toLimitItems(environment, [row]), (error) => ({ + type: "other" as const, + cause: error, + })).map((items) => items[0]) + ); + }, + reset: (environment: AuthenticatedEnvironment, name: string) => { + return findLimitByName(this.db, environment, name) + .andThen((row) => + syncResetToEngine(environment, row).orElse((error) => + error.type === "limit_not_overridden" + ? errAsync(error) + : compensateEngineFromFreshRow(this.db, environment, row.id) + .orElse(() => okAsync(undefined)) + .andThen(() => errAsync(error)) + ) + ) + .andThen((row) => + resetLimitOverrides(this.db, row).orElse((error) => + compensateEngineFromFreshRow(this.db, environment, row.id) + .orElse(() => okAsync(undefined)) + .andThen(() => errAsync(error)) + ) + ) + .andThen((row) => + fromPromise(toLimitItems(environment, [row]), (error) => ({ + type: "other" as const, + cause: error, + })).map((items) => items[0]) + ); + }, + }; + } +} + +function concurrencyLimitDisplayId(row: Pick): string { + return `climit_${row.friendlyId.replace(/^queue_/, "")}`; +} + +function concurrencyLimitNameFromRow(row: Pick): string { + return row.name.startsWith(LIMIT_QUEUE_PREFIX) + ? row.name.slice(LIMIT_QUEUE_PREFIX.length) + : row.name; +} + +/** + * Limits live in two places: named and shared-queue inline limits are LIMIT-role + * rows under the `limit/` prefix, while an inline limit on a task's own default + * queue compiles onto that V2 QUEUE row (the design's zero-gate-cost case). Its + * derived `task/` name resolves here too, so every declared limit is + * retrievable and overridable through this one surface. V1 queue rows never + * match: their limit is queue surface, managed through the queues API. Only the + * anonymous `limit/task/` namespace requires bounds — a boundless row there is + * retired (its inline limit moved onto the task's own queue) and must fall + * through to the live queue row — while a boundless NAMED limit is a real, + * deliberately uncapped row (referenced without a declaration) that stays + * visible and cappable. + */ +function limitRowsWhere(environment: AuthenticatedEnvironment) { + return { + runtimeEnvironmentId: environment.id, + OR: [ + { + role: "LIMIT" as const, + OR: [ + { name: { not: { startsWith: `${LIMIT_QUEUE_PREFIX}task/` } } }, + { concurrencyLimit: { not: null } }, + { totalConcurrencyLimit: { not: null } }, + ], + }, + { + role: "QUEUE" as const, + concurrencyVersion: "V2" as const, + OR: [{ concurrencyLimit: { not: null } }, { totalConcurrencyLimit: { not: null } }], + }, + ], + }; +} + +function findLimitByName( + db: PrismaClientOrTransaction, + environment: AuthenticatedEnvironment, + name: string +) { + if (!LIMIT_NAME_PATTERN.test(name)) { + return errAsync({ type: "limit_not_found" as const }); + } + + return fromPromise( + db.taskQueue.findFirst({ + where: { + runtimeEnvironmentId: environment.id, + name: `${LIMIT_QUEUE_PREFIX}${name}`, + role: "LIMIT", + ...(name.startsWith("task/") + ? { + OR: [{ concurrencyLimit: { not: null } }, { totalConcurrencyLimit: { not: null } }], + } + : {}), + }, + }), + (error) => ({ type: "other" as const, cause: error }) + ).andThen((row) => { + if (row) { + return okAsync(row); + } + if (!name.startsWith("task/")) { + return errAsync({ type: "limit_not_found" as const }); + } + return fromPromise( + db.taskQueue.findFirst({ + where: { + runtimeEnvironmentId: environment.id, + name, + role: "QUEUE", + concurrencyVersion: "V2", + }, + }), + (error) => ({ type: "other" as const, cause: error }) + ).andThen((queueRow) => { + if (!queueRow) { + return errAsync({ type: "limit_not_found" as const }); + } + return okAsync(queueRow); + }); + }); +} + +/** + * LIMIT rows read the gate machinery (group set + per-gate queued counter); a + * default-queue inline limit is its home queue, so the queue's own concurrency + * and length ARE the runs holding and waiting on the limit. + */ +async function toLimitItems( + environment: AuthenticatedEnvironment, + rows: TaskQueue[] +): Promise { + const limitNames = rows.filter((row) => row.role === "LIMIT").map((row) => row.name); + const queueNames = rows.filter((row) => row.role === "QUEUE").map((row) => row.name); + const [gateRunning, gateQueued, queueRunning, queueQueued] = await Promise.all([ + engine.totalConcurrencyOfQueues(environment, limitNames), + engine.gateQueuedCountOfQueues(environment, limitNames), + queueNames.length > 0 + ? engine.currentConcurrencyOfQueues(environment, queueNames) + : Promise.resolve({} as Record), + queueNames.length > 0 + ? engine.lengthOfQueues(environment, queueNames) + : Promise.resolve({} as Record), + ]); + const running = { ...queueRunning, ...gateRunning }; + const queued = { ...queueQueued, ...gateQueued }; + + return rows.map((row) => ({ + id: concurrencyLimitDisplayId(row), + name: concurrencyLimitNameFromRow(row), + perKey: toBound( + row.concurrencyLimit, + row.concurrencyLimitBase, + row.concurrencyLimitOverriddenAt + ), + total: toBound( + row.totalConcurrencyLimit, + row.totalConcurrencyLimitBase, + row.totalConcurrencyLimitOverriddenAt + ), + running: running[row.name] ?? 0, + queued: queued[row.name] ?? 0, + })); +} + +function toBound( + current: number | null, + base: number | null, + overriddenAt: Date | null +): ConcurrencyLimitBoundValue { + const overridden = overriddenAt !== null; + return { + current, + base: overridden ? base : current, + override: overridden ? current : null, + overriddenAt, + }; +} + +function applyLimitOverride( + db: PrismaClientOrTransaction, + row: TaskQueue, + override: ConcurrencyLimitOverrideInput, + overriddenBy?: User +) { + const now = new Date(); + const data: Record = {}; + + if (override.perKey !== undefined) { + data.concurrencyLimit = override.perKey; + data.concurrencyLimitBase = row.concurrencyLimitOverriddenAt + ? row.concurrencyLimitBase + : (row.concurrencyLimit ?? null); + data.concurrencyLimitOverriddenAt = now; + data.concurrencyLimitOverriddenBy = overriddenBy?.id ?? null; + data.concurrencyLimitOverridePercent = null; + } + + if (override.total !== undefined) { + data.totalConcurrencyLimit = override.total; + data.totalConcurrencyLimitBase = row.totalConcurrencyLimitOverriddenAt + ? row.totalConcurrencyLimitBase + : (row.totalConcurrencyLimit ?? null); + data.totalConcurrencyLimitOverriddenAt = now; + data.totalConcurrencyLimitOverriddenBy = overriddenBy?.id ?? null; + } + + return guardedLimitUpdate(db, row, data); +} + +/** + * Enforce first, then persist: the engine syncs to the declared base BEFORE the + * override markers clear, so an engine failure leaves the markers set and a retry + * converges instead of being rejected while the overridden limit stays enforced. + */ +/** + * A paused row's pause IS the engine per-key value 0 (the DB concurrencyLimit + * column keeps the configured value), so every per-key engine write from this + * surface must preserve it — otherwise an override or reset that only touched + * `total` would silently resume a queue every other surface still reports as + * paused. Named LIMIT rows are never paused (pausing a limit is an override to + * `{ total: 0 }`), so they always take the target branch. + */ +function perKeyEngineWrite( + environment: AuthenticatedEnvironment, + row: Pick, + target: number | null | undefined +) { + if (row.paused) { + return updateQueueConcurrencyLimits(environment, row.name, 0); + } + return typeof target === "number" + ? updateQueueConcurrencyLimits(environment, row.name, target) + : removeQueueConcurrencyLimits(environment, row.name); +} + +function syncResetToEngine( + environment: AuthenticatedEnvironment, + row: TaskQueue +): ResultAsync< + TaskQueue, + { type: "limit_not_overridden" } | { type: "sync_limit_to_engine_failed"; cause: unknown } +> { + if (row.concurrencyLimitOverriddenAt === null && row.totalConcurrencyLimitOverriddenAt === null) { + return errAsync({ type: "limit_not_overridden" as const }); + } + + const perKeyTarget = row.concurrencyLimitOverriddenAt + ? row.concurrencyLimitBase + : row.concurrencyLimit; + const totalTarget = row.totalConcurrencyLimitOverriddenAt + ? row.totalConcurrencyLimitBase + : row.totalConcurrencyLimit; + + const perKeySync = perKeyEngineWrite(environment, row, perKeyTarget); + + const totalSync = + typeof totalTarget === "number" + ? updateQueueTotalConcurrencyLimits(environment, row.name, totalTarget) + : removeQueueTotalConcurrencyLimits(environment, row.name); + + return fromPromise(settleBothEngineWrites(perKeySync, totalSync), (error) => ({ + type: "sync_limit_to_engine_failed" as const, + cause: error, + })).map(() => row); +} + +function resetLimitOverrides(db: PrismaClientOrTransaction, row: TaskQueue) { + const data: Record = {}; + + if (row.concurrencyLimitOverriddenAt !== null) { + data.concurrencyLimit = row.concurrencyLimitBase; + data.concurrencyLimitBase = null; + data.concurrencyLimitOverriddenAt = null; + data.concurrencyLimitOverriddenBy = null; + data.concurrencyLimitOverridePercent = null; + } + + if (row.totalConcurrencyLimitOverriddenAt !== null) { + data.totalConcurrencyLimit = row.totalConcurrencyLimitBase; + data.totalConcurrencyLimitBase = null; + data.totalConcurrencyLimitOverriddenAt = null; + data.totalConcurrencyLimitOverriddenBy = null; + } + + return guardedLimitUpdate(db, row, data); +} + +/** + * Both engine writes settle before a failure is reported, so no write is still in + * flight when a caller's compensation runs — a late sibling can never land after + * the compensating re-sync and leave one bound stale. + */ +async function settleBothEngineWrites(a: Promise, b: Promise): Promise { + const results = await Promise.allSettled([a, b]); + const failed = results.find((result) => result.status === "rejected"); + if (failed && failed.status === "rejected") { + throw failed.reason; + } +} + +/** + * Optimistic update: the where clause carries the row's updatedAt plus both override + * markers as read, so ANY concurrent write — another override or reset, or a deploy + * refreshing the declared values — makes this update miss (P2025) and the caller + * gets a conflict instead of persisting values computed from a stale row. The + * markers narrow the same-millisecond updatedAt window to writes that also leave + * both markers untouched. + */ +function guardedLimitUpdate( + db: PrismaClientOrTransaction, + row: TaskQueue, + data: Record +) { + return fromPromise( + db.taskQueue.update({ + where: { + id: row.id, + updatedAt: row.updatedAt, + concurrencyLimitOverriddenAt: row.concurrencyLimitOverriddenAt, + totalConcurrencyLimitOverriddenAt: row.totalConcurrencyLimitOverriddenAt, + }, + data, + }), + (error) => { + if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2025") { + return { type: "conflict" as const }; + } + return { type: "limit_update_failed" as const, cause: error }; + } + ); +} + +type SyncedLimitValues = { perKey: number | null; total: number | null; paused: boolean }; + +/** + * Re-syncs the engine from fresh reads of the row until the enforced values stop + * moving (bounded), the same convergence the deploy sync uses: every actor writes + * Postgres before its own engine sync, so re-syncing whatever is freshest + * converges. The fixpoint compares the values the engine enforces rather than + * updatedAt, because Prisma's @updatedAt has millisecond precision and two writes + * in the same millisecond are indistinguishable by timestamp. Matching values + * only prove this actor once synced them, not that the engine still holds them + * (another actor may have diverged it and written the same values back), but + * skipping keeps divergence non-silent: a failed engine write always surfaces + * to that actor's caller, which can retry, and a stale write landing after the + * loop's final read (the loop is bounded) is healed by the next sync or deploy, + * the same residual the deploy-time queue sync accepts. Callers use it two ways: after a failure (a reset's enforce-first engine write preceding a + * persist that then conflicts, or an override's sync failing after its persist), + * where the original error still reaches the caller; and after a successful sync + * with `alreadySynced` set to the values just synced, where an unchanged row + * costs one read and a moved row is re-synced. + */ +function compensateEngineFromFreshRow( + db: PrismaClientOrTransaction, + environment: AuthenticatedEnvironment, + rowId: string, + options?: { alreadySynced?: SyncedLimitValues } +) { + return fromPromise( + (async () => { + let lastSynced: SyncedLimitValues | null = options?.alreadySynced ?? null; + for (let i = 0; i < 3; i++) { + const fresh = await db.taskQueue.findFirst({ where: { id: rowId } }); + if ( + !fresh || + (lastSynced !== null && + fresh.concurrencyLimit === lastSynced.perKey && + fresh.totalConcurrencyLimit === lastSynced.total && + fresh.paused === lastSynced.paused) + ) { + return; + } + await settleBothEngineWrites( + perKeyEngineWrite(environment, fresh, fresh.concurrencyLimit), + typeof fresh.totalConcurrencyLimit === "number" + ? updateQueueTotalConcurrencyLimits( + environment, + fresh.name, + fresh.totalConcurrencyLimit + ) + : removeQueueTotalConcurrencyLimits(environment, fresh.name) + ); + lastSynced = { + perKey: fresh.concurrencyLimit, + total: fresh.totalConcurrencyLimit, + paused: fresh.paused, + }; + } + })(), + (error) => ({ type: "other" as const, cause: error }) + ); +} + +/** + * Pushes both engine keys from the row: the per-key limit and the total. Limit + * rows are never paused (pausing a limit is an override to `{ total: 0 }`), so + * both keys sync unconditionally, unlike queue rows. + */ +function syncLimitToEngine(environment: AuthenticatedEnvironment, row: TaskQueue) { + const perKeySync = perKeyEngineWrite(environment, row, row.concurrencyLimit); + + const totalSync = + typeof row.totalConcurrencyLimit === "number" + ? updateQueueTotalConcurrencyLimits(environment, row.name, row.totalConcurrencyLimit) + : removeQueueTotalConcurrencyLimits(environment, row.name); + + return fromPromise(settleBothEngineWrites(perKeySync, totalSync), (error) => ({ + type: "sync_limit_to_engine_failed" as const, + cause: error, + })).map(() => row); +} diff --git a/apps/webapp/app/v3/services/concurrencyLimitsSystemInstance.server.ts b/apps/webapp/app/v3/services/concurrencyLimitsSystemInstance.server.ts new file mode 100644 index 00000000000..eeaa6882e9f --- /dev/null +++ b/apps/webapp/app/v3/services/concurrencyLimitsSystemInstance.server.ts @@ -0,0 +1,15 @@ +import { prisma, $replica } from "~/db.server"; +import { ConcurrencyLimitsSystem } from "./concurrencyLimitsSystem.server"; +import { singleton } from "~/utils/singleton"; + +export const concurrencyLimitsSystem = singleton( + "concurrency-limits-system", + initializeConcurrencyLimitsSystemInstance +); + +function initializeConcurrencyLimitsSystemInstance() { + return new ConcurrencyLimitsSystem({ + db: prisma, + reader: $replica, + }); +} diff --git a/apps/webapp/app/v3/services/concurrencySystem.server.ts b/apps/webapp/app/v3/services/concurrencySystem.server.ts index 099f22bdd69..7d65353b798 100644 --- a/apps/webapp/app/v3/services/concurrencySystem.server.ts +++ b/apps/webapp/app/v3/services/concurrencySystem.server.ts @@ -67,17 +67,24 @@ export class ConcurrencySystem { environment: AuthenticatedEnvironment, queue: QueueInput, override: ConcurrencyLimitOverride, - overriddenBy?: User + overriddenBy?: User, + opts?: QueueMutationOpts ) => { return findQueueFromInput(this.db, environment, queue) + .andThen((queue) => guardQueueVersion(queue, opts)) .andThen((queue) => overrideQueueConcurrencyLimit(this.db, environment, queue, override, overriddenBy) ) .andThen((queue) => syncQueueConcurrencyToEngine(environment, queue)) .andThen((queue) => getQueueStats(environment, queue)); }, - resetConcurrencyLimit: (environment: AuthenticatedEnvironment, queue: QueueInput) => { + resetConcurrencyLimit: ( + environment: AuthenticatedEnvironment, + queue: QueueInput, + opts?: QueueMutationOpts + ) => { return findQueueFromInput(this.db, environment, queue) + .andThen((queue) => guardQueueVersion(queue, opts)) .andThen((queue) => resetQueueConcurrencyLimit(this.db, queue)) .andThen((queue) => syncQueueConcurrencyToEngine(environment, queue)) .andThen((queue) => getQueueStats(environment, queue)); @@ -180,6 +187,22 @@ function findQueueFromInput( return findQueueByName(db, environment, queueName); } +/** + * The public queue override/reset endpoints are the V1 lever; a V2 queue's + * limits are managed through the concurrency-limits endpoints (a default-queue + * inline limit under its derived task/ name), and a V2 response hides + * queue-level concurrency, so mutating one here would succeed invisibly. The + * dashboard's own actions pass no opts and keep working on every version. + */ +type QueueMutationOpts = { v1Only?: boolean }; + +function guardQueueVersion(queue: TaskQueue, opts: QueueMutationOpts | undefined) { + if (opts?.v1Only && queue.concurrencyVersion === "V2") { + return errAsync({ type: "queue_version_unsupported" as const }); + } + return okAsync(queue); +} + function findQueueByFriendlyId( db: PrismaClientOrTransaction, environment: AuthenticatedEnvironment, diff --git a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts index 9712d8d06f1..5a4babbb409 100644 --- a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts +++ b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts @@ -387,9 +387,112 @@ async function createWorkerTasks( if (entry) entries.push(entry); } } + + await retireStaleAnonymousConcurrencyLimitRows(metadata.tasks, environment, prisma); + return entries; } +/** + * A task whose inline limit no longer compiles to its anonymous LIMIT row (the + * limit moved onto the task's own default queue, or was removed) leaves that + * row behind, where it would shadow the live task/ name on the + * concurrency-limits surface and keep stale engine keys. Retiring clears the + * row's bounds and override state — dropping it from listing and name + * resolution so the queue-row fallback resolves — and removes its engine keys. + */ +async function retireStaleAnonymousConcurrencyLimitRows( + tasks: TaskResource[], + environment: AuthenticatedEnvironment, + prisma: PrismaClientOrTransaction +): Promise { + const candidateNames = tasks + .filter((task) => !(task.concurrency?.inline && task.queue?.name)) + .map((task) => anonymousConcurrencyLimitQueueName(task.id)); + if (candidateNames.length === 0) { + return; + } + + const staleRows = await prisma.taskQueue.findMany({ + where: { + runtimeEnvironmentId: environment.id, + role: "LIMIT", + name: { in: boundedIn(candidateNames) }, + OR: [{ concurrencyLimit: { not: null } }, { totalConcurrencyLimit: { not: null } }], + }, + select: { id: true, name: true, updatedAt: true }, + }); + if (staleRows.length === 0) { + return; + } + + /** Engine keys go first, and the row's bounds are only nulled once both + * removals succeeded: a failed removal leaves the row bounded, so the next + * deploy retries the retirement instead of stranding a stale engine limit + * (worst case a pause-by-zero) that nothing can see or clear. Each null is + * guarded on the row's read updatedAt, so a concurrent writer (an operator + * override, or another deploy re-creating the limit) wins and its own engine + * sync governs; a racing limits-surface sync converges via its freshness + * re-check against the nulled row. */ + for (const row of staleRows) { + try { + await Promise.all([ + removeQueueConcurrencyLimits(environment, row.name), + removeQueueTotalConcurrencyLimits(environment, row.name), + ]); + } catch (error) { + logger.error( + "retireStaleAnonymousConcurrencyLimitRows: engine cleanup failed, retrying next deploy", + { + environmentId: environment.id, + queueName: row.name, + error, + } + ); + continue; + } + const retired = await prisma.taskQueue.updateMany({ + where: { id: row.id, updatedAt: row.updatedAt }, + data: { + concurrencyLimit: null, + concurrencyLimitBase: null, + concurrencyLimitOverriddenAt: null, + concurrencyLimitOverriddenBy: null, + concurrencyLimitOverridePercent: null, + totalConcurrencyLimit: null, + totalConcurrencyLimitBase: null, + totalConcurrencyLimitOverriddenAt: null, + totalConcurrencyLimitOverriddenBy: null, + }, + }); + if (retired.count === 0) { + /** A concurrent writer (an operator override, or another deploy) took the + * row between the read and the null, and the key removal above may have + * erased the engine state that writer just synced — including a + * pause-by-zero. Restore the engine from the fresh row so the winner's + * bounds stay enforced; both writers write the same fresh values, so the + * race converges. */ + const fresh = await prisma.taskQueue.findFirst({ where: { id: row.id } }); + if (fresh) { + await Promise.allSettled([ + fresh.paused + ? updateQueueConcurrencyLimits(environment, fresh.name, 0) + : typeof fresh.concurrencyLimit === "number" + ? updateQueueConcurrencyLimits(environment, fresh.name, fresh.concurrencyLimit) + : removeQueueConcurrencyLimits(environment, fresh.name), + typeof fresh.totalConcurrencyLimit === "number" + ? updateQueueTotalConcurrencyLimits( + environment, + fresh.name, + fresh.totalConcurrencyLimit + ) + : removeQueueTotalConcurrencyLimits(environment, fresh.name), + ]); + } + } + } +} + async function createWorkerTask( task: TaskResource, queues: Array, @@ -426,7 +529,7 @@ async function createWorkerTask( if (concurrency?.inline) { if (!task.queue?.name) { - queueConcurrencyLimit = concurrency.inline.perKey ?? concurrency.inline.total; + queueConcurrencyLimit = concurrency.inline.perKey; queueTotalConcurrencyLimit = concurrency.inline.total; } else { if (compiledGates.length > 1) { @@ -438,7 +541,7 @@ async function createWorkerTask( await createWorkerQueue( { name: anonymousQueueName, - concurrencyLimit: concurrency.inline.perKey ?? concurrency.inline.total ?? null, + concurrencyLimit: concurrency.inline.perKey ?? null, combinedConcurrencyLimit: concurrency.inline.total ?? null, }, `task/${task.id}`, @@ -707,9 +810,9 @@ function assertNotReservedQueueName(name: string, context: string): void { /** * Materializes the worker's declared named concurrency limits (plus any names tasks - * reference without declaring, created uncapped) as LIMIT-role TaskQueue rows. A - * total-only limit stores the total as its per-key limit too, so no single key (or - * the keyless pool) can exceed it even before the group check applies. + * reference without declaring, created uncapped) as LIMIT-role TaskQueue rows. The + * columns mirror the declared shape exactly: perKey caps each key pool (and the + * keyless pool); total caps everything together via the group set. */ async function createWorkerConcurrencyLimits( metadata: BackgroundWorkerMetadata, @@ -731,7 +834,7 @@ async function createWorkerConcurrencyLimits( await createWorkerQueue( { name: concurrencyLimitQueueName(limit.name), - concurrencyLimit: limit.perKey ?? limit.total ?? null, + concurrencyLimit: limit.perKey ?? null, combinedConcurrencyLimit: limit.total ?? null, }, limit.name, diff --git a/apps/webapp/app/v3/services/pauseQueue.server.ts b/apps/webapp/app/v3/services/pauseQueue.server.ts index 87d2d339864..ad7b8f20e46 100644 --- a/apps/webapp/app/v3/services/pauseQueue.server.ts +++ b/apps/webapp/app/v3/services/pauseQueue.server.ts @@ -94,6 +94,7 @@ export class PauseQueueService extends BaseService { friendlyId: updatedQueue.friendlyId, name: updatedQueue.name, type: updatedQueue.type, + version: updatedQueue.concurrencyVersion, running: results[1]?.[updatedQueue.name] ?? 0, queued: results[0]?.[updatedQueue.name] ?? 0, concurrencyLimit: updatedQueue.concurrencyLimit ?? null, diff --git a/apps/webapp/test/concurrencyLimitsSystem.test.ts b/apps/webapp/test/concurrencyLimitsSystem.test.ts new file mode 100644 index 00000000000..4b7c3545058 --- /dev/null +++ b/apps/webapp/test/concurrencyLimitsSystem.test.ts @@ -0,0 +1,523 @@ +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { beforeEach, describe, expect, vi } from "vitest"; +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { ConcurrencyLimitsSystem } from "~/v3/services/concurrencyLimitsSystem.server"; + +/** + * These tests exercise the DB-write, marker and ordering logic against a real + * Postgres. The engine syncs are spies so tests can assert ordering and inject + * failures; the Redis side itself is covered by the run-engine suites. + */ +const { perKeySyncMock, perKeyRemoveMock, totalSyncMock, totalRemoveMock } = vi.hoisted(() => ({ + perKeySyncMock: vi.fn(async (..._args: unknown[]) => undefined), + perKeyRemoveMock: vi.fn(async (..._args: unknown[]) => undefined), + totalSyncMock: vi.fn(async (..._args: unknown[]) => undefined), + totalRemoveMock: vi.fn(async (..._args: unknown[]) => undefined), +})); + +vi.mock("~/v3/runQueue.server", () => ({ + updateQueueConcurrencyLimits: perKeySyncMock, + removeQueueConcurrencyLimits: perKeyRemoveMock, + updateQueueTotalConcurrencyLimits: totalSyncMock, + removeQueueTotalConcurrencyLimits: totalRemoveMock, +})); + +vi.mock("~/v3/runEngine.server", () => ({ + engine: { + totalConcurrencyOfQueues: async (_env: unknown, queues: string[]) => + Object.fromEntries(queues.map((q) => [q, 0])), + gateQueuedCountOfQueues: async (_env: unknown, queues: string[]) => + Object.fromEntries(queues.map((q) => [q, 0])), + currentConcurrencyOfQueues: async (_env: unknown, queues: string[]) => + Object.fromEntries(queues.map((q) => [q, 0])), + lengthOfQueues: async (_env: unknown, queues: string[]) => + Object.fromEntries(queues.map((q) => [q, 0])), + }, +})); + +vi.setConfig({ testTimeout: 30_000 }); + +async function seedEnvAndLimit( + prisma: PrismaClient, + opts: { perKey?: number | null; total?: number | null } = {} +) { + const slug = `s${Math.random().toString(36).slice(2, 10)}`; + + const organization = await prisma.organization.create({ data: { title: slug, slug } }); + const project = await prisma.project.create({ + data: { name: slug, slug, organizationId: organization.id, externalRef: slug }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + slug, + type: "PRODUCTION", + projectId: project.id, + organizationId: organization.id, + apiKey: slug, + pkApiKey: slug, + shortcode: slug, + maximumConcurrencyLimit: 100, + }, + }); + + const row = await prisma.taskQueue.create({ + data: { + friendlyId: `queue_${slug}`, + name: "limit/openai", + orderableName: "openai", + projectId: project.id, + runtimeEnvironmentId: environment.id, + role: "LIMIT", + concurrencyVersion: "V2", + concurrencyLimit: opts.perKey ?? null, + totalConcurrencyLimit: opts.total ?? null, + }, + }); + + const authEnv = { + id: environment.id, + maximumConcurrencyLimit: environment.maximumConcurrencyLimit, + } as unknown as AuthenticatedEnvironment; + + const system = new ConcurrencyLimitsSystem({ db: prisma, reader: prisma }); + + return { environment, row, authEnv, system }; +} + +describe("ConcurrencyLimitsSystem", () => { + /** Call counts must start at zero per test and leaked one-off implementations + * must not outlive the test that set them; mockReset also restores the default + * implementations given to vi.fn above. */ + beforeEach(() => { + perKeySyncMock.mockReset(); + perKeyRemoveMock.mockReset(); + totalSyncMock.mockReset(); + totalRemoveMock.mockReset(); + }); + + postgresTest( + "override changes only the given bound and keeps the declared base", + async ({ prisma }) => { + const { authEnv, system, row } = await seedEnvAndLimit(prisma, { total: 25 }); + + const result = await system.limits.override(authEnv, "openai", { total: 50 }); + expect(result.isOk()).toBe(true); + if (result.isOk()) { + expect(result.value.total).toMatchObject({ current: 50, base: 25, override: 50 }); + expect(result.value.perKey).toMatchObject({ current: null, override: null }); + } + + const updated = await prisma.taskQueue.findFirstOrThrow({ where: { id: row.id } }); + expect(updated.totalConcurrencyLimit).toBe(50); + expect(updated.totalConcurrencyLimitBase).toBe(25); + expect(updated.totalConcurrencyLimitOverriddenAt).not.toBeNull(); + expect(updated.concurrencyLimitOverriddenAt).toBeNull(); + + expect(totalSyncMock).toHaveBeenCalledWith(authEnv, "limit/openai", 50); + expect(perKeyRemoveMock).toHaveBeenCalledWith(authEnv, "limit/openai"); + } + ); + + postgresTest("override to zero pauses the limit in the engine", async ({ prisma }) => { + const { authEnv, system } = await seedEnvAndLimit(prisma, { total: 25 }); + + const result = await system.limits.override(authEnv, "openai", { total: 0 }); + expect(result.isOk()).toBe(true); + expect(totalSyncMock).toHaveBeenCalledWith(authEnv, "limit/openai", 0); + }); + + postgresTest("reset restores the declared values and clears the markers", async ({ prisma }) => { + const { authEnv, system, row } = await seedEnvAndLimit(prisma, { perKey: 2, total: 25 }); + + await system.limits.override(authEnv, "openai", { perKey: 10, total: 50 }); + const result = await system.limits.reset(authEnv, "openai"); + expect(result.isOk()).toBe(true); + if (result.isOk()) { + expect(result.value.total).toMatchObject({ current: 25, base: 25, override: null }); + expect(result.value.perKey).toMatchObject({ current: 2, base: 2, override: null }); + } + + const updated = await prisma.taskQueue.findFirstOrThrow({ where: { id: row.id } }); + expect(updated.concurrencyLimit).toBe(2); + expect(updated.totalConcurrencyLimit).toBe(25); + expect(updated.concurrencyLimitOverriddenAt).toBeNull(); + expect(updated.totalConcurrencyLimitOverriddenAt).toBeNull(); + }); + + postgresTest("reset without an override is rejected", async ({ prisma }) => { + const { authEnv, system } = await seedEnvAndLimit(prisma, { total: 25 }); + + const result = await system.limits.reset(authEnv, "openai"); + expect(result.isErr()).toBe(true); + if (result.isErr()) { + expect(result.error.type).toBe("limit_not_overridden"); + } + }); + + postgresTest( + "a failed engine sync during reset leaves the override intact so a retry converges", + async ({ prisma }) => { + const { authEnv, system, row } = await seedEnvAndLimit(prisma, { total: 25 }); + + await system.limits.override(authEnv, "openai", { total: 50 }); + + totalSyncMock.mockRejectedValueOnce(new Error("redis down")); + const failed = await system.limits.reset(authEnv, "openai"); + expect(failed.isErr()).toBe(true); + + /** The marker must survive the failed sync: the DB still says overridden. */ + const midway = await prisma.taskQueue.findFirstOrThrow({ where: { id: row.id } }); + expect(midway.totalConcurrencyLimitOverriddenAt).not.toBeNull(); + expect(midway.totalConcurrencyLimit).toBe(50); + + const retried = await system.limits.reset(authEnv, "openai"); + expect(retried.isOk()).toBe(true); + const final = await prisma.taskQueue.findFirstOrThrow({ where: { id: row.id } }); + expect(final.totalConcurrencyLimit).toBe(25); + expect(final.totalConcurrencyLimitOverriddenAt).toBeNull(); + } + ); + + postgresTest( + "a mutation whose markers moved underneath it conflicts instead of clobbering", + async ({ prisma }) => { + const { authEnv, system, row } = await seedEnvAndLimit(prisma, { total: 25 }); + + await system.limits.override(authEnv, "openai", { total: 50 }); + + /** + * Interleave a concurrent reset between this mutation's read and its write: + * the engine sync hook is the seam after the read, so clearing the markers + * there makes the guarded update miss and surface a conflict. + */ + totalSyncMock.mockImplementationOnce(async () => { + await prisma.taskQueue.update({ + where: { id: row.id }, + data: { + totalConcurrencyLimit: 25, + totalConcurrencyLimitBase: null, + totalConcurrencyLimitOverriddenAt: null, + totalConcurrencyLimitOverriddenBy: null, + }, + }); + }); + + const raced = await system.limits.reset(authEnv, "openai"); + expect(raced.isErr()).toBe(true); + if (raced.isErr()) { + expect(raced.error.type).toBe("conflict"); + } + + /** The concurrent actor's state stands untouched. */ + const final = await prisma.taskQueue.findFirstOrThrow({ where: { id: row.id } }); + expect(final.totalConcurrencyLimit).toBe(25); + expect(final.totalConcurrencyLimitOverriddenAt).toBeNull(); + } + ); + + postgresTest( + "a failed engine sync during override compensates from the fresh row", + async ({ prisma }) => { + const { authEnv, system, row } = await seedEnvAndLimit(prisma, { total: 25 }); + + totalSyncMock.mockRejectedValueOnce(new Error("redis down")); + const failed = await system.limits.override(authEnv, "openai", { total: 50 }); + expect(failed.isErr()).toBe(true); + if (failed.isErr()) { + expect(failed.error.type).toBe("sync_limit_to_engine_failed"); + } + + /** The persist already happened; compensation re-syncs it so the engine + * doesn't keep enforcing the old bound while the API reports the new one. + * Exactly two calls: the rejected primary sync, then the compensating + * re-sync from the fresh row — without compensation there is only one. */ + const updated = await prisma.taskQueue.findFirstOrThrow({ where: { id: row.id } }); + expect(updated.totalConcurrencyLimit).toBe(50); + expect(totalSyncMock).toHaveBeenCalledTimes(2); + expect(totalSyncMock).toHaveBeenLastCalledWith(authEnv, "limit/openai", 50); + } + ); + + postgresTest( + "an older override's engine write landing last is repaired by the freshness re-check", + async ({ prisma }) => { + const { authEnv, system, row } = await seedEnvAndLimit(prisma, { total: 25 }); + + /** `engineTotal` is written when a sync "lands", so landing order can differ + * from call order: the first override's write is delayed until a second, + * newer override has fully completed, then lands with the stale value. */ + let engineTotal: number | null = null; + let secondResult: Awaited> | undefined; + totalSyncMock.mockImplementation(async (_env, _name, value) => { + engineTotal = value as number; + }); + totalSyncMock.mockImplementationOnce(async (_env, _name, value) => { + secondResult = await system.limits.override(authEnv, "openai", { total: 75 }); + engineTotal = value as number; + }); + + const first = await system.limits.override(authEnv, "openai", { total: 50 }); + expect(first.isOk()).toBe(true); + expect(secondResult?.isOk()).toBe(true); + + const final = await prisma.taskQueue.findFirstOrThrow({ where: { id: row.id } }); + expect(final.totalConcurrencyLimit).toBe(75); + expect(engineTotal).toBe(75); + } + ); + + postgresTest( + "a default-queue inline limit resolves, overrides and resets under its task/ name", + async ({ prisma }) => { + const { authEnv, system, environment } = await seedEnvAndLimit(prisma, { total: 25 }); + + const queueRow = await prisma.taskQueue.create({ + data: { + friendlyId: `queue_t${environment.slug}`, + name: "task/send-email", + orderableName: "send-email", + projectId: environment.projectId, + runtimeEnvironmentId: environment.id, + role: "QUEUE", + concurrencyVersion: "V2", + concurrencyLimit: 1, + totalConcurrencyLimit: 10, + }, + }); + + const retrieved = await system.limits.retrieve(authEnv, "task/send-email"); + expect(retrieved.isOk()).toBe(true); + if (retrieved.isOk()) { + expect(retrieved.value.name).toBe("task/send-email"); + expect(retrieved.value.perKey).toMatchObject({ current: 1, base: 1 }); + expect(retrieved.value.total).toMatchObject({ current: 10, base: 10 }); + } + + const overridden = await system.limits.override(authEnv, "task/send-email", { total: 20 }); + expect(overridden.isOk()).toBe(true); + expect(totalSyncMock).toHaveBeenCalledWith(authEnv, "task/send-email", 20); + + const reset = await system.limits.reset(authEnv, "task/send-email"); + expect(reset.isOk()).toBe(true); + const final = await prisma.taskQueue.findFirstOrThrow({ where: { id: queueRow.id } }); + expect(final.totalConcurrencyLimit).toBe(10); + expect(final.totalConcurrencyLimitOverriddenAt).toBeNull(); + + const listed = await system.limits.list(authEnv, { page: 1, perPage: 50 }); + expect(listed.isOk()).toBe(true); + if (listed.isOk()) { + expect(listed.value.map((item) => item.name).sort()).toEqual(["openai", "task/send-email"]); + } + } + ); + + postgresTest( + "V1 queue rows and boundless V2 queue rows never surface as limits", + async ({ prisma }) => { + const { authEnv, system, environment } = await seedEnvAndLimit(prisma, { total: 25 }); + + await prisma.taskQueue.create({ + data: { + friendlyId: `queue_v1${environment.slug}`, + name: "task/legacy-task", + orderableName: "legacy-task", + projectId: environment.projectId, + runtimeEnvironmentId: environment.id, + role: "QUEUE", + concurrencyVersion: "V1", + concurrencyLimit: 5, + }, + }); + await prisma.taskQueue.create({ + data: { + friendlyId: `queue_nb${environment.slug}`, + name: "task/unbounded-task", + orderableName: "unbounded-task", + projectId: environment.projectId, + runtimeEnvironmentId: environment.id, + role: "QUEUE", + concurrencyVersion: "V2", + }, + }); + + const v1 = await system.limits.retrieve(authEnv, "task/legacy-task"); + expect(v1.isErr()).toBe(true); + + /** Boundless V2 queues stay out of the list but resolve by name, so an + * operator can still cap an undeclared task through this surface. */ + const boundless = await system.limits.retrieve(authEnv, "task/unbounded-task"); + expect(boundless.isOk()).toBe(true); + if (boundless.isOk()) { + expect(boundless.value.perKey.current).toBeNull(); + expect(boundless.value.total.current).toBeNull(); + } + + const listed = await system.limits.list(authEnv, { page: 1, perPage: 50 }); + expect(listed.isOk()).toBe(true); + if (listed.isOk()) { + expect(listed.value.map((item) => item.name)).toEqual(["openai"]); + } + } + ); + + postgresTest( + "a retired anonymous LIMIT row falls through to the live queue row", + async ({ prisma }) => { + const { authEnv, system, environment } = await seedEnvAndLimit(prisma, { total: 25 }); + + await prisma.taskQueue.create({ + data: { + friendlyId: `queue_rl${environment.slug}`, + name: "limit/task/send-email", + orderableName: "send-email", + projectId: environment.projectId, + runtimeEnvironmentId: environment.id, + role: "LIMIT", + concurrencyVersion: "V2", + }, + }); + await prisma.taskQueue.create({ + data: { + friendlyId: `queue_ql${environment.slug}`, + name: "task/send-email", + orderableName: "send-email-q", + projectId: environment.projectId, + runtimeEnvironmentId: environment.id, + role: "QUEUE", + concurrencyVersion: "V2", + concurrencyLimit: 1, + totalConcurrencyLimit: 10, + }, + }); + + const retrieved = await system.limits.retrieve(authEnv, "task/send-email"); + expect(retrieved.isOk()).toBe(true); + if (retrieved.isOk()) { + expect(retrieved.value.total).toMatchObject({ current: 10 }); + } + + const listed = await system.limits.list(authEnv, { page: 1, perPage: 50 }); + expect(listed.isOk()).toBe(true); + if (listed.isOk()) { + expect(listed.value.filter((item) => item.name === "task/send-email")).toHaveLength(1); + } + } + ); + + postgresTest("overrides and resets preserve a queue pause in the engine", async ({ prisma }) => { + const { authEnv, system, environment } = await seedEnvAndLimit(prisma, { total: 25 }); + + await prisma.taskQueue.create({ + data: { + friendlyId: `queue_p${environment.slug}`, + name: "task/paused-task", + orderableName: "paused-task", + projectId: environment.projectId, + runtimeEnvironmentId: environment.id, + role: "QUEUE", + concurrencyVersion: "V2", + concurrencyLimit: 1, + totalConcurrencyLimit: 10, + paused: true, + }, + }); + + const overridden = await system.limits.override(authEnv, "task/paused-task", { total: 20 }); + expect(overridden.isOk()).toBe(true); + expect(totalSyncMock).toHaveBeenCalledWith(authEnv, "task/paused-task", 20); + /** The pause IS the per-key engine value 0; the sync must rewrite 0, never + * the configured limit and never a removal. */ + expect(perKeySyncMock).toHaveBeenCalledWith(authEnv, "task/paused-task", 0); + expect(perKeySyncMock).not.toHaveBeenCalledWith(authEnv, "task/paused-task", 1); + expect(perKeyRemoveMock).not.toHaveBeenCalledWith(authEnv, "task/paused-task"); + + perKeySyncMock.mockClear(); + const reset = await system.limits.reset(authEnv, "task/paused-task"); + expect(reset.isOk()).toBe(true); + expect(perKeySyncMock).toHaveBeenCalledWith(authEnv, "task/paused-task", 0); + expect(perKeySyncMock).not.toHaveBeenCalledWith(authEnv, "task/paused-task", 1); + }); + + postgresTest("a perKey override clears a stale percent override source", async ({ prisma }) => { + const { authEnv, system, environment } = await seedEnvAndLimit(prisma, { total: 25 }); + + const row = await prisma.taskQueue.create({ + data: { + friendlyId: `queue_pc${environment.slug}`, + name: "task/percent-task", + orderableName: "percent-task", + projectId: environment.projectId, + runtimeEnvironmentId: environment.id, + role: "QUEUE", + concurrencyVersion: "V2", + concurrencyLimit: 50, + concurrencyLimitBase: 100, + concurrencyLimitOverriddenAt: new Date(), + concurrencyLimitOverridePercent: 50, + }, + }); + + const overridden = await system.limits.override(authEnv, "task/percent-task", { perKey: 3 }); + expect(overridden.isOk()).toBe(true); + const updated = await prisma.taskQueue.findFirstOrThrow({ where: { id: row.id } }); + expect(updated.concurrencyLimit).toBe(3); + expect(updated.concurrencyLimitOverridePercent).toBeNull(); + }); + + postgresTest("an uncapped named limit stays visible and cappable", async ({ prisma }) => { + const { authEnv, system, environment } = await seedEnvAndLimit(prisma, { total: 25 }); + + /** Deploys materialize referenced-but-undeclared names as uncapped LIMIT + * rows; only the anonymous limit/task/ namespace treats boundless as + * retired. */ + await prisma.taskQueue.create({ + data: { + friendlyId: `queue_un${environment.slug}`, + name: "limit/acme-api", + orderableName: "acme-api", + projectId: environment.projectId, + runtimeEnvironmentId: environment.id, + role: "LIMIT", + concurrencyVersion: "V2", + }, + }); + + const retrieved = await system.limits.retrieve(authEnv, "acme-api"); + expect(retrieved.isOk()).toBe(true); + if (retrieved.isOk()) { + expect(retrieved.value.total.current).toBeNull(); + } + + const capped = await system.limits.override(authEnv, "acme-api", { total: 5 }); + expect(capped.isOk()).toBe(true); + expect(totalSyncMock).toHaveBeenCalledWith(authEnv, "limit/acme-api", 5); + + const listed = await system.limits.list(authEnv, { page: 1, perPage: 50 }); + expect(listed.isOk()).toBe(true); + if (listed.isOk()) { + expect(listed.value.map((item) => item.name)).toContain("acme-api"); + } + }); + + postgresTest("retrieve misses queue-role rows and unknown names", async ({ prisma }) => { + const { authEnv, system, environment } = await seedEnvAndLimit(prisma, { total: 25 }); + + await prisma.taskQueue.create({ + data: { + friendlyId: `queue_q${environment.slug}`, + name: "limit/shadow", + orderableName: "shadow", + projectId: environment.projectId, + runtimeEnvironmentId: environment.id, + role: "QUEUE", + }, + }); + + const missing = await system.limits.retrieve(authEnv, "missing"); + expect(missing.isErr()).toBe(true); + + const shadow = await system.limits.retrieve(authEnv, "shadow"); + expect(shadow.isErr()).toBe(true); + }); +}); diff --git a/apps/webapp/test/dashboardAgentToolScopes.test.ts b/apps/webapp/test/dashboardAgentToolScopes.test.ts index 6eabb2d8eb7..7f850c3d3a4 100644 --- a/apps/webapp/test/dashboardAgentToolScopes.test.ts +++ b/apps/webapp/test/dashboardAgentToolScopes.test.ts @@ -34,7 +34,7 @@ const VIA_ENV_JWT: Read[] = [ { tool: "get_queue (metrics)", path: "/api/v1/queues/:name/metrics", - resource: { type: "query", id: "queue_metrics" }, + resource: { type: "query", id: "concurrency_metrics" }, }, { tool: "get_queue (live row)", path: "/api/v1/queues/:name", resource: { type: "queues" } }, { diff --git a/apps/webapp/test/reportHealth.test.ts b/apps/webapp/test/reportHealth.test.ts index 706291816f0..6357ac2f77b 100644 --- a/apps/webapp/test/reportHealth.test.ts +++ b/apps/webapp/test/reportHealth.test.ts @@ -94,8 +94,8 @@ describe("health cause tree (Golden A — env limit saturation)", () => { it("footer = raise the limit (self-serve) + docs + do-nothing (drains)", () => { expect(vm.footer).toEqual([ - { code: "raise_env_limit", link: "concurrency" }, - { code: "concurrency_docs", link: "concurrency" }, + { code: "raise_env_limit", link: "concurrency-limits" }, + { code: "concurrency_docs", link: "concurrency-limits" }, { code: "do_nothing_drains", value: 2.3 }, ]); }); diff --git a/apps/webapp/test/reportHealthData.test.ts b/apps/webapp/test/reportHealthData.test.ts index 6b12f33ea75..4beaf6546aa 100644 --- a/apps/webapp/test/reportHealthData.test.ts +++ b/apps/webapp/test/reportHealthData.test.ts @@ -51,7 +51,7 @@ function makeDeps(opts: { if (query.includes("dlq_total")) return wrap(opts.queueTotals); if (isEnv && query.includes("timeBucket")) return wrap(opts.envSeries); if (isEnv) return wrap(opts.envScalar ?? [{}]); - if (query.includes("FROM queue_metrics")) return wrap(opts.worst); + if (query.includes("FROM concurrency_metrics")) return wrap(opts.worst); if (query.includes("task_identifier")) return wrap([]); if (query.includes("FROM runs") && query.includes("timeBucket")) return wrap(opts.runsSeries); return wrap(opts.runs); @@ -76,7 +76,7 @@ const RUNS_SCALAR: Rows = [ ]; describe("loadHealthInput — orchestration (query seam)", () => { - it("measured path: queue_metrics source, real pending, parsed dlq, window from timeRange", async () => { + it("measured path: concurrency_metrics source, real pending, parsed dlq, window from timeRange", async () => { const input = await loadHealthInput( fakeEnv, "1h", diff --git a/apps/webapp/test/reportsApiRoute.test.ts b/apps/webapp/test/reportsApiRoute.test.ts index f027c0d9eb3..37043b6513b 100644 --- a/apps/webapp/test/reportsApiRoute.test.ts +++ b/apps/webapp/test/reportsApiRoute.test.ts @@ -105,7 +105,7 @@ describe("api.v1.reports.$key — authorization", () => { expect(requiredResources("health")).toEqual([ { type: "query", id: "runs" }, { type: "query", id: "env_metrics" }, - { type: "query", id: "queue_metrics" }, + { type: "query", id: "concurrency_metrics" }, ]); }); @@ -118,7 +118,7 @@ describe("api.v1.reports.$key — authorization", () => { describe("reportQueryTables — scope derivation from the registry", () => { const registry: Record = { - health: { tables: ["runs", "env_metrics", "queue_metrics"] }, + health: { tables: ["runs", "env_metrics", "concurrency_metrics"] }, narrow: { tables: ["runs"] }, }; @@ -127,7 +127,11 @@ describe("reportQueryTables — scope derivation from the registry", () => { }); it("still gives the wider report all of its tables", () => { - expect(reportQueryTables("health", registry)).toEqual(["runs", "env_metrics", "queue_metrics"]); + expect(reportQueryTables("health", registry)).toEqual([ + "runs", + "env_metrics", + "concurrency_metrics", + ]); }); it("returns no tables for an unknown key", () => { diff --git a/apps/webapp/test/resolveTriggerUri.test.ts b/apps/webapp/test/resolveTriggerUri.test.ts index d394beea591..dbe9baaa1d6 100644 --- a/apps/webapp/test/resolveTriggerUri.test.ts +++ b/apps/webapp/test/resolveTriggerUri.test.ts @@ -4,7 +4,7 @@ import { resolveTriggerUri, type TriggerUriScope } from "~/services/resolveTrigg import { v3DeploymentVersionPath, v3ErrorPath, - v3QueuesPath, + concurrencyPath, v3RunPath, v3RunSpanPath, } from "~/utils/pathBuilder"; @@ -58,7 +58,7 @@ describe("resolveTriggerUri", () => { const uri = formatTriggerUri({ kind: "queue", ...uriScope, name: "task/send email" }); expect(resolveTriggerUri(scope, uri)).toEqual({ label: "task/send email", - url: `${v3QueuesPath(org, project, env)}?query=task%2Fsend%20email`, + url: `${concurrencyPath(org, project, env)}?query=task%2Fsend%20email`, }); }); diff --git a/internal-packages/clickhouse/schema/044_add_queue_metrics_total_concurrency.sql b/internal-packages/clickhouse/schema/044_add_queue_metrics_total_concurrency.sql new file mode 100644 index 00000000000..c6ed50b40cb --- /dev/null +++ b/internal-packages/clickhouse/schema/044_add_queue_metrics_total_concurrency.sql @@ -0,0 +1,178 @@ +-- +goose Up + +-- Total-concurrency gauges: total_running is the in-flight count across ALL +-- concurrency-key variants of a queue (the groupConcurrency set), total_limit the +-- RAW stored total cap (0 = none, readers clamp against max_env_limit). Emitted on +-- base-queue gauge rows only. Per-key gauge rows carry the queue concurrency +-- limit that applied in queue_limit, surfaced in the ck tier as max_limit +-- (1000000 = no explicit limit). + +ALTER TABLE trigger_dev.queue_metrics_raw_v1 + ADD COLUMN IF NOT EXISTS total_running UInt32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS total_limit UInt32 DEFAULT 0; + +ALTER TABLE trigger_dev.queue_metrics_v1 + ADD COLUMN IF NOT EXISTS max_total_running SimpleAggregateFunction(max, UInt32), + ADD COLUMN IF NOT EXISTS max_total_limit SimpleAggregateFunction(max, UInt32); + +ALTER TABLE trigger_dev.queue_metrics_5m_v1 + ADD COLUMN IF NOT EXISTS max_total_running SimpleAggregateFunction(max, UInt32), + ADD COLUMN IF NOT EXISTS max_total_limit SimpleAggregateFunction(max, UInt32); + +ALTER TABLE trigger_dev.queue_metrics_ck_v1 + ADD COLUMN IF NOT EXISTS max_limit SimpleAggregateFunction(max, UInt32); + +-- Materialized views cannot be altered: recreate them with the new columns. The 5m +-- MV MUST keep reading raw, never cascade off queue_metrics_v1 (out-of-time-order +-- deltaSumTimestamp merges double-count bridging spans). + +DROP VIEW IF EXISTS trigger_dev.queue_metrics_mv_v1; +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_mv_v1 +TO trigger_dev.queue_metrics_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, + toStartOfInterval(event_time, INTERVAL 10 SECOND) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue' AND concurrency_key = '') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started' AND concurrency_key = '') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack' AND concurrency_key = '') AS ack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'nack' AND concurrency_key = '') AS nack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'dlq' AND concurrency_key = '') AS dlq_delta, + sum(throttled) AS throttled_count, + max(queued) AS max_queued, + max(running) AS max_running, + max(queue_limit) AS max_limit, + max(env_queued) AS max_env_queued, + max(env_running) AS max_env_running, + max(env_limit) AS max_env_limit, + max(ck_backlogged) AS max_ck_backlogged, + max(ck_max_wait_ms) AS max_ck_wait_ms, + max(total_running) AS max_total_running, + max(total_limit) AS max_total_limit, + sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, + quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles +FROM trigger_dev.queue_metrics_raw_v1 +GROUP BY organization_id, project_id, environment_id, queue_name, bucket_start; + +DROP VIEW IF EXISTS trigger_dev.queue_metrics_5m_mv_v1; +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_5m_mv_v1 +TO trigger_dev.queue_metrics_5m_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, + toStartOfInterval(event_time, INTERVAL 5 MINUTE) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue' AND concurrency_key = '') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started' AND concurrency_key = '') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack' AND concurrency_key = '') AS ack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'nack' AND concurrency_key = '') AS nack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'dlq' AND concurrency_key = '') AS dlq_delta, + sum(throttled) AS throttled_count, + max(queued) AS max_queued, + max(running) AS max_running, + max(queue_limit) AS max_limit, + max(env_queued) AS max_env_queued, + max(env_running) AS max_env_running, + max(env_limit) AS max_env_limit, + max(ck_backlogged) AS max_ck_backlogged, + max(ck_max_wait_ms) AS max_ck_wait_ms, + max(total_running) AS max_total_running, + max(total_limit) AS max_total_limit, + sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, + quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles +FROM trigger_dev.queue_metrics_raw_v1 +GROUP BY organization_id, project_id, environment_id, queue_name, bucket_start; + +DROP VIEW IF EXISTS trigger_dev.queue_metrics_ck_mv_v1; +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_ck_mv_v1 +TO trigger_dev.queue_metrics_ck_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, concurrency_key, + toStartOfInterval(event_time, INTERVAL 10 SECOND) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack') AS ack_delta, + maxIf(queued, op = 'gauge') AS max_queued, + maxIf(running, op = 'gauge') AS max_running, + maxIf(queue_limit, op = 'gauge') AS max_limit, + sumIf(wait_ms, op = 'started') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0) AS wait_ms_count +FROM trigger_dev.queue_metrics_raw_v1 +WHERE concurrency_key != '' +GROUP BY organization_id, project_id, environment_id, queue_name, concurrency_key, bucket_start; + +-- +goose Down +DROP VIEW IF EXISTS trigger_dev.queue_metrics_ck_mv_v1; +DROP VIEW IF EXISTS trigger_dev.queue_metrics_5m_mv_v1; +DROP VIEW IF EXISTS trigger_dev.queue_metrics_mv_v1; +ALTER TABLE trigger_dev.queue_metrics_ck_v1 DROP COLUMN IF EXISTS max_limit; +ALTER TABLE trigger_dev.queue_metrics_5m_v1 DROP COLUMN IF EXISTS max_total_running, DROP COLUMN IF EXISTS max_total_limit; +ALTER TABLE trigger_dev.queue_metrics_v1 DROP COLUMN IF EXISTS max_total_running, DROP COLUMN IF EXISTS max_total_limit; +ALTER TABLE trigger_dev.queue_metrics_raw_v1 DROP COLUMN IF EXISTS total_running, DROP COLUMN IF EXISTS total_limit; + +-- Recreate the pre-044 materialized views (the definitions from 036) so ingestion keeps +-- feeding every aggregate table after a rollback. +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_mv_v1 +TO trigger_dev.queue_metrics_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, + toStartOfInterval(event_time, INTERVAL 10 SECOND) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue' AND concurrency_key = '') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started' AND concurrency_key = '') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack' AND concurrency_key = '') AS ack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'nack' AND concurrency_key = '') AS nack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'dlq' AND concurrency_key = '') AS dlq_delta, + sum(throttled) AS throttled_count, + max(queued) AS max_queued, + max(running) AS max_running, + max(queue_limit) AS max_limit, + max(env_queued) AS max_env_queued, + max(env_running) AS max_env_running, + max(env_limit) AS max_env_limit, + max(ck_backlogged) AS max_ck_backlogged, + max(ck_max_wait_ms) AS max_ck_wait_ms, + sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, + quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles +FROM trigger_dev.queue_metrics_raw_v1 +GROUP BY organization_id, project_id, environment_id, queue_name, bucket_start; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_5m_mv_v1 +TO trigger_dev.queue_metrics_5m_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, + toStartOfInterval(event_time, INTERVAL 5 MINUTE) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue' AND concurrency_key = '') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started' AND concurrency_key = '') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack' AND concurrency_key = '') AS ack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'nack' AND concurrency_key = '') AS nack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'dlq' AND concurrency_key = '') AS dlq_delta, + sum(throttled) AS throttled_count, + max(queued) AS max_queued, + max(running) AS max_running, + max(queue_limit) AS max_limit, + max(env_queued) AS max_env_queued, + max(env_running) AS max_env_running, + max(env_limit) AS max_env_limit, + max(ck_backlogged) AS max_ck_backlogged, + max(ck_max_wait_ms) AS max_ck_wait_ms, + sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, + quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles +FROM trigger_dev.queue_metrics_raw_v1 +GROUP BY organization_id, project_id, environment_id, queue_name, bucket_start; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_ck_mv_v1 +TO trigger_dev.queue_metrics_ck_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, concurrency_key, + toStartOfInterval(event_time, INTERVAL 10 SECOND) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack') AS ack_delta, + maxIf(queued, op = 'gauge') AS max_queued, + maxIf(running, op = 'gauge') AS max_running, + sumIf(wait_ms, op = 'started') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0) AS wait_ms_count +FROM trigger_dev.queue_metrics_raw_v1 +WHERE concurrency_key != '' +GROUP BY organization_id, project_id, environment_id, queue_name, concurrency_key, bucket_start; diff --git a/internal-packages/clickhouse/src/queueMetrics.ts b/internal-packages/clickhouse/src/queueMetrics.ts index 39576b4a0a3..aa3cf5296d2 100644 --- a/internal-packages/clickhouse/src/queueMetrics.ts +++ b/internal-packages/clickhouse/src/queueMetrics.ts @@ -21,6 +21,8 @@ export const QueueMetricsRawV1Input = z.object({ throttled: z.number().optional(), ck_backlogged: z.number().optional(), ck_max_wait_ms: z.number().optional(), + total_running: z.number().optional(), + total_limit: z.number().optional(), wait_ms: z.number().optional(), cumulative: z.number().optional(), }); diff --git a/internal-packages/metrics-pipeline/src/consumer.test.ts b/internal-packages/metrics-pipeline/src/consumer.test.ts index 672fa426999..f9f59335249 100644 --- a/internal-packages/metrics-pipeline/src/consumer.test.ts +++ b/internal-packages/metrics-pipeline/src/consumer.test.ts @@ -43,6 +43,7 @@ redisTest( }); await consumer.start(); + await emitter.waitUntilReady(); emitter.emit("queueA", { op: "enqueue", q: "queueA" }); emitter.emit("queueB", { op: "started", q: "queueB", wait: 42 }); @@ -156,6 +157,7 @@ redisTest( }); await consumer.start(); + await emitter.waitUntilReady(); emitter.emit(a, { op: "enqueue", q: a }); emitter.emit(b, { op: "enqueue", q: b }); await waitFor(() => inserted.flatMap((i) => i.rows).length >= 2); diff --git a/internal-packages/metrics-pipeline/src/lua.ts b/internal-packages/metrics-pipeline/src/lua.ts index 64f3b896c0d..701f608308a 100644 --- a/internal-packages/metrics-pipeline/src/lua.ts +++ b/internal-packages/metrics-pipeline/src/lua.ts @@ -17,6 +17,10 @@ export type GaugeComputeLuaParams = { // CK-health extras (both or neither): appended as an optional gauge tail, gauge[8]/gauge[9]. ckBacklogged?: string; ckMaxWaitMs?: string; + // Total-concurrency extras (both or neither, and only with the CK extras): appended as + // gauge[10]/gauge[11]. totalLimit is the RAW stored limit (0 = none); readers clamp. + totalRunning?: string; + totalLimit?: string; }; // Computes an op=gauge snapshot into the enclosing script's `__qm_g` local (a flat @@ -26,11 +30,21 @@ export type GaugeComputeLuaParams = { export function createMetricsGaugeComputeLua(params: GaugeComputeLuaParams): string { const throttled = params.throttledExpr ?? "__cc >= __lim and __ql > 0"; const hasCk = params.ckBacklogged != null && params.ckMaxWaitMs != null; - const gauge = hasCk + const hasTotal = params.totalRunning != null && params.totalLimit != null; + if (hasTotal && !hasCk) { + throw new Error("gauge totalRunning/totalLimit extras require the CK extras"); + } + const gauge = hasTotal ? ` local __ckq = tonumber(${params.ckBacklogged}) or 0 local __ckw = tonumber(${params.ckMaxWaitMs}) or 0 + local __tcc = tonumber(${params.totalRunning}) or 0 + local __tlim = tonumber(${params.totalLimit}) or 0 + __qm_g = {__ql, __cc, __lim, __eql, __ec, __elim, __thr, __ckq, __ckw, __tcc, __tlim}` + : hasCk + ? ` local __ckq = tonumber(${params.ckBacklogged}) or 0 + local __ckw = tonumber(${params.ckMaxWaitMs}) or 0 __qm_g = {__ql, __cc, __lim, __eql, __ec, __elim, __thr, __ckq, __ckw}` - : ` __qm_g = {__ql, __cc, __lim, __eql, __ec, __elim, __thr}`; + : ` __qm_g = {__ql, __cc, __lim, __eql, __ec, __elim, __thr}`; return ` if ${params.enabledArg} then diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index 40e3d7bc336..85b9441efbb 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -1748,6 +1748,27 @@ export class RunEngine { return this.runQueue.currentConcurrencyOfQueues(environment, queues); } + async totalConcurrencyOfQueues( + environment: MinimalAuthenticatedEnvironment, + queues: string[] + ): Promise> { + return this.runQueue.totalConcurrencyOfQueues(environment, queues); + } + + async totalConcurrencyLimitsOfQueues( + environment: MinimalAuthenticatedEnvironment, + queues: string[] + ): Promise> { + return this.runQueue.totalConcurrencyLimitsOfQueues(environment, queues); + } + + async gateQueuedCountOfQueues( + environment: MinimalAuthenticatedEnvironment, + queues: string[] + ): Promise> { + return this.runQueue.gateQueuedCountOfQueues(environment, queues); + } + async concurrencyKeyBreakdown( environment: MinimalAuthenticatedEnvironment, queue: string, @@ -2987,6 +3008,7 @@ export class RunEngine { { select: { queue: true, + concurrencyKey: true, }, }, this.prisma @@ -3008,6 +3030,7 @@ export class RunEngine { runId, orgId: latestSnapshot.organizationId, queue: taskRun.queue, + concurrencyKey: taskRun.concurrencyKey ?? undefined, env: { id: latestSnapshot.environmentId, type: latestSnapshot.environmentType, diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 1dcbcb33a88..bc03d61f301 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -78,6 +78,10 @@ const SemanticAttributes = { * during a rolling upgrade), including runs parked in the DLQ or suspended on * checkpoints, which older rules based on the queue zset could never prune. */ +/** TTL for gate queued counters: refreshed on every delta and on every read, so an + * active or observed gate never re-anchors; the Lua helper inlines the same value. */ +const GATE_QUEUED_COUNTER_TTL_SECONDS = 86400; + const QUEUE_GATES_LUA_HELPERS = ` local function __gateKeys(gatesKeyPrefix, msg, gate) local base = gatesKeyPrefix .. '{org:' .. msg.orgId .. '}:proj:' .. msg.projectId .. ':env:' .. msg.environmentId .. ':queue:' .. gate.queue @@ -116,18 +120,12 @@ local function __gateReconcile(setKey, msgKeyPrefix, reconcileKeyPrefix) end end -local function __gatesHaveCapacity(gatesKeyPrefix, msg, messageId, envLimit, msgKeyPrefix, ckOverridesEnabled) +local function __gatesHaveCapacity(gatesKeyPrefix, msg, messageId, envLimit, msgKeyPrefix) if not msg.gates then return true end for _, gate in ipairs(msg.gates) do local base, variant, gateKey = __gateKeys(gatesKeyPrefix, msg, gate) local occupancy = tonumber(redis.call('SCARD', variant .. ':currentConcurrency') or '0') local perKeyLimit = math.min(tonumber(redis.call('GET', base .. ':concurrency') or '1000000'), envLimit) - if ckOverridesEnabled and gateKey and gateKey ~= '' then - local gateOverride = redis.call('HGET', base .. ':ckLimits', string.sub(variant, #gatesKeyPrefix + 1)) - if gateOverride then - perKeyLimit = math.min(tonumber(gateOverride), envLimit) - end - end if occupancy >= perKeyLimit and redis.call('SISMEMBER', variant .. ':currentConcurrency', messageId) == 0 then __gateReconcile(variant .. ':currentConcurrency', msgKeyPrefix, gatesKeyPrefix) return false @@ -166,6 +164,43 @@ local function __gatesRelease(gatesKeyPrefix, rawPayload, messageId) redis.call('SREM', base .. ':groupConcurrency', messageId) end end +end + +-- Per-gate queued counter: runs that are queued and must clear the gate to execute. +-- Callers gate the delta on the actual queue-zset transition (ZADD added == 1 / +-- ZREM removed == 1) so re-enqueues and already-removed members never double count; +-- gates sharing a base (duplicate entries, key variants) count once per run. The +-- 24h TTL refreshes on every delta, so an ACTIVE gate's count never resets while +-- drift from delta-less paths (a mixed-version rollout, a stale-entry cleanup) +-- clears once the gate has been quiet for a day. The residual gap is a gate idle +-- for 24h with runs still queued (e.g. paused with no new enqueues): its counter +-- expires and under-counts until the backlog fully drains. Payload-driven and +-- flag-independent, like release, so counts stay exact across flag flips. +local function __gateQueuedDelta(gatesKeyPrefix, msg, delta) + if type(msg) ~= 'table' or not msg.gates then return end + local seenBases = {} + for _, gate in ipairs(msg.gates) do + local base = __gateKeys(gatesKeyPrefix, msg, gate) + if not seenBases[base] then + seenBases[base] = true + local counterKey = base .. ':gateQueuedCounter' + if delta > 0 then + redis.call('INCRBY', counterKey, delta) + redis.call('EXPIRE', counterKey, '86400') + elseif tonumber(redis.call('GET', counterKey) or '0') > 0 then + redis.call('DECRBY', counterKey, -delta) + redis.call('EXPIRE', counterKey, '86400') + end + end + end +end + +local function __gateQueuedDeltaRaw(gatesKeyPrefix, rawPayload, delta) + if not rawPayload or rawPayload == false then return end + if not string.find(rawPayload, '"gates"', 1, true) then return end + local ok, msg = pcall(cjson.decode, rawPayload) + if not ok then return end + __gateQueuedDelta(gatesKeyPrefix, msg, delta) end`; // Prelude spliced at the top of every gauge-carrying script: declares the gauge slot and @@ -176,9 +211,26 @@ const QUEUE_METRICS_GAUGE_PRELUDE = ` local __qm_g = false local function __qmret(r) if r == nil then r = false end return {r, __qm_g} end`; +/** Total-concurrency tail (gauge[10]/gauge[11]): live group cardinality + raw stored cap. + * Requires the groupConcurrencyKey local and the __totalLimitRaw memo (one GET shared with + * the total-cap gate); every script that runs a gauge with this tail declares both. The + * group SCARD stays a fresh read: it must be post-admission. */ +const QUEUE_METRICS_TOTAL_GAUGE_EXTRAS = { + totalRunning: "redis.call('SCARD', groupConcurrencyKey)", + totalLimit: "__totalLimitRaw() or '0'", +}; + +/** The gauge layout is positional (totals ride behind the CK slots), so the plain + * scripts zero-fill the CK health fields: a base queue has no CK variants to backlog. */ +const QUEUE_METRICS_PLAIN_CK_ZERO_EXTRAS = { + ckBacklogged: "0", + ckMaxWaitMs: "0", +}; + // Fresh-read gauge for splice points with no reusable locals: enqueue slow-path (before -// return 0) and the base dequeue top. Gated on the last ARGV so it is inert unless the -// caller opts in. CK queues emit per-subqueue depth (queue_name aggregates via the MV). +// return 0) and the base dequeue's sample-at-return wrapper. Gated on the last ARGV so it +// is inert unless the caller opts in. CK queues emit per-subqueue depth (queue_name +// aggregates via the MV). const QUEUE_METRICS_GAUGE_LUA = createMetricsGaugeComputeLua({ enabledArg: "ARGV[#ARGV] == '1'", queued: "redis.call('ZCARD', queueKey)", @@ -187,6 +239,8 @@ const QUEUE_METRICS_GAUGE_LUA = createMetricsGaugeComputeLua({ envQueued: "redis.call('ZCARD', envQueueKey)", envRunning: "redis.call('SCARD', envCurrentConcurrencyKey)", envLimit: "redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit", + ...QUEUE_METRICS_PLAIN_CK_ZERO_EXTRAS, + ...QUEUE_METRICS_TOTAL_GAUGE_EXTRAS, }); // Enqueue fast-path gauge: the admission check already computed queueCurrent/envCurrent/ @@ -200,6 +254,8 @@ const QUEUE_METRICS_ENQUEUE_FASTPATH_GAUGE_LUA = createMetricsGaugeComputeLua({ envQueued: "redis.call('ZCARD', envQueueKey)", envRunning: "envCurrent", envLimit: "envLimit", + ...QUEUE_METRICS_PLAIN_CK_ZERO_EXTRAS, + ...QUEUE_METRICS_TOTAL_GAUGE_EXTRAS, }); // CK-health extras: distinct backlogged keys + most-starved head-of-line wait (ckIndex scores @@ -222,6 +278,7 @@ const QUEUE_METRICS_CK_ENQUEUE_GAUGE_LUA = createMetricsGaugeComputeLua({ envRunning: "redis.call('SCARD', envCurrentConcurrencyKey)", envLimit: "redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit", ...QUEUE_METRICS_CK_GAUGE_EXTRAS, + ...QUEUE_METRICS_TOTAL_GAUGE_EXTRAS, }); const QUEUE_METRICS_CK_ENQUEUE_FASTPATH_GAUGE_LUA = createMetricsGaugeComputeLua({ @@ -233,6 +290,7 @@ const QUEUE_METRICS_CK_ENQUEUE_FASTPATH_GAUGE_LUA = createMetricsGaugeComputeLua envRunning: "envCurrent", envLimit: "envLimit", ...QUEUE_METRICS_CK_GAUGE_EXTRAS, + ...QUEUE_METRICS_TOTAL_GAUGE_EXTRAS, }); // CK dequeue: depth/running from the per-base-queue aggregate counters the run-queue already @@ -248,6 +306,7 @@ const QUEUE_METRICS_CK_DEQUEUE_GAUGE_LUA = createMetricsGaugeComputeLua({ envLimit: "redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit", throttledExpr: "false", ...QUEUE_METRICS_CK_GAUGE_EXTRAS, + ...QUEUE_METRICS_TOTAL_GAUGE_EXTRAS, }); /** Injected queue-metrics stream emitter; all calls are no-ops when metrics are disabled. */ @@ -261,13 +320,6 @@ export interface RunQueueMetricsEmitter { emitGauge(shardKey: string, fields: Record): void; } -export class RunQueueConcurrencyKeyLimitExceededError extends Error { - constructor(message: string) { - super(message); - this.name = "RunQueueConcurrencyKeyLimitExceededError"; - } -} - export type RunQueueOptions = { name: string; tracer: Tracer; @@ -297,9 +349,11 @@ export type RunQueueOptions = { */ counterTtlSeconds?: number; /** - * When true, concurrency-keyed queues maintain a per-base-queue groupConcurrency SET - * (total in-flight across all key variants) and enforce the queue's total concurrency - * limit at admit time. Default false: admit paths are byte-identical to before, and + * When true, queues maintain a per-base-queue groupConcurrency SET (total in-flight + * across all key variants AND keyless runs) and enforce the queue's total concurrency + * limit at admit time. V2 concurrency semantics (a limit's `total` bound, including + * total-only declarations) are enforced solely through this flag: with it off, a + * total-only limit caps nothing. Default false: admit paths are byte-identical to before, and * only the release-side SREM mirror runs (a no-op on an absent set), so the flag can * be flipped on a fleet that has fully rolled onto this build without draining queues. * @@ -316,10 +370,6 @@ export type RunQueueOptions = { * that dead-lettered or suspended through a mirror-less path. Enabling only after * every instance runs this build avoids the noise but is no longer load-bearing * for correctness. - * - * Per-concurrency-key limit overrides are part of the same concurrency-limits - * feature and are deliberately enforced behind this flag too: writes are always - * accepted and durable, and enforcement of both arrives together. */ totalConcurrencyEnabled?: boolean; /** @@ -331,8 +381,6 @@ export type RunQueueOptions = { * the total cap covering releases from builds without the mirror. */ gatesEnabled?: boolean; - /** Cap on per-concurrency-key limit overrides stored per queue. Default 1000. */ - maxConcurrencyKeyOverridesPerQueue?: number; workerOptions?: { pollIntervalMs?: number; immediatePollIntervalMs?: number; @@ -446,7 +494,6 @@ export class RunQueue { private queueSelectionStrategy: RunQueueSelectionStrategy; private shardCount: number; private counterTtlSeconds: number; - private maxConcurrencyKeyOverridesPerQueue: number; private abortController: AbortController; private worker: Worker; private workerQueueResolver: WorkerQueueResolver; @@ -457,7 +504,6 @@ export class RunQueue { constructor(public readonly options: RunQueueOptions) { this.shardCount = options.shardCount ?? 2; this.counterTtlSeconds = options.counterTtlSeconds ?? 86400; - this.maxConcurrencyKeyOverridesPerQueue = options.maxConcurrencyKeyOverridesPerQueue ?? 1000; this.retryOptions = options.retryOptions ?? defaultRetrySettings; this.redis = createRedisClient(options.redis, { onError: (error) => { @@ -642,7 +688,7 @@ export class RunQueue { } /** - * Total in-flight runs across all concurrency-key variants of a queue (the + * Total in-flight runs on a queue, keyed and keyless together (the * groupConcurrency SET cardinality). Admits only populate the set while * totalConcurrencyEnabled is on. After the flag is turned off the set drains * to zero through the release-side mirrors, so a nonzero read reflects real @@ -653,59 +699,81 @@ export class RunQueue { } /** - * Sets a per-concurrency-key limit override for a queue. The stored value is the - * raw requested limit; admit paths clamp to the environment limit at read time. - * Throws RunQueueConcurrencyKeyLimitExceededError when a NEW key would push the - * queue past maxConcurrencyKeyOverridesPerQueue (updates to existing keys always - * succeed). + * Runs that are queued and must clear this gate queue to execute (the per-gate + * queued counter): incremented per gate on enqueue and decremented on admit and + * on every queued-removal path, floored at zero. Reads refresh the counter's + * TTL, so a gate anyone observes (dashboard, API) never expires while idle — + * e.g. a paused limit with a stalled backlog keeps its count over a quiet + * weekend; only gates nobody touches or reads for a day re-anchor. */ - public async updateQueueConcurrencyKeyLimit( + public async gateQueuedCountOfQueue(env: MinimalAuthenticatedEnvironment, queue: string) { + const counts = await this.gateQueuedCountOfQueues(env, [queue]); + return counts[queue] ?? 0; + } + + /** Batch variant: one pipeline of GETs, each with a TTL refresh. */ + public async gateQueuedCountOfQueues( env: MinimalAuthenticatedEnvironment, - queue: string, - concurrencyKey: string, - limit: number - ) { - const result = await this.redis.setQueueConcurrencyKeyLimit( - this.keys.queueCkLimitsKey(env, queue), - this.keys.queueKey(env, queue, concurrencyKey), - String(limit), - String(this.maxConcurrencyKeyOverridesPerQueue) - ); + queues: string[] + ): Promise> { + const pipeline = this.redis.pipeline(); + queues.forEach((queue) => { + const key = this.keys.gateQueuedCounterKey(env, queue); + pipeline.get(key); + pipeline.expire(key, GATE_QUEUED_COUNTER_TTL_SECONDS, "XX"); + }); - if (result === 0) { - throw new RunQueueConcurrencyKeyLimitExceededError( - `Cannot add a concurrency key override to queue ${queue}: the queue already has ${this.maxConcurrencyKeyOverridesPerQueue} overrides` - ); - } + const results = await pipeline.exec(); + + return queues.reduce( + (acc, queue, index) => { + const value = results?.[index * 2]?.[1]; + const parsed = typeof value === "string" ? Number(value) : 0; + acc[queue] = Number.isFinite(parsed) ? Math.max(parsed, 0) : 0; + return acc; + }, + {} as Record + ); } - public async removeQueueConcurrencyKeyLimit( + /** Batch variant of totalConcurrencyOfQueue: one pipeline of group SCARDs. */ + public async totalConcurrencyOfQueues( env: MinimalAuthenticatedEnvironment, - queue: string, - concurrencyKey: string - ) { - return this.redis.hdel( - this.keys.queueCkLimitsKey(env, queue), - this.keys.queueKey(env, queue, concurrencyKey) + queues: string[] + ): Promise> { + const pipeline = this.redis.pipeline(); + queues.forEach((queue) => { + pipeline.scard(this.keys.queueGroupConcurrencyKey(env, queue)); + }); + + const results = await pipeline.exec(); + + return queues.reduce( + (acc, queue, index) => { + const value = results?.[index]?.[1]; + acc[queue] = typeof value === "number" ? value : 0; + return acc; + }, + {} as Record ); } - /** Returns the raw per-concurrency-key limit overrides for a queue, keyed by concurrency key value. */ - public async getQueueConcurrencyKeyLimits( + /** Batch read of the RAW stored total concurrency limits (undefined = no cap). */ + public async totalConcurrencyLimitsOfQueues( env: MinimalAuthenticatedEnvironment, - queue: string - ): Promise> { - const raw = await this.redis.hgetall(this.keys.queueCkLimitsKey(env, queue)); + queues: string[] + ): Promise> { + const keys = queues.map((queue) => this.keys.queueTotalConcurrencyLimitKey(env, queue)); + const values = keys.length > 0 ? await this.redis.mget(...keys) : []; - const limits: Record = {}; - for (const [variantName, value] of Object.entries(raw)) { - const ckIndex = variantName.indexOf(":ck:"); - if (ckIndex === -1) { - continue; - } - limits[variantName.slice(ckIndex + 4)] = Number(value); - } - return limits; + return queues.reduce( + (acc, queue, index) => { + const value = values[index]; + acc[queue] = value != null ? Number(value) : undefined; + return acc; + }, + {} as Record + ); } public async updateEnvConcurrencyLimits(env: MinimalAuthenticatedEnvironment) { @@ -1510,6 +1578,7 @@ export class RunQueue { this.keys.queueCurrentDequeuedKeyFromQueue(message.queue), this.keys.envCurrentDequeuedKeyFromQueue(message.queue), this.keys.messageKey(message.orgId, messageId), + this.keys.queueGroupConcurrencyKeyFromQueue(message.queue), messageId, this.options.redis.keyPrefix ?? "" ); @@ -1568,6 +1637,7 @@ export class RunQueue { runId: string; orgId: string; queue: string; + concurrencyKey?: string; env: RunQueueKeyProducerEnvironment; }) { return this.#callClearMessageFromConcurrencySets(params); @@ -2355,6 +2425,10 @@ export class RunQueue { fields.ckq = ckq; fields.ckw = ckw; } + if (gauge.length >= 11) { + fields.tcc = gauge[9]; + fields.tlim = gauge[10]; + } this.options.queueMetrics?.emitGauge(queue, fields); } @@ -2457,7 +2531,6 @@ export class RunQueue { const totalConcurrencyLimitKey = this.keys.queueTotalConcurrencyLimitKeyFromQueue( message.queue ); - const ckLimitsKey = this.keys.queueCkLimitsKeyFromQueue(message.queue); const totalConcurrencyEnabledArg = this.options.totalConcurrencyEnabled ? "1" : "0"; if (ttlInfo) { @@ -2481,7 +2554,6 @@ export class RunQueue { baseQueueKey, groupConcurrencyKey, totalConcurrencyLimitKey, - ckLimitsKey, // args queueName, messageId, @@ -2521,7 +2593,6 @@ export class RunQueue { baseQueueKey, groupConcurrencyKey, totalConcurrencyLimitKey, - ckLimitsKey, // args queueName, messageId, @@ -2557,6 +2628,8 @@ export class RunQueue { queueConcurrencyLimitKey, envConcurrencyLimitKey, envConcurrencyLimitBurstFactorKey, + this.keys.queueGroupConcurrencyKeyFromQueue(message.queue), + this.keys.queueTotalConcurrencyLimitKeyFromQueue(message.queue), // args queueName, messageId, @@ -2589,6 +2662,8 @@ export class RunQueue { queueConcurrencyLimitKey, envConcurrencyLimitKey, envConcurrencyLimitBurstFactorKey, + this.keys.queueGroupConcurrencyKeyFromQueue(message.queue), + this.keys.queueTotalConcurrencyLimitKeyFromQueue(message.queue), // args queueName, messageId, @@ -2675,6 +2750,8 @@ export class RunQueue { envQueueKey, masterQueueKey, ttlQueueKey, + this.keys.queueGroupConcurrencyKeyFromQueue(messageQueue), + this.keys.queueTotalConcurrencyLimitKeyFromQueue(messageQueue), //args messageQueue, String(Date.now()), @@ -2812,7 +2889,6 @@ export class RunQueue { runningCounterKey, this.keys.queueGroupConcurrencyKeyFromQueue(ckWildcardQueue), this.keys.queueTotalConcurrencyLimitKeyFromQueue(ckWildcardQueue), - this.keys.queueCkLimitsKeyFromQueue(ckWildcardQueue), //args ckWildcardQueue, String(Date.now()), @@ -3091,6 +3167,7 @@ export class RunQueue { envCurrentDequeuedKey, envQueueKey, workerQueueKey, + this.keys.queueGroupConcurrencyKeyFromQueue(message.queue), messageId, messageQueue, messageKeyValue, @@ -3103,18 +3180,31 @@ export class RunQueue { runId, orgId, queue, + concurrencyKey, env, }: { runId: string; orgId: string; queue: string; + concurrencyKey?: string; env: RunQueueKeyProducerEnvironment; }) { const messageId = runId; const messageKey = this.keys.messageKey(orgId, messageId); - const queueCurrentConcurrencyKey = this.keys.queueCurrentConcurrencyKey(env, queue); + /** + * Callers pass the bare TaskRun queue name plus its concurrencyKey; the run's + * slots live on the ck variant. Both variants mirror the per-base-queue group + * set (keyed and keyless admits populate it); the tracked clear additionally + * maintains the counters that only keyed queues keep. + */ + const fullQueue = concurrencyKey ? this.keys.queueKey(env, queue, concurrencyKey) : queue; + const queueCurrentConcurrencyKey = this.keys.queueCurrentConcurrencyKey( + env, + queue, + concurrencyKey + ); const envCurrentConcurrencyKey = this.keys.envCurrentConcurrencyKey(env); - const queueCurrentDequeuedKey = this.keys.queueCurrentDequeuedKey(env, queue); + const queueCurrentDequeuedKey = this.keys.queueCurrentDequeuedKey(env, queue, concurrencyKey); const envCurrentDequeuedKey = this.keys.envCurrentDequeuedKey(env); this.logger.debug("Calling clearMessageFromConcurrencySets", { @@ -3129,15 +3219,15 @@ export class RunQueue { service: this.name, }); - if (queue.includes(":ck:")) { + if (fullQueue.includes(":ck:")) { return this.redis.clearMessageFromConcurrencySetsTracked( queueCurrentConcurrencyKey, envCurrentConcurrencyKey, queueCurrentDequeuedKey, envCurrentDequeuedKey, - this.keys.queueRunningCounterKeyFromQueue(queue), - this.keys.ckIndexKeyFromQueue(queue), - this.keys.queueGroupConcurrencyKeyFromQueue(queue), + this.keys.queueRunningCounterKeyFromQueue(fullQueue), + this.keys.ckIndexKeyFromQueue(fullQueue), + this.keys.queueGroupConcurrencyKeyFromQueue(fullQueue), messageKey, messageId, this.options.redis.keyPrefix ?? "", @@ -3151,6 +3241,7 @@ export class RunQueue { queueCurrentDequeuedKey, envCurrentDequeuedKey, messageKey, + this.keys.queueGroupConcurrencyKey(env, queue), messageId, this.options.redis.keyPrefix ?? "" ); @@ -3228,6 +3319,7 @@ export class RunQueue { queueCurrentDequeuedKey, envCurrentDequeuedKey, envQueueKey, + this.keys.queueGroupConcurrencyKeyFromQueue(message.queue), //args messageId, messageQueue, @@ -3289,6 +3381,7 @@ export class RunQueue { envCurrentDequeuedKey, envQueueKey, deadLetterQueueKey, + this.keys.queueGroupConcurrencyKeyFromQueue(message.queue), messageId, messageQueue, this.options.redis.keyPrefix ?? "" @@ -3653,7 +3746,7 @@ end // When enableFastPath == '0', the script skips the fast-path check entirely and behaves // identically to the pre-fast-path version (with the addition of returning 0). this.redis.defineCommand("enqueueMessage", { - numberOfKeys: 12, + numberOfKeys: 14, lua: ` local masterQueueKey = KEYS[1] local queueKey = KEYS[2] @@ -3668,6 +3761,16 @@ local workerQueueKey = KEYS[9] local queueConcurrencyLimitKey = KEYS[10] local envConcurrencyLimitKey = KEYS[11] local envConcurrencyLimitBurstFactorKey = KEYS[12] +-- Total-cap keys (KEYS 13-14) +local groupConcurrencyKey = KEYS[13] +local totalConcurrencyLimitKey = KEYS[14] +local __rawTotalLimit = nil +local function __totalLimitRaw() + if __rawTotalLimit == nil then + __rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) or false + end + return __rawTotalLimit +end local queueName = ARGV[1] local messageId = ARGV[2] @@ -3703,20 +3806,37 @@ if enableFastPath == '1' then ) if queueCurrent < queueLimit then + -- Total-cap gate: a fast-path admit consumes a group slot, so it must + -- respect the env-clamped total limit. At the cap we fall through to the + -- slow path (the message queues; the dequeue gate holds it). + local totalAllowsFastPath = true + if totalConcurrencyEnabled then + local rawTotalLimit = __totalLimitRaw() + if rawTotalLimit then + local totalLimit = math.min(tonumber(rawTotalLimit), envLimit) + if tonumber(redis.call('SCARD', groupConcurrencyKey) or '0') >= totalLimit then + totalAllowsFastPath = false + end + end + end + local gateMsg = nil local gatesAllowFastPath = true if gatesEnabled and string.find(messageData, '"gates"', 1, true) then local okDecode, decoded = pcall(cjson.decode, messageData) if okDecode and type(decoded) == 'table' and decoded.gates then gateMsg = decoded - gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil, totalConcurrencyEnabled) + gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil) end end - if gatesAllowFastPath then + if totalAllowsFastPath and gatesAllowFastPath then redis.call('SET', messageKey, messageData) redis.call('SADD', queueCurrentConcurrencyKey, messageId) redis.call('SADD', envCurrentConcurrencyKey, messageId) + if totalConcurrencyEnabled then + redis.call('SADD', groupConcurrencyKey, messageId) + end if gateMsg then __gatesAcquire(keyPrefix, gateMsg, messageId) end @@ -3734,7 +3854,10 @@ end redis.call('SET', messageKey, messageData) -- Add the message to the queue -redis.call('ZADD', queueKey, messageScore, messageId) +local added = redis.call('ZADD', queueKey, messageScore, messageId) +if added == 1 then + __gateQueuedDeltaRaw(keyPrefix, messageData, 1) +end -- Add the message to the env queue redis.call('ZADD', envQueueKey, messageScore, messageId) @@ -3748,8 +3871,12 @@ else redis.call('ZADD', masterQueueKey, earliestMessage[2], queueName) end --- Update the concurrency keys -redis.call('SREM', queueCurrentConcurrencyKey, messageId) +-- Update the concurrency keys. The groupConcurrency SREM mirrors the base SREM +-- unconditionally (no flag check) so a disabled flag still drains the group set. +local removedFromCurrentConcurrency = redis.call('SREM', queueCurrentConcurrencyKey, messageId) +if removedFromCurrentConcurrency == 1 then + redis.call('SREM', groupConcurrencyKey, messageId) +end redis.call('SREM', envCurrentConcurrencyKey, messageId) redis.call('SREM', queueCurrentDequeuedKey, messageId) redis.call('SREM', envCurrentDequeuedKey, messageId) @@ -3765,7 +3892,7 @@ return __qmret(0) // (scheduled independently before enqueue) handles TTL expiry. This mirrors what // dequeueMessagesFromQueue does: it removes from the TTL set when dequeuing. this.redis.defineCommand("enqueueMessageWithTtl", { - numberOfKeys: 13, + numberOfKeys: 15, lua: ` local masterQueueKey = KEYS[1] local queueKey = KEYS[2] @@ -3781,6 +3908,16 @@ local workerQueueKey = KEYS[10] local queueConcurrencyLimitKey = KEYS[11] local envConcurrencyLimitKey = KEYS[12] local envConcurrencyLimitBurstFactorKey = KEYS[13] +-- Total-cap keys (KEYS 14-15) +local groupConcurrencyKey = KEYS[14] +local totalConcurrencyLimitKey = KEYS[15] +local __rawTotalLimit = nil +local function __totalLimitRaw() + if __rawTotalLimit == nil then + __rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) or false + end + return __rawTotalLimit +end local queueName = ARGV[1] local messageId = ARGV[2] @@ -3818,20 +3955,37 @@ if enableFastPath == '1' then ) if queueCurrent < queueLimit then + -- Total-cap gate: a fast-path admit consumes a group slot, so it must + -- respect the env-clamped total limit. At the cap we fall through to the + -- slow path (the message queues; the dequeue gate holds it). + local totalAllowsFastPath = true + if totalConcurrencyEnabled then + local rawTotalLimit = __totalLimitRaw() + if rawTotalLimit then + local totalLimit = math.min(tonumber(rawTotalLimit), envLimit) + if tonumber(redis.call('SCARD', groupConcurrencyKey) or '0') >= totalLimit then + totalAllowsFastPath = false + end + end + end + local gateMsg = nil local gatesAllowFastPath = true if gatesEnabled and string.find(messageData, '"gates"', 1, true) then local okDecode, decoded = pcall(cjson.decode, messageData) if okDecode and type(decoded) == 'table' and decoded.gates then gateMsg = decoded - gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil, totalConcurrencyEnabled) + gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil) end end - if gatesAllowFastPath then + if totalAllowsFastPath and gatesAllowFastPath then redis.call('SET', messageKey, messageData) redis.call('SADD', queueCurrentConcurrencyKey, messageId) redis.call('SADD', envCurrentConcurrencyKey, messageId) + if totalConcurrencyEnabled then + redis.call('SADD', groupConcurrencyKey, messageId) + end if gateMsg then __gatesAcquire(keyPrefix, gateMsg, messageId) end @@ -3850,7 +4004,10 @@ end redis.call('SET', messageKey, messageData) -- Add the message to the queue -redis.call('ZADD', queueKey, messageScore, messageId) +local added = redis.call('ZADD', queueKey, messageScore, messageId) +if added == 1 then + __gateQueuedDeltaRaw(keyPrefix, messageData, 1) +end -- Add the message to the env queue redis.call('ZADD', envQueueKey, messageScore, messageId) @@ -3867,8 +4024,12 @@ else redis.call('ZADD', masterQueueKey, earliestMessage[2], queueName) end --- Update the concurrency keys -redis.call('SREM', queueCurrentConcurrencyKey, messageId) +-- Update the concurrency keys. The groupConcurrency SREM mirrors the base SREM +-- unconditionally (no flag check) so a disabled flag still drains the group set. +local removedFromCurrentConcurrency = redis.call('SREM', queueCurrentConcurrencyKey, messageId) +if removedFromCurrentConcurrency == 1 then + redis.call('SREM', groupConcurrencyKey, messageId) +end redis.call('SREM', envCurrentConcurrencyKey, messageId) redis.call('SREM', queueCurrentDequeuedKey, messageId) redis.call('SREM', envCurrentDequeuedKey, messageId) @@ -4103,7 +4264,7 @@ return __qmret(0) // *Tracked variants of dequeueMessageFromKey and the ack/nack/dlq/release/clear // scripts. this.redis.defineCommand("enqueueMessageCkTracked", { - numberOfKeys: 18, + numberOfKeys: 17, lua: ` local masterQueueKey = KEYS[1] local queueKey = KEYS[2] @@ -4125,7 +4286,13 @@ local baseQueueKey = KEYS[15] -- Total-cap keys (KEYS 16-17) local groupConcurrencyKey = KEYS[16] local totalConcurrencyLimitKey = KEYS[17] -local ckLimitsKey = KEYS[18] +local __rawTotalLimit = nil +local function __totalLimitRaw() + if __rawTotalLimit == nil then + __rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) or false + end + return __rawTotalLimit +end local queueName = ARGV[1] local messageId = ARGV[2] @@ -4163,12 +4330,6 @@ if enableFastPath == '1' then tonumber(redis.call('GET', queueConcurrencyLimitKey) or '1000000'), envLimit ) - if totalConcurrencyEnabled then - local perKeyOverride = redis.call('HGET', ckLimitsKey, queueName) - if perKeyOverride then - queueLimit = math.min(tonumber(perKeyOverride), envLimit) - end - end if queueCurrent < queueLimit then -- Total-cap gate: a fast-path admit consumes a group slot, so it must @@ -4176,7 +4337,7 @@ if enableFastPath == '1' then -- slow path (the message queues; the dequeue gate holds it). local totalAllowsFastPath = true if totalConcurrencyEnabled then - local rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) + local rawTotalLimit = __totalLimitRaw() if rawTotalLimit then local totalLimit = math.min(tonumber(rawTotalLimit), envLimit) if tonumber(redis.call('SCARD', groupConcurrencyKey) or '0') >= totalLimit then @@ -4191,7 +4352,7 @@ if enableFastPath == '1' then local okDecode, decoded = pcall(cjson.decode, messageData) if okDecode and type(decoded) == 'table' and decoded.gates then gateMsg = decoded - gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil, totalConcurrencyEnabled) + gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil) end end @@ -4241,6 +4402,7 @@ local added = redis.call('ZADD', queueKey, messageScore, messageId) redis.call('ZADD', envQueueKey, messageScore, messageId) if added == 1 then redis.call('INCR', lengthCounterKey) + __gateQueuedDeltaRaw(keyPrefix, messageData, 1) end -- Rebalance CK index @@ -4280,7 +4442,7 @@ return __qmret(0) }); this.redis.defineCommand("enqueueMessageWithTtlCkTracked", { - numberOfKeys: 19, + numberOfKeys: 18, lua: ` local masterQueueKey = KEYS[1] local queueKey = KEYS[2] @@ -4303,7 +4465,13 @@ local baseQueueKey = KEYS[16] -- Total-cap keys (KEYS 17-18) local groupConcurrencyKey = KEYS[17] local totalConcurrencyLimitKey = KEYS[18] -local ckLimitsKey = KEYS[19] +local __rawTotalLimit = nil +local function __totalLimitRaw() + if __rawTotalLimit == nil then + __rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) or false + end + return __rawTotalLimit +end local queueName = ARGV[1] local messageId = ARGV[2] @@ -4343,18 +4511,12 @@ if enableFastPath == '1' then tonumber(redis.call('GET', queueConcurrencyLimitKey) or '1000000'), envLimit ) - if totalConcurrencyEnabled then - local perKeyOverride = redis.call('HGET', ckLimitsKey, queueName) - if perKeyOverride then - queueLimit = math.min(tonumber(perKeyOverride), envLimit) - end - end if queueCurrent < queueLimit then -- Total-cap gate: see enqueueMessageCkTracked. local totalAllowsFastPath = true if totalConcurrencyEnabled then - local rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) + local rawTotalLimit = __totalLimitRaw() if rawTotalLimit then local totalLimit = math.min(tonumber(rawTotalLimit), envLimit) if tonumber(redis.call('SCARD', groupConcurrencyKey) or '0') >= totalLimit then @@ -4369,7 +4531,7 @@ if enableFastPath == '1' then local okDecode, decoded = pcall(cjson.decode, messageData) if okDecode and type(decoded) == 'table' and decoded.gates then gateMsg = decoded - gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil, totalConcurrencyEnabled) + gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil) end end @@ -4412,6 +4574,7 @@ redis.call('ZADD', envQueueKey, messageScore, messageId) redis.call('ZADD', ttlQueueKey, ttlScore, ttlMember) if added == 1 then redis.call('INCR', lengthCounterKey) + __gateQueuedDeltaRaw(keyPrefix, messageData, 1) end -- Rebalance CK index @@ -4616,6 +4779,9 @@ for i, member in ipairs(expiredMembers) do -- ZREM from queue; if successful AND this is a CK variant, DECR lengthCounter. local removedFromZset = redis.call('ZREM', queueKey, runId) + if removedFromZset == 1 then + __gateQueuedDeltaRaw(keyPrefix, rawPayload, -1) + end local envMatch = string.match(rawQueueKey, ":env:([^:]+)") if envMatch then @@ -4635,8 +4801,16 @@ for i, member in ipairs(expiredMembers) do redis.call('SREM', envConcurrencyKey, runId) redis.call('SREM', envDequeuedKey, runId) - -- Rebalance CK index AND update counters if this is a CK queue + -- Mirror the currentConcurrency SREM into the base groupConcurrency set for + -- keyed and keyless queues alike, so a defensive removal always drains the + -- total pool too. local ckMatch = string.match(rawQueueKey, "(.-):ck:") + if removedFromCurrent == 1 then + local groupBase = ckMatch or rawQueueKey + redis.call('SREM', keyPrefix .. groupBase .. ":groupConcurrency", runId) + end + + -- Rebalance CK index AND update counters if this is a CK queue if ckMatch then local lengthCounterKey = keyPrefix .. ckMatch .. ":lengthCounter" local runningCounterKey = keyPrefix .. ckMatch .. ":runningCounter" @@ -4646,10 +4820,6 @@ for i, member in ipairs(expiredMembers) do if removedFromDequeued == 1 then decrFloored(runningCounterKey) end - -- Mirror the per-CK currentConcurrency SREM into the base groupConcurrency set - if removedFromCurrent == 1 then - redis.call('SREM', keyPrefix .. ckMatch .. ":groupConcurrency", runId) - end local ckIndexKey = keyPrefix .. ckMatch .. ":ckIndex" local earliest = redis.call('ZRANGE', queueKey, 0, 0, 'WITHSCORES') @@ -4679,7 +4849,7 @@ return results }); this.redis.defineCommand("dequeueMessagesFromQueue", { - numberOfKeys: 10, + numberOfKeys: 12, lua: ` local queueKey = KEYS[1] local queueConcurrencyLimitKey = KEYS[2] @@ -4691,6 +4861,16 @@ local messageKeyPrefix = KEYS[7] local envQueueKey = KEYS[8] local masterQueueKey = KEYS[9] local ttlQueueKey = KEYS[10] -- Optional: TTL sorted set key (empty string if not used) +-- Total-cap keys (KEYS 11-12) +local groupConcurrencyKey = KEYS[11] +local totalConcurrencyLimitKey = KEYS[12] +local __rawTotalLimit = nil +local function __totalLimitRaw() + if __rawTotalLimit == nil then + __rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) or false + end + return __rawTotalLimit +end local queueName = ARGV[1] local currentTime = tonumber(ARGV[2]) @@ -4702,7 +4882,16 @@ local gatesEnabled = ARGV[7] == '1' local totalConcurrencyEnabled = ARGV[8] == '1' ${QUEUE_METRICS_GAUGE_PRELUDE} ${QUEUE_GATES_LUA_HELPERS} +-- Sample-at-return: the gauge is computed once, by the return wrapper, so every +-- exit emits the state as of that exit (post-admission on the success path) and +-- no path pays for a sample that a later one would overwrite. +local function __qmsample() ${QUEUE_METRICS_GAUGE_LUA} +end +do + local __qmret_inner = __qmret + __qmret = function(r) __qmsample() return __qmret_inner(r) end +end -- Check current env concurrency against the limit local envCurrentConcurrency = tonumber(redis.call('SCARD', envCurrentConcurrencyKey) or '0') @@ -4729,6 +4918,22 @@ local envAvailableCapacity = envConcurrencyLimitWithBurstFactor - envCurrentConc local queueAvailableCapacity = totalQueueConcurrencyLimit - queueCurrentConcurrency local actualMaxCount = math.min(maxCount, envAvailableCapacity, queueAvailableCapacity) +-- Total-cap gate: every admit joins the group set, so the batch is bounded by the +-- env-clamped total limit's remaining capacity. At saturation, run the bounded +-- reconcile once (heals leaked members) and re-check before giving up. +if totalConcurrencyEnabled then + local rawTotalLimit = __totalLimitRaw() + if rawTotalLimit then + local totalLimit = math.min(tonumber(rawTotalLimit), envConcurrencyLimit) + local groupCurrentConcurrency = tonumber(redis.call('SCARD', groupConcurrencyKey) or '0') + if groupCurrentConcurrency >= totalLimit then + __gateReconcile(groupConcurrencyKey, messageKeyPrefix, keyPrefix) + groupCurrentConcurrency = tonumber(redis.call('SCARD', groupConcurrencyKey) or '0') + end + actualMaxCount = math.min(actualMaxCount, totalLimit - groupCurrentConcurrency) + end +end + if actualMaxCount <= 0 then return __qmret(nil) end @@ -4763,8 +4968,11 @@ for i = 1, #messages, 2 do -- leave messageKey intact, and (re-)register the TTL entry so the -- TTL consumer can discover and properly expire the run. The entry -- is removed on first dequeue, so it cannot be assumed to exist. - redis.call('ZREM', queueKey, messageId) + local removedExpired = redis.call('ZREM', queueKey, messageId) redis.call('ZREM', envQueueKey, messageId) + if removedExpired == 1 then + __gateQueuedDelta(keyPrefix, messageData, -1) + end if ttlQueueKey and ttlQueueKey ~= '' then local ttlMember = queueName .. '|' .. messageId .. '|' .. (messageData.orgId or '') redis.call('ZADD', ttlQueueKey, ttlExpiresAt, ttlMember) @@ -4772,14 +4980,20 @@ for i = 1, #messages, 2 do else local gatesAllow = true if gatesEnabled then - gatesAllow = __gatesHaveCapacity(keyPrefix, messageData, messageId, envConcurrencyLimit, messageKeyPrefix, totalConcurrencyEnabled) + gatesAllow = __gatesHaveCapacity(keyPrefix, messageData, messageId, envConcurrencyLimit, messageKeyPrefix) end if gatesAllow then - redis.call('ZREM', queueKey, messageId) + local removedFromQueue = redis.call('ZREM', queueKey, messageId) redis.call('ZREM', envQueueKey, messageId) + if removedFromQueue == 1 then + __gateQueuedDelta(keyPrefix, messageData, -1) + end redis.call('SADD', queueCurrentConcurrencyKey, messageId) redis.call('SADD', envCurrentConcurrencyKey, messageId) + if totalConcurrencyEnabled then + redis.call('SADD', groupConcurrencyKey, messageId) + end if gatesEnabled then __gatesAcquire(keyPrefix, messageData, messageId) end @@ -4974,7 +5188,7 @@ return results // (normal dequeue, TTL-expired, or stale-orphan path — all of which were // counted at enqueue time). this.redis.defineCommand("dequeueMessagesFromCkQueueTracked", { - numberOfKeys: 14, + numberOfKeys: 13, lua: ` local ckIndexKey = KEYS[1] local queueConcurrencyLimitKey = KEYS[2] @@ -4989,7 +5203,13 @@ local lengthCounterKey = KEYS[10] local runningCounterKey = KEYS[11] local groupConcurrencyKey = KEYS[12] local totalConcurrencyLimitKey = KEYS[13] -local ckLimitsKey = KEYS[14] +local __rawTotalLimit = nil +local function __totalLimitRaw() + if __rawTotalLimit == nil then + __rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) or false + end + return __rawTotalLimit +end local ckWildcardName = ARGV[1] local currentTime = tonumber(ARGV[2]) @@ -5001,7 +5221,16 @@ local totalConcurrencyEnabled = ARGV[7] == '1' local gatesEnabled = ARGV[8] == '1' ${QUEUE_METRICS_GAUGE_PRELUDE} ${QUEUE_GATES_LUA_HELPERS} +-- Sample-at-return: the gauge is computed once, by the return wrapper, so every +-- exit emits the state as of that exit (post-admission on the success path) and +-- no path pays for a sample that a later one would overwrite. +local function __qmsample() ${QUEUE_METRICS_CK_DEQUEUE_GAUGE_LUA} +end +do + local __qmret_inner = __qmret + __qmret = function(r) __qmsample() return __qmret_inner(r) end +end local function decrLengthCounter() if tonumber(redis.call('GET', lengthCounterKey) or '0') > 0 then @@ -5032,7 +5261,7 @@ local actualMaxCount = math.min(maxCount, envAvailableCapacity) -- behind, and blocking on it would deadlock the run against itself). local totalHeadroom = nil if totalConcurrencyEnabled then - local rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) + local rawTotalLimit = __totalLimitRaw() if rawTotalLimit then local totalConcurrencyLimit = math.min(tonumber(rawTotalLimit), envConcurrencyLimit) local groupCurrentConcurrency = tonumber(redis.call('SCARD', groupConcurrencyKey) or '0') @@ -5081,12 +5310,6 @@ for _, ckQueueName in ipairs(ckQueues) do local ckCurrentConcurrency = tonumber(redis.call('SCARD', ckConcurrencyKey) or '0') local perKeyLimit = queueConcurrencyLimit - if totalConcurrencyEnabled then - local perKeyOverride = redis.call('HGET', ckLimitsKey, ckQueueName) - if perKeyOverride then - perKeyLimit = math.min(tonumber(perKeyOverride), envConcurrencyLimit) - end - end if ckCurrentConcurrency >= perKeyLimit then -- Back a blocked variant off so it cannot pin the bounded candidate window @@ -5111,9 +5334,12 @@ for _, ckQueueName in ipairs(ckQueues) do local ttlExpiresAt = messageData and messageData.ttlExpiresAt if ttlExpiresAt and ttlExpiresAt <= currentTime then - redis.call('ZREM', fullQueueKey, messageId) + local removedExpired = redis.call('ZREM', fullQueueKey, messageId) redis.call('ZREM', envQueueKey, messageId) decrLengthCounter() + if removedExpired == 1 then + __gateQueuedDelta(keyPrefix, messageData, -1) + end if ttlQueueKey and ttlQueueKey ~= '' then local ttlMember = ckQueueName .. '|' .. messageId .. '|' .. (messageData.orgId or '') redis.call('ZADD', ttlQueueKey, ttlExpiresAt, ttlMember) @@ -5121,7 +5347,7 @@ for _, ckQueueName in ipairs(ckQueues) do else local gatesAllow = true if gatesEnabled then - gatesAllow = __gatesHaveCapacity(keyPrefix, messageData, messageId, envConcurrencyLimit, messageKeyPrefix, totalConcurrencyEnabled) + gatesAllow = __gatesHaveCapacity(keyPrefix, messageData, messageId, envConcurrencyLimit, messageKeyPrefix) end if not gatesAllow then blockedByGates = true @@ -5135,9 +5361,12 @@ for _, ckQueueName in ipairs(ckQueues) do end if gatesAllow and totalAllows then - redis.call('ZREM', fullQueueKey, messageId) + local removedFromQueue = redis.call('ZREM', fullQueueKey, messageId) redis.call('ZREM', envQueueKey, messageId) decrLengthCounter() + if removedFromQueue == 1 then + __gateQueuedDelta(keyPrefix, messageData, -1) + end redis.call('SADD', ckConcurrencyKey, messageId) redis.call('SADD', envCurrentConcurrencyKey, messageId) if totalConcurrencyEnabled then @@ -5322,7 +5551,7 @@ return message }); this.redis.defineCommand("acknowledgeMessage", { - numberOfKeys: 9, + numberOfKeys: 10, lua: ` -- Keys: local masterQueueKey = KEYS[1] @@ -5334,6 +5563,7 @@ local queueCurrentDequeuedKey = KEYS[6] local envCurrentDequeuedKey = KEYS[7] local envQueueKey = KEYS[8] local workerQueueKey = KEYS[9] +local groupConcurrencyKey = KEYS[10] -- Args: local messageId = ARGV[1] @@ -5349,8 +5579,11 @@ local rawPayload = redis.call('GET', messageKey) redis.call('DEL', messageKey) -- Remove the message from the queue -redis.call('ZREM', messageQueueKey, messageId) +local removedFromQueueZset = redis.call('ZREM', messageQueueKey, messageId) redis.call('ZREM', envQueueKey, messageId) +if removedFromQueueZset == 1 then + __gateQueuedDeltaRaw(keyPrefix, rawPayload, -1) +end -- Rebalance the parent queues local earliestMessage = redis.call('ZRANGE', messageQueueKey, 0, 0, 'WITHSCORES') @@ -5360,8 +5593,12 @@ else redis.call('ZADD', masterQueueKey, earliestMessage[2], messageQueueName) end --- Update the concurrency keys -redis.call('SREM', queueCurrentConcurrencyKey, messageId) +-- Update the concurrency keys. The groupConcurrency SREM mirrors the base SREM +-- unconditionally (no flag check) so a disabled flag still drains the group set. +local removedFromCurrentConcurrency = redis.call('SREM', queueCurrentConcurrencyKey, messageId) +if removedFromCurrentConcurrency == 1 then + redis.call('SREM', groupConcurrencyKey, messageId) +end redis.call('SREM', envCurrentConcurrencyKey, messageId) redis.call('SREM', queueCurrentDequeuedKey, messageId) redis.call('SREM', envCurrentDequeuedKey, messageId) @@ -5375,7 +5612,7 @@ end }); this.redis.defineCommand("nackMessage", { - numberOfKeys: 8, + numberOfKeys: 9, lua: ` -- Keys: local masterQueueKey = KEYS[1] @@ -5386,6 +5623,7 @@ local envCurrentConcurrencyKey = KEYS[5] local queueCurrentDequeuedKey = KEYS[6] local envCurrentDequeuedKey = KEYS[7] local envQueueKey = KEYS[8] +local groupConcurrencyKey = KEYS[9] -- Args: local messageId = ARGV[1] @@ -5398,15 +5636,22 @@ ${QUEUE_GATES_LUA_HELPERS} -- Update the message data redis.call('SET', messageKey, messageData) --- Update the concurrency keys -redis.call('SREM', queueCurrentConcurrencyKey, messageId) +-- Update the concurrency keys. The groupConcurrency SREM mirrors the base SREM +-- unconditionally (no flag check) so a disabled flag still drains the group set. +local removedFromCurrentConcurrency = redis.call('SREM', queueCurrentConcurrencyKey, messageId) +if removedFromCurrentConcurrency == 1 then + redis.call('SREM', groupConcurrencyKey, messageId) +end redis.call('SREM', envCurrentConcurrencyKey, messageId) redis.call('SREM', queueCurrentDequeuedKey, messageId) redis.call('SREM', envCurrentDequeuedKey, messageId) __gatesRelease(keyPrefix, messageData, messageId) -- Enqueue the message into the queue -redis.call('ZADD', messageQueueKey, messageScore, messageId) +local added = redis.call('ZADD', messageQueueKey, messageScore, messageId) +if added == 1 then + __gateQueuedDeltaRaw(keyPrefix, messageData, 1) +end redis.call('ZADD', envQueueKey, messageScore, messageId) -- Rebalance the parent queues @@ -5420,7 +5665,7 @@ end }); this.redis.defineCommand("moveToDeadLetterQueue", { - numberOfKeys: 9, + numberOfKeys: 10, lua: ` -- Keys: local masterQueueKey = KEYS[1] @@ -5432,6 +5677,7 @@ local queueCurrentDequeuedKey = KEYS[6] local envCurrentDequeuedKey = KEYS[7] local envQueueKey = KEYS[8] local deadLetterQueueKey = KEYS[9] +local groupConcurrencyKey = KEYS[10] -- Args: local messageId = ARGV[1] @@ -5442,8 +5688,11 @@ ${QUEUE_GATES_LUA_HELPERS} local rawPayload = redis.call('GET', messageKey) -- Remove the message from the queue -redis.call('ZREM', messageQueue, messageId) +local removedFromQueueZset = redis.call('ZREM', messageQueue, messageId) redis.call('ZREM', envQueueKey, messageId) +if removedFromQueueZset == 1 then + __gateQueuedDeltaRaw(keyPrefix, rawPayload, -1) +end -- Rebalance the parent queues local earliestMessage = redis.call('ZRANGE', messageQueue, 0, 0, 'WITHSCORES') @@ -5456,8 +5705,12 @@ end -- Add the message to the dead letter queue redis.call('ZADD', deadLetterQueueKey, tonumber(redis.call('TIME')[1]), messageId) --- Update the concurrency keys -redis.call('SREM', queueCurrentConcurrencyKey, messageId) +-- Update the concurrency keys. The groupConcurrency SREM mirrors the base SREM +-- unconditionally (no flag check) so a disabled flag still drains the group set. +local removedFromCurrentConcurrency = redis.call('SREM', queueCurrentConcurrencyKey, messageId) +if removedFromCurrentConcurrency == 1 then + redis.call('SREM', groupConcurrencyKey, messageId) +end redis.call('SREM', envCurrentConcurrencyKey, messageId) redis.call('SREM', queueCurrentDequeuedKey, messageId) redis.call('SREM', envCurrentDequeuedKey, messageId) @@ -5700,6 +5953,7 @@ local removedFromZset = redis.call('ZREM', messageQueueKey, messageId) redis.call('ZREM', envQueueKey, messageId) if removedFromZset == 1 then decrFloored(lengthCounterKey) + __gateQueuedDeltaRaw(keyPrefix, rawPayload, -1) end -- Rebalance CK index @@ -5823,6 +6077,9 @@ end -- Enqueue the message back into the CK-specific queue. INCR lengthCounter only if -- it's a new entry (ZADD returns 1). local added = redis.call('ZADD', messageQueueKey, messageScore, messageId) +if added == 1 then + __gateQueuedDeltaRaw(keyPrefix, messageData, 1) +end redis.call('ZADD', envQueueKey, messageScore, messageId) if added == 1 then redis.call('INCR', lengthCounterKey) @@ -5894,6 +6151,7 @@ local removedFromZset = redis.call('ZREM', messageQueue, messageId) redis.call('ZREM', envQueueKey, messageId) if removedFromZset == 1 then decrFloored(lengthCounterKey) + __gateQueuedDeltaRaw(keyPrefix, rawPayload, -1) end -- Rebalance CK index @@ -5941,7 +6199,7 @@ __gatesRelease(keyPrefix, rawPayload, messageId) }); this.redis.defineCommand("releaseConcurrency", { - numberOfKeys: 5, + numberOfKeys: 6, lua: ` -- Keys: local queueCurrentConcurrencyKey = KEYS[1] @@ -5949,14 +6207,19 @@ local envCurrentConcurrencyKey = KEYS[2] local queueCurrentDequeuedKey = KEYS[3] local envCurrentDequeuedKey = KEYS[4] local messageKey = KEYS[5] +local groupConcurrencyKey = KEYS[6] -- Args: local messageId = ARGV[1] local keyPrefix = ARGV[2] ${QUEUE_GATES_LUA_HELPERS} --- Update the concurrency keys -redis.call('SREM', queueCurrentConcurrencyKey, messageId) +-- Update the concurrency keys. The groupConcurrency SREM mirrors the base SREM +-- unconditionally (no flag check) so a disabled flag still drains the group set. +local removedFromCurrentConcurrency = redis.call('SREM', queueCurrentConcurrencyKey, messageId) +if removedFromCurrentConcurrency == 1 then + redis.call('SREM', groupConcurrencyKey, messageId) +end redis.call('SREM', envCurrentConcurrencyKey, messageId) redis.call('SREM', queueCurrentDequeuedKey, messageId) redis.call('SREM', envCurrentDequeuedKey, messageId) @@ -6019,26 +6282,6 @@ __gatesRelease(keyPrefix, redis.call('GET', messageKey), messageId) `, }); - this.redis.defineCommand("setQueueConcurrencyKeyLimit", { - numberOfKeys: 1, - lua: ` -local ckLimitsKey = KEYS[1] - -local fieldName = ARGV[1] -local limit = ARGV[2] -local maxFields = tonumber(ARGV[3]) - -if redis.call('HEXISTS', ckLimitsKey, fieldName) == 0 then - if redis.call('HLEN', ckLimitsKey) >= maxFields then - return 0 - end -end - -redis.call('HSET', ckLimitsKey, fieldName, limit) -return 1 -`, - }); - this.redis.defineCommand("updateEnvironmentConcurrencyLimits", { numberOfKeys: 2, lua: ` @@ -6119,7 +6362,7 @@ return results }); this.redis.defineCommand("clearMessageFromConcurrencySets", { - numberOfKeys: 5, + numberOfKeys: 6, lua: ` -- Keys: local queueCurrentConcurrencyKey = KEYS[1] @@ -6127,14 +6370,19 @@ local envCurrentConcurrencyKey = KEYS[2] local queueCurrentDequeuedKey = KEYS[3] local envCurrentDequeuedKey = KEYS[4] local messageKey = KEYS[5] +local groupConcurrencyKey = KEYS[6] -- Args: local messageId = ARGV[1] local keyPrefix = ARGV[2] ${QUEUE_GATES_LUA_HELPERS} --- Update the concurrency keys -redis.call('SREM', queueCurrentConcurrencyKey, messageId) +-- Update the concurrency keys. The groupConcurrency SREM mirrors the base SREM +-- unconditionally (no flag check) so a disabled flag still drains the group set. +local removedFromCurrentConcurrency = redis.call('SREM', queueCurrentConcurrencyKey, messageId) +if removedFromCurrentConcurrency == 1 then + redis.call('SREM', groupConcurrencyKey, messageId) +end redis.call('SREM', envCurrentConcurrencyKey, messageId) redis.call('SREM', queueCurrentDequeuedKey, messageId) redis.call('SREM', envCurrentDequeuedKey, messageId) @@ -6217,6 +6465,8 @@ declare module "@internal/redis" { queueConcurrencyLimitKey: string, envConcurrencyLimitKey: string, envConcurrencyLimitBurstFactorKey: string, + groupConcurrencyKey: string, + totalConcurrencyLimitKey: string, //args queueName: string, messageId: string, @@ -6249,6 +6499,8 @@ declare module "@internal/redis" { queueConcurrencyLimitKey: string, envConcurrencyLimitKey: string, envConcurrencyLimitBurstFactorKey: string, + groupConcurrencyKey: string, + totalConcurrencyLimitKey: string, //args queueName: string, messageId: string, @@ -6294,6 +6546,8 @@ declare module "@internal/redis" { envQueueKey: string, masterQueueKey: string, ttlQueueKey: string, + groupConcurrencyKey: string, + totalConcurrencyLimitKey: string, //args childQueueName: string, currentTime: string, @@ -6331,6 +6585,7 @@ declare module "@internal/redis" { envCurrentDequeuedKey: string, envQueueKey: string, workerQueueKey: string, + groupConcurrencyKey: string, // args messageId: string, messageQueueName: string, @@ -6347,6 +6602,7 @@ declare module "@internal/redis" { queueCurrentDequeuedKey: string, envCurrentDequeuedKey: string, messageKey: string, + groupConcurrencyKey: string, // args messageId: string, keyPrefix: string, @@ -6363,6 +6619,7 @@ declare module "@internal/redis" { queueCurrentDequeuedKey: string, envCurrentDequeuedKey: string, envQueueKey: string, + groupConcurrencyKey: string, // args messageId: string, messageQueueName: string, @@ -6383,6 +6640,7 @@ declare module "@internal/redis" { envCurrentDequeuedKey: string, envQueueKey: string, deadLetterQueueKey: string, + groupConcurrencyKey: string, // args messageId: string, messageQueueName: string, @@ -6397,20 +6655,13 @@ declare module "@internal/redis" { queueCurrentDequeuedKey: string, envCurrentDequeuedKey: string, messageKey: string, + groupConcurrencyKey: string, // args messageId: string, keyPrefix: string, callback?: Callback ): Result; - setQueueConcurrencyKeyLimit( - ckLimitsKey: string, - fieldName: string, - limit: string, - maxFields: string, - callback?: Callback - ): Result; - updateEnvironmentConcurrencyLimits( // keys envConcurrencyLimitKey: string, @@ -6597,7 +6848,6 @@ declare module "@internal/redis" { baseQueueKey: string, groupConcurrencyKey: string, totalConcurrencyLimitKey: string, - ckLimitsKey: string, queueName: string, messageId: string, messageData: string, @@ -6635,7 +6885,6 @@ declare module "@internal/redis" { baseQueueKey: string, groupConcurrencyKey: string, totalConcurrencyLimitKey: string, - ckLimitsKey: string, queueName: string, messageId: string, messageData: string, @@ -6670,7 +6919,6 @@ declare module "@internal/redis" { runningCounterKey: string, groupConcurrencyKey: string, totalConcurrencyLimitKey: string, - ckLimitsKey: string, ckWildcardName: string, currentTime: string, defaultEnvConcurrencyLimit: string, diff --git a/internal-packages/run-engine/src/run-queue/keyProducer.ts b/internal-packages/run-engine/src/run-queue/keyProducer.ts index 120e04f8c38..ff7a51f91f3 100644 --- a/internal-packages/run-engine/src/run-queue/keyProducer.ts +++ b/internal-packages/run-engine/src/run-queue/keyProducer.ts @@ -26,6 +26,7 @@ const constants = { RUNNING_COUNTER_PART: "runningCounter", GROUP_CONCURRENCY_PART: "groupConcurrency", TOTAL_CONCURRENCY_LIMIT_PART: "totalConcurrency", + GATE_QUEUED_COUNTER_PART: "gateQueuedCounter", } as const; export class RunQueueFullKeyProducer implements RunQueueKeyProducer { @@ -353,6 +354,16 @@ export class RunQueueFullKeyProducer implements RunQueueKeyProducer { return `${this.baseQueueKeyFromQueue(queue)}:${constants.GROUP_CONCURRENCY_PART}`; } + /** + * Counter of queued runs holding this queue as a gate: runs that are not + * executing, are queued, and must clear this gate to execute. Maintained + * exactly like the CK length counter: incremented per gate on enqueue, + * decremented on admit and on every queued-removal path. + */ + gateQueuedCounterKey(env: RunQueueKeyProducerEnvironment, queue: string): string { + return `${this.queueKey(env, queue)}:${constants.GATE_QUEUED_COUNTER_PART}`; + } + /** * String key holding the queue's total concurrency limit (the cap across all * concurrency-key variants). Absent = no total cap. Readers clamp to the @@ -366,14 +377,6 @@ export class RunQueueFullKeyProducer implements RunQueueKeyProducer { return `${this.baseQueueKeyFromQueue(queue)}:${constants.TOTAL_CONCURRENCY_LIMIT_PART}`; } - queueCkLimitsKey(env: RunQueueKeyProducerEnvironment, queue: string): string { - return `${this.queueKey(env, queue)}:ckLimits`; - } - - queueCkLimitsKeyFromQueue(queue: string): string { - return `${this.baseQueueKeyFromQueue(queue)}:ckLimits`; - } - isCkWildcard(queue: string): boolean { return queue.endsWith(":ck:*"); } diff --git a/internal-packages/run-engine/src/run-queue/metrics.test.ts b/internal-packages/run-engine/src/run-queue/metrics.test.ts index ebfc295470e..c239b4aa937 100644 --- a/internal-packages/run-engine/src/run-queue/metrics.test.ts +++ b/internal-packages/run-engine/src/run-queue/metrics.test.ts @@ -24,6 +24,21 @@ const authenticatedEnvDev = { organization: { id: "o1234" }, }; +// A dead Redis leaves waitUntilReady() pending forever (the client retries +// indefinitely), which would burn the whole test timeout with no diagnostic. +// The abort releases the losing timer promptly so it cannot hold an event +// loop open for the remaining 15s after a fast ready. +async function emitterReady(emitter: MetricsStreamEmitter) { + const abort = new AbortController(); + const timedOut = setTimeout(15_000, "timeout", { signal: abort.signal }).catch(() => "aborted"); + const winner = await Promise.race([emitter.waitUntilReady().then(() => "ready"), timedOut]); + abort.abort(); + if (winner === "timeout") { + void emitter.close().catch(() => {}); + throw new Error("metrics emitter Redis connection never became ready"); + } +} + async function readAllEntries( redisOptions: { host: string; @@ -81,6 +96,7 @@ describe("RunQueue queue-metrics emission", () => { definition, flag: { enabled: () => true }, }); + await emitterReady(emitter); const queue = new RunQueue({ name: "rq", @@ -123,7 +139,10 @@ describe("RunQueue queue-metrics emission", () => { const entries = await waitForEntries(redis, definition, (es) => { const seen = es.map((e) => e.fields.op); - return ["enqueue", "gauge", "started", "ack"].every((o) => seen.includes(o)); + if (!["enqueue", "gauge", "started", "ack"].every((o) => seen.includes(o))) return false; + return es.some( + (e) => e.fields.op === "gauge" && e.fields.cc === "1" && e.fields.ql === "0" + ); }); const ops = entries.map((e) => e.fields.op); expect(ops).toContain("enqueue"); @@ -137,9 +156,23 @@ describe("RunQueue queue-metrics emission", () => { for (const f of ["ql", "cc", "lim", "eql", "ec", "elim", "thr"]) { expect(gauge!.fields[f]).toBeDefined(); } - // Non-CK scripts keep the 7-field gauge (no CK-health tail). - expect(gauge!.fields.ckq).toBeUndefined(); - expect(gauge!.fields.ckw).toBeUndefined(); + /** + * Non-CK scripts emit the full gauge tail too: zeroed CK-health fields (a base + * queue has no CK variants) followed by the total running/limit pair, so a + * keyless queue with a total limit still charts total concurrency. + */ + expect(gauge!.fields.ckq).toBe("0"); + expect(gauge!.fields.ckw).toBe("0"); + expect(gauge!.fields.tcc).toBeDefined(); + expect(gauge!.fields.tlim).toBeDefined(); + + // Pins the dequeue script's sample-at-return wrapper: only the dequeue emits the + // post-admission reading (running 1, queued 0); the enqueue gauge sees the inverse. + const dequeueGauge = entries.find( + (e) => e.fields.op === "gauge" && e.fields.cc === "1" && e.fields.ql === "0" + ); + assertGauge(dequeueGauge); + expect(dequeueGauge!.fields.q).toContain("task/my-task"); // The first counter emission also seeds a cum=0 baseline (no wait); the real reading // carries wait. Pick the reading (cum > 0). @@ -172,6 +205,7 @@ describe("RunQueue queue-metrics emission", () => { definition, flag: { enabled: () => true }, }); + await emitterReady(emitter); const queue = new RunQueue({ name: "rq", tracer: trace.getTracer("rq"), @@ -244,6 +278,7 @@ describe("RunQueue queue-metrics emission", () => { maxLen: 1000, }; const emitter = new MetricsStreamEmitter({ redis, definition, flag: { enabled: () => true } }); + await emitterReady(emitter); const queue = new RunQueue({ name: "rq", tracer: trace.getTracer("rq"), @@ -283,14 +318,13 @@ describe("RunQueue queue-metrics emission", () => { expect(dequeued?.messageId).toBe(message.runId); const entries = await waitForEntries(redis, definition, (es) => - es.some( - (e) => e.fields.op === "gauge" && e.fields.q.includes(":ck:") && e.fields.thr === "0" - ) + es.some((e) => e.fields.op === "gauge" && e.fields.q.includes(":ck:*")) ); const gauges = entries.filter((e) => e.fields.op === "gauge"); expect(gauges.length).toBeGreaterThan(0); - // The aggregate CK dequeue gauge targets the CK wildcard and never sets thr. - const aggregate = gauges.find((e) => e.fields.q.includes(":ck:") && e.fields.thr === "0"); + // The aggregate gauge targets the CK wildcard and only the CK dequeue script emits + // it, so this pins that script's sample-at-return wrapper. + const aggregate = gauges.find((e) => e.fields.q.includes(":ck:*")); assertGauge(aggregate); expect(Number(aggregate!.fields.ql)).toBeGreaterThanOrEqual(0); expect(Number(aggregate!.fields.cc)).toBeGreaterThanOrEqual(0); @@ -344,6 +378,7 @@ describe("RunQueue queue-metrics emission", () => { flag: { enabled: () => true }, gaugeSampleRate: 0, }); + await emitterReady(emitter); const queue = new RunQueue({ name: "rq", tracer: trace.getTracer("rq"), diff --git a/internal-packages/run-engine/src/run-queue/tests/queueGates.test.ts b/internal-packages/run-engine/src/run-queue/tests/queueGates.test.ts index c2195074426..272523b9595 100644 --- a/internal-packages/run-engine/src/run-queue/tests/queueGates.test.ts +++ b/internal-packages/run-engine/src/run-queue/tests/queueGates.test.ts @@ -231,6 +231,51 @@ describe("RunQueue gates", () => { } ); + redisTest( + "counts queued runs per gate and drains the counter on admit and ack", + async ({ redisContainer }) => { + const queue = createQueue(redisContainer, true); + try { + await queue.updateQueueConcurrencyLimits(authenticatedEnvDev, "task/my-task", 5); + await queue.updateQueueConcurrencyLimits(authenticatedEnvDev, "shared-gate", 1); + + const now = Date.now(); + for (const i of [0, 1]) { + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message: makeMessage({ + runId: `r${i}`, + timestamp: now - 1000 + i, + gates: [{ queue: "shared-gate" }], + }), + workerQueue: "main", + }); + } + + const oneAdmitted = await waitFor( + async () => + (await queue.currentConcurrencyOfQueue(authenticatedEnvDev, "shared-gate")) === 1 + ); + expect(oneAdmitted).toBe(true); + + /** One run executes, one waits: the gate's queued counter holds the waiter. */ + await setTimeout(2000); + expect(await queue.gateQueuedCountOfQueue(authenticatedEnvDev, "shared-gate")).toBe(1); + + expect(await popWorkerQueue(queue, "r0")).toBe(true); + await queue.acknowledgeMessage(authenticatedEnvDev.organization.id, "r0"); + + const drained = await waitFor(async () => { + if (!(await popWorkerQueue(queue, "r1"))) return false; + return (await queue.gateQueuedCountOfQueue(authenticatedEnvDev, "shared-gate")) === 0; + }); + expect(drained).toBe(true); + } finally { + await queue.quit(); + } + } + ); + redisTest("ignores gates and holds no gate slots when disabled", async ({ redisContainer }) => { const queue = createQueue(redisContainer, false); try { diff --git a/internal-packages/run-engine/src/run-queue/tests/totalConcurrency.test.ts b/internal-packages/run-engine/src/run-queue/tests/totalConcurrency.test.ts index f8f914b564f..0ccedd8881b 100644 --- a/internal-packages/run-engine/src/run-queue/tests/totalConcurrency.test.ts +++ b/internal-packages/run-engine/src/run-queue/tests/totalConcurrency.test.ts @@ -207,6 +207,52 @@ describe("RunQueue total concurrency limit", () => { } ); + redisTest("the total limit caps keyed and keyless runs together", async ({ redisContainer }) => { + const queue = createQueue(redisContainer, true); + try { + await queue.updateQueueConcurrencyLimits(authenticatedEnvDev, "task/my-task", 5); + await queue.updateQueueTotalConcurrencyLimits(authenticatedEnvDev, "task/my-task", 1); + + const now = Date.now(); + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message: makeMessage({ runId: "r0", timestamp: now - 1000 }), + workerQueue: "main", + }); + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message: makeMessage({ runId: "r1", concurrencyKey: "ck-a", timestamp: now - 999 }), + workerQueue: "main", + }); + + const oneAdmitted = await waitFor( + async () => (await queue.totalConcurrencyOfQueue(authenticatedEnvDev, "task/my-task")) === 1 + ); + expect(oneAdmitted).toBe(true); + + /** The second run must stay queued: the total pool spans keyed and keyless. */ + await setTimeout(2000); + expect(await queue.totalConcurrencyOfQueue(authenticatedEnvDev, "task/my-task")).toBe(1); + expect(await queue.lengthOfQueue(authenticatedEnvDev, "task/my-task")).toBe(1); + + const dequeued = await queue.dequeueMessageFromWorkerQueue("consumer-1", "main"); + assertNonNullable(dequeued); + await queue.acknowledgeMessage(authenticatedEnvDev.organization.id, dequeued.messageId); + + /** Acking the first holder frees the total pool; the other run is admitted. */ + const secondAdmitted = await waitFor(async () => { + const next = await queue.dequeueMessageFromWorkerQueue("consumer-1", "main", { + blockingPop: false, + }); + return next !== undefined && next.messageId !== dequeued.messageId; + }); + expect(secondAdmitted).toBe(true); + expect(await queue.totalConcurrencyOfQueue(authenticatedEnvDev, "task/my-task")).toBe(1); + } finally { + await queue.quit(); + } + }); + redisTest("enqueue fast path respects the total limit", async ({ redisContainer }) => { const queue = createQueue(redisContainer, true); try { diff --git a/internal-packages/run-engine/src/run-queue/types.ts b/internal-packages/run-engine/src/run-queue/types.ts index ad358a04cbb..80565b0f732 100644 --- a/internal-packages/run-engine/src/run-queue/types.ts +++ b/internal-packages/run-engine/src/run-queue/types.ts @@ -111,9 +111,7 @@ export interface RunQueueKeyProducer { queueGroupConcurrencyKeyFromQueue(queue: string): string; queueTotalConcurrencyLimitKey(env: RunQueueKeyProducerEnvironment, queue: string): string; queueTotalConcurrencyLimitKeyFromQueue(queue: string): string; - - queueCkLimitsKey(env: RunQueueKeyProducerEnvironment, queue: string): string; - queueCkLimitsKeyFromQueue(queue: string): string; + gateQueuedCounterKey(env: RunQueueKeyProducerEnvironment, queue: string): string; //env oncurrency envCurrentConcurrencyKey(env: EnvDescriptor): string; diff --git a/knip.json b/knip.json index c8e7f4027dd..84456756ca1 100644 --- a/knip.json +++ b/knip.json @@ -27,9 +27,6 @@ ], "ignoreDependencies": ["@sentry/cli", "assert", "util"] }, - "internal-packages/run-engine": { - "ignore": ["src/run-queue/index.ts"] - }, "internal-packages/dashboard-agent": { "entry": ["trigger.config.ts", "src/investigation-sweep.ts", "src/maintenance.ts"], "ignoreBinaries": ["rg"] diff --git a/packages/core/src/v3/schemas/queues.ts b/packages/core/src/v3/schemas/queues.ts index 5ebc8258cf6..d66d51e9704 100644 --- a/packages/core/src/v3/schemas/queues.ts +++ b/packages/core/src/v3/schemas/queues.ts @@ -13,7 +13,7 @@ export type QueueType = z.infer; export const RetrieveQueueType = z.enum([...queueTypes, "id"]); export type RetrieveQueueType = z.infer; -export const QueueItem = z.object({ +const QueueItemCommon = { /** The queue id, e.g. queue_12345 */ id: z.string(), /** The queue name */ @@ -30,26 +30,60 @@ export const QueueItem = z.object({ queued: z.number(), /** Whether the queue is paused. If it's paused, no new runs will be started. */ paused: z.boolean(), - /** The concurrency limit of the queue */ + /** + * The queue's own concurrency limit. Meaningful on V1 queues only; always + * null on V2 queues (kept on both so existing clients keep parsing). + */ concurrencyLimit: z.number().nullable(), - /** The concurrency limit of the queue */ - concurrency: z - .object({ - /** The effective/current concurrency limit */ - current: z.number().nullable(), - /** The base concurrency limit (default) */ - base: z.number().nullable(), - /** The effective/current concurrency limit */ - override: z.number().nullable(), - /** When the override was applied */ - overriddenAt: z.coerce.date().nullable(), - /** Who overrode the concurrency limit (will be null if overridden via the API) */ - overriddenBy: z.string().nullable(), - }) - .optional(), -}); +}; -export type QueueItem = z.infer; +/** + * The queue's `version` discriminates its shape. V1 queues carry their own + * concurrency limit (applied per key when runs pass a `concurrencyKey`, to the + * whole queue when they don't) and its override state. V2 queues are only the + * line runs wait in: concurrency is declared with the task `concurrency` + * option and read or overridden through `concurrencyLimits`. A response from a + * server that predates the discriminator has V1 semantics by definition, so a + * missing `version` defaults to "V1" rather than failing the parse. + */ +const QueueItemUnion = z.discriminatedUnion("version", [ + z.object({ + ...QueueItemCommon, + version: z.literal("V1"), + /** The queue's concurrency limit override state */ + concurrency: z + .object({ + /** The effective/current concurrency limit */ + current: z.number().nullable(), + /** The base concurrency limit (default) */ + base: z.number().nullable(), + /** The overridden concurrency limit, when an override is active */ + override: z.number().nullable(), + /** When the override was applied */ + overriddenAt: z.coerce.date().nullable(), + /** Who overrode the concurrency limit (will be null if overridden via the API) */ + overriddenBy: z.string().nullable(), + }) + .optional(), + }), + z.object({ + ...QueueItemCommon, + version: z.literal("V2"), + /** Never present on V2 queues; declared so existing `queue.concurrency?.…` + * reads keep compiling across the union and see undefined. */ + concurrency: z.undefined().optional(), + }), +]); + +export const QueueItem = z.preprocess( + (value) => + value && typeof value === "object" && !("version" in value) + ? { ...value, version: "V1" } + : value, + QueueItemUnion +); + +export type QueueItem = z.infer; export const ListQueueOptions = z.object({ /** The page number */