feat(webapp): notify customers of incidents via BetterStack webhook

When a status report is published on the status page, an inbound BetterStack
webhook fans the update out over Slack, email, and Discord. Fires only on published incident
updates, not monitor auto-alerts. Deduped per update; each channel no-ops
unless its own config is present.

Verified end-to-end against a real BetterStack test status page: real webhook
delivered to Slack and Discord.

Author: kathiekiwi

.server-changes/incident-notifications.md

@@ -0,0 +1,12 @@
+---
+area: webapp
+type: feature
+---
+
+Add an inbound webhook (`POST /webhooks/v1/betterstack-incidents`) that receives
+status-page incident updates and proactively notifies customers over Slack
+(channels matching a configurable name prefix), email (org admins, via the
+alerts email transport), and Discord (an incoming webhook). Delivery runs on the
+alerts redis-worker with per-surface jobs and is deduped on the incident update
+id. Gated by `INCIDENT_NOTIFY_ENABLED` plus a shared-secret token in the webhook
+URL; each surface no-ops unless its own config is present.

apps/webapp/app/env.server.ts

@@ -1606,6 +1606,17 @@ const EnvironmentSchema = z
     BETTERSTACK_API_KEY: z.string().optional(),
     BETTERSTACK_STATUS_PAGE_ID: z.string().optional(),
 
+    // Incident notifications: when a status report is published on the status
+    // page, BetterStack calls our inbound webhook and we fan the update out over
+    // Slack/email/Discord. Each surface is a no-op unless its own config below
+    // is present. The status-page webhook has no signature, so the endpoint is
+    // gated by a shared secret in the URL. The Slack channel prefix has no
+    // default — Slack delivery only runs when a prefix is explicitly configured.
+    INCIDENT_NOTIFY_ENABLED: z.string().default("0"),
+    BETTERSTACK_INCIDENT_WEBHOOK_SECRET: z.string().optional(),
+    INCIDENT_NOTIFY_SLACK_CHANNEL_PREFIX: z.string().optional(),
+    INCIDENT_NOTIFY_DISCORD_WEBHOOK_URL: z.string().optional(),
+
     RUN_REPLICATION_REDIS_HOST: z
       .string()
       .optional()
@@ -2010,6 +2021,14 @@ const EnvironmentSchema = z
   .and(GithubAppEnvSchema)
   .and(S2EnvSchema)
   .superRefine((env, ctx) => {
+    if (env.INCIDENT_NOTIFY_ENABLED === "1" && !env.BETTERSTACK_INCIDENT_WEBHOOK_SECRET) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        path: ["BETTERSTACK_INCIDENT_WEBHOOK_SECRET"],
+        message: "BETTERSTACK_INCIDENT_WEBHOOK_SECRET is required when INCIDENT_NOTIFY_ENABLED=1",
+      });
+    }
+
     const presets = new Set(env.COMPUTE_TEMPLATE_MACHINE_PRESETS);
     for (const required of env.COMPUTE_TEMPLATE_MACHINE_PRESETS_REQUIRED) {
       if (!presets.has(required)) {

apps/webapp/app/routes/webhooks.v1.betterstack-incidents.ts

@@ -0,0 +1,81 @@
+import { type ActionFunctionArgs, json } from "@remix-run/server-runtime";
+import { createHash, timingSafeEqual } from "node:crypto";
+import { env } from "~/env.server";
+import {
+  IncidentWebhookSchema,
+  isCustomerNotifiableEvent,
+  normalizeIncidentUpdate,
+} from "~/services/betterstack/incidentWebhook";
+import { logger } from "~/services/logger.server";
+import { alertsWorker } from "~/v3/alertsWorker.server";
+
+// Inbound webhook for status-page incident updates. BetterStack calls this when
+// we publish or edit a status report; we fan the update out to customers over
+// Slack/email/Discord. The webhook is unsigned and BetterStack cannot attach
+// custom headers to status-page subscriptions (it explicitly recommends
+// URL-based auth), so the endpoint is gated by a shared secret passed as
+// `?token=`. The token is redacted from access logs and the request logger at
+// ingress (see redactSensitiveQueryParams in server.ts). When the feature is
+// disabled or no secret is configured we 404 so the endpoint isn't advertised.
+//
+// We respond 200 quickly and hand delivery to the background worker. BetterStack
+// redelivers on failure (up to 10 times), so the enqueue is deduped on the
+// update id to avoid notifying twice for the same update.
+export async function action({ request }: ActionFunctionArgs) {
+  if (request.method.toUpperCase() !== "POST") {
+    return json({ error: "Method not allowed" }, { status: 405 });
+  }
+
+  const secret = env.BETTERSTACK_INCIDENT_WEBHOOK_SECRET;
+  if (env.INCIDENT_NOTIFY_ENABLED !== "1" || !secret) {
+    return json({ error: "Not found" }, { status: 404 });
+  }
+
+  const token = new URL(request.url).searchParams.get("token") ?? "";
+  if (!secretsMatch(token, secret)) {
+    return json({ error: "Invalid token" }, { status: 401 });
+  }
+
+  const rawBody = await request.text();
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(rawBody);
+  } catch {
+    return json({ error: "Invalid JSON" }, { status: 400 });
+  }
+
+  const payload = IncidentWebhookSchema.safeParse(parsed);
+  if (!payload.success) {
+    logger.warn("BetterStack incident webhook: invalid payload", {
+      issues: payload.error.issues,
+    });
+    return json({ error: "Invalid payload", issues: payload.error.issues }, { status: 400 });
+  }
+
+  // Maintenance and component-update events are not customer incidents.
+  if (!isCustomerNotifiableEvent(payload.data)) {
+    return json({ ignored: true, reason: "non_incident_event" }, { status: 200 });
+  }
+
+  const update = normalizeIncidentUpdate(payload.data);
+  if (!update) {
+    return json({ ignored: true, reason: "no_updates" }, { status: 200 });
+  }
+
+  await alertsWorker.enqueueOnce({
+    id: `incident-notify:${update.updateId}`,
+    job: "v3.fanoutIncidentNotification",
+    payload: update,
+  });
+
+  return json({ received: true }, { status: 200 });
+}
+
+// timingSafeEqual requires equal-length buffers; hashing both sides to a fixed
+// width keeps the comparison constant-time without leaking the secret length.
+function secretsMatch(a: string, b: string): boolean {
+  const aHash = createHash("sha256").update(a).digest();
+  const bHash = createHash("sha256").update(b).digest();
+  return timingSafeEqual(aHash, bHash);
+}

apps/webapp/app/services/betterstack/incidentWebhook.ts

@@ -0,0 +1,117 @@
+import { z } from "zod";
+
+// Payload shape for the BetterStack status-page "subscribe via webhook"
+// integration. This fires when a status report is published or updated on the
+// status page (event_type "incident"). Maintenance windows and individual
+// component changes arrive as separate event types that we ignore.
+//
+// Note: this webhook is NOT cryptographically signed by BetterStack, so the
+// route gates access with a shared secret in the URL instead.
+
+// BetterStack sends resource ids as JSON numbers, not strings. Accept either
+// and normalize to a string so the rest of the pipeline (dedup keys, logs) can
+// treat every id uniformly.
+const IdSchema = z.union([z.string(), z.number()]).transform((v) => String(v));
+
+export const IncidentUpdateSchema = z.object({
+  id: IdSchema,
+  status_report_id: IdSchema.optional(),
+  body: z.string().nullish(),
+  created_at: z.string().nullish(),
+  updated_at: z.string().nullish(),
+});
+
+export const IncidentWebhookSchema = z.object({
+  event_type: z.string(),
+  page: z
+    .object({
+      id: IdSchema.optional(),
+      status_indicator: z.string().nullish(),
+      status_description: z.string().nullish(),
+    })
+    .optional(),
+  // Optional so maintenance/component-update callbacks (which carry no
+  // `incident`) still parse and fall through to the ignore path, rather than
+  // failing with a 400 that BetterStack would retry.
+  incident: z
+    .object({
+      id: IdSchema,
+      name: z.string().nullish(),
+      created_at: z.string().nullish(),
+      updated_at: z.string().nullish(),
+      shortlink: z.string().nullish(),
+      incident_updates: z.array(IncidentUpdateSchema).default([]),
+    })
+    .optional(),
+});
+
+export type IncidentWebhook = z.infer<typeof IncidentWebhookSchema>;
+
+export const NormalizedIncidentUpdateSchema = z.object({
+  incidentId: z.string(),
+  updateId: z.string(),
+  name: z.string(),
+  statusIndicator: z.string(),
+  body: z.string(),
+  shortlink: z.string().nullable(),
+  updatedAt: z.string().nullable(),
+});
+
+export type NormalizedIncidentUpdate = {
+  /** Stable id of the incident itself, for grouping all of its updates. */
+  incidentId: string;
+  /** Id of the specific status report update — our idempotency key. */
+  updateId: string;
+  /** Incident title shown on the status page. */
+  name: string;
+  /** Latest aggregate state of the page: operational | degraded | downtime | maintenance. */
+  statusIndicator: string;
+  /** Body of the most recent status report update (markdown). */
+  body: string;
+  /** Public shortlink to the incident on the status page. */
+  shortlink: string | null;
+  /** ISO timestamp of the most recent update, when provided. */
+  updatedAt: string | null;
+};
+
+/**
+ * Only status-report ("incident") events feed customer notifications. Auto
+ * alerting from monitors arrives via a different webhook and never as this
+ * event type, which is exactly the separation we want.
+ */
+export function isCustomerNotifiableEvent(payload: IncidentWebhook): boolean {
+  return payload.event_type === "incident" && !!payload.incident;
+}
+
+/**
+ * Collapse the webhook into the single most-recent update. Returns null when
+ * there are no updates to relay (nothing to notify about yet).
+ */
+export function normalizeIncidentUpdate(payload: IncidentWebhook): NormalizedIncidentUpdate | null {
+  if (!payload.incident) {
+    return null;
+  }
+
+  const updates = payload.incident.incident_updates;
+  if (updates.length === 0) {
+    return null;
+  }
+
+  // BetterStack lists updates newest-first; fall back to the last entry if the
+  // ordering ever changes by sorting on created_at when available.
+  const mostRecent = [...updates].sort((a, b) => {
+    const aTime = a.created_at ? Date.parse(a.created_at) : 0;
+    const bTime = b.created_at ? Date.parse(b.created_at) : 0;
+    return bTime - aTime;
+  })[0];
+
+  return {
+    incidentId: payload.incident.id,
+    updateId: mostRecent.id,
+    name: payload.incident.name?.trim() || "Service incident",
+    statusIndicator: payload.page?.status_indicator?.trim() || "downtime",
+    body: mostRecent.body?.trim() || "",
+    shortlink: payload.incident.shortlink ?? null,
+    updatedAt: mostRecent.created_at ?? payload.incident.updated_at ?? null,
+  };
+}

apps/webapp/app/utils/redactUrl.ts

@@ -0,0 +1,32 @@
+// Query params that carry credentials and must never reach access logs, the
+// structured request logger, or traces. Some inbound webhooks (e.g. the
+// BetterStack status-page webhook) can only authenticate via a URL query token
+// because the sender cannot attach custom headers, so we redact at ingress.
+export const SENSITIVE_QUERY_PARAMS = ["token", "secret", "access_token", "api_key"];
+
+/**
+ * Return `url` with the values of any sensitive query params replaced by
+ * `[redacted]`. Accepts absolute URLs or path+query strings. Never throws — a
+ * malformed URL is returned unchanged so redaction can't break request handling
+ * or span creation.
+ */
+export function redactSensitiveQueryParams(url: string): string {
+  const queryStart = url.indexOf("?");
+  if (queryStart === -1) {
+    return url;
+  }
+
+  try {
+    const params = new URLSearchParams(url.slice(queryStart + 1));
+    let didRedact = false;
+    for (const key of SENSITIVE_QUERY_PARAMS) {
+      if (params.has(key)) {
+        params.set(key, "[redacted]");
+        didRedact = true;
+      }
+    }
+    return didRedact ? `${url.slice(0, queryStart)}?${params.toString()}` : url;
+  } catch {
+    return url;
+  }
+}

apps/webapp/app/v3/alertsWorker.server.ts

@@ -4,9 +4,17 @@ import { z } from "zod";
 import { env } from "~/env.server";
 import { logger } from "~/services/logger.server";
 import { singleton } from "~/utils/singleton";
+import { NormalizedIncidentUpdateSchema } from "~/services/betterstack/incidentWebhook";
 import { DeliverAlertService } from "./services/alerts/deliverAlert.server";
 import { DeliverErrorGroupAlertService } from "./services/alerts/deliverErrorGroupAlert.server";
 import { ErrorAlertEvaluator } from "./services/alerts/errorAlertEvaluator.server";
+import { deliverIncidentToDiscord } from "./services/alerts/incidentNotifications/deliverDiscord.server";
+import {
+  deliverIncidentEmailPage,
+  deliverIncidentEmailToRecipient,
+} from "./services/alerts/incidentNotifications/deliverEmail.server";
+import { deliverIncidentToSlack } from "./services/alerts/incidentNotifications/deliverSlack.server";
+import { fanoutIncidentNotification } from "./services/alerts/incidentNotifications/fanout.server";
 import { PerformDeploymentAlertsService } from "./services/alerts/performDeploymentAlerts.server";
 import { PerformTaskRunAlertsService } from "./services/alerts/performTaskRunAlerts.server";
 
@@ -93,6 +101,52 @@ function initializeWorker() {
         },
         logErrors: true,
       },
+      "v3.fanoutIncidentNotification": {
+        schema: NormalizedIncidentUpdateSchema,
+        visibilityTimeoutMs: 30_000,
+        retry: {
+          maxAttempts: 3,
+        },
+        logErrors: true,
+      },
+      "v3.deliverIncidentSlack": {
+        schema: z.object({ update: NormalizedIncidentUpdateSchema }),
+        visibilityTimeoutMs: 60_000,
+        retry: {
+          maxAttempts: 3,
+        },
+        logErrors: true,
+      },
+      "v3.deliverIncidentDiscord": {
+        schema: z.object({ update: NormalizedIncidentUpdateSchema }),
+        visibilityTimeoutMs: 30_000,
+        retry: {
+          maxAttempts: 3,
+        },
+        logErrors: true,
+      },
+      "v3.deliverIncidentEmail": {
+        schema: z.object({
+          update: NormalizedIncidentUpdateSchema,
+          cursor: z.string().nullable(),
+        }),
+        visibilityTimeoutMs: 60_000,
+        retry: {
+          maxAttempts: 3,
+        },
+        logErrors: true,
+      },
+      "v3.deliverIncidentEmailRetry": {
+        schema: z.object({
+          update: NormalizedIncidentUpdateSchema,
+          recipient: z.object({ userId: z.string(), email: z.string() }),
+        }),
+        visibilityTimeoutMs: 30_000,
+        retry: {
+          maxAttempts: 3,
+        },
+        logErrors: true,
+      },
     },
     concurrency: {
       workers: env.ALERTS_WORKER_CONCURRENCY_WORKERS,
@@ -126,6 +180,21 @@ function initializeWorker() {
         const service = new DeliverErrorGroupAlertService();
         await service.call(payload);
       },
+      "v3.fanoutIncidentNotification": async ({ payload }) => {
+        await fanoutIncidentNotification(payload);
+      },
+      "v3.deliverIncidentSlack": async ({ payload }) => {
+        await deliverIncidentToSlack(payload.update);
+      },
+      "v3.deliverIncidentDiscord": async ({ payload }) => {
+        await deliverIncidentToDiscord(payload.update);
+      },
+      "v3.deliverIncidentEmail": async ({ payload }) => {
+        await deliverIncidentEmailPage(payload);
+      },
+      "v3.deliverIncidentEmailRetry": async ({ payload }) => {
+        await deliverIncidentEmailToRecipient(payload);
+      },
     },
   });