diff --git a/.changeset/convex-agent-skill.md b/.changeset/convex-agent-skill.md new file mode 100644 index 00000000000..7ec8e33df2d --- /dev/null +++ b/.changeset/convex-agent-skill.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/convex": patch +--- + +Add a bundled agent skill for integrating Trigger.dev with Convex, including transactional triggering, Realtime progress, recovery, and scoped-token safety. Install it manually from the package; it is not part of the CLI skills installer. diff --git a/apps/webapp/.gitignore b/apps/webapp/.gitignore index f825411d640..dfd5391315d 100644 --- a/apps/webapp/.gitignore +++ b/apps/webapp/.gitignore @@ -23,4 +23,3 @@ storybook-static /prisma/seed.js /prisma/populate.js -.memory-snapshots diff --git a/apps/webapp/app/components/navigation/sideMenuSections.tsx b/apps/webapp/app/components/navigation/sideMenuSections.tsx index d7c0970a6d1..a6bb3503d6a 100644 --- a/apps/webapp/app/components/navigation/sideMenuSections.tsx +++ b/apps/webapp/app/components/navigation/sideMenuSections.tsx @@ -291,7 +291,7 @@ export function buildSideMenuSections({ ? [ { id: "oidc", - name: "OIDC", + name: "Workload identity", icon: KeyIcon, activeIconColor: "text-text-bright", to: v3OidcPath(organization, project, environment), diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index e2fad985099..260eee18685 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -429,6 +429,7 @@ const EnvironmentSchema = z ADMIN_EMAILS: z.string().refine(isValidRegex, "ADMIN_EMAILS must be a valid regex.").optional(), // Instance-level kill switch for the admin dashboard and user impersonation. ADMIN_DASHBOARD_ENABLED: BoolEnv.default(true), + BILLING_SERVICE_SECRET: z.string().min(32).optional(), REMIX_APP_PORT: z.string().optional(), // Opt-in, dev-only: stream this process's logs over a local telnet/TCP socket on this port. // Read directly from process.env in server.ts (before this schema loads); declared here for discoverability. @@ -1359,6 +1360,8 @@ const EnvironmentSchema = z RUN_ENGINE_CONCURRENCY_SWEEPER_PROCESS_MARKED_SCHEDULE: z.string().optional(), RUN_ENGINE_CONCURRENCY_SWEEPER_SCAN_JITTER_IN_MS: z.coerce.number().int().optional(), RUN_ENGINE_CONCURRENCY_SWEEPER_PROCESS_MARKED_JITTER_IN_MS: z.coerce.number().int().optional(), + // Blank or 0 means the engine default of 1000; clamped to 1..10000 in RunQueue. + RUN_ENGINE_CONCURRENCY_SWEEPER_SCAN_COUNT: OptionalLimitEnv, // TTL System settings for automatic run expiration RUN_ENGINE_TTL_SYSTEM_DISABLED: BoolEnv.default(false), diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.oidc/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.oidc/route.tsx index 787338d8358..d392406d9d6 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.oidc/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.oidc/route.tsx @@ -27,7 +27,7 @@ import { AWS_S3_READ_PERMISSIONS, awsOidcTaskExample, awsOidcTrustPolicy } from import { pageMeta } from "~/utils/pageTitle"; import { docsPath, EnvironmentParamSchema } from "~/utils/pathBuilder"; -export const meta = pageMeta("OpenID Connect (OIDC)"); +export const meta = pageMeta("Workload identity"); export const loader = dashboardLoader( { @@ -80,21 +80,21 @@ export default function OidcSetupPage() { return ( - + - OIDC documentation + Workload identity documentation - Access AWS from deployed tasks using short-lived credentials. + Access AWS from your tasks without storing long-lived credentials. {state === "development" ? ( - OIDC requires a deployed environment. Select Production, Staging or Preview. For local - development, use your own AWS credentials. + Workload identity requires a deployed environment. Select Production, Staging or + Preview. For local development, use your own AWS credentials. ) : setup ? ( <> diff --git a/apps/webapp/app/routes/admin.api.v1.gc.ts b/apps/webapp/app/routes/admin.api.v1.gc.ts deleted file mode 100644 index 7df456298b7..00000000000 --- a/apps/webapp/app/routes/admin.api.v1.gc.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { type DataFunctionArgs } from "@remix-run/node"; -import { PerformanceObserver } from "node:perf_hooks"; -import { runInNewContext } from "node:vm"; -import v8 from "v8"; -import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; - -async function waitTillGcFinishes() { - let resolver: (value: PerformanceEntry) => void; - let _rejector: (reason?: any) => void; - - const promise = new Promise((resolve, reject) => { - resolver = resolve; - _rejector = reject; - }); - - const obs = new PerformanceObserver((list) => { - const entry = list.getEntries()[0]; - - if (entry.name === "gc") { - resolver(entry); - } - }); - - obs.observe({ entryTypes: ["gc"] }); - - v8.setFlagsFromString("--expose-gc"); - const gc = global.gc ?? runInNewContext("gc"); - - gc(); - - // disable expose-gc - v8.setFlagsFromString("--noexpose-gc"); - - return promise; -} - -export async function loader({ request }: DataFunctionArgs) { - await requireAdminApiRequest(request); - - const entry = await waitTillGcFinishes(); - - return new Response(JSON.stringify(entry), { - status: 200, - headers: { - "Content-Type": "application/json", - }, - }); -} diff --git a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.billing-limit.hit.ts b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.billing-limit.hit.ts index e6e7cd984c1..01a6290e476 100644 --- a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.billing-limit.hit.ts +++ b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.billing-limit.hit.ts @@ -6,7 +6,7 @@ import { type BillingLimitHitWebhookBody, } from "~/services/billingLimit.schemas"; import { logger } from "~/services/logger.server"; -import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; +import { requireBillingServiceOrAdminApiRequest } from "~/services/billingServiceAuth.server"; import { bustBillingLimitCaches } from "~/services/platform.v3.server"; import { enqueueBillingLimitCancelInProgressRuns, @@ -21,7 +21,7 @@ const ParamsSchema = z.object({ /** Billing platform webhook: org entered billing limit grace. Idempotent — returns 202. */ export async function action({ request, params }: ActionFunctionArgs) { - await requireAdminApiRequest(request); + await requireBillingServiceOrAdminApiRequest(request); if (request.method.toLowerCase() !== "post") { return json({ error: "Method not allowed" }, { status: 405 }); diff --git a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.billing-limit.reject.ts b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.billing-limit.reject.ts index de4362e5ef2..3832d9c3031 100644 --- a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.billing-limit.reject.ts +++ b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.billing-limit.reject.ts @@ -1,7 +1,7 @@ import { type ActionFunctionArgs, json } from "@remix-run/server-runtime"; import { z } from "zod"; import { prisma } from "~/db.server"; -import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; +import { requireBillingServiceOrAdminApiRequest } from "~/services/billingServiceAuth.server"; import { bustBillingLimitCaches } from "~/services/platform.v3.server"; import { BillingLimitConvergeEnvironmentsService } from "~/v3/services/billingLimit/billingLimitConvergeEnvironmentsService.server"; import { enqueueBillingLimitConverge } from "~/v3/billingLimitWorker.server"; @@ -12,7 +12,7 @@ const ParamsSchema = z.object({ /** Billing platform webhook: org billing limit grace expired. Idempotent — returns 202. */ export async function action({ request, params }: ActionFunctionArgs) { - await requireAdminApiRequest(request); + await requireBillingServiceOrAdminApiRequest(request); if (request.method.toLowerCase() !== "post") { return json({ error: "Method not allowed" }, { status: 405 }); diff --git a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.billing-limit.resolve.ts b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.billing-limit.resolve.ts index e41519ec81b..81e8787e949 100644 --- a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.billing-limit.resolve.ts +++ b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.billing-limit.resolve.ts @@ -1,7 +1,7 @@ import { type ActionFunctionArgs, json } from "@remix-run/server-runtime"; import { z } from "zod"; import { prisma } from "~/db.server"; -import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; +import { requireBillingServiceOrAdminApiRequest } from "~/services/billingServiceAuth.server"; import { bustBillingLimitCaches } from "~/services/platform.v3.server"; import { logger } from "~/services/logger.server"; import { enqueueBillingLimitResolve } from "~/v3/billingLimitWorker.server"; @@ -19,7 +19,7 @@ const BodySchema = z.object({ /** Billing platform webhook: org resolved billing limit to ok. Idempotent — returns 202. */ export async function action({ request, params }: ActionFunctionArgs) { - await requireAdminApiRequest(request); + await requireBillingServiceOrAdminApiRequest(request); if (request.method.toLowerCase() !== "post") { return json({ error: "Method not allowed" }, { status: 405 }); diff --git a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.concurrency.ts b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.concurrency.ts index a2273af1c13..05b933db5a1 100644 --- a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.concurrency.ts +++ b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.concurrency.ts @@ -2,7 +2,7 @@ import type { ActionFunctionArgs } from "@remix-run/server-runtime"; import { json } from "@remix-run/server-runtime"; import { z } from "zod"; import { prisma } from "~/db.server"; -import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; +import { requireBillingServiceOrAdminApiRequest } from "~/services/billingServiceAuth.server"; import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; import { setOrgConcurrencyLimits } from "~/v3/services/orgPlanProvisioning.server"; import { enqueueOrgProvisioningJob } from "~/v3/services/orgProvisioningJobs.server"; @@ -19,7 +19,7 @@ const RequestBodySchema = z.object({ }); export async function action({ request, params }: ActionFunctionArgs) { - await requireAdminApiRequest(request); + await requireBillingServiceOrAdminApiRequest(request); const { organizationId } = ParamsSchema.parse(params); diff --git a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.environments.staging.ts b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.environments.staging.ts index 017b6b78c03..ff198ddc167 100644 --- a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.environments.staging.ts +++ b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.environments.staging.ts @@ -1,7 +1,7 @@ import { type ActionFunctionArgs, json } from "@remix-run/server-runtime"; import { z } from "zod"; import { prisma } from "~/db.server"; -import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; +import { requireBillingServiceOrAdminApiRequest } from "~/services/billingServiceAuth.server"; import { enqueueOrgProvisioningJob } from "~/v3/services/orgProvisioningJobs.server"; const ParamsSchema = z.object({ @@ -9,7 +9,7 @@ const ParamsSchema = z.object({ }); export async function action({ request, params }: ActionFunctionArgs) { - await requireAdminApiRequest(request); + await requireBillingServiceOrAdminApiRequest(request); const { organizationId } = ParamsSchema.parse(params); diff --git a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.runs.enable.ts b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.runs.enable.ts index ae43e3f4a4f..5f4951db5fe 100644 --- a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.runs.enable.ts +++ b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.runs.enable.ts @@ -1,7 +1,7 @@ import { type ActionFunctionArgs, json } from "@remix-run/server-runtime"; import { z } from "zod"; import { prisma } from "~/db.server"; -import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; +import { requireBillingServiceOrAdminApiRequest } from "~/services/billingServiceAuth.server"; import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; import { enqueueOrgProvisioningJob } from "~/v3/services/orgProvisioningJobs.server"; @@ -14,7 +14,7 @@ const BodySchema = z.object({ }); export async function action({ request, params }: ActionFunctionArgs) { - await requireAdminApiRequest(request); + await requireBillingServiceOrAdminApiRequest(request); const { organizationId } = ParamsSchema.parse(params); const body = BodySchema.safeParse(await request.json()); diff --git a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.stream-basin.ts b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.stream-basin.ts index 20305e393e6..59966932a05 100644 --- a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.stream-basin.ts +++ b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.stream-basin.ts @@ -1,7 +1,7 @@ import type { ActionFunctionArgs } from "@remix-run/server-runtime"; import { json } from "@remix-run/server-runtime"; import { z } from "zod"; -import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; +import { requireBillingServiceOrAdminApiRequest } from "~/services/billingServiceAuth.server"; import { isValidDuration } from "~/services/realtime/duration.server"; import { deprovisionBasinForOrg, @@ -21,7 +21,7 @@ const BodySchema = z.discriminatedUnion("action", [ ]); export async function action({ request, params }: ActionFunctionArgs) { - await requireAdminApiRequest(request); + await requireBillingServiceOrAdminApiRequest(request); const { organizationId } = ParamsSchema.parse(params); diff --git a/apps/webapp/app/routes/admin.api.v1.snapshot.ts b/apps/webapp/app/routes/admin.api.v1.snapshot.ts deleted file mode 100644 index daba88f2a42..00000000000 --- a/apps/webapp/app/routes/admin.api.v1.snapshot.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { type DataFunctionArgs } from "@remix-run/node"; -import fs from "fs"; -import os from "os"; -import path from "path"; -import { PassThrough } from "stream"; -import v8 from "v8"; -import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; - -// Format date as yyyy-MM-dd HH_mm_ss_SSS -function formatDate(date: Date) { - const year = date.getFullYear(); - const month = date.getMonth() + 1; - const day = date.getDate(); - const hours = date.getHours(); - const minutes = date.getMinutes(); - const seconds = date.getSeconds(); - const milliseconds = date.getMilliseconds(); - - return `${year}-${month.toString().padStart(2, "0")}-${day.toString().padStart(2, "0")} ${hours - .toString() - .padStart(2, "0")}_${minutes.toString().padStart(2, "0")}_${seconds - .toString() - .padStart(2, "0")}_${milliseconds.toString().padStart(3, "0")}`; -} - -export async function loader({ request }: DataFunctionArgs) { - await requireAdminApiRequest(request); - - const tempDir = os.tmpdir(); - const filepath = path.join( - tempDir, - `${getTaskIdentifier()}-${formatDate(new Date())}.heapsnapshot` - ); - - const snapshotPath = v8.writeHeapSnapshot(filepath); - if (!snapshotPath) { - throw new Response("No snapshot saved", { status: 500 }); - } - - const body = new PassThrough(); - const stream = fs.createReadStream(snapshotPath); - stream.on("open", () => stream.pipe(body)); - stream.on("error", (err) => body.end(err)); - stream.on("end", () => body.end()); - - return new Response(body as any, { - status: 200, - headers: { - "Content-Type": "application/octet-stream", - "Content-Disposition": `attachment; filename="${path.basename(snapshotPath)}"`, - "Content-Length": (await fs.promises.stat(snapshotPath)).size.toString(), - }, - }); -} - -function getTaskIdentifier() { - if (!process.env.ECS_CONTAINER_METADATA_URI) { - return "local"; - } - - const url = new URL(process.env.ECS_CONTAINER_METADATA_URI); - - return url.pathname.split("/")[2].split("-")[0]; -} diff --git a/apps/webapp/app/services/billingServiceAuth.server.ts b/apps/webapp/app/services/billingServiceAuth.server.ts new file mode 100644 index 00000000000..df3bbb49929 --- /dev/null +++ b/apps/webapp/app/services/billingServiceAuth.server.ts @@ -0,0 +1,35 @@ +import { createHash, timingSafeEqual } from "node:crypto"; +import { env } from "~/env.server"; +import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; + +/** + * Guards the admin API routes the billing service calls. Accepts the billing service secret + * (`BILLING_SERVICE_SECRET`) or, as on every other admin route, an admin's personal access token. + */ +export async function requireBillingServiceOrAdminApiRequest(request: Request): Promise { + if (isBillingServiceRequest(request, env.BILLING_SERVICE_SECRET)) { + return; + } + + await requireAdminApiRequest(request); +} + +function isBillingServiceRequest(request: Request, secret: string | undefined): boolean { + if (!secret) { + return false; + } + + const authorization = request.headers.get("Authorization"); + if (!authorization?.startsWith("Bearer ")) { + return false; + } + + return digestEquals(authorization.slice("Bearer ".length), secret); +} + +function digestEquals(a: string, b: string): boolean { + return timingSafeEqual( + createHash("sha256").update(a).digest(), + createHash("sha256").update(b).digest() + ); +} diff --git a/apps/webapp/app/services/oidc/externalOidc.server.ts b/apps/webapp/app/services/oidc/externalOidc.server.ts index 38f23cc73a2..ebec23527d2 100644 --- a/apps/webapp/app/services/oidc/externalOidc.server.ts +++ b/apps/webapp/app/services/oidc/externalOidc.server.ts @@ -10,7 +10,9 @@ import { env } from "~/env.server"; import { isCloud } from "~/services/platform.v3.server"; import { createRedisRateLimitClient } from "~/services/rateLimiterCore.server"; import { singleton } from "~/utils/singleton"; +import { globalFlagsRegistry } from "~/v3/globalFlagsRegistry.server"; import { runStore } from "~/v3/runStore.server"; +import { resolveExternalOidcFlags } from "./externalOidcFeatureFlags"; import { loadExternalOidcPlugin } from "./externalOidcPlugin.server"; import { ExternalOidcRateLimits } from "./externalOidcRateLimit.server"; import { externalOidcSigningService } from "./instance.server"; @@ -37,7 +39,11 @@ export function getExternalOidcController(): Promise { if (!runTokenInjectionEnabled(orgFeatureFlags)) return false; const controller = await getExternalOidcController(); - return controller?.isEnabled(orgFeatureFlags) ?? false; + return ( + controller?.isEnabled( + resolveExternalOidcFlags(globalFlagsRegistry.current() ?? {}, orgFeatureFlags) + ) ?? false + ); } async function findEnvironment( @@ -70,7 +76,10 @@ async function findEnvironment( organizationId: environment.organizationId, projectId: environment.projectId, projectRef: environment.project.externalRef, - orgFeatureFlags: environment.organization.featureFlags, + orgFeatureFlags: resolveExternalOidcFlags( + globalFlagsRegistry.current() ?? {}, + environment.organization.featureFlags + ), }; } diff --git a/apps/webapp/app/services/oidc/externalOidcFeatureFlags.ts b/apps/webapp/app/services/oidc/externalOidcFeatureFlags.ts new file mode 100644 index 00000000000..0bf09c75c10 --- /dev/null +++ b/apps/webapp/app/services/oidc/externalOidcFeatureFlags.ts @@ -0,0 +1,13 @@ +import { FeatureFlagCatalog, isPlainFlagObject } from "~/v3/featureFlags"; + +export function resolveExternalOidcFlags( + globalFlags: Partial, + orgFeatureFlags: unknown +): { externalOidc: boolean } { + const override = FeatureFlagCatalog.externalOidc.safeParse( + isPlainFlagObject(orgFeatureFlags) ? orgFeatureFlags.externalOidc : undefined + ); + return { + externalOidc: override.success ? override.data : (globalFlags.externalOidc ?? false), + }; +} diff --git a/apps/webapp/app/v3/featureFlags.ts b/apps/webapp/app/v3/featureFlags.ts index 9ad47d6dc2c..461be6794fb 100644 --- a/apps/webapp/app/v3/featureFlags.ts +++ b/apps/webapp/app/v3/featureFlags.ts @@ -236,6 +236,7 @@ export const FeatureFlagCatalog = { // Injects a signed per-attempt TRIGGER_RUN_TOKEN into deployed runs. Org override wins over // the global value. Requires RUN_TOKEN_ISSUER_URL and an active internal run signing key. [FEATURE_FLAG.runTokenInjection]: z.boolean(), + // Org override wins over the global value; off by default. [FEATURE_FLAG.externalOidc]: z.boolean(), // Shows the session page's Trace tab and serves its routes. Org override wins over the global // value; off by default. Admins always have it. @@ -254,7 +255,6 @@ export const GLOBAL_LOCKED_FLAGS: FeatureFlagKey[] = [ FEATURE_FLAG.runOpsMintShard, FEATURE_FLAG.runOpsMintShardEnvPins, FEATURE_FLAG.apiRateLimitMetricsEnabled, - FEATURE_FLAG.externalOidc, FEATURE_FLAG.workerQueueCompatibility, FEATURE_FLAG.workerQueueChannel, FEATURE_FLAG.workerQueueEnvironmentOverrides, diff --git a/apps/webapp/app/v3/runEngine.server.ts b/apps/webapp/app/v3/runEngine.server.ts index 571c1c364c4..1f8eb851ae0 100644 --- a/apps/webapp/app/v3/runEngine.server.ts +++ b/apps/webapp/app/v3/runEngine.server.ts @@ -112,6 +112,7 @@ function createRunEngine() { processMarkedSchedule: env.RUN_ENGINE_CONCURRENCY_SWEEPER_PROCESS_MARKED_SCHEDULE, scanJitterInMs: env.RUN_ENGINE_CONCURRENCY_SWEEPER_SCAN_JITTER_IN_MS, processMarkedJitterInMs: env.RUN_ENGINE_CONCURRENCY_SWEEPER_PROCESS_MARKED_JITTER_IN_MS, + scanCount: env.RUN_ENGINE_CONCURRENCY_SWEEPER_SCAN_COUNT, }, ttlSystem: { disabled: env.RUN_ENGINE_TTL_SYSTEM_DISABLED, diff --git a/apps/webapp/memory-leak-detector.js b/apps/webapp/memory-leak-detector.js deleted file mode 100644 index 7bec919d156..00000000000 --- a/apps/webapp/memory-leak-detector.js +++ /dev/null @@ -1,979 +0,0 @@ -#!/usr/bin/env node - -/** - * Memory Leak Detector for Trigger.dev Webapp - * - * This script starts the server, performs memory snapshots, executes API requests, - * and analyzes memory usage patterns to detect potential memory leaks. - * - * Usage: node memory-leak-detector.js [options] - */ - -const { spawn, exec: _exec } = require("child_process"); -const fs = require("fs"); -const path = require("path"); -const http = require("http"); -const { performance } = require("perf_hooks"); - -class MemoryLeakDetector { - constructor(options = {}) { - // Create timestamped directory for this run - const runTimestamp = new Date().toISOString().replace(/[:.]/g, "-"); - const baseDir = options.baseDir || "./.memory-snapshots"; - const runDir = `${baseDir}/${runTimestamp}-${options.label || "memory-leak-detector"}`; - - this.options = { - // Server configuration - serverPort: options.serverPort || 3030, - serverStartTimeout: options.serverStartTimeout || 30000, - - // Sentry configuration - sentryDsn: options.sentryDsn || "", - - // Memory snapshot configuration - heapSnapshotPath: `${runDir}/snapshots`, - adminToken: options.adminToken, // Bearer token for admin API access - - apiKey: options.apiKey, - - // Testing configuration - warmupRequests: options.warmupRequests || 10, - testRequests: options.testRequests || 100, - requestDelay: options.requestDelay || 50, // ms between requests - requestTimeout: options.requestTimeout || 5000, - - // API endpoints to test (configurable) - apiEndpoints: options.apiEndpoints || ["/api/v1/runs"], - postApiEndpoints: options.postApiEndpoints || ["/api/v1/mock"], - - // Memory analysis thresholds - memoryLeakThreshold: options.memoryLeakThreshold || 50, // MB increase - heapGrowthThreshold: options.heapGrowthThreshold || 0.2, // 20% growth - - // Output configuration - verbose: options.verbose || false, - outputFile: `${runDir}/memory-leak-report.json`, - runDir: runDir, - runTimestamp: runTimestamp, - - ...options, - }; - - this.serverProcess = null; - this.results = { - startTime: new Date().toISOString(), - runTimestamp: runTimestamp, - runDirectory: runDir, - serverInfo: {}, - snapshots: [], - testPhases: [], // Track each phase of testing - totalRequests: { - warmup: 0, - phase1: 0, - phase2: 0, - successful: 0, - failed: 0, - errors: [], - }, - memoryAnalysis: {}, - recommendations: [], - }; - } - - log(message, level = "info") { - const timestamp = new Date().toISOString(); - const prefix = `[${timestamp}] [${level.toUpperCase()}]`; - - if (level === "error") { - console.error(`${prefix} ${message}`); - } else if (this.options.verbose || level === "info") { - console.log(`${prefix} ${message}`); - } - } - - async ensureSnapshotDir() { - if (!fs.existsSync(this.options.runDir)) { - fs.mkdirSync(this.options.runDir, { recursive: true }); - this.log(`Created run directory: ${this.options.runDir}`); - } - if (!fs.existsSync(this.options.heapSnapshotPath)) { - fs.mkdirSync(this.options.heapSnapshotPath, { recursive: true }); - this.log(`Created snapshot directory: ${this.options.heapSnapshotPath}`); - } - } - - async takeHeapSnapshot(label) { - const timestamp = Date.now(); - const filename = `heap-${label}-${timestamp}.heapsnapshot`; - const filepath = path.join(this.options.heapSnapshotPath, filename); - - try { - let snapshotData; - - if (this.options.adminToken) { - this.log(`Running GC before snapshot...`); - await this.runGc(); - - // Use the admin API endpoint to get actual V8 heap snapshot - this.log(`Taking V8 heap snapshot via admin API: ${label}...`); - - snapshotData = await this.takeV8HeapSnapshot(label, filepath, timestamp); - } else { - // Fallback to basic memory usage info - this.log(`Taking basic memory snapshot: ${label} (no admin token provided)...`); - - const memUsage = process.memoryUsage(); - snapshotData = { - label, - timestamp, - filename: filename.replace(".heapsnapshot", ".json"), - filepath: filepath.replace(".heapsnapshot", ".json"), - processMemory: memUsage, - type: "basic", - }; - - fs.writeFileSync(snapshotData.filepath, JSON.stringify(snapshotData, null, 2)); - } - - this.results.snapshots.push(snapshotData); - this.log(`Snapshot completed: ${label}`); - - return snapshotData; - } catch (error) { - this.log(`Failed to take heap snapshot: ${error.message}`, "error"); - throw error; - } - } - - async takeV8HeapSnapshot(label, filepath, timestamp) { - return new Promise((resolve, reject) => { - const requestOptions = { - hostname: "localhost", - port: this.options.serverPort, - path: "/admin/api/v1/snapshot", - method: "GET", - timeout: 120000, // Heap snapshots can take a while - headers: { - Authorization: `Bearer ${this.options.adminToken}`, - "User-Agent": "memory-leak-detector/1.0", - }, - }; - - const req = http.request(requestOptions, (res) => { - if (res.statusCode !== 200) { - reject(new Error(`Admin API returned ${res.statusCode}: ${res.statusMessage}`)); - return; - } - - const writeStream = fs.createWriteStream(filepath); - let downloadedBytes = 0; - - res.on("data", (chunk) => { - downloadedBytes += chunk.length; - this.log(`Downloaded ${downloadedBytes} bytes`); - writeStream.write(chunk); - }); - - res.on("end", () => { - writeStream.end(); - - const snapshotData = { - label, - timestamp, - filename: path.basename(filepath), - filepath, - size: downloadedBytes, - type: "v8-heapsnapshot", - }; - - this.log(`V8 heap snapshot saved: ${Math.round(downloadedBytes / 1024 / 1024)}MB`); - resolve(snapshotData); - }); - - res.on("error", (error) => { - writeStream.destroy(); - fs.unlink(filepath, () => {}); // Clean up partial file - reject(error); - }); - - writeStream.on("error", (error) => { - res.destroy(); - reject(error); - }); - }); - - req.on("error", (error) => { - reject(new Error(`Failed to connect to admin API: ${error.message}`)); - }); - - req.on("timeout", () => { - req.destroy(); - reject(new Error("Admin API request timeout")); - }); - - req.end(); - }); - } - - async runGc() { - return new Promise((resolve, reject) => { - const requestOptions = { - hostname: "localhost", - port: this.options.serverPort, - path: "/admin/api/v1/gc", - method: "GET", - timeout: 120000, // GC can take a while - headers: { - Authorization: `Bearer ${this.options.adminToken}`, - "User-Agent": "memory-leak-detector/1.0", - }, - }; - - const req = http.request(requestOptions, (res) => { - if (res.statusCode !== 200) { - reject(new Error(`Admin API returned ${res.statusCode}: ${res.statusMessage}`)); - return; - } - - res.on("data", (chunk) => { - this.log(`GC run completed`); - }); - - res.on("end", () => { - resolve(); - }); - - res.on("error", (error) => { - reject(error); - }); - }); - - req.on("error", (error) => { - reject(new Error(`Failed to connect to admin API: ${error.message}`)); - }); - - req.on("timeout", () => { - req.destroy(); - reject(new Error("Admin API request timeout")); - }); - - req.end(); - }); - } - - async startServer() { - return new Promise((resolve, reject) => { - this.log("Starting server..."); - - // First, build the project - const buildProcess = spawn("npm", ["run", "build"], { - cwd: process.cwd(), - stdio: this.options.verbose ? "inherit" : "pipe", - }); - - buildProcess.on("close", (code) => { - if (code !== 0) { - reject(new Error(`Build failed with code ${code}`)); - return; - } - - this.log("Build completed, starting server..."); - - const nodePath = path.resolve(process.cwd(), "../../node_modules/.pnpm/node_modules"); - - this.log(`Using NODE_PATH: ${nodePath}`); - - // Start the server with memory inspection flags - this.serverProcess = spawn( - "node", - ["--max-old-space-size=16384", "--expose-gc", "./build/server.js"], - { - cwd: process.cwd(), - stdio: ["ignore", "pipe", "pipe"], - env: { - ...process.env, - NODE_ENV: "production", - NODE_PATH: nodePath, - PORT: this.options.serverPort, - // Disable Sentry to prevent memory leaks from it - SENTRY_DSN: this.options.sentryDsn, - }, - } - ); - - let serverReady = false; - const timeout = setTimeout(() => { - if (!serverReady) { - this.serverProcess.kill(); - reject(new Error("Server failed to start within timeout")); - } - }, this.options.serverStartTimeout); - - this.serverProcess.stdout.on("data", (data) => { - const output = data.toString(); - if (this.options.verbose) { - console.log("SERVER:", output.trim()); - } - - if (output.includes("server ready") && !serverReady) { - serverReady = true; - clearTimeout(timeout); - - this.results.serverInfo = { - port: this.options.serverPort, - startTime: Date.now(), - nodeVersion: process.version, - }; - - this.log(`Server started on port ${this.options.serverPort}`); - - // Wait a bit more for server to fully initialize - setTimeout(() => resolve(), 2000); - } - }); - - this.serverProcess.stderr.on("data", (data) => { - const error = data.toString(); - if (this.options.verbose || error.includes("Error")) { - console.error("SERVER ERROR:", error.trim()); - } - }); - - this.serverProcess.on("close", (code) => { - if (!serverReady && code !== 0) { - clearTimeout(timeout); - reject(new Error(`Server process exited with code ${code}`)); - } - }); - }); - }); - } - - async makeRequest(endpoint, options = {}) { - return new Promise((resolve, reject) => { - const requestOptions = { - hostname: "localhost", - port: this.options.serverPort, - path: endpoint, - method: options.method || "GET", - timeout: this.options.requestTimeout, - headers: { - "User-Agent": "memory-leak-detector/1.0", - Accept: "application/json", - Authorization: `Bearer ${this.options.apiKey}`, - ...options.headers, - }, - }; - - const req = http.request(requestOptions, (res) => { - let data = ""; - - res.on("data", (chunk) => { - data += chunk; - }); - - res.on("end", () => { - resolve({ - statusCode: res.statusCode, - headers: res.headers, - body: data, - endpoint: endpoint, - }); - }); - }); - - req.on("error", (error) => { - reject({ error, endpoint }); - }); - - req.on("timeout", () => { - req.destroy(); - reject({ error: new Error("Request timeout"), endpoint }); - }); - - if (options.body) { - req.write(options.body); - } - - req.end(); - }); - } - - async performRequestPhase(phaseName, numRequests) { - this.log(`Starting ${phaseName} phase with ${numRequests} GET requests...`); - - const startTime = performance.now(); - let successfulRequests = 0; - let failedRequests = 0; - const errors = []; - - for (let i = 0; i < numRequests; i++) { - const endpoint = this.options.apiEndpoints[i % this.options.apiEndpoints.length]; - - try { - const response = await this.makeRequest(endpoint); - - if (response.statusCode >= 200 && response.statusCode < 300) { - successfulRequests++; - } else { - failedRequests++; - errors.push({ - endpoint, - statusCode: response.statusCode, - error: `HTTP ${response.statusCode}`, - phase: phaseName, - }); - } - } catch (error) { - failedRequests++; - errors.push({ - endpoint: error.endpoint, - error: error.error?.message || "Unknown error", - phase: phaseName, - }); - } - - // Add delay between requests - if (i < numRequests - 1) { - await this.delay(this.options.requestDelay); - } - - // Progress reporting - if (this.options.verbose && numRequests > 25 && (i + 1) % 25 === 0) { - this.log(`${phaseName}: ${i + 1}/${numRequests} requests completed`); - } - } - - this.log(`Continuing with ${phaseName} phase with ${numRequests} POST requests...`); - - for (let i = 0; i < numRequests; i++) { - const endpoint = this.options.postApiEndpoints[i % this.options.postApiEndpoints.length]; - - try { - // Send a LARGE body to try and trigger a memory leak - const response = await this.makeRequest(endpoint, { - method: "POST", - body: JSON.stringify( - Array.from({ length: 1000 }, (_, index) => ({ - id: index, - name: `Mock ${index}`, - description: `Mock ${index} description`, - createdAt: new Date().toISOString(), - updatedAt: new Date().toISOString(), - uuid: crypto.randomUUID(), - })) - ), - }); - - if (response.statusCode >= 200 && response.statusCode < 300) { - successfulRequests++; - } else { - failedRequests++; - errors.push({ - endpoint, - statusCode: response.statusCode, - error: `HTTP ${response.statusCode}`, - phase: phaseName, - }); - } - } catch (error) { - failedRequests++; - errors.push({ - endpoint: error.endpoint, - error: error.error?.message || "Unknown error", - phase: phaseName, - }); - } - - // Add delay between requests - if (i < numRequests - 1) { - await this.delay(this.options.requestDelay); - } - - // Progress reporting - if (this.options.verbose && numRequests > 25 && (i + 1) % 25 === 0) { - this.log(`${phaseName}: ${i + 1}/${numRequests} requests completed`); - } - } - - const endTime = performance.now(); - const duration = endTime - startTime; - - const phaseResults = { - phase: phaseName, - total: numRequests, - successful: successfulRequests, - failed: failedRequests, - errors: errors.slice(0, 5), // Keep only first 5 errors per phase - duration: Math.round(duration), - requestsPerSecond: Math.round((numRequests / duration) * 1000), - }; - - this.results.testPhases.push(phaseResults); - - // Update totals - this.results.totalRequests.successful += successfulRequests; - this.results.totalRequests.failed += failedRequests; - this.results.totalRequests.errors.push(...errors.slice(0, 3)); - - if (phaseName === "warmup") { - this.results.totalRequests.warmup = numRequests; - } else if (phaseName === "load-test-1") { - this.results.totalRequests.phase1 = numRequests; - } else if (phaseName === "load-test-2") { - this.results.totalRequests.phase2 = numRequests; - } - - this.log( - `${phaseName} completed: ${successfulRequests}/${numRequests} successful (${Math.round( - duration - )}ms total)` - ); - - return phaseResults; - } - - analyzeMemoryUsage() { - if (this.results.snapshots.length < 3) { - this.log("Need at least 3 snapshots to analyze memory usage", "warn"); - return; - } - - const snapshot1 = this.results.snapshots[0]; // after warmup - const snapshot2 = this.results.snapshots[1]; // after first load test - const snapshot3 = this.results.snapshots[2]; // after second load test - - let analysis = {}; - let heapGrowth; - let heapGrowthPercent; - - // Handle different snapshot types - if ( - snapshot1.type === "v8-heapsnapshot" && - snapshot2.type === "v8-heapsnapshot" && - snapshot3.type === "v8-heapsnapshot" - ) { - const size1 = snapshot1.size || 0; - const size2 = snapshot2.size || 0; - const size3 = snapshot3.size || 0; - - const growth1to2 = size2 - size1; - const growth2to3 = size3 - size2; - const totalGrowth = size3 - size1; - - const growth1to2Percent = size1 > 0 ? (growth1to2 / size1) * 100 : 0; - const growth2to3Percent = size2 > 0 ? (growth2to3 / size2) * 100 : 0; - const totalGrowthPercent = size1 > 0 ? (totalGrowth / size1) * 100 : 0; - - analysis = { - snapshotAnalysis: { - type: "v8-heapsnapshot", - phase1: { - size: Math.round(size1 / 1024 / 1024), - file: snapshot1.filename, - }, - phase2: { - size: Math.round(size2 / 1024 / 1024), - file: snapshot2.filename, - growthFromPhase1: Math.round(growth1to2 / 1024 / 1024), - growthPercentFromPhase1: Math.round(growth1to2Percent * 100) / 100, - }, - phase3: { - size: Math.round(size3 / 1024 / 1024), - file: snapshot3.filename, - growthFromPhase2: Math.round(growth2to3 / 1024 / 1024), - growthPercentFromPhase2: Math.round(growth2to3Percent * 100) / 100, - }, - total: { - growth: Math.round(totalGrowth / 1024 / 1024), - growthPercent: Math.round(totalGrowthPercent * 100) / 100, - }, - }, - snapshots: this.results.snapshots.length, - snapshotPaths: this.results.snapshots.map((s) => s.filepath), - }; - - // Use total growth for recommendations - heapGrowth = totalGrowth; - heapGrowthPercent = totalGrowthPercent; - } else if (snapshot1.processMemory && snapshot2.processMemory && snapshot3.processMemory) { - // Traditional process memory analysis with 3 snapshots - const heap1 = snapshot1.processMemory.heapUsed; - const heap2 = snapshot2.processMemory.heapUsed; - const heap3 = snapshot3.processMemory.heapUsed; - const rss1 = snapshot1.processMemory.rss; - const rss2 = snapshot2.processMemory.rss; - const rss3 = snapshot3.processMemory.rss; - - const heapGrowth1to2 = heap2 - heap1; - const heapGrowth2to3 = heap3 - heap2; - const totalHeapGrowth = heap3 - heap1; - const rssGrowth1to2 = rss2 - rss1; - const rssGrowth2to3 = rss3 - rss2; - const totalRssGrowth = rss3 - rss1; - - analysis = { - heapUsage: { - phase1: Math.round(heap1 / 1024 / 1024), - phase2: Math.round(heap2 / 1024 / 1024), - phase3: Math.round(heap3 / 1024 / 1024), - growth1to2: Math.round(heapGrowth1to2 / 1024 / 1024), - growth2to3: Math.round(heapGrowth2to3 / 1024 / 1024), - totalGrowth: Math.round(totalHeapGrowth / 1024 / 1024), - totalGrowthPercent: Math.round((totalHeapGrowth / heap1) * 100 * 100) / 100, - }, - rssUsage: { - phase1: Math.round(rss1 / 1024 / 1024), - phase2: Math.round(rss2 / 1024 / 1024), - phase3: Math.round(rss3 / 1024 / 1024), - growth1to2: Math.round(rssGrowth1to2 / 1024 / 1024), - growth2to3: Math.round(rssGrowth2to3 / 1024 / 1024), - totalGrowth: Math.round(totalRssGrowth / 1024 / 1024), - totalGrowthPercent: Math.round((totalRssGrowth / rss1) * 100 * 100) / 100, - }, - snapshots: this.results.snapshots.length, - }; - - heapGrowth = totalHeapGrowth; - heapGrowthPercent = (totalHeapGrowth / heap1) * 100; - } else { - this.log("Mixed or incompatible snapshot types - cannot analyze memory growth", "warn"); - analysis = { - error: "Incompatible snapshot types", - snapshots: this.results.snapshots.length, - snapshotTypes: this.results.snapshots.map((s) => ({ label: s.label, type: s.type })), - }; - this.results.memoryAnalysis = analysis; - return; - } - - this.results.memoryAnalysis = analysis; - - // Generate recommendations - const recommendations = []; - - if (Math.abs(heapGrowth) > this.options.memoryLeakThreshold * 1024 * 1024) { - recommendations.push({ - type: "warning", - message: `Significant heap growth detected: ${Math.round( - heapGrowth / 1024 / 1024 - )}MB increase`, - suggestion: "Consider investigating object retention and event listener cleanup", - }); - } - - if (Math.abs(heapGrowthPercent) > this.options.heapGrowthThreshold * 100) { - recommendations.push({ - type: "warning", - message: `High heap growth percentage: ${heapGrowthPercent.toFixed(1)}%`, - suggestion: "Review memory allocation patterns and garbage collection behavior", - }); - } - - if ( - this.results.totalRequests.failed > - (this.results.totalRequests.phase1 + this.results.totalRequests.phase2) * 0.1 - ) { - recommendations.push({ - type: "error", - message: `High request failure rate: ${this.results.totalRequests.failed}/${ - this.results.totalRequests.phase1 + this.results.totalRequests.phase2 - }`, - suggestion: "Fix failing endpoints before analyzing memory patterns", - }); - } - - if (recommendations.length === 0) { - recommendations.push({ - type: "info", - message: "No obvious memory leaks detected in this test run", - suggestion: "Consider running longer tests or testing different API endpoints", - }); - } - - this.results.recommendations = recommendations; - - // Log analysis results - this.log("=== Memory Analysis Results ==="); - - if (this.results.memoryAnalysis.snapshotAnalysis) { - const analysis = this.results.memoryAnalysis.snapshotAnalysis; - this.log(`V8 Heap Snapshot Analysis:`); - this.log(` Phase 1 (after warmup): ${analysis.phase1.size}MB`); - this.log( - ` Phase 2 (after load test 1): ${analysis.phase2.size}MB (${ - analysis.phase2.growthPercentFromPhase1 > 0 ? "+" : "" - }${analysis.phase2.growthPercentFromPhase1}%)` - ); - this.log( - ` Phase 3 (after load test 2): ${analysis.phase3.size}MB (${ - analysis.phase3.growthPercentFromPhase2 > 0 ? "+" : "" - }${analysis.phase3.growthPercentFromPhase2}%)` - ); - this.log( - ` Total Growth: ${analysis.total.growthPercent > 0 ? "+" : ""}${ - analysis.total.growth - }MB (${analysis.total.growthPercent}%)` - ); - - this.log(`\nSnapshot files saved:`); - this.log(` 1. ${analysis.phase1.file}`); - this.log(` 2. ${analysis.phase2.file}`); - this.log(` 3. ${analysis.phase3.file}`); - this.log(`\nšŸ’” Analyze snapshots in Chrome DevTools:`); - this.log(` 1. Open Chrome DevTools → Memory tab`); - this.log(` 2. Load all 3 .heapsnapshot files`); - this.log(` 3. Compare snapshots to identify memory leaks`); - this.log(` 4. Focus on comparing snapshot 1 vs 3 for overall growth`); - } else if (this.results.memoryAnalysis.heapUsage) { - const heap = this.results.memoryAnalysis.heapUsage; - const rss = this.results.memoryAnalysis.rssUsage; - this.log(`Heap Usage Analysis:`); - this.log(` Phase 1: ${heap.phase1}MB`); - this.log( - ` Phase 2: ${heap.phase2}MB (${heap.growth1to2 > 0 ? "+" : ""}${heap.growth1to2}MB)` - ); - this.log( - ` Phase 3: ${heap.phase3}MB (${heap.growth2to3 > 0 ? "+" : ""}${heap.growth2to3}MB)` - ); - this.log( - ` Total Growth: ${heap.totalGrowthPercent > 0 ? "+" : ""}${heap.totalGrowth}MB (${ - heap.totalGrowthPercent - }%)` - ); - - this.log(`RSS Usage Analysis:`); - this.log(` Phase 1: ${rss.phase1}MB`); - this.log(` Phase 2: ${rss.phase2}MB (${rss.growth1to2 > 0 ? "+" : ""}${rss.growth1to2}MB)`); - this.log(` Phase 3: ${rss.phase3}MB (${rss.growth2to3 > 0 ? "+" : ""}${rss.growth2to3}MB)`); - this.log( - ` Total Growth: ${rss.totalGrowthPercent > 0 ? "+" : ""}${rss.totalGrowth}MB (${ - rss.totalGrowthPercent - }%)` - ); - } - - recommendations.forEach((rec, i) => { - this.log( - `${i + 1}. [${rec.type.toUpperCase()}] ${rec.message}`, - rec.type === "error" ? "error" : "info" - ); - this.log(` Suggestion: ${rec.suggestion}`, "info"); - }); - } - - async generateReport() { - const reportData = { - ...this.results, - endTime: new Date().toISOString(), - configuration: { - serverPort: this.options.serverPort, - testRequests: this.options.testRequests, - warmupRequests: this.options.warmupRequests, - apiEndpoints: this.options.apiEndpoints, - memoryLeakThreshold: this.options.memoryLeakThreshold, - heapGrowthThreshold: this.options.heapGrowthThreshold, - adminToken: this.options.adminToken ? "[REDACTED]" : null, - runDirectory: this.options.runDir, - }, - }; - - try { - fs.writeFileSync(this.options.outputFile, JSON.stringify(reportData, null, 2)); - this.log(`Report saved to: ${this.options.outputFile}`); - } catch (error) { - this.log(`Failed to save report: ${error.message}`, "error"); - } - - return reportData; - } - - async delay(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); - } - - async cleanup() { - if (this.serverProcess && !this.serverProcess.killed) { - this.log("Stopping server..."); - this.serverProcess.kill("SIGTERM"); - - // Wait a bit for graceful shutdown - await this.delay(2000); - - if (!this.serverProcess.killed) { - this.serverProcess.kill("SIGKILL"); - } - } - } - - async run() { - try { - this.log("=== Memory Leak Detection Started ==="); - this.log(`Run directory: ${this.options.runDir}`); - - await this.ensureSnapshotDir(); - await this.startServer(); - - // Wait for server to fully initialize - await this.delay(3000); - - // Phase 1: Warmup requests + first snapshot - this.log("\n=== Phase 1: Warmup ==="); - await this.performRequestPhase("warmup", this.options.warmupRequests); - - // Force GC and wait before snapshot - if (global.gc) { - global.gc(); - await this.delay(1000); - } - await this.takeHeapSnapshot("after-warmup"); - - // Phase 2: First load test + second snapshot - this.log("\n=== Phase 2: Load Test 1 ==="); - await this.performRequestPhase("load-test-1", this.options.testRequests); - - // Force GC and wait before snapshot - if (global.gc) { - global.gc(); - await this.delay(1000); - } - await this.takeHeapSnapshot("after-load-test-1"); - - // Phase 3: Second load test + third snapshot - this.log("\n=== Phase 3: Load Test 2 ==="); - await this.performRequestPhase("load-test-2", this.options.testRequests); - - // Force GC and wait before final snapshot - if (global.gc) { - global.gc(); - await this.delay(2000); - } - await this.takeHeapSnapshot("after-load-test-2"); - - // Analyze results - this.analyzeMemoryUsage(); - - // Generate report - await this.generateReport(); - - this.log("\n=== Memory Leak Detection Completed ==="); - this.log(`šŸ“ All results saved to: ${this.options.runDir}`); - this.log(`šŸ“Š Report: ${this.options.outputFile}`); - this.log(`šŸ“ˆ Snapshots: ${this.options.heapSnapshotPath}`); - - if ( - this.results.snapshots.length === 3 && - this.results.snapshots[0].type === "v8-heapsnapshot" - ) { - this.log(`\nšŸ” Next steps:`); - this.log(`1. Analyze snapshots in Chrome DevTools Memory tab`); - this.log(`2. Compare snapshot 1 vs 3 to identify memory leaks`); - this.log(`3. Look for growing Sentry-related objects`); - this.log(`4. Run with SENTRY_DSN enabled to compare results`); - } - } catch (error) { - this.log(`Detection failed: ${error.message}`, "error"); - throw error; - } finally { - await this.cleanup(); - } - } -} - -// CLI interface -function parseArgs() { - const args = process.argv.slice(2); - const options = {}; - - for (let i = 0; i < args.length; i += 2) { - const arg = args[i]; - const value = args[i + 1]; - - switch (arg) { - case "--port": - options.serverPort = parseInt(value); - break; - case "--requests": - options.testRequests = parseInt(value); - break; - case "--delay": - options.requestDelay = parseInt(value); - break; - case "--endpoints": - options.apiEndpoints = value.split(","); - break; - case "--threshold": - options.memoryLeakThreshold = parseFloat(value); - break; - case "--output": - options.outputFile = value; - break; - case "--token": - options.adminToken = value; - break; - case "--api-key": - options.apiKey = value; - break; - case "--sentry-dsn": - options.sentryDsn = value; - break; - case "--label": - options.label = value; - break; - case "--verbose": - options.verbose = true; - i--; // No value for this flag - break; - case "--help": - console.log(` -Memory Leak Detector Usage: - -node memory-leak-detector.js [options] - -Options: - --port Server port (default: 3030) - --requests Number of test requests (default: 100) - --delay Delay between requests (default: 50) - --endpoints Comma-separated API endpoints to test - --label Label for the run - --threshold Memory leak threshold in MB (default: 50) - --token Admin Bearer token for V8 heap snapshots - --api-key API key for API requests - --sentry-dsn Sentry DSN for memory leak detection - --output Output report file (default: memory-leak-report.json) - --verbose Enable verbose logging - --help Show this help - -Examples: - node memory-leak-detector.js --verbose --requests 200 - node memory-leak-detector.js --token "your-admin-token" --api-key "your-api-key" --verbose - node memory-leak-detector.js --endpoints "/api/v1/whoami,/api/v1/projects" --threshold 25 - `); - process.exit(0); - } - } - - return options; -} - -// Main execution -if (require.main === module) { - const options = parseArgs(); - const detector = new MemoryLeakDetector(options); - - detector - .run() - .then(() => { - console.log("\nāœ… Memory leak detection completed successfully!"); - console.log(`šŸ“ Results directory: ${detector.options.runDir}`); - console.log(`šŸ“Š Report: ${detector.options.outputFile}`); - console.log(`šŸ“ˆ Snapshots: ${detector.options.heapSnapshotPath}`); - process.exit(0); - }) - .catch((error) => { - console.error("\nāŒ Memory leak detection failed:", error.message); - process.exit(1); - }); -} - -module.exports = MemoryLeakDetector; diff --git a/apps/webapp/test/billingServiceAuth.test.ts b/apps/webapp/test/billingServiceAuth.test.ts new file mode 100644 index 00000000000..45909728124 --- /dev/null +++ b/apps/webapp/test/billingServiceAuth.test.ts @@ -0,0 +1,108 @@ +import { beforeEach, describe, expect, vi } from "vitest"; + +const SECRET = "billing-service-secret-0123456789abcdef"; + +const db = vi.hoisted(() => ({ client: null as any })); +const env = vi.hoisted(() => ({ + SESSION_SECRET: "test-session-secret-for-billing-service-auth", + ENCRYPTION_KEY: "0123456789abcdef0123456789abcdef", + BILLING_SERVICE_SECRET: undefined as string | undefined, +})); + +vi.mock("~/db.server", () => ({ + get prisma() { + return db.client; + }, + get $replica() { + return db.client; + }, +})); +vi.mock("~/env.server", () => ({ env })); +vi.mock("~/services/logger.server", () => ({ + logger: { debug: vi.fn(), error: vi.fn(), warn: vi.fn(), info: vi.fn() }, +})); +vi.mock("~/services/rbac.server", () => ({ + rbac: { isUsingPlugin: async () => false, setTokenRole: async () => ({ ok: true }) }, +})); + +import { postgresTest } from "@internal/testcontainers"; +import { requireBillingServiceOrAdminApiRequest } from "~/services/billingServiceAuth.server"; +import { + createPersonalAccessToken, + requireAdminApiRequest, +} from "~/services/personalAccessToken.server"; + +vi.setConfig({ testTimeout: 60_000 }); + +let userCounter = 0; + +async function patFor(prisma: any, admin: boolean) { + userCounter += 1; + const user = await prisma.user.create({ + data: { + email: `billing-service-auth-${userCounter}-${Date.now()}@example.com`, + authenticationMethod: "MAGIC_LINK", + admin, + }, + }); + const pat = await createPersonalAccessToken({ name: "test", userId: user.id }); + return (pat as { token: string }).token; +} + +function bearer(token: string) { + return new Request("http://localhost/admin/api/v1/orgs/org_1/runs/enable", { + method: "POST", + headers: { Authorization: `Bearer ${token}` }, + }); +} + +async function statusOf(guard: Promise) { + try { + await guard; + return 200; + } catch (error) { + if (error instanceof Response) return error.status; + throw error; + } +} + +describe("billing service admin API auth", () => { + beforeEach(() => { + env.BILLING_SERVICE_SECRET = SECRET; + }); + + /** Catches: billing's calls failing once the secret is configured. */ + postgresTest("billing routes accept the billing service secret", async ({ prisma }) => { + db.client = prisma; + expect(await statusOf(requireBillingServiceOrAdminApiRequest(bearer(SECRET)))).toBe(200); + }); + + /** Catches: admins losing access to the billing routes, or the fallback accepting any user's PAT. */ + postgresTest("billing routes still take admin PATs and refuse others", async ({ prisma }) => { + db.client = prisma; + const adminPat = await patFor(prisma, true); + const userPat = await patFor(prisma, false); + + expect(await statusOf(requireBillingServiceOrAdminApiRequest(bearer(adminPat)))).toBe(200); + expect(await statusOf(requireBillingServiceOrAdminApiRequest(bearer(userPat)))).toBe(403); + }); + + /** Catches: a near-match of the secret being accepted. */ + postgresTest("billing routes refuse a wrong secret", async ({ prisma }) => { + db.client = prisma; + expect(await statusOf(requireBillingServiceOrAdminApiRequest(bearer(`${SECRET}x`)))).toBe(401); + }); + + /** Catches: installs without billing accepting a bearer value because no secret is configured. */ + postgresTest("the secret does nothing when it isn't configured", async ({ prisma }) => { + db.client = prisma; + env.BILLING_SERVICE_SECRET = undefined; + expect(await statusOf(requireBillingServiceOrAdminApiRequest(bearer(SECRET)))).toBe(401); + }); + + /** Catches: the billing secret working as an admin credential on the other admin routes. */ + postgresTest("other admin routes refuse the secret", async ({ prisma }) => { + db.client = prisma; + expect(await statusOf(requireAdminApiRequest(bearer(SECRET)))).toBe(401); + }); +}); diff --git a/apps/webapp/test/externalOidcFeatureFlags.test.ts b/apps/webapp/test/externalOidcFeatureFlags.test.ts new file mode 100644 index 00000000000..40ed33b9230 --- /dev/null +++ b/apps/webapp/test/externalOidcFeatureFlags.test.ts @@ -0,0 +1,28 @@ +import { expect, it } from "vitest"; +import { resolveExternalOidcFlags } from "~/services/oidc/externalOidcFeatureFlags"; + +// Catches: an invalid sibling flag discarding an explicit OIDC authorization override. +it.each([ + { global: true, override: false }, + { global: false, override: true }, + { global: undefined, override: true }, +])( + "preserves org OIDC $override over global $global despite an invalid sibling", + ({ global, override }) => { + expect( + resolveExternalOidcFlags( + { externalOidc: global }, + { externalOidc: override, realtimeBackend: "not-a-backend" } + ) + ).toEqual({ externalOidc: override }); + } +); + +it.each([ + { global: true, expected: true }, + { global: undefined, expected: false }, +])("falls back to global $global for an invalid org OIDC value", ({ global, expected }) => { + expect( + resolveExternalOidcFlags({ externalOidc: global }, { externalOidc: "not-a-boolean" }) + ).toEqual({ externalOidc: expected }); +}); diff --git a/apps/webapp/test/globalFlagWriteRouting.test.ts b/apps/webapp/test/globalFlagWriteRouting.test.ts index 3193919b5ca..f7aa44a936b 100644 --- a/apps/webapp/test/globalFlagWriteRouting.test.ts +++ b/apps/webapp/test/globalFlagWriteRouting.test.ts @@ -96,6 +96,11 @@ describe("lockedFlagsInPayload — what the global page refuses", () => { ]); }); + // Catches: rejecting the external OIDC global toggle as an org-only flag. + it("allows external OIDC to be configured globally", () => { + expect(lockedFlagsInPayload([FEATURE_FLAG.externalOidc], true)).toEqual([]); + }); + it("allows the shard list, because that is the page's ramp lever", () => { expect(lockedFlagsInPayload([FEATURE_FLAG.runOpsMintShardSet], true)).toEqual([]); }); diff --git a/docs/apikeys.mdx b/docs/apikeys.mdx index 72adc4e1324..7e44d2851a1 100644 --- a/docs/apikeys.mdx +++ b/docs/apikeys.mdx @@ -12,7 +12,7 @@ description: "Authenticate backend requests with environment-specific API keys." To access AWS or another external service from a deployed task without stored cloud keys, use - [OpenID Connect (OIDC)](/oidc). Trigger.dev API keys cannot mint OIDC tokens. + [workload identity](/oidc). Trigger.dev API keys cannot mint OIDC tokens. ## Create an API key diff --git a/docs/docs.json b/docs/docs.json index ca629c4ded6..454ecc49018 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -165,6 +165,7 @@ "group": "Webhooks", "pages": [ "webhooks/overview", + "webhooks/public-beta", "webhooks/sources", "webhooks/connect", "webhooks/deliveries", @@ -804,6 +805,11 @@ ] }, "redirects": [ + { + "source": "/webhooks/private-beta", + "destination": "/webhooks/public-beta", + "permanent": true + }, { "source": "/queue-concurrency", "destination": "/concurrency", diff --git a/docs/guides/frameworks/bun.mdx b/docs/guides/frameworks/bun.mdx index c704899a663..b7a50d58767 100644 --- a/docs/guides/frameworks/bun.mdx +++ b/docs/guides/frameworks/bun.mdx @@ -9,11 +9,6 @@ import Prerequisites from "/snippets/framework-prerequisites.mdx"; import CliRunTestStep from "/snippets/step-run-test.mdx"; import CliViewRunStep from "/snippets/step-view-run.mdx"; - - The trigger.dev CLI does not yet support Bun. So you will need to run the CLI using Node.js. Bun - will still be used to execute your tasks, even in the `dev` environment. - - **`runtime: "bun"` is deprecated.** It deploys on Bun 1.3.3 with a Node.js 20 binary alongside it (`bun-legacy` is the same runtime). Deploys that use it will be rejected from **January 5, 2027**. diff --git a/docs/oidc.mdx b/docs/oidc.mdx index bb5922e61d6..ed93fb2a973 100644 --- a/docs/oidc.mdx +++ b/docs/oidc.mdx @@ -1,14 +1,14 @@ --- -title: "OpenID Connect (OIDC)" -description: "Access AWS and other services from deployed tasks using short-lived credentials instead of stored cloud keys." +title: "Workload identity" +description: "Access AWS from your tasks without storing long-lived credentials." --- - OIDC is in private beta on Trigger.dev Cloud. [Contact us](https://trigger.dev/contact) to request - access. + Workload identity is in private beta on Trigger.dev Cloud. [Contact us](https://trigger.dev/contact) + to request access. -OIDC lets a deployed task on Trigger.dev Cloud prove its identity to an external service without storing a long-lived cloud key. Trigger.dev issues a short-lived identity token. The service validates that token and grants only the permissions you configured for that identity. +Workload identity lets a deployed task on Trigger.dev Cloud access AWS and other services without storing long-lived credentials. It uses OIDC-based workload identity federation: Trigger.dev issues a short-lived OpenID Connect (OIDC) token, and the service validates it before granting the permissions configured for that identity. ## Prerequisites @@ -21,7 +21,7 @@ OIDC lets a deployed task on Trigger.dev Cloud prove its identity to an external Open your project in the dashboard, select the Production, Staging, or Preview environment that - needs AWS access, then open [**OIDC**](https://cloud.trigger.dev/_/oidc) in the environment menu. Configure each environment separately. + needs AWS access, then open [**Workload identity**](https://cloud.trigger.dev/_/oidc) in the environment menu. Configure each environment separately. In the AWS IAM console, open **Identity providers → Add provider → OpenID Connect**. Paste the @@ -137,9 +137,9 @@ The provider requires `roleArn` and `region`, and accepts optional `roleSessionN ## Terraform setup Use these resources instead of creating the provider and role through the console. Copy the exact -issuer URL and subject from the environment's OIDC page into the input variables. This example uses -exact subject matching; attach a least-privilege permissions policy to the role separately, as in -the setup steps above. +issuer URL and subject from the environment's Workload identity page into the input variables. This +example uses exact subject matching; attach a least-privilege permissions policy to the role separately, +as in the setup steps above. ```hcl terraform { diff --git a/docs/webhooks/private-beta.mdx b/docs/webhooks/private-beta.mdx deleted file mode 100644 index ea13c9788b9..00000000000 --- a/docs/webhooks/private-beta.mdx +++ /dev/null @@ -1,157 +0,0 @@ ---- -title: "Webhooks private beta" -description: "How to install the webhooks beta packages, keep them up to date, and send feedback." -sidebarTitle: "Private beta" -hidden: true ---- - -Thanks for trying webhooks. Your organization has access, so endpoints you declare sync when you deploy or run `trigger dev`, and they show up on each environment's **Webhooks** page. - -## Install the beta packages - -Webhooks ship as prerelease packages on the `webhooks` npm tag. Install the SDK and the CLI from that tag: - - - -```bash pnpm -pnpm add @trigger.dev/sdk@webhooks -pnpm add -D trigger.dev@webhooks -``` - -```bash npm -npm i @trigger.dev/sdk@webhooks -npm i -D trigger.dev@webhooks -``` - - - -Add the connector packages if you're building Slack or Linear agents: - - - -```bash pnpm -pnpm add @trigger.dev/slack@webhooks @trigger.dev/linear@webhooks -``` - -```bash npm -npm i @trigger.dev/slack@webhooks @trigger.dev/linear@webhooks -``` - - - -Use the same tag everywhere, including `@trigger.dev/build` if your project has it. The CLI checks that its version matches the SDK, so run the beta CLI too: `pnpm dlx trigger.dev@webhooks dev` and `pnpm dlx trigger.dev@webhooks deploy` (or `npx trigger.dev@webhooks ...`). - -The webhook tools in the Trigger.dev MCP server come with the beta CLI as well. Install the MCP server pinned to the tag: - -```bash -pnpm dlx trigger.dev@webhooks install-mcp --tag webhooks -``` - -## If your package manager blocks new versions - -Every beta build is new, usually hours or minutes old when you install it. A minimum release age policy skips versions that young, so installing from the `webhooks` tag either fails or installs the wrong version: - -- **pnpm 11** applies `minimumReleaseAge` (one day) by default. It doesn't fail: it quietly installs an older prerelease that has no webhooks, such as `@trigger.dev/sdk@4.5.0-rc.7`. Check `pnpm ls @trigger.dev/sdk` shows a `-webhooks.` version. -- **Yarn** applies `npmMinimalAgeGate` by default. -- **npm** (`min-release-age`) and **Bun** (`install.minimumReleaseAge`) only do this if you've turned it on. - -Exempt the Trigger.dev packages instead of turning the policy off. The CLI package is `trigger.dev`, outside the `@trigger.dev` scope, so it needs its own entry. - - - -```yaml pnpm-workspace.yaml -minimumReleaseAgeExclude: - - "@trigger.dev/*" - - trigger.dev -``` - -```ini .npmrc -min-release-age-exclude[]=@trigger.dev/* -min-release-age-exclude[]=trigger.dev -``` - -```yaml .yarnrc.yml -npmPreapprovedPackages: - - "@trigger.dev/*" - - trigger.dev -``` - -```toml bunfig.toml -[install] -# Bun matches names exactly, so list each package you install, and the ones they depend on -minimumReleaseAgeExcludes = [ - "@trigger.dev/sdk", - "@trigger.dev/core", - "@trigger.dev/build", - "@trigger.dev/schema-to-json", - "@trigger.dev/slack", - "@trigger.dev/linear", - "trigger.dev", -] -``` - - - -pnpm needs 10.17 or later for the `@trigger.dev/*` pattern; on 10.16, list the packages by name as in the Bun example. Run the install again after adding the exemption. - -## Stay up to date - -Beta versions look like `4.8.0-webhooks.20261020092920`, the next release plus a timestamp. Run the install commands again to move to the newest build. To stay on one build, install that exact version instead of the tag. - -When webhooks become generally available they ship in a normal release, and you switch back to `@latest`. You can remove the release age exemption then. - -## Changelog - -### 4.8.0-webhooks.20261020092920 (October 20) - -**Breaking:** `slack()` now requires `startOn`. Without it, every message in every channel the bot was in started a session. `appMentions()` answers when the bot is @mentioned, and needs no bot user id: - -```ts -import { appMentions, slack } from "@trigger.dev/slack"; - -slack({ - id: "slack", - endpoint: slackEvents, - token: process.env.SLACK_BOT_TOKEN!, - startOn: appMentions(), -}); -``` - -Add `app_mention` to the app's bot events and the `app_mentions:read` scope, then reinstall the app. See [Slack agents](/webhooks/slack). - -New: - -- 12 more sources: `webhooks.loops()`, `granola()`, `autumn()`, `metronome()`, `mux()`, `tailscale()`, `langfuse()`, `infisical()`, `planetscale()`, `plain()`, `firecrawl()` and `supabase()`, each with a setup prompt. Supabase database webhooks aren't signed, so `supabase()` checks a bearer token you add as a header. -- Retries from PagerDuty, Jira, Zendesk, Attio, Shopify, Notion, Vercel, Typeform, Intercom, WorkOS and Firecrawl are deduplicated on the provider's own event or delivery id, so a retried event runs once. See [Deliveries](/webhooks/deliveries). -- Filters accept `== null` and `!= null` to check whether a field is present, and `event any ( ... )` matches a payload that is an array of events (Tailscale, SendGrid, HubSpot, Auth0) when any of them matches. -- `webhook()` takes `task()` options such as `queue`, `machine` and `retry`. -- A failed turn in Slack, Linear or a custom channel posts "Something went wrong answering this. Try again." instead of the raw error. `errorMessage` changes it, as a fixed line or a function of the error, run id and event. See [When a turn fails](/webhooks/channels#when-a-turn-fails). -- Slack: tool calls show as one plan in the reply, each task titled by the tool's `title` (`progress: "tasks"` gives one card per call). `ownWorkspaceOnly: true` ignores Slack Connect guests, and `SlackRunSurface` types `chat.surface()`. -- Dashboard: filter an endpoint's deliveries by status, live or test mode, and delivery id (ours or the provider's). - -Fixed: - -- Plain and Metronome deliveries verify, with each provider's replay window. - -This build also includes what's on `main` since the last one, such as span redaction with `telemetry.spanProcessors`, managing metric dashboards from the API, and `trigger tasks trigger`. - -### 4.8.0-webhooks.20261015065339 (October 15) - -The first beta build. - -## Get started - -- [Overview](/webhooks/overview): endpoints, sources, and what can subscribe. -- [Connect a provider](/webhooks/connect): register the URL and signing secret, then check that events arrive. -- [Slack agents](/webhooks/slack) and [Linear agents](/webhooks/linear): turn a `chat.agent` into an agent that lives in Slack or Linear. -- [Deliveries and endpoints](/webhooks/deliveries): inspect payloads, replay deliveries, and see the runs each one started. - -## What to expect - -- APIs may change before general availability. -- Access is per organization. A project in an organization without access deploys normally, but its endpoints don't sync. -- Webhooks are open source, so they work self-hosted too. - -## Feedback - -Send bugs, questions and ideas to [eric@trigger.dev](mailto:eric@trigger.dev). Delivery ids (`whd_...`) and run ids help us find what happened. diff --git a/docs/webhooks/public-beta.mdx b/docs/webhooks/public-beta.mdx new file mode 100644 index 00000000000..d8074998df1 --- /dev/null +++ b/docs/webhooks/public-beta.mdx @@ -0,0 +1,70 @@ +--- +title: "Webhooks public beta" +description: "Webhooks are available to every organization in the normal Trigger.dev release. Install the packages, see what may change before general availability, and send feedback." +sidebarTitle: "Public beta" +--- + +**Webhooks are in public beta: every organization can use them, and they ship in the normal Trigger.dev release.** Endpoints you declare sync when you deploy or run `trigger dev`, and they show up on each environment's **Webhooks** page. + +## What public beta means + +- The API may still change before general availability. We announce changes in the [changelog](https://trigger.dev/changelog). +- Webhooks are open source, so they work self-hosted too. See [Self-hosting webhooks](/self-hosting/env/webapp#webhooks) to turn them on. + +## Install the packages + +Webhooks are in `@trigger.dev/sdk` 4.8.1 and later. Install the SDK and the CLI from `@latest`: + + + +```bash npm +npm i @trigger.dev/sdk@latest +npm i -D trigger.dev@latest +``` + +```bash pnpm +pnpm add @trigger.dev/sdk@latest +pnpm add -D trigger.dev@latest +``` + +```bash bun +bun add @trigger.dev/sdk@latest +bun add -D trigger.dev@latest +``` + + + +Add the connector packages if you're building Slack or Linear agents: + + + +```bash npm +npm i @trigger.dev/slack@latest @trigger.dev/linear@latest +``` + +```bash pnpm +pnpm add @trigger.dev/slack@latest @trigger.dev/linear@latest +``` + +```bash bun +bun add @trigger.dev/slack@latest @trigger.dev/linear@latest +``` + + + +The webhook tools in the [MCP server](/mcp-introduction) come with the CLI, so a current `trigger.dev` install has them. + +## Moving from a private beta build + +If you installed from the `webhooks` npm tag, switch every Trigger.dev package to `@latest`, including `@trigger.dev/build` if your project has it. Versions like `4.8.0-webhooks.20261020092920` are older than 4.8.1. You can also remove any minimum release age exemption you added for the beta builds, and reinstall the MCP server without `--tag webhooks`. + +## Get started + +- [Overview](/webhooks/overview): endpoints, sources, and what can subscribe. +- [Connect a provider](/webhooks/connect): register the URL and signing secret, then check that events arrive. +- [Slack agents](/webhooks/slack) and [Linear agents](/webhooks/linear): turn a `chat.agent` into an agent that lives in Slack or Linear. +- [Deliveries and endpoints](/webhooks/deliveries): inspect payloads, replay deliveries, and see the runs each one started. + +## Feedback + +Send bugs, questions and ideas to [eric@trigger.dev](mailto:eric@trigger.dev). Delivery ids (`whd_...`) and run ids help us find what happened. diff --git a/internal-packages/observability-map/src/adapters/remix.test.ts b/internal-packages/observability-map/src/adapters/remix.test.ts index be45ce6c62c..8d5222432d3 100644 --- a/internal-packages/observability-map/src/adapters/remix.test.ts +++ b/internal-packages/observability-map/src/adapters/remix.test.ts @@ -4,7 +4,7 @@ describe("familyOf", () => { it("classifies each family from the flat-route filename", () => { expect(familyOf("api.v1.runs.$runId.ts")).toBe("api.v1"); expect(familyOf("api.something.ts")).toBe("api.other"); - expect(familyOf("admin.api.v1.gc.ts")).toBe("admin"); + expect(familyOf("admin.api.v1.feature-flags.ts")).toBe("admin"); expect(familyOf("resources.queues.ts")).toBe("resources"); expect(familyOf("_app.orgs.$slug.ts")).toBe("dashboard"); expect(familyOf("otel.v1.logs.ts")).toBe("ingest"); diff --git a/internal-packages/run-engine/src/engine/errors.ts b/internal-packages/run-engine/src/engine/errors.ts index 05f9e0d0eb0..30f53ba2eba 100644 --- a/internal-packages/run-engine/src/engine/errors.ts +++ b/internal-packages/run-engine/src/engine/errors.ts @@ -188,6 +188,61 @@ export class StoreWaitpointRegistrationUnsupported extends Error { } } +/** + * A store-format blocker reached the block seam without a caller-owned block-operation identity. The + * store's compare-and-set needs a stable, caller-supplied identity that survives retries and is captured + * with the logical operation — NOT re-derived from whatever snapshot is latest when the operation runs — + * so a delayed operation from an older cycle carries its ORIGINAL identity and is refused stale after a + * newer cycle is installed. Without it, a delayed replay would silently adopt the current cycle's identity + * and be accepted. Legacy (Postgres) blockers need no identity and never reach this. In the dark default + * (no store minting) this is unreachable. + */ +export class StoreWaitpointBlockOperationIdentityRequired extends Error { + readonly runId: string; + readonly waitpointIds: string[]; + constructor(params: { runId: string; waitpointIds: string[] }) { + super( + `Cannot register store-format waitpoint(s) [${params.waitpointIds.join( + ", " + )}] against run ${params.runId}: a store block operation requires a caller-owned block identity ` + + `(blockOperation.id) that survives retries; re-deriving it from the latest snapshot cannot fence a ` + + `delayed operation from an older cycle` + ); + this.name = "StoreWaitpointBlockOperationIdentityRequired"; + this.runId = params.runId; + this.waitpointIds = params.waitpointIds; + } +} + +/** + * Reject an operation from an earlier run cycle before it changes MemoryDB. + * Check the snapshot anchor even if cleanup has already removed the old block. + * Legacy waits do not use this check. + */ +export class StoreWaitpointBlockCycleAdvanced extends Error { + readonly runId: string; + readonly waitpointIds: string[]; + readonly expectedCycleAnchor: string; + readonly currentCycleAnchor: string; + constructor(params: { + runId: string; + waitpointIds: string[]; + expectedCycleAnchor: string; + currentCycleAnchor: string; + }) { + super( + `Store block operation for run ${params.runId} carries cycle anchor ${params.expectedCycleAnchor}, ` + + `but the run's active cycle has advanced to ${params.currentCycleAnchor}; refusing to re-block a ` + + `run whose attempt is over` + ); + this.name = "StoreWaitpointBlockCycleAdvanced"; + this.runId = params.runId; + this.waitpointIds = params.waitpointIds; + this.expectedCycleAnchor = params.expectedCycleAnchor; + this.currentCycleAnchor = params.currentCycleAnchor; + } +} + /** * A run whose organization could hold MemoryDB block edges was asked to clear its block state inside a * rollback-capable Postgres lifecycle transaction (`tx`). Clearing MemoryDB cannot join that @@ -210,6 +265,30 @@ export class TransactionalStoreClearUnsupported extends Error { } } +/** + * A mixed-residency absorption accepted the MemoryDB half but its legacy half failed AND the immediate + * compensating rollback of the store contribution also failed. Carries the exact fenced targets so the + * caller can arm a durable retry rather than report the rejected operation as clean while leaving store + * residue. `compensationCause` is the failed cleanup error. + */ +export class StoreAbsorptionCompensationFailed extends Error { + readonly runId: string; + readonly blockId: string; + readonly absorptionId: string; + readonly compensationCause: unknown; + constructor(args: { runId: string; blockId: string; absorptionId: string; cause: unknown }) { + super( + `Mixed absorption for run ${args.runId} block ${args.blockId} accepted MemoryDB but failed legacy, ` + + `and compensating the store contribution failed.` + ); + this.name = "StoreAbsorptionCompensationFailed"; + this.runId = args.runId; + this.blockId = args.blockId; + this.absorptionId = args.absorptionId; + this.compensationCause = args.cause; + } +} + /** * A store-format waitpoint id reached the router but no store coordinator is wired. A recognised * store id must never fall back to Postgres, so this fails loud rather than mis-routing. In the dark diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index e07d7da8bcb..8b51b8e58d2 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -251,6 +251,7 @@ export class RunEngine { processMarkedSchedule: options.queue?.concurrencySweeper?.processMarkedSchedule, scanJitterInMs: options.queue?.concurrencySweeper?.scanJitterInMs, processMarkedJitterInMs: options.queue?.concurrencySweeper?.processMarkedJitterInMs, + scanCount: options.queue?.concurrencySweeper?.scanCount, callback: this.#concurrencySweeperCallback.bind(this), }, shardCount: options.queue?.shardCount, @@ -292,6 +293,7 @@ export class RunEngine { finishWaitpoint: async ({ payload }) => { await this.waitpointSystem.completeWaitpoint({ id: payload.waitpointId, + organizationId: payload.organizationId, output: payload.error ? { value: payload.error, @@ -350,8 +352,20 @@ export class RunEngine { await this.waitpointSystem.ensureWaitpointCompleted({ waitpointId: payload.waitpointId, output: payload.output, + organizationId: payload.organizationId, }); }, + ensureAbsorptionCompensated: async ({ payload }) => { + await this.waitpointSystem.ensureAbsorptionCompensated({ + runId: payload.runId, + blockId: payload.blockId, + absorptionId: payload.absorptionId, + organizationId: payload.organizationId, + }); + }, + recoverWaitpointBlock: async ({ payload }) => { + await this.waitpointSystem.recoverWaitpointBlock(payload); + }, enqueueDelayedRun: async ({ payload }) => { await this.delayedRunSystem.enqueueDelayedRun({ runId: payload.runId }); }, @@ -436,6 +450,8 @@ export class RunEngine { this.checkpointSystem = new CheckpointSystem({ resources, + waitpointStoreEnabled: (organizationId) => + this.waitpointSystem.isStoreRoutingEnabled(organizationId), executionSnapshotSystem: this.executionSnapshotSystem, enqueueSystem: this.enqueueSystem, }); @@ -473,6 +489,7 @@ export class RunEngine { executionSnapshotSystem: this.executionSnapshotSystem, enqueueSystem: this.enqueueSystem, completionGuardDelayMs: options.completionGuardDelayMs, + waitpointStore: options.waitpointStore, }); this.ttlSystem = new TtlSystem({ @@ -2067,6 +2084,7 @@ export class RunEngine { workerId, runnerId, expectedEnvironmentId, + blockOperation, tx, }: { runId: string; @@ -2083,6 +2101,14 @@ export class RunEngine { // id and yields nothing (404) for a foreign run, before any edge is registered. Omitted by trusted // internal callers. expectedEnvironmentId?: string; + // Use the block ID and cycle anchor captured by the caller. + // Required for MemoryDB waits; ignored for Postgres waits. + blockOperation?: { + id: string; + absorptionId: string; + cycleAnchor: string; + expectedPreviousBlockId?: string; + }; tx?: PrismaClientOrTransaction; }): Promise { return this.waitpointSystem.blockRunWithWaitpoint({ @@ -2096,6 +2122,7 @@ export class RunEngine { workerId, runnerId, expectedEnvironmentId, + blockOperation, tx, }); } diff --git a/internal-packages/run-engine/src/engine/systems/checkpointSystem.ts b/internal-packages/run-engine/src/engine/systems/checkpointSystem.ts index 6a8d328bf44..6120900b486 100644 --- a/internal-packages/run-engine/src/engine/systems/checkpointSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/checkpointSystem.ts @@ -16,6 +16,7 @@ import type { SystemResources } from "./systems.js"; import { snapshotRouteFromSnapshot } from "@internal/run-store"; import { ServiceValidationError } from "../errors.js"; import type { EnqueueSystem } from "./enqueueSystem.js"; +import { cycleAnchorSnapshotId } from "./waitpointSystem.js"; /** The description of the EXECUTING snapshot `continueRunExecution` writes. */ export const CONTINUED_AFTER_SUSPENDED_DESCRIPTION = "Run was continued after being suspended"; @@ -24,6 +25,7 @@ export type CheckpointSystemOptions = { resources: SystemResources; executionSnapshotSystem: ExecutionSnapshotSystem; enqueueSystem: EnqueueSystem; + waitpointStoreEnabled?: (organizationId: string) => boolean | Promise; }; export class CheckpointSystem { @@ -240,13 +242,28 @@ export class CheckpointSystem { checkpoint: taskRunCheckpoint, } satisfies CreateCheckpointResult; } else { + // An unwired capability check returns false synchronously. + const storeEnabled = + (snapshot.executionStatus === "EXECUTING_WITH_WAITPOINTS" || + snapshot.executionStatus === "SUSPENDED") && + this.options.waitpointStoreEnabled?.(env.organizationId); //create a new execution snapshot, with the checkpoint const newSnapshot = await this.executionSnapshotSystem.createExecutionSnapshot(prisma, { run, snapshot: { executionStatus: "SUSPENDED", description: "Run was suspended after creating a checkpoint.", - metadata: snapshot.metadata, + metadata: + storeEnabled && (await storeEnabled) === true + ? { + ...(snapshot.metadata && + typeof snapshot.metadata === "object" && + !Array.isArray(snapshot.metadata) + ? snapshot.metadata + : {}), + waitpointCycleAnchor: cycleAnchorSnapshotId(snapshot), + } + : snapshot.metadata, }, previousSnapshotId: snapshot.id, previousSnapshot: snapshot, diff --git a/internal-packages/run-engine/src/engine/systems/waitpointResultRunStore.ts b/internal-packages/run-engine/src/engine/systems/waitpointResultRunStore.ts index be8fa3bd944..65b4cd08e3a 100644 --- a/internal-packages/run-engine/src/engine/systems/waitpointResultRunStore.ts +++ b/internal-packages/run-engine/src/engine/systems/waitpointResultRunStore.ts @@ -29,7 +29,7 @@ export function createWaitpointResultRunStore( requestedIds: ids, legacyReader: async () => [], storeReader: async (requested) => { - const entries = await options.store.readCompletedRecords(requested); + const entries = await options.store.readCompletedRecords(requested, context.runId); for (const [id, entry] of entries) { if (entry.record.environmentId !== context.environmentId) { throw new UnresolvableCompletedWaitpointRecordsError( diff --git a/internal-packages/run-engine/src/engine/systems/waitpointSystem.test.ts b/internal-packages/run-engine/src/engine/systems/waitpointSystem.test.ts index 7c2d7ca1903..907ca6a2c48 100644 --- a/internal-packages/run-engine/src/engine/systems/waitpointSystem.test.ts +++ b/internal-packages/run-engine/src/engine/systems/waitpointSystem.test.ts @@ -331,6 +331,8 @@ describe("WaitpointSystem create/block write routing", () => { store: wired ? destination : undefined, isStoreEnabled: isEnabled, mintPolicy: { resolveMintMode: () => "redis", isCapabilityReady: () => true }, + withLegacyRegistrationLock: (runId, operation) => + resources.runLock.lock("routing-test", [runId], operation), }); const system = new WaitpointSystem({ ...engine.waitpointSystem["options"], @@ -342,8 +344,9 @@ describe("WaitpointSystem create/block write routing", () => { } : undefined, }); - const expectOneFlagCheck = (before: number) => { - expect(checkedOrganizations.slice(before)).toEqual(wired ? [organizationId] : []); + const expectFlagChecks = (before: number, enabledCount = 1) => { + const count = !wired ? 0 : mode === "enabled" ? enabledCount : 1; + expect(checkedOrganizations.slice(before)).toEqual(Array(count).fill(organizationId)); }; for (const native of [false, true]) { const id = native ? generateWaitpointId("DATETIME") : WaitpointId.generate().id; @@ -372,23 +375,20 @@ describe("WaitpointSystem create/block write routing", () => { let beforeFlagChecks = checkedOrganizations.length; if (mode === "enabled" && native) { await expect(router.registerBlocks(params)).rejects.toThrow("three-step barrier"); - expectOneFlagCheck(beforeFlagChecks); + expectFlagChecks(beforeFlagChecks); beforeFlagChecks = checkedOrganizations.length; await expect(router.registerBlocksLockless(params)).rejects.toThrow( "three-step barrier" ); - expectOneFlagCheck(beforeFlagChecks); + expectFlagChecks(beforeFlagChecks); expect(legacyStore.calls).toEqual([]); } else { - expect(await router.registerBlocks(params)).toEqual( - mode === "enabled" - ? { pendingCount: 1, storeRoutingEnabled: true } - : { pendingCount: 1 } - ); - expectOneFlagCheck(beforeFlagChecks); + expect(await router.registerBlocks(params)).toEqual({ pendingCount: 1 }); + // The cleanup fence has its own independent gate after registration's routing gate. + expectFlagChecks(beforeFlagChecks, 2); beforeFlagChecks = checkedOrganizations.length; await router.registerBlocksLockless(params); - expectOneFlagCheck(beforeFlagChecks); + expectFlagChecks(beforeFlagChecks, 3); // Direct delegation preserves the original collection, including duplicates. if (mode !== "enabled") expect(legacyStore.blockInputs.at(-1)).toBe(ids); expect(legacyStore.calls).toEqual([ @@ -404,7 +404,8 @@ describe("WaitpointSystem create/block write routing", () => { projectId: env.projectId, organizationId, }); - expectOneFlagCheck(beforeFlagChecks); + // System admission, absorption, and cleanup-fence gates remain separate stages. + expectFlagChecks(beforeFlagChecks, 3); expect(destinationStore.calls).toEqual([]); } // Ensure a fan-out edge even when enabled registration correctly rejected this parent. @@ -432,8 +433,8 @@ describe("WaitpointSystem create/block write routing", () => { if (!nativeDispatch) { // Complete-before-register uses the same gated payload on the immediate unblock path. beforeFlagChecks = checkedOrganizations.length; - // A flag change after the check affects the next operation, not this registration's - // route or derived continuation payload. There is no decision cache across operations. + // A flag change after system admission is rechecked by absorption. The system still + // carries its admitted org onto the continuation; the next operation evaluates afresh. disableAfterNextCheck = mode === "enabled"; await system.blockRunWithWaitpoint({ runId: run.id, @@ -441,7 +442,7 @@ describe("WaitpointSystem create/block write routing", () => { projectId: env.projectId, organizationId, }); - expectOneFlagCheck(beforeFlagChecks); + expectFlagChecks(beforeFlagChecks, 2); const immediate = await resources.worker.getJob(`continueRunIfUnblocked:${run.id}`); expect(immediate?.item).toEqual( mode === "enabled" ? { runId: run.id, organizationId } : { runId: run.id } @@ -450,7 +451,7 @@ describe("WaitpointSystem create/block write routing", () => { if (mode === "enabled") { beforeFlagChecks = checkedOrganizations.length; await system.completeWaitpoint({ id, organizationId }); - expectOneFlagCheck(beforeFlagChecks); + expectFlagChecks(beforeFlagChecks); const disabledJob = await resources.worker.getJob( `continueRunIfUnblocked:${run.id}` ); @@ -463,7 +464,7 @@ describe("WaitpointSystem create/block write routing", () => { beforeFlagChecks = checkedOrganizations.length; const direct = await router.complete({ waitpointId: id, organizationId }); expect(direct.waitpoint.status).toBe("COMPLETED"); - expectOneFlagCheck(beforeFlagChecks); + expectFlagChecks(beforeFlagChecks); const missing = generateWaitpointId("DATETIME"); legacyStore.calls.length = 0; destinationStore.calls.length = 0; @@ -489,7 +490,7 @@ describe("WaitpointSystem create/block write routing", () => { completedAfter: new Date(), mintContext: { organizationId, parentResidency: "redis-primary" }, }); - expectOneFlagCheck(beforeFlagChecks); + expectFlagChecks(beforeFlagChecks); expect(mode === "enabled" ? legacyStore.calls : destinationStore.calls).toEqual([]); expect(mode === "enabled" ? destinationStore.calls : legacyStore.calls).toEqual([ "upsertWaitpoint", diff --git a/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts b/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts index ce636b6173d..bfeda341774 100644 --- a/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { timeoutError } from "@trigger.dev/core/v3"; import type { ShardKey } from "@trigger.dev/core/v3/isomorphic"; import { boundedIn } from "@trigger.dev/database"; @@ -9,21 +10,29 @@ import type { Waitpoint, } from "@trigger.dev/database"; import { assertNever } from "assert-never"; -import { WaitpointCompletionGuardArmedError } from "../errors.js"; +import { + StoreAbsorptionCompensationFailed, + StoreWaitpointBlockCycleAdvanced, + WaitpointCompletionGuardArmedError, +} from "../errors.js"; +import { registrationEvidence } from "../waitpointCoordinator/keys.js"; +import { classifyWaitpointCoordinator } from "../waitpointCoordinator/routing.js"; import { buildCompletedWaitpointRecords } from "./completedWaitpointResolver.js"; import { resolveCompletedWaitpointRecords } from "./completedWaitpointRecordAdapter.js"; import { sendNotificationToWorker } from "../eventBus.js"; import { isFinalRunStatus } from "../statuses.js"; import { LegacyPostgresWaitpointCoordinator } from "../waitpointCoordinator/legacyPostgresCoordinator.js"; +import type { WaitpointStoreCoordinator } from "../waitpointCoordinator/storeCoordinator.js"; import { RoutingWaitpointCoordinator, type RoutingWaitpointCoordinatorMintPolicy, type WaitpointRunKey, } from "../waitpointCoordinator/routingCoordinator.js"; -import type { WaitpointStoreCoordinator } from "../waitpointCoordinator/storeCoordinator.js"; import type { + AbsorbBlockersParams, + AbsorbBlockersResult, + BlockerRegistration, CompleteParams, - RegisterBlocksParams, RunBlockEdge, WaitpointCoordinator, } from "../waitpointCoordinator/types.js"; @@ -33,6 +42,8 @@ import { getLatestExecutionSnapshot, isExpiredSnapshotOfFinishedRun, } from "./executionSnapshotSystem.js"; +import type { workerCatalog } from "../workerCatalog.js"; +import type { z } from "zod"; import type { SystemResources } from "./systems.js"; import { hasStoreWaitpoints, @@ -72,8 +83,46 @@ export type WaitpointSystemOptions = { waitpointStore?: Pick; }; +type RecoverWaitpointBlock = z.infer; + const DEFAULT_COMPLETION_GUARD_DELAY_MS = 30_000; +// Delay before the worker retries failed registration cleanup under the run lock. +const BLOCKER_CLEANUP_OBLIGATION_DELAY_MS = 30_000; + +// Keep the caller's block identity on retries. Do not replace it with the current block ID. +// When resuming, read the installed block ID with currentStoreBlockId. + +// Use this exact description to identify a snapshot blocked by a waitpoint. +const WAITPOINT_BLOCKED_SNAPSHOT_DESCRIPTION = "Run was blocked by a waitpoint."; + +// Return the snapshot ID that identifies the current wait cycle. +// Before blocking, use the current snapshot. While blocked, use its predecessor. +// A checkpoint preserves the cycle anchor in its metadata. +// Callers capture this ID before registration and check it under the run lock. +// The check rejects old operations after resume or retry, even if the old block is gone. +export function cycleAnchorSnapshotId(snapshot: TaskRunExecutionSnapshot): string { + // A checkpoint suspends this same active wait, preserving the anchor captured under the run lock. + if ( + snapshot.executionStatus === "SUSPENDED" && + snapshot.description === "Run was suspended after creating a checkpoint." + ) { + const metadata = snapshot.metadata; + if ( + metadata && + typeof metadata === "object" && + !Array.isArray(metadata) && + typeof metadata.waitpointCycleAnchor === "string" + ) + return metadata.waitpointCycleAnchor; + } + const isWaitpointBlocked = + snapshot.executionStatus === "EXECUTING_WITH_WAITPOINTS" || + (snapshot.executionStatus === "SUSPENDED" && + snapshot.description === WAITPOINT_BLOCKED_SNAPSHOT_DESCRIPTION); + return isWaitpointBlocked ? (snapshot.previousSnapshotId ?? snapshot.id) : snapshot.id; +} + type WaitpointContinuationWaitpoint = Pick; export type WaitpointContinuationResult = @@ -112,9 +161,9 @@ export class WaitpointSystem { prisma: this.$.prisma, logger: this.$.logger, }); - this.legacyCoordinator = legacyCoordinator; // Dark by default: with no store coordinator the router delegates every operation to Postgres and // never consults the mint policy, so production behaves exactly as before. + this.legacyCoordinator = legacyCoordinator; this.coordinator = options.waitpointRouting?.store ? new RoutingWaitpointCoordinator({ legacy: legacyCoordinator, @@ -130,6 +179,17 @@ export class WaitpointSystem { isCapabilityReady: options.waitpointRouting?.mintPolicy?.isCapabilityReady ?? (() => false), }, + markStoreWaitpoints: (runId, organizationId) => + this.$.runLock.lock("markRunWaitpointsOnStore", [runId], async () => { + // Reuse the run lock for registration and association creation. + // Child marking also takes the start-attempt lock, even if the flag changes. + if (!(await this.isWaitpointStoreEnabled(organizationId))) { + throw new Error("Waitpoint store was disabled before marking the run"); + } + await this.$.runStore.markRunWaitpointsOnStore(runId); + }), + withLegacyRegistrationLock: (runId, operation) => + this.$.runLock.lock("registerLegacyWaitpoint", [runId], operation), mightHaveStoreBlockers: options.waitpointRouting?.mightHaveStoreBlockers ?? (async ({ runId }) => { @@ -144,20 +204,85 @@ export class WaitpointSystem { : legacyCoordinator; } - public async clearBlockingWaitpoints({ + public isStoreRoutingEnabled(organizationId?: string): boolean | Promise { + return ( + this.coordinator instanceof RoutingWaitpointCoordinator && + this.coordinator.isStoreRoutingEnabled(organizationId) + ); + } + + public async isWaitpointStoreEnabled( + organizationId?: string, + organizationFlags?: unknown + ): Promise { + return ( + organizationId !== undefined && + (await this.options.waitpointStoreEnabled?.(organizationId, organizationFlags)) === true + ); + } + + /** + * If Postgres registration fails after MemoryDB accepts its part, undo that part. + * If immediate cleanup also fails, queue a cleanup retry before returning the error. + * Scope the retry to this block and operation. + */ + async #absorbBlockersWithCompensationGuard( + params: AbsorbBlockersParams + ): Promise { + try { + return await this.coordinator.absorbBlockers(params); + } catch (error) { + if (error instanceof StoreAbsorptionCompensationFailed) { + // Use one job per operation. Repeated retries reuse the same job. + // Store only IDs in the payload; read the added edges from the saved record. + await this.$.worker.enqueueOnce({ + id: this.#absorptionCompensationObligationId( + error.runId, + error.blockId, + error.absorptionId + ), + job: "ensureAbsorptionCompensated", + payload: { + runId: error.runId, + blockId: error.blockId, + absorptionId: error.absorptionId, + organizationId: params.organizationId, + }, + availableAt: new Date(Date.now() + BLOCKER_CLEANUP_OBLIGATION_DELAY_MS), + }); + } + throw error; + } + } + + #absorptionCompensationObligationId( + runId: string, + blockId: string, + absorptionId: string + ): string { + return `ensureAbsorptionCompensated:${runId}:${blockId}:${absorptionId}`; + } + + /** + * Retry cleanup under the run lock so it cannot race with registration. + * Check the saved operation record inside the lock. After successful registration, + * its proof prevents cleanup from removing accepted edges, even if the job was not acknowledged. + */ + public async ensureAbsorptionCompensated({ runId, + blockId, + absorptionId, organizationId, - tx, }: { runId: string; - // The run's org, so the router can skip MemoryDB for a run whose org could not hold store edges. - // Undefined keeps legacy routing without discovering an organization. + blockId: string; + absorptionId: string; organizationId?: string; - tx?: PrismaClientOrTransaction; - }) { - const deleted = await this.coordinator.clearRunBlockState({ runId, organizationId, tx }); - - return deleted.count; + }): Promise { + if (!(await this.isWaitpointStoreEnabled(organizationId))) return; + await this.$.runLock.lock("ensureAbsorptionCompensated", [runId], async () => { + await this.coordinator.compensateAbsorption({ runId, blockId, absorptionId, organizationId }); + }); } /** Completes a waitpoint and schedules a continue for each run it blocked. Safe under races. */ @@ -176,12 +301,9 @@ export class WaitpointSystem { /** Arms the write-ahead completion guard first. The guard's replay passes false. */ armGuard?: boolean; /** - * The waitpoint's organization, from the completing caller's context. A waitpoint is environment- - * scoped and every blocking run is resolved within its own environment (see the wait routes), so a - * waitpoint cannot block runs across organizations — this one org is carried onto each blocked run's - * `continueRunIfUnblocked` so the delayed continuation routes MemoryDB by the run's org rather than - * an unknown one, without any per-edge lookup. Undefined (a caller without it, or an old guard - * replay) retains the original legacy route, including its error behavior. + * Use the caller's organisation for each resume job. Blocking runs belong to the + * same organisation, so no extra lookup is needed for each edge. + * Without an organisation, use Postgres routing and reject MemoryDB IDs. */ organizationId?: string; }): Promise { @@ -194,7 +316,11 @@ export class WaitpointSystem { // Armed before the first mutation, so a committed completion always has a durable watcher. An // arm failure is not wrapped, so the caller cannot mistake it for one the guard will replay. if (armGuard) { - await this.#scheduleCompletionGuard(id, output); + await this.#scheduleCompletionGuard( + id, + output, + storeRoutingEnabled ? organizationId : undefined + ); } // From here the guard owns eventual completion. A failure is wrapped as @@ -227,9 +353,7 @@ export class WaitpointSystem { // The fixed job id debounces repeat enqueues. id: jobId, job: "continueRunIfUnblocked", - // Carry the completion's org (== every blocked run's org; see the doc on this method) so the - // delayed continuation can route MemoryDB when enabled. Disabled/unwired jobs retain - // the original runId-only payload. + // Pass the organisation to the resume job. Without it, use Postgres routing. payload: storeRoutingEnabled ? { runId: run.taskRunId, organizationId } : { runId: run.taskRunId }, @@ -272,12 +396,16 @@ export class WaitpointSystem { // status-guarded update fixes the winning output, so a replay never overwrites it. async #scheduleCompletionGuard( waitpointId: string, - output?: { value: string; type?: string; isError: boolean } + output?: { value: string; type?: string; isError: boolean }, + organizationId?: string ): Promise { await this.$.worker.enqueueOnce({ id: this.#completionGuardId(waitpointId), job: "ensureWaitpointCompleted", - payload: { waitpointId, output }, + payload: + organizationId !== undefined + ? { waitpointId, output, organizationId } + : { waitpointId, output }, availableAt: new Date(Date.now() + this.completionGuardDelayMs), }); } @@ -294,11 +422,13 @@ export class WaitpointSystem { public async ensureWaitpointCompleted({ waitpointId, output, + organizationId, }: { waitpointId: string; output?: { value: string; type?: string; isError: boolean }; + organizationId?: string; }): Promise { - await this.completeWaitpoint({ id: waitpointId, output, armGuard: false }); + await this.completeWaitpoint({ id: waitpointId, output, organizationId, armGuard: false }); } /** @@ -420,6 +550,8 @@ export class WaitpointSystem { workerId, runnerId, expectedEnvironmentId, + blockOperation, + recoverOnly = false, tx, }: { runId: string; @@ -431,15 +563,26 @@ export class WaitpointSystem { batch?: { id: string; index?: number }; workerId?: string; runnerId?: string; + // Use the caller's block ID and cycle anchor for MemoryDB waits. + // Keep them unchanged within a wait cycle. Check the anchor under the run lock + // before writing to MemoryDB, even if the old block is gone. + // Postgres-only waits ignore these fields. + blockOperation?: { + id: string; + absorptionId: string; + cycleAnchor: string; + expectedPreviousBlockId?: string; + }; // An environment the run MUST belong to. When set, the run's snapshot read is scoped to it, so a // run from another org/project/env yields no snapshot and throws (404) BEFORE any edge is // registered — the cross-org registration guard for the public token-wait route. The run's // snapshots are all in its own environment, so a snapshot found under this id proves membership. expectedEnvironmentId?: string; + /** Recovery can finish an accepted request. It cannot accept a new request. */ + recoverOnly?: boolean; tx?: PrismaClientOrTransaction; }): Promise { const prisma = tx ?? this.$.prisma; - await this.$.raceSimulationSystem.waitForRacepoint({ runId }); let $waitpoints = typeof waitpoints === "string" ? [waitpoints] : waitpoints; @@ -455,22 +598,125 @@ export class WaitpointSystem { organizationId ); - // Two statements, in this order: see the READ COMMITTED note on the method. - const params: RegisterBlocksParams = { - runId, - waitpointIds: $waitpoints, - projectId, - spanIdToComplete, - batchId: batch?.id, - batchIndex: batch?.index, - client: prisma, - }; - if (this.coordinator instanceof RoutingWaitpointCoordinator) { - params.organizationId = organizationId; + const storeRoutingEnabled = + this.coordinator instanceof RoutingWaitpointCoordinator && + (await this.coordinator.isStoreRoutingEnabled(organizationId)); + // Reject operations from an earlier cycle before changing either store. + // The run lock protects this check. Check the anchor even if the old block is gone. + if ( + storeRoutingEnabled && + $waitpoints.some((id) => classifyWaitpointCoordinator(id) === "store") + ) { + const currentAnchor = cycleAnchorSnapshotId(snapshot); + if (blockOperation && blockOperation.cycleAnchor !== currentAnchor) { + throw new StoreWaitpointBlockCycleAdvanced({ + runId, + waitpointIds: $waitpoints.filter((id) => classifyWaitpointCoordinator(id) === "store"), + expectedCycleAnchor: blockOperation.cycleAnchor, + currentCycleAnchor: currentAnchor, + }); + } + + if (blockOperation && batch) { + // Give each batch position its own registration ID, including its waitpoint set. + // Derive it from the original request so caller retries and recovery use the same ID. + // Hash the inputs to limit key length. Keep the block and cycle IDs unchanged. + const registrationId = createHash("sha256") + .update( + JSON.stringify([batch.id, batch.index ?? null, [...new Set($waitpoints)].sort()]) + ) + .digest("hex"); + blockOperation = { + ...blockOperation, + absorptionId: `${blockOperation.id}:batch:${registrationId}`, + }; + } } - const { pendingCount, storeRoutingEnabled } = await this.coordinator.registerBlocks(params); - const isRunBlocked = pendingCount > 0; + if ( + recoverOnly && + (!storeRoutingEnabled || + !blockOperation || + !(await this.coordinator.hasAbsorbedBlockers({ + runId, + blockId: blockOperation.id, + absorptionId: blockOperation.absorptionId, + waitpointIds: $waitpoints, + batchIndex: batch?.index, + organizationId, + }))) + ) { + if (storeRoutingEnabled && blockOperation) { + // A crash may leave source holds before absorption was accepted. + await this.coordinator.compensateAbsorption({ + runId, + blockId: blockOperation.id, + absorptionId: blockOperation.absorptionId, + organizationId, + unacceptedEvidence: registrationEvidence($waitpoints, batch?.index), + }); + } + return snapshot; + } + + if ( + storeRoutingEnabled && + blockOperation && + $waitpoints.some((id) => classifyWaitpointCoordinator(id) === "store") + ) { + await this.$.worker.enqueueOnce({ + id: `recoverWaitpointBlock:${runId}:${blockOperation.absorptionId}`, + job: "recoverWaitpointBlock", + payload: { + runId, + organizationId, + projectId, + waitpoints: $waitpoints, + blockOperation, + timeout, + spanIdToComplete, + batch, + workerId, + runnerId, + expectedEnvironmentId, + }, + availableAt: new Date(Date.now() + (this.options.completionGuardDelayMs ?? 30_000)), + }); + } + + // Phase 1: record the waits and count pending results. + // MemoryDB waits require the caller's block identity. + // Pass the returned registration to phase 3 without changes. + let registration: BlockerRegistration | undefined; + let pendingCount: number; + if (storeRoutingEnabled) { + const absorbed = await this.#absorbBlockersWithCompensationGuard({ + runId, + blockId: blockOperation?.id, + absorptionId: blockOperation?.absorptionId, + expectedPreviousBlockId: blockOperation?.expectedPreviousBlockId, + organizationId, + waitpointIds: $waitpoints, + projectId, + spanIdToComplete, + batchId: batch?.id, + batchIndex: batch?.index, + client: prisma, + parentResidency: snapshotRouteFromSnapshot(snapshot)?.residency, + }); + registration = absorbed.registration; + pendingCount = absorbed.pendingCount; + } else { + ({ pendingCount } = await this.legacyCoordinator.registerBlocks({ + runId, + waitpointIds: $waitpoints, + projectId, + spanIdToComplete, + batchId: batch?.id, + batchIndex: batch?.index, + client: prisma, + })); + } let newStatus: TaskRunExecutionStatus = "SUSPENDED"; if ( @@ -480,6 +726,7 @@ export class WaitpointSystem { newStatus = "EXECUTING_WITH_WAITPOINTS"; } + // Phase 2: save the blocked snapshot before registering MemoryDB watchers. if (newStatus !== snapshot.executionStatus) { snapshot = await this.executionSnapshotSystem.createExecutionSnapshot(prisma, { run: { @@ -489,7 +736,7 @@ export class WaitpointSystem { }, snapshot: { executionStatus: newStatus, - description: "Run was blocked by a waitpoint.", + description: WAITPOINT_BLOCKED_SNAPSHOT_DESCRIPTION, }, previousSnapshotId: snapshot.id, previousSnapshot: snapshot, @@ -508,15 +755,51 @@ export class WaitpointSystem { await sendNotificationToWorker({ runId, snapshot, eventBus: this.$.eventBus }); } + // Phase 3: register watchers and collect results that completed during registration. + // Use the updated pending count to decide whether the run can resume. + if (registration) { + const result = await this.coordinator.registerBlockers(registration); + pendingCount = result.pendingCount; + const cachedSpanCompletions = result.cachedSpanCompletions; + + // Registration succeeded. Remove this operation's cleanup job. + // Keep its acceptance proof until the remaining jobs and events are handled. + if (blockOperation?.absorptionId) { + await this.$.worker.ack( + this.#absorptionCompensationObligationId( + runId, + blockOperation.id, + blockOperation.absorptionId + ) + ); + } + + // Close trace spans for results that completed before watcher registration. + // Other waits can still block the run. Repeated completion events are safe. + for (const span of cachedSpanCompletions ?? []) { + this.$.eventBus.emit("cachedRunCompleted", { + time: new Date(), + span: { id: span.spanId, createdAt: new Date(span.spanCreatedAt) }, + blockedRunId: span.blockedRunId, + hasError: span.hasError, + cachedRunId: span.cachedRunId, + }); + } + } + const isRunBlocked = pendingCount > 0; + if (timeout) { for (const waitpoint of $waitpoints) { await this.$.worker.enqueue({ id: `finishWaitpoint.${waitpoint}`, job: "finishWaitpoint", - payload: { - waitpointId: waitpoint, - error: JSON.stringify(timeoutError(timeout)), - }, + payload: storeRoutingEnabled + ? { + waitpointId: waitpoint, + organizationId, + error: JSON.stringify(timeoutError(timeout)), + } + : { waitpointId: waitpoint, error: JSON.stringify(timeoutError(timeout)) }, availableAt: timeout, }); } @@ -527,15 +810,37 @@ export class WaitpointSystem { // The fixed job id debounces rapid repeat blocks. id: `continueRunIfUnblocked:${runId}`, job: "continueRunIfUnblocked", - payload: storeRoutingEnabled ? { runId, organizationId } : { runId: runId }, + payload: storeRoutingEnabled ? { runId, organizationId } : { runId }, availableAt: new Date(Date.now() + 50), }); } + if (storeRoutingEnabled && blockOperation) { + await this.coordinator.finalizeBlockers({ + runId, + blockId: blockOperation.id, + absorptionId: blockOperation.absorptionId, + waitpointIds: $waitpoints, + batchIndex: batch?.index, + organizationId, + }); + await this.$.worker.ack(`recoverWaitpointBlock:${runId}:${blockOperation.absorptionId}`); + } return snapshot; }); } + /** Use the same cleanup guard for the batch header and its Postgres edge. */ + async withLegacyRegistrationLock( + runId: string, + organizationId: string, + operation: () => Promise + ): Promise { + return this.coordinator instanceof RoutingWaitpointCoordinator + ? this.coordinator.withLegacyRegistrationLock(runId, { organizationId }, operation) + : operation(); + } + /** * blockRunWithWaitpoint without the run lock, for batch items. blockRunWithCreatedBatch has * already moved the parent to EXECUTING_WITH_WAITPOINTS, and a per-item lock causes @@ -545,6 +850,7 @@ export class WaitpointSystem { runId, waitpoints, projectId, + organizationId, timeout, spanIdToComplete, batch, @@ -552,14 +858,19 @@ export class WaitpointSystem { runId: string; waitpoints: string | string[]; projectId: string; + organizationId?: string; timeout?: Date; spanIdToComplete?: string; batch: { id: string; index?: number }; }): Promise { const $waitpoints = typeof waitpoints === "string" ? [waitpoints] : waitpoints; + // Keep the lockless path when disabled. When enabled, take the run lock before + // checking the marker and writing the edge. Unmarked parents do not access MemoryDB. + // The batch already has a blocked snapshot, so no pending count is needed. await this.coordinator.registerBlocksLockless({ runId, + organizationId, waitpointIds: $waitpoints, projectId, spanIdToComplete, @@ -1036,7 +1347,7 @@ export class WaitpointSystem { ) ), storeReader: this.#waitpointStore - ? (ids) => this.#waitpointStore!.readCompletedRecords(ids) + ? (ids) => this.#waitpointStore!.readCompletedRecords(ids, runId) : undefined, }); } @@ -1057,6 +1368,47 @@ export class WaitpointSystem { }); } + public async recoverWaitpointBlock(payload: RecoverWaitpointBlock): Promise { + if (!(await this.isWaitpointStoreEnabled(payload.organizationId))) return; + try { + await this.blockRunWithWaitpoint({ ...payload, recoverOnly: true }); + } catch (error) { + // The run has moved to a later cycle. Clean up this old operation. + if (error instanceof StoreWaitpointBlockCycleAdvanced) { + // Reacquire the run lock before cleanup. Leave newer and terminal blocks unchanged. + await this.ensureAbsorptionCompensated({ + runId: payload.runId, + organizationId: payload.organizationId, + blockId: payload.blockOperation.id, + absorptionId: payload.blockOperation.absorptionId, + }); + return; + } + throw error; + } + } + + /** + * Create the IDs for a MemoryDB wait registration. + * The block ID belongs to the run and wait cycle. Concurrent waits share it. + * The absorption ID also includes the waitpoint. Batch registration adds the batch and position. + * Retries in the same cycle reuse these IDs. A new cycle gets a new block ID. + * Capture the cycle anchor from the current snapshot. + * For Postgres waits, return undefined without reading a snapshot. + */ + public async buildStoreBlockOperation(params: { + runId: string; + waitpointId: string; + organizationId?: string; + }): Promise<{ id: string; absorptionId: string; cycleAnchor: string } | undefined> { + if (!(await this.isStoreRoutingEnabled(params.organizationId))) return undefined; + if (classifyWaitpointCoordinator(params.waitpointId) !== "store") return undefined; + const snapshot = await getLatestExecutionSnapshot(this.$.prisma, params.runId, this.$.runStore); + const cycleAnchor = cycleAnchorSnapshotId(snapshot); + const id = `block:${params.runId}:${cycleAnchor}`; + return { id, absorptionId: `${id}:${params.waitpointId}`, cycleAnchor }; + } + /** Undefined for a success with no output, or for a run that is not final. */ public buildWaitpointOutputFromRun( run: Pick @@ -1171,4 +1523,19 @@ export class WaitpointSystem { return waitpoint; }); } + + public async clearBlockingWaitpoints({ + runId, + organizationId, + tx, + }: { + runId: string; + // Use the run's organisation for routing. If absent, use Postgres without an org lookup. + organizationId?: string; + tx?: PrismaClientOrTransaction; + }) { + const deleted = await this.coordinator.clearRunBlockState({ runId, organizationId, tx }); + + return deleted.count; + } } diff --git a/internal-packages/run-engine/src/engine/tests/checkpointSystemStore.test.ts b/internal-packages/run-engine/src/engine/tests/checkpointSystemStore.test.ts index c9a4d636015..3ec508a408d 100644 --- a/internal-packages/run-engine/src/engine/tests/checkpointSystemStore.test.ts +++ b/internal-packages/run-engine/src/engine/tests/checkpointSystemStore.test.ts @@ -150,7 +150,14 @@ describe("CheckpointSystem store routing (single-DB passthrough)", () => { "checkpoint create routes the TaskRunCheckpoint write + SUSPENDED snapshot through the store", async ({ prisma, redisOptions }) => { const countingStore = new CountingPostgresRunStore({ prisma, readOnlyPrisma: prisma }); - const engine = new RunEngine(createEngineOptions(redisOptions, prisma, countingStore)); + let checkpointFlagCalls = 0; + const engine = new RunEngine({ + ...createEngineOptions(redisOptions, prisma, countingStore), + waitpointStoreEnabled: async () => { + checkpointFlagCalls++; + return false; + }, + }); try { const { run, blockingSnapshotId } = await driveToCheckpointable( @@ -161,6 +168,7 @@ describe("CheckpointSystem store routing (single-DB passthrough)", () => { const checkpointsBefore = countingStore.checkpointCreates; const snapshotsBefore = countingStore.snapshotCreates; + checkpointFlagCalls = 0; const checkpointResult = await engine.createCheckpoint({ runId: run.id, @@ -174,6 +182,10 @@ describe("CheckpointSystem store routing (single-DB passthrough)", () => { }); expect(checkpointResult.ok).toBe(true); + // No store router is installed: a checkpoint must not resolve the waitpoint flag. + expect(checkpointFlagCalls).toBe(0); + // An unwired check must not create a Promise on the checkpoint path. + expect(engine.waitpointSystem.isStoreRoutingEnabled()).toBe(false); // (1) a TaskRunCheckpoint row exists for the run (joined via the SUSPENDED snapshot). const persistedCheckpoint = await prisma.taskRunCheckpoint.findFirst({ diff --git a/internal-packages/run-engine/src/engine/types.ts b/internal-packages/run-engine/src/engine/types.ts index 39ba58855fd..5fed2507d45 100644 --- a/internal-packages/run-engine/src/engine/types.ts +++ b/internal-packages/run-engine/src/engine/types.ts @@ -24,6 +24,7 @@ import { type GlobalRateLimiter, } from "@trigger.dev/redis-worker"; import type { ControlPlaneResolver } from "./controlPlaneResolver.js"; +import type { WaitpointSystemOptions } from "./systems/waitpointSystem.js"; import type { FairQueueSelectionStrategyOptions } from "../run-queue/fairQueueSelectionStrategy.js"; import type { RunQueueMetricsEmitter, RunQueueOptions } from "../run-queue/index.js"; import type { MinimalAuthenticatedEnvironment } from "../shared/index.js"; @@ -146,6 +147,7 @@ export type RunEngineOptions = { processMarkedSchedule?: string; scanJitterInMs?: number; processMarkedJitterInMs?: number; + scanCount?: number; }; /** TTL system options for automatic run expiration */ ttlSystem?: { @@ -277,6 +279,12 @@ export type RunEngineOptions = { * completion whose inline side effects never acked it. Default: 30s. */ completionGuardDelayMs?: number; + + /** + * The MemoryDB completed-record reader, for resolving store-format completed waitpoints in a + * Redis-primary resume cycle. Dark by default (undefined leaves the resolver Postgres-only). + */ + waitpointStore?: WaitpointSystemOptions["waitpointStore"]; heartbeatTimeoutsMs?: Partial; repairSnapshotTimeoutMs?: number; treatProductionExecutionStallsAsOOM?: boolean; diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/fanoutWorker.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/fanoutWorker.test.ts index 5d31f5d5687..711525233d2 100644 --- a/internal-packages/run-engine/src/engine/waitpointCoordinator/fanoutWorker.test.ts +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/fanoutWorker.test.ts @@ -15,6 +15,9 @@ import { fanoutRetryDelayMs } from "./fanoutPolicy.js"; import { WaitpointFanoutWorker, type FanoutWorkerHooks } from "./fanoutWorker.js"; import { edgeField, + runBlockKeys, + runResultKey, + runWithdrawalKey, fanoutIndexKeys, fanoutPartition, waitpointKeys, @@ -225,7 +228,10 @@ describe("completion with one watcher", () => { expect(tick.visits).toHaveLength(1); expect(tick.visits[0]).toMatchObject({ delivered: 1, outcome: "drained", pages: 1 }); - expect(JSON.parse((await receiptFor(probe, "run_1", "w_a"))!)).toEqual(completion()); + expect(JSON.parse((await receiptFor(probe, "run_1", "w_a"))!)).toEqual({ + ...completion(), + record: record("w_a"), + }); expect(await probe.scard("wp:v1:run:{run_1}:pend")).toBe(0); const keys = waitpointKeys("w_a"); @@ -609,7 +615,10 @@ describe("worker failure and reclaiming", () => { // run still holds exactly one receipt. expect(redelivered).toMatchObject({ duplicates: 1, delivered: 0, outcome: "drained" }); expect(await probe.hlen("wp:v1:run:{run_1}:done")).toBe(1); - expect(JSON.parse((await receiptFor(probe, "run_1", "w_a"))!)).toEqual(completion()); + expect(JSON.parse((await receiptFor(probe, "run_1", "w_a"))!)).toEqual({ + ...completion(), + record: record("w_a"), + }); } finally { probe.disconnect(); await store.quit(); @@ -2532,7 +2541,10 @@ describe("block-scoped watcher registration", () => { // Refused, and told which block the run is actually on. It neither overwrote the // receipt nor re-armed a handoff block two already owns. expect(late).toMatchObject({ outcome: "stale", currentBlockId: BLOCK_2 }); - expect(JSON.parse((await receiptFor(probe, RUN, "w_a"))!)).toEqual(completion()); + expect(JSON.parse((await receiptFor(probe, RUN, "w_a"))!)).toEqual({ + ...completion(), + record: record("w_a"), + }); const state = await store.readBlockState(RUN); expect(state.blockId).toBe(BLOCK_2); expect(state.deliveredIds).toEqual(["w_a"]); @@ -3253,7 +3265,10 @@ describe("completion encoding reuse", () => { // And the envelope every run received is unchanged. for (const runId of runIds) { - expect(JSON.parse((await receiptFor(probe, runId, "w_a"))!)).toEqual(completion()); + expect(JSON.parse((await receiptFor(probe, runId, "w_a"))!)).toEqual({ + ...completion(), + record: record("w_a"), + }); } } finally { probe.disconnect(); @@ -3949,3 +3964,73 @@ describe("a quarantined claim reports failures, not attempts", () => { } ); }); + +describe("delivery after terminal retention", () => { + redisTest( + "restarted fanout cannot resurrect a run after its terminal marker expires while withdrawal remains owed", + async ({ redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions, terminalRetentionMs: 20 }); + const probe = createRedisClient(redisOptions); + const clock = fakeClock(); + let first = true; + const stopped = worker(store, { + clock: clock.now, + hooks: { + beforeDeliver: async () => { + if (!first) return; + first = false; + await store.terminalRetainAndClear({ + runId: "expired-run", + blockId: `${BLOCK}_expired-run`, + }); + throw new Error("withdrawal/worker interrupted before delivery"); + }, + }, + }); + const restarted = worker(store, { workerId: "restarted", clock: clock.now }); + try { + await pending(store, "late-source"); + await blockRuns(store, "late-source", ["expired-run"]); + await store.complete({ waitpointId: "late-source", completion: completion() }); + clock.advance(1_000); + expect((await stopped.visit("late-source")).failures).toBe(1); + await expect + .poll(() => probe.exists(runBlockKeys("expired-run").state), { + timeout: 2000, + interval: 5, + message: "terminal marker did not expire", + }) + .toBe(0); + expect(await probe.pttl(waitpointKeys("late-source").record)).toBe(-1); + expect(await probe.exists(runWithdrawalKey("expired-run", `${BLOCK}_expired-run`))).toBe(1); + // Worker/queue processing may stop for arbitrarily long, independently of retry count. + clock.advance(7 * 24 * 60 * 60 * 1000); + const result = await restarted.visit("late-source"); + expect(result.outcome).toBe("drained"); + expect(result.rejected).toBe(1); + expect( + await probe.exists( + runResultKey("expired-run"), + ...Object.values(runBlockKeys("expired-run")) + ) + ).toBe(0); + // Explicit primitive early delivery remains supported for callers that own absorption. + expect( + ( + await store.deliverCompletion({ + runId: "early-run", + blockId: "early", + waitpointId: "late-source", + completion: completion(), + }) + ).outcome + ).toBe("delivered"); + } finally { + await stopped.stop(); + await restarted.stop(); + await store.quit(); + await probe.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/fanoutWorker.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/fanoutWorker.ts index 00b597a96c8..1bede64769b 100644 --- a/internal-packages/run-engine/src/engine/waitpointCoordinator/fanoutWorker.ts +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/fanoutWorker.ts @@ -583,7 +583,7 @@ export class WaitpointFanoutWorker { // that will never succeed is worth telling apart from one that might. let encoded: EncodedWaitpointCompletion; try { - encoded = encodeCompletionForDelivery(completion, `fanout:${waitpointId}`); + encoded = encodeCompletionForDelivery(completion, `fanout:${waitpointId}`, claim.record); } catch (error) { this.logger.error("Waitpoint fanout completion is undeliverable", { waitpointId, @@ -702,6 +702,7 @@ export class WaitpointFanoutWorker { blockId, waitpointId, encoded, + requireCurrentBlock: true, }); // `stale`, `terminal` and `acked` all mean the run's shard will refuse a // redelivery identically, so the watcher is retired rather than retried. diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.ts index 554db00dedc..785a0adcd40 100644 --- a/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.ts +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.ts @@ -1,3 +1,5 @@ +import { createHash } from "node:crypto"; + /** * Waitpoint coordination keyspace. Three hash tags, deliberately: * @@ -40,7 +42,20 @@ export type WaitpointKeys = { /** The durable fanout entry: claim, lease, failure streak and drain state. */ fanout: string; }; -export type RunBlockKeys = { pend: string; done: string; edge: string; state: string }; +export type RunBlockKeys = { + pend: string; + done: string; + edge: string; + state: string; + /** Edge counts by waitpoint ID and operation claim counts by `!edgeField`. */ + refs: string; + /** Stable edge order and immutable first-completion ordinals for bounded snapshot reads. */ + edgeIndex: string; + doneOrder: string; + /** Unreferenced receipts are detached in constant work by selective reconciliation. */ + early: string; + earlyOrder: string; +}; export type FanoutIndexKeys = { due: string; quarantine: string }; export function waitpointKeys(waitpointId: string): WaitpointKeys { @@ -55,9 +70,55 @@ export function runBlockKeys(runId: string): RunBlockKeys { done: `${base}:done`, edge: `${base}:edge`, state: `${base}:st`, + refs: `${base}:refs`, + edgeIndex: `${base}:ei`, + doneOrder: `${base}:do`, + early: `${base}:early`, + earlyOrder: `${base}:eo`, }; } +/** Immutable completions referenced by this parent's durable snapshots, retained until terminal cleanup. */ +export function runResultKey(runId: string): string { + return `${NAMESPACE}:run:{${runId}}:results`; +} + +/** + * The durable manifest retained for ONE block while its watchers are being + * withdrawn (terminal release or non-terminal abandon). Its target hash is detached atomically from + * the active edge hash. The manifest survives even an empty hash so a failed or interrupted + * withdrawal can be retried from an authoritative target list even when the active block is gone or a + * newer block is installed. Keyed by block id (same hash slot as the run's block keys) so it never + * collides with another cycle's record. + */ +export function runWithdrawalKey(runId: string, blockId: string): string { + return `${NAMESPACE}:run:{${runId}}:wd:${blockId}`; +} + +/** The detached edge hash owned by a retained withdrawal; never expires before convergence. */ +export function runWithdrawalTargetsKey(runId: string, blockId: string): string { + return `${runWithdrawalKey(runId, blockId)}:edges`; +} + +/** + * Store the edges claimed by one registration, plus its acceptance markers. + * Cleanup releases these claims without removing edges still claimed by another operation. + * After phase 3, keep only the request proof until registration finishes. + * Each operation has its own key in the run's Redis hash slot. + */ +export function runContributionKey(runId: string, blockId: string, absorptionId: string): string { + return `${NAMESPACE}:run:{${runId}}:cn:${blockId}:${absorptionId}`; +} + +/** Outstanding contribution identities for a known run; bounded pages, no TTL or keyspace scan. */ +export function runContributionIndexKey(runId: string): string { + return `${NAMESPACE}:run:{${runId}}:contributions`; +} + +export function contributionMember(blockId: string, absorptionId: string): string { + return JSON.stringify([blockId, absorptionId]); +} + /** * How many partitions the fanout indexes are split across. * @@ -206,3 +267,22 @@ export function assertSingleSlot(operation: string, keys: string[]): void { } } } + +/** Source holds that an unfinished registration must release, in its parent's slot. */ +export function runRegistrationSourcesKey( + runId: string, + blockId: string, + absorptionId: string +): string { + return `${runContributionKey(runId, blockId, absorptionId)}:sources`; +} + +/** Hash the full request. Reserved proof markers cannot match edge fields. */ +export function registrationEvidence(waitpointIds: string[], batchIndex?: number): string { + return ( + "!request:" + + createHash("sha256") + .update(JSON.stringify([batchIndex ?? null, [...new Set(waitpointIds)].sort()])) + .digest("hex") + ); +} diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts index 4f87f655570..96897ebb94a 100644 --- a/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts @@ -6,18 +6,26 @@ import { UnclassifiableRunId, } from "@trigger.dev/core/v3/isomorphic"; import type { Logger } from "@trigger.dev/core/logger"; -import type { PrismaClient, Waitpoint } from "@trigger.dev/database"; +import type { PrismaClient, PrismaClientOrTransaction, Waitpoint } from "@trigger.dev/database"; import { boundedIn, Prisma } from "@trigger.dev/database"; import { nanoid } from "nanoid"; import { UnclassifiableWaitpointId } from "../errors.js"; import type { + AbandonRunBlockersParams, + AbsorbBlockersParams, + AbsorbedBlockersParams, + AbsorbBlockersResult, + AcknowledgeResumeParams, AssociatedWaitpointData, + BlockerRegistration, ClearRunBlockStateParams, + ReleaseRunBlockersParams, CompleteParams, CompleteResult, CreateDateTimeWaitpointParams, CreateManualWaitpointParams, CreateWaitpointResult, + RegisterBlockersResult, RegisterBlocksLocklessParams, RegisterBlocksParams, RunBlockEdge, @@ -119,6 +127,80 @@ export class LegacyPostgresWaitpointCoordinator implements WaitpointCoordinator await this.#writeBlockEdges(params); } + async finalizeBlockers(_params: AbsorbedBlockersParams): Promise {} + + async hasAbsorbedBlockers(_params: AbsorbedBlockersParams): Promise { + return false; + } + + // Phase 1. Legacy has no absorb/register split: the edge write IS the absorption, and the separate + // pending re-count is the same READ COMMITTED re-read `registerBlocks` does. The count is carried in + // the noop continuation so phase 3 can return it unchanged. + async absorbBlockers({ client, ...edge }: AbsorbBlockersParams): Promise { + await this.#writeBlockEdges(edge); + const pendingCount = await this.runStore.countPendingWaitpoints( + edge.waitpointIds, + client, + edge.runId + ); + return { pendingCount, registration: { kind: "noop", pendingCount } }; + } + + // Phase 3. Nothing to register for a Postgres blocker — completion fans out at complete() time by + // querying the edges. Returns the pending count absorb carried, so the caller's resume decision is + // identical to today's single-call behaviour. + async registerBlockers(registration: BlockerRegistration): Promise { + if (registration.kind !== "noop") { + throw new Error( + `LegacyPostgresWaitpointCoordinator.registerBlockers received a ${registration.kind} continuation` + ); + } + return { pendingCount: registration.pendingCount }; + } + + // Postgres has no store block-operation id. + async currentStoreBlockId(): Promise { + return undefined; + } + + // Postgres has no watcher registration or resume handoff to acknowledge — completion already fanned + // out at complete() time — so the resume cleanup is just deleting the run's consumed edge rows. + async acknowledgeResume(params: AcknowledgeResumeParams): Promise { + await this.clearRunBlockState({ runId: params.runId, edgeIds: params.edgeIds }); + } + + // No terminal-fence to install for Postgres: deleting the edge rows is the cancel/terminal cleanup. + async releaseRunBlockers(params: ReleaseRunBlockersParams): Promise<{ count: number }> { + return this.clearRunBlockState({ runId: params.runId, tx: params.tx }); + } + + // No terminal marker to avoid either: a retry abandon is the same edge delete for Postgres. + async abandonRunBlockers(params: AbandonRunBlockersParams): Promise<{ count: number }> { + return this.clearRunBlockState({ runId: params.runId, tx: params.tx }); + } + + // No store watchers to withdraw on the legacy half — the retained store replay never reaches here. + async resumeRetainedWithdrawal(): Promise<{ count: number }> { + return { count: 0 }; + } + + // A pure-Postgres run IS legacy-only: delete its edges in the caller's client, before the transition. + async clearLegacyBlockersBeforeTransition(params: { + runId: string; + organizationId?: string; + tx?: PrismaClientOrTransaction; + }): Promise<{ count: number }> { + return this.clearRunBlockState({ runId: params.runId, tx: params.tx }); + } + + // No store contribution to compensate on the legacy half of a mixed absorption. + async compensateAbsorption(): Promise {} + + // No store block to fence: a pure-Postgres run never installs a MemoryDB cycle over a previous one. + async armCleanupFence(): Promise {} + + async assertNoCleanupOwed(): Promise {} + async complete({ waitpointId, output }: CompleteParams): Promise { // Residency store-selection guard. complete arrives with only (waitpointId, output) — no run // id — so the owning run-ops store is selected by the waitpoint's own residency. In single-DB diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/routingCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/routingCoordinator.ts index 28b212bfffb..b6440d4e5c4 100644 --- a/internal-packages/run-engine/src/engine/waitpointCoordinator/routingCoordinator.ts +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/routingCoordinator.ts @@ -1,6 +1,8 @@ import type { Logger } from "@trigger.dev/core/logger"; -import type { Waitpoint } from "@trigger.dev/database"; +import type { PrismaClientOrTransaction, Waitpoint } from "@trigger.dev/database"; import { + StoreAbsorptionCompensationFailed, + StoreWaitpointBlockOperationIdentityRequired, StoreWaitpointRegistrationUnsupported, StoreWaitpointWithoutStoreCoordinator, TransactionalStoreClearUnsupported, @@ -13,16 +15,24 @@ import { } from "./mintPolicy.js"; import { classifyWaitpointCoordinator, partitionByResidency } from "./routing.js"; import type { + AbandonRunBlockersParams, + AbsorbBlockersParams, + AbsorbedBlockersParams, + AbsorbBlockersResult, + AcknowledgeResumeParams, AssociatedWaitpointData, + BlockerRegistration, ClearRunBlockStateParams, + CurrentBlockContext, + ReleaseRunBlockersParams, CompleteParams, CompleteResult, CreateDateTimeWaitpointParams, CreateManualWaitpointParams, CreateWaitpointResult, + RegisterBlockersResult, RegisterBlocksLocklessParams, RegisterBlocksParams, - RegisterBlocksResult, RunBlockEdge, RunKeyedRoutingContext, WaitpointCoordinator, @@ -58,6 +68,10 @@ export type RoutingWaitpointCoordinatorOptions = { * current or historical enrollment. False skips MemoryDB; unavailable/missing readers are conservative. */ mightHaveStoreBlockers?: (runKey: WaitpointRunKey) => boolean | Promise; + /** Persist the run marker BEFORE the first store mutation; failure must abort the operation. */ + markStoreWaitpoints?: (runId: string, organizationId?: string) => Promise; + /** Serialize marked parents' lockless legacy writes with their cross-store cleanup. */ + withLegacyRegistrationLock?: (runId: string, operation: () => Promise) => Promise; logger?: Logger; }; @@ -78,6 +92,8 @@ export type RoutingWaitpointCoordinatorOptions = { export class RoutingWaitpointCoordinator implements WaitpointCoordinator { readonly #legacy: WaitpointCoordinator; readonly #store: WaitpointCoordinator | undefined; + readonly #markStoreWaitpoints?: (runId: string, organizationId?: string) => Promise; + readonly #withLegacyRegistrationLock?: RoutingWaitpointCoordinatorOptions["withLegacyRegistrationLock"]; readonly #mintPolicy: RoutingWaitpointCoordinatorMintPolicy; readonly #isStoreEnabled?: RoutingWaitpointCoordinatorOptions["isStoreEnabled"]; readonly #mightHaveStoreBlockers?: (runKey: WaitpointRunKey) => boolean | Promise; @@ -86,6 +102,8 @@ export class RoutingWaitpointCoordinator implements WaitpointCoordinator { this.#legacy = options.legacy; this.#store = options.store; this.#mintPolicy = options.mintPolicy; + this.#markStoreWaitpoints = options.markStoreWaitpoints; + this.#withLegacyRegistrationLock = options.withLegacyRegistrationLock; this.#isStoreEnabled = options.isStoreEnabled; this.#mightHaveStoreBlockers = options.mightHaveStoreBlockers; } @@ -129,6 +147,7 @@ export class RoutingWaitpointCoordinator implements WaitpointCoordinator { return decision.residency; } + // Native absorption validates wiring and block identity independently of parent TRES residency. // The registration guard, shared by both register paths. Fails loud BEFORE any write and returns // only the legacy subset — store-format registration is a three-step barrier the single-call seam // cannot express (see StoreWaitpointRegistrationUnsupported), so it is refused here rather than @@ -176,7 +195,7 @@ export class RoutingWaitpointCoordinator implements WaitpointCoordinator { organizationId: params.organizationId, })) ) { - // Never-enabled / no store: legacy only, and the caller's tx joins the single-store clear. + // Unmarked / no store: legacy only, and the caller's tx joins the single-store clear. return this.#legacy.clearRunBlockState(params); } // The run may hold MemoryDB blockers. A clear bound to a rollback-capable Postgres lifecycle @@ -193,6 +212,152 @@ export class RoutingWaitpointCoordinator implements WaitpointCoordinator { return { count: legacyResult.count + storeResult.count }; } + async currentStoreBlockId( + runId: string, + context?: CurrentBlockContext + ): Promise { + if ( + !(await this.#runMightHaveStoreBlockers({ runId, organizationId: context?.organizationId })) + ) { + return undefined; + } + return this.#store!.currentStoreBlockId(runId, context); + } + + async abandonRunBlockers(params: AbandonRunBlockersParams): Promise<{ count: number }> { + if ( + !(await this.#runMightHaveStoreBlockers({ + runId: params.runId, + organizationId: params.organizationId, + })) + ) { + return this.#legacy.abandonRunBlockers(params); + } + // Like releaseRunBlockers: the store abandon cannot join a Postgres transaction, so a run that may + // hold store blockers is abandoned only AFTER the durable retry transition, never inside its tx. + if (params.tx) { + throw new TransactionalStoreClearUnsupported(params.runId); + } + // SEQUENCED, legacy FIRST: delete this cycle's legacy edges, then atomically clear+withdraw the store + // block. Load-bearing — if the legacy delete fails the store block stays active and the obligation can + // retry the whole initial cleanup safely; once the store block is atomically cleared into its retained + // withdrawal record, legacy is already gone. + const legacyResult = await this.#legacy.abandonRunBlockers(params); + const storeResult = await this.#store!.abandonRunBlockers(params); + return { count: legacyResult.count + storeResult.count }; + } + + async resumeRetainedWithdrawal(params: { + runId: string; + blockId: string; + organizationId?: string; + }): Promise<{ count: number }> { + // Store-only by contract: a delayed obligation whose block is no longer current must never reach the + // legacy coordinator (its block-id-less delete would drop a newer block's edges). + if ( + !(await this.#runMightHaveStoreBlockers({ + runId: params.runId, + organizationId: params.organizationId, + })) + ) { + return { count: 0 }; + } + return this.#store!.resumeRetainedWithdrawal(params); + } + + async acknowledgeResume(params: AcknowledgeResumeParams): Promise { + // Legacy deletes its edge rows; the store acknowledges the handoff and drains under the block-id + // fence. A unmarked run reaches only Postgres. Both touch different stores, so run concurrently. + if ( + !(await this.#runMightHaveStoreBlockers({ + runId: params.runId, + organizationId: params.organizationId, + })) + ) { + await this.#legacy.acknowledgeResume(params); + return; + } + await Promise.all([ + this.#legacy.acknowledgeResume(params), + this.#store!.acknowledgeResume(params), + ]); + } + + async releaseRunBlockers(params: ReleaseRunBlockersParams): Promise<{ count: number }> { + if ( + !(await this.#runMightHaveStoreBlockers({ + runId: params.runId, + organizationId: params.organizationId, + })) + ) { + // Unmarked / no store: legacy only, and the caller's tx joins the single-store delete. + return this.#legacy.releaseRunBlockers(params); + } + // The store's terminal-first release cannot join a Postgres transaction, so a run that may hold + // store blockers must be released AFTER the lifecycle transition commits — never inside its tx. + if (params.tx) { + throw new TransactionalStoreClearUnsupported(params.runId); + } + // ORDERED, store FIRST: terminalize the exact block and finish its retained withdrawal (the store + // adapter is replay-safe — it finishes an already-terminalized block's retained withdrawal), THEN + // delete the Postgres edges. On a partial failure the whole call rejects and the obligation replays: + // the store half converges (or no-ops) and the legacy delete is retried, so a legacy failure after a + // successful store terminalization can never strand the Postgres edges. Never concurrent — a store + // success + legacy failure under Promise.all cleared the block and dropped the legacy retry. + const storeResult = await this.#store!.releaseRunBlockers(params); + const legacyResult = await this.#legacy.releaseRunBlockers(params); + return { count: legacyResult.count + storeResult.count }; + } + + async armCleanupFence(params: { + runId: string; + blockId: string; + organizationId?: string; + }): Promise { + // Store-only, and only for a run that may hold store blockers: a unmarked run has no store block + // to fence and reaches nothing here (no MemoryDB write on the dark path). + if ( + !(await this.#runMightHaveStoreBlockers({ + runId: params.runId, + organizationId: params.organizationId, + })) + ) { + return; + } + await this.#store!.armCleanupFence(params); + } + + async assertNoCleanupOwed(runId: string, context?: RunKeyedRoutingContext): Promise { + if (await this.#runMightHaveStoreBlockers({ runId, organizationId: context?.organizationId })) { + await this.#store!.assertNoCleanupOwed(runId); + } + } + + async clearLegacyBlockersBeforeTransition(params: { + runId: string; + organizationId?: string; + tx?: PrismaClientOrTransaction; + }): Promise<{ count: number }> { + // Unconditional legacy edge delete in the caller's tx. The DECISION to call this (only when the run has + // no current store block) is made once, per-run, by the caller, so a legacy-only run is still + // pre-cleared even when other runs in its org use MemoryDB. Store side never + // touched here. + return this.#legacy.clearLegacyBlockersBeforeTransition(params); + } + + async compensateAbsorption(params: { + runId: string; + blockId: string; + absorptionId: string; + /** Recovery may discard only preparation for this request, never an accepted operation. */ + unacceptedEvidence?: string; + organizationId?: string; + }): Promise { + if (!(await this.isStoreRoutingEnabled(params.organizationId))) return; + // Store-only concern: the failed mixed absorption's legacy half wrote nothing (store went first). + await this.#store!.compensateAbsorption(params); + } + async readRunBlockState( runId: string, context?: RunKeyedRoutingContext @@ -212,19 +377,17 @@ export class RoutingWaitpointCoordinator implements WaitpointCoordinator { return [...legacyEdges, ...storeEdges]; } - async registerBlocks(params: RegisterBlocksParams): Promise { + async registerBlocks(params: RegisterBlocksParams): Promise<{ pendingCount: number }> { if (!(await this.isStoreRoutingEnabled(params.organizationId))) { - // Drop routing-only context so the disabled legacy call retains its original shape. - const { organizationId: _organizationId, ...legacyParams } = params; - return this.#legacy.registerBlocks(legacyParams); + return this.#legacy.registerBlocks(params); } // Store-format blockers are refused inside the guard; only legacy ids reach a write. const legacyIds = this.#legacyRegistrationSubset(params); if (legacyIds.length === 0) { - return { pendingCount: 0, storeRoutingEnabled: true }; + return { pendingCount: 0 }; } - const result = await this.#legacy.registerBlocks({ ...params, waitpointIds: legacyIds }); - return { ...result, storeRoutingEnabled: true }; + await this.assertNoCleanupOwed(params.runId, params); + return this.#legacy.registerBlocks({ ...params, waitpointIds: legacyIds }); } async registerBlocksLockless(params: RegisterBlocksLocklessParams): Promise { @@ -232,9 +395,177 @@ export class RoutingWaitpointCoordinator implements WaitpointCoordinator { return this.#legacy.registerBlocksLockless(params); } const legacyIds = this.#legacyRegistrationSubset(params); - if (legacyIds.length > 0) { - await this.#legacy.registerBlocksLockless({ ...params, waitpointIds: legacyIds }); + if (legacyIds.length === 0) return; + await this.withLegacyRegistrationLock(params.runId, params, () => + this.#legacy.registerBlocksLockless({ ...params, waitpointIds: legacyIds }) + ); + } + + /** Guard a legacy registration's preparation and write with the existing cleanup run lock. */ + async withLegacyRegistrationLock( + runId: string, + context: RunKeyedRoutingContext | undefined, + operation: () => Promise + ): Promise { + if ( + !this.#store || + (this.#isStoreEnabled && !(await this.#isStoreEnabled(context?.organizationId))) + ) { + return operation(); + } + if (!this.#withLegacyRegistrationLock) { + throw new Error("Enabled legacy registration requires the run lock"); + } + return this.#withLegacyRegistrationLock(runId, async () => { + // The marker is monotonic but can change from false to true when the first native wait + // joins. Decide under the parent lock too, so a stale false cannot authorize child creation + // across that first block's retry cleanup. Unmarked parents still never read MemoryDB. + await this.assertNoCleanupOwed(runId, context); + return operation(); + }); + } + + // Validate wiring and caller-owned block identity before either coordinator is mutated. + // Parent snapshot residency is independent of a shared waitpoint's storage home. + #assertStoreBlockersEligible( + runId: string, + storeIds: string[], + blockId: string | undefined + ): void { + if (storeIds.length === 0) { + return; + } + if (!this.#store) { + throw new StoreWaitpointWithoutStoreCoordinator(storeIds[0]!); + } + // A store block operation must carry a caller-owned identity — re-deriving it here from run state + // could not fence a delayed operation from an older cycle. Fail loud before any mutation. + if (!blockId) { + throw new StoreWaitpointBlockOperationIdentityRequired({ runId, waitpointIds: storeIds }); + } + } + + async finalizeBlockers(params: AbsorbedBlockersParams): Promise { + if (!(await this.isStoreRoutingEnabled(params.organizationId))) return; + await this.#store!.finalizeBlockers(params); + } + + async hasAbsorbedBlockers(params: AbsorbedBlockersParams): Promise { + if (!(await this.isStoreRoutingEnabled(params.organizationId))) return false; + return this.#store!.hasAbsorbedBlockers(params); + } + + async absorbBlockers(params: AbsorbBlockersParams): Promise { + if (!(await this.isStoreRoutingEnabled(params.organizationId))) { + return this.#legacy.absorbBlockers(params); } + const partition = partitionByResidency(params.waitpointIds, (id) => id); + const legacyIds = partition.legacy.map((e) => e.item); + const storeIds = partition.store.map((e) => e.item); + + this.#assertStoreBlockersEligible(params.runId, storeIds, params.blockId); + + // Native/mixed absorption checks the fence atomically in its store CAS. Legacy-only writes + // need the same admission barrier before Postgres can accept a later attempt's edges. + if (storeIds.length === 0 && legacyIds.length > 0) { + await this.assertNoCleanupOwed(params.runId, params); + } + + // Commit the discovery marker first. A crash after it leaves only a harmless extra lookup; + // a crash after absorption can never leave Redis edges hidden from a restarted/rolled-back process. + if (storeIds.length > 0) await this.#markStoreWaitpoints?.(params.runId, params.organizationId); + + // Store CAS FIRST, then legacy — the two mutating absorptions never run concurrently. A stale/refused + // store operation throws here BEFORE any legacy edge is written. It cannot change the active store + // block or register watchers, but may finish an already-retained predecessor withdrawal. Legacy + // edges are written only after the store half is durably accepted. + const storeResult = + storeIds.length > 0 + ? await this.#store!.absorbBlockers({ + ...params, + waitpointIds: storeIds, + registrationWaitpointIds: params.waitpointIds, + }) + : undefined; + + let legacyResult: AbsorbBlockersResult | undefined; + try { + legacyResult = + legacyIds.length > 0 + ? await this.#legacy.absorbBlockers({ ...params, waitpointIds: legacyIds }) + : undefined; + } catch (legacyError) { + // Store accepted but legacy failed: compensate ONLY this invocation's store contribution, fenced by + // the block id and the exact edges just contributed, so an earlier additive call's blockers on the + // same block and any early-delivery receipts survive. If compensation itself fails, surface the + // exact fenced targets so the caller arms a durable retry — never report clean while leaving residue. + if (storeResult && storeIds.length > 0) { + try { + await this.#store!.compensateAbsorption({ + runId: params.runId, + blockId: params.blockId!, + absorptionId: params.absorptionId!, + organizationId: params.organizationId, + }); + } catch (compensationError) { + throw new StoreAbsorptionCompensationFailed({ + runId: params.runId, + blockId: params.blockId!, + absorptionId: params.absorptionId!, + cause: compensationError, + }); + } + } + throw legacyError; + } + + const parts: BlockerRegistration[] = []; + let pendingCount = 0; + if (legacyResult) { + parts.push(legacyResult.registration); + pendingCount += legacyResult.pendingCount; + } + if (storeResult) { + parts.push(storeResult.registration); + pendingCount += storeResult.pendingCount; + } + + return { pendingCount, registration: { kind: "routing", parts } }; + } + + async registerBlockers(registration: BlockerRegistration): Promise { + if (registration.kind === "noop") return this.#legacy.registerBlockers(registration); + if (registration.kind !== "routing") { + throw new Error( + `RoutingWaitpointCoordinator.registerBlockers received a ${registration.kind} continuation` + ); + } + + // Validate every store part before dispatching either store or legacy mutations. + for (const part of registration.parts) { + if ( + part.kind === "store" && + this.#isStoreEnabled && + !(await this.#isStoreEnabled(part.organizationId)) + ) { + throw new Error("Store waitpoint registration is disabled for this organization"); + } + } + + // Dispatch each part to the coordinator that produced it, concurrently: `noop` is the legacy half, + // `store` the MemoryDB half. Their pending counts sum to the run's post-registration pending set. + const results = await Promise.all( + registration.parts.map((part) => + part.kind === "store" + ? this.#store!.registerBlockers(part) + : this.#legacy.registerBlockers(part) + ) + ); + + return { + pendingCount: results.reduce((sum, r) => sum + r.pendingCount, 0), + cachedSpanCompletions: results.flatMap((r) => r.cachedSpanCompletions ?? []), + }; } async complete(params: CompleteParams): Promise { diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts index b763924e8df..cb8b37105a4 100644 --- a/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts @@ -1,14 +1,18 @@ -import type { Callback, Redis, Result } from "@internal/redis"; +import type { Callback, RedisClient, Result } from "@internal/redis"; /** - * Lua for the waitpoint coordination protocol. Four rules hold throughout: + * Lua for the waitpoint coordination protocol. Scripts assume protocol-owned key types and command + * permissions. Atomic execution excludes interleaving; Redis does not roll back a command error. + * Recovery handles interruption and lost replies between script invocations and cross-shard work. + * + * Four rules hold throughout: * * 1. Every key a script touches is declared in KEYS. No script builds a key name inside * Lua. ioredis prefixes only the KEYS array, so a key minted in Lua would be * unprefixed while the client wrote a prefixed one — and a script with a single * declared key gives the caller's single-slot assertion nothing to compare. - * 2. Lua never parses JSON. Each script branches only on a short status string and moves - * opaque blobs, so every encoding decision stays in TypeScript. + * 2. Completion/edge payloads remain opaque. Watcher registration decodes watcher JSON + * only to fill missing cached-span metadata on a duplicate registration. * 3. A missing HGET returns Lua `false`, not `nil` — measured directly against a live * Redis: `EVAL "return {'a', false, 'c'}"` and a table holding a missing-field HGET * result both come back as 3 elements; only `EVAL "return {'a', nil, 'c'}"` comes back @@ -53,6 +57,9 @@ import type { Callback, Redis, Result } from "@internal/redis"; * delivery. */ +/** Hard ceiling for a single cleanup script's work and target reply. */ +export const RUN_CLEANUP_PAGE_SIZE = 128; + const STORED_COMPLETED = "COMPLETED"; const MISSING = "missing"; @@ -83,6 +90,7 @@ const ACKNOWLEDGED = "acknowledged"; const UNKNOWN = "unknown"; const RETAINED = "retained"; const ARMED = "armed"; +const CLEANUP_OWED = "cleanupOwed"; const PENDING_RECORD = "pending-record"; const ACKED = "acked"; const NOT_DUE = "notdue"; @@ -101,7 +109,14 @@ const PERSIST_RUN_KEYS = ` for i = 1, 4 do redis.call('PERSIST', KEYS[i]) end `; -export function registerWaitpointCommands(redis: Redis): void { +// An accepted registration retains its source until a watcher or parent result owns completion. +export const RETAIN_WAITPOINT_RECORD = ` + if (tonumber(redis.call('HGET', record, 'rh')) or 0) == 0 then + redis.call('PEXPIRE', record, ttl) + end +`; + +export function registerWaitpointCommands(redis: RedisClient): void { /** * KEYS: record. ARGV: recordJson, status ('PENDING'|'COMPLETED'), completionJson (''), * terminalTtlMs. @@ -135,6 +150,11 @@ export function registerWaitpointCommands(redis: Redis): void { if ARGV[3] ~= '' then redis.call('HSET', record, 'c', ARGV[3]) end + -- The immutable completed-run identity, stored as its own field so fanout and registration can read + -- it once per page (as cachedRunId) without decoding the whole record. + if ARGV[5] ~= nil and ARGV[5] ~= '' then + redis.call('HSET', record, 'cbt', ARGV[5]) + end if ARGV[2] == '${STORED_COMPLETED}' then redis.call('PEXPIRE', record, ttl) end @@ -156,19 +176,38 @@ export function registerWaitpointCommands(redis: Redis): void { end if redis.call('HGET', record, 'status') == '${STORED_COMPLETED}' then - return { '${DID_COMPLETE}', redis.call('HGET', record, 'c') or '' } + -- Return the completed-run identity (cbt) in the SAME atomic reply as the completion envelope, so a + -- delivered completion and its cachedRunId can never diverge at the record's expiry. + return { + '${DID_COMPLETE}', + redis.call('HGET', record, 'c') or '', + redis.call('HGET', record, 'cbt') or '', + redis.call('HGET', record, 'r') or '' + } end -- The watcher lands before any flip can read the watcher set, because this script -- and wpComplete are both atomic on this same shard. So a register either appears -- in the flip's watcher set, or it observes COMPLETED above. -- - -- HSETNX: the first registration wins, mirroring the edge's ON CONFLICT DO NOTHING. + -- HSETNX: keep one watcher per run/block/index. Source-publication recovery can + -- register before the foreground creates its cached span, so a duplicate may fill + -- an absent span (and its creation time). The first actual span still wins. -- The queue is pushed only when HSETNX actually set the field, keeping the two in -- step: the hash is the authority for whether a watcher is still live (a cancellation -- HDELs it), the queue is the paging order the fanout worker walks. if redis.call('HSETNX', watchers, ARGV[1], ARGV[2]) == 1 then redis.call('RPUSH', queue, ARGV[1]) + else + local incoming = cjson.decode(ARGV[2]) + if incoming.spanIdToComplete and incoming.spanIdToComplete ~= cjson.null and incoming.spanIdToComplete ~= '' then + local existing = cjson.decode(redis.call('HGET', watchers, ARGV[1])) + if not existing.spanIdToComplete or existing.spanIdToComplete == cjson.null or existing.spanIdToComplete == '' then + existing.spanIdToComplete = incoming.spanIdToComplete + existing.createdAt = incoming.createdAt + redis.call('HSET', watchers, ARGV[1], cjson.encode(existing)) + end + end end return { '${REGISTERED}' } `, @@ -250,7 +289,7 @@ export function registerWaitpointCommands(redis: Redis): void { -- watchers that were withdrawn while the waitpoint was pending, and those would -- otherwise sit on a key with no reader and no TTL. redis.call('DEL', queue, watchers) - redis.call('PEXPIRE', record, ttl) +${RETAIN_WAITPOINT_RECORD} return { '${DID_COMPLETE}', ARGV[1], '${ABSENT}' } `, }); @@ -349,6 +388,7 @@ export function registerWaitpointCommands(redis: Redis): void { redis.call('HGET', fanout, 'cAt') or '0', redis.call('HGET', fanout, 'fail') or '0', epoch, + redis.call('HGET', record, 'cbt') or '', tostring(#page) } for i = 1, #page do @@ -358,6 +398,8 @@ export function registerWaitpointCommands(redis: Redis): void { out[#out + 1] = redis.call('HGET', watchers, page[i]) or '' end + -- Immutable metadata is returned in the same atomic read as the completion. + out[#out + 1] = redis.call('HGET', record, 'r') or '' return out `, }); @@ -448,7 +490,7 @@ export function registerWaitpointCommands(redis: Redis): void { -- left. redis.call('DEL', queue, watchers) redis.call('PEXPIRE', fanout, ttl) - redis.call('PEXPIRE', record, ttl) +${RETAIN_WAITPOINT_RECORD} return { '${DRAINED}', redis.call('HGET', fanout, 'del') or '0' } `, @@ -774,6 +816,13 @@ export function registerWaitpointCommands(redis: Redis): void { `, }); + // Bounded admission read for legacy-only edges. The caller holds the run lock through the + // subsequent Postgres write, serializing it with cleanup-fence installation and replay. + redis.defineCommand("runReadCleanupFence", { + numberOfKeys: 1, + lua: `return { redis.call('HGET', KEYS[1], 'co') or '' }`, + }); + /** * KEYS: pend, done, edge, st. * ARGV: blockId, n, then n groups of 5 — waitpointId, edgeField, edgeJson, reportedFlag @@ -798,16 +847,20 @@ export function registerWaitpointCommands(redis: Redis): void { * The caller therefore owes a fresh id per block operation. */ redis.defineCommand("runAbsorbBlockers", { - numberOfKeys: 4, + numberOfKeys: 14, lua: ` - local pend, done, edge, st = KEYS[1], KEYS[2], KEYS[3], KEYS[4] + local pend, done, edge, st, contrib, results = KEYS[1], KEYS[2], KEYS[3], KEYS[4], KEYS[5], KEYS[6] local n = tonumber(ARGV[4]) + local edgeIndex, doneOrder, early, earlyOrder = KEYS[8], KEYS[9], KEYS[10], KEYS[11] + local wd, targets = KEYS[12], KEYS[13] -- Guard before any write: a wrong n must not half-apply the script. HDEL/HSETNX below - -- are irreversible mid-script, and Redis does not roll back a script that errors. - if n == nil or #ARGV ~= 4 + n * 5 then + -- are irreversible mid-script, and Redis does not roll back a script that errors. The trailing + -- ARGV carries the absorption id, index member and read revision after the edge groups. + if n == nil or (#ARGV ~= 8 + n * 5 and #ARGV ~= 7 + n * 5) then return redis.error_reply('runAbsorbBlockers: arity mismatch') end + local evidenceOffset = #ARGV == 8 + n * 5 and 3 or 2 if ARGV[1] == '' then return redis.error_reply('runAbsorbBlockers: ARGV[1] must be a block id') end @@ -825,6 +878,16 @@ export function registerWaitpointCommands(redis: Redis): void { return { '${TERMINAL}', previous } end + -- The cleanup-owed fence: a previous mixed block still owes its cross-store cleanup (its block-id-less + -- Postgres delete plus store withdrawal). A DIFFERENT block must not become active until that clears, + -- or a delayed B1 cleanup could delete a reused B2 legacy edge. A same-block operation (an idempotent + -- retry of the fenced block itself) is not blocked; the higher layer's cycle-anchor precondition + -- fences a genuine delayed replay of it. + local cleanupOwed = redis.call('HGET', st, 'co') or '' + if cleanupOwed ~= '' and cleanupOwed ~= ARGV[1] then + return { '${CLEANUP_OWED}', cleanupOwed } + end + -- Block ids are random, so they carry no order and cannot be compared. The caller names -- the predecessor it believes it is replacing instead, which is what makes a delayed -- retry of an older operation distinguishable from a legitimate rollover. @@ -835,38 +898,33 @@ export function registerWaitpointCommands(redis: Redis): void { if not (isRetry or isRollover or isFirst) then return { '${STALE}', previous } end + -- Validate the declared predecessor's retained keys before any mutation. Rollover moves the + -- hash instead of enumerating it; the manifest survives a lost reply and empty old blocks. + if isRollover and (redis.call('EXISTS', wd, targets) > 0) then + return redis.error_reply('runAbsorbBlockers: predecessor withdrawal already retained') + end ${PERSIST_RUN_KEYS} - -- Returned to the caller, because this edge hash is the only record of which waitpoint - -- shards the previous cycle registered on, and the sweep below destroys it. + for i = 7, 11 do redis.call('PERSIST', KEYS[i]) end + redis.call('PERSIST', results) local supersededBlock = '' - local supersededFields = {} - if isRollover then - -- A clean sweep. A receipt carries no block id, so nothing here can tell one left by - -- the previous cycle from one belonging to this cycle — and there is no such thing - -- while another block is installed, because a delivery quoting a block id that is not - -- current is refused as stale. Receipts survive only for a first block, where they are - -- a genuine early delivery, and for a same-block retry. - supersededBlock = previous - supersededFields = redis.call('HKEYS', edge) - redis.call('DEL', pend, done, edge) - end - - if not isRetry then - redis.call('HSET', st, 'blk', ARGV[1], 'hs', '') + redis.call('SET', wd, previous) + if redis.call('EXISTS', edge) == 1 then redis.call('RENAME', edge, targets) end + redis.call('PERSIST', targets) + redis.call('UNLINK', pend, done, KEYS[7], edgeIndex, doneOrder, early, earlyOrder) + end + -- Retry discovery is independent of the fresh-rollover branch. + if expectsPrevious and redis.call('GET', wd) == expected then supersededBlock = expected end + redis.call('HSET', st, 'rev', ARGV[#ARGV]) + if not isRetry then redis.call('HSET', st, 'blk', ARGV[1], 'hs', '') end + if evidenceOffset == 3 and ARGV[#ARGV - evidenceOffset] ~= '' then + redis.call('SADD', contrib, '!accepted', ARGV[#ARGV - 1]) + redis.call('ZADD', KEYS[14], 0, ARGV[#ARGV - evidenceOffset + 1]) end - - -- seenDelivered makes the delivered-pair output DISTINCT BY ID: two edges for one - -- waitpoint must contribute one pair, not two. local requestedIds = {} local seenDelivered = {} - -- Slot 1 is the outcome on EVERY path, so a rejection is explicit rather than inferred - -- from a slot that would otherwise hold a count. Slots 4 and 5 delimit the superseded - -- fields, keeping the delivered pairs after them unambiguous however many there are. - local out = { '${ABSORBED}', '0', '0', supersededBlock, tostring(#supersededFields) } - for i = 1, #supersededFields do - out[#out + 1] = supersededFields[i] - end + -- Keep the old field-count slot at zero; targets are read only in hard-bounded pages. + local out = { '${ABSORBED}', '0', '0', supersededBlock, '0' } for i = 0, n - 1 do local id = ARGV[5 + i * 5] @@ -875,10 +933,38 @@ ${PERSIST_RUN_KEYS} local reportedFlag = ARGV[8 + i * 5] local reported = ARGV[9 + i * 5] - -- HSETNX is the ON CONFLICT DO NOTHING of the edge write: a retry must not - -- overwrite the first attempt's metadata. - redis.call('HSETNX', edge, field, edgeJson) + -- Preserve the first edge metadata, but count each operation's claim separately. + -- A failed earlier operation must not remove an edge adopted by a later one. + local inserted = redis.call('HSETNX', edge, field, edgeJson) + if inserted == 1 then + redis.call('HINCRBY', KEYS[7], id, 1) + redis.call('ZADD', edgeIndex, redis.call('HINCRBY', st, 'edgeSeq', 1), field) + end + local claimField = '!' .. field + if ARGV[#ARGV - evidenceOffset] ~= '' then + if redis.call('SISMEMBER', contrib, '!registered') == 0 and redis.call('SADD', contrib, field) == 1 then + -- Existing edges without claim counts predate this protocol or have no operation ID. + if inserted == 0 then redis.call('HSETNX', KEYS[7], claimField, 1) end + if redis.call('HGET', KEYS[7], claimField) ~= '-1' then + redis.call('HINCRBY', KEYS[7], claimField, 1) + end + redis.call('ZADD', KEYS[14], 0, ARGV[#ARGV - evidenceOffset + 1]) + end + else + -- Calls without an operation ID have no compensation path. Keep their claim until cleanup. + redis.call('HSET', KEYS[7], claimField, -1) + end requestedIds[id] = true + -- Adopt the first early receipt before considering a reported completion. Its ordinal and + -- envelope are immutable; moving it invalidates the structural read revision above. + local earlyReceipt = redis.call('HGET', early, id) + if earlyReceipt then + if redis.call('HSETNX', done, id, earlyReceipt) == 1 then + redis.call('ZADD', doneOrder, redis.call('ZSCORE', earlyOrder, id), id) + end + redis.call('HDEL', early, id) + redis.call('ZREM', earlyOrder, id) + end if reportedFlag == '1' then -- Already COMPLETED when the watcher registered. It never becomes pending, even @@ -887,7 +973,12 @@ ${PERSIST_RUN_KEYS} -- HSETNX, not HSET: a fanout delivery that raced ahead of this absorb has -- already written the frozen envelope, and this branch's reported value may be ''. -- Overwriting would trade a real envelope for an empty one. - redis.call('HSETNX', done, id, reported) + if redis.call('HSETNX', done, id, reported) == 1 then + redis.call('ZADD', doneOrder, redis.call('HINCRBY', st, 'seq', 1), id) + end + -- Retain the snapshot result in the same fenced commit as the active block. A source + -- captured before absorption can expire immediately without erasing the accepted result. + redis.call('HSETNX', results, id, redis.call('HGET', done, id)) redis.call('SREM', pend, id) if not seenDelivered[id] then seenDelivered[id] = true @@ -921,8 +1012,15 @@ ${PERSIST_RUN_KEYS} end end + local remaining = redis.call('SCARD', pend) + -- An already-completed source can finish this block during absorption, before registration. + -- Preserve the same durable handoff obligation as delivery, without re-arming an acknowledged + -- cycle or making an empty block with no completion receipts owe a resume. + if remaining == 0 and (redis.call('HLEN', done) + redis.call('HLEN', early)) > 0 and redis.call('HGET', st, 'hs') ~= 'ack' then + redis.call('HSET', st, 'hs', 'owed') + end out[2] = tostring(pendingOfRequested) - out[3] = tostring(redis.call('SCARD', pend)) + out[3] = tostring(remaining) return out `, }); @@ -950,22 +1048,23 @@ ${PERSIST_RUN_KEYS} * HSETNX, so a redelivery is visibly a duplicate, cannot rewrite a frozen receipt, and * cannot signal a resume the fresh delivery has already signalled. * - * One residual case is deliberately left to the reconciliation lane: a delivery that - * arrives after the whole run partition, `term` marker included, has aged out of its - * retention window reads as an early delivery and writes a receipt with no owning block. - * It cannot resume anything — a resume needs a block id and a complete pending set — but - * it is an orphan, and reaping orphans is that lane's work, not this script's. + * Production registration/fanout requires the current block: absorption always precedes watcher + * publication. This also refuses delayed work after terminal state expires. Explicit primitive + * early delivery may omit that requirement when the caller owns a subsequent absorption. */ redis.defineCommand("runDeliverCompletion", { - numberOfKeys: 3, + numberOfKeys: 8, lua: ` - local pend, done, st = KEYS[1], KEYS[2], KEYS[3] + local pend, done, st, results = KEYS[1], KEYS[2], KEYS[3], KEYS[4] if redis.call('HGET', st, 'term') == '1' then return { '${TERMINAL}', '0', '0', '' } end local current = redis.call('HGET', st, 'blk') + if ARGV[4] == '1' and (not current or current == '') then + return { '${STALE}', '0', '0', '' } + end if current and current ~= '' and current ~= ARGV[1] then return { '${STALE}', tostring(redis.call('SCARD', pend)), '0', current } end @@ -977,7 +1076,16 @@ ${PERSIST_RUN_KEYS} return { '${ACKED}', tostring(redis.call('SCARD', pend)), '0', current } end - local fresh = redis.call('HSETNX', done, ARGV[2], ARGV[3]) + local receipts, order = done, KEYS[6] + if redis.call('HEXISTS', KEYS[5], ARGV[2]) == 0 then receipts, order = KEYS[7], KEYS[8] end + -- Initialize a fresh epoch after absence/expiry, but ongoing deliveries do not invalidate a + -- read. Its captured seq reconstructs pending-at-start even after SREM below. + redis.call('HSETNX', st, 'rev', ARGV[5]) + local fresh = redis.call('HSETNX', receipts, ARGV[2], ARGV[3]) + if fresh == 1 then redis.call('ZADD', order, redis.call('HINCRBY', st, 'seq', 1), ARGV[2]) end + -- The snapshot result and active receipt commit together, after all block/terminal + -- fences. Immutable waitpoint IDs make replay and later cycles unable to rewrite a result. + redis.call('HSETNX', results, ARGV[2], redis.call('HGET', receipts, ARGV[2])) redis.call('SREM', pend, ARGV[2]) local remaining = redis.call('SCARD', pend) @@ -1025,7 +1133,7 @@ ${PERSIST_RUN_KEYS} return { '${STALE}', current } end - redis.call('HSET', st, 'hs', 'ack') + redis.call('HSET', st, 'hs', 'ack', 'rev', ARGV[2]) return { '${ACKNOWLEDGED}' } `, }); @@ -1044,7 +1152,7 @@ ${PERSIST_RUN_KEYS} * abandoned. State is expired, never deleted, so receipts stay readable for the window. */ redis.defineCommand("runTerminalCleanup", { - numberOfKeys: 4, + numberOfKeys: 10, lua: ` local st = KEYS[4] @@ -1056,7 +1164,8 @@ ${PERSIST_RUN_KEYS} -- A run with no coordination state at all is a no-op, not a tombstone. Writing one -- per terminal run would cost a key carrying a fortnight's TTL for every run in the -- system, and a run that never absorbed a blocker has no receipt to protect. - if redis.call('EXISTS', KEYS[1], KEYS[2], KEYS[3], KEYS[4]) == 0 then + if redis.call('EXISTS', KEYS[1], KEYS[2], KEYS[3], KEYS[4]) == 0 and + (ARGV[3] ~= '1' or redis.call('EXISTS', KEYS[5]) == 0) then return { '${ARMED}', '', 'absent' } end @@ -1065,47 +1174,62 @@ ${PERSIST_RUN_KEYS} return { '${RETAINED}', 'handoff-owed' } end - redis.call('HSET', st, 'term', '1', 'hs', '', 'termAt', ARGV[1]) + redis.call('HSET', st, 'term', '1', 'hs', '', 'termAt', ARGV[1], 'rev', ARGV[4]) for i = 1, 4 do redis.call('PEXPIRE', KEYS[i], ttl) end + for i = 6, 10 do redis.call('PEXPIRE', KEYS[i], ttl) end + -- Resume is not result consumption. Only the run's terminal lifecycle arms this TTL. + if ARGV[3] == '1' then redis.call('PEXPIRE', KEYS[5], ttl) end return { '${ARMED}', hs, 'present' } `, }); - // KEYS: pend, done, edge, st. + /** + * KEYS: edge, st, edgeIndex, doneOrder, earlyOrder. + * ARGV: start|page, structural revision, completion cutoff, edge offset, last early ordinal. + * A snapshot is (rev, seq): membership/deletion changes rev; fresh immutable receipts only advance + * seq. Pages therefore reconstruct statuses at start without restarting for ordinary fanout. + */ redis.defineCommand("runReadBlockState", { - numberOfKeys: 4, + numberOfKeys: 5, lua: ` - local pend, done, edge, st = KEYS[1], KEYS[2], KEYS[3], KEYS[4] - - local pendIds = redis.call('SMEMBERS', pend) - -- HKEYS, never HGETALL: the delivered set's values are completion envelopes with - -- inline outputs, and materializing those inside a single-threaded script would - -- block the shard. - local doneIds = redis.call('HKEYS', done) - local edges = redis.call('HGETALL', edge) - local state = redis.call('HMGET', st, 'blk', 'hs', 'term') - - local out = { - tostring(#pendIds), tostring(#doneIds), tostring(#edges), - state[1] or '', state[2] or '', state[3] or '' - } - for i = 1, #pendIds do out[#out + 1] = pendIds[i] end - for i = 1, #doneIds do out[#out + 1] = doneIds[i] end - for i = 1, #edges do out[#out + 1] = edges[i] end + local edge, st, edgeIndex, doneOrder, earlyOrder = unpack(KEYS) + local state = redis.call('HMGET', st, 'rev', 'seq', 'blk', 'hs', 'term') + local revision = state[1] or '' + local cutoff = tonumber(ARGV[3]) or 0 + if ARGV[1] == 'start' then cutoff = tonumber(state[2]) or 0 + elseif revision ~= ARGV[2] then return { '${STALE}' } end + local edgeOffset = tonumber(ARGV[4]) + local earlyAfter = ARGV[5] == '' and '-inf' or '(' .. ARGV[5] + local fields = redis.call('ZRANGE', edgeIndex, edgeOffset, edgeOffset + ${RUN_CLEANUP_PAGE_SIZE} - 1) + local earlyIds = redis.call('ZRANGEBYSCORE', earlyOrder, earlyAfter, cutoff, 'WITHSCORES', 'LIMIT', 0, ${RUN_CLEANUP_PAGE_SIZE}) + local out = { 'page', revision, tostring(cutoff), state[3] or '', state[4] or '', state[5] or '', tostring(#fields), tostring(#earlyIds / 2) } + for i = 1, #fields do + local field = fields[i] + local sep = string.find(field, '#[^#]*$') + local id = string.sub(field, 1, sep - 1) + local ordinal = tonumber(redis.call('ZSCORE', doneOrder, id)) + out[#out + 1] = field + out[#out + 1] = redis.call('HGET', edge, field) or '' + out[#out + 1] = ordinal and ordinal <= cutoff and '1' or '0' + end + for i = 1, #earlyIds do out[#out + 1] = earlyIds[i] end return out `, }); - // KEYS: pend, done, edge, st. ARGV: n, then n edge fields. n = 0 clears everything. + // KEYS: pend, done, edge, st, refs. ARGV: blockId, n, then n edge fields. + // n = 0 unlinks the full active partition, including co; the exact-block CAS still applies. + // Only callers that already own lifecycle/legacy cleanup may use full-clear. Retry cleanup uses + // retain-and-clear plus finalizeRetainedWithdrawal so its cross-store fence survives interruption. redis.defineCommand("runClear", { - numberOfKeys: 4, + numberOfKeys: 9, lua: ` local pend, done, edge, st = KEYS[1], KEYS[2], KEYS[3], KEYS[4] local n = tonumber(ARGV[2]) -- Guard before any write, same reasoning as runAbsorbBlockers. - if n == nil or #ARGV ~= 2 + n then + if n == nil or #ARGV ~= 3 + n or n > ${RUN_CLEANUP_PAGE_SIZE} then return redis.error_reply('runClear: arity mismatch') end if ARGV[1] == '' then @@ -1127,43 +1251,289 @@ ${PERSIST_RUN_KEYS} end if n == 0 then - redis.call('DEL', pend, done, edge, st) + redis.call('UNLINK', unpack(KEYS)) return { '${CLEARED}', current } end + redis.call('HSET', st, 'rev', ARGV[#ARGV]) for i = 1, n do - redis.call('HDEL', edge, ARGV[2 + i]) + local field = ARGV[2 + i] + if redis.call('HDEL', edge, field) == 1 then + redis.call('HDEL', KEYS[5], '!' .. field) + redis.call('ZREM', KEYS[6], field) + local sep = string.find(field, '#[^#]*$') + if sep then + local id = string.sub(field, 1, sep - 1) + if redis.call('HINCRBY', KEYS[5], id, -1) <= 0 then + redis.call('HDEL', KEYS[5], id) + redis.call('HDEL', done, id) + redis.call('ZREM', KEYS[7], id) + redis.call('SREM', pend, id) + end + end + end end - -- Reconcile rather than delete by name. The edge set is the authority: after the - -- drain, pend and done may only hold ids that some surviving edge still references. - -- A superset of "remove the drained ids", because one waitpoint can hold several - -- edges at different batch indexes, and because a delivery that landed before its - -- blockers were absorbed leaves a receipt no name-derived drain could reach. - local remaining = {} - local fields = redis.call('HKEYS', edge) - for i = 1, #fields do - local sep = string.find(fields[i], '#[^#]*$') - if sep then - remaining[string.sub(fields[i], 1, sep - 1)] = true - end + -- Every unreferenced receipt is isolated here. Detach them all just as the old full-parent + -- reconciliation did, while references above preserve another batch index's receipt. + redis.call('UNLINK', KEYS[8], KEYS[9]) + + return { '${DRAINED}', current } + `, + }); + + /** + * Detach one exact block in constant work. RENAME transfers ownership of the complete edge hash, + * including after a lost reply, without copying/encoding its fields. The separate manifest keeps an + * empty withdrawal discoverable. Neither retained key expires; page acknowledgement owns reclamation. + * KEYS: pend, done, edge, st, wd, targets, refs. ARGV: blockId. + */ + redis.defineCommand("runAbandonRetainAndClear", { + numberOfKeys: 11, + lua: ` + local pend, done, edge, st, wd, targets, refs = unpack(KEYS) + if ARGV[1] == '' then return redis.error_reply('runAbandonRetainAndClear: missing block id') end + local current = redis.call('HGET', st, 'blk') or '' + if redis.call('HGET', st, 'term') == '1' then return { '${TERMINAL}', current } end + if current ~= ARGV[1] then return { '${STALE}', current } end + local retained = redis.call('GET', wd) + if retained and retained ~= ARGV[1] then + return redis.error_reply('runAbandonRetainAndClear: invalid withdrawal identity') + end + local hasEdge = redis.call('EXISTS', edge) == 1 + if hasEdge and redis.call('EXISTS', targets) == 1 then + return redis.error_reply('runAbandonRetainAndClear: active and retained targets conflict') + end + -- The manifest is the first durable write. If it cannot be recorded, nothing is detached. + redis.call('SET', wd, ARGV[1]) + if redis.call('EXISTS', edge) == 1 then redis.call('RENAME', edge, targets) end + redis.call('PERSIST', targets) + redis.call('UNLINK', pend, done, refs, KEYS[8], KEYS[9], KEYS[10], KEYS[11]) + redis.call('HDEL', st, 'blk', 'hs') + redis.call('HSET', st, 'rev', ARGV[2]) + return { '${RETAINED}' } + `, + }); + + /** KEYS: pend, done, edge, st, wd, results, targets, refs. ARGV: blockId, nowMs, ttlMs. */ + redis.defineCommand("runTerminalRetainAndClear", { + numberOfKeys: 12, + lua: ` + local pend, done, edge, st, wd, results, targets, refs = unpack(KEYS) + if ARGV[1] == '' then return redis.error_reply('runTerminalRetainAndClear: missing block id') end + local ttl = tonumber(ARGV[3]) + if ttl == nil or ttl <= 0 then return redis.error_reply('runTerminalRetainAndClear: invalid TTL') end + local current = redis.call('HGET', st, 'blk') or '' + if redis.call('HGET', st, 'term') == '1' then return { '${TERMINAL}', current } end + if current ~= '' and current ~= ARGV[1] then return { '${STALE}', current } end + local retained = redis.call('GET', wd) + if retained and retained ~= ARGV[1] then + return redis.error_reply('runTerminalRetainAndClear: invalid withdrawal identity') + end + local hasEdge = redis.call('EXISTS', edge) == 1 + if current == ARGV[1] and hasEdge and redis.call('EXISTS', targets) == 1 then + return redis.error_reply('runTerminalRetainAndClear: active and retained targets conflict') + end + local owesCleanup = current == ARGV[1] or retained or redis.call('HGET', st, 'co') == ARGV[1] + if current == '' and not owesCleanup and redis.call('EXISTS', results) == 0 then + return { '${ABSENT}' } end + if current == ARGV[1] then + -- Publish recovery evidence before detaching the active hash. + redis.call('SET', wd, ARGV[1]) + if hasEdge then redis.call('RENAME', edge, targets) end + redis.call('PERSIST', targets) + end + redis.call('UNLINK', pend, done, edge, refs, KEYS[9], KEYS[10], KEYS[11], KEYS[12]) + redis.call('HSET', st, 'term', '1', 'hs', '', 'termAt', ARGV[2], 'rev', ARGV[4]) + redis.call('HDEL', st, 'blk') + redis.call('PEXPIRE', st, ttl) + redis.call('PEXPIRE', results, ttl) + if owesCleanup then return { '${RETAINED}' } end + return { '${ABSENT}' } + `, + }); - local doneIds = redis.call('HKEYS', done) - for i = 1, #doneIds do - if not remaining[doneIds[i]] then - redis.call('HDEL', done, doneIds[i]) - end + /** Read without consuming; ack only after the cross-shard watcher withdrawals converge. */ + redis.defineCommand("runWithdrawalPage", { + numberOfKeys: 2, + lua: ` + local wd, targets = KEYS[1], KEYS[2] + if ARGV[1] == '' then return redis.error_reply('runWithdrawalPage: missing block id') end + local identity = redis.call('GET', wd) + if not identity then return { '${ABSENT}' } end + if identity ~= ARGV[1] then return redis.error_reply('runWithdrawalPage: invalid block identity') end + if ARGV[2] == 'read' then + local fields = redis.call('HRANDFIELD', targets, ${RUN_CLEANUP_PAGE_SIZE}) + local out = { '${RETAINED}' } + for i = 1, #fields do out[#out + 1] = fields[i] end + return out + end + if ARGV[2] ~= 'ack' or #ARGV - 2 > ${RUN_CLEANUP_PAGE_SIZE} then + return redis.error_reply('runWithdrawalPage: invalid acknowledgement') end + for i = 3, #ARGV do redis.call('HDEL', targets, ARGV[i]) end + return { '${ACKNOWLEDGED}' } + `, + }); - local pendIds = redis.call('SMEMBERS', pend) - for i = 1, #pendIds do - if not remaining[pendIds[i]] then - redis.call('SREM', pend, pendIds[i]) - end + /** + * KEYS: st. ARGV: blockId. + * + * Durably mark this run's exact current store block as cleanup-owed, so `runAbsorbBlockers` refuses a + * DIFFERENT block until the fence is cleared. Set BEFORE the retry/requeue transition. A terminal run + * (already fenced by `term`) is a no-op — its own fence already refuses every absorb. + */ + redis.defineCommand("runArmCleanupFence", { + numberOfKeys: 1, + lua: ` + local st = KEYS[1] + if ARGV[1] == '' then + return redis.error_reply('runArmCleanupFence: ARGV[1] must be the block id') end + if redis.call('HGET', st, 'term') == '1' then + return { '${TERMINAL}' } + end + redis.call('HSET', st, 'co', ARGV[1]) + return { '${ARMED}' } + `, + }); - return { '${DRAINED}', current } + /** + * KEYS: st. ARGV: blockId. + * + * Remove the cleanup-owed fence, but ONLY when it names this exact block — so a stale clear can never + * lift a newer block's owed fence. Called as the LAST step of a block's cleanup, after both its legacy + * delete and its store retain-and-clear/withdrawal have converged. + */ + redis.defineCommand("runClearCleanupFence", { + numberOfKeys: 1, + lua: ` + local st = KEYS[1] + if ARGV[1] == '' then + return redis.error_reply('runClearCleanupFence: ARGV[1] must be the block id') + end + if redis.call('HGET', st, 'co') == ARGV[1] then + redis.call('HDEL', st, 'co') + return { '${CLEARED}' } + end + return { '${ABSENT}' } + `, + }); + + /** + * KEYS: wd (the exact block's withdrawal manifest), st, targets. ARGV: blockId. + * + * ATOMICALLY retire a converged block's recovery evidence: delete its retained-withdrawal record AND + * clear its cleanup-owed fence, in one same-slot step, so a process death can never leave the fence set + * with no durable record to recover from (which would reject every later block forever). Both keys share + * the run's `{runId}` hash slot. Idempotent: a missing record and/or an already-cleared or differently- + * owned fence are all fine. The fence is cleared ONLY when it names THIS block — a newer block's `co` is + * never touched. Refuses while any unacknowledged target remains. + */ + redis.defineCommand("runFinalizeRetainedWithdrawal", { + numberOfKeys: 3, + lua: ` + local wd, st = KEYS[1], KEYS[2] + if ARGV[1] == '' then + return redis.error_reply('runFinalizeRetainedWithdrawal: ARGV[1] must be the block id') + end + local identity = redis.call('GET', wd) + if identity and identity ~= ARGV[1] then + return redis.error_reply('runFinalizeRetainedWithdrawal: invalid block identity') + end + if redis.call('HLEN', KEYS[3]) ~= 0 then + return { '${RETAINED}' } + end + local cleanupOwed = redis.call('HGET', st, 'co') + redis.call('UNLINK', wd, KEYS[3]) + if cleanupOwed == ARGV[1] then + redis.call('HDEL', st, 'co') + return { '${CLEARED}' } + end + return { '${ABSENT}' } + `, + }); + + /** + * Retire successful registration bookkeeping and its discovery entry in one same-slot commit. + */ + redis.defineCommand("runDeleteContribution", { + numberOfKeys: 2, + lua: ` + redis.call('UNLINK', KEYS[1]) + redis.call('ZREM', KEYS[2], ARGV[1]) + return { '${CLEARED}' } + `, + }); + + /** + * Compensate at most one page, preserving unrelated edges and duplicate waitpoint references. + * SPOP progress, edge removal and final index retirement commit together, including lost replies. + * KEYS: pend, done, edge, st, contribution, refs, edgeIndex, doneOrder, early, earlyOrder, index. + * ARGV: blockId, absorptionId, indexMember, revision. + */ + redis.defineCommand("runCompensateAbsorption", { + numberOfKeys: 11, + lua: ` + local pend, done, edge, st, contrib, refs = unpack(KEYS) + if ARGV[1] == '' or ARGV[2] == '' then + return redis.error_reply('runCompensateAbsorption: missing identity') + end + local current = redis.call('HGET', st, 'blk') or '' + if current ~= ARGV[1] or redis.call('HGET', st, 'term') == '1' then + redis.call('UNLINK', contrib) + redis.call('ZREM', KEYS[11], ARGV[3]) + return { '${STALE}', '0', '0', '0' } + end + -- Registered operations owe tail recovery, never compensation of accepted edges. + if redis.call('SISMEMBER', contrib, '!registered') == 1 then + return { '${RETAINED}', tostring(redis.call('HLEN', edge)), '0', '0' } + end + redis.call('HSET', st, 'rev', ARGV[4]) + redis.call('SREM', contrib, '!accepted') + local removed = redis.call('SPOP', contrib, ${RUN_CLEANUP_PAGE_SIZE}) + local removedFields = 0 + for i = 1, #removed do + local field = removed[i] + if string.sub(field, 1, 1) ~= '!' then removedFields = removedFields + 1 end + local releaseEdge = false + if string.sub(field, 1, 1) ~= '!' and redis.call('HEXISTS', edge, field) == 1 then + local claimField = '!' .. field + local claims = tonumber(redis.call('HGET', refs, claimField)) or 1 + if claims > 1 then + redis.call('HINCRBY', refs, claimField, -1) + elseif claims ~= -1 then + redis.call('HDEL', refs, claimField) + releaseEdge = true + end + end + if releaseEdge and redis.call('HDEL', edge, field) == 1 then + redis.call('ZREM', KEYS[7], field) + local sep = string.find(field, '#[^#]*$') + if sep then + local id = string.sub(field, 1, sep - 1) + if redis.call('HINCRBY', refs, id, -1) <= 0 then + redis.call('HDEL', refs, id) + redis.call('HDEL', done, id) + redis.call('ZREM', KEYS[8], id) + redis.call('SREM', pend, id) + end + end + end + end + local surviving = redis.call('HLEN', edge) + local remaining = redis.call('SCARD', contrib) + if remaining == 0 then redis.call('ZREM', KEYS[11], ARGV[3]) end + local outcome = '${RETAINED}' + if surviving == 0 and remaining == 0 then + redis.call('UNLINK', pend, done, edge, refs, KEYS[7], KEYS[8], KEYS[9], KEYS[10]) + if redis.call('HGET', st, 'co') ~= ARGV[1] then + redis.call('HDEL', st, 'blk', 'hs') + outcome = '${CLEARED}' + end + end + return { outcome, tostring(surviving), tostring(removedFields), tostring(remaining) } `, }); } @@ -1176,6 +1546,7 @@ declare module "@internal/redis" { status: string, completionJson: string, terminalTtlMs: string, + completedByTaskRunId: string, callback?: Callback ): Result; wpRegisterOrReport( @@ -1271,20 +1642,38 @@ declare module "@internal/redis" { doneKey: string, edgeKey: string, stateKey: string, + contributionKey: string, + resultsKey: string, + refsKey: string, + edgeIndexKey: string, + doneOrderKey: string, + earlyKey: string, + earlyOrderKey: string, + predecessorWithdrawalKey: string, + predecessorTargetsKey: string, + contributionIndexKey: string, ...args: Array> ): Result; runDeliverCompletion( pendKey: string, doneKey: string, stateKey: string, + resultsKey: string, + refsKey: string, + doneOrderKey: string, + earlyKey: string, + earlyOrderKey: string, blockId: string, waitpointId: string, completionJson: string, + requireCurrentBlock: string, + revision: string, callback?: Callback ): Result; runMarkHandoffAcked( stateKey: string, blockId: string, + revision: string, callback?: Callback ): Result; runTerminalCleanup( @@ -1292,24 +1681,124 @@ declare module "@internal/redis" { doneKey: string, edgeKey: string, stateKey: string, + resultsKey: string, + refsKey: string, + edgeIndexKey: string, + doneOrderKey: string, + earlyKey: string, + earlyOrderKey: string, nowMs: string, terminalTtlMs: string, voidHandoff: string, + revision: string, callback?: Callback ): Result; runReadBlockState( + edgeKey: string, + stateKey: string, + edgeIndexKey: string, + doneOrderKey: string, + earlyOrderKey: string, + mode: string, + revision: string, + cutoff: string, + edgeOffset: string, + earlyAfter: string, + callback?: Callback + ): Result; + runClear( pendKey: string, doneKey: string, edgeKey: string, stateKey: string, + refsKey: string, + edgeIndexKey: string, + doneOrderKey: string, + earlyKey: string, + earlyOrderKey: string, + ...args: Array> + ): Result; + runAbandonRetainAndClear( + pendKey: string, + doneKey: string, + edgeKey: string, + stateKey: string, + withdrawalKey: string, + targetsKey: string, + refsKey: string, + edgeIndexKey: string, + doneOrderKey: string, + earlyKey: string, + earlyOrderKey: string, + expectedBlockId: string, + revision: string, callback?: Callback ): Result; - runClear( + runTerminalRetainAndClear( pendKey: string, doneKey: string, edgeKey: string, stateKey: string, + withdrawalKey: string, + resultsKey: string, + targetsKey: string, + refsKey: string, + edgeIndexKey: string, + doneOrderKey: string, + earlyKey: string, + earlyOrderKey: string, + expectedBlockId: string, + nowMs: string, + terminalTtlMs: string, + revision: string, + callback?: Callback + ): Result; + runReadCleanupFence(stateKey: string, callback?: Callback): Result; + runArmCleanupFence( + stateKey: string, + blockId: string, + callback?: Callback + ): Result; + runClearCleanupFence( + stateKey: string, + blockId: string, + callback?: Callback + ): Result; + runFinalizeRetainedWithdrawal( + withdrawalKey: string, + stateKey: string, + targetsKey: string, + blockId: string, + callback?: Callback + ): Result; + runWithdrawalPage( + withdrawalKey: string, + targetsKey: string, ...args: Array> ): Result; + runDeleteContribution( + contributionKey: string, + contributionIndexKey: string, + member: string, + callback?: Callback + ): Result; + runCompensateAbsorption( + pendKey: string, + doneKey: string, + edgeKey: string, + stateKey: string, + contributionKey: string, + refsKey: string, + edgeIndexKey: string, + doneOrderKey: string, + earlyKey: string, + earlyOrderKey: string, + contributionIndexKey: string, + blockId: string, + absorptionId: string, + member: string, + revision: string, + callback?: Callback + ): Result; } } diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.handoff.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.handoff.test.ts new file mode 100644 index 00000000000..9ee5a4b5725 --- /dev/null +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.handoff.test.ts @@ -0,0 +1,212 @@ +import { createRedisClient } from "@internal/redis"; +import { redisTest } from "@internal/testcontainers"; +import { describe, expect } from "vitest"; +import { + runBlockKeys, + runContributionKey, + runContributionIndexKey, + contributionMember, + registrationEvidence, +} from "./keys.js"; +import { WaitpointStoreCoordinator } from "./storeCoordinator.js"; + +describe("fanout handoff reads", () => { + redisTest( + "fanout handoff read is independent of blocker collections and preserves failure semantics", + async ({ redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions }); + const redis = createRedisClient(redisOptions); + const keys = runBlockKeys("handoff-read-run"); + try { + expect(await store.readBlockHandoff("handoff-read-run")).toBe("none"); + // These unrelated collections are intentionally unreadable. The fixed-size handoff read + // must not touch them, regardless of their cardinality or shape. + await redis.set(keys.pend, "wrong-type"); + await redis.set(keys.done, "wrong-type"); + await redis.set(keys.edge, "wrong-type"); + await redis.hset(keys.state, "hs", "owed"); + expect(await store.readBlockHandoff("handoff-read-run")).toBe("owed"); + await redis.hset(keys.state, "blk", "current-block"); + expect(await store.readBlockHandoff("handoff-read-run", "current-block")).toBe("owed"); + expect(await store.readBlockHandoff("handoff-read-run", "old-block")).toBe("none"); + await redis.hset(keys.state, "term", "1"); + expect(await store.readBlockHandoff("handoff-read-run", "current-block")).toBe("none"); + await redis.hdel(keys.state, "term"); + await redis.hset(keys.state, "hs", "ack"); + expect(await store.readBlockHandoff("handoff-read-run")).toBe("acked"); + await redis.hset(keys.state, "hs", "unknown"); + expect(await store.readBlockHandoff("handoff-read-run")).toBe("none"); + await redis.del(keys.state); + await redis.set(keys.state, "wrong-type"); + await expect(store.readBlockHandoff("handoff-read-run")).rejects.toThrow("WRONGTYPE"); + } finally { + await store.quit(); + await redis.quit(); + } + } + ); +}); + +describe("contribution discovery", () => { + redisTest( + "an operation-free registration keeps its claim until ordinary block cleanup", + async ({ redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions }); + const redis = createRedisClient(redisOptions); + const args = { + runId: "shared-unowned-run", + blockId: "B1", + edges: [{ waitpointId: "W1", type: "RUN" as const, createdAt: new Date().toISOString() }], + }; + try { + await store.absorbBlockers({ ...args, absorptionId: "A1" }); + await store.absorbBlockers(args); + await store.absorbBlockers({ ...args, absorptionId: "A2" }); + for (const absorptionId of ["A1", "A2"]) { + await store.compensateAbsorption({ ...args, absorptionId }); + expect((await store.readBlockState(args.runId)).pendingIds).toEqual(["W1"]); + } + await store.clearBlockState({ runId: args.runId, blockId: args.blockId }); + expect((await store.readBlockState(args.runId)).edges).toEqual([]); + expect(await redis.exists(runBlockKeys(args.runId).refs)).toBe(0); + } finally { + await redis.quit(); + await store.quit(); + } + } + ); + + redisTest( + "recovery requires this block's contribution and the complete logical request", + async ({ redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions }); + const runId = "recovery-evidence-run"; + const operation = { runId, blockId: "B1", absorptionId: "A1", waitpointIds: ["W1"] }; + const edge = (waitpointId: string, batchIndex?: number) => ({ + waitpointId, + batchIndex, + type: "RUN" as const, + createdAt: new Date().toISOString(), + }); + try { + expect(await store.hasAbsorbedBlockers(operation)).toBe(false); + await store.absorbBlockers({ ...operation, edges: [edge("W1")] }); + // Acceptance is scoped to the original complete request, never a superset. + expect(await store.hasAbsorbedBlockers({ ...operation, waitpointIds: ["W1", "W2"] })).toBe( + false + ); + expect(await store.hasAbsorbedBlockers({ ...operation, blockId: "old" })).toBe(false); + expect( + await store.hasAbsorbedBlockers({ ...operation, absorptionId: "never-accepted" }) + ).toBe(false); + await store.markRegistrationComplete({ + ...operation, + evidence: registrationEvidence(operation.waitpointIds), + }); + expect(await store.hasAbsorbedBlockers(operation)).toBe(true); + await store.compensateAbsorption(operation); + expect(await store.hasAbsorbedBlockers(operation)).toBe(true); + expect(await store.hasAbsorbedBlockers({ ...operation, waitpointIds: ["W1", "W2"] })).toBe( + false + ); + expect(await store.hasAbsorbedBlockers({ ...operation, batchIndex: 0 })).toBe(false); + await store.absorbBlockers({ + ...operation, + absorptionId: "duplicate", + edges: [edge("W1")], + }); + expect(await store.readContribution({ ...operation, absorptionId: "duplicate" })).toEqual([ + "W1#", + ]); + expect(await store.hasAbsorbedBlockers({ ...operation, absorptionId: "duplicate" })).toBe( + true + ); + await store.absorbBlockers({ + ...operation, + absorptionId: "indexed", + edges: [edge("W1", 0)], + }); + await store.markRegistrationComplete({ + ...operation, + absorptionId: "indexed", + evidence: registrationEvidence(["W1"], 0), + }); + expect( + await store.hasAbsorbedBlockers({ ...operation, absorptionId: "indexed", batchIndex: 0 }) + ).toBe(true); + await store.clearBlockState({ runId, blockId: "B1" }); + expect(await store.hasAbsorbedBlockers(operation)).toBe(false); + } finally { + await store.quit(); + } + } + ); + + redisTest( + "indexes only outstanding operations and retires them atomically without touching another generation", + async ({ redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions }); + const redis = createRedisClient(redisOptions); + const runId = "indexed-run"; + const index = runContributionIndexKey(runId); + const edge = (waitpointId: string) => ({ + waitpointId, + type: "RUN" as const, + createdAt: new Date().toISOString(), + }); + try { + await store.absorbBlockers({ + runId, + blockId: "B1", + absorptionId: "A1", + edges: [edge("W1")], + }); + await store.absorbBlockers({ + runId, + blockId: "B1", + absorptionId: "A2", + edges: [edge("W2")], + }); + await store.absorbBlockers({ + runId, + blockId: "B1", + absorptionId: "A3", + edges: [edge("W1")], + }); + expect(await redis.zrange(index, 0, 9)).toEqual([ + contributionMember("B1", "A1"), + contributionMember("B1", "A2"), + contributionMember("B1", "A3"), + ]); + expect(await redis.pttl(index)).toBe(-1); + await store.deleteContribution({ runId, blockId: "B1", absorptionId: "A1" }); + expect(await redis.exists(runContributionKey(runId, "B1", "A1"))).toBe(0); + await store.compensateAbsorption({ runId, blockId: "B1", absorptionId: "A2" }); + await store.deleteContribution({ runId, blockId: "B1", absorptionId: "A3" }); + expect(await redis.exists(index)).toBe(0); + expect((await store.readBlockState(runId)).edges.map((e) => e.waitpointId)).toEqual(["W1"]); + await store.absorbBlockers({ + runId, + blockId: "B1", + absorptionId: "old", + edges: [edge("old")], + }); + await store.absorbBlockers({ + runId, + blockId: "B2", + expectedPreviousBlockId: "B1", + absorptionId: "new", + edges: [edge("new")], + }); + await store.compensateAbsorption({ runId, blockId: "B1", absorptionId: "old" }); + expect(await redis.zrange(index, 0, 9)).toEqual([contributionMember("B2", "new")]); + expect((await store.readBlockState(runId)).edges.map((e) => e.waitpointId)).toEqual([ + "new", + ]); + } finally { + await store.quit(); + await redis.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.parentBounds.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.parentBounds.test.ts new file mode 100644 index 00000000000..35f26e13ed6 --- /dev/null +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.parentBounds.test.ts @@ -0,0 +1,355 @@ +import { createRedisClient } from "@internal/redis"; +import { redisTest } from "@internal/testcontainers"; +import { describe, expect } from "vitest"; +import { + runBlockKeys, + runResultKey, + runContributionIndexKey, + contributionMember, + runWithdrawalKey, + runWithdrawalTargetsKey, + waitpointKeys, + watcherField, +} from "./keys.js"; +import { registerWaitpointCommands, RUN_CLEANUP_PAGE_SIZE } from "./scripts.js"; +import { + WaitpointStoreCoordinator, + type BlockEdge, + type WaitpointCompletion, + type WaitpointRecordInput, +} from "./storeCoordinator.js"; + +const PAGE = RUN_CLEANUP_PAGE_SIZE; +const COUNT = PAGE * 3 + 7; +const edge = (waitpointId: string, batchIndex?: number): BlockEdge => ({ + waitpointId, + type: "RUN", + batchIndex, + createdAt: "2026-10-20T12:00:00.000Z", +}); +const completion: WaitpointCompletion = { + completedAt: "2026-10-20T12:00:00.000Z", + outputType: "application/json", + outputIsError: false, + output: { inline: "first" }, +}; +const record = (id: string): WaitpointRecordInput => ({ + id, + friendlyId: id, + type: "RUN", + environmentId: "env", + projectId: "project", + createdAt: completion.completedAt, + updatedAt: completion.completedAt, + userProvidedIdempotencyKey: false, + tags: [], +}); +const calls = (stats: string, command: string) => + Number( + stats + .split("\n") + .find((line) => line.startsWith(`cmdstat_${command}:`)) + ?.split("calls=")[1] + ?.split(",")[0] ?? 0 + ); + +// Cumulative blocks are assembled by many small calls, as production children do. +async function add(store: WaitpointStoreCoordinator, runId: string, count = COUNT) { + for (let i = 0; i < count; i++) + await store.absorbBlockers({ runId, blockId: "B1", edges: [edge(`w_${i}`)] }); +} + +describe("bounded cumulative parent operations", () => { + redisTest( + "reads hard-bounded pages and selectively clears without enumerating surviving collections", + async ({ redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions }); + const redis = createRedisClient(redisOptions); + registerWaitpointCommands(redis); + const runId = "large-parent", + keys = runBlockKeys(runId); + try { + await add(store, runId); + await store.absorbBlockers({ runId, blockId: "B1", edges: [edge("w_0", 1)] }); + await store.deliverCompletion({ runId, blockId: "B1", waitpointId: "w_0", completion }); + for (let i = 0; i < COUNT; i++) + await store.deliverCompletion({ + runId, + blockId: "B1", + waitpointId: `early_${i}`, + completion, + }); + const before = await redis.info("commandstats"); + let offset = 0, + earlyOffset = 0, + earlyAfter = "", + revision = "", + cutoff = "0", + pages = 0; + for (let visits = 0; visits < 10; visits++) { + const reply = await redis.runReadBlockState( + keys.edge, + keys.state, + keys.edgeIndex, + keys.doneOrder, + keys.earlyOrder, + pages ? "page" : "start", + revision, + cutoff, + String(offset), + earlyAfter + ); + expect(reply[0]).toBe("page"); + revision = reply[1]!; + cutoff = reply[2]!; + const edgeCount = Number(reply[6]), + earlyCount = Number(reply[7]); + expect(edgeCount).toBeLessThanOrEqual(PAGE); + expect(earlyCount).toBeLessThanOrEqual(PAGE); + expect(reply).toHaveLength(8 + edgeCount * 3 + earlyCount * 2); + offset += edgeCount; + earlyOffset += earlyCount; + pages++; + if (earlyCount) earlyAfter = reply[reply.length - 1]!; + if (edgeCount < PAGE && earlyCount < PAGE) break; + } + expect(pages).toBe(4); + expect(offset).toBe(COUNT + 1); + expect(earlyOffset).toBe(COUNT); + const state = await store.readBlockState(runId); + expect(state.edges).toHaveLength(COUNT + 1); + expect(state.pendingIds).toHaveLength(COUNT - 1); + expect(state.deliveredIds).toHaveLength(COUNT + 1); + await store.clearBlockState({ runId, blockId: "B1", edgeIds: ["w_0#"] }); + expect((await store.readBlockState(runId)).deliveredIds).toEqual(["w_0"]); + expect(await redis.exists(keys.early, keys.earlyOrder)).toBe(0); + // Large selective requests are split; each atomic command touches <=128 requested edges. + await store.clearBlockState({ + runId, + blockId: "B1", + edgeIds: Array.from({ length: COUNT - 1 }, (_, i) => `w_${i + 1}#`), + }); + const remaining = await store.readBlockState(runId); + expect(remaining.edges.map((item) => item.edgeId)).toEqual(["w_0#1"]); + expect(remaining.deliveredIds).toEqual(["w_0"]); + await store.clearBlockState({ runId, blockId: "B1", edgeIds: ["w_0#1"] }); + expect((await store.readBlockState(runId)).deliveredIds).toEqual([]); + expect(await redis.hget(runResultKey(runId), "w_0")).toBe(JSON.stringify(completion)); + const after = await redis.info("commandstats"); + for (const command of ["hkeys", "hgetall", "smembers", "scan", "hscan", "sscan"]) + expect(calls(after, command)).toBe(calls(before, command)); + } finally { + await store.quit(); + await redis.quit(); + } + } + ); + + redisTest( + "returns the read-start snapshot while completions land before every later page", + async ({ redisOptions }) => { + const writer = new WaitpointStoreCoordinator({ redisOptions }); + let pages = 0; + const reader = new WaitpointStoreCoordinator({ + redisOptions, + beforeCommand: async () => { + pages++; + if (pages === 1) return; + // A completes after its page; B before its page. Mixing current statuses yields no snapshot. + const start = (pages - 2) * PAGE; + for (let i = start; i < Math.min(COUNT, start + PAGE * 2); i++) + await writer.deliverCompletion({ + runId: "arrival", + blockId: "B1", + waitpointId: `w_${i}`, + completion, + }); + await writer.deliverCompletion({ + runId: "arrival", + blockId: "B1", + waitpointId: `later_${pages}`, + completion, + }); + }, + }); + try { + await add(writer, "arrival"); + await writer.deliverCompletion({ + runId: "arrival", + blockId: "B1", + waitpointId: "w_0", + completion, + }); + const snapshot = await reader.readBlockState("arrival"); + expect(pages).toBe(4); + expect(snapshot.edges).toHaveLength(COUNT); + expect(snapshot.deliveredIds).toEqual(["w_0"]); + expect(new Set(snapshot.pendingIds)).toEqual( + new Set(Array.from({ length: COUNT - 1 }, (_, i) => `w_${i + 1}`)) + ); + expect(snapshot.handoff).toBe("none"); + const now = await writer.readBlockState("arrival"); + expect(now.pendingIds).toEqual([]); + expect(now.handoff).toBe("owed"); + } finally { + await reader.quit(); + await writer.quit(); + } + } + ); + + redisTest( + "restarts on structural changes, including clearing and recreating the same block identity", + async ({ redisOptions }) => { + const writer = new WaitpointStoreCoordinator({ redisOptions }); + let pages = 0; + const reader = new WaitpointStoreCoordinator({ + redisOptions, + beforeCommand: async () => { + expect(++pages, "structural retry did not converge").toBeLessThanOrEqual(4); + if (pages !== 2) return; + await writer.clearBlockState({ runId: "recreated", blockId: "B1" }); + await writer.absorbBlockers({ + runId: "recreated", + blockId: "B1", + edges: [edge("replacement")], + }); + await writer.deliverCompletion({ + runId: "recreated", + blockId: "B1", + waitpointId: "replacement", + completion, + }); + }, + }); + try { + await add(writer, "recreated"); + const snapshot = await reader.readBlockState("recreated"); + expect(pages).toBe(3); + expect(snapshot.edges.map((item) => item.waitpointId)).toEqual(["replacement"]); + expect(snapshot.pendingIds).toEqual([]); + expect(snapshot.deliveredIds).toEqual(["replacement"]); + expect(snapshot.handoff).toBe("owed"); + } finally { + await reader.quit(); + await writer.quit(); + } + } + ); + + redisTest( + "preserves first early receipt, empty-absorption handoff, and terminal retention of every index", + async ({ redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions }); + const redis = createRedisClient(redisOptions); + const runId = "early", + keys = runBlockKeys(runId); + try { + await store.deliverCompletion({ runId, blockId: "B1", waitpointId: "a", completion }); + const ordinal = await redis.zscore(keys.earlyOrder, "a"); + await store.deliverCompletion({ + runId, + blockId: "B1", + waitpointId: "a", + completion: { ...completion, output: { inline: "duplicate" } }, + }); + expect(await redis.zscore(keys.earlyOrder, "a")).toBe(ordinal); + await store.absorbBlockers({ runId, blockId: "B1", edges: [] }); + expect((await store.readBlockState(runId)).handoff).toBe("owed"); + const adopted = await store.absorbBlockers({ + runId, + blockId: "B1", + edges: [ + { + ...edge("a"), + reported: { completion: { ...completion, output: { inline: "reported" } } }, + }, + ], + }); + expect(adopted.alreadyDelivered).toEqual([{ waitpointId: "a", completion }]); + expect(await redis.zscore(keys.doneOrder, "a")).toBe(ordinal); + expect(await redis.zscore(keys.earlyOrder, "a")).toBeNull(); + await store.deliverCompletion({ + runId, + blockId: "B1", + waitpointId: "unreferenced", + completion, + }); + await store.cleanupRunBlockState({ runId, reason: "terminal" }); + for (const key of [keys.state, keys.edgeIndex, keys.doneOrder, keys.early, keys.earlyOrder]) + expect(await redis.pttl(key)).toBeGreaterThan(0); + } finally { + await store.quit(); + await redis.quit(); + } + } + ); + + redisTest( + "a lost rollover reply remains recoverable after a newer block and stale caller retry", + async ({ redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions }); + const redis = createRedisClient(redisOptions); + registerWaitpointCommands(redis); + const runId = "rollover", + keys = runBlockKeys(runId); + try { + await store.createIfAbsent({ record: record("shared"), status: "PENDING" }); + for (let i = 0; i < COUNT; i++) + await store.registerBlocks({ runId, blockId: "B1", edges: [edge("shared", i)] }); + const before = await redis.info("commandstats"); + // Execute the real commit but omit its JS continuation: model lost reply/process restart. + const rolled = await redis.runAbsorbBlockers( + keys.pend, + keys.done, + keys.edge, + keys.state, + "wp:v1:run:{rollover}:unused", + runResultKey(runId), + keys.refs, + keys.edgeIndex, + keys.doneOrder, + keys.early, + keys.earlyOrder, + runWithdrawalKey(runId, "B1"), + runWithdrawalTargetsKey(runId, "B1"), + runContributionIndexKey(runId), + "B2", + "1", + "B1", + "0", + "", + contributionMember("B2", ""), + "lost-reply" + ); + expect(rolled).toEqual(["absorbed", "0", "0", "B1", "0"]); + expect(await redis.hlen(runWithdrawalTargetsKey(runId, "B1"))).toBe(COUNT); + await store.registerBlocks({ + runId, + blockId: "B3", + expectedPreviousBlockId: "B2", + edges: [edge("shared", 0)], + }); + const stale = await store.absorbBlockers({ + runId, + blockId: "B2", + expectedPreviousBlockId: "B1", + edges: [], + }); + expect(stale.outcome).toBe("stale"); + expect(await store.readWithdrawalPage({ runId, blockId: "B1" })).toBeUndefined(); + expect(await redis.hlen(waitpointKeys("shared").watchers)).toBe(1); + expect( + await redis.hexists(waitpointKeys("shared").watchers, watcherField(runId, "B3", 0)) + ).toBe(1); + expect((await store.readBlockState(runId)).blockId).toBe("B3"); + const after = await redis.info("commandstats"); + for (const command of ["hkeys", "hgetall", "smembers", "scan", "hscan", "sscan"]) + expect(calls(after, command)).toBe(calls(before, command)); + } finally { + await store.quit(); + await redis.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.parentResults.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.parentResults.test.ts new file mode 100644 index 00000000000..a64a30f130a --- /dev/null +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.parentResults.test.ts @@ -0,0 +1,106 @@ +import { createRedisClient } from "@internal/redis"; +import { redisTest } from "@internal/testcontainers"; +import { generateWaitpointId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect } from "vitest"; +import { runResultKey, waitpointKeys } from "./keys.js"; +import { StoreWaitpointCoordinator } from "./storeWaitpointCoordinator.js"; +import { WaitpointStoreCoordinator, type WaitpointRecordInput } from "./storeCoordinator.js"; + +describe("parent snapshot results", () => { + redisTest( + "snapshot results survive replay and newer blocks, then results-only terminal cleanup fences delivery", + async ({ redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions }); + const adapter = new StoreWaitpointCoordinator({ store }); + const redis = createRedisClient(redisOptions); + const now = new Date().toISOString(); + const completion = { + completedAt: now, + outputType: "application/json", + outputIsError: true, + output: { inline: '{"message":"original"}' }, + }; + const ids = [generateWaitpointId("RUN"), generateWaitpointId("RUN")]; + const records: WaitpointRecordInput[] = ids.map((id) => ({ + id, + friendlyId: `waitpoint_${id}`, + type: "RUN", + environmentId: "env", + projectId: "project", + createdAt: now, + updatedAt: now, + completedByTaskRunId: `child_${id}`, + userProvidedIdempotencyKey: false, + tags: [], + })); + try { + await store.createIfAbsent({ record: records[0]!, status: "COMPLETED", completion }); + await store.registerBlocks({ + runId: "parent", + blockId: "first", + edges: [{ waitpointId: ids[0]!, type: "RUN", createdAt: now }], + }); + await redis.del(waitpointKeys(ids[0]!).record); + await store.acknowledgeResumeHandoff({ runId: "parent", blockId: "first" }); + await store.clearBlockState({ runId: "parent", blockId: "first" }); + expect((await store.readCompletedRecords([ids[0]!], "parent")).get(ids[0]!)).toEqual({ + status: "COMPLETED", + record: records[0], + completion, + }); + await store.createIfAbsent({ record: records[1]!, status: "PENDING" }); + await store.registerBlocks({ + runId: "parent", + blockId: "second", + edges: [{ waitpointId: ids[1]!, type: "RUN", createdAt: now }], + }); + expect( + ( + await store.deliverCompletion({ + runId: "parent", + blockId: "first", + waitpointId: ids[0]!, + record: records[0], + completion: { ...completion, output: { inline: "wrong" } }, + }) + ).outcome + ).toBe("stale"); + expect( + (await store.terminalRetainAndClear({ runId: "parent", blockId: "first" })).outcome + ).toBe("stale"); + expect(await redis.pttl(runResultKey("parent"))).toBe(-1); + expect( + (await store.readCompletedRecords([ids[0]!], "parent")).get(ids[0]!)?.completion + ).toEqual(completion); + await store.complete({ waitpointId: ids[1]!, completion }); + await store.registerBlocks({ + runId: "parent", + blockId: "second", + edges: [{ waitpointId: ids[1]!, type: "RUN", createdAt: now }], + }); + await store.acknowledgeResumeHandoff({ runId: "parent", blockId: "second" }); + await store.clearBlockState({ runId: "parent", blockId: "second" }); + expect((await store.readCompletedRecords(ids, "parent")).size).toBe(2); + await adapter.releaseRunBlockers({ runId: "parent", reason: "terminal" }); + expect(await redis.pttl(runResultKey("parent"))).toBeGreaterThan(0); + expect( + ( + await store.deliverCompletion({ + runId: "parent", + blockId: "second", + waitpointId: ids[1]!, + record: records[1], + completion, + }) + ).outcome + ).toBe("terminal"); + const retained = await store.readCompletedRecords(ids, "parent"); + for (const rec of records) + expect(retained.get(rec.id)).toEqual({ status: "COMPLETED", record: rec, completion }); + } finally { + await redis.quit(); + await store.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.readCompletedRecords.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.readCompletedRecords.test.ts index 5dd8b7a09ec..d994e59f9f4 100644 --- a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.readCompletedRecords.test.ts +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.readCompletedRecords.test.ts @@ -6,6 +6,9 @@ import { redisTest } from "@internal/testcontainers"; import { type RedisOptions } from "@internal/redis"; import { generateWaitpointId } from "@trigger.dev/core/v3/isomorphic"; import { describe, expect } from "vitest"; +import { createRedisClient } from "@internal/redis"; +import { runResultKey, waitpointKeys } from "./keys.js"; +import { WaitpointFanoutWorker } from "./fanoutWorker.js"; import { WaitpointStoreCoordinator, type WaitpointCompletion, @@ -134,3 +137,65 @@ describe("WaitpointStoreCoordinator.readCompletedRecords", () => { } ); }); + +redisTest( + "parent snapshots retain atomic completed metadata through acknowledgement and terminal cleanup", + async ({ redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions }); + const redis = createRedisClient(redisOptions); + const worker = new WaitpointFanoutWorker({ coordinator: store, enabled: false }); + try { + const early = generateWaitpointId("DATETIME"); + const late = generateWaitpointId("RUN"); + const pending = generateWaitpointId("RUN"); + const records = [ + record(early, { type: "DATETIME", completedAfter: NOW }), + record(late, { type: "RUN", completedByTaskRunId: "run_child" }), + ]; + for (const rec of records) await store.createIfAbsent({ record: rec, status: "PENDING" }); + await store.complete({ waitpointId: early, completion: completion() }); + await store.createIfAbsent({ record: record(pending), status: "PENDING" }); + await store.registerBlocks({ + runId: "run_parent", + blockId: "block_parent", + edges: [early, late, pending].map((waitpointId) => ({ waitpointId, createdAt: NOW })), + }); + await store.complete({ waitpointId: late, completion: completion() }); + expect((await worker.visit(late)).outcome).toBe("drained"); + // Simulate expiry only after both the registration and fanout receipts are durable. + await redis.del(waitpointKeys(early).record, waitpointKeys(late).record); + expect((await store.readCompletedRecords([early, late])).size).toBe(0); + expect((await store.readBlockState("run_parent")).pendingIds).toEqual([pending]); + const retained = await store.readCompletedRecords([early, late], "run_parent"); + for (const rec of records) + expect(retained.get(rec.id)).toEqual({ + status: "COMPLETED", + record: rec, + completion: completion(), + }); + expect((await store.readCompletedRecords([early], "run_other")).size).toBe(0); + await store.clearBlockState({ runId: "run_parent", blockId: "block_parent" }); + expect((await store.readCompletedRecords([early, late], "run_parent")).size).toBe(2); + expect(await redis.pttl(runResultKey("run_parent"))).toBe(-1); + await store.cleanupRunBlockState({ runId: "run_parent", reason: "resume" }); + expect(await redis.pttl(runResultKey("run_parent"))).toBe(-1); + await store.releaseRunWatchers({ runId: "run_parent", reason: "terminal" }); + expect(await redis.pttl(runResultKey("run_parent"))).toBeGreaterThan(0); + expect( + ( + await store.deliverCompletion({ + runId: "run_parent", + blockId: "block_parent", + waitpointId: early, + completion: completion(), + record: records[0], + }) + ).outcome + ).toBe("terminal"); + } finally { + await worker.stop(); + await redis.quit(); + await store.quit(); + } + } +); diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.registrationReplay.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.registrationReplay.test.ts new file mode 100644 index 00000000000..c4bb133bff3 --- /dev/null +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.registrationReplay.test.ts @@ -0,0 +1,130 @@ +import { createRedisClient } from "@internal/redis"; +import { redisTest } from "@internal/testcontainers"; +import { generateWaitpointId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect } from "vitest"; +import { runBlockKeys, waitpointKeys } from "./keys.js"; +import { WaitpointStoreCoordinator, type WaitpointRecordInput } from "./storeCoordinator.js"; + +describe("registration replay", () => { + redisTest( + "registration replay consumes its fenced receipt after source expiry", + async ({ redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions }); + const redis = createRedisClient(redisOptions); + const id = generateWaitpointId("RUN"); + const now = new Date().toISOString(); + const record: WaitpointRecordInput = { + id, + friendlyId: `waitpoint_${id}`, + type: "RUN", + environmentId: "env", + projectId: "project", + createdAt: now, + updatedAt: now, + completedByTaskRunId: "child", + userProvidedIdempotencyKey: false, + tags: [], + }; + const args = { + runId: "parent", + blockId: "block", + absorptionId: "absorption", + edges: [ + { waitpointId: id, type: "RUN" as const, createdAt: now, spanIdToComplete: "span" }, + ], + }; + try { + await store.createIfAbsent({ + record, + status: "COMPLETED", + completion: { + completedAt: now, + outputType: "application/json", + outputIsError: true, + output: { inline: '{"message":"failed"}' }, + }, + }); + await store.registerBlocks(args); + expect((await store.readBlockState(args.runId)).handoff).toBe("owed"); + // Expire the source after receipt delivery but before the caller schedules continuation. + await redis.pexpire(waitpointKeys(id).record, 1); + await expect + .poll(() => redis.exists(waitpointKeys(id).record), { + timeout: 5_000, + interval: 20, + message: "completed source record never expired", + }) + .toBe(0); + expect((await store.readCompletedRecords([id], args.runId)).size).toBe(1); + const replay = await store.registerBlocks(args); + expect(replay.pendingOfRequested).toBe(0); + expect(replay.storePendingTotal).toBe(0); + expect(replay.cachedSpanCompletions).toEqual([ + { + spanId: "span", + spanCreatedAt: now, + blockedRunId: "parent", + hasError: true, + cachedRunId: "child", + }, + ]); + expect((await store.readBlockState(args.runId)).handoff).toBe("owed"); + await store.acknowledgeResumeHandoff({ runId: args.runId, blockId: args.blockId }); + await store.registerBlocks({ + ...args, + blockId: "next", + absorptionId: "next", + expectedPreviousBlockId: args.blockId, + edges: [], + }); + expect((await store.registerBlocks(args)).outcome).toBe("stale"); + } finally { + await redis.quit(); + await store.quit(); + } + } + ); + + redisTest( + "registration replay refuses corrupt metadata and cannot infer missing source metadata", + async ({ redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions }); + const redis = createRedisClient(redisOptions); + const id = generateWaitpointId("RUN"); + const now = new Date().toISOString(); + const args = { + runId: "parent", + blockId: "block", + edges: [{ waitpointId: id, type: "RUN" as const, createdAt: now }], + }; + const completion = { + completedAt: now, + outputType: "application/json", + outputIsError: false, + output: null, + }; + try { + await store.absorbBlockers(args); + // A metadata-less historical receipt still needs a live, valid source record. + await store.deliverCompletion({ + runId: args.runId, + blockId: args.blockId, + waitpointId: id, + completion, + }); + await expect(store.registerBlocks(args)).rejects.toThrow("is not present in the store"); + await redis.hset( + runBlockKeys(args.runId).done, + id, + JSON.stringify({ ...completion, record: { id: "wrong" } }) + ); + await expect(store.registerBlocks(args)).rejects.toThrow("Invalid retained completion"); + await redis.hset(runBlockKeys(args.runId).done, id, "{"); + await expect(store.registerBlocks(args)).rejects.toThrow(); + } finally { + await redis.quit(); + await store.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts index d415c6e80a0..8c566969362 100644 --- a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts @@ -11,11 +11,19 @@ import { describe, expect } from "vitest"; import { edgeField, idempotencyKey, + parseEdgeField, runBlockKeys, + runContributionKey, + runContributionIndexKey, + contributionMember, + runResultKey, + runWithdrawalKey, + runWithdrawalTargetsKey, + waitpointKeys, watcherField, WaitpointKeyTagError, } from "./keys.js"; -import { registerWaitpointCommands } from "./scripts.js"; +import { registerWaitpointCommands, RUN_CLEANUP_PAGE_SIZE } from "./scripts.js"; import { encodeCompletionForDelivery, EncodedWaitpointCompletion, @@ -347,10 +355,16 @@ describe("registerOrReport", () => { } }); - redisTest("keeps the first registration's watcher on a re-register", async ({ redisOptions }) => { + redisTest("fills a watcher's first span and preserves it on retry", async ({ redisOptions }) => { const store = coordinator(redisOptions); try { await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + await store.registerOrReport({ + blockId: BLOCK_ID, + waitpointId: "w_a", + runId: "run_1", + createdAt: "2026-01-01T00:00:00.000Z", + }); await store.registerOrReport({ blockId: BLOCK_ID, waitpointId: "w_a", @@ -358,20 +372,27 @@ describe("registerOrReport", () => { spanIdToComplete: "span_first", createdAt: NOW, }); - // Same run, same (absent) batch index, so the watcher field collides. HSETNX must - // not let this second registration overwrite the first one's span. + // Same watcher identity: neither a later span nor a bare recovery replay may replace + // the first actual span or its creation time, or add another queue entry. await store.registerOrReport({ blockId: BLOCK_ID, waitpointId: "w_a", runId: "run_1", spanIdToComplete: "span_second", - createdAt: NOW, + createdAt: "2026-02-01T00:00:00.000Z", + }); + await store.registerOrReport({ + blockId: BLOCK_ID, + waitpointId: "w_a", + runId: "run_1", + createdAt: "2026-03-01T00:00:00.000Z", }); await store.complete({ waitpointId: "w_a", completion: completion() }); const watchers = await queuedWatchers(store, "w_a"); expect(watchers).toHaveLength(1); expect(watchers[0]!.spanIdToComplete).toBe("span_first"); + expect(watchers[0]!.createdAt).toBe(NOW); } finally { await store.quit(); } @@ -584,6 +605,16 @@ describe("reply framing (direct Lua — pins the wire shape the coordinator deco keys.done, keys.edge, keys.state, + runContributionKey("run_1", "block_test", ""), + runResultKey("run_1"), + keys.refs, + keys.edgeIndex, + keys.doneOrder, + keys.early, + keys.earlyOrder, + runWithdrawalKey("run_1", ""), + runWithdrawalTargetsKey("run_1", ""), + runContributionIndexKey("run_1"), BLOCK_ID, "0", "", @@ -597,7 +628,10 @@ describe("reply framing (direct Lua — pins the wire shape the coordinator deco fieldB, "{}", "1", - envelope + envelope, + "", + contributionMember("block_test", ""), + "raw-test-revision" ); // Slots 2 and 3 are the superseded block id and its edge-field count, both empty here @@ -625,6 +659,16 @@ describe("reply framing (direct Lua — pins the wire shape the coordinator deco keys.done, keys.edge, keys.state, + runContributionKey("run_1", "block_test", ""), + runResultKey("run_1"), + keys.refs, + keys.edgeIndex, + keys.doneOrder, + keys.early, + keys.earlyOrder, + runWithdrawalKey("run_1", ""), + runWithdrawalTargetsKey("run_1", ""), + runContributionIndexKey("run_1"), BLOCK_ID, "0", "", @@ -638,7 +682,10 @@ describe("reply framing (direct Lua — pins the wire shape the coordinator deco fieldA, "{}", "0", - "" + "", + "", + contributionMember("block_test", ""), + "raw-test-revision" ); // Slots 2 and 3 are the superseded block id and its edge-field count, both empty here @@ -662,6 +709,16 @@ describe("reply framing (direct Lua — pins the wire shape the coordinator deco keys.done, keys.edge, keys.state, + runContributionKey("run_1", "block_test", ""), + runResultKey("run_1"), + keys.refs, + keys.edgeIndex, + keys.doneOrder, + keys.early, + keys.earlyOrder, + runWithdrawalKey("run_1", ""), + runWithdrawalTargetsKey("run_1", ""), + runContributionIndexKey("run_1"), BLOCK_ID, "0", "", @@ -675,7 +732,10 @@ describe("reply framing (direct Lua — pins the wire shape the coordinator deco edgeField("w_b", 0), "{}", "0", - "" + "", + "", + contributionMember("block_test", ""), + "raw-test-revision" ); expect(reply).toEqual(["absorbed", "2", "2", "", "0"]); @@ -698,6 +758,16 @@ describe("reply framing (direct Lua — pins the wire shape the coordinator deco keys.done, keys.edge, keys.state, + runContributionKey("run_1", "block_test", ""), + runResultKey("run_1"), + keys.refs, + keys.edgeIndex, + keys.doneOrder, + keys.early, + keys.earlyOrder, + runWithdrawalKey("run_1", ""), + runWithdrawalTargetsKey("run_1", ""), + runContributionIndexKey("run_1"), BLOCK_ID, "0", "", @@ -711,7 +781,10 @@ describe("reply framing (direct Lua — pins the wire shape the coordinator deco edgeField("w_b", 0), "{}", "1", - envelope + envelope, + "", + contributionMember("block_test", ""), + "raw-test-revision" ); expect(reply).toEqual(["absorbed", "1", "1", "", "0", "w_b", envelope]); @@ -737,6 +810,16 @@ describe("reply framing (direct Lua — pins the wire shape the coordinator deco keys.done, keys.edge, keys.state, + runContributionKey("run_1", "block_test", ""), + runResultKey("run_1"), + keys.refs, + keys.edgeIndex, + keys.doneOrder, + keys.early, + keys.earlyOrder, + runWithdrawalKey("run_1", ""), + runWithdrawalTargetsKey("run_1", ""), + runContributionIndexKey("run_1"), BLOCK_ID, "0", "", @@ -745,7 +828,10 @@ describe("reply framing (direct Lua — pins the wire shape the coordinator deco edgeField("w_a", 0), "{}", "1", - "" + "", + "", + contributionMember("block_test", ""), + "raw-test-revision" ); expect(reply).toEqual(["absorbed", "0", "0", "", "0", "w_a", ""]); @@ -769,6 +855,16 @@ describe("reply framing (direct Lua — pins the wire shape the coordinator deco keys.done, keys.edge, keys.state, + runContributionKey("run_1", "block_test", ""), + runResultKey("run_1"), + keys.refs, + keys.edgeIndex, + keys.doneOrder, + keys.early, + keys.earlyOrder, + runWithdrawalKey("run_1", ""), + runWithdrawalTargetsKey("run_1", ""), + runContributionIndexKey("run_1"), BLOCK_ID, "0", "", @@ -782,7 +878,10 @@ describe("reply framing (direct Lua — pins the wire shape the coordinator deco edgeField("w_a", 1), "{}", "0", - "" + "", + "", + contributionMember("block_test", ""), + "raw-test-revision" ); expect(reply).toEqual(["absorbed", "1", "1", "", "0"]); @@ -809,6 +908,16 @@ describe("reply framing (direct Lua — pins the wire shape the coordinator deco keys.done, keys.edge, keys.state, + runContributionKey("run_1", "block_test", ""), + runResultKey("run_1"), + keys.refs, + keys.edgeIndex, + keys.doneOrder, + keys.early, + keys.earlyOrder, + runWithdrawalKey("run_1", ""), + runWithdrawalTargetsKey("run_1", ""), + runContributionIndexKey("run_1"), BLOCK_ID, "0", "", @@ -817,7 +926,10 @@ describe("reply framing (direct Lua — pins the wire shape the coordinator deco edgeField("w_a", 0), "{}", "0", - "" + "", + "", + contributionMember("block_test", ""), + "raw-test-revision" ); expect(reply).toEqual(["absorbed", "0", "0", "", "0", "w_a", envelope]); @@ -843,6 +955,16 @@ describe("reply framing (direct Lua — pins the wire shape the coordinator deco keys.done, keys.edge, keys.state, + runContributionKey("run_1", "block_test", ""), + runResultKey("run_1"), + keys.refs, + keys.edgeIndex, + keys.doneOrder, + keys.early, + keys.earlyOrder, + runWithdrawalKey("run_1", ""), + runWithdrawalTargetsKey("run_1", ""), + runContributionIndexKey("run_1"), BLOCK_ID, "0", "", @@ -851,7 +973,10 @@ describe("reply framing (direct Lua — pins the wire shape the coordinator deco field, "{}", "0", - "" + "", + "", + contributionMember("block_test", ""), + "raw-test-revision" ) ).rejects.toThrow(); @@ -876,7 +1001,21 @@ describe("reply framing (direct Lua — pins the wire shape the coordinator deco // n says 2 fields but only one field is supplied. await expect( - client.runClear(keys.pend, keys.done, keys.edge, keys.state, "2", field) + client.runClear( + keys.pend, + keys.done, + keys.edge, + keys.state, + keys.refs, + keys.edgeIndex, + keys.doneOrder, + keys.early, + keys.earlyOrder, + BLOCK_ID, + "2", + field, + "raw-test-revision" + ) ).rejects.toThrow(); expect(await client.hexists(keys.edge, field)).toBe(1); @@ -1380,6 +1519,436 @@ describe("absorbBlockers", () => { await store.quit(); } }); + + redisTest( + "the cleanup-owed fence refuses a DIFFERENT block until it is cleared, but never a same-block retry", + async ({ redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions }); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + await store.createIfAbsent({ record: record("w_b"), status: "PENDING" }); + + // B1 is installed. + const b1 = await store.absorbBlockers({ + runId: RUN_ID, + blockId: "B1", + edges: [edge("w_a")], + absorptionId: "B1", + }); + expect(b1.outcome).toBe("absorbed"); + + // B1's cross-store cleanup is owed (armed before its retry transition). + await store.armCleanupFence({ runId: RUN_ID, blockId: "B1" }); + + // A rollover to a DIFFERENT block B2 is REFUSED while B1's cleanup is owed — and it mutated nothing. + const b2 = await store.absorbBlockers({ + runId: RUN_ID, + blockId: "B2", + edges: [edge("w_b")], + expectedPreviousBlockId: "B1", + absorptionId: "B2", + }); + expect(b2.outcome).toBe("cleanupOwed"); + expect(b2.currentBlockId).toBe("B1"); + expect((await store.readBlockState(RUN_ID)).blockId).toBe("B1"); + + // A same-block (B1) retry is NOT blocked by the fence — idempotent. + const b1Retry = await store.absorbBlockers({ + runId: RUN_ID, + blockId: "B1", + edges: [edge("w_a")], + absorptionId: "B1", + }); + expect(b1Retry.outcome).toBe("absorbed"); + + // A stale clear for a DIFFERENT block does not lift B1's fence. + await store.clearCleanupFence({ runId: RUN_ID, blockId: "B_other" }); + expect( + ( + await store.absorbBlockers({ + runId: RUN_ID, + blockId: "B2", + edges: [edge("w_b")], + expectedPreviousBlockId: "B1", + absorptionId: "B2", + }) + ).outcome + ).toBe("cleanupOwed"); + + // Clearing the fence for B1 lets B2 finally install. + await store.clearCleanupFence({ runId: RUN_ID, blockId: "B1" }); + const b2Ok = await store.absorbBlockers({ + runId: RUN_ID, + blockId: "B2", + edges: [edge("w_b")], + expectedPreviousBlockId: "B1", + absorptionId: "B2", + }); + expect(b2Ok.outcome).toBe("absorbed"); + expect((await store.readBlockState(RUN_ID)).blockId).toBe("B2"); + } finally { + await store.quit(); + } + } + ); + + redisTest( + "finalizeRetainedWithdrawal atomically retires the record and the matching fence, is idempotent, and never lifts a different block's fence", + async ({ redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions }); + const probe = createRedisClient(redisOptions); + const stKey = `wp:v1:run:{${RUN_ID}}:st`; + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + + // Establish withdrawal:B1 (via an abandon that retained it) AND co=B1. + await store.absorbBlockers({ + runId: RUN_ID, + blockId: "B1", + edges: [edge("w_a")], + absorptionId: "B1", + }); + await store.armCleanupFence({ runId: RUN_ID, blockId: "B1" }); + await store.abandonRetainAndClear({ runId: RUN_ID, blockId: "B1" }); + expect(await store.readWithdrawalPage({ runId: RUN_ID, blockId: "B1" })).toBeDefined(); + expect(await probe.hget(stKey, "co")).toBe("B1"); + + // One atomic op retires BOTH. + await expect( + store.finalizeRetainedWithdrawal({ runId: RUN_ID, blockId: "B1" }) + ).rejects.toThrow("unacknowledged"); + await store.acknowledgeWithdrawalPage({ runId: RUN_ID, blockId: "B1", fields: ["w_a#"] }); + const r1 = await store.finalizeRetainedWithdrawal({ runId: RUN_ID, blockId: "B1" }); + expect(r1.fenceCleared).toBe(true); + expect(await store.readWithdrawalPage({ runId: RUN_ID, blockId: "B1" })).toBeUndefined(); + expect(await probe.hget(stKey, "co")).toBeNull(); + + // Idempotent no-op when both are already absent. + const r2 = await store.finalizeRetainedWithdrawal({ runId: RUN_ID, blockId: "B1" }); + expect(r2.fenceCleared).toBe(false); + expect(await store.readWithdrawalPage({ runId: RUN_ID, blockId: "B1" })).toBeUndefined(); + + // Fenced: with co=B2 owned by a newer block, finalizing B1 deletes ONLY B1's record and leaves co=B2. + await store.absorbBlockers({ + runId: RUN_ID, + blockId: "B1", + edges: [edge("w_a")], + absorptionId: "B1", + }); + await store.abandonRetainAndClear({ runId: RUN_ID, blockId: "B1" }); + await store.armCleanupFence({ runId: RUN_ID, blockId: "B2" }); + expect(await probe.hget(stKey, "co")).toBe("B2"); + await store.acknowledgeWithdrawalPage({ runId: RUN_ID, blockId: "B1", fields: ["w_a#"] }); + const r3 = await store.finalizeRetainedWithdrawal({ runId: RUN_ID, blockId: "B1" }); + expect(r3.fenceCleared).toBe(false); + expect(await store.readWithdrawalPage({ runId: RUN_ID, blockId: "B1" })).toBeUndefined(); + expect(await probe.hget(stKey, "co")).toBe("B2"); + } finally { + probe.disconnect(); + await store.quit(); + } + } + ); +}); + +redisTest( + "terminal empty withdrawal remains discoverable, while an invalid manifest retains its evidence and fence", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + const probe = createRedisClient(redisOptions); + const runId = "run_empty_terminal_withdrawal"; + const blockId = "B1"; + const key = runWithdrawalKey(runId, blockId); + try { + await store.absorbBlockers({ + runId, + blockId, + absorptionId: "A1", + edges: [{ waitpointId: "w_a", type: "RUN", createdAt: NOW }], + }); + await store.armCleanupFence({ runId, blockId }); + await store.compensateAbsorption({ runId, blockId, absorptionId: "A1" }); + await store.terminalRetainAndClear({ runId, blockId }); + const raw = (await probe.get(key))!; + expect(raw).toBe(blockId); + expect(await store.readWithdrawalPage({ runId, blockId })).toEqual({ + blockId, + fields: [], + }); + + await probe.set(key, "B2"); + await expect( + store.releaseRunWatchers({ runId, blockId, reason: "terminal" }) + ).rejects.toThrow(); + expect(await probe.get(key)).toBe("B2"); + expect(await probe.hget(runBlockKeys(runId).state, "co")).toBe(blockId); + + await probe.set(key, raw); + expect( + (await store.releaseRunWatchers({ runId, blockId, reason: "terminal" })).unregistered + ).toBe(0); + expect(await probe.get(key)).toBeNull(); + expect(await probe.hget(runBlockKeys(runId).state, "co")).toBeNull(); + expect((await store.readBlockState(runId)).terminal).toBe(true); + } finally { + probe.disconnect(); + await store.quit(); + } + } +); + +describe("bounded cleanup", () => { + redisTest( + "retains large additive blocks without enumerating them and replays acknowledged pages across a newer cycle", + async ({ redisOptions }) => { + let store = coordinator(redisOptions); + const probe = createRedisClient(redisOptions); + registerWaitpointCommands(probe); + const runId = "run_paged_withdrawal"; + const blockId = "B1"; + const count = RUN_CLEANUP_PAGE_SIZE * 3 + 7; + const keys = runBlockKeys(runId); + const wd = runWithdrawalKey(runId, blockId); + const targets = runWithdrawalTargetsKey(runId, blockId); + try { + await store.createIfAbsent({ record: record("w_shared"), status: "PENDING" }); + // Several independently bounded additive calls accumulate an unbounded logical block. + for (let start = 0; start < count; start += RUN_CLEANUP_PAGE_SIZE) { + const edges = Array.from( + { length: Math.min(RUN_CLEANUP_PAGE_SIZE, count - start) }, + (_, i) => edge("w_shared", { batchIndex: start + i }) + ); + await store.registerBlocks({ runId, blockId, edges }); + } + const statsBefore = await probe.info("commandstats"); + const reply = await probe.runAbandonRetainAndClear( + keys.pend, + keys.done, + keys.edge, + keys.state, + wd, + targets, + keys.refs, + keys.edgeIndex, + keys.doneOrder, + keys.early, + keys.earlyOrder, + blockId, + contributionMember("block_test", ""), + "raw-test-revision" + ); + expect(reply).toEqual(["retained"]); + expect(await probe.hlen(targets)).toBe(count); + expect(await probe.pttl(wd)).toBe(-1); + expect(await probe.pttl(targets)).toBe(-1); + await expect(store.finalizeRetainedWithdrawal({ runId, blockId })).rejects.toThrow( + "unacknowledged" + ); + const first = (await store.readWithdrawalPage({ runId, blockId }))!; + expect(first.fields).toHaveLength(RUN_CLEANUP_PAGE_SIZE); + // A crash after a watcher withdrawal but before acknowledgement retains the target for replay. + const firstIndex = parseEdgeField(first.fields[0]!)!.batchIndex!; + await store.unregisterWatcher({ + waitpointId: "w_shared", + runId, + blockId, + batchIndex: firstIndex, + }); + await store.quit(); + store = coordinator(redisOptions); + await store.registerBlocks({ + runId, + blockId: "B2", + edges: [edge("w_shared", { batchIndex: firstIndex })], + }); + let acknowledged = 0; + for (let pages = 0; pages < 5; pages++) { + const page = (await store.readWithdrawalPage({ runId, blockId }))!; + expect(page.fields.length).toBeLessThanOrEqual(RUN_CLEANUP_PAGE_SIZE); + if (!page.fields.length) break; + for (const field of page.fields) { + await store.unregisterWatcher({ + waitpointId: "w_shared", + runId, + blockId, + batchIndex: parseEdgeField(field)!.batchIndex, + }); + } + await store.acknowledgeWithdrawalPage({ runId, blockId, fields: page.fields }); + acknowledged += page.fields.length; + } + expect(acknowledged).toBe(count); + await store.finalizeRetainedWithdrawal({ runId, blockId }); + expect(await probe.exists(wd, targets)).toBe(0); + expect(await probe.hget(keys.state, "blk")).toBe("B2"); + expect(await probe.hlen(waitpointKeys("w_shared").watchers)).toBe(1); + expect( + await probe.hexists( + waitpointKeys("w_shared").watchers, + watcherField(runId, "B2", firstIndex) + ) + ).toBe(1); + // Terminal detach is equally constant-sized; its targets survive ordinary terminal-key expiry. + const terminal = await probe.runTerminalRetainAndClear( + keys.pend, + keys.done, + keys.edge, + keys.state, + runWithdrawalKey(runId, "B2"), + runResultKey(runId), + runWithdrawalTargetsKey(runId, "B2"), + keys.refs, + keys.edgeIndex, + keys.doneOrder, + keys.early, + keys.earlyOrder, + "B2", + String(Date.now()), + "10000", + contributionMember("block_test", ""), + "raw-test-revision" + ); + expect(terminal).toEqual(["retained"]); + await probe.unlink(keys.state); + await store.releaseRunWatchers({ runId, blockId: "B2", reason: "terminal" }); + expect(await probe.hlen(waitpointKeys("w_shared").watchers)).toBe(0); + expect(await store.readWithdrawalPage({ runId, blockId: "B2" })).toBeUndefined(); + // Exercise the actual terminal release loop across several pages, including its default + // block lookup. These targets have no watcher, so absent is successful convergence. + const terminalRunId = `${runId}_terminal`; + for (let start = 0; start < count; start += RUN_CLEANUP_PAGE_SIZE) { + await store.absorbBlockers({ + runId: terminalRunId, + blockId, + edges: Array.from({ length: Math.min(RUN_CLEANUP_PAGE_SIZE, count - start) }, (_, i) => + edge("w_shared", { batchIndex: start + i }) + ), + }); + } + await store.releaseRunWatchers({ runId: terminalRunId, reason: "terminal" }); + expect(await store.readWithdrawalPage({ runId: terminalRunId, blockId })).toBeUndefined(); + // No full-collection command is used by either retain, paging, finalization, or release's target lookup. + const statsAfter = await probe.info("commandstats"); + const calls = (stats: string, command: string) => { + const line = stats.split("\n").find((line) => line.startsWith(`cmdstat_${command}:`)); + return line ? Number(line.split("calls=")[1]!.split(",")[0]) : 0; + }; + expect(calls(statsAfter, "hget")).toBeGreaterThan(calls(statsBefore, "hget")); + for (const command of ["hkeys", "hgetall", "smembers"]) { + expect(calls(statsAfter, command)).toBe(calls(statsBefore, command)); + } + } finally { + probe.disconnect(); + await store.quit(); + } + } + ); + + redisTest( + "compensates bounded contribution pages while preserving concurrent additions, duplicate references, and stale generations", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + const probe = createRedisClient(redisOptions); + registerWaitpointCommands(probe); + const runId = "run_paged_compensation"; + const blockId = "B1"; + const keys = runBlockKeys(runId); + const contrib = runContributionKey(runId, blockId, "A1"); + const count = RUN_CLEANUP_PAGE_SIZE * 3 + 7; + try { + await store.absorbBlockers({ + runId, + blockId, + absorptionId: "kept", + edges: [edge("w_shared", { batchIndex: count + 1 })], + }); + for (let start = 0; start < count; start += RUN_CLEANUP_PAGE_SIZE) { + await store.absorbBlockers({ + runId, + blockId, + absorptionId: "A1", + edges: Array.from({ length: Math.min(RUN_CLEANUP_PAGE_SIZE, count - start) }, (_, i) => + edge("w_shared", { batchIndex: start + i }) + ), + }); + } + // HSETNX retries cannot increment the refcount twice. + await store.absorbBlockers({ + runId, + blockId, + absorptionId: "A1", + edges: [edge("w_shared", { batchIndex: 0 })], + }); + await store.deliverCompletion({ + runId, + blockId, + waitpointId: "w_shared", + completion: completion(), + }); + await store.deliverCompletion({ + runId, + blockId, + waitpointId: "w_early", + completion: completion(), + }); + const first = await probe.runCompensateAbsorption( + keys.pend, + keys.done, + keys.edge, + keys.state, + contrib, + keys.refs, + keys.edgeIndex, + keys.doneOrder, + keys.early, + keys.earlyOrder, + runContributionIndexKey(runId), + blockId, + "A1", + contributionMember(blockId, "A1"), + "raw-test-revision" + ); + expect(first).toHaveLength(4); + expect(Number(first[2])).toBeGreaterThan(0); + expect(Number(first[2])).toBeLessThanOrEqual(RUN_CLEANUP_PAGE_SIZE); + expect(Number(first[3])).toBeGreaterThanOrEqual(count - RUN_CLEANUP_PAGE_SIZE); + expect(Number(first[3])).toBeLessThan(count); + // New independent same-block registration while an interrupted compensation has pages left. + await store.absorbBlockers({ + runId, + blockId, + absorptionId: "A2", + edges: [edge("w_shared", { batchIndex: count }), edge("w_early")], + }); + const result = await store.compensateAbsorption({ runId, blockId, absorptionId: "A1" }); + expect(result).toEqual({ + outcome: "retained", + surviving: 3, + removed: count - Number(first[2]), + }); + expect(await probe.hget(keys.refs, "w_shared")).toBe("2"); + expect(await probe.hexists(keys.done, "w_shared")).toBe(1); + expect(await probe.hexists(keys.done, "w_early")).toBe(1); + expect(await probe.scard(keys.pend)).toBe(0); + await store.compensateAbsorption({ runId, blockId, absorptionId: "A2" }); + expect(await probe.hget(keys.refs, "w_shared")).toBe("1"); + expect(await probe.hexists(keys.done, "w_shared")).toBe(1); + await store.compensateAbsorption({ runId, blockId, absorptionId: "kept" }); + expect(await probe.exists(keys.edge, keys.refs, keys.done, keys.pend)).toBe(0); + expect(await store.currentBlockId(runId)).toBeUndefined(); + await store.absorbBlockers({ runId, blockId: "B2", edges: [edge("w_shared")] }); + await probe.sadd(contrib, "w_shared#"); + expect( + (await store.compensateAbsorption({ runId, blockId, absorptionId: "A1" })).outcome + ).toBe("stale"); + expect(await probe.hget(keys.refs, "w_shared")).toBe("1"); + expect(await probe.hexists(keys.edge, "w_shared#")).toBe(1); + } finally { + probe.disconnect(); + await store.quit(); + } + } + ); }); describe("deliverCompletion", () => { @@ -3616,6 +4185,31 @@ describe("cancellation fences delivery before withdrawal", () => { await store.quit(); } }); + + redisTest( + "a terminal release on a run with no coordination state writes neither a tombstone nor a withdrawal record", + async ({ redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions }); + const probe = createRedisClient(redisOptions); + try { + const released = await store.releaseRunWatchers({ + runId: "run_never_blocked", + reason: "terminal", + }); + expect(released.unregistered).toBe(0); + expect(released.cleanup).toMatchObject({ outcome: "armed", state: "absent" }); + + // No terminal tombstone on the run-state shard, and no withdrawal record. + expect(await probe.exists(`wp:v1:run:{run_never_blocked}:st`)).toBe(0); + expect( + await store.readWithdrawalPage({ runId: "run_never_blocked", blockId: "any" }) + ).toBeUndefined(); + } finally { + probe.disconnect(); + await store.quit(); + } + } + ); }); /** diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts index 7d1df7febeb..ec1799250a6 100644 --- a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts @@ -1,4 +1,5 @@ -import { createRedisClient, type Redis, type RedisOptions } from "@internal/redis"; +import { createHash, randomUUID } from "node:crypto"; +import { createRedisClient, type RedisClient, type RedisOptions } from "@internal/redis"; import type { Counter, Meter } from "@internal/tracing"; import { Logger } from "@trigger.dev/core/logger"; import { parseWaitpointId } from "@trigger.dev/core/v3/isomorphic"; @@ -13,10 +14,22 @@ import { idempotencyKey, parseEdgeField, runBlockKeys, + runContributionKey, + runContributionIndexKey, + runRegistrationSourcesKey, + contributionMember, + registrationEvidence, + runWithdrawalKey, + runWithdrawalTargetsKey, + runResultKey, waitpointKeys, watcherField, } from "./keys.js"; -import { registerWaitpointCommands } from "./scripts.js"; +import { + registerWaitpointCommands, + RUN_CLEANUP_PAGE_SIZE, + RETAIN_WAITPOINT_RECORD, +} from "./scripts.js"; /** The values written into a record's `status` field. Uppercase, and never a token. */ export type WaitpointStatus = "PENDING" | "COMPLETED"; @@ -39,7 +52,16 @@ type ScriptName = | "runMarkHandoffAcked" | "runTerminalCleanup" | "runReadBlockState" - | "runClear"; + | "runClear" + | "runAbandonRetainAndClear" + | "runTerminalRetainAndClear" + | "runArmCleanupFence" + | "runReadCleanupFence" + | "runClearCleanupFence" + | "runFinalizeRetainedWithdrawal" + | "runWithdrawalPage" + | "runCompensateAbsorption" + | "runDeleteContribution"; /** * The immutable half of a waitpoint, written once at creation. Carries every field the @@ -86,6 +108,44 @@ export type WaitpointCompletion = { output: WaitpointCompletionOutput; }; +/** A receipt may replace a source read only when it carries a complete, matching frozen record. */ +function assertRetainedCompletion( + receipt: WaitpointCompletion & { record?: WaitpointRecordInput }, + waitpointId: string +): asserts receipt is WaitpointCompletion & { record: WaitpointRecordInput } { + const record = receipt.record; + const parsed = parseWaitpointId(waitpointId); + const output = receipt.output; + if ( + !record || + parsed.format !== "b32hexW" || + record.id !== waitpointId || + record.type !== parsed.type || + typeof record.friendlyId !== "string" || + typeof record.environmentId !== "string" || + typeof record.projectId !== "string" || + !Number.isFinite(Date.parse(record.createdAt)) || + !Number.isFinite(Date.parse(record.updatedAt)) || + typeof record.userProvidedIdempotencyKey !== "boolean" || + !Array.isArray(record.tags) || + !record.tags.every((tag) => typeof tag === "string") || + (record.type === "RUN" && + (typeof record.completedByTaskRunId !== "string" || !record.completedByTaskRunId)) || + typeof receipt.outputType !== "string" || + typeof receipt.outputIsError !== "boolean" || + !Number.isFinite(Date.parse(receipt.completedAt)) || + (output !== null && + (typeof output !== "object" || + !( + ("inline" in output && typeof output.inline === "string") || + ("ref" in output && typeof output.ref === "string") + ))) + ) { + throw new Error(`Invalid retained completion for waitpoint ${waitpointId}`); + } + assertCompletionWithinLimits(receipt, "registerAndDeliver.retained"); +} + /** * The largest inline completion output this coordinator will accept, in BYTES. * @@ -273,10 +333,11 @@ export class EncodedWaitpointCompletion { /** The ONLY way to obtain one. Validates, then serializes exactly once. */ static encode( completion: WaitpointCompletion | undefined, - context: string + context: string, + record?: WaitpointRecordInput ): EncodedWaitpointCompletion { assertCompletionWithinLimits(completion, context); - const json = completion ? JSON.stringify(completion) : ""; + const json = completion ? JSON.stringify(record ? { ...completion, record } : completion) : ""; return new EncodedWaitpointCompletion( ENCODED_CONSTRUCTOR_TOKEN, json, @@ -292,9 +353,10 @@ export class EncodedWaitpointCompletion { export function encodeCompletionForDelivery( completion: WaitpointCompletion | undefined, - context: string + context: string, + record?: WaitpointRecordInput ): EncodedWaitpointCompletion { - return EncodedWaitpointCompletion.encode(completion, context); + return EncodedWaitpointCompletion.encode(completion, context, record); } /** @@ -338,7 +400,13 @@ export type CreateIfAbsentResult = export type RegisterOrReportResult = | { outcome: "registered" } - | { outcome: "completed"; completion?: WaitpointCompletion }; + | { + outcome: "completed"; + completion?: WaitpointCompletion; + record?: WaitpointRecordInput; + /** The completing run's id (record `cbt`), returned ATOMICALLY with the envelope for cachedRunId. */ + completedByTaskRunId?: string; + }; /** The lifecycle of a waitpoint's fanout obligation. `absent` means none was ever owed. */ export type FanoutState = "absent" | "pending" | "done" | "quarantined"; @@ -366,10 +434,10 @@ export type BlockEdge = { createdAt: string; type: WaitpointRecordInput["type"]; completedAfter?: string; - // Set when the register step already reported this waitpoint COMPLETED. The box, not + // Set when a source read or register step reported this waitpoint COMPLETED. The box, not // `completion`, carries the "reported" fact: box present + no completion means // COMPLETED-with-no-envelope, box absent means never reported. - reported?: { completion?: WaitpointCompletion }; + reported?: { completion?: WaitpointCompletion & { record?: WaitpointRecordInput } }; }; export type AbsorbResult = { @@ -378,7 +446,7 @@ export type AbsorbResult = { * on to a block this operation did not expect; `terminal` means it has been through * cleanup. Both mutated nothing, and neither may proceed to registration. */ - outcome: "absorbed" | "stale" | "terminal"; + outcome: "absorbed" | "stale" | "terminal" | "cleanupOwed"; /** The block the run is actually on, for diagnosing a refusal. */ currentBlockId?: string; /** @@ -394,7 +462,10 @@ export type AbsorbResult = { * a decision to resume. */ storePendingTotal: number; - alreadyDelivered: Array<{ waitpointId: string; completion?: WaitpointCompletion }>; + alreadyDelivered: Array<{ + waitpointId: string; + completion?: WaitpointCompletion & { record?: WaitpointRecordInput }; + }>; /** * What the rollover did about the SUPERSEDED cycle's watcher registrations, when this call * replaced one. Absent when nothing was superseded — a first block, or a same-block retry. @@ -403,6 +474,18 @@ export type AbsorbResult = { * just installed, so the caller gets the block AND the residue. */ supersededCleanup?: SupersededCleanup; + /** + * Cached spans closed by an early/racing delivery during registerAndDeliver, for the caller to emit as + * cachedRunCompleted. Empty from a bare absorb (delivery is registration's job). Shape matches the + * contract layer's CachedSpanCompletion; kept inline so this low-level module stays free of that import. + */ + cachedSpanCompletions?: Array<{ + spanId: string; + spanCreatedAt: string; + blockedRunId: string; + hasError: boolean; + cachedRunId?: string; + }>; }; export type SupersededCleanup = { @@ -487,6 +570,13 @@ export type FanoutClaim = failures: number; /** Completion time in epoch ms, for the completion-to-delivery latency signal. */ completedAtMs: number; + /** + * The waitpoint record's `completedByTaskRunId`, carried once per claimed page so cached-span + * completion can supply it as `cachedRunId` without a per-watcher record read. Undefined when the + * record does not carry it. + */ + completedByTaskRunId?: string; + record?: WaitpointRecordInput; page: FanoutPageEntry[]; }; @@ -604,7 +694,11 @@ export class WaitpointCompletionConflictError extends Error { export const DEFAULT_TERMINAL_RETENTION_MS = 14 * 24 * 60 * 60 * 1000; export type WaitpointStoreCoordinatorOptions = { - redisOptions: RedisOptions; + redisOptions?: RedisOptions; + /** App-owned standalone or MemoryDB cluster client. No eager connection is required. */ + client?: RedisClient; + /** Connect and validate infrastructure before commands; inert until a store operation. */ + beforeCommand?: () => Promise; logger?: Logger; meter?: Meter; /** Defaults to {@link DEFAULT_TERMINAL_RETENTION_MS}. */ @@ -643,7 +737,9 @@ function decodeHandoff(raw: string | undefined): HandoffState { } export class WaitpointStoreCoordinator { - private readonly redis: Redis; + private readonly redis: RedisClient; + private readonly beforeCommand?: () => Promise; + private readonly ownsClient: boolean; private readonly logger: Logger; private readonly terminalRetentionMs: number; private readonly clock: () => number; @@ -663,10 +759,17 @@ export class WaitpointStoreCoordinator { this.logger = options.logger ?? new Logger("WaitpointStoreCoordinator", "debug"); this.terminalRetentionMs = options.terminalRetentionMs ?? DEFAULT_TERMINAL_RETENTION_MS; this.clock = options.clock ?? Date.now; - this.redis = createRedisClient(options.redisOptions, { - onError: (error) => - this.logger.error("WaitpointStoreCoordinator redis client error", { error }), - }); + if (!options.client && !options.redisOptions) { + throw new Error("Waitpoint store requires a Redis client or connection options"); + } + this.beforeCommand = options.beforeCommand; + this.ownsClient = !options.client; + this.redis = + options.client ?? + createRedisClient(options.redisOptions!, { + onError: (error) => + this.logger.error("WaitpointStoreCoordinator redis client error", { error }), + }); registerWaitpointCommands(this.redis); if (options.meter) { @@ -712,6 +815,11 @@ export class WaitpointStoreCoordinator { // Idempotent and error-swallowing: every test calls this in a finally, and a double quit // must never mask the real assertion failure. async quit(): Promise { + if (!this.ownsClient) return; + if (this.redis.status === "wait" || this.redis.status === "end") { + this.redis.disconnect(); + return; + } if (!this.#quit) { this.#quit = this.redis.quit().then( () => undefined, @@ -731,7 +839,32 @@ export class WaitpointStoreCoordinator { */ #call(script: ScriptName, keys: string[], ...argv: string[]): Promise { assertSingleSlot(script, keys); - const command = this.redis[script] as (...args: string[]) => Promise; + // Each read-affecting mutation gets a fresh epoch, including retries of a logical operation. + // Delivery initializes the epoch only; its immutable receipt ordinal preserves an ongoing read. + if ( + [ + "runAbsorbBlockers", + "runDeliverCompletion", + "runMarkHandoffAcked", + "runTerminalCleanup", + "runClear", + "runAbandonRetainAndClear", + "runTerminalRetainAndClear", + "runCompensateAbsorption", + ].includes(script) + ) { + argv.push(randomUUID()); + } + const command = Reflect.get(this.redis, script) as (...args: string[]) => Promise; + return this.#dispatch(command, keys, argv); + } + + async #dispatch( + command: (...args: string[]) => Promise, + keys: string[], + argv: string[] + ): Promise { + await this.beforeCommand?.(); return command.call(this.redis, ...keys, ...argv); } @@ -764,7 +897,8 @@ export class WaitpointStoreCoordinator { JSON.stringify(args.record), args.status, args.completion ? JSON.stringify(args.completion) : "", - String(this.terminalRetentionMs) + String(this.terminalRetentionMs), + args.record.completedByTaskRunId ?? "" ); if (reply[0] === "created") { @@ -794,6 +928,8 @@ export class WaitpointStoreCoordinator { batchIndex?: number | null; spanIdToComplete?: string; createdAt: string; + /** The watching run's org, stored on the watcher so the fanout continuation routes without a lookup. */ + organizationId?: string; }): Promise { const keys = waitpointKeys(args.waitpointId); @@ -805,6 +941,7 @@ export class WaitpointStoreCoordinator { batchIndex: args.batchIndex ?? undefined, spanIdToComplete: args.spanIdToComplete, createdAt: args.createdAt, + organizationId: args.organizationId, }; const reply = await this.#call( @@ -821,7 +958,12 @@ export class WaitpointStoreCoordinator { // Registration after completion needs no fanout: the frozen envelope comes straight // back and the caller delivers it against the run's own shard. this.metrics.registrations?.add(1, { outcome: "reported_complete" }); - return { outcome: "completed", completion: parseJson(reply[1]) }; + return { + outcome: "completed", + completion: parseJson(reply[1]), + completedByTaskRunId: reply[2] || undefined, + record: parseJson(reply[3]), + }; } this.metrics.registrations?.add(1, { outcome: "registered" }); @@ -1026,15 +1168,29 @@ export class WaitpointStoreCoordinator { * predecessor makes the two distinguishable, and a mismatch is refused as `stale`. */ expectedPreviousBlockId?: string; + /** + * Stable across retries and unique to this additive operation. Each operation claims its edges; + * compensation releases those claims and removes only edges with no remaining claims. + * The adapter always supplies it. Direct calls without an ID retain their claims until block cleanup. + */ + absorptionId?: string; + registrationEvidence?: string; + batchIndex?: number; }): Promise { // Every reported completion, checked before the loop below serializes any of them. A // reported completion rides into absorption as a receipt, so it reaches the same // `JSON.stringify` an inline completion does and needs the same ceiling. for (const item of args.edges) { assertCompletionWithinLimits(item.reported?.completion, `absorbBlockers:${item.waitpointId}`); + if (item.reported?.completion?.record !== undefined) { + assertRetainedCompletion(item.reported.completion, item.waitpointId); + } } const keys = runBlockKeys(args.runId); + const replayCleanup = args.expectedPreviousBlockId + ? await this.#withdrawSupersededWatchers(args.runId, args.expectedPreviousBlockId) + : undefined; // No fast path for an empty list: storePendingTotal is defined as the run's WHOLE // store-resident pending set, so it has to be read even when nothing is requested. @@ -1060,16 +1216,40 @@ export class WaitpointStoreCoordinator { const reply = await this.#call( "runAbsorbBlockers", - [keys.pend, keys.done, keys.edge, keys.state], - ...argv + [ + keys.pend, + keys.done, + keys.edge, + keys.state, + runContributionKey(args.runId, args.blockId, args.absorptionId ?? ""), + runResultKey(args.runId), + keys.refs, + keys.edgeIndex, + keys.doneOrder, + keys.early, + keys.earlyOrder, + runWithdrawalKey(args.runId, args.expectedPreviousBlockId ?? ""), + runWithdrawalTargetsKey(args.runId, args.expectedPreviousBlockId ?? ""), + runContributionIndexKey(args.runId), + ], + ...argv, + args.absorptionId ?? "", + contributionMember(args.blockId, args.absorptionId ?? ""), + args.registrationEvidence ?? + registrationEvidence( + args.edges.map((edge) => edge.waitpointId), + args.batchIndex ?? args.edges[0]?.batchIndex ?? undefined + ) ); - // Refused outcomes mutated NOTHING — not pend, done, edge, blk, hs, term, and no - // watchers. The caller must not go on to register, so this is a result and not a partial + // Refused scripts mutate no active block state. An earlier retained predecessor can still + // converge on a stale retry; the caller must not register, so this is a result and not a partial // absorb: `registerBlocks` stops here, and a direct caller reads `outcome`. - if (reply[0] === "stale" || reply[0] === "terminal") { + if (reply[0] === "stale" || reply[0] === "terminal" || reply[0] === "cleanupOwed") { this.metrics.staleAbsorbs?.add(1, { outcome: reply[0] }); return { + // `cleanupOwed`: a previous mixed block's cross-store cleanup is still owed, so this DIFFERENT block + // is refused (active block unchanged) until the fence clears — reply[1] is the block that owes cleanup. outcome: reply[0], currentBlockId: reply[1] || undefined, pendingOfRequested: 0, @@ -1082,21 +1262,22 @@ export class WaitpointStoreCoordinator { // after the fields those two delimit. See the `out` table in runAbsorbBlockers. const supersededBlockId = reply[3] ?? ""; const supersededFieldCount = Number(reply[4] ?? 0); - const supersededFields = reply.slice(5, 5 + supersededFieldCount); const alreadyDelivered: AbsorbResult["alreadyDelivered"] = []; for (let i = 5 + supersededFieldCount; i < reply.length; i += 2) { alreadyDelivered.push({ waitpointId: reply[i]!, - completion: parseJson(reply[i + 1]), + completion: parseJson( + reply[i + 1] + ), }); } - // AFTER the script committed, never before: the block is installed either way, and a - // cleanup that ran first could withdraw registrations for a rollover that then failed. + // Fresh targets exist only after the atomic rollover committed. A retry may also have + // converged an already-retained predecessor before the CAS, without touching active state. const supersededCleanup = supersededBlockId - ? await this.#withdrawSupersededWatchers(args.runId, supersededBlockId, supersededFields) - : undefined; + ? await this.#withdrawSupersededWatchers(args.runId, supersededBlockId) + : replayCleanup; return { outcome: "absorbed", @@ -1108,84 +1289,50 @@ export class WaitpointStoreCoordinator { } /** - * Withdraw a superseded block's watcher registrations from their waitpoint shards. - * - * Each waitpoint is its own shard, so this cannot be part of the rollover script and cannot - * be atomic with it. That is the whole reason the fields are returned rather than acted on - * inside the Lua. - * - * Idempotent by construction: the watcher field carries the block id, so withdrawing the - * superseded block's registration cannot touch the new block's — even when the new block - * blocks on the very same waitpoint. `absent` and `missing` are both SUCCESS: a watcher - * already gone, or a waitpoint that has since expired, is the state this is trying to reach. - * - * THE BOUNDARY: a crash, a lost reply, or a cross-shard failure between the rollover - * committing and these unregistrations completing leaves registrations behind. They are inert - * — a completion delivering under a superseded block id is refused as stale — but they are - * unreachable from the run side, because the edge set that named them is gone. Repairing that - * residue is the reconciliation lane's, and this method reports it rather than retrying forever. + * Withdraw a detached predecessor in bounded, replay-safe pages. Watcher fields contain the block + * identity, so this never removes a newer block's registration. Retire only successful targets; + * failures remain in the no-TTL manifest for an explicit predecessor retry/reconciliation. Current + * application block-operation builders do not request rollover; this is the primitive's contract, + * not an automatically scheduled recovery promise. */ async #withdrawSupersededWatchers( runId: string, - blockId: string, - fields: string[] - ): Promise { - const targets = fields.flatMap((field) => { - const parsed = parseEdgeField(field); - if (!parsed) { - // An unparseable field cannot name a waitpoint to unregister from. Reported as residue - // rather than dropped silently, because it means the edge encoding drifted. - this.logger.error("Waitpoint rollover found an unparseable superseded edge field", { - runId, - blockId, - field, - }); - return []; - } - return [parsed]; - }); - + blockId: string + ): Promise { + let page = await this.readWithdrawalPage({ runId, blockId }); + if (!page) return undefined; const failed: SupersededCleanup["failed"] = []; let withdrawn = 0; - - await pMap( - targets, - async (target) => { - try { - await this.unregisterWatcher({ - waitpointId: target.waitpointId, - runId, - blockId, - batchIndex: target.batchIndex, - }); - // Every outcome the call can return — unregistered, absent, missing — means no - // registration under this block id remains, which is what was asked for. - withdrawn++; - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - failed.push({ ...target, error: message }); - this.metrics.supersededWatcherCleanupFailures?.add(1); - this.logger.error("Waitpoint rollover could not withdraw a superseded watcher", { - runId, - blockId, - waitpointId: target.waitpointId, - error, - }); - } - }, - { concurrency: SUPERSEDED_CLEANUP_CONCURRENCY } - ); - - const unparseable = fields.length - targets.length; - if (failed.length > 0 || unparseable > 0) { - this.logger.error("Waitpoint rollover left reconciliation residue", { - runId, - blockId, - failed: failed.length, - unparseable, - }); + while (page.fields.length > 0) { + const acknowledged: string[] = []; + await pMap( + page.fields, + async (field) => { + const target = parseEdgeField(field); + if (!target) throw new Error(`Invalid retained withdrawal target: ${field}`); + try { + await this.unregisterWatcher({ ...target, runId, blockId }); + acknowledged.push(field); + withdrawn++; + } catch (error) { + failed.push({ + ...target, + error: error instanceof Error ? error.message : String(error), + }); + this.metrics.supersededWatcherCleanupFailures?.add(1); + } + }, + { concurrency: SUPERSEDED_CLEANUP_CONCURRENCY } + ); + // Successful targets can be retired even if their page has failures. Failed targets remain + // durably discoverable by the same predecessor identity on a stale or ordinary caller retry. + await this.acknowledgeWithdrawalPage({ runId, blockId, fields: acknowledged }); + if (failed.length > 0) break; + const next = await this.readWithdrawalPage({ runId, blockId }); + if (!next) return { blockId, withdrawn, failed }; // Another retrier finalized the same manifest. + page = next; } - + if (failed.length === 0) await this.finalizeRetainedWithdrawal({ runId, blockId }); return { blockId, withdrawn, failed }; } @@ -1213,6 +1360,8 @@ export class WaitpointStoreCoordinator { runId: string; blockId: string; edges: BlockEdge[]; + /** The watching run's org, stored on each watcher so the fanout continuation routes without a lookup. */ + organizationId?: string; /** * What `absorbBlockers` reported for this same block. REQUIRED: registration is step 4 * of the block sequence and step 2 always precedes it, so there is no supported flow in @@ -1227,7 +1376,21 @@ export class WaitpointStoreCoordinator { ); let storePendingTotal = args.absorbed.storePendingTotal; + // Absorption returns receipts under the current block's atomic fence. Their immutable metadata + // survives source expiry, so replay must consume it rather than registering an already-delivered + // waitpoint again. Older receipts without metadata still require the ordinary source validation. + const cbtByWaitpoint = new Map(); + const retained = new Map( + args.absorbed.alreadyDelivered.map((entry) => [entry.waitpointId, entry]) + ); + for (const item of args.edges) { + const retainedCompletion = retained.get(item.waitpointId)?.completion; + if (retainedCompletion?.record !== undefined) { + assertRetainedCompletion(retainedCompletion, item.waitpointId); + cbtByWaitpoint.set(item.waitpointId, retainedCompletion.record.completedByTaskRunId); + continue; + } const result = await this.registerOrReport({ waitpointId: item.waitpointId, runId: args.runId, @@ -1235,8 +1398,13 @@ export class WaitpointStoreCoordinator { batchIndex: item.batchIndex, spanIdToComplete: item.spanIdToComplete, createdAt: item.createdAt, + organizationId: args.organizationId, }); + if (result.outcome === "completed" && !cbtByWaitpoint.has(item.waitpointId)) { + cbtByWaitpoint.set(item.waitpointId, result.completedByTaskRunId); + } + if (result.outcome !== "completed" || delivered.has(item.waitpointId)) { continue; } @@ -1248,6 +1416,8 @@ export class WaitpointStoreCoordinator { blockId: args.blockId, waitpointId: item.waitpointId, completion: result.completion, + requireCurrentBlock: true, + record: result.record, }); delivered.set(item.waitpointId, result.completion); storePendingTotal = receipt.storePendingTotal; @@ -1261,7 +1431,22 @@ export class WaitpointStoreCoordinator { } } + // Cached-span completions for edges whose delivery happened here (early or racing) and that carry a + // span. Independent of resumability — one blocker may close its span while others stay pending. The + // completed run identity (cachedRunId) came back ATOMICALLY with each completion above, so it can never + // diverge from the envelope at the record's expiry. + const cachedSpanCompletions = args.edges + .filter((edge) => edge.spanIdToComplete && delivered.has(edge.waitpointId)) + .map((edge) => ({ + spanId: edge.spanIdToComplete!, + spanCreatedAt: edge.createdAt, + blockedRunId: args.runId, + hasError: delivered.get(edge.waitpointId)?.outputIsError ?? false, + cachedRunId: cbtByWaitpoint.get(edge.waitpointId), + })); + return { + cachedSpanCompletions, // This half only runs once absorption succeeded — a refusal never reaches it. outcome: "absorbed", pendingOfRequested, @@ -1292,6 +1477,8 @@ export class WaitpointStoreCoordinator { runId: string; /** A FRESH id per block operation — see `runAbsorbBlockers` for why reuse is unsafe. */ blockId: string; + /** The contribution identity; defaults to the block id when the caller runs one absorption per block. */ + absorptionId?: string; edges: BlockEdge[]; /** Forwarded to `absorbBlockers` as the compare-and-set precondition. */ expectedPreviousBlockId?: string; @@ -1301,6 +1488,7 @@ export class WaitpointStoreCoordinator { blockId: args.blockId, edges: args.edges.map(({ reported: _reported, ...edge }) => edge), expectedPreviousBlockId: args.expectedPreviousBlockId, + absorptionId: args.absorptionId ?? args.blockId, }); // A refusal stops here. Registering for a block the store declined to install would leave @@ -1330,10 +1518,13 @@ export class WaitpointStoreCoordinator { * done; the receipt is simply empty. */ completion?: WaitpointCompletion; + record?: WaitpointRecordInput; + /** Production delivery follows absorption; refuse absence after cleanup/expiry. */ + requireCurrentBlock?: boolean; }): Promise { return this.deliverEncodedCompletion({ ...args, - encoded: encodeCompletionForDelivery(args.completion, "deliverCompletion"), + encoded: encodeCompletionForDelivery(args.completion, "deliverCompletion", args.record), }); } @@ -1349,6 +1540,7 @@ export class WaitpointStoreCoordinator { blockId: string; waitpointId: string; encoded: EncodedWaitpointCompletion; + requireCurrentBlock?: boolean; }): Promise { if (!EncodedWaitpointCompletion.isEncoded(args.encoded)) { throw new TypeError( @@ -1360,10 +1552,20 @@ export class WaitpointStoreCoordinator { const reply = await this.#call( "runDeliverCompletion", - [keys.pend, keys.done, keys.state], + [ + keys.pend, + keys.done, + keys.state, + runResultKey(args.runId), + keys.refs, + keys.doneOrder, + keys.early, + keys.earlyOrder, + ], args.blockId, args.waitpointId, - args.encoded.json + args.encoded.json, + args.requireCurrentBlock ? "1" : "0" ); const outcome = reply[0] as DeliverResult["outcome"]; @@ -1435,7 +1637,18 @@ export class WaitpointStoreCoordinator { const reply = await this.#call( "runTerminalCleanup", - [keys.pend, keys.done, keys.edge, keys.state], + [ + keys.pend, + keys.done, + keys.edge, + keys.state, + runResultKey(args.runId), + keys.refs, + keys.edgeIndex, + keys.doneOrder, + keys.early, + keys.earlyOrder, + ], String(this.clock()), String(this.terminalRetentionMs), voidHandoff @@ -1452,12 +1665,14 @@ export class WaitpointStoreCoordinator { } /** - * Withdraw this run's registrations from a set of waitpoints, then arm its terminal - * window. Used on cancellation and on terminal completion of a blocked run. - * - * The loop is over the run's OWN edges, so its width is the run's blocker count — the - * same bound the block operation already had — and never the waitpoint's watcher count. - * Each waitpoint sits on its own shard, so this cannot be one script. + * Withdraw this run's registrations on cancellation or terminal completion of a blocked run, TERMINAL + * FIRST: {@link terminalRetainAndClear} atomically installs the fence, clears the active block, arms the + * tombstone TTL and captures the withdrawal targets into the durable (no-TTL) record BEFORE any watcher + * is withdrawn cross-shard. So a fanout page claimed before the withdrawal refuses as `terminal`, and a + * partial withdrawal (or crash) leaves every unacknowledged target in the retained hash for + * {@link resumeRetainedWithdrawal} to finish — even after the ordinary run-state keys have expired. The + * record is removed only once every target has converged (a thrown Redis error rejects here and keeps + * it). The loop width is the run's own blocker count, never the waitpoint's watcher count. */ async releaseRunWatchers(args: { runId: string; @@ -1467,98 +1682,575 @@ export class WaitpointStoreCoordinator { * is the one a cancellation or terminal completion is interrupting. */ blockId?: string; - /** Defaults to every edge currently recorded for the run. */ - edges?: Array<{ waitpointId: string; batchIndex?: number | null }>; /** Concurrent unregistrations. Bounded so a wide batch cannot flood the cluster. */ concurrency?: number; }): Promise<{ unregistered: number; cleanup: CleanupResult }> { - const state = args.blockId && args.edges ? undefined : await this.readBlockState(args.runId); - const blockId = args.blockId ?? state?.blockId; - const edges = - args.edges ?? - (state?.edges ?? []).map((edge) => ({ - waitpointId: edge.waitpointId, - batchIndex: edge.batchIndex, - })); - - // TERMINAL FIRST, then withdraw. The reverse order looked tidier and did not fence - // anything: a fanout page claimed before the withdrawal already holds its watcher data, - // so deleting the registration cannot stop that delivery — it would land while `term` was - // still unset, empty `pend` and set `hs='owed'`, publishing a resume for a run being - // cancelled. Installing `term` on the run's own shard first makes any such delivery - // refuse as `terminal`, which is a decision on one slot and needs no cross-shard ordering. - const cleanup = await this.cleanupRunBlockState({ runId: args.runId, reason: args.reason }); - - // `retained` means the run is NOT terminal, so the fence is not in place. Withdrawing now - // would be the old unsafe order, so it is not done. Unreachable for these reasons — they - // all void an owed handoff — but the guard is what makes that a fact rather than a hope. - if (cleanup.outcome === "retained") { + const target = args.blockId ?? (await this.currentBlockId(args.runId)); + if (!target) { + // Resumed snapshots may still own results after their active block was acknowledged. + // This terminal-only operation fences delivery and arms result retention in one slot. + const cleanup = await this.cleanupRunBlockState({ runId: args.runId, reason: args.reason }); + await this.retireRunRegistrationSources(args.runId); return { unregistered: 0, cleanup }; } - // Now inert: the registrations can no longer produce a resume, so a cross-shard failure - // below leaves reconciliation residue and never a wrong wake-up. - const outcomes = blockId - ? await pMap( - edges, - (edge) => - this.unregisterWatcher({ - waitpointId: edge.waitpointId, - runId: args.runId, - blockId, - batchIndex: edge.batchIndex, - }), - { concurrency: args.concurrency ?? 10 } - ) - : []; - const unregistered = outcomes.filter((o) => o.outcome === "unregistered").length; + const retained = await this.terminalRetainAndClear({ runId: args.runId, blockId: target }); + if (retained.outcome === "stale") { + return { + unregistered: 0, + cleanup: { outcome: "armed", handoffWas: "none", state: "present" }, + }; + } + + await this.retireRunRegistrationSources(args.runId); + let unregistered = 0; + for (;;) { + const page = await this.readWithdrawalPage({ runId: args.runId, blockId: target }); + if (!page || page.fields.length === 0) break; + const outcomes = await pMap( + page.fields, + (field) => { + const parsed = parseEdgeField(field); + if (!parsed) throw new Error(`Invalid retained withdrawal target: ${field}`); + return this.unregisterWatcher({ + waitpointId: parsed.waitpointId, + runId: args.runId, + blockId: target, + batchIndex: parsed.batchIndex, + }); + }, + { concurrency: Math.max(1, Math.min(args.concurrency ?? 10, RUN_CLEANUP_PAGE_SIZE)) } + ); + unregistered += outcomes.filter((o) => o.outcome === "unregistered").length; + // A crash before this acknowledgement replays the whole page idempotently. Never acknowledge + // a partially failed page; retained targets outlive source records and the terminal tombstone. + await this.acknowledgeWithdrawalPage({ + runId: args.runId, + blockId: target, + fields: page.fields, + }); + } + await this.finalizeRetainedWithdrawal({ runId: args.runId, blockId: target }); - return { unregistered, cleanup }; + return { + unregistered, + cleanup: { outcome: "armed", handoffWas: "none", state: "present" }, + }; } - async readBlockState(runId: string): Promise { - const keys = runBlockKeys(runId); - const reply = await this.#call("runReadBlockState", [ - keys.pend, - keys.done, - keys.edge, - keys.state, - ]); + /** + * ATOMICALLY record ONE block's watcher-withdrawal targets and clear its active run-side state, so a + * crash before withdrawal can never strand an active block with no watchers. Fenced to the exact block: + * a stale/terminal block writes and clears nothing. Read and acknowledge bounded pages to drain it. + */ + async abandonRetainAndClear(args: { + runId: string; + blockId: string; + }): Promise< + { outcome: "retained" } | { outcome: "stale" | "terminal"; currentBlockId?: string } + > { + const keys = runBlockKeys(args.runId); + const reply = await this.#call( + "runAbandonRetainAndClear", + [ + keys.pend, + keys.done, + keys.edge, + keys.state, + runWithdrawalKey(args.runId, args.blockId), + runWithdrawalTargetsKey(args.runId, args.blockId), + keys.refs, + keys.edgeIndex, + keys.doneOrder, + keys.early, + keys.earlyOrder, + ], + args.blockId + ); + if (reply[0] === "retained") { + return { outcome: "retained" }; + } + return { outcome: reply[0] as "stale" | "terminal", currentBlockId: reply[1] || undefined }; + } - // Slots 0 and 1 are true element counts, but slot 2 is the FLAT length of the edge - // HGETALL — two entries per edge, field then value. The cursor arithmetic below relies - // on that asymmetry, so do not "normalise" it without changing the Lua too. - const pendCount = Number(reply[0]); - const doneCount = Number(reply[1]); - const edgeCount = Number(reply[2]); - const blockId = reply[3] || undefined; - const handoff = decodeHandoff(reply[4]); - const terminal = reply[5] === "1"; - - let cursor = 6; - const pendingIds = reply.slice(cursor, cursor + pendCount); - cursor += pendCount; - const deliveredIds = reply.slice(cursor, cursor + doneCount); - cursor += doneCount; - - const edges: BlockStateEdge[] = []; - for (let i = 0; i < edgeCount; i += 2) { - const edgeId = reply[cursor + i]!; - // An edge value is always a non-empty JSON.stringify, so a missing slot here means - // the cursor walked off the end of the reply. That must fail loudly, not decode a - // BlockEdge with no waitpointId — the exact off-by-one this task's arithmetic guards - // against. - const edgeJson = reply[cursor + i + 1]; - if (!edgeJson) { - throw new Error( - `readBlockState(${runId}): missing edge payload at reply index ${cursor + i + 1}` + /** + * ATOMICALLY retain ONE block's watcher-withdrawal targets and terminalize the run: it installs the + * terminal fence, clears the active block and arms the retention TTL on the run-state tombstone ONLY, + * capturing the withdrawal targets in the SAME durable (no-TTL) record the non-terminal abandon uses. So + * a watcher withdrawal still owed after the fortnight tombstone expires never loses its target list. + * Fenced to the exact block; a run with no coordination state writes nothing. + */ + async terminalRetainAndClear(args: { + runId: string; + blockId: string; + }): Promise< + { outcome: "retained" } | { outcome: "absent" | "stale" | "terminal"; currentBlockId?: string } + > { + const keys = runBlockKeys(args.runId); + const reply = await this.#call( + "runTerminalRetainAndClear", + [ + keys.pend, + keys.done, + keys.edge, + keys.state, + runWithdrawalKey(args.runId, args.blockId), + runResultKey(args.runId), + runWithdrawalTargetsKey(args.runId, args.blockId), + keys.refs, + keys.edgeIndex, + keys.doneOrder, + keys.early, + keys.earlyOrder, + ], + args.blockId, + String(this.clock()), + String(this.terminalRetentionMs) + ); + if (reply[0] === "retained") { + return { outcome: "retained" }; + } + return { + outcome: reply[0] as "absent" | "stale" | "terminal", + currentBlockId: reply[1] || undefined, + }; + } + + /** + * Durably mark this run's exact current store block as cleanup-owed. While set, {@link absorbBlockers} + * refuses a DIFFERENT block. Armed before the retry/requeue transition; cleared only once the block's + * cross-store cleanup has fully converged. + */ + async armCleanupFence(args: { runId: string; blockId: string }): Promise { + await this.#call("runArmCleanupFence", [runBlockKeys(args.runId).state], args.blockId); + } + + /** Read only the existing cleanup fence; never install an empty block for a legacy wait. */ + async readCleanupFence(runId: string): Promise { + const reply = await this.#call("runReadCleanupFence", [runBlockKeys(runId).state]); + return reply[0] || undefined; + } + + /** Lift the cleanup-owed fence, but only when it names this exact block. */ + async clearCleanupFence(args: { runId: string; blockId: string }): Promise { + await this.#call("runClearCleanupFence", [runBlockKeys(args.runId).state], args.blockId); + } + + /** Read at most one hard-bounded, non-consuming page of this block's durable withdrawal targets. */ + async readWithdrawalPage(args: { + runId: string; + blockId: string; + }): Promise<{ blockId: string; fields: string[] } | undefined> { + const reply = await this.#call( + "runWithdrawalPage", + [ + runWithdrawalKey(args.runId, args.blockId), + runWithdrawalTargetsKey(args.runId, args.blockId), + ], + args.blockId, + "read" + ); + return reply[0] === "absent" ? undefined : { blockId: args.blockId, fields: reply.slice(1) }; + } + + /** Acknowledge only targets whose watchers have converged. Duplicate/stale acknowledgements are safe. */ + async acknowledgeWithdrawalPage(args: { + runId: string; + blockId: string; + fields: string[]; + }): Promise { + if (args.fields.length > RUN_CLEANUP_PAGE_SIZE) + throw new Error("Withdrawal page exceeds limit"); + await this.#call( + "runWithdrawalPage", + [ + runWithdrawalKey(args.runId, args.blockId), + runWithdrawalTargetsKey(args.runId, args.blockId), + ], + args.blockId, + "ack", + ...args.fields + ); + } + + /** + * ATOMICALLY retire a converged block's recovery evidence: delete its retained-withdrawal record and, + * in the same same-slot step, clear its cleanup-owed fence when it names this exact block. Called only + * after every retained target is absent/unregistered, so the record and its matching fence can never be + * separated by a process crash. Returns whether the fence was cleared. + */ + async finalizeRetainedWithdrawal(args: { + runId: string; + blockId: string; + }): Promise<{ fenceCleared: boolean }> { + const keys = runBlockKeys(args.runId); + const reply = await this.#call( + "runFinalizeRetainedWithdrawal", + [ + runWithdrawalKey(args.runId, args.blockId), + keys.state, + runWithdrawalTargetsKey(args.runId, args.blockId), + ], + args.blockId + ); + if (reply[0] === "retained") throw new Error("Withdrawal still has unacknowledged targets"); + return { fenceCleared: reply[0] === "cleared" }; + } + + /** Terminal callers hold the parent lock and fence registration before retiring its sources. */ + async retireRunRegistrationSources(runId: string): Promise { + await this.beforeCommand?.(); + for (;;) { + const members = await this.redis.zrange( + runContributionIndexKey(runId), + 0, + RUN_CLEANUP_PAGE_SIZE - 1 + ); + if (members.length === 0) return; + for (const member of members) { + const identity: unknown = JSON.parse(member); + if ( + !Array.isArray(identity) || + identity.length !== 2 || + !identity.every((part) => typeof part === "string") + ) { + throw new Error("Invalid registration contribution identity"); + } + await this.deleteContribution({ runId, blockId: identity[0], absorptionId: identity[1] }); + } + } + } + + /** Record release targets before acquiring source holds. The caller holds the parent lock. */ + async prepareRegistrationSources(args: { + runId: string; + blockId: string; + absorptionId: string; + waitpointIds: string[]; + evidence: string; + expectedPreviousBlockId?: string; + }): Promise { + await this.beforeCommand?.(); + const keys = [ + runBlockKeys(args.runId).state, + runContributionKey(args.runId, args.blockId, args.absorptionId), + runRegistrationSourcesKey(args.runId, args.blockId, args.absorptionId), + runContributionIndexKey(args.runId), + ]; + assertSingleSlot("prepareRegistrationSources", keys); + const prepared = await this.redis.eval( + ` + local current = redis.call('HGET', KEYS[1], 'blk') or '' + local fence = redis.call('HGET', KEYS[1], 'co') or '' + if redis.call('HGET', KEYS[1], 'term') == '1' or + (fence ~= '' and fence ~= ARGV[1]) or + (current ~= ARGV[1] and current ~= ARGV[2]) then return 0 end + if redis.call('SCARD', KEYS[2]) > 0 and + redis.call('SISMEMBER', KEYS[2], ARGV[3]) == 0 then + return redis.error_reply('Registration request does not match its preparation') + end + if redis.call('SISMEMBER', KEYS[2], '!registered') == 1 then return 0 end + redis.call('SADD', KEYS[2], ARGV[3]) + redis.call('ZADD', KEYS[4], 0, ARGV[4]) + for i = 5, #ARGV do redis.call('ZADD', KEYS[3], 0, ARGV[i]) end + return 1 + `, + keys.length, + ...keys, + args.blockId, + args.expectedPreviousBlockId ?? "", + args.evidence, + contributionMember(args.blockId, args.absorptionId), + ...args.waitpointIds + ); + if (prepared !== 1) return; // Absorption still performs the authoritative block CAS. + const token = this.#registrationHoldToken(args); + for (const waitpointId of args.waitpointIds) { + // A missing source may have a retained parent result; the subsequent read decides. + await this.redis.eval( + `if redis.call('EXISTS', KEYS[1]) == 0 then return 0 end + if redis.call('HSETNX', KEYS[1], ARGV[1], '1') == 1 then + redis.call('HINCRBY', KEYS[1], 'rh', 1) + end + redis.call('PERSIST', KEYS[1]) + return 1`, + 1, + waitpointKeys(waitpointId).record, + token + ); + } + } + + #registrationHoldToken(args: { runId: string; blockId: string; absorptionId: string }): string { + return ( + "rh:" + + createHash("sha256") + .update(JSON.stringify([args.runId, args.blockId, args.absorptionId])) + .digest("hex") + ); + } + + /** Keep each target discoverable until its exact hold is released. Replays are harmless. */ + async releaseRegistrationSources(args: { + runId: string; + blockId: string; + absorptionId: string; + }): Promise { + await this.beforeCommand?.(); + const manifest = runRegistrationSourcesKey(args.runId, args.blockId, args.absorptionId); + const token = this.#registrationHoldToken(args); + for (;;) { + const sources = await this.redis.zrange(manifest, 0, RUN_CLEANUP_PAGE_SIZE - 1); + if (sources.length === 0) return; + for (const waitpointId of sources) { + const keys = waitpointKeys(waitpointId); + await this.redis.eval( + `local record, ttl = KEYS[1], tonumber(ARGV[2]) + if redis.call('EXISTS', record) == 0 then return 0 end + if redis.call('HDEL', KEYS[1], ARGV[1]) == 1 then + local remaining = redis.call('HINCRBY', KEYS[1], 'rh', -1) + if remaining == 0 then redis.call('HDEL', KEYS[1], 'rh') end + end + local fanout = redis.call('HGET', KEYS[2], 'state') or '' + if (tonumber(redis.call('HGET', KEYS[1], 'rh')) or 0) == 0 and + redis.call('HGET', KEYS[1], 'status') == 'COMPLETED' and + (fanout == '' or fanout == 'done') and redis.call('PTTL', KEYS[1]) < 0 then +${RETAIN_WAITPOINT_RECORD} + end + return 1`, + 2, + keys.record, + keys.fanout, + token, + String(this.terminalRetentionMs) ); + await this.redis.zrem(manifest, waitpointId); + } + } + } + + /** + * Check the request hash with a fixed number of Redis reads. Do not read or change edge lists. + * Hold the parent run lock across this check and the recovery attempt. + */ + async hasAbsorbedBlockers(args: { + runId: string; + blockId: string; + absorptionId: string; + waitpointIds: string[]; + batchIndex?: number; + }): Promise { + await this.beforeCommand?.(); + const keys = runBlockKeys(args.runId); + const [blockId, terminal] = await this.redis.hmget(keys.state, "blk", "term"); + if (blockId !== args.blockId || terminal === "1") return false; + if ( + !(await this.redis.sismember( + runContributionKey(args.runId, args.blockId, args.absorptionId), + "!accepted" + )) + ) + return false; + return ( + (await this.redis.sismember( + runContributionKey(args.runId, args.blockId, args.absorptionId), + registrationEvidence(args.waitpointIds, args.batchIndex) + )) === 1 + ); + } + + /** Remove cleanup targets after phase 3. Keep the request proof until registration finishes. */ + async markRegistrationComplete(args: { + runId: string; + blockId: string; + absorptionId: string; + evidence: string; + }): Promise { + await this.beforeCommand?.(); + const key = runContributionKey(args.runId, args.blockId, args.absorptionId); + await this.redis.eval( + ` + if redis.call('SISMEMBER', KEYS[1], ARGV[1]) == 0 then + return redis.error_reply('Missing registration acceptance proof') + end + redis.call('UNLINK', KEYS[1]) + redis.call('SADD', KEYS[1], ARGV[1], '!accepted', '!registered') + return 1 + `, + 1, + key, + args.evidence + ); + await this.releaseRegistrationSources(args); + } + + /** The edge fields claimed by one absorption, or [] once registered/compensated. */ + async readContribution(args: { + runId: string; + blockId: string; + absorptionId: string; + }): Promise { + await this.beforeCommand?.(); + return ( + await this.redis.smembers(runContributionKey(args.runId, args.blockId, args.absorptionId)) + ).filter((field) => !field.startsWith("!")); + } + + /** Remove the request proof and its index entry after registration finishes. */ + async deleteContribution(args: { + runId: string; + blockId: string; + absorptionId: string; + }): Promise { + await this.releaseRegistrationSources(args); + await this.#call( + "runDeleteContribution", + [ + runContributionKey(args.runId, args.blockId, args.absorptionId), + runContributionIndexKey(args.runId), + ], + contributionMember(args.blockId, args.absorptionId) + ); + } + + /** + * Roll back one absorption in bounded atomic pages, each fenced to the exact current block. Remove + * only edges with no remaining claims and their now-unreferenced receipts/pending ids; unrelated early receipts + * may still belong to concurrent additive registrations. The last page clears a phantom block unless + * its cleanup fence still owns it. Each page consumes its contribution in the same commit as removal, + * so replay never loses an unfinished target or touches a newer/terminal block. `removed` counts fields + * consumed during this invocation, not earlier pages whose reply was lost. + */ + async compensateAbsorption(args: { + runId: string; + blockId: string; + absorptionId: string; + unacceptedEvidence?: string; + }): Promise<{ outcome: "cleared" | "retained" | "stale"; surviving: number; removed: number }> { + if (args.unacceptedEvidence !== undefined) { + await this.beforeCommand?.(); + const matches = await this.redis.eval( + `return redis.call('SISMEMBER', KEYS[1], '!accepted') == 0 and + redis.call('SISMEMBER', KEYS[1], ARGV[1]) == 1 and 1 or 0`, + 1, + runContributionKey(args.runId, args.blockId, args.absorptionId), + args.unacceptedEvidence + ); + if (matches !== 1) return { outcome: "retained", surviving: 0, removed: 0 }; + } + await this.releaseRegistrationSources(args); + const keys = runBlockKeys(args.runId); + let removed = 0; + for (;;) { + const reply = await this.#call( + "runCompensateAbsorption", + [ + keys.pend, + keys.done, + keys.edge, + keys.state, + runContributionKey(args.runId, args.blockId, args.absorptionId), + keys.refs, + keys.edgeIndex, + keys.doneOrder, + keys.early, + keys.earlyOrder, + runContributionIndexKey(args.runId), + ], + args.blockId, + args.absorptionId, + contributionMember(args.blockId, args.absorptionId) + ); + removed += Number(reply[2]); + if (Number(reply[3]) === 0) { + return { + outcome: reply[0] as "cleared" | "retained" | "stale", + surviving: Number(reply[1]), + removed, + }; } - const stored = JSON.parse(edgeJson) as BlockEdge; - edges.push({ ...stored, edgeId }); } + } - return { pendingIds, deliveredIds, edges, blockId, handoff, terminal }; + /** Fixed-size identity lookup for lifecycle cleanup; never materializes blocker collections. */ + async currentBlockId(runId: string): Promise { + await this.beforeCommand?.(); + return (await this.redis.hget(runBlockKeys(runId).state, "blk")) || undefined; + } + + /** Read fixed-size handoff state, optionally fenced to one block, without blocker collections. */ + async readBlockHandoff(runId: string, expectedBlockId?: string): Promise { + await this.beforeCommand?.(); + const keys = runBlockKeys(runId); + const [blockId, handoff, terminal] = await this.redis.hmget(keys.state, "blk", "hs", "term"); + if (expectedBlockId !== undefined && (blockId !== expectedBlockId || terminal === "1")) { + return "none"; + } + return decodeHandoff(handoff ?? undefined); + } + + async readBlockState(runId: string): Promise { + const keys = runBlockKeys(runId); + for (;;) { + const pendingIds: string[] = [], + deliveredIds: string[] = []; + const seenPending = new Set(), + seenDelivered = new Set(); + const edges: BlockStateEdge[] = []; + let revision = "", + cutoff = "0", + edgeOffset = 0, + earlyAfter = ""; + let snapshot: Pick | undefined; + for (;;) { + const reply = await this.#call( + "runReadBlockState", + [keys.edge, keys.state, keys.edgeIndex, keys.doneOrder, keys.earlyOrder], + snapshot ? "page" : "start", + revision, + cutoff, + String(edgeOffset), + earlyAfter + ); + if (reply[0] === "stale") break; + if (!snapshot) { + revision = reply[1]!; + cutoff = reply[2]!; + snapshot = { + blockId: reply[3] || undefined, + handoff: decodeHandoff(reply[4]), + terminal: reply[5] === "1", + }; + } + const edgeCount = Number(reply[6]), + earlyCount = Number(reply[7]); + let cursor = 8; + for (let i = 0; i < edgeCount; i++, cursor += 3) { + const edgeJson = reply[cursor + 1]; + if (!edgeJson) throw new Error(`readBlockState(${runId}): missing edge payload`); + const stored = JSON.parse(edgeJson) as BlockEdge; + edges.push({ ...stored, edgeId: reply[cursor]! }); + const delivered = reply[cursor + 2] === "1"; + const seen = delivered ? seenDelivered : seenPending; + if (!seen.has(stored.waitpointId)) { + seen.add(stored.waitpointId); + (delivered ? deliveredIds : pendingIds).push(stored.waitpointId); + } + } + for (let i = 0; i < earlyCount; i++) { + const id = reply[cursor + i * 2]!; + if (!seenDelivered.has(id)) { + seenDelivered.add(id); + deliveredIds.push(id); + } + earlyAfter = reply[cursor + i * 2 + 1]!; + } + edgeOffset += edgeCount; + if (edgeCount < RUN_CLEANUP_PAGE_SIZE && earlyCount < RUN_CLEANUP_PAGE_SIZE) { + return { ...snapshot, pendingIds, deliveredIds, edges }; + } + } + } } /** @@ -1567,8 +2259,10 @@ export class WaitpointStoreCoordinator { * The selective form RECONCILES: any pending or delivered entry that no surviving edge * references goes too, not only the named ones. See runClear in scripts.ts for why. * - * This is the low-level state operation. It does NOT consult the handoff obligation — - * `cleanupRunBlockState` is the lifecycle gate that does. + * This low-level exact-block operation does not consult the handoff obligation. A full clear + * also removes the cleanup fence. Its caller must already own lifecycle and legacy-edge cleanup; + * failed-attempt recovery instead retains withdrawal evidence and finalizes the fence last. + * `cleanupRunBlockState` supplies the handoff gate for its lifecycle callers. */ async clearBlockState(args: { runId: string; @@ -1596,16 +2290,33 @@ export class WaitpointStoreCoordinator { const keys = runBlockKeys(args.runId); const edgeIds = args.edgeIds ?? []; - const reply = await this.#call( - "runClear", - [keys.pend, keys.done, keys.edge, keys.state], - args.blockId, - String(edgeIds.length), - ...edgeIds - ); - - const outcome = reply[0] as "cleared" | "drained" | "stale" | "terminal"; - return { outcome, currentBlockId: reply[1] || undefined }; + let result: Awaited>; + for (let offset = 0; offset < Math.max(1, edgeIds.length); offset += RUN_CLEANUP_PAGE_SIZE) { + const page = edgeIds.slice(offset, offset + RUN_CLEANUP_PAGE_SIZE); + const reply = await this.#call( + "runClear", + [ + keys.pend, + keys.done, + keys.edge, + keys.state, + keys.refs, + keys.edgeIndex, + keys.doneOrder, + keys.early, + keys.earlyOrder, + ], + args.blockId, + String(page.length), + ...page + ); + result = { + outcome: reply[0] as "cleared" | "drained" | "stale" | "terminal", + currentBlockId: reply[1] || undefined, + }; + if (result.outcome === "stale" || result.outcome === "terminal") return result; + } + return result!; } // --------------------------------------------------------------------------- @@ -1659,16 +2370,17 @@ export class WaitpointStoreCoordinator { throw new Error(`claimFanoutPage(${args.waitpointId}): claim returned no fence token`); } - const pageLength = Number(reply[7]); + const completedByTaskRunId = reply[7] || undefined; + const pageLength = Number(reply[8]); const page: FanoutPageEntry[] = []; for (let i = 0; i < pageLength; i++) { - const field = reply[8 + i * 2]; + const field = reply[9 + i * 2]; if (field === undefined) { throw new Error( - `claimFanoutPage(${args.waitpointId}): missing watcher field at reply index ${8 + i * 2}` + `claimFanoutPage(${args.waitpointId}): missing watcher field at reply index ${9 + i * 2}` ); } - page.push({ field, watcher: parseJson(reply[8 + i * 2 + 1]) }); + page.push({ field, watcher: parseJson(reply[9 + i * 2 + 1]) }); } return { @@ -1679,6 +2391,8 @@ export class WaitpointStoreCoordinator { completedAtMs: Number(reply[4]), failures: Number(reply[5]), epoch, + completedByTaskRunId, + record: parseJson(reply[9 + pageLength * 2]), page, }; } @@ -1909,12 +2623,33 @@ export class WaitpointStoreCoordinator { * signal to fall back to Postgres. A Redis error THROWS out of the `HMGET` and is never seen here * as an absent record, so an availability failure can never be mistaken for a missing waitpoint. */ - async readCompletedRecords(ids: string[]): Promise> { + async readCompletedRecords( + ids: string[], + runId?: string + ): Promise> { const out = new Map(); const distinct = [...new Set(ids)]; + await this.beforeCommand?.(); + // Parent snapshot results have no active TTL and survive block acknowledgement/rollover. + // Read only the IDs owned by the snapshot; original waitpoint records may have expired. + if (runId && distinct.length > 0) { + const receipts = await this.redis.hmget(runResultKey(runId), ...distinct); + for (let i = 0; i < distinct.length; i++) { + const receipt = parseJson( + receipts[i] ?? undefined + ); + if (!receipt?.record) continue; + const { record, ...completion } = receipt; + if (record.id !== distinct[i]) + throw new Error(`Parent result has mismatched waitpoint identity: ${distinct[i]}`); + out.set(distinct[i]!, { status: "COMPLETED", record, completion }); + } + } + await pMap( distinct, async (id) => { + if (out.has(id)) return; const keys = waitpointKeys(id); const [r, status, c] = await this.redis.hmget(keys.record, "r", "status", "c"); // No 'r' means no record hash under this id at all — absent, not an error. diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeWaitpointCoordinator.blockProtocol.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeWaitpointCoordinator.blockProtocol.test.ts new file mode 100644 index 00000000000..81ef4f25445 --- /dev/null +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeWaitpointCoordinator.blockProtocol.test.ts @@ -0,0 +1,616 @@ +import { registrationEvidence } from "./keys.js"; +// Test the three-phase protocol with the real router, adapter, Redis and Postgres. +// Cover block identity, resume acknowledgement, cleanup and fanout. +// Restart tests reconstruct registration from saved data. They do not run production recovery jobs; +// those jobs are tested in the engine integration suite. + +import { containerTest } from "@internal/testcontainers"; +import { generateWaitpointId, WaitpointId } from "@trigger.dev/core/v3/isomorphic"; +import type { PrismaClient } from "@trigger.dev/database"; +import { PostgresRunStore } from "@internal/run-store"; +import { Logger } from "@trigger.dev/core/logger"; +import { LegacyPostgresWaitpointCoordinator } from "./legacyPostgresCoordinator.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, +} from "../../../../run-store/src/testFixtures/snapshotIdFixture.js"; +import { describe, expect } from "vitest"; +import { StoreWaitpointBlockOperationIdentityRequired } from "../errors.js"; + +import { RoutingWaitpointCoordinator } from "./routingCoordinator.js"; +import { + StoreWaitpointBlockRefusedError, + StoreWaitpointCoordinator, +} from "./storeWaitpointCoordinator.js"; +import { WaitpointStoreCoordinator } from "./storeCoordinator.js"; +import type { + AbsorbBlockersParams, + BlockerRegistration, + StoreBlockerRegistration, +} from "./types.js"; + +function buildRouting( + store: WaitpointStoreCoordinator, + prisma: PrismaClient, + opts?: { clock?: () => number; markStoreWaitpoints?: () => Promise } +) { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const legacy = new LegacyPostgresWaitpointCoordinator({ + runStore: () => runStore, + prisma, + logger: new Logger("block-protocol-test"), + }); + const adapter = new StoreWaitpointCoordinator({ store, clock: opts?.clock }); + const routing = new RoutingWaitpointCoordinator({ + legacy, + store: adapter, + mintPolicy: { resolveMintMode: () => "redis", isCapabilityReady: () => true }, + mightHaveStoreBlockers: () => true, + markStoreWaitpoints: opts?.markStoreWaitpoints, + }); + return { routing, adapter }; +} + +// Count live watchers. Fanout queue length is not a watcher count. +async function liveWatchers( + store: WaitpointStoreCoordinator, + waitpointId: string +): Promise { + const d = await store.describeWaitpoint(waitpointId); + return d.outcome === "exists" ? d.liveWatchers : 0; +} + +// Create a pending waitpoint before registering its watcher. +async function seedPending( + store: WaitpointStoreCoordinator, + waitpointId: string, + completedByTaskRunId?: string +): Promise { + const now = new Date().toISOString(); + await store.createIfAbsent({ + record: { + id: waitpointId, + friendlyId: WaitpointId.toFriendlyId(waitpointId), + type: "MANUAL", + environmentId: "env_1", + projectId: "proj_1", + createdAt: now, + updatedAt: now, + userProvidedIdempotencyKey: false, + tags: [], + completedByTaskRunId, + }, + status: "PENDING", + }); +} + +const ORG = "org_1"; + +const absorbArgs = ( + client: PrismaClient, + runId: string, + blockId: string, + waitpointIds: string[], + over: Partial = {} +): AbsorbBlockersParams => ({ + runId, + blockId, + absorptionId: blockId, + organizationId: ORG, + waitpointIds, + projectId: "proj_1", + client, + parentResidency: "redis-primary", + ...over, +}); + +// Test helper: rebuild registration from saved data after a simulated crash. +// Production recovery jobs are covered by the engine integration tests. +async function reconstructFromDurableState( + store: WaitpointStoreCoordinator, + runId: string, + organizationId?: string +): Promise { + const state = await store.readBlockState(runId); + if (!state.blockId) throw new Error(`no durable block state to reconstruct for ${runId}`); + const registration: StoreBlockerRegistration = { + kind: "store", + registrationEvidence: registrationEvidence( + state.edges.map((e) => e.waitpointId), + state.edges[0]?.batchIndex ?? undefined + ), + runId, + blockId: state.blockId, + absorptionId: state.blockId, + organizationId, + edges: state.edges.map((e) => ({ + waitpointId: e.waitpointId, + batchId: e.batchId ?? undefined, + batchIndex: e.batchIndex ?? undefined, + spanIdToComplete: e.spanIdToComplete, + createdAt: e.createdAt, + type: e.type, + })), + storePendingTotal: state.pendingIds.length, + alreadyDelivered: [], + }; + return { kind: "routing", parts: [registration] }; +} + +const completionOf = (value: string) => ({ + completedAt: new Date().toISOString(), + outputType: "application/json", + outputIsError: false, + output: { inline: value }, +}); + +// Publish a real Postgres blocked snapshot between the two store protocol phases. +function transitionMarker(prisma: PrismaClient, runId: string) { + return { + async publish() { + const env = await seedSnapshotEnvironment(prisma); + await new PostgresRunStore({ prisma, readOnlyPrisma: prisma }).createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + engine: "V2", + executionStatus: "SUSPENDED", + description: "blocked transition", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + return env; + }, + async published() { + return (await prisma.taskRunExecutionSnapshot.count({ where: { runId } })) > 0; + }, + }; +} + +describe("Three-phase block protocol and lifecycle (real store adapter through the routing seam)", () => { + containerTest( + "compensation preserves a later registration of the same edge and releases failed claims", + async ({ prisma, redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions }); + const { adapter } = buildRouting(store, prisma); + try { + for (const laterSucceeds of [true, false]) { + const runId = `run_shared_claim_${laterSucceeds}`; + const wp = generateWaitpointId("MANUAL"); + await seedPending(store, wp); + const request = (absorptionId: string) => + absorbArgs(prisma, runId, "B1", [wp], { absorptionId }); + + // A1's store write survives a failed legacy write and failed inline compensation. + await adapter.absorbBlockers(request("A1")); + // A lost reply must not add a second claim for the same operation. + await adapter.absorbBlockers(request("A1")); + const second = await adapter.absorbBlockers(request("A2")); + if (laterSucceeds) { + await adapter.registerBlockers(second.registration); + await adapter.finalizeBlockers({ ...request("A2"), blockId: "B1", absorptionId: "A2" }); + } + + await adapter.compensateAbsorption({ runId, blockId: "B1", absorptionId: "A1" }); + await adapter.compensateAbsorption({ runId, blockId: "B1", absorptionId: "A1" }); + const retained = await store.readBlockState(runId); + expect(retained.blockId).toBe("B1"); + expect(retained.edges).toHaveLength(1); + expect(retained.pendingIds).toEqual([wp]); + + if (laterSucceeds) { + await adapter.complete({ waitpointId: wp, output: { value: "ok", isError: false } }); + const delivered = await store.registerAndDeliver({ + runId, + blockId: "B1", + edges: second.registration.kind === "store" ? second.registration.edges : [], + absorbed: { storePendingTotal: 1, alreadyDelivered: [] }, + }); + expect(delivered.pendingOfRequested).toBe(0); + expect((await store.readBlockState(runId)).pendingIds).toEqual([]); + } else { + await adapter.compensateAbsorption({ runId, blockId: "B1", absorptionId: "A2" }); + const cleared = await store.readBlockState(runId); + expect(cleared.blockId).toBeUndefined(); + expect(cleared.edges).toEqual([]); + expect(cleared.pendingIds).toEqual([]); + } + } + } finally { + await store.quit(); + } + } + ); + + containerTest( + "recovery rejects an unaccepted mixed request sharing a native edge and retains accepted duplicate tail proof", + async ({ prisma, redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions }); + let failMarker = false; + const { routing } = buildRouting(store, prisma, { + markStoreWaitpoints: async () => { + if (failMarker) throw new Error("marker unavailable"); + }, + }); + const runId = "run_exact_registration"; + try { + // Add the existing production partial index. The Prisma-only fixture omits it. + // Nonbatch retries must use the same uniqueness rule as production. + await prisma.$executeRaw` + CREATE UNIQUE INDEX IF NOT EXISTS "TaskRunWaitpoint_taskRunId_waitpointId_batchIndex_null_key" + ON "TaskRunWaitpoint"("taskRunId", "waitpointId") WHERE "batchIndex" IS NULL + `; + const env = await transitionMarker(prisma, runId).publish(); + const legacyWp = ( + await prisma.waitpoint.create({ + data: { + friendlyId: "waitpoint_exact_registration", + type: "MANUAL", + status: "PENDING", + projectId: env.projectId, + environmentId: env.id, + idempotencyKey: "exact_registration", + userProvidedIdempotencyKey: false, + }, + }) + ).id; + const native = generateWaitpointId("RUN"); + await seedPending(store, native); + const request = (absorptionId: string, ids = [native, legacyWp]) => + absorbArgs(prisma, runId, "B1", ids, { + absorptionId, + projectId: env.projectId, + }); + const accepted = await routing.absorbBlockers(request("A", [native])); + await routing.registerBlockers(accepted.registration); + await routing.finalizeBlockers({ + ...request("A", [native]), + blockId: "B1", + absorptionId: "A", + }); + failMarker = true; + await expect(routing.absorbBlockers(request("C"))).rejects.toThrow("marker unavailable"); + const recovery = { ...request("C"), blockId: "B1", absorptionId: "C" }; + failMarker = false; + // Use the recovery check. A rejected request must not add the Postgres wait L. + if (await routing.hasAbsorbedBlockers(recovery)) { + await routing.absorbBlockers(request("C")); + } + expect(await routing.hasAbsorbedBlockers(recovery)).toBe(false); + expect( + await prisma.taskRunWaitpoint.count({ + where: { taskRunId: runId, waitpointId: legacyWp }, + }) + ).toBe(0); + + failMarker = false; + // Accepting an existing edge still needs proof for this request. + const duplicate = await routing.absorbBlockers(request("D")); + await routing.registerBlockers(duplicate.registration); + const duplicateRecovery = { ...request("D"), blockId: "B1", absorptionId: "D" }; + expect(await routing.hasAbsorbedBlockers(duplicateRecovery)).toBe(true); + expect( + await routing.hasAbsorbedBlockers({ ...duplicateRecovery, waitpointIds: [native] }) + ).toBe(false); + // Late cleanup must not revoke proof after phase 3 succeeds. + await routing.compensateAbsorption(duplicateRecovery); + expect(await routing.hasAbsorbedBlockers(duplicateRecovery)).toBe(true); + await routing.registerBlockers((await routing.absorbBlockers(request("D"))).registration); + expect(await liveWatchers(store, native)).toBe(1); + expect( + await prisma.taskRunWaitpoint.count({ + where: { taskRunId: runId, waitpointId: legacyWp }, + }) + ).toBe(1); + await routing.finalizeBlockers(duplicateRecovery); + expect(await routing.hasAbsorbedBlockers(duplicateRecovery)).toBe(false); + + // An undone request stays rejected even if another operation has the same edge. + await routing.absorbBlockers(request("compensated", [native])); + await routing.compensateAbsorption({ ...recovery, absorptionId: "compensated" }); + await routing.absorbBlockers(request("other", [native])); + expect( + await routing.hasAbsorbedBlockers({ + ...recovery, + absorptionId: "compensated", + waitpointIds: [native], + }) + ).toBe(false); + } finally { + await store.quit(); + } + } + ); + + containerTest( + "ORDERING: absorb installs run-side pending but registers NO watcher until phase 3", + async ({ prisma, redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions }); + const { routing } = buildRouting(store, prisma); + try { + const runId = "run_order"; + const wp = generateWaitpointId("MANUAL"); + await seedPending(store, wp); + const marker = transitionMarker(prisma, runId); + + const { registration } = await routing.absorbBlockers( + absorbArgs(prisma, runId, "B1", [wp]) + ); + expect((await store.readBlockState(runId)).pendingIds).toEqual([wp]); + expect(await marker.published()).toBe(false); + expect(await liveWatchers(store, wp)).toBe(0); + + await marker.publish(); + const { pendingCount } = await routing.registerBlockers(registration); + expect(pendingCount).toBe(1); + expect(await liveWatchers(store, wp)).toBe(1); + } finally { + await store.quit(); + } + } + ); + + containerTest( + "ORG THREADING: the run's org lands on the store watcher, for the async continuation", + async ({ prisma, redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions }); + const { routing } = buildRouting(store, prisma); + try { + const runId = "run_org"; + const wp = generateWaitpointId("MANUAL"); + await seedPending(store, wp); + + const { registration } = await routing.absorbBlockers( + absorbArgs(prisma, runId, "B1", [wp]) + ); + await routing.registerBlockers(registration); + + // Check that the watcher carries its organisation for later routing. + const claim = await store.claimFanoutPage({ + waitpointId: wp, + workerId: "probe", + pageSize: 10, + leaseMs: 10_000, + }); + // Complete the pending waitpoint to read the watcher from the fanout result. + expect(claim.outcome).toBe("pending-record"); + await store.complete({ waitpointId: wp, completion: completionOf("1") }); + const claimed = await store.claimFanoutPage({ + waitpointId: wp, + workerId: "probe", + pageSize: 10, + leaseMs: 10_000, + }); + expect(claimed.outcome).toBe("claimed"); + if (claimed.outcome === "claimed") { + expect(claimed.page[0]?.watcher?.organizationId).toBe(ORG); + } + } finally { + await store.quit(); + } + } + ); + + containerTest( + "EARLY COMPLETION: a completion in the absorb→register gap is reported and delivered", + async ({ prisma, redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions }); + const { routing } = buildRouting(store, prisma); + try { + const runId = "run_early"; + const wp = generateWaitpointId("MANUAL"); + await seedPending(store, wp); + + const { registration } = await routing.absorbBlockers( + absorbArgs(prisma, runId, "B1", [wp]) + ); + const completed = await routing.complete({ + waitpointId: wp, + output: { value: '{"ok":true}', isError: false }, + }); + expect(completed.waitpoint.output).toBe('{"ok":true}'); + + const { pendingCount } = await routing.registerBlockers(registration); + expect(pendingCount).toBe(0); // reported complete and delivered, not stuck + const state = await store.readBlockState(runId); + expect(state.pendingIds).toEqual([]); + expect(state.deliveredIds).toEqual([wp]); + } finally { + await store.quit(); + } + } + ); + + containerTest( + "MIXED RESIDENCY: legacy + store blockers combine counts and route by shape", + async ({ prisma, redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions }); + const { routing } = buildRouting(store, prisma); + try { + const runId = "run_mixed"; + const env = await transitionMarker(prisma, runId).publish(); + const legacyWp = ( + await prisma.waitpoint.create({ + data: { + friendlyId: "waitpoint_legacy_mixed", + type: "MANUAL", + status: "PENDING", + projectId: env.projectId, + environmentId: env.id, + idempotencyKey: "legacy_mixed", + userProvidedIdempotencyKey: false, + }, + }) + ).id; + const storeWp = generateWaitpointId("MANUAL"); // b32hexW → store + await seedPending(store, storeWp); + + const { pendingCount, registration } = await routing.absorbBlockers( + absorbArgs(prisma, runId, "B1", [legacyWp, storeWp], { + client: prisma, + projectId: env.projectId, + }) + ); + expect(pendingCount).toBe(2); + expect( + await prisma.taskRunWaitpoint.count({ + where: { taskRunId: runId, waitpointId: legacyWp }, + }) + ).toBe(1); + expect((await store.readBlockState(runId)).pendingIds).toEqual([storeWp]); + + await routing.registerBlockers(registration); + await routing.complete({ waitpointId: storeWp, output: { value: "x", isError: false } }); + const diag = await store.describeWaitpoint(storeWp); + expect(diag.outcome === "exists" && diag.status).toBe("COMPLETED"); + } finally { + await store.quit(); + } + } + ); + + containerTest( + "CAS: block1 drains, block2 gets a fresh id; a delayed block1 absorb is refused stale without mutating block2", + async ({ prisma, redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions }); + const { routing } = buildRouting(store, prisma); + try { + const runId = "run_cas"; + const wp1 = generateWaitpointId("MANUAL"); + const wp2 = generateWaitpointId("MANUAL"); + await seedPending(store, wp1); + await seedPending(store, wp2); + + // Retry the same operation on block 1. Do not add duplicate edges. + await routing.absorbBlockers(absorbArgs(prisma, runId, "B1", [wp1])); + await routing.absorbBlockers(absorbArgs(prisma, runId, "B1", [wp1])); + let state = await store.readBlockState(runId); + expect(state.blockId).toBe("B1"); + expect(state.pendingIds).toEqual([wp1]); + + // Complete block 1 and acknowledge its resume. + const { registration: reg1 } = await routing.absorbBlockers( + absorbArgs(prisma, runId, "B1", [wp1]) + ); + await routing.registerBlockers(reg1); + await routing.complete({ waitpointId: wp1, output: { value: "1", isError: false } }); + await store.deliverCompletion({ + runId, + blockId: "B1", + waitpointId: wp1, + completion: completionOf("1"), + }); + const obs = await routing.currentStoreBlockId(runId, { organizationId: ORG }); + await routing.acknowledgeResume({ runId, blockId: obs, organizationId: ORG }); + expect((await store.readBlockState(runId)).blockId).toBeUndefined(); // drained + + // Create block 2 with a new ID after block 1 has drained. + await routing.absorbBlockers(absorbArgs(prisma, runId, "B2", [wp2])); + state = await store.readBlockState(runId); + expect(state.blockId).toBe("B2"); + expect(state.pendingIds).toEqual([wp2]); + + // Reject a delayed block-1 request without changing block 2. + await expect( + routing.absorbBlockers(absorbArgs(prisma, runId, "B1", [wp1])) + ).rejects.toBeInstanceOf(StoreWaitpointBlockRefusedError); + state = await store.readBlockState(runId); + expect(state.blockId).toBe("B2"); + expect(state.pendingIds).toEqual([wp2]); + } finally { + await store.quit(); + } + } + ); + + containerTest( + "RESTART after absorb, before the transition: recovery reconstructs the block from durable records and converges", + async ({ prisma, redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions }); + const { routing } = buildRouting(store, prisma); + try { + const runId = "run_restart1"; + const wp = generateWaitpointId("MANUAL"); + await seedPending(store, wp); + const marker = transitionMarker(prisma, runId); + + // Simulate a crash after absorption. Discard the in-memory registration. + await routing.absorbBlockers(absorbArgs(prisma, runId, "B1", [wp])); + // (registration deliberately dropped) + + // Only the pending block is saved. No blocked snapshot or watcher exists. + expect((await store.readBlockState(runId)).pendingIds).toEqual([wp]); + expect(await marker.published()).toBe(false); + expect(await liveWatchers(store, wp)).toBe(0); + + // Rebuild from saved data, publish the snapshot, then register watchers. + const recovered = await reconstructFromDurableState(store, runId, ORG); + await marker.publish(); + const { pendingCount } = await routing.registerBlockers(recovered); + expect(pendingCount).toBe(1); + expect(await liveWatchers(store, wp)).toBe(1); + } finally { + await store.quit(); + } + } + ); + + containerTest( + "RESTART after the transition, before registration: recovery reconstructs from durable records and registers", + async ({ prisma, redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions }); + const { routing } = buildRouting(store, prisma); + try { + const runId = "run_restart2"; + const wp = generateWaitpointId("MANUAL"); + await seedPending(store, wp); + const marker = transitionMarker(prisma, runId); + + await routing.absorbBlockers(absorbArgs(prisma, runId, "B1", [wp])); + await marker.publish(); + // Simulate a crash after the snapshot write, before watcher registration. + + expect(await marker.published()).toBe(true); + expect((await store.readBlockState(runId)).pendingIds).toEqual([wp]); + expect(await liveWatchers(store, wp)).toBe(0); + + const recovered = await reconstructFromDurableState(store, runId, ORG); + const { pendingCount } = await routing.registerBlockers(recovered); + expect(pendingCount).toBe(1); + expect(await liveWatchers(store, wp)).toBe(1); + + // A later completion delivers through the durable watcher. + await routing.complete({ waitpointId: wp, output: { value: "ok", isError: false } }); + const diag = await store.describeWaitpoint(wp); + expect(diag.outcome === "exists" && diag.status).toBe("COMPLETED"); + } finally { + await store.quit(); + } + } + ); + + containerTest( + "FAIL LOUD: a store-format blocker with no caller-owned identity is refused before any coordinator is mutated", + async ({ prisma, redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions }); + const { routing } = buildRouting(store, prisma); + try { + const runId = "run_noidentity"; + const wp = generateWaitpointId("MANUAL"); + await seedPending(store, wp); + + // Call without the required block identity. + const { blockId: _drop, ...noIdentity } = absorbArgs(prisma, runId, "unused", [wp]); + await expect( + routing.absorbBlockers(noIdentity as AbsorbBlockersParams) + ).rejects.toBeInstanceOf(StoreWaitpointBlockOperationIdentityRequired); + expect((await store.readBlockState(runId)).blockId).toBeUndefined(); + } finally { + await store.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeWaitpointCoordinator.preDeliveryExpiry.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeWaitpointCoordinator.preDeliveryExpiry.test.ts new file mode 100644 index 00000000000..f79a823ef88 --- /dev/null +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeWaitpointCoordinator.preDeliveryExpiry.test.ts @@ -0,0 +1,359 @@ +import { createRedisClient } from "@internal/redis"; +import { containerTest, redisTest } from "@internal/testcontainers"; +import { generateWaitpointId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect } from "vitest"; +import { + registrationEvidence, + runContributionIndexKey, + runRegistrationSourcesKey, + runResultKey, + waitpointKeys, +} from "./keys.js"; +import { WaitpointFanoutWorker } from "./fanoutWorker.js"; +import { StoreWaitpointCoordinator } from "./storeWaitpointCoordinator.js"; +import { WaitpointStoreCoordinator, type WaitpointRecordInput } from "./storeCoordinator.js"; + +describe("completed sources before delivery", () => { + redisTest( + "pending absorption retains its source until registration and then releases it", + async ({ redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions, terminalRetentionMs: 50 }); + const redis = createRedisClient(redisOptions); + const adapter = new StoreWaitpointCoordinator({ store }); + const id = generateWaitpointId("DATETIME"); + const now = new Date().toISOString(); + const args = { + runId: "pending-expiry", + blockId: "block", + absorptionId: "operation", + organizationId: "org", + projectId: "project", + waitpointIds: [id], + client: undefined as never, + }; + try { + await store.createIfAbsent({ + record: { + id, + friendlyId: `waitpoint_${id}`, + type: "DATETIME", + environmentId: "env", + projectId: "project", + createdAt: now, + updatedAt: now, + completedAfter: now, + userProvidedIdempotencyKey: false, + tags: [], + }, + status: "PENDING", + }); + await adapter.absorbBlockers(args); + await store.complete({ + waitpointId: id, + completion: { + completedAt: now, + outputType: "application/json", + outputIsError: false, + output: null, + }, + }); + expect(await redis.pttl(waitpointKeys(id).record)).toBe(-1); + // Exceed terminal retention while no watcher is installed. + await new Promise((resolve) => setTimeout(resolve, 100)); + const retry = await adapter.absorbBlockers(args); + expect((await adapter.registerBlockers(retry.registration)).pendingCount).toBe(0); + expect( + await redis.exists(runRegistrationSourcesKey(args.runId, args.blockId, args.absorptionId)) + ).toBe(0); + await expect.poll(() => redis.exists(waitpointKeys(id).record)).toBe(0); + // A lost phase-3 reply still replays from the retained result after source expiry. + const recovered = await adapter.absorbBlockers(args); + expect((await adapter.registerBlockers(recovered.registration)).pendingCount).toBe(0); + expect((await store.readCompletedRecords([id], args.runId)).get(id)?.status).toBe( + "COMPLETED" + ); + await adapter.finalizeBlockers(args); + expect(await redis.zcard(runContributionIndexKey(args.runId))).toBe(0); + } finally { + await redis.quit(); + await store.quit(); + } + } + ); + + redisTest( + "unaccepted preparation cleanup releases only its own source hold", + async ({ redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions, terminalRetentionMs: 50 }); + const redis = createRedisClient(redisOptions); + const id = generateWaitpointId("DATETIME"); + const now = new Date().toISOString(); + const args = { + runId: "prepared-expiry", + blockId: "block", + absorptionId: "A1", + waitpointIds: [id], + evidence: registrationEvidence([id]), + }; + try { + await store.createIfAbsent({ + record: { + id, + friendlyId: `waitpoint_${id}`, + type: "DATETIME", + environmentId: "env", + projectId: "project", + createdAt: now, + updatedAt: now, + completedAfter: now, + userProvidedIdempotencyKey: false, + tags: [], + }, + status: "PENDING", + }); + // Crash after source acquisition but before run absorption; both operations remain discoverable. + await store.prepareRegistrationSources(args); + await store.prepareRegistrationSources({ ...args, absorptionId: "A2" }); + await store.registerOrReport({ + waitpointId: id, + runId: "other-parent", + blockId: "other-block", + createdAt: now, + }); + await store.complete({ + waitpointId: id, + completion: { + completedAt: now, + outputType: "application/json", + outputIsError: false, + output: null, + }, + }); + await store.compensateAbsorption({ ...args, unacceptedEvidence: "!wrong-request" }); + expect(await redis.hget(waitpointKeys(id).record, "rh")).toBe("2"); + await store.compensateAbsorption({ ...args, unacceptedEvidence: args.evidence }); + await store.compensateAbsorption({ ...args, unacceptedEvidence: args.evidence }); + expect(await redis.hget(waitpointKeys(id).record, "rh")).toBe("1"); + const fanout = new WaitpointFanoutWorker({ coordinator: store, enabled: true }); + try { + expect((await fanout.visit(id)).outcome).toBe("drained"); + } finally { + await fanout.stop(); + } + expect(await redis.pttl(waitpointKeys(id).record)).toBe(-1); + await store.deleteContribution({ ...args, absorptionId: "A2" }); + expect(await redis.zcard(runContributionIndexKey(args.runId))).toBe(0); + await expect.poll(() => redis.exists(waitpointKeys(id).record)).toBe(0); + } finally { + await redis.quit(); + await store.quit(); + } + } + ); + + redisTest( + "terminal cleanup releases interrupted registration sources and can be replayed", + async ({ redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions, terminalRetentionMs: 50 }); + const redis = createRedisClient(redisOptions); + const adapter = new StoreWaitpointCoordinator({ store }); + const id = generateWaitpointId("DATETIME"); + const now = new Date().toISOString(); + const args = { + runId: "terminal-source", + blockId: "block", + absorptionId: "operation", + organizationId: "org", + projectId: "project", + waitpointIds: [id], + client: undefined as never, + }; + try { + await store.createIfAbsent({ + record: { + id, + friendlyId: `waitpoint_${id}`, + type: "DATETIME", + environmentId: "env", + projectId: "project", + createdAt: now, + updatedAt: now, + completedAfter: now, + userProvidedIdempotencyKey: false, + tags: [], + }, + status: "PENDING", + }); + await adapter.absorbBlockers(args); + await store.complete({ + waitpointId: id, + completion: { + completedAt: now, + outputType: "application/json", + outputIsError: false, + output: null, + }, + }); + expect(await redis.pttl(waitpointKeys(id).record)).toBe(-1); + // A failed cross-slot release keeps its manifest for the next terminal attempt. + const manifest = runRegistrationSourcesKey(args.runId, args.blockId, args.absorptionId); + await redis.rename(manifest, `${manifest}:saved`); + await redis.set(manifest, "wrong-type"); + await expect( + store.releaseRunWatchers({ runId: args.runId, blockId: args.blockId, reason: "terminal" }) + ).rejects.toThrow(); + expect(await redis.zcard(runContributionIndexKey(args.runId))).toBe(1); + await redis.del(manifest); + await redis.rename(`${manifest}:saved`, manifest); + await store.releaseRunWatchers({ + runId: args.runId, + blockId: args.blockId, + reason: "terminal", + }); + await store.releaseRunWatchers({ + runId: args.runId, + blockId: args.blockId, + reason: "terminal", + }); + expect(await redis.exists(manifest)).toBe(0); + expect(await redis.zcard(runContributionIndexKey(args.runId))).toBe(0); + await expect.poll(() => redis.exists(waitpointKeys(id).record)).toBe(0); + } finally { + await redis.quit(); + await store.quit(); + } + } + ); + + containerTest( + "absorption retains completed sources before first registration and replays after their actual expiry", + async ({ prisma, redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions }); + const redis = createRedisClient(redisOptions); + const now = new Date().toISOString(); + try { + for (const type of ["DATETIME", "RUN"] as const) { + const id = generateWaitpointId(type); + const record: WaitpointRecordInput = { + id, + friendlyId: `waitpoint_${id}`, + type, + environmentId: "env", + projectId: "project", + createdAt: now, + updatedAt: now, + completedAfter: type === "DATETIME" ? now : undefined, + completedByTaskRunId: type === "RUN" ? "cached-child" : undefined, + userProvidedIdempotencyKey: true, + idempotencyKey: "shared-token", + tags: ["retained"], + }; + const completion = { + completedAt: now, + outputType: "application/json", + outputIsError: type === "RUN", + output: { inline: "" }, + }; + await store.createIfAbsent({ record, status: "COMPLETED", completion }); + const args = { + client: prisma, + runId: `parent-${type}`, + blockId: "block", + absorptionId: "absorption", + organizationId: "org", + projectId: "project", + waitpointIds: [id], + spanIdToComplete: "span", + batchIndex: 4, + }; + const adapter = new StoreWaitpointCoordinator({ store }); + const first = await adapter.absorbBlockers(args); + if (first.registration.kind !== "store") throw new Error("Expected store continuation"); + expect(await redis.hlen(waitpointKeys(id).watchers)).toBe(0); + // Simulate source expiry after the snapshot write but before watcher registration. + // No receipt or watcher exists yet. + await redis.pexpire(waitpointKeys(id).record, 1); + await expect + .poll(() => redis.exists(waitpointKeys(id).record), { + timeout: 5_000, + interval: 20, + message: "completed source record never expired", + }) + .toBe(0); + expect((await store.readCompletedRecords([id], args.runId)).get(id)).toEqual({ + status: "COMPLETED", + record, + completion, + }); + expect(await redis.pttl(runResultKey(args.runId))).toBe(-1); + expect((await store.readBlockState(args.runId)).handoff).toBe("owed"); + // Test source expiry with both the original registration and recovery. + expect((await adapter.registerBlockers(first.registration)).pendingCount).toBe(0); + const restarted = new StoreWaitpointCoordinator({ store }); + const replay = await restarted.absorbBlockers(args); + if (replay.registration.kind !== "store") + throw new Error("Expected store replay continuation"); + expect(await restarted.registerBlockers(replay.registration)).toEqual({ + pendingCount: 0, + cachedSpanCompletions: [ + { + spanId: "span", + spanCreatedAt: replay.registration.edges[0]!.createdAt, + blockedRunId: args.runId, + hasError: completion.outputIsError, + cachedRunId: record.completedByTaskRunId, + }, + ], + }); + const state = await store.readBlockState(args.runId); + expect(state.pendingIds).toEqual([]); + expect(state.edges[0]?.completedAfter).toBe(record.completedAfter); + await store.acknowledgeResumeHandoff({ runId: args.runId, blockId: args.blockId }); + await restarted.absorbBlockers(args); + expect((await store.readBlockState(args.runId)).handoff).toBe("acked"); + // Remove the old result expiry when reusing results in an active cycle. + await redis.pexpire(runResultKey(args.runId), 60_000); + await restarted.absorbBlockers({ + ...args, + blockId: "next", + absorptionId: "next", + expectedPreviousBlockId: args.blockId, + }); + expect(await redis.pttl(runResultKey(args.runId))).toBe(-1); + // An old operation must not change the result expiry. + await redis.pexpire(runResultKey(args.runId), 60_000); + const frozen = await redis.hgetall(runResultKey(args.runId)); + await expect(restarted.absorbBlockers(args)).rejects.toThrow("refused as stale"); + expect(await redis.pttl(runResultKey(args.runId))).toBeGreaterThan(0); + expect(await redis.hgetall(runResultKey(args.runId))).toEqual(frozen); + } + } finally { + await redis.quit(); + await store.quit(); + } + } + ); + + containerTest( + "missing source is rejected before installing an active block", + async ({ prisma, redisOptions }) => { + const store = new WaitpointStoreCoordinator({ redisOptions }); + const adapter = new StoreWaitpointCoordinator({ store }); + try { + await expect( + adapter.absorbBlockers({ + runId: "missing-parent", + client: prisma, + blockId: "block", + absorptionId: "absorption", + projectId: "project", + waitpointIds: [generateWaitpointId("DATETIME")], + }) + ).rejects.toThrow("is not present in the store"); + expect((await store.readBlockState("missing-parent")).blockId).toBeUndefined(); + } finally { + await store.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeWaitpointCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeWaitpointCoordinator.ts new file mode 100644 index 00000000000..0a689e14163 --- /dev/null +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeWaitpointCoordinator.ts @@ -0,0 +1,530 @@ +import { parseWaitpointId, WaitpointId } from "@trigger.dev/core/v3/isomorphic"; +import type { Waitpoint } from "@trigger.dev/database"; +import pMap from "p-map"; +import { + StoreWaitpointBlockOperationIdentityRequired, + StoreWaitpointRegistrationUnsupported, +} from "../errors.js"; +import { parseEdgeField, registrationEvidence } from "./keys.js"; +import { + WaitpointNotFoundError, + type AbsorbResult, + type BlockEdge, + type WaitpointCompletion, + type WaitpointStoredRecord, + type WaitpointStoreCoordinator, +} from "./storeCoordinator.js"; +import type { + AbandonRunBlockersParams, + AbsorbBlockersParams, + AbsorbedBlockersParams, + AbsorbBlockersResult, + AcknowledgeResumeParams, + AssociatedWaitpointData, + BlockerRegistration, + ClearRunBlockStateParams, + ReleaseRunBlockersParams, + CompleteParams, + CompleteResult, + CreateDateTimeWaitpointParams, + CreateManualWaitpointParams, + CreateWaitpointResult, + RegisterBlockersResult, + RegisterBlocksLocklessParams, + RegisterBlocksParams, + RunBlockEdge, + StoreBlockEdge, + StoreBlockerRegistration, + WaitpointCoordinator, +} from "./types.js"; + +// Limit concurrent watcher removals to avoid a burst of Redis commands. +const ABANDON_UNREGISTER_CONCURRENCY = 10; + +/** + * Reject registration if the run has moved to another block or has completed cleanup. + * Neither case can add watchers. + */ +export class StoreWaitpointBlockRefusedError extends Error { + readonly runId: string; + readonly outcome: "stale" | "terminal" | "cleanupOwed"; + readonly currentBlockId?: string; + constructor(params: { + runId: string; + outcome: "stale" | "terminal" | "cleanupOwed"; + currentBlockId?: string; + }) { + super( + `Store block absorption for run ${params.runId} was refused as ${params.outcome}` + + (params.currentBlockId ? ` (current block ${params.currentBlockId})` : "") + ); + this.name = "StoreWaitpointBlockRefusedError"; + this.runId = params.runId; + this.outcome = params.outcome; + this.currentBlockId = params.currentBlockId; + } +} + +export type StoreWaitpointCoordinatorOptions = { + store: WaitpointStoreCoordinator; + /** Injectable so a test can pin edge timestamps. Defaults to Date.now. */ + clock?: () => number; +}; + +/** + * Adapt MemoryDB waitpoints to the {@link WaitpointCoordinator} interface. + * This adapter handles registration, completion, reads and cleanup. Creation methods are not wired here. + * Never route a MemoryDB ID to Postgres. + * + * Register waits in three phases: + * 1. {@link absorbBlockers}: record waits and completed results under the caller's block ID. + * 2. The caller saves the blocked snapshot. + * 3. {@link registerBlockers}: add watchers and deliver completed results. + * + * Keep the caller's block ID unchanged so Redis can reject retries from an old cycle. + */ +export class StoreWaitpointCoordinator implements WaitpointCoordinator { + readonly #store: WaitpointStoreCoordinator; + readonly #clock: () => number; + + constructor(options: StoreWaitpointCoordinatorOptions) { + this.#store = options.store; + this.#clock = options.clock ?? Date.now; + } + + // ---- Block protocol ------------------------------------------------------- + + async finalizeBlockers(params: AbsorbedBlockersParams): Promise { + await this.#store.deleteContribution(params); + } + + async hasAbsorbedBlockers(params: AbsorbedBlockersParams): Promise { + return this.#store.hasAbsorbedBlockers(params); + } + + async absorbBlockers(params: AbsorbBlockersParams): Promise { + // Require the caller's block identity to reject old operations. + // Check direct calls as well as calls through the router. + if (!params.blockId || !params.absorptionId) { + throw new StoreWaitpointBlockOperationIdentityRequired({ + runId: params.runId, + waitpointIds: params.waitpointIds, + }); + } + const blockId = params.blockId; + const absorptionId = params.absorptionId; + const evidence = registrationEvidence( + params.registrationWaitpointIds ?? params.waitpointIds, + params.batchIndex + ); + const edges = this.#buildEdges(params); + // Read completed results before creating the block. On retry, use retained results first. + // Save the results with the block in one atomic operation. + // Source expiry must not leave an active block without its result. + try { + await this.#store.prepareRegistrationSources({ + runId: params.runId, + blockId, + absorptionId, + waitpointIds: params.waitpointIds, + evidence, + expectedPreviousBlockId: params.expectedPreviousBlockId, + }); + const sources = await this.#store.readCompletedRecords(params.waitpointIds, params.runId); + const reportedEdges: BlockEdge[] = edges.map((edge) => { + const source = sources.get(edge.waitpointId); + if (!source) throw new WaitpointNotFoundError(edge.waitpointId); + edge.completedAfter = source.record.completedAfter; + return source.status === "COMPLETED" && source.completion + ? { + ...edge, + reported: { completion: { ...source.completion, record: source.record } }, + } + : edge; + }); + + // Pass the caller's block ID and expected previous ID unchanged. + // If Redis rejects the block as stale or terminal, do not register watchers. + const absorbed = await this.#store.absorbBlockers({ + runId: params.runId, + blockId, + edges: reportedEdges, + registrationEvidence: evidence, + batchIndex: params.batchIndex, + expectedPreviousBlockId: params.expectedPreviousBlockId, + absorptionId, + }); + + if (absorbed.outcome !== "absorbed") { + throw new StoreWaitpointBlockRefusedError({ + runId: params.runId, + outcome: absorbed.outcome, + currentBlockId: absorbed.currentBlockId, + }); + } + + const registration: StoreBlockerRegistration = { + kind: "store", + registrationEvidence: evidence, + runId: params.runId, + blockId, + absorptionId, + organizationId: params.organizationId, + edges, + storePendingTotal: absorbed.storePendingTotal, + alreadyDelivered: absorbed.alreadyDelivered, + supersededCleanup: absorbed.supersededCleanup, + }; + + return { pendingCount: absorbed.pendingOfRequested, registration }; + } catch (error) { + // Failed preparation has no accepted block; release its source holds immediately. + // If cleanup fails too, the prearmed recovery job retains the indexed targets. + await this.#store.compensateAbsorption({ + runId: params.runId, + blockId, + absorptionId, + unacceptedEvidence: evidence, + }); + throw error; + } + } + + #buildEdges(params: AbsorbBlockersParams): StoreBlockEdge[] { + const nowIso = new Date(this.#clock()).toISOString(); + return params.waitpointIds.map((waitpointId) => { + const parsed = parseWaitpointId(waitpointId); + if (parsed.format !== "b32hexW") { + // Postgres IDs belong in the legacy coordinator. Reject a routing error here. + throw new Error( + `StoreWaitpointCoordinator.absorbBlockers received a non-store waitpoint id ${waitpointId}` + ); + } + return { + waitpointId, + batchId: params.batchId, + batchIndex: params.batchIndex, + spanIdToComplete: params.spanIdToComplete, + createdAt: nowIso, + type: parsed.type, + }; + }); + } + + async registerBlockers(registration: BlockerRegistration): Promise { + if (registration.kind !== "store") { + throw new Error( + `StoreWaitpointCoordinator.registerBlockers received a ${registration.kind} continuation` + ); + } + + const result = await this.#store.registerAndDeliver({ + runId: registration.runId, + blockId: registration.blockId, + organizationId: registration.organizationId, + edges: registration.edges as BlockEdge[], + absorbed: { + storePendingTotal: registration.storePendingTotal, + alreadyDelivered: registration.alreadyDelivered as AbsorbResult["alreadyDelivered"], + supersededCleanup: registration.supersededCleanup as AbsorbResult["supersededCleanup"], + }, + }); + + // After phase 3, prevent cleanup from undoing accepted waits. + // Keep request proof until the caller finishes timeout jobs, resume jobs and events. + await this.#store.markRegistrationComplete({ + runId: registration.runId, + blockId: registration.blockId, + absorptionId: registration.absorptionId, + evidence: registration.registrationEvidence, + }); + + return { + pendingCount: result.pendingOfRequested, + cachedSpanCompletions: result.cachedSpanCompletions, + }; + } + + async currentStoreBlockId(runId: string): Promise { + return this.#store.currentBlockId(runId); + } + + async acknowledgeResume(params: AcknowledgeResumeParams): Promise { + // Acknowledge the block that the caller resumed, then clear that same block. + // Keep receipts until acknowledgement succeeds so a failed snapshot write can retry. + // If a newer block has replaced it, leave the newer block unchanged. + if (!params.blockId) { + return; + } + const ack = await this.#store.acknowledgeResumeHandoff({ + runId: params.runId, + blockId: params.blockId, + }); + if (ack.outcome !== "acknowledged") { + return; + } + await this.#store.clearBlockState({ runId: params.runId, blockId: params.blockId }); + } + + async abandonRunBlockers(params: AbandonRunBlockersParams): Promise<{ count: number }> { + // Save this block's watcher-removal list and clear its active state atomically. + // Then remove the watchers. If removal fails, keep the list for retry. + // Do not change a newer block. The router removes Postgres edges first. + const target = params.blockId ?? (await this.#store.currentBlockId(params.runId)); + if (!target) { + return { count: 0 }; + } + const result = await this.#store.abandonRetainAndClear({ + runId: params.runId, + blockId: target, + }); + if (result.outcome !== "retained") { + // This block is no longer active. Nothing needs to be abandoned. + return { count: 0 }; + } + return this.#withdrawRetained(params.runId, target); + } + + async resumeRetainedWithdrawal(params: { + runId: string; + blockId: string; + organizationId?: string; + }): Promise<{ count: number }> { + // Finish watcher removal from the saved list. Leave Postgres and newer blocks unchanged. + // Do nothing if the list was already removed. + return this.#withdrawRetained(params.runId, params.blockId); + } + + /** Remove watchers in bounded pages. Keep a page until every removal succeeds. */ + async #withdrawRetained(runId: string, blockId: string): Promise<{ count: number }> { + let retained = await this.#store.readWithdrawalPage({ runId, blockId }); + if (!retained) return { count: 0 }; + let count = 0; + while (retained && retained.fields.length > 0) { + await pMap( + retained.fields, + (field) => { + const parsed = parseEdgeField(field); + if (!parsed) throw new Error(`Invalid retained withdrawal target: ${field}`); + return this.#store.unregisterWatcher({ + waitpointId: parsed.waitpointId, + runId, + blockId, + batchIndex: parsed.batchIndex, + }); + }, + { concurrency: ABANDON_UNREGISTER_CONCURRENCY } + ); + await this.#store.acknowledgeWithdrawalPage({ runId, blockId, fields: retained.fields }); + // Count completed removals, including watchers that were already absent. + count += retained.fields.length; + retained = await this.#store.readWithdrawalPage({ runId, blockId }); + } + // Remove the empty list and this block's cleanup guard atomically. + await this.#store.finalizeRetainedWithdrawal({ runId, blockId }); + return { count }; + } + + async releaseRunBlockers(params: ReleaseRunBlockersParams): Promise<{ count: number }> { + // Mark the run terminal before removing watchers. Fanout must not resume a terminal run. + // Keep the saved edges and terminal marker if removal fails, so cleanup can retry. + // If a block ID is supplied, clean up only that block. + const result = await this.#store.releaseRunWatchers({ + runId: params.runId, + reason: params.reason, + blockId: params.blockId, + }); + return { count: result.unregistered }; + } + + // This adapter has no Postgres edges to clear. + async clearLegacyBlockersBeforeTransition(): Promise<{ count: number }> { + return { count: 0 }; + } + + async armCleanupFence(params: { runId: string; blockId: string }): Promise { + // Prevent new blocks until this block's cleanup finishes in both stores. + await this.#store.armCleanupFence({ runId: params.runId, blockId: params.blockId }); + } + + async assertNoCleanupOwed(runId: string): Promise { + const blockId = await this.#store.readCleanupFence(runId); + if (blockId) { + throw new StoreWaitpointBlockRefusedError({ + runId, + outcome: "cleanupOwed", + currentBlockId: blockId, + }); + } + } + + async compensateAbsorption(params: { + runId: string; + blockId: string; + absorptionId: string; + /** Recovery may discard only preparation for this request, never an accepted operation. */ + unacceptedEvidence?: string; + }): Promise { + // Release this operation's claims. Preserve edges claimed by other operations. + // Clear the block ID if no edges remain. Leave newer and terminal blocks unchanged. + // The test hook can pause cleanup; production callers hold the run lock. + await this._beforeCompensateDrain(); + await this.#store.compensateAbsorption(params); + } + + /** Test hook to pause cleanup before the atomic operation. */ + protected async _beforeCompensateDrain(): Promise {} + + // ---- Completion, read, clear (routed here by id shape) -------------------- + + async complete({ waitpointId, output }: CompleteParams): Promise { + const completion: WaitpointCompletion = { + completedAt: new Date(this.#clock()).toISOString(), + outputType: output?.type ?? "application/json", + outputIsError: output?.isError ?? false, + output: output ? { inline: output.value } : null, + }; + + // Complete the waitpoint and queue fanout. Repeating the same result is safe. + // A different result throws WaitpointCompletionConflictError. + // The fanout worker delivers results in bounded pages. + await this.#store.complete({ waitpointId, completion }); + + const waitpoint = await this.#waitpointFromStore(waitpointId); + return { waitpoint, blockedRuns: [] }; + } + + async readRunBlockState(runId: string): Promise { + const state = await this.#store.readBlockState(runId); + const delivered = new Set(state.deliveredIds); + return state.edges.map((edge) => ({ + id: edge.edgeId, + batchId: edge.batchId ?? null, + batchIndex: edge.batchIndex ?? null, + waitpoint: { + id: edge.waitpointId, + status: delivered.has(edge.waitpointId) ? "COMPLETED" : "PENDING", + type: edge.type, + completedAfter: edge.completedAfter ? new Date(edge.completedAfter) : null, + }, + })); + } + + async clearRunBlockState(params: ClearRunBlockStateParams): Promise<{ count: number }> { + const state = await this.#store.readBlockState(params.runId); + if (!state.blockId) { + return { count: 0 }; + } + + if (params.edgeIds && params.edgeIds.length > 0) { + // Ignore Postgres edge IDs. The router sends mixed lists to both coordinators. + const owned = new Set(state.edges.map((edge) => edge.edgeId)); + const toDrain = params.edgeIds.filter((id) => owned.has(id)); + if (toDrain.length === 0) { + return { count: 0 }; + } + await this.#store.clearBlockState({ + runId: params.runId, + blockId: state.blockId, + edgeIds: toDrain, + }); + return { count: toDrain.length }; + } + + await this.#store.clearBlockState({ runId: params.runId, blockId: state.blockId }); + return { count: state.edges.length }; + } + + // Creation is added by later PRs. + + registerBlocks(params: RegisterBlocksParams): Promise<{ pendingCount: number }> { + // MemoryDB waits require all three registration phases. + // Reject direct calls to this single-step method, as the router does. + return Promise.reject( + new StoreWaitpointRegistrationUnsupported({ + runId: params.runId, + waitpointIds: params.waitpointIds, + }) + ); + } + + registerBlocksLockless(params: RegisterBlocksLocklessParams): Promise { + return Promise.reject( + new StoreWaitpointRegistrationUnsupported({ + runId: params.runId, + waitpointIds: params.waitpointIds, + }) + ); + } + + createDateTimeWaitpoint(_params: CreateDateTimeWaitpointParams): Promise { + return Promise.reject(this.#mintNotWired("DATETIME")); + } + + createManualWaitpoint(_params: CreateManualWaitpointParams): Promise { + return Promise.reject(this.#mintNotWired("MANUAL")); + } + + mintAssociatedWaitpointData(): AssociatedWaitpointData { + throw this.#mintNotWired("RUN"); + } + + createAssociatedWaitpoint(): Promise { + return Promise.reject(this.#mintNotWired("RUN")); + } + + #mintNotWired(type: string): Error { + return new Error( + `Store-format ${type} minting is not wired in the shared block-protocol commit; the per-type ` + + `migration ticket owns it` + ); + } + + // Convert the stored record to the Waitpoint return type. + // Reject missing records or envelopes instead of returning incomplete data. + async #waitpointFromStore(waitpointId: string): Promise { + const records = await this.#store.readCompletedRecords([waitpointId]); + const stored = records.get(waitpointId); + if (!stored) { + throw new WaitpointNotFoundError(waitpointId); + } + return waitpointFromStoredRecord(waitpointId, stored); + } +} + +/** + * Convert a store record to the existing Waitpoint return type. + * Fill the fields callers read. Set the remaining fields to null. + */ +function waitpointFromStoredRecord(waitpointId: string, stored: WaitpointStoredRecord): Waitpoint { + const record = stored.record; + const completion = stored.completion; + return { + id: waitpointId, + friendlyId: record.friendlyId ?? WaitpointId.toFriendlyId(waitpointId), + type: record.type, + status: stored.status, + completedAt: completion ? new Date(completion.completedAt) : null, + idempotencyKey: record.idempotencyKey ?? null, + idempotencyKeyExpiresAt: record.idempotencyKeyExpiresAt + ? new Date(record.idempotencyKeyExpiresAt) + : null, + userProvidedIdempotencyKey: record.userProvidedIdempotencyKey, + inactiveIdempotencyKey: null, + projectId: record.projectId, + environmentId: record.environmentId, + completedByTaskRunId: record.completedByTaskRunId ?? null, + completedByBatchId: record.completedByBatchId ?? null, + completedAfter: record.completedAfter ? new Date(record.completedAfter) : null, + output: completion?.output + ? "inline" in completion.output + ? completion.output.inline + : completion.output.ref + : null, + outputType: completion?.outputType ?? null, + outputIsError: completion?.outputIsError ?? false, + createdAt: record.createdAt ? new Date(record.createdAt) : new Date(), + updatedAt: record.updatedAt ? new Date(record.updatedAt) : new Date(), + tags: record.tags ?? [], + } as unknown as Waitpoint; +} diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts index bd620ee9d23..10383d296e5 100644 --- a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts @@ -29,19 +29,128 @@ export type WaitpointMintContext = { * never reads the value. */ /** - * Context a run-keyed operation forwards so a routing coordinator can decide whether the run could hold - * MemoryDB edges (rollout is keyed by organization). Optional and opaque: the legacy coordinator never - * reads it. An absent organizationId keeps the legacy route without store discovery. + * Optional trusted org context for a run-keyed operation. The router checks the organization's current + * flag before consulting the individual run's durable storage marker. Legacy coordinators ignore it. */ export type RunKeyedRoutingContext = { organizationId?: string; }; +/** Identify the registration to check. Hold the parent run lock when reading its proof. */ +export type AbsorbedBlockersParams = { + runId: string; + blockId: string; + absorptionId: string; + waitpointIds: string[]; + batchIndex?: number; + organizationId?: string; +}; + export type WaitpointCoordinator = { clearRunBlockState(params: ClearRunBlockStateParams): Promise<{ count: number }>; readRunBlockState(runId: string, context?: RunKeyedRoutingContext): Promise; - registerBlocks(params: RegisterBlocksParams): Promise; + registerBlocks(params: RegisterBlocksParams): Promise<{ pendingCount: number }>; registerBlocksLockless(params: RegisterBlocksLocklessParams): Promise; + /** + * Phase 1 of the three-phase block protocol: absorb the blocker set into run-side coordination and + * report how many distinct requested blockers are still pending. The caller publishes the durable + * blocked TRES transition AFTER this returns and BEFORE calling {@link registerBlockers} — so no + * watcher is ever installed for a run that never durably blocked. Returns an opaque + * {@link BlockerRegistration} continuation to hand to phase 3 unchanged. + * + * Legacy (Postgres) absorbs by writing the block edges and re-reading the pending count under READ + * COMMITTED, so its two phases straddle the transition safely with the register phase a no-op. Store + * (MemoryDB) absorbs run-side pending state atomically, and phase 3 registers the waitpoint-side + * watchers. A routing coordinator partitions by id shape and absorbs MemoryDB before Postgres. + */ + absorbBlockers(params: AbsorbBlockersParams): Promise; + /** Remove recovery proof after registration and its follow-up work finish. */ + finalizeBlockers(params: AbsorbedBlockersParams): Promise; + /** Check whether this exact request was accepted before recovery retries it. */ + hasAbsorbedBlockers(params: AbsorbedBlockersParams): Promise; + /** + * Phase 3 of the block protocol: register the run as a watcher on each store-resident waitpoint and + * immediately deliver any completion the registration reports (an early completion that landed in the + * absorb→transition→register gap is returned, never lost). A no-op for legacy blockers. Returns the + * distinct-of-requested pending count AFTER registration and early delivery: 0 means the run is now + * resumable. A throw partway leaves earlier watchers registered and the run absorbed — nothing + * resumes early, and the residue is recoverable. + */ + registerBlockers(registration: BlockerRegistration): Promise; + /** + * Acknowledge the durable resume-handoff for a run's current block and drain its edges under the + * block-id fence. Called at the resume-durability boundary — AFTER TRES has durably accepted the + * resumed snapshot — so a transient TRES failure costs a retry, not the run's completed receipts. Not + * a substitute for terminal cleanup, and never raw `clearRunBlockState`. + */ + /** + * The run's current store block-operation id, for a caller to OBSERVE before resuming and hand back to + * {@link acknowledgeResume}, so the acknowledgement targets that exact block and not whichever block is + * installed later. Undefined for a legacy-only run or an unmarked run (no store read). + */ + currentStoreBlockId(runId: string, context?: CurrentBlockContext): Promise; + acknowledgeResume(params: AcknowledgeResumeParams): Promise; + /** + * Non-terminal retry/requeue cleanup: fence and abandon the run's current block and withdraw its + * watchers so an in-flight old delivery is refused, WITHOUT terminalizing the run — its next attempt + * must be able to open a fresh block cycle. Returns how many edges were abandoned. + */ + abandonRunBlockers(params: AbandonRunBlockersParams): Promise<{ count: number }>; + /** + * STORE-ONLY replay of a block's retained watcher withdrawal, from its durable record. Used by a + * delayed cleanup obligation whose block is no longer current: it must NOT touch the legacy coordinator + * (whose block-id-less edge delete would drop a newer block's Postgres edges) or the current block. + */ + resumeRetainedWithdrawal(params: { + runId: string; + blockId: string; + organizationId?: string; + }): Promise<{ count: number }>; + /** + * Terminal/cancel cleanup: withdraw the run's watchers terminal-first and arm terminal retention, so a + * cancelled or terminal run can never be resumed by an in-flight delivery. Returns how many edges were + * released. + */ + releaseRunBlockers(params: ReleaseRunBlockersParams): Promise<{ count: number }>; + /** + * Clear a LEGACY-ONLY run's block edges BEFORE its failed-attempt transition, in the caller's Postgres + * client, restoring the pre-store ordering so a crash after the transition cannot strand Postgres edges. + * A run that may hold store blockers is a no-op here — its cleanup is the fenced post-transition path — + * and a legacy/unmarked run never consults MemoryDB. + */ + clearLegacyBlockersBeforeTransition(params: { + runId: string; + organizationId?: string; + tx?: PrismaClientOrTransaction; + }): Promise<{ count: number }>; + /** + * Durably mark this run's exact current store block as cleanup-owed BEFORE its retry/requeue transition, + * so a DIFFERENT store block cannot be absorbed until the previous mixed block's cross-store cleanup (its + * block-id-less legacy delete plus store withdrawal) has fully converged. Store-only; legacy and + * unmarked runs are a no-op. Lifted as the last step of that cleanup. + */ + armCleanupFence(params: { + runId: string; + blockId: string; + organizationId?: string; + }): Promise; + /** Refuse a legacy edge write while an earlier block still owes cross-store cleanup. */ + assertNoCleanupOwed(runId: string, context?: RunKeyedRoutingContext): Promise; + /** + * Roll back ONE absorption's store contribution, by identity: read the exact edge fields it inserted + * from the durable contribution record, remove only those under the block-id fence (preserving an + * earlier additive call's pre-existing same-block edges and any receipts), then delete the record. A + * stale compensation against a newer block mutates nothing but may retire its own obsolete record. + * Legacy has no store edges (no-op). + */ + compensateAbsorption(params: { + runId: string; + blockId: string; + absorptionId: string; + /** Recovery may discard only preparation for this request, never an accepted operation. */ + unacceptedEvidence?: string; + organizationId?: string; + }): Promise; complete(params: CompleteParams): Promise; createDateTimeWaitpoint(params: CreateDateTimeWaitpointParams): Promise; createManualWaitpoint(params: CreateManualWaitpointParams): Promise; @@ -86,8 +195,9 @@ export type RunBlockEdge = { waitpoint: Pick; }; -export type RegisterBlocksParams = RunKeyedRoutingContext & { +export type RegisterBlocksParams = { runId: string; + organizationId?: string; waitpointIds: string[]; projectId: string; spanIdToComplete?: string; @@ -102,18 +212,196 @@ export type RegisterBlocksParams = RunKeyedRoutingContext & { client: ReadClient; }; -export type RegisterBlocksResult = { - pendingCount: number; - /** This operation's router decision, used only to carry context onto its continuation job. */ - storeRoutingEnabled?: true; -}; - /** * The lockless variant writes the edge and does not count. Two methods rather than * one method with a flag, so "the batch path issues no extra query" is structural. */ export type RegisterBlocksLocklessParams = Omit; +/** + * Phase-1 inputs. The caller supplies the block-operation identity; the store never sources it by + * reading and reusing whatever id is currently installed. + */ +export type AbsorbBlockersParams = { + /** All requested IDs, before splitting them by store. */ + registrationWaitpointIds?: string[]; + runId: string; + waitpointIds: string[]; + projectId: string; + spanIdToComplete?: string; + batchId?: string; + batchIndex?: number; + /** + * The CALLER-OWNED block-operation identity: FRESH and stable per logical block cycle, RECOMPUTED + * identically on a retry of the same operation, and captured WITH the operation — never re-derived from + * whatever snapshot is latest when the operation runs. It is the CAS identity: a re-absorb under the + * same id is an idempotent additive registration, a new id whose predecessor is named rolls the + * previous cycle over, and a new id with a different predecessor installed is refused as stale. + * REQUIRED for a store-format blocker (absent → fail loud); legacy ignores it. + */ + blockId?: string; + /** + * The CALLER-OWNED contribution identity for THIS absorption: shared by retries of this exact additive + * operation, fresh for a separate additive operation on the same block. It keys the durable contribution + * record of which edges this operation actually inserted, so a failed mixed absorption compensates only + * its own edges. REQUIRED for a store-format blocker (absent → fail loud); legacy ignores it. + */ + absorptionId?: string; + /** + * The predecessor cycle this operation supersedes, for a rollover — captured with the operation, not + * re-read at execution time, so a delayed replay carries its ORIGINAL expectation and the store's CAS + * rejects it as stale rather than rolling over whatever is currently installed. Undefined for a first + * block or a same-operation retry. Legacy ignores it. + */ + expectedPreviousBlockId?: string; + /** + * The run's organization, carried onto the store watcher so the ASYNCHRONOUS fanout continuation can + * route by the run's org without an extra lookup. Legacy ignores it. + */ + organizationId?: string; + /** Read client for the legacy pending re-count, resolved to `tx ?? prisma` by the caller. */ + client: ReadClient; + /** + * Optional compatibility hint from older callers. Parent snapshot residency does not gate registration. + */ + parentResidency?: SnapshotResidency; +}; + +/** + * Inputs to the durable resume-handoff acknowledgement. Called ONLY after TRES has durably accepted the + * run's resume transition: the store marks the block's handoff acknowledged and drains its edges under + * the block-id fence; legacy deletes the named edge rows. Never a substitute for terminal cleanup. + */ +export type AcknowledgeResumeParams = { + runId: string; + /** + * The EXACT store block the continuation observed and is resuming, read once before the resume + * transition. The store acknowledges and clears THAT block; if a newer cycle has since superseded it, + * the acknowledgement is stale and a no-op, so the newer cycle is never cleared. Undefined for a + * legacy-only run (no store block). Legacy ignores it. + */ + blockId?: string; + /** The edges the resume consumed. Omit to acknowledge without draining specific edges. */ + edgeIds?: string[]; + /** The run's org, so a routing coordinator can skip MemoryDB for a unmarked run. */ + organizationId?: string; +}; + +/** Context a run-keyed store-block read forwards, so a routing coordinator can skip a unmarked run. */ +export type CurrentBlockContext = { + organizationId?: string; +}; + +/** + * Inputs to non-terminal retry/requeue cleanup. Runs AFTER the durable retry transition, outside the + * Postgres transaction, so it never joins a rollback-capable tx. The store fences and abandons the old + * block; legacy deletes the run's edge rows. + */ +export type AbandonRunBlockersParams = { + runId: string; + organizationId?: string; + /** + * The exact block to abandon. When set, the store abandons that block (active or resumed from its + * retained withdrawal record) and never touches a different one; when omitted, the run's current block + * is abandoned. A retry from the cleanup obligation always sets it. + */ + blockId?: string; + tx?: PrismaClientOrTransaction; +}; + +/** + * Inputs to terminal/cancel cleanup. The store withdraws the run's watchers TERMINAL-FIRST (install the + * terminal fence, then unregister), so an in-flight fanout delivery can never publish a resume for a run + * being torn down. Legacy deletes the run's edge rows. Never deletes state before the lifecycle + * transition — this IS the lifecycle transition's cleanup. + */ +export type ReleaseRunBlockersParams = { + runId: string; + reason: "cancelled" | "terminal"; + organizationId?: string; + /** + * The exact block to release. When set, the terminal-first withdrawal targets that block and never a + * newer one; when omitted, the run's current block is released. A retry from the cleanup obligation and + * the cancellation path set it. + */ + blockId?: string; + /** + * A caller lifecycle transaction. A routing coordinator refuses it when the run might hold store + * blockers — MemoryDB cannot join a Postgres transaction, so the store release must run after the + * transition commits. Legacy joins it. + */ + tx?: PrismaClientOrTransaction; +}; + +export type AbsorbBlockersResult = { + /** Distinct-of-requested blockers still pending after absorb — gates the blocked TRES transition. */ + pendingCount: number; + /** Opaque continuation for {@link WaitpointCoordinator.registerBlockers}. The caller does not read it. */ + registration: BlockerRegistration; +}; + +/** + * A cached trace span a store completion durably closed. The store's async fanout and its synchronous + * early delivery both emit this (independently of whole-run resumability), so a store completion finishes + * the same cached spans the legacy inline path did. At-least-once; the emitted event must be idempotent. + */ +type CachedSpanCompletion = { + spanId: string; + spanCreatedAt: string; + blockedRunId: string; + hasError: boolean; + cachedRunId?: string; +}; + +export type RegisterBlockersResult = { + /** Distinct-of-requested pending AFTER registration and early delivery. 0 ⇒ the run is resumable. */ + pendingCount: number; + /** Cached spans closed by an early/racing delivery during THIS registration, for the caller to emit. */ + cachedSpanCompletions?: CachedSpanCompletion[]; +}; + +/** One run→waitpoint edge as the store block protocol records it. A plain shape, no store-internal types. */ +export type StoreBlockEdge = { + waitpointId: string; + batchIndex?: number | null; + batchId?: string; + spanIdToComplete?: string; + createdAt: string; + type: WaitpointMintType; + completedAfter?: string; +}; + +/** + * The store variant of the opaque block continuation. Produced and consumed only by the store adapter; + * `completion` and `supersededCleanup` are typed `unknown` so store-internal envelope types never leak + * into this contract layer — the adapter casts them back. + */ +export type StoreBlockerRegistration = { + kind: "store"; + registrationEvidence: string; + runId: string; + blockId: string; + /** Keep this operation ID through phase 3 and the caller's follow-up work. */ + absorptionId: string; + /** Carried onto each watcher so the asynchronous fanout continuation routes by the run's org. */ + organizationId?: string; + edges: StoreBlockEdge[]; + storePendingTotal: number; + alreadyDelivered: Array<{ waitpointId: string; completion?: unknown }>; + supersededCleanup?: unknown; +}; + +/** + * The opaque continuation threaded from {@link WaitpointCoordinator.absorbBlockers} to + * {@link WaitpointCoordinator.registerBlockers}. `noop` is legacy (nothing to register; the edge write + * and pending count already happened in absorb). `routing` wraps the per-coordinator parts a routing + * layer absorbed. Each coordinator only ever registers a continuation it produced. + */ +export type BlockerRegistration = + | { kind: "noop"; pendingCount: number } + | StoreBlockerRegistration + | { kind: "routing"; parts: BlockerRegistration[] }; + export type CompleteParams = RunKeyedRoutingContext & { waitpointId: string; output?: { diff --git a/internal-packages/run-engine/src/engine/workerCatalog.ts b/internal-packages/run-engine/src/engine/workerCatalog.ts index 53cc54490a6..538c64297eb 100644 --- a/internal-packages/run-engine/src/engine/workerCatalog.ts +++ b/internal-packages/run-engine/src/engine/workerCatalog.ts @@ -1,9 +1,32 @@ import { z } from "zod"; export const workerCatalog = { + recoverWaitpointBlock: { + schema: z.object({ + runId: z.string(), + organizationId: z.string(), + projectId: z.string(), + waitpoints: z.array(z.string()), + blockOperation: z.object({ + id: z.string(), + absorptionId: z.string(), + cycleAnchor: z.string(), + expectedPreviousBlockId: z.string().optional(), + }), + timeout: z.coerce.date().optional(), + spanIdToComplete: z.string().optional(), + batch: z.object({ id: z.string(), index: z.number().optional() }).optional(), + workerId: z.string().optional(), + runnerId: z.string().optional(), + expectedEnvironmentId: z.string().optional(), + }), + visibilityTimeoutMs: 30_000, + retry: { maxAttempts: 10_000, minTimeoutInMs: 1_000, maxTimeoutInMs: 300_000 }, + }, finishWaitpoint: { schema: z.object({ waitpointId: z.string(), + organizationId: z.string().optional(), error: z.string().optional(), }), visibilityTimeoutMs: 30_000, @@ -114,6 +137,7 @@ export const workerCatalog = { ensureWaitpointCompleted: { schema: z.object({ waitpointId: z.string(), + organizationId: z.string().optional(), output: z .object({ value: z.string(), @@ -135,6 +159,25 @@ export const workerCatalog = { }), visibilityTimeoutMs: 30_000, }, + /** + * Retry cleanup when Postgres registration and immediate MemoryDB cleanup both fail. + * Hold the parent run lock. Remove only this operation's added edges. + * Preserve successful registrations and newer blocks. + */ + ensureAbsorptionCompensated: { + schema: z.object({ + runId: z.string(), + blockId: z.string(), + absorptionId: z.string(), + organizationId: z.string().optional(), + }), + visibilityTimeoutMs: 30_000, + retry: { + maxAttempts: 10_000, + minTimeoutInMs: 1_000, + maxTimeoutInMs: 300_000, + }, + }, }; for (const job of Object.values(workerCatalog)) { diff --git a/internal-packages/run-engine/src/run-queue/bench/concurrencySweeperScan.bench.test.ts b/internal-packages/run-engine/src/run-queue/bench/concurrencySweeperScan.bench.test.ts new file mode 100644 index 00000000000..6d727210772 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/bench/concurrencySweeperScan.bench.test.ts @@ -0,0 +1,144 @@ +/** + * Cost of one concurrency-sweeper pass (`scanConcurrencySets`) at different SCAN COUNTs. + * + * Seeds a testcontainers Redis with BENCH_SETS queue currentConcurrency sets plus + * BENCH_UNRELATED_PER_SET non-matching keys per set (default 4,000 sets x (1 set + 99 + * unrelated) = 400k keys, 1% matching), then runs one full sweep per entry in + * BENCH_SCAN_COUNTS and reports SCAN calls, SSCAN calls, total commands, server usec for + * each, Redis process CPU, wall time, peak RSS during the sweep, and whether the callback + * saw exactly the seeded run ids. + * + * Run on demand: `pnpm run test:bench src/run-queue/bench/concurrencySweeperScan --silent=false`. + * Knobs: BENCH_SETS, BENCH_UNRELATED_PER_SET, BENCH_SCAN_COUNTS (default "10,1000"). + */ +import { redisTest } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { Logger } from "@trigger.dev/core/logger"; +import { describe } from "vitest"; +import { FairQueueSelectionStrategy } from "../fairQueueSelectionStrategy.js"; +import { RunQueue } from "../index.js"; +import { RunQueueFullKeyProducer } from "../keyProducer.js"; +import { seedSweeperKeyspace } from "../tests/sweeperKeyspace.js"; + +vi.setConfig({ testTimeout: 900_000 }); + +const SETS = Number(process.env.BENCH_SETS ?? 4_000); +const UNRELATED_PER_SET = Number(process.env.BENCH_UNRELATED_PER_SET ?? 99); +const SCAN_COUNTS = (process.env.BENCH_SCAN_COUNTS ?? "10,1000").split(",").map(Number); +const KEY_PREFIX = "engine:runqueue:"; + +function createQueue(redisContainer: any, scanCount: number, seen: Set) { + const redis = { + keyPrefix: KEY_PREFIX, + host: redisContainer.getHost(), + port: redisContainer.getPort(), + }; + return new RunQueue({ + name: "rq", + tracer: trace.getTracer("rq"), + logger: new Logger("RunQueue", "warn"), + defaultEnvConcurrency: 10, + keys: new RunQueueFullKeyProducer(), + masterQueueConsumersDisabled: true, + workerOptions: { disabled: true }, + queueSelectionStrategy: new FairQueueSelectionStrategy({ + redis, + keys: new RunQueueFullKeyProducer(), + }), + redis, + concurrencySweeper: { + scanCount, + callback: async (runIds) => { + runIds.forEach((id) => seen.add(id)); + return []; + }, + }, + }); +} + +async function stats(queue: RunQueue) { + const [commandstats, info, cpu] = await Promise.all([ + queue.redis.info("commandstats"), + queue.redis.info("stats"), + queue.redis.info("cpu"), + ]); + const cpuSeconds = (name: string) => Number(cpu.match(new RegExp(`${name}:([\\d.]+)`))![1]); + const cmd = (name: string) => { + const m = commandstats.match(new RegExp(`cmdstat_${name}:calls=(\\d+),usec=(\\d+)`)); + return { calls: m ? Number(m[1]) : 0, usec: m ? Number(m[2]) : 0 }; + }; + return { + scan: cmd("scan"), + sscan: cmd("sscan"), + total: Number(info.match(/total_commands_processed:(\d+)/)![1]), + // Whole-process Redis CPU, including the per-call network/parse cost commandstats leaves out. + cpuMs: (cpuSeconds("used_cpu_user") + cpuSeconds("used_cpu_sys")) * 1000, + }; +} + +describe("concurrency sweeper SCAN COUNT benchmark", () => { + redisTest("one full sweep per SCAN COUNT", async ({ redisContainer }) => { + const seedQueue = createQueue(redisContainer, 1000, new Set()); + let expected: Set; + let dbsize: number; + try { + const startedAt = performance.now(); + expected = await seedSweeperKeyspace(seedQueue.redis, { + concurrencySets: SETS, + unrelatedPerSet: UNRELATED_PER_SET, + }); + dbsize = await seedQueue.redis.dbsize(); + console.log( + `seeded ${dbsize} keys (${SETS} matching sets, ${expected.size} run ids) in ${Math.round( + performance.now() - startedAt + )}ms` + ); + } finally { + await seedQueue.quit(); + } + + const results: Array> = []; + for (const scanCount of SCAN_COUNTS) { + const seen = new Set(); + const queue = createQueue(redisContainer, scanCount, seen); + try { + const rssBefore = process.memoryUsage.rss(); + let rssPeak = rssBefore; + const sampler = setInterval(() => { + rssPeak = Math.max(rssPeak, process.memoryUsage.rss()); + }, 5); + + const before = await stats(queue); + const startedAt = performance.now(); + const sweep = await queue["scanConcurrencySets"](); + const wallMs = performance.now() - startedAt; + const after = await stats(queue); + clearInterval(sampler); + rssPeak = Math.max(rssPeak, process.memoryUsage.rss()); + + const identical = seen.size === expected.size && [...seen].every((id) => expected.has(id)); + results.push({ + scanCount, + scanCalls: after.scan.calls - before.scan.calls, + scanServerMs: Math.round((after.scan.usec - before.scan.usec) / 1000), + sscanCalls: after.sscan.calls - before.sscan.calls, + sscanServerMs: Math.round((after.sscan.usec - before.sscan.usec) / 1000), + totalCommands: after.total - before.total, + redisCpuMs: Math.round(after.cpuMs - before.cpuMs), + wallMs: Math.round(wallMs), + rssBeforeMB: Math.round(rssBefore / 2 ** 20), + rssPeakMB: Math.round(rssPeak / 2 ** 20), + keysProcessed: sweep?.processedKeys ?? 0, + runIdsSeen: seen.size, + identical, + }); + expect(identical).toBe(true); + } finally { + await queue.quit(); + } + } + + console.log(`dbsize=${dbsize}`); + console.table(results); + }); +}); diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 4d47ceb1c92..1d4bf74179c 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -88,6 +88,9 @@ const SemanticAttributes = { * active or observed gate never re-anchors; the Lua helper inlines the same value. */ const GATE_QUEUED_COUNTER_TTL_SECONDS = 86400; +/** Concurrency sets the sweeper processes at once from one SCAN page (the old COUNT 10 page width). */ +export const SWEEPER_SET_CONCURRENCY = 10; + /** Counters a dequeue script tallies for its reconcile work; positional on the reply's third slot. */ type ReconcileCounters = { passesGroup: number; @@ -1324,6 +1327,8 @@ export type RunQueueOptions = { scanJitterInMs?: number; processMarkedSchedule?: string; processMarkedJitterInMs?: number; + /** SCAN COUNT hint per call while looking for currentConcurrency sets */ + scanCount?: number; callback: ConcurrencySweeperCallback; }; /** TTL system for automatic run expiration */ @@ -2867,7 +2872,7 @@ export class RunQueue { * Create a scan stream for queue current concurrency keys */ public currentConcurrencyScanStream( - count: number = 10, + count: number, onEnd?: () => void, onError?: (error: Error) => void ): { stream: Readable; redis: Redis } { @@ -4960,13 +4965,17 @@ export class RunQueue { }; const { promise, resolve, reject } = promiseWithResolvers(); + // "end" can fire while the last page is still being processed; finish it first. + let processing: Promise = Promise.resolve(); const { stream, redis } = this.currentConcurrencyScanStream( - 10, + boundedInt(this.options.concurrencySweeper?.scanCount, 1000, 1, 10_000), () => { - this.logger.debug("Concurrency scan stream closed", { stats }); + void processing.then(() => { + this.logger.debug("Concurrency scan stream closed", { stats }); - resolve(stats); + resolve(stats); + }); }, (error) => { this.logger.error("Concurrency scan stream error", { @@ -5011,11 +5020,21 @@ export class RunQueue { stats.processedKeys += uniqueKeys.length; - await Promise.allSettled(uniqueKeys.map((key) => this.processConcurrencySet(key))).finally( - () => { - stream.resume(); + // A full SCAN page can hold hundreds of matches; cap the fan-out so the per-set callback + // (a DB query in the engine) doesn't burst. + processing = (async () => { + for (let i = 0; i < uniqueKeys.length; i += SWEEPER_SET_CONCURRENCY) { + await Promise.allSettled( + uniqueKeys + .slice(i, i + SWEEPER_SET_CONCURRENCY) + .map((key) => this.processConcurrencySet(key)) + ); } - ); + })().finally(() => { + stream.resume(); + }); + + await processing; }); return promise; @@ -5027,9 +5046,11 @@ export class RunQueue { }); const { promise, resolve, reject } = promiseWithResolvers(); + // Same as the SCAN page level: "end" can fire while the last SSCAN page is in flight. + let processing: Promise = Promise.resolve(); stream.on("end", () => { - resolve(); + void processing.then(() => resolve()); }); stream.on("error", (error) => { @@ -5041,7 +5062,7 @@ export class RunQueue { reject(error); }); - stream.on("data", async (runIds: string[]) => { + const processPage = async (runIds: string[]) => { stream.pause(); if (this.abortController.signal.aborted) { @@ -5069,6 +5090,10 @@ export class RunQueue { } stream.resume(); + }; + + stream.on("data", (runIds: string[]) => { + processing = processPage(runIds); }); return promise; diff --git a/internal-packages/run-engine/src/run-queue/tests/concurrencySweeper.test.ts b/internal-packages/run-engine/src/run-queue/tests/concurrencySweeper.test.ts index 67033aafdef..af241f29c76 100644 --- a/internal-packages/run-engine/src/run-queue/tests/concurrencySweeper.test.ts +++ b/internal-packages/run-engine/src/run-queue/tests/concurrencySweeper.test.ts @@ -4,9 +4,10 @@ import { Logger } from "@trigger.dev/core/logger"; import { describe } from "node:test"; import { setTimeout } from "node:timers/promises"; import { FairQueueSelectionStrategy } from "../fairQueueSelectionStrategy.js"; -import { RunQueue } from "../index.js"; +import { RunQueue, SWEEPER_SET_CONCURRENCY } from "../index.js"; import { RunQueueFullKeyProducer } from "../keyProducer.js"; import type { InputPayload } from "../types.js"; +import { commandCalls, seedSweeperKeyspace } from "./sweeperKeyspace.js"; import { Decimal } from "@trigger.dev/database"; const testOptions = { @@ -175,4 +176,187 @@ describe("RunQueue Concurrency Sweeper", () => { } } ); + + redisTest( + "scans the keyspace in large SCAN batches, bounds callback fan-out, and finds the same members", + async ({ redisContainer }) => { + // Catches: the sweeper falling back to tiny SCAN pages (~1.7M SCANs per prod sweep) or bursting one callback per set in a page. + const createQueue = ( + scanCount: number | undefined, + seen: Set, + inFlight: { now: number; max: number } + ) => + new RunQueue({ + ...testOptions, + logger: new Logger("RunQueue", "warn"), + masterQueueConsumersDisabled: true, + workerOptions: { disabled: true }, + queueSelectionStrategy: new FairQueueSelectionStrategy({ + redis: { + keyPrefix: "runqueue:test:", + host: redisContainer.getHost(), + port: redisContainer.getPort(), + }, + keys: testOptions.keys, + }), + redis: { + keyPrefix: "runqueue:test:", + host: redisContainer.getHost(), + port: redisContainer.getPort(), + }, + concurrencySweeper: { + scanCount, + callback: async (runIds) => { + inFlight.max = Math.max(inFlight.max, ++inFlight.now); + await setTimeout(1); + runIds.forEach((id) => seen.add(id)); + inFlight.now--; + return []; + }, + }, + }); + + const sweep = async (scanCount: number | undefined) => { + const seen = new Set(); + const inFlight = { now: 0, max: 0 }; + const queue = createQueue(scanCount, seen, inFlight); + try { + const before = await commandCalls(queue.redis, "scan"); + await queue["scanConcurrencySets"](); + const scans = (await commandCalls(queue.redis, "scan")) - before; + return { seen, scans, maxInFlight: inFlight.max, dbsize: await queue.redis.dbsize() }; + } finally { + await queue.quit(); + } + }; + + const seedQueue = createQueue(undefined, new Set(), { now: 0, max: 0 }); + let expected: Set; + try { + expected = await seedSweeperKeyspace(seedQueue.redis, { + concurrencySets: 2_000, + unrelatedPerSet: 9, + }); + } finally { + await seedQueue.quit(); + } + + const legacy = await sweep(10); + const current = await sweep(undefined); + + // Same members reach the callback whatever the batch size. + expect(legacy.seen).toEqual(expected); + expect(current.seen).toEqual(expected); + + // ~20k keys: the old COUNT 10 needs ~2,000 SCANs, the default must stay near dbsize/1000. + expect(current.dbsize).toBeGreaterThanOrEqual(20_000); + expect(legacy.scans).toBeGreaterThan(1_000); + expect(current.scans).toBeLessThanOrEqual(Math.ceil(current.dbsize / 1000) * 2); + + // A 1000-key SCAN page can hold hundreds of matches; the callback (a DB query in + // the engine) must not fan out wider than the old 10-key page did. + expect(current.maxInFlight).toBeGreaterThan(1); + expect(current.maxInFlight).toBeLessThanOrEqual(SWEEPER_SET_CONCURRENCY); + } + ); + + redisTest( + "does not resolve the sweep until the last SCAN page has been processed", + async ({ redisContainer }) => { + // Catches: the sweep resolving on SCAN "end" while the last page's sets are still being processed. + const seen = new Set(); + const queue = new RunQueue({ + ...testOptions, + logger: new Logger("RunQueue", "warn"), + masterQueueConsumersDisabled: true, + workerOptions: { disabled: true }, + queueSelectionStrategy: new FairQueueSelectionStrategy({ + redis: { + keyPrefix: "runqueue:test:", + host: redisContainer.getHost(), + port: redisContainer.getPort(), + }, + keys: testOptions.keys, + }), + redis: { + keyPrefix: "runqueue:test:", + host: redisContainer.getHost(), + port: redisContainer.getPort(), + }, + concurrencySweeper: { + scanCount: 30, + callback: async (runIds) => { + await setTimeout(50); + runIds.forEach((id) => seen.add(id)); + return []; + }, + }, + }); + + try { + // 40 sets at COUNT 30 is two pages, the second non-empty. While the first is processed + // the stream reads ahead to the end, so "end" fires as soon as the second page is handed + // out, before its callbacks finish. A single page can't show this ("end" waits for + // resume()), nor can an empty last page. + const expected = await seedSweeperKeyspace(queue.redis, { + concurrencySets: 40, + unrelatedPerSet: 0, + }); + + const stats = await queue["scanConcurrencySets"](); + expect(seen).toEqual(expected); + expect(stats?.streamCallbacks).toBeGreaterThan(1); + } finally { + await queue.quit(); + } + } + ); + + redisTest( + "does not resolve a set until its last SSCAN page has been processed", + async ({ redisContainer }) => { + // Catches: a concurrency set resolving on SSCAN "end" while its last page's callback is still pending. + const seen = new Set(); + const queue = new RunQueue({ + ...testOptions, + logger: new Logger("RunQueue", "warn"), + masterQueueConsumersDisabled: true, + workerOptions: { disabled: true }, + queueSelectionStrategy: new FairQueueSelectionStrategy({ + redis: { + keyPrefix: "runqueue:test:", + host: redisContainer.getHost(), + port: redisContainer.getPort(), + }, + keys: testOptions.keys, + }), + redis: { + keyPrefix: "runqueue:test:", + host: redisContainer.getHost(), + port: redisContainer.getPort(), + }, + concurrencySweeper: { + callback: async (runIds) => { + await setTimeout(50); + runIds.forEach((id) => seen.add(id)); + return []; + }, + }, + }); + + try { + // 250 members is past the listpack limit, so SSCAN COUNT 100 pages through a hashtable. + const expected = Array.from({ length: 250 }, (_, i) => `run_${i}`); + await queue.redis.sadd( + testOptions.keys.queueCurrentConcurrencyKey(authenticatedEnvDev, messageDev.queue), + ...expected + ); + + await queue["scanConcurrencySets"](); + expect(seen).toEqual(new Set(expected)); + } finally { + await queue.quit(); + } + } + ); }); diff --git a/internal-packages/run-engine/src/run-queue/tests/sweeperKeyspace.ts b/internal-packages/run-engine/src/run-queue/tests/sweeperKeyspace.ts new file mode 100644 index 00000000000..ca88d87d820 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/tests/sweeperKeyspace.ts @@ -0,0 +1,54 @@ +import type { Redis } from "@internal/redis"; +import { RunQueueFullKeyProducer } from "../keyProducer.js"; + +/** + * Seeds a keyspace shaped like prod for the concurrency sweeper: `concurrencySets` queue + * currentConcurrency sets (the only keys the sweeper's MATCH accepts), each surrounded by + * `unrelatedPerSet` keys the MATCH/TYPE filter has to skip (env currentConcurrency set, + * currentDequeued set, queue zset, message strings). Returns every run id put in a + * matching set, i.e. exactly what one full sweep must hand to the callback. + */ +export async function seedSweeperKeyspace( + redis: Redis, + { concurrencySets, unrelatedPerSet }: { concurrencySets: number; unrelatedPerSet: number } +): Promise> { + const keys = new RunQueueFullKeyProducer(); + const expected = new Set(); + const batch = 500; + + for (let start = 0; start < concurrencySets; start += batch) { + const pipeline = redis.pipeline(); + for (let i = start; i < Math.min(start + batch, concurrencySets); i++) { + const env = { + id: `e${i % 997}`, + type: "PRODUCTION" as const, + project: { id: `p${i % 997}` }, + organization: { id: `o${i % 331}` }, + }; + const queue = `task/q${i}`; + const runIds = Array.from({ length: (i % 3) + 1 }, (_, j) => `run_${i}_${j}`); + runIds.forEach((id) => expected.add(id)); + pipeline.sadd(keys.queueCurrentConcurrencyKey(env, queue), ...runIds); + + const unrelated = [ + () => pipeline.sadd(keys.envCurrentConcurrencyKey(env), ...runIds), + () => pipeline.sadd(keys.queueCurrentDequeuedKey(env, queue), ...runIds), + () => pipeline.zadd(keys.queueKey(env, queue), Date.now(), runIds[0]!), + ]; + for (let u = 0; u < unrelatedPerSet; u++) { + if (u < unrelated.length) unrelated[u]!(); + else pipeline.set(keys.messageKey(env.organization.id, `msg_${i}_${u}`), "{}"); + } + } + await pipeline.exec(); + } + + return expected; +} + +/** Cumulative calls of `command` the server has handled, from INFO commandstats. */ +export async function commandCalls(redis: Redis, command: string): Promise { + const info = await redis.info("commandstats"); + const match = info.match(new RegExp(`cmdstat_${command}:calls=(\\d+)`)); + return match ? Number(match[1]) : 0; +} diff --git a/internal-packages/run-store/src/PostgresRunStore.dualSchema.test.ts b/internal-packages/run-store/src/PostgresRunStore.dualSchema.test.ts index 1770481300f..bf757048561 100644 --- a/internal-packages/run-store/src/PostgresRunStore.dualSchema.test.ts +++ b/internal-packages/run-store/src/PostgresRunStore.dualSchema.test.ts @@ -376,3 +376,62 @@ describe("PostgresRunStore dual-schema (P2-store-bodies)", () => { } ); }); + +describe("native waitpoint marker", () => { + for (const dedicated of [false, true]) { + const fixture = dedicated ? heteroRunOpsPostgresTest : heteroPostgresTest; + fixture( + `marking is monotonic without redundant row versions (${dedicated ? "dedicated" : "legacy"})`, + async ({ prisma17 }) => { + const schemaVariant = dedicated ? "dedicated" : "legacy"; + const client = prisma17 as unknown as PrismaClient; + const store = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant, + }); + const env = await seedEnvironment(prisma17, schemaVariant, "marker"); + for (const initial of [undefined, { triggerSource: "sdk", workerCell: "cell-a" }]) { + const runId = initial ? "marker-annotated" : "marker-null"; + const input = buildCreateRunInput({ + runId, + friendlyId: `run_${runId}`, + taskIdentifier: "task", + organizationId: env.organization.id, + projectId: env.project.id, + runtimeEnvironmentId: env.environment.id, + }); + if (initial) input.data.annotations = initial; + await store.createRun(input); + const versions = () => + client.$queryRaw< + Array<{ xmin: string; annotations: Record | null }> + >`SELECT xmin::text, "annotations" FROM "TaskRun" WHERE id = ${runId}`; + const before = (await versions())[0]!; + expect(before.annotations).toEqual(initial ?? null); + // Race the FIRST mark with an unrelated JSONB merge on the unmarked row. + await Promise.all([ + store.markRunWaitpointsOnStore(runId), + client.$executeRaw` + UPDATE "TaskRun" + SET "annotations" = COALESCE("annotations", '{}'::jsonb) || '{"independent":"kept"}'::jsonb + WHERE id = ${runId} + `, + ]); + const first = (await versions())[0]!; + expect(first.annotations).toEqual({ + ...initial, + waitpointsOnStore: true, + independent: "kept", + }); + expect(first.xmin).not.toBe(before.xmin); + await store.markRunWaitpointsOnStore(runId); + expect((await versions())[0]!.xmin).toBe(first.xmin); + } + await expect(store.markRunWaitpointsOnStore("missing-run")).rejects.toMatchObject({ + code: "P2025", + }); + } + ); + } +}); diff --git a/internal-packages/run-store/src/PostgresRunStore.ts b/internal-packages/run-store/src/PostgresRunStore.ts index 7a1a35389f2..c281e1e8d7e 100644 --- a/internal-packages/run-store/src/PostgresRunStore.ts +++ b/internal-packages/run-store/src/PostgresRunStore.ts @@ -1053,6 +1053,23 @@ export class PostgresRunStore implements RunStore { ) as Promise>; } + async markRunWaitpointsOnStore(runId: string): Promise { + const marked = await this.prisma.$executeRaw` + UPDATE "TaskRun" + SET "annotations" = + (CASE WHEN jsonb_typeof("annotations") = 'object' + THEN "annotations" ELSE '{}'::jsonb END) || '{"waitpointsOnStore":true}'::jsonb, + "updatedAt" = CURRENT_TIMESTAMP + WHERE "id" = ${runId} + AND ("annotations" -> 'waitpointsOnStore') IS DISTINCT FROM 'true'::jsonb + `; + if (marked === 0) { + // An existing marker is already durable; avoid another row version/replication event. + // Preserve the missing-run error before the caller may absorb native edges. + await this.prisma.taskRun.findFirstOrThrow({ where: { id: runId }, select: { id: true } }); + } + } + async recordBulkActionMembership( runId: string, bulkActionId: string, diff --git a/internal-packages/run-store/src/delegatingRunStore.ts b/internal-packages/run-store/src/delegatingRunStore.ts index 9334bf76a0e..95831f6e60b 100644 --- a/internal-packages/run-store/src/delegatingRunStore.ts +++ b/internal-packages/run-store/src/delegatingRunStore.ts @@ -125,6 +125,10 @@ export class DelegatingRunStore implements RunStore { return this.delegate.requeueRun(runId, args, tx); } + markRunWaitpointsOnStore(runId: string): Promise { + return this.delegate.markRunWaitpointsOnStore(runId); + } + recordBulkActionMembership( runId: string, bulkActionId: string, diff --git a/internal-packages/run-store/src/placementCatalog.ts b/internal-packages/run-store/src/placementCatalog.ts index b11c317b779..7725fadbac3 100644 --- a/internal-packages/run-store/src/placementCatalog.ts +++ b/internal-packages/run-store/src/placementCatalog.ts @@ -24,6 +24,7 @@ export const ROUTES_BY_GIVEN_RUN_ID: readonly string[] = [ "recordRetryOutcome", "requeueRun", "recordBulkActionMembership", + "markRunWaitpointsOnStore", "cancelRun", "failRunPermanently", "finalizeRun", diff --git a/internal-packages/run-store/src/redisSnapshotStore.recoveryLease.test.ts b/internal-packages/run-store/src/redisSnapshotStore.recoveryLease.test.ts index ebc37b0b68b..4d0ba10f068 100644 --- a/internal-packages/run-store/src/redisSnapshotStore.recoveryLease.test.ts +++ b/internal-packages/run-store/src/redisSnapshotStore.recoveryLease.test.ts @@ -188,6 +188,8 @@ describe("RecoverySweeper pending gauges and the recovery lease", () => { }); expect(prepared.outcome).toBe("prepared"); + // Age is measured from Redis's stream timestamp; establish a positive age independently of + // host/Redis clock skew or a prepare and sweep happening in the same millisecond. const { oldestMs } = await index.summary(runToPartition(runId)); expect(oldestMs).toBeTypeOf("number"); vi.useFakeTimers({ toFake: ["Date"] }); diff --git a/internal-packages/run-store/src/runOpsStore.ts b/internal-packages/run-store/src/runOpsStore.ts index 8e0bfa7818f..af3b4fafa67 100644 --- a/internal-packages/run-store/src/runOpsStore.ts +++ b/internal-packages/run-store/src/runOpsStore.ts @@ -954,6 +954,10 @@ export class RoutingRunStore implements RunStore { return (await this.#routeForWrite(runId)).requeueRun(runId, args); } + async markRunWaitpointsOnStore(runId: string): Promise { + return (await this.#routeForWrite(runId)).markRunWaitpointsOnStore(runId); + } + async recordBulkActionMembership( runId: string, bulkActionId: string, diff --git a/internal-packages/run-store/src/runStoreMethodNames.ts b/internal-packages/run-store/src/runStoreMethodNames.ts index 007b16959a2..2b44695192c 100644 --- a/internal-packages/run-store/src/runStoreMethodNames.ts +++ b/internal-packages/run-store/src/runStoreMethodNames.ts @@ -19,6 +19,7 @@ export const RUN_STORE_METHOD_NAMES = [ "recordRetryOutcome", "requeueRun", "recordBulkActionMembership", + "markRunWaitpointsOnStore", "cancelRun", "failRunPermanently", "finalizeRun", diff --git a/internal-packages/run-store/src/types.ts b/internal-packages/run-store/src/types.ts index 93d43e2713d..fa522782a55 100644 --- a/internal-packages/run-store/src/types.ts +++ b/internal-packages/run-store/src/types.ts @@ -554,6 +554,12 @@ export interface RunStore { args: { select: S; workerQueue?: string }, tx?: PrismaClientOrTransaction ): Promise>; + /** + * Write the sticky internal annotations marker before MemoryDB waitpoint state. The caller holds + * the same run lifecycle lock as attempt stamping, so a stamp cannot overwrite a concurrent mark. + */ + markRunWaitpointsOnStore(runId: string): Promise; + recordBulkActionMembership( runId: string, bulkActionId: string, diff --git a/knip.ts b/knip.ts index 806c8597104..2b7062859b7 100644 --- a/knip.ts +++ b/knip.ts @@ -22,7 +22,6 @@ export default { "evalite.config.ts", "vitest.*.config.ts", "evals/**/*.eval.ts", - "memory-leak-detector.js", "prisma/populate.ts", "seed*.{ts,mts}", "scripts/**/*.{js,mjs,cjs,ts,mts,cts}", @@ -79,6 +78,9 @@ export default { ignoreDependencies: ["@epic-web/test-server", "execa", "find-up"], ignoreBinaries: ["xdg-open"], }, + "packages/convex": { + ignore: ["example/**"], + }, "packages/core": { ignoreFiles: ["src/**/*-cjs.cts"], ignoreDependencies: ["ai-v7"], diff --git a/packages/convex/README.md b/packages/convex/README.md index fdc7ab0dfe5..564ab426168 100644 --- a/packages/convex/README.md +++ b/packages/convex/README.md @@ -6,6 +6,12 @@ Use Convex for application state and Trigger.dev for work that needs longer exec **Experimental MVP.** No HTTP routes, webhooks, Trigger lifecycle hooks, or changes to your Trigger server are required. +## Example + +[Try the live demo](https://ardent-rabbit-965.convex.site). + +The [image-editing example](https://github.com/triggerdotdev/trigger.dev/tree/main/packages/convex/example) demonstrates private image processing, Realtime progress, and Convex synchronization. It uses Trigger.dev `4.8.3` from npmjs and runs as a standalone pnpm workspace. The example is not included in npm packages or component builds. + ## Install ```sh @@ -119,6 +125,21 @@ const run = useQuery(api.jobs.status, { jobId }); The ID returned by `trigger.trigger` identifies the **component record**, not the Trigger.dev run. The latter becomes available as `run.runId` after submission. +## Agent skill + +The package ships a versioned [`trigger-convex` skill](https://github.com/triggerdotdev/trigger.dev/blob/main/packages/convex/skills/trigger-convex/SKILL.md) covering transactional triggering, owned reactive queries, callbacks, Realtime + refresh, reload recovery, and token safety. + +Ask your coding assistant to read `node_modules/@trigger.dev/convex/skills/trigger-convex/SKILL.md`, or copy it into your assistant's native skill directory. For example, from a project with this package installed: + +```sh +mkdir -p .agents/skills/trigger-convex +cp node_modules/@trigger.dev/convex/skills/trigger-convex/SKILL.md .agents/skills/trigger-convex/SKILL.md +``` + +Use `.claude/skills/`, `.cursor/skills/`, or `.github/skills/` instead if appropriate for your assistant. Re-copy after upgrading `@trigger.dev/convex`; copied files do not update automatically. For non-hoisted installs, locate the package with `node -p "require.resolve('@trigger.dev/convex/package.json')"`. + +This skill is installed manually, not through `npx trigger.dev@latest skills`. Older package versions may not contain it. + ## API - `trigger(ctx, taskId, payload, options?)`: enqueue work. Calling from a mutation makes the request atomic with your application's writes. diff --git a/packages/convex/example/.env.example b/packages/convex/example/.env.example new file mode 100644 index 00000000000..c7bb693809b --- /dev/null +++ b/packages/convex/example/.env.example @@ -0,0 +1,9 @@ +# Set by convex deployment create --select or convex dev. +VITE_CONVEX_URL= +VITE_CONVEX_SITE_URL= + +# Server-only configuration belongs in Convex, not in VITE_ variables: +# DEMO_ARTIFACT_SECRET (shared with Trigger; configured by scripts/cloud.ts) +# TRIGGER_SECRET_KEY +# TRIGGER_API_URL=https://test-api.trigger.dev +# DEMO_ENABLED=false diff --git a/packages/convex/example/.gitignore b/packages/convex/example/.gitignore new file mode 100644 index 00000000000..8c069238866 --- /dev/null +++ b/packages/convex/example/.gitignore @@ -0,0 +1,7 @@ +node_modules/ +dist/ +.env* +!.env.example +.trigger/ +.convex/ +*.log diff --git a/packages/convex/example/README.md b/packages/convex/example/README.md new file mode 100644 index 00000000000..c9021d45b1a --- /dev/null +++ b/packages/convex/example/README.md @@ -0,0 +1,146 @@ +# Convex Ɨ Trigger.dev: prompt-driven image editing + +A small demo by Trigger.dev: choose one of ten bundled photographs, describe a modification, and get a private, downloadable edited image. + +**Public app:** https://ardent-rabbit-965.convex.site + +## Current state + +Backend, frontend, and Trigger tasks are deployed, and **image editing is enabled**. All Trigger.dev dependencies are pinned to `4.8.3` from npmjs, including `@trigger.dev/convex` and `@trigger.dev/react-hooks`. + +A live Realtime verification completed the prepare → model edit → parallel exports workflow in **26.2 seconds**, streamed real phase metadata, requested authoritative Convex synchronization on completion, and verified all three private AVIF downloads. Unauthorized token issuance, cross-session refresh, cross-run token reads, and unauthenticated image access were rejected. The readable task payload contained no worker credential. A separate browser run verified the React subscription, terminal refresh, and final image delivery. + +`AI_GATEWAY_API_KEY` belongs only in Trigger environment variables, never Convex, Vite, source files, or the browser. + +## Page flow + +The page has three vertical steps: choose an image, describe the edit, and view/download the result. Only one source photo is shown; a native dialog offers the other photographs. Export settings, internal task details, and budget counters are deliberately not exposed as controls. + +A fresh visit starts with an empty prompt and result. Old results appear only after an explicit submission or history selection, never merely because their inputs match. The current draft, selected job, and submission request ID are saved for this browser session. Refreshing restores the exact photo, prompt, and job; an owned active job is also recovered from Convex when local editor state is missing. The request ID finds accepted work even if the browser missed its submission response. Recovery does not automatically enqueue another generation. + +Restored results are labeled, and **New edit** clears the prompt/result while retaining the selected photo. Every disabled Submit state explains its reason. Selecting a previous edit restores its inputs. The page caches all generated variants privately, while showing just the primary image. + +During work, one progress bar and three labels, **Prepare → Edit → Finish**, show actual workflow progress. There are no elapsed-time estimates or artificial progress increments. Convex still controls job completion and when verified results become visible. + +## Actual Trigger workflow + +The parent task `convex-edit-image` orchestrates: + +1. **Prepare** (`convex-image-prepare`): load the selected bundled JPEG, strip metadata, and downsize to at most 1024 pixels per side. +2. **Edit** (`convex-image-model-edit`): send the photo and prompt to `meta/muse-image-1.0` through Vercel AI Gateway. Request exactly one image, with no automatic paid retries. Normalize the result to a bounded PNG. +3. **Export** (`convex-image-export`): fan out three child tasks concurrently, creating 1024 Ɨ 768, 800 Ɨ 600, and 400 Ɨ 400 AVIF versions using Sharp. +4. **Finish**: the parent collects successful exports. The component's terminal callback verifies the files, publishes safe metadata, and releases the admission lock. + +Child tasks have deterministic job-specific idempotency keys. Every task has one attempt. A failed export does not rerun the image model automatically. There are no fake timers or artificial delays. + +Convex transactionally admits the request and enqueues the Trigger component operation. For the selected active job, the browser uses `useRealtimeRun` with an ownership-authorized, exact-run token. Parent metadata reports each real phase; export children set idempotent completion flags. A terminal Realtime event invokes the ownership-checked `jobs.refresh` mutation, which calls the component’s `trigger.refresh()` API. It never forwards browser-provided status or output. + +The component still polls Trigger every 15–60 seconds as recovery when browsers close or Realtime is unavailable. Registered artifacts also update Convex reactively. Progress is held at or below 95 until the completion callback verifies all files. + +## Privacy and retention + +- Each browser creates a cryptographically random 256-bit session capability. The server stores only its hash and scopes every job query to it. +- No shared gallery, public result query, or cross-user prompt cache. +- Images are served through an authenticated HTTP action, not public Convex storage URLs. Downloads require the browser's capability in an Authorization header. Tokens never go in image URLs. +- Trigger workers derive a job-scoped HMAC capability from `DEMO_ARTIFACT_SECRET`, configured only in Trigger and Convex server environments. No worker capability or shared secret appears in a task payload, metadata, or output. Browsers cannot upload artifacts or read intermediate files. +- Browser Realtime tokens authorize only the owned parent run, expire within ten minutes or job expiry, and remain in memory. Realtime omits payload/output columns, but this is not REST protection: the task payload must still be credential-free. Legacy jobs created before this migration cannot receive Realtime tokens. +- Task payloads and outputs contain references/metadata, not image bytes. Provider errors are sanitized to avoid recording image bodies in task errors. Prompt/task metadata is still available to trusted operators in private dashboards; this is not end-to-end encryption. +- The browser fetches final images as Blobs, previews them with object URLs, and keeps up to 20 jobs in session-scoped IndexedDB. Server URLs are not persisted in the local cache. +- Normal server files and job records expire one hour after admission. Intermediate files are removed on completion, and failed jobs have their registered files removed immediately. An hourly indexed sweep collects unregistered files left by process interruption, normally within two hours of upload. +- Root Convex storage is dedicated to this demo's artifacts. Static hosting uses isolated component storage. Scope the collector before introducing unrelated root-storage files. + +Losing the browser session key loses access to pending remote jobs. Clearing local history does not erase the session key or immediately erase unexpired remote jobs. Browser storage can be evicted; downloads are the reliable way to retain files permanently. If IndexedDB is unavailable, current-page download links still work. + +The image provider receives the selected bundled image and prompt. There are no visitor uploads or arbitrary input URLs. + +## Spending and abuse limits + +- Fixed model, exactly one generated image, and prompts capped at 500 characters. +- **500 lifetime admitted edits and 50 per UTC day**, enforced atomically before enqueueing. Counters persist across deployments and browser sessions. +- At the catalog's current $0.01/image rate, 500 generations is approximately **$5 in model charges**, excluding compute, storage, and traffic. These are admission limits, not a guarantee against future provider price changes. +- One outstanding job globally, a shared 20-second cooldown, five-minute task queue TTLs, and bounded execution durations. +- No automatic retries at either the Trigger task or AI SDK level. Failed jobs still consume their reserved slot, including definite pre-model failures. No automatic refunds on ambiguous failures. +- Fixed image inputs, pixel limits, bounded streamed artifact uploads, and no user-selected model/provider/size/count. +- `DEMO_ENABLED` is an operator kill switch. The manual deployment procedure pauses new work until deployment and hosting checks finish. +- Optional Turnstile is implemented. Set both `TURNSTILE_SITE_KEY` and `TURNSTILE_SECRET_KEY` in Convex to require a successful challenge with the deployment's hostname. A partially configured pair fails closed. Without these keys, spending caps still apply but CAPTCHA is not active. + +These measures bound admitted image-generation work. They are not DDoS protection for public Convex endpoints. Use additional traffic controls before broad promotion. + +## Setup and local development + +This is a standalone, private pnpm workspace under `packages/convex/example`. Its dependencies, tests, assets, and build output are excluded from the `@trigger.dev/convex` package build and npm tarball. Run its commands from this directory, not the monorepo root. + +```sh +corepack pnpm install --frozen-lockfile +corepack pnpm dev +corepack pnpm dev:convex +``` + +Without `VITE_CONVEX_URL`, the page is an honest read-only preview. + +Current development deployment: `chris-arderne:tiny-worlds:dev/demo`, https://ardent-rabbit-965.convex.cloud. The historical Convex project name is unchanged. [Convex dashboard](https://dashboard.convex.dev/d/ardent-rabbit-965). + +Trigger project: `convex-image-processing`, reference `proj_wcwcgcodbajkuocrovpq`. CLI profile **test**, environment **prod on test cloud**, not production cloud. [Trigger dashboard](https://test-cloud.trigger.dev/projects/v3/proj_wcwcgcodbajkuocrovpq). + +Use Node.js 24 and pnpm 10.33.2. Authenticate the Convex CLI for the selected deployment and the Trigger CLI's `test` profile for test cloud. Ensure `.env.local` contains the selected deployment's `VITE_CONVEX_URL` and `VITE_CONVEX_SITE_URL` (see `.env.example`). Add `AI_GATEWAY_API_KEY` to the Trigger environment above before deploying tasks. + +`configure` safely resolves the project's Trigger credentials from the CLI profile, creates or reuses the server-only artifact secret, and injects Convex configuration over stdin without printing secret values. Run it before the first task deployment so worker authentication is configured. Omit `--enable` to keep new work paused. For a different Trigger project, deliberately update the config and operator helper. + +Follow the manual deployment steps below. The helper and `deploy:trigger` command target the existing test-cloud project; for another project, update `trigger.config.ts` and `scripts/cloud.ts` first. + +Disable new work with: + +```sh +corepack pnpm exec convex env set DEMO_ENABLED false +``` + +If component synchronization pauses, restore credentials and invoke the authenticated internal `jobs:resume` mutation with the job ID. Visitors cannot invoke it. + +## Manual deployment + +There is no demo CI or automatic deployment. Run these commands from `packages/convex/example`; no GitHub environments or repository secrets are required. They update the existing demo's selected **development** Convex deployment and its hosted frontend. + +Install dependencies, check the example, pause new edits, and configure server credentials before deploying: + +```sh +corepack pnpm install --frozen-lockfile +corepack pnpm test +node scripts/cloud.ts configure +corepack pnpm exec convex dev --once --typecheck enable +corepack pnpm deploy:trigger --local-bundle +node scripts/cloud.ts status +``` + +Confirm the Trigger status is `DEPLOYED` before continuing. Then build/upload the frontend and check its public assets: + +```sh +corepack pnpm upload:site +node scripts/verify-hosting.ts +``` + +`upload:site` runs the frontend build. Once deployment and hosting checks pass, explicitly enable new edits and check readiness without creating a generation: + +```sh +node scripts/cloud.ts configure --enable +node scripts/verify.ts +``` + +Optionally run one paid end-to-end verification: + +```sh +node scripts/verify.ts --launch --realtime +``` + +`--realtime` checks scoped token access, consumes live run updates, and requests a Convex refresh on the terminal frame. `--launch` explicitly spends one generation. The stable operator session/request is reused while the server job exists. If Turnstile is configured, use the browser for live generation; the CLI does not bypass the challenge. + +`verify.ts` without `--launch` checks readiness and hosted samples without creating a model job. With an existing verification job it checks that job and authenticated downloads. Tests cover ownership of token issuance and refresh, job-scoped worker authentication, Realtime progress identity/clamping, cross-session isolation, authenticated file delivery/expiry, orphan cleanup, limits/deduplication, invalid result handling, native AVIF decoding, and bounded model-call configuration. + +Live browser recovery QA submitted a Coast edit with Enter, refreshed mid-generation, left the page entirely for 35 seconds, and returned to the same job and verified result. Draft restoration, Shift+Enter, New edit/reset persistence, and explicit history selection were also checked without additional paid work. + +Convex does not currently support renaming the generated `*.convex.site` subdomain ([feature request](https://ship.convex.dev/requests/js7ch5fteyhskr871g4w9ce5n58e1zh7)). An owned [custom domain](https://docs.convex.dev/production/custom-domains) requires Convex Pro and DNS configuration. Changing origins also creates a separate browser session/history unless a migration is provided. + +`upload:site` updates the selected development frontend. `deploy:site` targets the default production deployment and deploys backend plus frontend. Do not confuse them. + +The ten frozen JPEGs in `public/samples` come from Unsplash; source URLs are in `shared/catalog.ts`. See the [Unsplash license](https://unsplash.com/license). `node scripts/download-samples.ts` refreshes and bounds them to 1600 pixels per side; `--resize-only` normalizes existing local files without fetching again. + +The example source lives alongside the component in `packages/convex/example` and syncs with the public repository. It is not included in published package bundles. diff --git a/packages/convex/example/convex/_generated/api.d.ts b/packages/convex/example/convex/_generated/api.d.ts new file mode 100644 index 00000000000..050d478c7bb --- /dev/null +++ b/packages/convex/example/convex/_generated/api.d.ts @@ -0,0 +1,52 @@ +/* eslint-disable */ +/** + * Generated `api` utility. + * + * THIS CODE IS AUTOMATICALLY GENERATED. + * + * To regenerate, run `npx convex dev`. + * @module + */ + +import type * as auth from "../auth.js"; +import type * as crons from "../crons.js"; +import type * as http from "../http.js"; +import type * as jobs from "../jobs.js"; +import type * as realtime from "../realtime.js"; +import type * as trigger from "../trigger.js"; + +import type { ApiFromModules, FilterApi, FunctionReference } from "convex/server"; + +declare const fullApi: ApiFromModules<{ + auth: typeof auth; + crons: typeof crons; + http: typeof http; + jobs: typeof jobs; + realtime: typeof realtime; + trigger: typeof trigger; +}>; + +/** + * A utility for referencing Convex functions in your app's public API. + * + * Usage: + * ```js + * const myFunctionReference = api.myModule.myFunction; + * ``` + */ +export declare const api: FilterApi>; + +/** + * A utility for referencing Convex functions in your app's internal API. + * + * Usage: + * ```js + * const myFunctionReference = internal.myModule.myFunction; + * ``` + */ +export declare const internal: FilterApi>; + +export declare const components: { + trigger: import("@trigger.dev/convex/_generated/component.js").ComponentApi<"trigger">; + staticHosting: import("@convex-dev/static-hosting/_generated/component.js").ComponentApi<"staticHosting">; +}; diff --git a/packages/convex/example/convex/_generated/api.js b/packages/convex/example/convex/_generated/api.js new file mode 100644 index 00000000000..44bf9858320 --- /dev/null +++ b/packages/convex/example/convex/_generated/api.js @@ -0,0 +1,23 @@ +/* eslint-disable */ +/** + * Generated `api` utility. + * + * THIS CODE IS AUTOMATICALLY GENERATED. + * + * To regenerate, run `npx convex dev`. + * @module + */ + +import { anyApi, componentsGeneric } from "convex/server"; + +/** + * A utility for referencing Convex functions in your app's API. + * + * Usage: + * ```js + * const myFunctionReference = api.myModule.myFunction; + * ``` + */ +export const api = anyApi; +export const internal = anyApi; +export const components = componentsGeneric(); diff --git a/packages/convex/example/convex/_generated/dataModel.d.ts b/packages/convex/example/convex/_generated/dataModel.d.ts new file mode 100644 index 00000000000..4824efde609 --- /dev/null +++ b/packages/convex/example/convex/_generated/dataModel.d.ts @@ -0,0 +1,56 @@ +/* eslint-disable */ +/** + * Generated data model types. + * + * THIS CODE IS AUTOMATICALLY GENERATED. + * + * To regenerate, run `npx convex dev`. + * @module + */ + +import type { + DataModelFromSchemaDefinition, + DocumentByName, + TableNamesInDataModel, + SystemTableNames, +} from "convex/server"; +import type { GenericId } from "convex/values"; +import schema from "../schema.js"; + +/** + * The names of all of your Convex tables. + */ +export type TableNames = TableNamesInDataModel; + +/** + * The type of a document stored in Convex. + * + * @typeParam TableName - A string literal type of the table name (like "users"). + */ +export type Doc = DocumentByName; + +/** + * An identifier for a document in Convex. + * + * Convex documents are uniquely identified by their `Id`, which is accessible + * on the `_id` field. To learn more, see [Document IDs](https://docs.convex.dev/using/document-ids). + * + * Documents can be loaded using `db.get(tableName, id)` in query and mutation functions. + * + * IDs are just strings at runtime, but this type can be used to distinguish them from other + * strings when type checking. + * + * @typeParam TableName - A string literal type of the table name (like "users"). + */ +export type Id = GenericId; + +/** + * A type describing your Convex data model. + * + * This type includes information about what tables you have, the type of + * documents stored in those tables, and the indexes defined on them. + * + * This type is used to parameterize methods like `queryGeneric` and + * `mutationGeneric` to make them type-safe. + */ +export type DataModel = DataModelFromSchemaDefinition; diff --git a/packages/convex/example/convex/_generated/server.d.ts b/packages/convex/example/convex/_generated/server.d.ts new file mode 100644 index 00000000000..f235db4a112 --- /dev/null +++ b/packages/convex/example/convex/_generated/server.d.ts @@ -0,0 +1,162 @@ +/* eslint-disable */ +/** + * Generated utilities for implementing server-side Convex query and mutation functions. + * + * THIS CODE IS AUTOMATICALLY GENERATED. + * + * To regenerate, run `npx convex dev`. + * @module + */ + +import { + ActionBuilder, + HttpActionBuilder, + MutationBuilder, + QueryBuilder, + GenericActionCtx, + GenericMutationCtx, + GenericQueryCtx, + GenericDatabaseReader, + GenericDatabaseWriter, +} from "convex/server"; +import type { DataModel } from "./dataModel.js"; + +/** + * Typesafe environment variables. + * + * This includes platform-provided env vars and any variables declared in + * `convex.config.ts`. + */ +type Env = { + readonly CONVEX_CLOUD_URL: string; + readonly CONVEX_SITE_URL: string; +}; + +/** + * Define a query in this Convex app's public API. + * + * This function will be allowed to read your Convex database and will be accessible from the client. + * + * @param func - The query function. It receives a {@link QueryCtx} as its first argument. + * @returns The wrapped query. Include this as an `export` to name it and make it accessible. + */ +export declare const query: QueryBuilder; + +/** + * Define a query that is only accessible from other Convex functions (but not from the client). + * + * This function will be allowed to read from your Convex database. It will not be accessible from the client. + * + * @param func - The query function. It receives a {@link QueryCtx} as its first argument. + * @returns The wrapped query. Include this as an `export` to name it and make it accessible. + */ +export declare const internalQuery: QueryBuilder; + +/** + * Define a mutation in this Convex app's public API. + * + * This function will be allowed to modify your Convex database and will be accessible from the client. + * + * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. + * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. + */ +export declare const mutation: MutationBuilder; + +/** + * Define a mutation that is only accessible from other Convex functions (but not from the client). + * + * This function will be allowed to modify your Convex database. It will not be accessible from the client. + * + * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. + * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. + */ +export declare const internalMutation: MutationBuilder; + +/** + * Define an action in this Convex app's public API. + * + * An action is a function which can execute any JavaScript code, including non-deterministic + * code and code with side-effects, like calling third-party services. + * They can be run in Convex's JavaScript environment or in Node.js using the "use node" directive. + * They can interact with the database indirectly by calling queries and mutations using the {@link ActionCtx}. + * + * @param func - The action. It receives an {@link ActionCtx} as its first argument. + * @returns The wrapped action. Include this as an `export` to name it and make it accessible. + */ +export declare const action: ActionBuilder; + +/** + * Define an action that is only accessible from other Convex functions (but not from the client). + * + * @param func - The function. It receives an {@link ActionCtx} as its first argument. + * @returns The wrapped function. Include this as an `export` to name it and make it accessible. + */ +export declare const internalAction: ActionBuilder; + +/** + * Define an HTTP action. + * + * The wrapped function will be used to respond to HTTP requests received + * by a Convex deployment if the requests matches the path and method where + * this action is routed. Be sure to route your httpAction in `convex/http.js`. + * + * @param func - The function. It receives an {@link ActionCtx} as its first argument + * and a Fetch API `Request` object as its second. + * @returns The wrapped function. Import this function from `convex/http.js` and route it to hook it up. + */ +export declare const httpAction: HttpActionBuilder; + +/** + * Typesafe environment variables. + * + * This includes platform-provided env vars and any variables declared in + * `convex.config.ts`. + */ +export declare const env: Env; + +/** + * A set of services for use within Convex query functions. + * + * The query context is passed as the first argument to any Convex query + * function run on the server. + * + * This differs from the {@link MutationCtx} because all of the services are + * read-only. + */ +export type QueryCtx = GenericQueryCtx; + +/** + * A set of services for use within Convex mutation functions. + * + * The mutation context is passed as the first argument to any Convex mutation + * function run on the server. + */ +export type MutationCtx = GenericMutationCtx; + +/** + * A set of services for use within Convex action functions. + * + * The action context is passed as the first argument to any Convex action + * function run on the server. + */ +export type ActionCtx = GenericActionCtx; + +/** + * An interface to read from the database within Convex query functions. + * + * The two entry points are {@link DatabaseReader.get}, which fetches a single + * document by its {@link Id}, or {@link DatabaseReader.query}, which starts + * building a query. + */ +export type DatabaseReader = GenericDatabaseReader; + +/** + * An interface to read from and write to the database within Convex mutation + * functions. + * + * Convex guarantees that all writes within a single mutation are + * executed atomically, so you never have to worry about partial writes leaving + * your data in an inconsistent state. See [the Convex Guide](https://docs.convex.dev/understanding/convex-fundamentals/functions#atomicity-and-optimistic-concurrency-control) + * for the guarantees Convex provides your functions. + */ +export type DatabaseWriter = GenericDatabaseWriter; diff --git a/packages/convex/example/convex/_generated/server.js b/packages/convex/example/convex/_generated/server.js new file mode 100644 index 00000000000..2dbe0dbd475 --- /dev/null +++ b/packages/convex/example/convex/_generated/server.js @@ -0,0 +1,101 @@ +/* eslint-disable */ +/** + * Generated utilities for implementing server-side Convex query and mutation functions. + * + * THIS CODE IS AUTOMATICALLY GENERATED. + * + * To regenerate, run `npx convex dev`. + * @module + */ + +import { + actionGeneric, + httpActionGeneric, + queryGeneric, + mutationGeneric, + internalActionGeneric, + internalMutationGeneric, + internalQueryGeneric, +} from "convex/server"; + +/** + * Define a query in this Convex app's public API. + * + * This function will be allowed to read your Convex database and will be accessible from the client. + * + * @param func - The query function. It receives a {@link QueryCtx} as its first argument. + * @returns The wrapped query. Include this as an `export` to name it and make it accessible. + */ +export const query = queryGeneric; + +/** + * Define a query that is only accessible from other Convex functions (but not from the client). + * + * This function will be allowed to read from your Convex database. It will not be accessible from the client. + * + * @param func - The query function. It receives a {@link QueryCtx} as its first argument. + * @returns The wrapped query. Include this as an `export` to name it and make it accessible. + */ +export const internalQuery = internalQueryGeneric; + +/** + * Define a mutation in this Convex app's public API. + * + * This function will be allowed to modify your Convex database and will be accessible from the client. + * + * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. + * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. + */ +export const mutation = mutationGeneric; + +/** + * Define a mutation that is only accessible from other Convex functions (but not from the client). + * + * This function will be allowed to modify your Convex database. It will not be accessible from the client. + * + * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. + * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. + */ +export const internalMutation = internalMutationGeneric; + +/** + * Define an action in this Convex app's public API. + * + * An action is a function which can execute any JavaScript code, including non-deterministic + * code and code with side-effects, like calling third-party services. + * They can be run in Convex's JavaScript environment or in Node.js using the "use node" directive. + * They can interact with the database indirectly by calling queries and mutations using the {@link ActionCtx}. + * + * @param func - The action. It receives an {@link ActionCtx} as its first argument. + * @returns The wrapped action. Include this as an `export` to name it and make it accessible. + */ +export const action = actionGeneric; + +/** + * Define an action that is only accessible from other Convex functions (but not from the client). + * + * @param func - The function. It receives an {@link ActionCtx} as its first argument. + * @returns The wrapped function. Include this as an `export` to name it and make it accessible. + */ +export const internalAction = internalActionGeneric; + +/** + * Define an HTTP action. + * + * The wrapped function will be used to respond to HTTP requests received + * by a Convex deployment if the requests matches the path and method where + * this action is routed. Be sure to route your httpAction in `convex/http.js`. + * + * @param func - The function. It receives an {@link ActionCtx} as its first argument + * and a Fetch API `Request` object as its second. + * @returns The wrapped function. Import this function from `convex/http.js` and route it to hook it up. + */ +export const httpAction = httpActionGeneric; + +/** + * Typesafe environment variables. + * + * This includes platform-provided env vars and any variables declared in + * `convex.config.ts`. + */ +export const env = process.env; diff --git a/packages/convex/example/convex/auth.ts b/packages/convex/example/convex/auth.ts new file mode 100644 index 00000000000..26a503189d9 --- /dev/null +++ b/packages/convex/example/convex/auth.ts @@ -0,0 +1,28 @@ +import type { QueryCtx } from "./_generated/server"; +import type { Id } from "./_generated/dataModel"; + +export async function ownedEdit( + ctx: Pick, + jobId: Id<"edits">, + sessionToken: string +) { + const owner = await ownerHash(sessionToken); + const job = await ctx.db.get(jobId); + if (!job || job.ownerHash !== owner || job.expiresAt <= Date.now()) + throw new Error("Job not found"); + return job; +} + +export async function digest(value: string) { + const bytes = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(value)); + return Array.from(new Uint8Array(bytes), (byte) => byte.toString(16).padStart(2, "0")).join(""); +} +export async function ownerHash(token: string) { + if (!/^[a-f0-9]{64}$/.test(token)) throw new Error("Invalid browser session"); + return digest(token); +} +export function randomToken() { + return Array.from(crypto.getRandomValues(new Uint8Array(32)), (byte) => + byte.toString(16).padStart(2, "0") + ).join(""); +} diff --git a/packages/convex/example/convex/convex.config.ts b/packages/convex/example/convex/convex.config.ts new file mode 100644 index 00000000000..37d91650ace --- /dev/null +++ b/packages/convex/example/convex/convex.config.ts @@ -0,0 +1,8 @@ +import { defineApp } from "convex/server"; +import staticHosting from "@convex-dev/static-hosting/convex.config.js"; +import trigger from "@trigger.dev/convex/convex.config.js"; + +const app = defineApp({ httpPrefix: "/api" }); +app.use(trigger); +app.use(staticHosting, { httpPrefix: "/" }); +export default app; diff --git a/packages/convex/example/convex/crons.ts b/packages/convex/example/convex/crons.ts new file mode 100644 index 00000000000..c984dc7eb43 --- /dev/null +++ b/packages/convex/example/convex/crons.ts @@ -0,0 +1,6 @@ +import { cronJobs } from "convex/server"; +import { internal } from "./_generated/api"; + +const crons = cronJobs(); +crons.interval("collect orphaned image files", { hours: 1 }, internal.jobs.sweepStorage, {}); +export default crons; diff --git a/packages/convex/example/convex/http.ts b/packages/convex/example/convex/http.ts new file mode 100644 index 00000000000..30c72e8a11f --- /dev/null +++ b/packages/convex/example/convex/http.ts @@ -0,0 +1,104 @@ +import { httpRouter } from "convex/server"; +import { httpAction } from "./_generated/server"; +import { internal } from "./_generated/api"; +import type { Id } from "./_generated/dataModel"; + +const MAX_ARTIFACT_BYTES = 5 * 1024 * 1024; +const CONTENT_TYPES = { + prepared: "image/jpeg", + edited: "image/png", + hero: "image/avif", + card: "image/avif", + square: "image/avif", +} as const; +const DEV_ORIGINS = [ + "http://127.0.0.1:5173", + "http://127.0.0.1:5174", + "http://localhost:5173", + "http://localhost:5174", +]; +const http = httpRouter(); + +const artifact = httpAction(async (ctx, request) => { + const origin = request.headers.get("Origin"); + const allowed = [process.env.CONVEX_SITE_URL, ...DEV_ORIGINS]; + const headers = new Headers({ + "Cache-Control": "private, no-store", + Vary: "Authorization, Origin", + "X-Content-Type-Options": "nosniff", + }); + if (origin && !allowed.includes(origin)) return new Response(null, { status: 403, headers }); + if (origin) headers.set("Access-Control-Allow-Origin", origin); + if (request.method === "OPTIONS") { + headers.set("Access-Control-Allow-Methods", "GET, POST, OPTIONS"); + headers.set("Access-Control-Allow-Headers", "Authorization, Content-Type"); + return new Response(null, { status: 204, headers }); + } + const token = request.headers.get("Authorization")?.match(/^Bearer ([a-f0-9]{64})$/)?.[1]; + if (!token) return new Response(null, { status: 401, headers }); + const url = new URL(request.url); + const slot = url.searchParams.get("slot"); + const jobId = url.searchParams.get("jobId") as Id<"edits"> | null; + if (!slot || !Object.hasOwn(CONTENT_TYPES, slot) || !jobId) + return new Response(null, { status: 404, headers }); + const kind = slot as keyof typeof CONTENT_TYPES; + const write = request.method === "POST"; + let access; + try { + access = await ctx.runQuery(internal.jobs.fileAccess, { jobId, token, slot: kind, write }); + } catch { + return new Response(null, { status: 404, headers }); + } + if (!access) return new Response(null, { status: 404, headers }); + if (!write) { + const blob = access.storageId ? await ctx.storage.get(access.storageId) : null; + if (!blob) return new Response(null, { status: 404, headers }); + headers.set("Content-Type", CONTENT_TYPES[kind]); + headers.set( + "Content-Disposition", + `inline; filename="${kind}.${kind === "prepared" ? "jpg" : kind === "edited" ? "png" : "avif"}"` + ); + return new Response(blob, { headers }); + } + if (request.headers.get("Content-Type") !== CONTENT_TYPES[kind]) + return new Response(null, { status: 415, headers }); + if (Number(request.headers.get("Content-Length")) > MAX_ARTIFACT_BYTES) + return new Response(null, { status: 413, headers }); + const reader = request.body?.getReader(); + if (!reader) return new Response(null, { status: 400, headers }); + const chunks: Uint8Array[] = []; + let length = 0; + while (true) { + const { value, done } = await reader.read(); + if (done) break; + length += value.byteLength; + if (length > MAX_ARTIFACT_BYTES) { + await reader.cancel(); + return new Response(null, { status: 413, headers }); + } + chunks.push(new Uint8Array(value)); + } + if (!length) return new Response(null, { status: 400, headers }); + const blob = new Blob(chunks, { type: CONTENT_TYPES[kind] }); + // A process failure between storage and registration is recovered by the storage sweep. + const storageId = await ctx.storage.store(blob); + try { + if ( + !(await ctx.runMutation(internal.jobs.registerFile, { jobId, token, slot: kind, storageId })) + ) { + await ctx.storage.delete(storageId); + return new Response(null, { status: 410, headers }); + } + } catch { + await ctx.storage.delete(storageId); + return new Response(null, { status: 503, headers }); + } + return new Response(null, { status: 201, headers }); +}); + +http.route({ path: "/image", method: "GET", handler: artifact }); +http.route({ path: "/image", method: "OPTIONS", handler: artifact }); +http.route({ path: "/artifacts", method: "POST", handler: artifact }); +http.route({ path: "/artifacts", method: "GET", handler: artifact }); +http.route({ path: "/artifacts", method: "OPTIONS", handler: artifact }); +export default http; diff --git a/packages/convex/example/convex/jobs.test.ts b/packages/convex/example/convex/jobs.test.ts new file mode 100644 index 00000000000..2aca1a1563e --- /dev/null +++ b/packages/convex/example/convex/jobs.test.ts @@ -0,0 +1,241 @@ +import { afterEach, beforeEach, expect, test, vi } from "vitest"; +import { convexTest } from "convex-test"; +import componentSchema from "../node_modules/@trigger.dev/convex/dist/component/schema.js"; +import { api, internal } from "./_generated/api"; +import schema from "./schema"; +import { jobWorkerToken } from "../shared/workerAuth"; +import { COOLDOWN_MS, MAX_JOBS, DAILY_JOB_LIMIT, RESULT_TTL_MS, VARIANTS } from "../shared/catalog"; +import type { Id } from "./_generated/dataModel"; + +const OWNER = "a".repeat(64); +const OTHER = "b".repeat(64); +const DAY_MS = 24 * 60 * 60_000; +const args = { + sessionToken: OWNER, + sample: "forest" as const, + preset: "balanced" as const, + prompt: "Make it winter", + requestId: "request-first", +}; +const modules = import.meta.glob("./**/*.ts"); +const componentModules = { + "./_generated/component.js": () => + import("../node_modules/@trigger.dev/convex/dist/component/_generated/component.js"), + "./runs.js": () => import("../node_modules/@trigger.dev/convex/dist/component/runs.js"), + "./worker.js": () => import("../node_modules/@trigger.dev/convex/dist/component/worker.js"), +}; +function setup() { + const t = convexTest(schema, modules); + t.registerComponent("trigger", componentSchema, componentModules); + return t; +} +beforeEach(() => { + vi.useFakeTimers(); + vi.stubEnv("DEMO_ENABLED", "true"); + vi.stubEnv("TRIGGER_SECRET_KEY", "test-only"); + vi.stubEnv("DEMO_ARTIFACT_SECRET", "d".repeat(64)); + vi.stubEnv("CONVEX_SITE_URL", "https://example.convex.site"); + vi.stubEnv("TURNSTILE_SITE_KEY", undefined); + vi.stubEnv("TURNSTILE_SECRET_KEY", undefined); +}); +afterEach(() => { + vi.useRealTimers(); + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); +}); + +async function finish(t: ReturnType, id: string, output?: unknown) { + const job = await t.run((ctx) => ctx.db.get(id as Id<"edits">)); + if (!job?.componentRunId) throw new Error("Job missing"); + await t.mutation(internal.jobs.complete, { + run: { + id: job.componentRunId, + taskId: "convex-edit-image", + status: output ? "COMPLETED" : "FAILED", + syncStatus: "complete", + createdAt: Date.now(), + updatedAt: Date.now(), + ...(output ? { output } : {}), + }, + }); +} +const board = (t: ReturnType, sessionToken = OWNER) => + t.query(api.jobs.board, { sessionToken }); + +test("jobs and cached requests are private to their browser session", async () => { + const t = setup(); + const first = await t.action(api.jobs.process, args); + expect(await t.action(api.jobs.process, args)).toEqual({ jobId: first.jobId, cached: true }); + expect((await board(t)).jobs.map((job) => job.prompt)).toEqual([args.prompt]); + expect((await board(t, OTHER)).jobs).toEqual([]); + await expect(t.query(api.jobs.board, { sessionToken: "guessable" })).rejects.toThrow( + "Invalid browser session" + ); + await expect(t.action(api.jobs.process, { ...args, prompt: "different edit" })).rejects.toThrow( + "different edits" + ); + await finish(t, first.jobId); + vi.setSystemTime(Date.now() + COOLDOWN_MS); + const second = await t.action(api.jobs.process, { ...args, sessionToken: OTHER }); + expect(second.cached).toBe(false); + expect(second.jobId).not.toBe(first.jobId); + expect((await board(t, OTHER)).jobs).toHaveLength(1); +}); + +test("Realtime credentials and refresh require ownership, and legacy runs stay polling-only", async () => { + const t = setup(); + const first = await t.action(api.jobs.process, args); + const input = { jobId: first.jobId as Id<"edits">, sessionToken: OTHER }; + await expect(t.action(api.realtime.token, input)).rejects.toThrow("Job not found"); + await expect(t.mutation(api.jobs.refresh, input)).rejects.toThrow("Job not found"); + await t.run((ctx) => ctx.db.patch(input.jobId, { realtimeEligible: undefined })); + expect(await t.action(api.realtime.token, { ...input, sessionToken: OWNER })).toBeNull(); + await t.mutation(api.jobs.refresh, { ...input, sessionToken: OWNER }); + expect((await board(t)).quota.used).toBe(1); +}); + +test("global lifetime and daily budgets cannot be reset with another session", async () => { + const t = setup(); + await t.run((ctx) => + ctx.db.insert("launchControl", { used: 4, modelUsed: MAX_JOBS - 1, lastLaunchAt: 0 }) + ); + const first = await t.action(api.jobs.process, args); + await finish(t, first.jobId); + vi.setSystemTime(Date.now() + COOLDOWN_MS); + await expect(t.action(api.jobs.process, { ...args, sessionToken: OTHER })).rejects.toThrow( + "lifetime generation budget" + ); + expect((await board(t)).quota.used).toBe(MAX_JOBS); + const daily = setup(); + await daily.run((ctx) => + ctx.db.insert("launchControl", { + used: 4, + modelUsed: 0, + dayStart: Math.floor(Date.now() / DAY_MS) * DAY_MS, + dayUsed: DAILY_JOB_LIMIT, + lastLaunchAt: 0, + }) + ); + await expect(daily.action(api.jobs.process, args)).rejects.toThrow("Today's generation budget"); + vi.setSystemTime(Date.now() + DAY_MS); + await daily.action(api.jobs.process, args); + expect((await board(daily)).quota.dailyUsed).toBe(1); +}); + +test("active-job fencing, cooldown and the kill switch prevent extra paid work", async () => { + const t = setup(); + const first = await t.action(api.jobs.process, args); + await expect( + t.action(api.jobs.process, { ...args, requestId: "request-other", prompt: "Make it summer" }) + ).rejects.toThrow("Another background job"); + await finish(t, first.jobId); + await expect(t.action(api.jobs.process, { ...args, requestId: "request-other" })).rejects.toThrow( + "cooldown" + ); + vi.setSystemTime(Date.now() + COOLDOWN_MS); + await t.action(api.jobs.process, { ...args, requestId: "request-second" }); + await finish(t, first.jobId); + expect((await board(t)).quota.active).toBe(true); + vi.stubEnv("DEMO_ENABLED", "false"); + await expect( + t.action(api.jobs.process, { ...args, requestId: "request-disabled", prompt: "New edit" }) + ).rejects.toThrow("paused"); + expect(await t.action(api.jobs.process, args)).toEqual({ jobId: first.jobId, cached: true }); + expect((await board(t)).quota.used).toBe(2); +}); + +test("private HTTP artifacts reject other sessions and expire with intermediate files cleaned up", async () => { + const t = setup(); + const first = await t.action(api.jobs.process, args); + const WORKER = await jobWorkerToken("d".repeat(64), first.jobId); + const path = (slot: string, endpoint = "artifacts") => + `/${endpoint}?jobId=${first.jobId}&slot=${slot}`; + const auth = (token: string) => ({ Authorization: `Bearer ${token}` }); + expect((await t.fetch(path("prepared"))).status).toBe(401); + expect( + ( + await t.fetch(path("prepared"), { + method: "POST", + headers: { ...auth(OWNER), "Content-Type": "image/jpeg" }, + body: "input", + }) + ).status + ).toBe(404); + expect( + ( + await t.fetch(path("prepared"), { + method: "POST", + headers: { ...auth(WORKER), "Content-Type": "image/jpeg" }, + body: "input", + }) + ).status + ).toBe(201); + expect((await t.fetch(path("prepared"), { headers: auth(OWNER) })).status).toBe(404); + const bytes = "private avif fixture"; + for (const variant of VARIANTS) { + expect( + ( + await t.fetch(path(variant.kind), { + method: "POST", + headers: { ...auth(WORKER), "Content-Type": "image/avif" }, + body: bytes, + }) + ).status + ).toBe(201); + } + await finish(t, first.jobId, { + inputBytes: 100, + processingMs: 1000, + variants: VARIANTS.map((variant) => ({ + kind: variant.kind, + width: variant.width, + height: variant.height, + bytes: bytes.length, + format: "avif", + })), + }); + expect((await t.fetch(path("prepared"), { headers: auth(WORKER) })).status).toBe(404); + const response = await t.fetch(path("hero", "image"), { headers: auth(OWNER) }); + expect(response.status).toBe(200); + expect(response.headers.get("Cache-Control")).toContain("no-store"); + expect(await response.text()).toBe(bytes); + expect((await t.fetch(path("hero", "image"), { headers: auth(OTHER) })).status).toBe(404); + vi.setSystemTime(Date.now() + RESULT_TTL_MS); + await t.mutation(internal.jobs.expire, { jobId: first.jobId as Id<"edits"> }); + expect((await board(t)).jobs).toEqual([]); + expect((await t.fetch(path("hero", "image"), { headers: auth(OWNER) })).status).toBe(404); +}); + +test("invalid completed output fails visibly and releases the active reservation", async () => { + const t = setup(); + const first = await t.action(api.jobs.process, args); + await finish(t, first.jobId, { variants: [] }); + const state = await board(t); + expect(state.jobs[0]?.status).toBe("FAILED"); + expect(state.jobs[0]?.error).toContain("could not be verified"); + expect(state.quota.active).toBe(false); +}); + +test("the storage sweep collects unregistered files left by interrupted uploads", async () => { + const t = setup(); + const id = await t.run((ctx) => ctx.storage.store(new Blob(["orphan"], { type: "image/png" }))); + await t.mutation(internal.jobs.sweepStorage, {}); + expect(await t.run(async (ctx) => (await ctx.storage.get(id)) !== null)).toBe(true); + vi.setSystemTime(Date.now() + RESULT_TTL_MS + 1); + await t.mutation(internal.jobs.sweepStorage, {}); + expect(await t.run(async (ctx) => (await ctx.storage.get(id)) === null)).toBe(true); +}); + +test("configured human verification fails closed before reserving any generation", async () => { + const t = setup(); + vi.stubEnv("TURNSTILE_SITE_KEY", "test-site"); + vi.stubEnv("TURNSTILE_SECRET_KEY", "test-secret"); + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue(Response.json({ success: true, hostname: "attacker.example" })) + ); + await expect( + t.action(api.jobs.process, { ...args, challengeToken: "challenge" }) + ).rejects.toThrow("Human verification failed"); + expect((await board(t)).quota.used).toBe(0); +}); diff --git a/packages/convex/example/convex/jobs.ts b/packages/convex/example/convex/jobs.ts new file mode 100644 index 00000000000..460a36a160a --- /dev/null +++ b/packages/convex/example/convex/jobs.ts @@ -0,0 +1,437 @@ +import { vRun } from "@trigger.dev/convex"; +import { v } from "convex/values"; +import { action, mutation, query, internalMutation, internalQuery } from "./_generated/server"; +import { internal } from "./_generated/api"; +import { trigger } from "./trigger"; +import { vSample, vPreset, vSlot } from "./schema"; +import { digest, ownedEdit, ownerHash } from "./auth"; +import { jobWorkerToken } from "../shared/workerAuth"; +import { + COOLDOWN_MS, + MAX_JOBS, + DAILY_JOB_LIMIT, + PROMPT_LIMIT, + RESULT_TTL_MS, + PROCESSOR_VERSION, +} from "../shared/catalog"; +import { outputSchema } from "../shared/output"; +import type { Board } from "../shared/types"; +import type { processImage } from "../trigger/process-image"; + +const DAY_MS = 24 * 60 * 60_000; +const CLEANUP_BATCH_SIZE = 100; +const requestArgs = { + sessionToken: v.string(), + sample: vSample, + preset: vPreset, + prompt: v.string(), + requestId: v.string(), +}; +type ProcessResult = { jobId: string; cached: boolean }; + +export const board = query({ + args: { sessionToken: v.string() }, + handler: async (ctx, { sessionToken }): Promise => { + const owner = await ownerHash(sessionToken); + const control = await ctx.db.query("launchControl").unique(); + const records = await ctx.db + .query("edits") + .withIndex("by_owner", (q) => q.eq("ownerHash", owner).gt("expiresAt", Date.now())) + .order("desc") + .take(20); + const jobs = await Promise.all( + records.map(async (record) => { + const run = record.componentRunId ? await trigger.getRun(ctx, record.componentRunId) : null; + const exportCount = record.files.filter( + (file) => !["prepared", "edited"].includes(file.slot) + ).length; + const artifactProgress = exportCount + ? 65 + exportCount * 10 + : record.files.some((file) => file.slot === "edited") + ? 65 + : record.files.length + ? 25 + : 0; + const reportedProgress = Math.max(0, Math.min(95, Number(run?.metadata?.progress) || 0)); + const artifactStage = exportCount + ? `${exportCount} of 3 parallel exports ready; waiting for completion` + : artifactProgress === 65 + ? "Model edit ready; parallel exports next" + : "Photograph prepared; image model edit next"; + const observedStatus = + record.files.length && ["PENDING", "QUEUED"].includes(run?.status ?? "PENDING") + ? "EXECUTING" + : run?.status; + const status = record.finalStatus ?? observedStatus ?? "PENDING"; + return { + id: record._id, + requestId: record.requestId, + sample: record.sample, + preset: record.preset, + prompt: record.prompt, + createdAt: record._creationTime, + completedAt: record.completedAt, + expiresAt: record.expiresAt, + status, + syncStatus: run?.syncStatus ?? "pending", + runId: run?.runId ?? null, + realtimeEligible: record.realtimeEligible === true, + progress: + record.finalStatus === "COMPLETED" ? 100 : Math.max(reportedProgress, artifactProgress), + stage: record.deliveryError + ? "Result verification failed" + : record.finalStatus === "COMPLETED" + ? "Images ready" + : record.finalStatus + ? "Workflow stopped" + : Number(run?.metadata?.progress) >= 100 + ? "Verifying exported images" + : artifactProgress > reportedProgress + ? artifactStage + : String(run?.metadata?.stage ?? "Waiting for Trigger"), + error: + record.deliveryError ?? + (run?.syncError + ? "Status synchronization paused. An operator needs to resume this job." + : record.finalStatus && status !== "COMPLETED" + ? "The job did not finish. No automatic paid retry was made." + : null), + results: (record.variants ?? []).map((variant) => ({ + ...variant, + url: `${globalThis.process.env.CONVEX_SITE_URL}/api/image?jobId=${record._id}&slot=${variant.kind}`, + })), + inputBytes: record.inputBytes, + processingMs: record.processingMs, + }; + }) + ); + const today = Math.floor(Date.now() / DAY_MS) * DAY_MS; + return { + jobs, + quota: { + used: control?.modelUsed ?? 0, + limit: MAX_JOBS, + dailyUsed: control?.dayStart === today ? (control.dayUsed ?? 0) : 0, + dailyLimit: DAILY_JOB_LIMIT, + active: !!control?.activeEdit, + enabled: globalThis.process.env.DEMO_ENABLED === "true", + cooldownUntil: control ? control.lastLaunchAt + COOLDOWN_MS : 0, + }, + turnstileSiteKey: globalThis.process.env.TURNSTILE_SITE_KEY ?? null, + }; + }, +}); + +export const existingRequest = internalQuery({ + args: requestArgs, + handler: async (ctx, args): Promise => { + const owner = await ownerHash(args.sessionToken); + const existing = await ctx.db + .query("edits") + .withIndex("by_request", (q) => q.eq("ownerHash", owner).eq("requestId", args.requestId)) + .unique(); + if (!existing) return null; + if ( + existing.sample !== args.sample || + existing.preset !== args.preset || + existing.prompt !== args.prompt.trim() + ) + throw new Error("Request identifier already used for different edits"); + return { jobId: existing._id, cached: true }; + }, +}); + +export const process = action({ + args: { ...requestArgs, challengeToken: v.optional(v.string()) }, + handler: async (ctx, { challengeToken, ...args }): Promise => { + await ownerHash(args.sessionToken); + if (!args.prompt.trim() || args.prompt.length > PROMPT_LIMIT) + throw new Error(`Enter an edit instruction of at most ${PROMPT_LIMIT} characters.`); + if (!/^[a-zA-Z0-9-]{8,64}$/.test(args.requestId)) throw new Error("Invalid request identifier"); + const existing = await ctx.runQuery(internal.jobs.existingRequest, args); + if (existing) return existing; + const siteKey = globalThis.process.env.TURNSTILE_SITE_KEY; + const secret = globalThis.process.env.TURNSTILE_SECRET_KEY; + if (siteKey || secret) { + if (!siteKey || !secret) throw new Error("Human verification is not configured correctly."); + if (!challengeToken || challengeToken.length > 2048) + throw new Error("Complete human verification first."); + let verified = false; + try { + const response = await fetch("https://challenges.cloudflare.com/turnstile/v0/siteverify", { + method: "POST", + body: new URLSearchParams({ secret, response: challengeToken }), + signal: AbortSignal.timeout(10_000), + }); + const result = await response.json(); + verified = + response.ok && + result.success === true && + result.hostname === new URL(globalThis.process.env.CONVEX_SITE_URL!).hostname; + } catch { + /* Verification fails closed; a retry must not create duplicate paid work. */ + } + if (!verified) { + const received = await ctx.runQuery(internal.jobs.existingRequest, args); + if (received) return received; + throw new Error("Human verification failed. Please try again."); + } + } + return ctx.runMutation(internal.jobs.reserve, args); + }, +}); + +export const reserve = internalMutation({ + args: requestArgs, + handler: async ( + ctx, + { sessionToken, sample, preset, prompt: rawPrompt, requestId } + ): Promise => { + const owner = await ownerHash(sessionToken); + const prompt = rawPrompt.trim(); + if (!prompt || prompt.length > PROMPT_LIMIT || !/^[a-zA-Z0-9-]{8,64}$/.test(requestId)) + throw new Error("Invalid edit request"); + const existing = await ctx.db + .query("edits") + .withIndex("by_request", (q) => q.eq("ownerHash", owner).eq("requestId", requestId)) + .unique(); + if (existing) { + if (existing.sample !== sample || existing.preset !== preset || existing.prompt !== prompt) + throw new Error("Request identifier already used for different edits"); + return { jobId: existing._id, cached: true }; + } + const key = await digest(JSON.stringify([PROCESSOR_VERSION, sample, preset, prompt])); + const cached = await ctx.db + .query("edits") + .withIndex("by_key", (q) => q.eq("ownerHash", owner).eq("key", key)) + .order("desc") + .first(); + if ( + cached && + cached.expiresAt > Date.now() && + (!cached.finalStatus || cached.finalStatus === "COMPLETED") + ) + return { jobId: cached._id, cached: true }; + if (globalThis.process.env.DEMO_ENABLED !== "true") + throw new Error("New image edits are paused while the demo is being configured."); + const control = await ctx.db.query("launchControl").unique(); + const used = control?.modelUsed ?? 0; + const dayStart = Math.floor(Date.now() / DAY_MS) * DAY_MS; + const dayUsed = control?.dayStart === dayStart ? (control.dayUsed ?? 0) : 0; + if (used >= MAX_JOBS) throw new Error("The demo's lifetime generation budget is exhausted."); + if (dayUsed >= DAILY_JOB_LIMIT) + throw new Error("Today's generation budget is exhausted. Please come back tomorrow."); + if (control?.activeEdit) + throw new Error("Another background job is running. Please wait for it to finish."); + if (control && Date.now() < control.lastLaunchAt + COOLDOWN_MS) + throw new Error("Please wait for the shared cooldown."); + const serviceUrl = globalThis.process.env.CONVEX_SITE_URL; + if (!serviceUrl) throw new Error("The image service is not configured."); + const jobId = await ctx.db.insert("edits", { + ownerHash: owner, + workerHash: "", + realtimeEligible: true, + sample, + preset, + prompt, + requestId, + key, + expiresAt: Date.now() + RESULT_TTL_MS, + files: [], + }); + const componentRunId = await trigger.trigger( + ctx, + "convex-edit-image", + { + sample, + preset, + prompt, + jobId, + serviceUrl, + }, + { idempotencyKey: `edit:${jobId}`, tags: ["convex-image-demo"], ttl: "5m" } + ); + const workerToken = await jobWorkerToken(globalThis.process.env.DEMO_ARTIFACT_SECRET, jobId); + await ctx.db.patch(jobId, { componentRunId, workerHash: await digest(workerToken) }); + const next = { + modelUsed: used + 1, + dayStart, + dayUsed: dayUsed + 1, + lastLaunchAt: Date.now(), + activeEdit: jobId, + }; + if (control) await ctx.db.patch(control._id, next); + else await ctx.db.insert("launchControl", { used: 0, ...next }); + await ctx.scheduler.runAfter(RESULT_TTL_MS, internal.jobs.expire, { jobId }); + return { jobId, cached: false }; + }, +}); + +export const refresh = mutation({ + args: { jobId: v.id("edits"), sessionToken: v.string() }, + handler: async (ctx, { jobId, sessionToken }): Promise => { + const job = await ownedEdit(ctx, jobId, sessionToken); + if (job.componentRunId && !job.finalStatus) await trigger.refresh(ctx, job.componentRunId); + return null; + }, +}); + +export const realtimeAccess = internalQuery({ + args: { jobId: v.id("edits"), sessionToken: v.string() }, + handler: async (ctx, { jobId, sessionToken }) => { + const job = await ownedEdit(ctx, jobId, sessionToken); + // Old runs may contain a worker credential in their payload and must never get browser tokens. + if (!job.realtimeEligible || !job.componentRunId) return null; + const run = await trigger.getRun(ctx, job.componentRunId); + if (!run?.runId) return null; + return { runId: run.runId, cached: job.realtimeToken ?? null, expiresAt: job.expiresAt }; + }, +}); + +export const cacheRealtimeToken = internalMutation({ + args: { + jobId: v.id("edits"), + sessionToken: v.string(), + runId: v.string(), + accessToken: v.string(), + expiresAt: v.number(), + }, + handler: async (ctx, args): Promise => { + const job = await ownedEdit(ctx, args.jobId, args.sessionToken); + if (!job.realtimeEligible) return null; + const run = job.componentRunId ? await trigger.getRun(ctx, job.componentRunId) : null; + if (run?.runId !== args.runId) throw new Error("Run changed"); + await ctx.db.patch(job._id, { + realtimeToken: { + runId: args.runId, + accessToken: args.accessToken, + expiresAt: args.expiresAt, + }, + }); + return null; + }, +}); + +export const fileAccess = internalQuery({ + args: { jobId: v.id("edits"), token: v.string(), slot: vSlot, write: v.boolean() }, + handler: async (ctx, { jobId, token, slot, write }) => { + if (!/^[a-f0-9]{64}$/.test(token)) return null; + const job = await ctx.db.get(jobId); + if (!job || job.expiresAt <= Date.now()) return null; + const hash = await digest(token); + const worker = hash === job.workerHash; + if ( + write + ? !worker || !!job.finalStatus + : !worker && (hash !== job.ownerHash || slot === "prepared" || slot === "edited") + ) + return null; + return { storageId: job.files.find((file) => file.slot === slot)?.storageId ?? null }; + }, +}); + +export const registerFile = internalMutation({ + args: { jobId: v.id("edits"), token: v.string(), slot: vSlot, storageId: v.id("_storage") }, + handler: async (ctx, { jobId, token, slot, storageId }): Promise => { + const job = await ctx.db.get(jobId); + if ( + !job || + job.expiresAt <= Date.now() || + job.finalStatus || + (await digest(token)) !== job.workerHash + ) + return false; + if (job.files.some((file) => file.slot === slot)) await ctx.storage.delete(storageId); + else await ctx.db.patch(jobId, { files: [...job.files, { slot, storageId }] }); + return true; + }, +}); + +export const expire = internalMutation({ + args: { jobId: v.id("edits") }, + handler: async (ctx, { jobId }): Promise => { + const job = await ctx.db.get(jobId); + if (!job) return null; + if (job.expiresAt > Date.now()) throw new Error("Job has not expired"); + for (const file of job.files) await ctx.storage.delete(file.storageId); + const control = await ctx.db.query("launchControl").unique(); + if (control?.activeEdit === jobId) await ctx.db.patch(control._id, { activeEdit: undefined }); + await ctx.db.delete(jobId); + return null; + }, +}); + +export const sweepStorage = internalMutation({ + args: {}, + handler: async (ctx): Promise => { + // Root storage is reserved for image artifacts; static hosting uses isolated component storage. + const files = await ctx.db.system + .query("_storage") + .withIndex("by_creation_time", (q) => q.lt("_creationTime", Date.now() - RESULT_TTL_MS)) + .take(CLEANUP_BATCH_SIZE); + for (const file of files) await ctx.storage.delete(file._id); + if (files.length === CLEANUP_BATCH_SIZE) + await ctx.scheduler.runAfter(0, internal.jobs.sweepStorage, {}); + return null; + }, +}); + +export const resume = internalMutation({ + args: { jobId: v.id("edits") }, + handler: async (ctx, { jobId }): Promise => { + const job = await ctx.db.get(jobId); + if (!job?.componentRunId || job.finalStatus) + throw new Error("Job is missing or already finished"); + return trigger.resume(ctx, job.componentRunId); + }, +}); + +export const complete = internalMutation({ + args: { run: vRun }, + handler: async (ctx, { run }): Promise => { + const job = await ctx.db + .query("edits") + .withIndex("by_run", (q) => q.eq("componentRunId", run.id)) + .unique(); + if (!job || job.finalStatus) return null; + let deliveryError: string | undefined; + let output: ReturnType | undefined; + if (run.status === "COMPLETED") { + try { + output = outputSchema.parse(run.output); + for (const variant of output.variants) { + const file = job.files.find((file) => file.slot === variant.kind); + const info = file ? await ctx.db.system.get(file.storageId) : null; + if (!info || info.size !== variant.bytes || info.contentType !== "image/avif") + throw new Error("Missing image"); + } + } catch { + output = undefined; + deliveryError = "Processing finished, but the generated files could not be verified."; + } + } + const successful = !!output; + const retained = []; + for (const file of job.files) { + if (!successful || file.slot === "prepared" || file.slot === "edited") + await ctx.storage.delete(file.storageId); + else retained.push(file); + } + await ctx.db.patch(job._id, { + finalStatus: deliveryError ? "FAILED" : run.status, + deliveryError, + completedAt: Date.now(), + files: retained, + ...(output + ? { + variants: output.variants, + inputBytes: output.inputBytes, + processingMs: output.processingMs, + } + : {}), + }); + const control = await ctx.db.query("launchControl").unique(); + if (control?.activeEdit === job._id) await ctx.db.patch(control._id, { activeEdit: undefined }); + return null; + }, +}); diff --git a/packages/convex/example/convex/realtime.ts b/packages/convex/example/convex/realtime.ts new file mode 100644 index 00000000000..8193678a78b --- /dev/null +++ b/packages/convex/example/convex/realtime.ts @@ -0,0 +1,49 @@ +"use node"; + +import { auth } from "@trigger.dev/sdk"; +import { v } from "convex/values"; +import { action } from "./_generated/server"; +import { internal } from "./_generated/api"; + +const TOKEN_LIFETIME_MS = 10 * 60_000; +const TOKEN_RENEWAL_MARGIN_MS = 60_000; +type Credentials = { accessToken: string; baseURL: string; expiresAt: number; runId: string }; + +export const token = action({ + args: { jobId: v.id("edits"), sessionToken: v.string() }, + handler: async (ctx, args): Promise => { + const access = await ctx.runQuery(internal.jobs.realtimeAccess, args); + if (!access) return null; + const baseURL = process.env.TRIGGER_API_URL; + if (!baseURL) throw new Error("Trigger API is not configured"); + if ( + access.cached?.runId === access.runId && + access.cached.expiresAt > Date.now() + TOKEN_RENEWAL_MARGIN_MS + ) { + return { ...access.cached, baseURL }; + } + const expiresAt = Math.min(Date.now() + TOKEN_LIFETIME_MS, access.expiresAt); + if (expiresAt <= Date.now()) return null; + let accessToken: string; + try { + accessToken = await auth.withAuth( + { secretKey: process.env.TRIGGER_SECRET_KEY, baseURL }, + () => + auth.createPublicToken({ + scopes: { read: { runs: [access.runId] } }, + expirationTime: new Date(expiresAt), + realtime: { skipColumns: ["payload", "output"] }, + }) + ); + } catch { + throw new Error("Live updates are unavailable. Convex will continue syncing this job."); + } + await ctx.runMutation(internal.jobs.cacheRealtimeToken, { + ...args, + runId: access.runId, + accessToken, + expiresAt, + }); + return { accessToken, baseURL, expiresAt, runId: access.runId }; + }, +}); diff --git a/packages/convex/example/convex/schema.ts b/packages/convex/example/convex/schema.ts new file mode 100644 index 00000000000..1e4b151908d --- /dev/null +++ b/packages/convex/example/convex/schema.ts @@ -0,0 +1,102 @@ +import { defineSchema, defineTable } from "convex/server"; +import { v } from "convex/values"; +import type { SampleId } from "../shared/types"; + +export const vSample = v.union( + ...[ + "forest", + "coast", + "mountains", + "city", + "desert", + "lake", + "dog", + "flowers", + "architecture", + "canyon", + ].map((value) => v.literal(value as SampleId)) +); +export const vPreset = v.union(v.literal("balanced"), v.literal("compact"), v.literal("quality")); + +export const vSlot = v.union( + v.literal("prepared"), + v.literal("edited"), + v.literal("hero"), + v.literal("card"), + v.literal("square") +); +export default defineSchema({ + edits: defineTable({ + ownerHash: v.string(), + workerHash: v.string(), + realtimeEligible: v.optional(v.boolean()), + realtimeToken: v.optional( + v.object({ accessToken: v.string(), expiresAt: v.number(), runId: v.string() }) + ), + sample: vSample, + preset: vPreset, + prompt: v.string(), + requestId: v.string(), + key: v.string(), + componentRunId: v.optional(v.string()), + expiresAt: v.number(), + finalStatus: v.optional(v.string()), + deliveryError: v.optional(v.string()), + completedAt: v.optional(v.number()), + inputBytes: v.optional(v.number()), + processingMs: v.optional(v.number()), + files: v.array(v.object({ slot: vSlot, storageId: v.id("_storage") })), + variants: v.optional( + v.array( + v.object({ + kind: v.union(v.literal("hero"), v.literal("card"), v.literal("square")), + width: v.number(), + height: v.number(), + bytes: v.number(), + format: v.literal("avif"), + }) + ) + ), + }) + .index("by_owner", ["ownerHash", "expiresAt"]) + .index("by_request", ["ownerHash", "requestId"]) + .index("by_key", ["ownerHash", "key"]) + .index("by_run", ["componentRunId"]), + + jobs: defineTable({ + sample: vSample, + preset: vPreset, + key: v.string(), + requestId: v.string(), + componentRunId: v.string(), + finalStatus: v.optional(v.string()), + deliveryError: v.optional(v.string()), + completedAt: v.optional(v.number()), + inputBytes: v.optional(v.number()), + processingMs: v.optional(v.number()), + variants: v.optional( + v.array( + v.object({ + kind: v.union(v.literal("hero"), v.literal("card"), v.literal("square")), + width: v.number(), + height: v.number(), + bytes: v.number(), + format: v.literal("avif"), + storageId: v.id("_storage"), + }) + ) + ), + }) + .index("by_key", ["key"]) + .index("by_request", ["requestId"]) + .index("by_run", ["componentRunId"]), + launchControl: defineTable({ + used: v.number(), + modelUsed: v.optional(v.number()), + dayStart: v.optional(v.number()), + dayUsed: v.optional(v.number()), + activeEdit: v.optional(v.id("edits")), + lastLaunchAt: v.number(), + activeJob: v.optional(v.id("jobs")), + }), +}); diff --git a/packages/convex/example/convex/trigger.ts b/packages/convex/example/convex/trigger.ts new file mode 100644 index 00000000000..d45c13d57c3 --- /dev/null +++ b/packages/convex/example/convex/trigger.ts @@ -0,0 +1,7 @@ +import { Trigger } from "@trigger.dev/convex"; +import { components, internal } from "./_generated/api"; + +export const trigger: Trigger = new Trigger(components.trigger, { + baseUrl: process.env.TRIGGER_API_URL, + onComplete: internal.jobs.complete, +}); diff --git a/packages/convex/example/convex/tsconfig.json b/packages/convex/example/convex/tsconfig.json new file mode 100644 index 00000000000..95f3dc08fba --- /dev/null +++ b/packages/convex/example/convex/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "types": ["node"] + }, + "include": ["./**/*.ts"], + "exclude": ["./_generated", "./**/*.test.ts"] +} diff --git a/packages/convex/example/index.html b/packages/convex/example/index.html new file mode 100644 index 00000000000..55ca1a83f2f --- /dev/null +++ b/packages/convex/example/index.html @@ -0,0 +1,21 @@ + + + + + + + + + Convex Ɨ Trigger.dev Ā· Image editing demo + + +
+ + + diff --git a/packages/convex/example/package.json b/packages/convex/example/package.json new file mode 100644 index 00000000000..99ff0614355 --- /dev/null +++ b/packages/convex/example/package.json @@ -0,0 +1,41 @@ +{ + "name": "convex-image-processing", + "private": true, + "version": "0.0.0", + "type": "module", + "packageManager": "pnpm@10.33.2", + "scripts": { + "dev": "vite --host 127.0.0.1", + "build": "tsc --noEmit && vite build", + "typecheck": "tsc --noEmit", + "test": "vitest run", + "dev:convex": "convex dev", + "deploy:convex": "convex deploy", + "deploy:site": "static-hosting deploy --build-command 'pnpm build'", + "upload:site": "static-hosting upload --build --build-command 'pnpm build'", + "deploy:trigger": "trigger deploy --profile test --env prod --native-build --skip-update-check" + }, + "dependencies": { + "@convex-dev/static-hosting": "0.2.1", + "@trigger.dev/convex": "4.8.3", + "@trigger.dev/react-hooks": "4.8.3", + "@trigger.dev/sdk": "4.8.3", + "ai": "7.0.140", + "convex": "1.48.0", + "react": "19.3.1", + "react-dom": "19.3.1", + "sharp": "0.35.5", + "zod": "4.5.4" + }, + "devDependencies": { + "@trigger.dev/build": "4.8.3", + "@types/node": "24.13.3", + "@types/react": "19.2.17", + "@types/react-dom": "19.2.3", + "convex-test": "0.0.65", + "trigger.dev": "4.8.3", + "typescript": "5.9.3", + "vite": "8.3.5", + "vitest": "4.1.11" + } +} diff --git a/packages/convex/example/pnpm-lock.yaml b/packages/convex/example/pnpm-lock.yaml new file mode 100644 index 00000000000..a3ec2aae72d --- /dev/null +++ b/packages/convex/example/pnpm-lock.yaml @@ -0,0 +1,4495 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + '@convex-dev/static-hosting': + specifier: 0.2.1 + version: 0.2.1(convex@1.48.0(react@19.3.1))(react@19.3.1) + '@trigger.dev/convex': + specifier: 4.8.3 + version: 4.8.3(convex@1.48.0(react@19.3.1)) + '@trigger.dev/react-hooks': + specifier: 4.8.3 + version: 4.8.3(react-dom@19.3.1(react@19.3.1))(react@19.3.1)(zod@4.5.4) + '@trigger.dev/sdk': + specifier: 4.8.3 + version: 4.8.3(ai@7.0.140(zod@4.5.4))(react@19.3.1)(zod@4.5.4) + ai: + specifier: 7.0.140 + version: 7.0.140(zod@4.5.4) + convex: + specifier: 1.48.0 + version: 1.48.0(react@19.3.1) + react: + specifier: 19.3.1 + version: 19.3.1 + react-dom: + specifier: 19.3.1 + version: 19.3.1(react@19.3.1) + sharp: + specifier: 0.35.5 + version: 0.35.5(@types/node@24.13.3) + zod: + specifier: 4.5.4 + version: 4.5.4 + devDependencies: + '@trigger.dev/build': + specifier: 4.8.3 + version: 4.8.3(magicast@0.3.5)(supports-color@10.2.2)(typescript@5.9.3)(zod@4.5.4) + '@types/node': + specifier: 24.13.3 + version: 24.13.3 + '@types/react': + specifier: 19.2.17 + version: 19.2.17 + '@types/react-dom': + specifier: 19.2.3 + version: 19.2.3(@types/react@19.2.17) + convex-test: + specifier: 0.0.65 + version: 0.0.65(convex@1.48.0(react@19.3.1)) + trigger.dev: + specifier: 4.8.3 + version: 4.8.3(typescript@5.9.3) + typescript: + specifier: 5.9.3 + version: 5.9.3 + vite: + specifier: 8.3.5 + version: 8.3.5(@types/node@24.13.3)(esbuild@0.27.0)(jiti@2.7.0) + vitest: + specifier: 4.1.11 + version: 4.1.11(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(vite@8.3.5(@types/node@24.13.3)(esbuild@0.27.0)(jiti@2.7.0)) + +packages: + + '@ai-sdk/gateway@4.0.113': + resolution: {integrity: sha512-fYov5+sH1A20/eN1PaL5z8bZ9RUq0CcL9yAsAhCHz/qUCKliDOg9HXjNYzOUlH6o21lIrp/aKQc8QLGkEEZGlQ==} + engines: {node: '>=22'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + + '@ai-sdk/provider-utils@5.0.59': + resolution: {integrity: sha512-eaXBTkguxBFzS+KvxGRsbdkX9A7+R1g/C4jWx9mup0oSs3Cx5Mb427/byGpD2XL8SnMYT9O8LmXBmrgo+N3TJw==} + engines: {node: '>=22'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + + '@ai-sdk/provider@4.0.27': + resolution: {integrity: sha512-TtWwm6SxFiPomlMIbTQa+yhJte+w4/oCsW+wJrJzMC3MqHpAY9e5dJF1Y4HxyKt3W2Csac0KiI76Ckio11TFYw==} + engines: {node: '>=22'} + + '@babel/helper-string-parser@7.29.11': + resolution: {integrity: sha512-snZbO7MEDOof+hvNa97EjEY7qD7h2kwvQr5AJC3ZhQudKW5dnej6k3kP9NQV88hJwaS8vDqRLFo9rivLEhle2g==} + engines: {node: '>=6.9.0'} + + '@babel/helper-validator-identifier@7.29.7': + resolution: {integrity: sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==} + engines: {node: '>=6.9.0'} + + '@babel/parser@7.29.11': + resolution: {integrity: sha512-OZ6+oaK7GMyg3IfCpj8JOmn8n+w2rJbDT0LLfVDTLb1ZFeKDh3AiNELgCMUw175IJHSkLl74hG0M0VLsvx+gag==} + engines: {node: '>=6.0.0'} + hasBin: true + + '@babel/types@7.29.11': + resolution: {integrity: sha512-WCsemW40Cpc6gouC1SrY6PBaZ+oLRlEVY9XBvZBI7il/HyUrb2nJ505wE1MHEFyDZNnrWDEuy4jL50WVHfTTSQ==} + engines: {node: '>=6.9.0'} + + '@bugsnag/cuid@3.2.2': + resolution: {integrity: sha512-7onuYLTMqMmHE9BBPG0YER4nFsU1rB+me1/YIeMusqcLbVbKKuG9u9+BDVDpje5e0llkkrVNOKYwmzM9DRIo7A==} + + '@clack/core@0.5.0': + resolution: {integrity: sha512-p3y0FIOwaYRUPRcMO7+dlmLh8PSRcrjuTndsiA0WAFbWES0mLZlrjVoBRZ9DzkPFJZG6KGkJmoEAY0ZcVWTkow==} + + '@clack/prompts@0.11.0': + resolution: {integrity: sha512-pMN5FcrEw9hUkZA4f+zLlzivQSeQf5dRGJjSUbvVYDLvpKCdQx5OaknvKzgbtXOizhP+SJJJjqEbOe55uKKfAw==} + + '@colors/colors@1.5.0': + resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==} + engines: {node: '>=0.1.90'} + + '@convex-dev/static-hosting@0.2.1': + resolution: {integrity: sha512-QMcfKUOB8dHeLa81xzk2GEIn7LzhEvtdgYhfFpIFer6uE/+xMStDxHAZsX5h1qsttYi7wNbEKAMf7MfOQ7ZrMw==} + hasBin: true + peerDependencies: + convex: ^1.37.0 + react: ^18.3.1 || ^19.0.0 + + '@depot/cli-darwin-arm64@0.0.1-cli.2.80.0': + resolution: {integrity: sha512-H7tQ0zWXVmdYXGFvt3d/v5fmquMlMM1I9JC8C2yiBZ9En9a20hzSbKoiym92RtcfqjKQFvhXL0DT6vQmJ8bgQA==} + engines: {node: '>=14'} + cpu: [arm64] + os: [darwin] + + '@depot/cli-darwin-x64@0.0.1-cli.2.80.0': + resolution: {integrity: sha512-3RDyybnCC2YSvbJCkHxXBYbKR7adUO7U00O9DHreX8EyS2E/C3kQgET+rFXR+iDdeG0h+h+At2rDng7AxvUhDw==} + engines: {node: '>=14'} + cpu: [x64] + os: [darwin] + + '@depot/cli-linux-arm64@0.0.1-cli.2.80.0': + resolution: {integrity: sha512-Fc1+Guqdsl2WM+b+76FirRjjRvEoYZX0MI4uuIQF9c9gwp6UZSZ5o3U7lAUpdkclVMKXSAW5bsIpgcOfUiJX3w==} + engines: {node: '>=14'} + cpu: [arm64] + os: [linux] + + '@depot/cli-linux-arm@0.0.1-cli.2.80.0': + resolution: {integrity: sha512-95kjKwKxP6RKkAhJoPlO6g3drEYpFWy9AC7QfvQMHDRnI3HXgbxA/BSTp4R3C4TTfp5JUPShVf8Js/NAcrUpUQ==} + engines: {node: '>=14'} + cpu: [arm] + os: [linux] + + '@depot/cli-linux-ia32@0.0.1-cli.2.80.0': + resolution: {integrity: sha512-2KoyIoYqnyIuynPN+mWbugoOB2APiNNvFLKERWOwPa0KYLSfcENT3sYxDW8tDuIw2dftHlmZN73uvXrGFeMcDQ==} + engines: {node: '>=14'} + cpu: [ia32] + os: [linux] + + '@depot/cli-linux-x64@0.0.1-cli.2.80.0': + resolution: {integrity: sha512-B8xDlXFxhYHD2tUk0jUv7SKipLfWwJVlY7ZhDQixyQzatibXakahdqpvZJgE6FLZJd3lgFIEAVsKURZHa1l57A==} + engines: {node: '>=14'} + cpu: [x64] + os: [linux] + + '@depot/cli-win32-arm64@0.0.1-cli.2.80.0': + resolution: {integrity: sha512-CPJX691MKEhnKZX25xS+opWESgPQ73HKkEwkAvvVqdBzahndcHoqAeIxYLv2hoCqrDlkH3YCF+DJleLiEP3blA==} + engines: {node: '>=14'} + cpu: [arm64] + os: [win32] + + '@depot/cli-win32-ia32@0.0.1-cli.2.80.0': + resolution: {integrity: sha512-6eewZ1zPEyNL3zcCwC01s3rZNMqMGVu1iv/EPywj3/uk4CNyUBaTlZ3StQ1BLFHPQnU0vk29yOzA5ZEuRLSyxA==} + engines: {node: '>=14'} + cpu: [ia32] + os: [win32] + + '@depot/cli-win32-x64@0.0.1-cli.2.80.0': + resolution: {integrity: sha512-9CRcc7D0/x4UrBkDuc35WVPMQG5gKMD1JckGLEl6VREE0Ppdny6n+hunQ8prwVc8aqzKG134XCC2U4DUjYg18A==} + engines: {node: '>=14'} + cpu: [x64] + os: [win32] + + '@depot/cli@0.0.1-cli.2.80.0': + resolution: {integrity: sha512-KvmOiQdpbamFziqnzzgqBm6RjfGhLJimBBYEOVriTxCPtVJuBIFm34xZllM36OQzPZIWpWBP+2/UnOyRG5smUg==} + engines: {node: '>=14'} + hasBin: true + + '@electric-sql/client@1.0.14': + resolution: {integrity: sha512-LtPAfeMxXRiYS0hyDQ5hue2PjljUiK9stvzsVyVb4nwxWQxfOWTSF42bHTs/o5i3x1T4kAQ7mwHpxa4A+f8X7Q==} + + '@emnapi/runtime@1.11.3': + resolution: {integrity: sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==} + + '@esbuild/aix-ppc64@0.23.1': + resolution: {integrity: sha512-6VhYk1diRqrhBAqpJEdjASR/+WVRtfjpqKuNw11cLiaWpAT/Uu+nokB+UJnevzy/P9C/ty6AOe0dwueMrGh/iQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/aix-ppc64@0.27.0': + resolution: {integrity: sha512-KuZrd2hRjz01y5JK9mEBSD3Vj3mbCvemhT466rSuJYeE/hjuBrHfjjcjMdTm/sz7au+++sdbJZJmuBwQLuw68A==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.23.1': + resolution: {integrity: sha512-xw50ipykXcLstLeWH7WRdQuysJqejuAGPd30vd1i5zSyKK3WE+ijzHmLKxdiCMtH1pHz78rOg0BKSYOSB/2Khw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm64@0.27.0': + resolution: {integrity: sha512-CC3vt4+1xZrs97/PKDkl0yN7w8edvU2vZvAFGD16n9F0Cvniy5qvzRXjfO1l94efczkkQE6g1x0i73Qf5uthOQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.23.1': + resolution: {integrity: sha512-uz6/tEy2IFm9RYOyvKl88zdzZfwEfKZmnX9Cj1BHjeSGNuGLuMD1kR8y5bteYmwqKm1tj8m4cb/aKEorr6fHWQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-arm@0.27.0': + resolution: {integrity: sha512-j67aezrPNYWJEOHUNLPj9maeJte7uSMM6gMoxfPC9hOg8N02JuQi/T7ewumf4tNvJadFkvLZMlAq73b9uwdMyQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.23.1': + resolution: {integrity: sha512-nlN9B69St9BwUoB+jkyU090bru8L0NA3yFvAd7k8dNsVH8bi9a8cUAUSEcEEgTp2z3dbEDGJGfP6VUnkQnlReg==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/android-x64@0.27.0': + resolution: {integrity: sha512-wurMkF1nmQajBO1+0CJmcN17U4BP6GqNSROP8t0X/Jiw2ltYGLHpEksp9MpoBqkrFR3kv2/te6Sha26k3+yZ9Q==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.23.1': + resolution: {integrity: sha512-YsS2e3Wtgnw7Wq53XXBLcV6JhRsEq8hkfg91ESVadIrzr9wO6jJDMZnCQbHm1Guc5t/CdDiFSSfWP58FNuvT3Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-arm64@0.27.0': + resolution: {integrity: sha512-uJOQKYCcHhg07DL7i8MzjvS2LaP7W7Pn/7uA0B5S1EnqAirJtbyw4yC5jQ5qcFjHK9l6o/MX9QisBg12kNkdHg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.23.1': + resolution: {integrity: sha512-aClqdgTDVPSEGgoCS8QDG37Gu8yc9lTHNAQlsztQ6ENetKEO//b8y31MMu2ZaPbn4kVsIABzVLXYLhCGekGDqw==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/darwin-x64@0.27.0': + resolution: {integrity: sha512-8mG6arH3yB/4ZXiEnXof5MK72dE6zM9cDvUcPtxhUZsDjESl9JipZYW60C3JGreKCEP+p8P/72r69m4AZGJd5g==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.23.1': + resolution: {integrity: sha512-h1k6yS8/pN/NHlMl5+v4XPfikhJulk4G+tKGFIOwURBSFzE8bixw1ebjluLOjfwtLqY0kewfjLSrO6tN2MgIhA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-arm64@0.27.0': + resolution: {integrity: sha512-9FHtyO988CwNMMOE3YIeci+UV+x5Zy8fI2qHNpsEtSF83YPBmE8UWmfYAQg6Ux7Gsmd4FejZqnEUZCMGaNQHQw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.23.1': + resolution: {integrity: sha512-lK1eJeyk1ZX8UklqFd/3A60UuZ/6UVfGT2LuGo3Wp4/z7eRTRYY+0xOu2kpClP+vMTi9wKOfXi2vjUpO1Ro76g==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.27.0': + resolution: {integrity: sha512-zCMeMXI4HS/tXvJz8vWGexpZj2YVtRAihHLk1imZj4efx1BQzN76YFeKqlDr3bUWI26wHwLWPd3rwh6pe4EV7g==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.23.1': + resolution: {integrity: sha512-/93bf2yxencYDnItMYV/v116zff6UyTjo4EtEQjUBeGiVpMmffDNUyD9UN2zV+V3LRV3/on4xdZ26NKzn6754g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm64@0.27.0': + resolution: {integrity: sha512-AS18v0V+vZiLJyi/4LphvBE+OIX682Pu7ZYNsdUHyUKSoRwdnOsMf6FDekwoAFKej14WAkOef3zAORJgAtXnlQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.23.1': + resolution: {integrity: sha512-CXXkzgn+dXAPs3WBwE+Kvnrf4WECwBdfjfeYHpMeVxWE0EceB6vhWGShs6wi0IYEqMSIzdOF1XjQ/Mkm5d7ZdQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-arm@0.27.0': + resolution: {integrity: sha512-t76XLQDpxgmq2cNXKTVEB7O7YMb42atj2Re2Haf45HkaUpjM2J0UuJZDuaGbPbamzZ7bawyGFUkodL+zcE+jvQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.23.1': + resolution: {integrity: sha512-VTN4EuOHwXEkXzX5nTvVY4s7E/Krz7COC8xkftbbKRYAl96vPiUssGkeMELQMOnLOJ8k3BY1+ZY52tttZnHcXQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-ia32@0.27.0': + resolution: {integrity: sha512-Mz1jxqm/kfgKkc/KLHC5qIujMvnnarD9ra1cEcrs7qshTUSksPihGrWHVG5+osAIQ68577Zpww7SGapmzSt4Nw==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.23.1': + resolution: {integrity: sha512-Vx09LzEoBa5zDnieH8LSMRToj7ir/Jeq0Gu6qJ/1GcBq9GkfoEAoXvLiW1U9J1qE/Y/Oyaq33w5p2ZWrNNHNEw==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-loong64@0.27.0': + resolution: {integrity: sha512-QbEREjdJeIreIAbdG2hLU1yXm1uu+LTdzoq1KCo4G4pFOLlvIspBm36QrQOar9LFduavoWX2msNFAAAY9j4BDg==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.23.1': + resolution: {integrity: sha512-nrFzzMQ7W4WRLNUOU5dlWAqa6yVeI0P78WKGUo7lg2HShq/yx+UYkeNSE0SSfSure0SqgnsxPvmAUu/vu0E+3Q==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-mips64el@0.27.0': + resolution: {integrity: sha512-sJz3zRNe4tO2wxvDpH/HYJilb6+2YJxo/ZNbVdtFiKDufzWq4JmKAiHy9iGoLjAV7r/W32VgaHGkk35cUXlNOg==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.23.1': + resolution: {integrity: sha512-dKN8fgVqd0vUIjxuJI6P/9SSSe/mB9rvA98CSH2sJnlZ/OCZWO1DJvxj8jvKTfYUdGfcq2dDxoKaC6bHuTlgcw==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-ppc64@0.27.0': + resolution: {integrity: sha512-z9N10FBD0DCS2dmSABDBb5TLAyF1/ydVb+N4pi88T45efQ/w4ohr/F/QYCkxDPnkhkp6AIpIcQKQ8F0ANoA2JA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.23.1': + resolution: {integrity: sha512-5AV4Pzp80fhHL83JM6LoA6pTQVWgB1HovMBsLQ9OZWLDqVY8MVobBXNSmAJi//Csh6tcY7e7Lny2Hg1tElMjIA==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-riscv64@0.27.0': + resolution: {integrity: sha512-pQdyAIZ0BWIC5GyvVFn5awDiO14TkT/19FTmFcPdDec94KJ1uZcmFs21Fo8auMXzD4Tt+diXu1LW1gHus9fhFQ==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.23.1': + resolution: {integrity: sha512-9ygs73tuFCe6f6m/Tb+9LtYxWR4c9yg7zjt2cYkjDbDpV/xVn+68cQxMXCjUpYwEkze2RcU/rMnfIXNRFmSoDw==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-s390x@0.27.0': + resolution: {integrity: sha512-hPlRWR4eIDDEci953RI1BLZitgi5uqcsjKMxwYfmi4LcwyWo2IcRP+lThVnKjNtk90pLS8nKdroXYOqW+QQH+w==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.23.1': + resolution: {integrity: sha512-EV6+ovTsEXCPAp58g2dD68LxoP/wK5pRvgy0J/HxPGB009omFPv3Yet0HiaqvrIrgPTBuC6wCH1LTOY91EO5hQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/linux-x64@0.27.0': + resolution: {integrity: sha512-1hBWx4OUJE2cab++aVZ7pObD6s+DK4mPGpemtnAORBvb5l/g5xFGk0vc0PjSkrDs0XaXj9yyob3d14XqvnQ4gw==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.27.0': + resolution: {integrity: sha512-6m0sfQfxfQfy1qRuecMkJlf1cIzTOgyaeXaiVaaki8/v+WB+U4hc6ik15ZW6TAllRlg/WuQXxWj1jx6C+dfy3w==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.23.1': + resolution: {integrity: sha512-aevEkCNu7KlPRpYLjwmdcuNz6bDFiE7Z8XC4CPqExjTvrHugh28QzUXVOZtiYghciKUacNktqxdpymplil1beA==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.27.0': + resolution: {integrity: sha512-xbbOdfn06FtcJ9d0ShxxvSn2iUsGd/lgPIO2V3VZIPDbEaIj1/3nBBe1AwuEZKXVXkMmpr6LUAgMkLD/4D2PPA==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.23.1': + resolution: {integrity: sha512-3x37szhLexNA4bXhLrCC/LImN/YtWis6WXr1VESlfVtVeoFJBRINPJ3f0a/6LV8zpikqoUg4hyXw0sFBt5Cr+Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-arm64@0.27.0': + resolution: {integrity: sha512-fWgqR8uNbCQ/GGv0yhzttj6sU/9Z5/Sv/VGU3F5OuXK6J6SlriONKrQ7tNlwBrJZXRYk5jUhuWvF7GYzGguBZQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.23.1': + resolution: {integrity: sha512-aY2gMmKmPhxfU+0EdnN+XNtGbjfQgwZj43k8G3fyrDM/UdZww6xrWxmDkuz2eCZchqVeABjV5BpildOrUbBTqA==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.27.0': + resolution: {integrity: sha512-aCwlRdSNMNxkGGqQajMUza6uXzR/U0dIl1QmLjPtRbLOx3Gy3otfFu/VjATy4yQzo9yFDGTxYDo1FfAD9oRD2A==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.27.0': + resolution: {integrity: sha512-nyvsBccxNAsNYz2jVFYwEGuRRomqZ149A39SHWk4hV0jWxKM0hjBPm3AmdxcbHiFLbBSwG6SbpIcUbXjgyECfA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.23.1': + resolution: {integrity: sha512-RBRT2gqEl0IKQABT4XTj78tpk9v7ehp+mazn2HbUeZl1YMdaGAQqhapjGTCe7uw7y0frDi4gS0uHzhvpFuI1sA==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/sunos-x64@0.27.0': + resolution: {integrity: sha512-Q1KY1iJafM+UX6CFEL+F4HRTgygmEW568YMqDA5UV97AuZSm21b7SXIrRJDwXWPzr8MGr75fUZPV67FdtMHlHA==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.23.1': + resolution: {integrity: sha512-4O+gPR5rEBe2FpKOVyiJ7wNDPA8nGzDuJ6gN4okSA1gEOYZ67N8JPk58tkWtdtPeLz7lBnY6I5L3jdsr3S+A6A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-arm64@0.27.0': + resolution: {integrity: sha512-W1eyGNi6d+8kOmZIwi/EDjrL9nxQIQ0MiGqe/AWc6+IaHloxHSGoeRgDRKHFISThLmsewZ5nHFvGFWdBYlgKPg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.23.1': + resolution: {integrity: sha512-BcaL0Vn6QwCwre3Y717nVHZbAa4UBEigzFm6VdsVdT/MbZ38xoj1X9HPkZhbmaBGUD1W8vxAfffbDe8bA6AKnQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-ia32@0.27.0': + resolution: {integrity: sha512-30z1aKL9h22kQhilnYkORFYt+3wp7yZsHWus+wSKAJR8JtdfI76LJ4SBdMsCopTR3z/ORqVu5L1vtnHZWVj4cQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.23.1': + resolution: {integrity: sha512-BHpFFeslkWrXWyUPnbKm+xYYVYruCinGcftSBaa8zoF9hZO4BcSCFUvHVTtzpIY6YzUnYtuEhZ+C9iEXjxnasg==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@esbuild/win32-x64@0.27.0': + resolution: {integrity: sha512-aIitBcjQeyOhMTImhLZmtxfdOcuNRpwlPNmlFKPcHQYPhEssw75Cl1TSXJXpMkzaua9FUetx/4OQKq7eJul5Cg==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@google-cloud/precise-date@4.0.0': + resolution: {integrity: sha512-1TUx3KdaU3cN7nfCdNf+UVqA/PSX29Cjcox3fZZBtINlRrXVTmUkQnCKv2MbBUbCopbK4olAT1IHl76uZyCiVA==} + engines: {node: '>=14.0.0'} + + '@hono/node-server@2.1.5': + resolution: {integrity: sha512-MgP/I06GdvsilDspMk/qmRzpk8LOUStP6jFAbSRi40W09DhoyUn+Mj7eyKICPqeYKybOjVFLVIAQzvu1bfIa7Q==} + engines: {node: '>=20'} + peerDependencies: + hono: ^4 || ^5.0.0-0 + + '@img/colour@1.1.0': + resolution: {integrity: sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==} + engines: {node: '>=18'} + + '@img/sharp-darwin-arm64@0.35.5': + resolution: {integrity: sha512-QRUlFQ0WxvdWyqqG/WtI3iupfD5rBzmCHXSdPsY91sAtVtTo7Q4cb6zOccZ3gqEqkr0f1As1ehLqmEpDsRf+lg==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [darwin] + + '@img/sharp-darwin-x64@0.35.5': + resolution: {integrity: sha512-+BR255RhDlpygUpOc/Jdt1nT6DQ3XG/ERo5wbcdOf5Q320dKtPCKPLR1LJs9VGXRaMa8l1uUa0tkCNOXiAxZUw==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [darwin] + + '@img/sharp-freebsd-wasm32@0.35.5': + resolution: {integrity: sha512-Y/z91nEZ4uIBX5X3nfTovjU9lHNKFYbL2lpHCLVNmXQK03VIZvXBBt0KxbPGp2SdGSF+2mQU4e+hQaWOt86iAw==} + engines: {node: '>=20.9.0'} + os: [freebsd] + + '@img/sharp-libvips-darwin-arm64@1.3.4': + resolution: {integrity: sha512-5R89nBYiRdUlSWJxPhO+GVtaXzXSxKnRu/xqMn3KTA3L9EB9Oy/P+Nn2f2vlhPuUdy/Zusb2DarbyTpGCfEDuw==} + cpu: [arm64] + os: [darwin] + + '@img/sharp-libvips-darwin-x64@1.3.4': + resolution: {integrity: sha512-iR2OKH80yi0U+dUplyh3/xdpFvps6YkCwsXenIJxqxR1v9o+xtKTGbS9H7cps+2Vxjc8B1j96p75NmTGjIhtpQ==} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-linux-arm64@1.3.4': + resolution: {integrity: sha512-Y3dgX/6lE2QhQb+Gxy0WZxfg9MEm/JBjamZpS2IklP7xIQoKN4hzAm7KcMVGtaVDt3neE9OKBC7vAfonA/Lr1A==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-arm@1.3.4': + resolution: {integrity: sha512-LmRtTsOHuvM2+wlO2Db37dx5MiZhB0FvSunciw48YjdOkZz9KAiRbm8ujeMOA1INqmei5NapFxYEK1D1ZSidmw==} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-ppc64@1.3.4': + resolution: {integrity: sha512-Le6boB8Tai0Nis+gIxIpKx68UDVVIqdR8Tin5Yf1z2LJJQLDJvCDRqRu+jC2qCoD+eIomonmOwB4smBRxfVpYQ==} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-riscv64@1.3.4': + resolution: {integrity: sha512-aHkkIEHPRdQEegJN20MLmGtxYD9R2wQr3Cwpddnu5+YKMt6Uzax7S9h5gpZTo8wyrGuZSlfQ63OevL5mTyOC7Q==} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-s390x@1.3.4': + resolution: {integrity: sha512-ra/mB6MikESDUO7Yg+Mi95bFBb9GsObURuhnOv3OqknjGe9sZrG8tCe9q0xSIGrtLgvgw0gKnFWcK4blSgQOuQ==} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-x64@1.3.4': + resolution: {integrity: sha512-GJ//SSXbnwSDes02umB3nDJLFcQzw8a18V8fyhqr6tV515tOEMdImjjxj1AoafMRz56F3PHgftnj1QEKSU1zkw==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linuxmusl-arm64@1.3.4': + resolution: {integrity: sha512-hvulFwtjUcagsis6BBxHwGFwWoNZjgYmULGVrZcyfNbjA8hKILbRxGg15/7w5HDyXHXUos/j6baAWqnCyQ2DWA==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@img/sharp-libvips-linuxmusl-x64@1.3.4': + resolution: {integrity: sha512-6zXKeE/p39I1AmA3cJG35eyBGNqNddLnUXjhwBnsGjFPWqf5VKkDBEqaEkPDoTEtkxwi2vv8Tcr2mDyP4So7Fg==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@img/sharp-linux-arm64@0.35.5': + resolution: {integrity: sha512-LYVx5JTsOM2CBzmxreh+nl64/3H6Xb09iSLknqH47z2T2DFFxDeFLP5y4dJwe6H7uGQlHPyEEtIqyo3DYsRwdQ==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-arm@0.35.5': + resolution: {integrity: sha512-LEaXK2WdXVK5ykcw0buWyPMsmLLL2vpHLD6yrNSW+JGEL3BZPA4tpKN6iaMc4AxTTAoaX/sU1rOL51lcIz48ZQ==} + engines: {node: '>=20.9.0'} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-ppc64@0.35.5': + resolution: {integrity: sha512-QVxAAq8evVRI9ia2vqgwrmWucn5Dfv+JdWzj75pD8omHLPSP7f8p20O8jxzjCcuCEQEOtYOZUmX1hkiZ0kdevA==} + engines: {node: '>=20.9.0'} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-riscv64@0.35.5': + resolution: {integrity: sha512-LtdreXguaavKODPIfzJ4kffx7UNt1omwtK0rch4EBbbSTXPnxWmYSayXdLJw0fJzQ97kHt1gL/yh4tvU+nCyRQ==} + engines: {node: '>=20.9.0'} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-s390x@0.35.5': + resolution: {integrity: sha512-UZasTOFiYzotTsGOCu42BfUzP6Tu6Do/947iRm1RsLKvlllxwGcn4RN27LibGWceix4Y+Pmw3jsnTcCQIgWjqA==} + engines: {node: '>=20.9.0'} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-x64@0.35.5': + resolution: {integrity: sha512-SxFtLTeJInhAA9Q836kux2vZNeOBQEx658qvbboZScr0wIARym3IcGmW7KpVD5sbVg0Ojy+udFQdayYIZyoNog==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@img/sharp-linuxmusl-arm64@0.35.5': + resolution: {integrity: sha512-9HbMclmI1zlNkFRs3z9/eBtDjfD0sGlrX1z6b1qwmiFY5ElDLh4BC0LPBdVp7z1DXFiKlIcznf+ZlsuZzLxQqg==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@img/sharp-linuxmusl-x64@0.35.5': + resolution: {integrity: sha512-4KOphqB035HrVdqLZfCgMzzERrQkkzOwRhl4OAkRO1YCldbaFjySXMaK534Mo0V+LndnlJk+sbUyLeU0ULyD1A==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [linux] + libc: [musl] + + '@img/sharp-wasm32@0.35.5': + resolution: {integrity: sha512-Ptsga1su4tQx+LLF1ECS9U6nz5kmrXKo6XVbtR48Ke3ZRxxgaWBu7IDtEe1quo8hiupwm6WFqxVlXaSf7IINGQ==} + engines: {node: '>=20.9.0'} + + '@img/sharp-webcontainers-wasm32@0.35.5': + resolution: {integrity: sha512-hfhF/FmoQyTUkA0bIKFOtw536BQSeBMe6BF6QyWlrPxT754+TFLaZ7sKKTfvvM0yJgKgaYTwnFCIZ/GuDw5SUA==} + engines: {node: '>=20.9.0'} + cpu: [wasm32] + + '@img/sharp-win32-arm64@0.35.5': + resolution: {integrity: sha512-X4t7g+7ZA5DKblCBEXGjUqqemj4vczING/5viFwAL8h4N3qYeyjwdCvRLHi4EdOUI+2Z7UFlp1VM+p/AuEtm6Q==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [win32] + + '@img/sharp-win32-ia32@0.35.5': + resolution: {integrity: sha512-5Zm82LoBc43nhwNybZlG7Y1KO//Zhsn306fQl29ZOuStHLGTo3BWL83q3cznX0poxSAMuYL1On/BHBxkBeKr6A==} + engines: {node: ^20.9.0} + cpu: [ia32] + os: [win32] + + '@img/sharp-win32-x64@0.35.5': + resolution: {integrity: sha512-x76eH0vEiHlcMQu8Y8IenntaACtddpT6W0wmXtWrnKcnKI7ME5DdgqhAD6SEWOEl1v2zDvkZDhFA9KnURwpfqg==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [win32] + + '@isaacs/fs-minipass@4.0.1': + resolution: {integrity: sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==} + engines: {node: '>=18.0.0'} + + '@jridgewell/sourcemap-codec@1.6.0': + resolution: {integrity: sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==} + + '@jsonhero/path@1.0.21': + resolution: {integrity: sha512-gVUDj/92acpVoJwsVJ/RuWOaHyG4oFzn898WNGQItLCTQ+hOaVlEaImhwE1WqOTf+l3dGOUkbSiVKlb3q1hd1Q==} + + '@microsoft/fetch-event-source@2.0.1': + resolution: {integrity: sha512-W6CLUJ2eBMw3Rec70qrsEW0jOm/3twwJv21mrmj2yORiaVmVYGS4sSS5yUwvQc1ZlDLYGPnClVWmUUMagKNsfA==} + + '@modelcontextprotocol/sdk@1.32.2': + resolution: {integrity: sha512-SWsVtH/0vULSQStoZ6xxH2K09Li8fXYTMC64ZeogCdpZ3XSi1eCPOByV1OpuJxuI3OcmHhosDWYyii07tpPsLw==} + engines: {node: '>=18'} + peerDependencies: + '@cfworker/json-schema': ^4.1.1 + zod: ^3.25 || ^4.0 + peerDependenciesMeta: + '@cfworker/json-schema': + optional: true + + '@opentelemetry/api-logs@0.219.0': + resolution: {integrity: sha512-FFx7YnaYJlIjqWW/AG/yAZ0L/NEY724PipXXXQLdtZPbLwBGbUMTGL1i/esI56TWfTUXxhLfpgrnWJCG8aUJyg==} + engines: {node: '>=8.0.0'} + + '@opentelemetry/api@1.9.1': + resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==} + engines: {node: '>=8.0.0'} + + '@opentelemetry/context-async-hooks@2.8.0': + resolution: {integrity: sha512-/3FIraneMcng67SUJCxvyInk/oxzwsxyadufk0wwfOBLf5wqtAGX4MoQASwSbndBPeARzBryUM9Azr5kHIdWLw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/core@2.8.0': + resolution: {integrity: sha512-hd1Lfh8p545nNz+jq1Ejfz+Mn1hyLuxYn1YzTfFNrxr8urEWMNQLPf1Th8kjOH+HxwawCrtgBp8JpBUR4ZSgww==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/exporter-logs-otlp-http@0.219.0': + resolution: {integrity: sha512-mhl2HL6GmZI8b8PwPfqMws/5ovJfbRTxwc9Y5agVVHiQ+e5SL1btsFr/kJDgt7YCexDtsUn5HAreHQO9szFS0A==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/exporter-metrics-otlp-http@0.219.0': + resolution: {integrity: sha512-6CaDRbMVHZSDWzNXwrR8y/H4B/Z1eMNnkHiPQlTx3Ojz2OHY4X/aff/UC4P/3pHUQSuTfi3oh2UsPPZppw+Vrg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/exporter-trace-otlp-http@0.219.0': + resolution: {integrity: sha512-9t6SvBXXBEjOBcIzgozvBbd3jWrv3Gt3ngGhl1fhdZ/zRc7oZDVOFEqbi2zlBpW9BXhgDMKv422J0DL/3iQWfw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/host-metrics@0.38.3': + resolution: {integrity: sha512-8iSOA8VPGoB5p/RIC8n/dcSe4cluCEWoznWENZfXR8sWQOQvergFu7v798xp7S5WQlZo1zfn1nVXx8dbyQ9m6Q==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation@0.219.0': + resolution: {integrity: sha512-X5t7I8GyIO9rmGHwoedZLREpQqrF1WW2nxzNNym6HOKpFiE+rvqV3ngC0xcZVO2YwIGf3KKmRdWrYwdwz3H9RQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/otlp-exporter-base@0.219.0': + resolution: {integrity: sha512-zvIxQX/AZUVKDU+hCuYx+7UkiP7GRdnk1ZbFQRYzHvYp47cAWR4j3IhoPhV9KaeXEv2xdGq3IA6PnpzDmLcmSA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/otlp-transformer@0.219.0': + resolution: {integrity: sha512-aaYKAyXhw9VchKZVGOopD3Gw/kPsyrX2c6IQ0AW32mTjqmZOh5Y6Gf5OYqTNqVktAeBjmFinhyFaCwW6GYK9YQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/resources@2.8.0': + resolution: {integrity: sha512-qmXQ27ilDbUK/vGMqwL8D4/rhn76C+sherM4wTbjlfknR8Nvfc/hCxjRJPhkzZzUsPiNg16SA31NxMabwttRjg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/sdk-logs@0.219.0': + resolution: {integrity: sha512-s6lTKRakaPClvKoWHRChxnXjDMkM/TQ30ff78jN6EBGf7MI7VzANE5PU3f4z9qDUudWjvZjOLHG0rBnBKYvoXA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.4.0 <1.10.0' + + '@opentelemetry/sdk-metrics@2.8.0': + resolution: {integrity: sha512-UDBGaj6W0Rgy5rTTaoxs8gVGF/aGkAKyjurJv7se6wjRxJu7FoquTLT/vt54DZfo4crbprYfhX/SOK9+BPw1qg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.9.0 <1.10.0' + + '@opentelemetry/sdk-trace-base@2.8.0': + resolution: {integrity: sha512-mhU4jp+vW0mGbFRd+GeXHvmfA4aDqWjBjLC3pE5XMpLs0IE2ryYb019Ts2AQrOq67gaTF25D91+fgvEHDZEnuQ==} + engines: {node: ^18.19.0 || >=20.6.0} + deprecated: Use @opentelemetry/sdk-trace package + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/sdk-trace-node@2.8.0': + resolution: {integrity: sha512-nZt9OGufioAc3AfoLTqA9bsAeaMJAictYDdI2VcNQ+PmT+3rfKjAZDZvgPfd8VPX0O5Bw1hdQF6kDK8VSpZiWg==} + engines: {node: ^18.19.0 || >=20.6.0} + deprecated: Use @opentelemetry/sdk-trace package + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/semantic-conventions@1.41.1': + resolution: {integrity: sha512-/UhIkaZgPutTFmQ7RnIJGgDXZmtEJ7Dvi86xNTFWcnRxVRNk/aotsqDJYeEvDP+FSMB2SdW+pQzNMcWP0rwuNA==} + engines: {node: '>=14'} + + '@opentelemetry/semantic-conventions@1.43.0': + resolution: {integrity: sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==} + engines: {node: '>=14'} + + '@oxc-project/types@0.155.0': + resolution: {integrity: sha512-SeE8JPwW1xs1tHeNycjUtUBdhorRa8o4jlvjwk49R7/Z291i+4DQa7JJIgOm/TaYQZx+QrPQbVJ/9c8dMIZN8Q==} + + '@posthog/core@1.29.13': + resolution: {integrity: sha512-7Me5zaeAue/wmA364Go8ChYbsVAfNAHbtDxXopWu3D6hq9PVScUcauRgjD1njgvP8NzN91SrIllE+pri3XvJVw==} + + '@posthog/types@1.376.4': + resolution: {integrity: sha512-EoDEvA925lf6yxPpbP4wozlXgu4b9WEqxZlFBUDd4k2akP5R/RWyHpvQT8aYyfY6BtSLn8TnVwxPQOM4b90isA==} + + '@prisma/config@6.19.3': + resolution: {integrity: sha512-CBPT44BjlQxEt8kiMEauji2WHTDoVBOKl7UlewXmUgBPnr/oPRZC3psci5chJnYmH0ivEIog2OU9PGWoki3DLQ==} + + '@protobuf-ts/runtime@2.11.1': + resolution: {integrity: sha512-KuDaT1IfHkugM2pyz+FwiY80ejWrkH1pAtOBOZFuR6SXEFTsnb/jiQWQ1rCIrcKx2BtyxnxW6BWwsVSA/Ie+WQ==} + + '@rolldown/binding-android-arm-eabi@1.2.15': + resolution: {integrity: sha512-aLcgK9gIJ3YRvrrtfSBbRa27F5WrpiTaPulnFpsHPxpUHgFlSgnU4854MHwoRGEBhfRJDWCS2WQ7lIjskE9Olw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [android] + + '@rolldown/binding-android-arm64@1.2.15': + resolution: {integrity: sha512-xD7DdZCrbThKuxNg7ooQmvp07ys4J/pnSEMTG3/OdOWl6daC67m/Q4fL78sRHMxTH/N4fGUbYhcPM0MF5WFHkg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [android] + + '@rolldown/binding-darwin-arm64@1.2.15': + resolution: {integrity: sha512-m2WuQ6r0B4OTYxyPpNLtXaMRbRDUbG8YQV2Q3KTdSSK8Th6KHCnC8jVCJt8bvYeSB5WivkkNbc85mKadaSdOxA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [darwin] + + '@rolldown/binding-darwin-x64@1.2.15': + resolution: {integrity: sha512-5RtnsHM36nodM9WbMy+zip0LGS/G+y5wro5RHrpOlr4ac24+1zSi9kiV2MP1myZeZ/ifEHfulQcWjnAXzc2C2A==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [darwin] + + '@rolldown/binding-freebsd-x64@1.2.15': + resolution: {integrity: sha512-g3l09pyFqhu+135JKkbkwiKGQmwU5hUMUPJ4VdBADvuqR8d33L6H578rh8m7rtxbdp+VmX/+nzI7nvhSZlzpUQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [freebsd] + + '@rolldown/binding-linux-arm-gnueabihf@1.2.15': + resolution: {integrity: sha512-nIe3VugI97nGQtspPxDqA4sXMUpeiocKIP/CqpY/u/vgbT+alX+cVxu1C/oVBPKg4feNTmYovE91v4DgXvXNwQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [linux] + + '@rolldown/binding-linux-arm64-gnu@1.2.15': + resolution: {integrity: sha512-iTBjXx+B5QSvgI89lJfmqh4wTh8NabR1YZIIN3hNx4zSG0EVfVqkOvLulNpG4251pbd6/IAmZ/LefVk1Y4rPhw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-arm64-musl@1.2.15': + resolution: {integrity: sha512-uN26XOnVR7rqQupG3V9oAG2EGGatWjcJHOeDOHUFCh4TBWmP2tHTFoVN13aYbwfm0aHqJAEmcwhAu6sooIjfqQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@rolldown/binding-linux-ppc64-gnu@1.2.15': + resolution: {integrity: sha512-9nWkcdd+H+XhaA3fQIwpHnKIrfx/PQ6rO0pivr78Og07tXlUlctBBJoEYLPfhTmCR/QxEVdn6uPjHV5SymPhdw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-s390x-gnu@1.2.15': + resolution: {integrity: sha512-Ggh0ciFjLwIusLSLz7LZYEaGdxOAUJXI+Qu5JCbhPJJHEt8vP+IjRATTJheiJsdzzTYgUiHUiw3X1Ea3ZpMBdA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-x64-gnu@1.2.15': + resolution: {integrity: sha512-FHg60rS6DBePAYZ9X9F67x+JObFdmgn6tupA1C/a3B+CufFmTD4mX+e8M75tCiAO3foR/6xwVBI0KQiI7sn/rw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@rolldown/binding-linux-x64-musl@1.2.15': + resolution: {integrity: sha512-/cvV/7v5VGz/moiz0XKRj2dE8DzsKyhIu1KpOvJGPjRywg3yscON889UGvAe8zfiJLHup7VLjBduoGWi3HusWQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [musl] + + '@rolldown/binding-openharmony-arm64@1.2.15': + resolution: {integrity: sha512-MyzC5Hrv/sm+Z7TSGKP/j8001RO2xQm1zOpCNVL4j0cY/VhBZviludA2YFQ7i4lH6ZAXxQm0lcMdBzM+99mE7w==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [openharmony] + + '@rolldown/binding-win32-arm64-msvc@1.2.15': + resolution: {integrity: sha512-+PgYiS+AH5omy6L45ZLxybasauJaj3M8buoT83D+nzPmCayXlxUV9hydhciCKVyIg0B/v0BtXPDE1uGqNwCPpg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [win32] + + '@rolldown/binding-win32-x64-msvc@1.2.15': + resolution: {integrity: sha512-HFZd2QNQYn1JbIi+Hpd7H+r/lkWxcfIu28nirZ6eCn2LafuvmExMlb/P96Sfx5tf7pPArw39IdhmCF8vmV+NhQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [win32] + + '@rolldown/pluginutils@1.0.1': + resolution: {integrity: sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==} + + '@rollup/rollup-darwin-arm64@4.65.1': + resolution: {integrity: sha512-zkZrXYwNlIcSUmcwY2DpsaDHr0XjVUeQHCuSunG3eQD7/5Wn/DvI7OMUE8NRRweK87yG8ebJ1xGOzmwpMsbZQA==} + cpu: [arm64] + os: [darwin] + + '@s2-dev/streamstore@0.25.0': + resolution: {integrity: sha512-oB8OJObT/s2Q68Tr01NLfDnQPFJp/Kn2pP3seWybgP7VrjQ+j1efXReOsyVCDsUBVMxkyPlzt75o/+GuL5Q3vg==} + + '@s2-dev/streamstore@0.25.1': + resolution: {integrity: sha512-P9wWkTi1qFNP+XuPHzz/jYPV5LY3fugMTQ6AJM0dn3DSpvL+9GBfGyIHe6WLGl1qcnLiUlWMPpwkKjWCHjMqhg==} + + '@socket.io/component-emitter@3.1.2': + resolution: {integrity: sha512-9BCxFwvbGg/RsZK9tjXd8s4UcwR0MWeFQ1XEKIQVVvAGJyINdrqKMcTRyLoK8Rse1GjzLV9cwjWV1olXRWEXVA==} + + '@sodaru/yup-to-json-schema@2.0.1': + resolution: {integrity: sha512-lWb0Wiz8KZ9ip/dY1eUqt7fhTPmL24p6Hmv5Fd9pzlzAdw/YNcWZr+tiCT4oZ4Zyxzi9+1X4zv82o7jYvcFxYA==} + + '@standard-schema/spec@1.1.0': + resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} + + '@trigger.dev/build@4.8.3': + resolution: {integrity: sha512-4KohHZdx0urxL96zisBAqvL4J4uKZ5u4rejWYyE8UkxDdOviR1YD3nxVmbYN4lWnpHEGxue116ZZrRoZr0+1MQ==} + engines: {node: '>=18.20.0'} + peerDependencies: + '@typescript/typescript6': ^6.0.0 + typescript: '>=5.0.0' + peerDependenciesMeta: + '@typescript/typescript6': + optional: true + typescript: + optional: true + + '@trigger.dev/convex@4.8.3': + resolution: {integrity: sha512-RMJpT4px0GGTGQPQk9U9VaYUEAPn9dNhhP0Z1bDqS5Ub5BpfNfBqO7tgO6EkoL4QH6MgfbBZLu1JF+hDfNE8Yw==} + peerDependencies: + convex: ^1.48.0 + + '@trigger.dev/core@4.8.3': + resolution: {integrity: sha512-V/7nOgnshOMMIUQ392wKAojLmE1t/xuod7WOnFE5s62d2IhnOaYGLBFtN0WnuNQP/GVxmRVGARwRv8j43XH0XA==} + engines: {node: '>=18.20.0'} + peerDependencies: + zod: ^3.25.56 || ^4.0.0 + + '@trigger.dev/react-hooks@4.8.3': + resolution: {integrity: sha512-X5VfUkhbLofKKQLI4FnbFPuW8g5R5+K498Zwkw8b3DMyQZMraCKK8D/BG7yFWlK6sSJitrebKS8Rdy4o4gO/pg==} + engines: {node: '>=18.20.0'} + peerDependencies: + react: ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^18.0 || ^19.0 || ^19.0.0-rc + + '@trigger.dev/schema-to-json@4.8.3': + resolution: {integrity: sha512-bmEdNExiuLEP+Mirj6J1oJ748VOlcWIRGVmGvJR0u40H5dqYA0I3qx2RHiF9KjR4Tk5fVi/fE60CakUireUQUA==} + engines: {node: '>=18.20.0'} + peerDependencies: + '@sinclair/typebox': '>=0.34.30' + arktype: '>=2.0.0' + runtypes: '>=5.0.0' + superstruct: '>=0.14.2' + valibot: '>=0.41.0' + zod: ^3.25.56 || ^4.0.0 + peerDependenciesMeta: + '@sinclair/typebox': + optional: true + arktype: + optional: true + runtypes: + optional: true + superstruct: + optional: true + valibot: + optional: true + + '@trigger.dev/sdk@4.8.3': + resolution: {integrity: sha512-0IkDT/mjRXVhRaaXEMlCbqb0YwafgUW75LvkVQMLe3w1L0sO/+Ud/YjNjQqrl8foZO1w2a+a08NEueyNKu9reg==} + engines: {node: '>=18.20.0'} + peerDependencies: + '@ai-sdk/otel': '>=1.0.0-0 <2' + '@aws-sdk/client-sts': ^3.840.0 + ai: ^5.0.0 || ^6.0.0 || >=7.0.0-canary <8 + react: ^18.0 || ^19.0 + zod: ^3.25.56 || ^4.0.0 + peerDependenciesMeta: + '@ai-sdk/otel': + optional: true + '@aws-sdk/client-sts': + optional: true + ai: + optional: true + react: + optional: true + + '@types/chai@5.2.3': + resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} + + '@types/deep-eql@4.0.2': + resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} + + '@types/estree@1.0.9': + resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + + '@types/node@24.13.3': + resolution: {integrity: sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==} + + '@types/react-dom@19.2.3': + resolution: {integrity: sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==} + peerDependencies: + '@types/react': ^19.2.0 + + '@types/react@19.2.17': + resolution: {integrity: sha512-MXfmqaVPEVgkBT/aY0aGCkRWWtByiYQXo3xdQ8r5RzuFrPiRn8Gar2tQdXSUQ2GKV3bkXckek89V8wQBY2Q/Aw==} + + '@types/retry@0.12.2': + resolution: {integrity: sha512-XISRgDJ2Tc5q4TRqvgJtzsRkFYNJzZrhTdtMoGVBttwzzQJkPnS3WWTFc7kuDRoPtPakl+T+OfdEUjYJj7Jbow==} + + '@vercel/oidc@3.2.0': + resolution: {integrity: sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==} + engines: {node: '>= 20'} + + '@vitest/expect@4.1.11': + resolution: {integrity: sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==} + + '@vitest/mocker@4.1.11': + resolution: {integrity: sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ==} + peerDependencies: + msw: ^2.4.9 + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + + '@vitest/pretty-format@4.1.11': + resolution: {integrity: sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw==} + + '@vitest/runner@4.1.11': + resolution: {integrity: sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw==} + + '@vitest/snapshot@4.1.11': + resolution: {integrity: sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog==} + + '@vitest/spy@4.1.11': + resolution: {integrity: sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==} + + '@vitest/utils@4.1.11': + resolution: {integrity: sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==} + + '@workflow/serde@4.1.0': + resolution: {integrity: sha512-pav4F2BoirECWR7Nf1TKt+2eETcBj7jj4cBefQ8VXQCA6NPkaKeLfj/zMgi+3zYV5ZIBT4GuUiphsj0/b9hPQQ==} + + accepts@2.0.0: + resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} + engines: {node: '>= 0.6'} + + acorn-import-attributes@1.9.5: + resolution: {integrity: sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==} + peerDependencies: + acorn: ^8 + + acorn@8.19.0: + resolution: {integrity: sha512-oJlA3XiRm7Cyk6qFD2Jn8ak9B7jSy0qy00ADO3+8dpT0LSjFihQYv4C02LFCSYJV3Q37xYwwRy5m+IpIiUzqWw==} + engines: {node: '>=0.4.0'} + hasBin: true + + ai@7.0.140: + resolution: {integrity: sha512-GHWqpUcY7EsRHc7jC99SqzpgabExb9w7anxDYa5N79AIVsGavIZFxZKhVQ/SORkuXFkPO4FG8cCSweaGutvwSw==} + engines: {node: '>=22'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + + ajv-formats@3.0.1: + resolution: {integrity: sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==} + peerDependencies: + ajv: ^8.0.0 + peerDependenciesMeta: + ajv: + optional: true + + ajv@8.20.0: + resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==} + + ansi-escapes@7.3.0: + resolution: {integrity: sha512-BvU8nYgGQBxcmMuEeUEmNTvrMVjJNSH7RgW24vXexN4Ven6qCvy4TntnvlnwnMLTVlcRQQdbRY8NKnaIoeWDNg==} + engines: {node: '>=18'} + + ansi-regex@5.0.1: + resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} + engines: {node: '>=8'} + + ansi-regex@6.4.0: + resolution: {integrity: sha512-KzTVk2tCWAHtYrvvvaP8bJKJq2pVinhLcGEQdtLIYPbmNGNyYe8QwNaTUYQp2J7/vIsUKt5QCqAfUkYyG9DkOw==} + engines: {node: '>=12'} + + anymatch@3.1.3: + resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} + engines: {node: '>= 8'} + + assertion-error@2.0.1: + resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} + engines: {node: '>=12'} + + balanced-match@4.0.4: + resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==} + engines: {node: 18 || 20 || >=22} + + binary-extensions@2.3.0: + resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} + engines: {node: '>=8'} + + bintrees@1.0.2: + resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==} + + body-parser@2.4.0: + resolution: {integrity: sha512-Ifk1KcLrxaGt1occQagQNFNNOqRt1LFuehEnHHzB8Z6cRgIxRU1TBaiIsUiCdA+gy5PL7Y1IX31DNKkRmxtXjQ==} + engines: {node: '>=18'} + + brace-expansion@5.0.12: + resolution: {integrity: sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==} + engines: {node: 20 || >=22} + + braces@3.0.3: + resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} + engines: {node: '>=8'} + + buffer-from@1.1.2: + resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} + + bundle-name@4.1.1: + resolution: {integrity: sha512-DdH81/zPLVS11EUgWq3tEu/xn+EzljlMYooDNdzWEnFha3R3NBMpMV1UqYIpjYHV/SgpFKMIX1Oh7o06SjM/oA==} + engines: {node: '>=18'} + + bytes@3.1.2: + resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} + engines: {node: '>= 0.8'} + + c12@1.11.2: + resolution: {integrity: sha512-oBs8a4uvSDO9dm8b7OCFW7+dgtVrwmwnrVXYzLm43ta7ep2jCn/0MhoUFygIWtxhyy6+/MG7/agvpY0U1Iemew==} + peerDependencies: + magicast: ^0.3.4 + peerDependenciesMeta: + magicast: + optional: true + + c12@3.1.0: + resolution: {integrity: sha512-uWoS8OU1MEIsOv8p/5a82c3H31LsWVR5qiyXVfBNOzfffjUWtPnhAb4BYI2uG2HfGmZmFjCtui5XNWaps+iFuw==} + peerDependencies: + magicast: ^0.3.5 + peerDependenciesMeta: + magicast: + optional: true + + call-bind-apply-helpers@1.0.2: + resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} + engines: {node: '>= 0.4'} + + call-bind@1.0.9: + resolution: {integrity: sha512-a/hy+pNsFUTR+Iz8TCJvXudKVLAnz/DyeSUo10I5yvFDQJBFU2s9uqQpoSrJlroHUKoKqzg+epxyP9lqFdzfBQ==} + engines: {node: '>= 0.4'} + + call-bound@1.0.4: + resolution: {integrity: sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==} + engines: {node: '>= 0.4'} + + chai@6.3.0: + resolution: {integrity: sha512-XWAtwJ6OHO+tj0EKCs0Y2UamnyOxseZWltU4x2U2wh8g4AigdjwvtUjvLP2tqkA/avxHEtzxNaqGq/YGNwckKg==} + engines: {node: '>=18'} + + chalk@5.6.2: + resolution: {integrity: sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==} + engines: {node: ^12.17.0 || ^14.13 || >=16.0.0} + + chokidar@3.6.0: + resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==} + engines: {node: '>= 8.10.0'} + + chokidar@4.0.3: + resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} + engines: {node: '>= 14.16.0'} + + chownr@2.0.0: + resolution: {integrity: sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==} + engines: {node: '>=10'} + + chownr@3.0.0: + resolution: {integrity: sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==} + engines: {node: '>=18'} + + citty@0.1.6: + resolution: {integrity: sha512-tskPPKEs8D2KPafUypv2gxwJP8h/OaJmC82QQGGDQcHvXX43xF2VDACcJVmZ0EuSxkpO9Kc4MlrA3q0+FG58AQ==} + + citty@0.2.2: + resolution: {integrity: sha512-+6vJA3L98yv+IdfKGZHBNiGW5KHn22e/JwID0Strsz8h4S/csAu/OuICwxrg44k5MRiZHWIo8XXuJgQTriRP4w==} + + cjs-module-lexer@2.3.0: + resolution: {integrity: sha512-lmNyBzi6iYiqyrExTctfmjd8tInx+dHu9PBomTfH5RXS53c27JMFJgzNv5TyAnm3PgGf+MLy0+Ws41/HlVxdKQ==} + + cli-table3@0.6.5: + resolution: {integrity: sha512-+W/5efTR7y5HRD7gACw9yQjqMVvEMLBHmboM/kPWam+H+Hmyrgjh6YncVKK122YZkXrLudzTuAukUw9FnMf7IQ==} + engines: {node: 10.* || >= 12.*} + + commander@9.5.0: + resolution: {integrity: sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==} + engines: {node: ^12.20.0 || >=14} + + confbox@0.1.8: + resolution: {integrity: sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==} + + confbox@0.2.4: + resolution: {integrity: sha512-ysOGlgTFbN2/Y6Cg3Iye8YKulHw+R2fNXHrgSmXISQdMnomY6eNDprVdW9R5xBguEqI954+S6709UyiO7B+6OQ==} + + confbox@0.3.1: + resolution: {integrity: sha512-cKUSoKa8YxFZZSmraVi7onONx3amu77ngK3kGpsYHDH7drPwCRkQE1RYMPlLRrMtnciRj274XNRxcHxnKmDSnA==} + + consola@3.4.2: + resolution: {integrity: sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==} + engines: {node: ^14.18.0 || >=16.10.0} + + content-disposition@2.0.1: + resolution: {integrity: sha512-e+H0ZXHSWYrENhQzw1LPuP4oF5MzVKmDU6d3hxlvaPEYLLg62MxtQNPRx4SYSuYJSBUgnQIG4HIN2tEtNv7Dog==} + engines: {node: '>=18'} + + content-type@1.0.5: + resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} + engines: {node: '>= 0.6'} + + content-type@2.1.0: + resolution: {integrity: sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==} + engines: {node: '>=18'} + + convert-source-map@2.0.0: + resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} + + convex-test@0.0.65: + resolution: {integrity: sha512-2FgHlLGraM2O3tEVpOfvoHSleK0x92J8xjQ/qClWzNJX3NcmgvuNeWBrmma7sLJ8lB3ENPxx6ykJTfvZmWg3BQ==} + peerDependencies: + convex: ^1.43.0 + + convex@1.48.0: + resolution: {integrity: sha512-M9jEgTvCN8ziP/VgQQUH+j0ijM7g+NmQd8tFlvETK8/HMC/aaqHZwcu4hBO30X5DV2VIyYK4mOoLQGIb1766ew==} + engines: {node: '>=20.0.0', npm: '>=7.0.0'} + hasBin: true + peerDependencies: + '@auth0/auth0-react': ^2.0.1 + '@clerk/clerk-react': ^4.12.8 || ^5.0.0 + '@clerk/react': ^6.4.3 + react: ^18.0.0 || ^19.0.0-0 || ^19.0.0 + peerDependenciesMeta: + '@auth0/auth0-react': + optional: true + '@clerk/clerk-react': + optional: true + '@clerk/react': + optional: true + react: + optional: true + + cookie-signature@1.2.2: + resolution: {integrity: sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==} + engines: {node: '>=6.6.0'} + + cookie@0.7.2: + resolution: {integrity: sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==} + engines: {node: '>= 0.6'} + + cors@2.8.6: + resolution: {integrity: sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==} + engines: {node: '>= 0.10'} + + cross-spawn@7.0.6: + resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} + engines: {node: '>= 8'} + + csstype@3.2.3: + resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + deepmerge-ts@7.1.5: + resolution: {integrity: sha512-HOJkrhaYsweh+W+e74Yn7YStZOilkoPb6fycpwNLKzSPtruFs48nYis0zy5yJz1+ktUhHxoRDJ27RQAWLIJVJw==} + engines: {node: '>=16.0.0'} + + default-browser-id@5.0.1: + resolution: {integrity: sha512-x1VCxdX4t+8wVfd1so/9w+vQ4vx7lKd2Qp5tDRutErwmR85OgmfX7RlLRMWafRMY7hbEiXIbudNrjOAPa/hL8Q==} + engines: {node: '>=18'} + + default-browser@5.5.1: + resolution: {integrity: sha512-m1pAzaJgZ/gssEqlOhJkPJp8Xly7QyW6xcrkUa2KKcDeDSEMP7X8xipU3snUcfisTQx0w1AGae+9UtJSfVnXGw==} + engines: {node: '>=18'} + + define-data-property@1.1.4: + resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==} + engines: {node: '>= 0.4'} + + define-lazy-prop@3.0.0: + resolution: {integrity: sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg==} + engines: {node: '>=12'} + + defu@6.1.7: + resolution: {integrity: sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ==} + + depd@2.0.0: + resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} + engines: {node: '>= 0.8'} + + dequal@2.0.3: + resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} + engines: {node: '>=6'} + + destr@2.0.5: + resolution: {integrity: sha512-ugFTXCtDZunbzasqBxrK93Ik/DRYsO6S/fedkWEMKqt04xZ4csmnmwGDBAb07QWNaGMAmnTIemsYZCksjATwsA==} + + detect-libc@2.1.2: + resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} + engines: {node: '>=8'} + + dotenv@16.6.1: + resolution: {integrity: sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==} + engines: {node: '>=12'} + + dunder-proto@1.0.1: + resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} + engines: {node: '>= 0.4'} + + ee-first@1.1.1: + resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} + + effect@3.21.0: + resolution: {integrity: sha512-PPN80qRokCd1f015IANNhrwOnLO7GrrMQfk4/lnZRE/8j7UPWrNNjPV0uBrZutI/nHzernbW+J0hdqQysHiSnQ==} + + effect@3.22.4: + resolution: {integrity: sha512-RNshu7AXVxxDh6aS4lothhJ+QoZ4xSykbWNaaVdvDN72RMyVwhCGdALkyl1TbEydgAmRIdD1eTyJqBUGwSBGDA==} + + emoji-regex@8.0.0: + resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + + empathic@2.0.0: + resolution: {integrity: sha512-i6UzDscO/XfAcNYD75CfICkmfLedpyPDdozrLMmQc5ORaQcdMoc21OnlEylMIqI7U8eniKrPMxxtj8k0vhmJhA==} + engines: {node: '>=14'} + + encodeurl@2.0.0: + resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} + engines: {node: '>= 0.8'} + + engine.io-client@6.6.7: + resolution: {integrity: sha512-GyAvmPA7gnRCpDQbYACAy79Lg4ewevrmaPzySZz0OVvgyZc3gEfLLqPTYk8wVBScjOEpM9ZO4KryyFpVItjGEA==} + + engine.io-parser@5.2.3: + resolution: {integrity: sha512-HqD3yTBfnBxIrbnM1DoD6Pcq8NECnh8d4As1Qgh0z5Gg3jRRIqijury0CL3ghu/edArpUYiYqQiDUQBIs4np3Q==} + engines: {node: '>=10.0.0'} + + environment@1.1.0: + resolution: {integrity: sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==} + engines: {node: '>=18'} + + es-define-property@1.0.1: + resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} + engines: {node: '>= 0.4'} + + es-errors@1.3.0: + resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} + engines: {node: '>= 0.4'} + + es-module-lexer@2.3.2: + resolution: {integrity: sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==} + + es-object-atoms@1.1.2: + resolution: {integrity: sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==} + engines: {node: '>= 0.4'} + + esbuild@0.23.1: + resolution: {integrity: sha512-VVNz/9Sa0bs5SELtn3f7qhJCDPCF5oMEl5cO9/SSinpE9hbPVvxbd572HH5AKiP7WD8INO53GgfDDhRjkylHEg==} + engines: {node: '>=18'} + hasBin: true + + esbuild@0.27.0: + resolution: {integrity: sha512-jd0f4NHbD6cALCyGElNpGAOtWxSq46l9X/sWB0Nzd5er4Kz2YTm+Vl0qKFT9KUJvD8+fiO8AvoHhFvEatfVixA==} + engines: {node: '>=18'} + hasBin: true + + escape-html@1.0.3: + resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==} + + estree-walker@3.0.3: + resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + + etag@1.8.1: + resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} + engines: {node: '>= 0.6'} + + eventsource-parser@3.1.1: + resolution: {integrity: sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==} + engines: {node: '>=18.0.0'} + + eventsource@3.0.7: + resolution: {integrity: sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==} + engines: {node: '>=18.0.0'} + + evt@2.5.9: + resolution: {integrity: sha512-GpjX476FSlttEGWHT8BdVMoI8wGXQGbEOtKcP4E+kggg+yJzXBZN2n4x7TS/zPBJ1DZqWI+rguZZApjjzQ0HpA==} + + expect-type@1.4.0: + resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} + engines: {node: '>=12.0.0'} + + express-rate-limit@8.7.1: + resolution: {integrity: sha512-2FXbqkygil3mZiXzEtM0IjE/UBmBNot0J1GEVrJT4JbrvwOSoEtqtO+wHbiMQELWe1eNvHoYRg2dkjOid+6JkA==} + engines: {node: '>= 16'} + peerDependencies: + express: '>= 4.11' + + express@5.3.0: + resolution: {integrity: sha512-qCqy1xSoaugxOD4IVHb1ZWdkvCF2W3aE2okKouWj8VwJ+ztskSiXoBnCSJqfKT+cVPJhuJqe9YLbY425+Zh00g==} + engines: {node: '>= 18'} + + exsolve@1.1.3: + resolution: {integrity: sha512-kuzL9zherLQoqDeQaS55AiiIg1kb4KYXaSusTtgZTPlvKP2gGCI82LNmBNh5RjDVUXIcmgx+/Mx7vPwRzBbhJA==} + + fast-check@3.23.2: + resolution: {integrity: sha512-h5+1OzzfCC3Ef7VbtKdcv7zsstUQwUDlYpUTvjeUsJAssPgLn7QzbboPtL5ro04Mq0rPOsMzl7q5hIbRs2wD1A==} + engines: {node: '>=8.0.0'} + + fast-deep-equal@3.1.3: + resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + + fast-npm-meta@0.2.2: + resolution: {integrity: sha512-E+fdxeaOQGo/CMWc9f4uHFfgUPJRAu7N3uB8GBvB3SDPAIWJK4GKyYhkAGFq+GYrcbKNfQIz5VVQyJnDuPPCrg==} + + fast-uri@3.1.9: + resolution: {integrity: sha512-hS/0Wt3Ya043oKXs0sIxrMpKQITXhMLIrVAKKXi4gozGC9v7YRWVnXIZzjNPTO02M9BdxLCWqK1yRXgNziVyJQ==} + + fdir@6.5.0: + resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} + engines: {node: '>=12.0.0'} + peerDependencies: + picomatch: ^3 || ^4 + peerDependenciesMeta: + picomatch: + optional: true + + fill-range@7.1.1: + resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} + engines: {node: '>=8'} + + finalhandler@2.1.1: + resolution: {integrity: sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==} + engines: {node: '>= 18.0.0'} + + forwarded@0.2.0: + resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} + engines: {node: '>= 0.6'} + + fresh@2.0.0: + resolution: {integrity: sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==} + engines: {node: '>= 0.8'} + + fs-minipass@2.1.0: + resolution: {integrity: sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==} + engines: {node: '>= 8'} + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + + get-intrinsic@1.3.0: + resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} + engines: {node: '>= 0.4'} + + get-proto@1.0.1: + resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} + engines: {node: '>= 0.4'} + + giget@1.2.5: + resolution: {integrity: sha512-r1ekGw/Bgpi3HLV3h1MRBIlSAdHoIMklpaQ3OQLFcRw9PwAj2rqigvIbg+dBUI51OxVI2jsEtDywDBjSiuf7Ug==} + hasBin: true + + giget@2.0.0: + resolution: {integrity: sha512-L5bGsVkxJbJgdnwyuheIunkGatUF/zssUoxxjACCseZYAVbaqdh9Tsmmlkl8vYan09H7sbvKt4pS8GqKLBrEzA==} + hasBin: true + + git-last-commit@1.0.1: + resolution: {integrity: sha512-FDSgeMqa7GnJDxt/q0AbrxbfeTyxp4ImxEw1e4nw6NUHA5FMhFUq33dTXI4Xdgcj1VQ1q5QLWF6WxFrJ8KCBOg==} + + glob-parent@5.1.2: + resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} + engines: {node: '>= 6'} + + gopd@1.2.0: + resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} + engines: {node: '>= 0.4'} + + has-flag@5.0.1: + resolution: {integrity: sha512-CsNUt5x9LUdx6hnk/E2SZLsDyvfqANZSUq4+D3D8RzDJ2M+HDTIkF60ibS1vHaK55vzgiZw1bEPFG9yH7l33wA==} + engines: {node: '>=12'} + + has-property-descriptors@1.0.2: + resolution: {integrity: sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==} + + has-symbols@1.1.0: + resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} + engines: {node: '>= 0.4'} + + hasown@2.0.4: + resolution: {integrity: sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==} + engines: {node: '>= 0.4'} + + hono@4.13.14: + resolution: {integrity: sha512-yhRLQ21U6qa9Hsu2N7Qcbh8tE+WKEcoErjIo56LeSGiGXGp9sA7UpE86Ui9uexb61KakiGwuHMn2QdV7w6i6kQ==} + engines: {node: '>=16.9.0'} + + http-errors@2.0.1: + resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} + engines: {node: '>= 0.8'} + + humanize-duration@3.35.0: + resolution: {integrity: sha512-+KYh+R85n1yadHHwy/QXF0JgPq5NPPBNrvigA5aAKrlKFd0bJpMx7wE3Xd6rM0kEaBQMCFrdMihkaDSgWERNvQ==} + + iconv-lite@0.7.3: + resolution: {integrity: sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==} + engines: {node: '>=0.10.0'} + + ignore@7.0.14: + resolution: {integrity: sha512-wCBiZ77u1tuWde6G/nfOtLaCTYcFDU5yCzARzK10y+rLhzYoL7a7yEemcopcc2Mzow7gZOIjnV2ym1NYmdiJOQ==} + engines: {node: '>= 4'} + + import-in-the-middle@3.0.1: + resolution: {integrity: sha512-pYkiyXVL2Mf3pozdlDGV6NAObxQx13Ae8knZk1UJRJ6uRW/ZRmTGHlQYtrsSl7ubuE5F8CD1z+s1n4RHNuTtuA==} + engines: {node: '>=18'} + + inherits@2.0.4: + resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + + ini@5.0.0: + resolution: {integrity: sha512-+N0ngpO3e7cRUWOJAS7qw0IZIVc6XPrW4MlFBdD066F2L4k1L6ker3hLqSq7iXxU5tgS4WGkIUElWn5vogAEnw==} + engines: {node: ^18.17.0 || >=20.5.0} + + ip-address@10.7.5: + resolution: {integrity: sha512-t2v0rkILYv9WXmSL7TpAaU3HXgaFYOPUcH3QHgwQ++gpJuH3bqyMy4Fvt/DJCTKwzH3Cf+bAp5W/mFL39KPLzA==} + engines: {node: '>= 12'} + + ipaddr.js@1.9.1: + resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} + engines: {node: '>= 0.10'} + + is-binary-path@2.1.0: + resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==} + engines: {node: '>=8'} + + is-core-module@2.17.0: + resolution: {integrity: sha512-J/vG0zBCbIKOQFfufSwyXdMrsohyJIUNkrnmo6WZGzoM7tr/lsbfW5b2BvisL6zsyMzK9UxV9L6c7AoFbyXHOA==} + engines: {node: '>= 0.4'} + + is-docker@3.0.0: + resolution: {integrity: sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + hasBin: true + + is-extglob@2.1.1: + resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} + engines: {node: '>=0.10.0'} + + is-fullwidth-code-point@3.0.0: + resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} + engines: {node: '>=8'} + + is-glob@4.0.3: + resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} + engines: {node: '>=0.10.0'} + + is-inside-container@1.0.0: + resolution: {integrity: sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA==} + engines: {node: '>=14.16'} + hasBin: true + + is-network-error@1.3.2: + resolution: {integrity: sha512-PhBY86zaxNZUuWP6h13Vu5oFe0XY6/UlKzQnYFELzGVHygP3MxmvTfYSG7GN3aIab/iWudSMgjSnG9Dq+nHrgA==} + engines: {node: '>=16'} + + is-number@7.0.0: + resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} + engines: {node: '>=0.12.0'} + + is-promise@4.0.0: + resolution: {integrity: sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==} + + is-wsl@3.1.1: + resolution: {integrity: sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw==} + engines: {node: '>=16'} + + isarray@2.0.5: + resolution: {integrity: sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==} + + isexe@2.0.0: + resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + + jiti@1.21.7: + resolution: {integrity: sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==} + hasBin: true + + jiti@2.7.0: + resolution: {integrity: sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==} + hasBin: true + + jose@5.10.0: + resolution: {integrity: sha512-s+3Al/p9g32Iq+oqXxkW//7jk2Vig6FF1CFqzVXoTUXt2qz89YWbL+OwS17NFYEvxC35n0FKeGO2LGYSxeM2Gg==} + + jose@6.2.13: + resolution: {integrity: sha512-7TIg10ZE1x82YediN+lcrPNvGMKge9yFHQD+Qho/b8y2adfugU3AsvvHfxV0fzyFllT6l9xCjf4RWY9g/Z9PJQ==} + + json-schema-traverse@1.0.0: + resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + + json-schema-typed@8.0.2: + resolution: {integrity: sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==} + + json-schema@0.4.0: + resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==} + + json-stable-stringify@1.3.0: + resolution: {integrity: sha512-qtYiSSFlwot9XHtF9bD9c7rwKjr+RecWT//ZnPvSmEjpV5mmPOCN4j8UjY5hbjNkOwZ/jQv3J6R1/pL7RwgMsg==} + engines: {node: '>= 0.4'} + + jsonc-parser@3.2.1: + resolution: {integrity: sha512-AilxAyFOAcK5wA1+LeaySVBrHsGQvUFCDWXKpZjzaL0PqW+xfBOttn8GNtWKFWqneyMZj41MWF9Kl6iPWLwgOA==} + + jsonify@0.0.1: + resolution: {integrity: sha512-2/Ki0GcmuqSrgFyelQq9M05y7PS0mEwuIzrf3f1fPqkVDVRvZrPZtVSMHxdgo8Aq0sxAOb/cr2aqqA3LeWHVPg==} + + lightningcss-android-arm64@1.33.0: + resolution: {integrity: sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [android] + + lightningcss-darwin-arm64@1.33.0: + resolution: {integrity: sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [darwin] + + lightningcss-darwin-x64@1.33.0: + resolution: {integrity: sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [darwin] + + lightningcss-freebsd-x64@1.33.0: + resolution: {integrity: sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [freebsd] + + lightningcss-linux-arm-gnueabihf@1.33.0: + resolution: {integrity: sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm] + os: [linux] + + lightningcss-linux-arm64-gnu@1.33.0: + resolution: {integrity: sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + lightningcss-linux-arm64-musl@1.33.0: + resolution: {integrity: sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + libc: [musl] + + lightningcss-linux-x64-gnu@1.33.0: + resolution: {integrity: sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + libc: [glibc] + + lightningcss-linux-x64-musl@1.33.0: + resolution: {integrity: sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + libc: [musl] + + lightningcss-win32-arm64-msvc@1.33.0: + resolution: {integrity: sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [win32] + + lightningcss-win32-x64-msvc@1.33.0: + resolution: {integrity: sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [win32] + + lightningcss@1.33.0: + resolution: {integrity: sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==} + engines: {node: '>= 12.0.0'} + + magic-string@0.30.21: + resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + + magicast@0.3.5: + resolution: {integrity: sha512-L0WhttDl+2BOsybvEOLK7fW3UA0OQ0IQ2d6Zl2x/a6vVRs3bAY0ECOSHHeL5jD+SbOpOCUEi0y1DgHEn9Qn1AQ==} + + math-intrinsics@1.1.0: + resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} + engines: {node: '>= 0.4'} + + media-typer@1.1.1: + resolution: {integrity: sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==} + engines: {node: '>= 0.8'} + + merge-descriptors@2.0.0: + resolution: {integrity: sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==} + engines: {node: '>=18'} + + mime-db@1.54.0: + resolution: {integrity: sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==} + engines: {node: '>= 0.6'} + + mime-types@3.0.2: + resolution: {integrity: sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==} + engines: {node: '>=18'} + + minimal-polyfills@2.2.3: + resolution: {integrity: sha512-oxdmJ9cL+xV72h0xYxp4tP2d5/fTBpP45H8DIOn9pASuF8a3IYTf+25fMGDYGiWW+MFsuog6KD6nfmhZJQ+uUw==} + + minimatch@10.2.6: + resolution: {integrity: sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==} + engines: {node: 18 || 20 || >=22} + + minipass@3.3.6: + resolution: {integrity: sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==} + engines: {node: '>=8'} + + minipass@5.0.0: + resolution: {integrity: sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==} + engines: {node: '>=8'} + + minipass@7.1.3: + resolution: {integrity: sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==} + engines: {node: '>=16 || 14 >=14.17'} + + minizlib@2.1.2: + resolution: {integrity: sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==} + engines: {node: '>= 8'} + + minizlib@3.1.0: + resolution: {integrity: sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==} + engines: {node: '>= 18'} + + mkdirp@1.0.4: + resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==} + engines: {node: '>=10'} + hasBin: true + + mlly@1.8.2: + resolution: {integrity: sha512-d+ObxMQFmbt10sretNDytwt85VrbkhhUA/JBGm1MPaWJ65Cl4wOgLaB1NYvJSZ0Ef03MMEU/0xpPMXUIQ29UfA==} + + module-details-from-path@1.0.4: + resolution: {integrity: sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + nanoid@3.3.18: + resolution: {integrity: sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + + nanoid@3.3.20: + resolution: {integrity: sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + + negotiator@1.1.0: + resolution: {integrity: sha512-NMPBRMJgiQHjbd8phG3Vebdx4kZ1H121rbl5IkMqeOsahptB9BKo/d7oJ3zTXqTgagn2bWlNSXkh0QUGM31RYg==} + engines: {node: '>=18'} + + node-fetch-native@1.6.7: + resolution: {integrity: sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q==} + + normalize-path@3.0.0: + resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} + engines: {node: '>=0.10.0'} + + nypm@0.5.4: + resolution: {integrity: sha512-X0SNNrZiGU8/e/zAB7sCTtdxWTMSIO73q+xuKgglm2Yvzwlo8UoC5FNySQFCvl84uPaeADkqHUZUkWy4aH4xOA==} + engines: {node: ^14.16.0 || >=16.10.0} + hasBin: true + + nypm@0.6.12: + resolution: {integrity: sha512-4AhWntz+ds/P5kz1rN25yGegTPYsO5ffOy+8S0TeuXKZrgjlmgcEBLA1xubzcP/dWX4zYfNybv4TQERlL9Z0LQ==} + engines: {node: '>=18'} + hasBin: true + + object-assign@4.1.1: + resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} + engines: {node: '>=0.10.0'} + + object-inspect@1.13.4: + resolution: {integrity: sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==} + engines: {node: '>= 0.4'} + + object-keys@1.1.1: + resolution: {integrity: sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==} + engines: {node: '>= 0.4'} + + obug@2.2.1: + resolution: {integrity: sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q==} + engines: {node: '>=12.20.0'} + + ohash@1.1.6: + resolution: {integrity: sha512-TBu7PtV8YkAZn0tSxobKY2n2aAQva936lhRrj6957aDaCf9IEtqsKbgMzXE/F/sjqYOwmrukeORHNLe5glk7Cg==} + + ohash@2.0.12: + resolution: {integrity: sha512-65S/5gk9YSsaRjcyf7Nfa6h/d3E8/1gslpXfI4W7Dxn/oap8IKRuNT5VXkLQ1YFKIEg4apRY4Pj6aiwFzrDdmw==} + + on-finished@2.4.2: + resolution: {integrity: sha512-ATWUv387i+nLD49EiDoOft5scf32Vsdz3ktrQJG9bi7jRFmaXTssvyvj1EmeKVVgvesbqUtvQQvaS2TpTukAZQ==} + engines: {node: '>= 0.8'} + + once@1.4.0: + resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + + open@10.2.0: + resolution: {integrity: sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==} + engines: {node: '>=18'} + + os-paths@7.4.0: + resolution: {integrity: sha512-Ux1J4NUqC6tZayBqLN1kUlDAEvLiQlli/53sSddU4IN+h+3xxnv2HmRSMpVSvr1hvJzotfMs3ERvETGK+f4OwA==} + engines: {node: '>= 4.0'} + + p-limit@6.2.0: + resolution: {integrity: sha512-kuUqqHNUqoIWp/c467RI4X6mmyuojY5jGutNU0wVTmEOOfcuwLqyMVoAi9MKi2Ak+5i9+nhmrK4ufZE8069kHA==} + engines: {node: '>=18'} + + p-retry@6.2.1: + resolution: {integrity: sha512-hEt02O4hUct5wtwg4H4KcWgDdm+l1bOaEy/hWzd8xtXB9BqxTWBBhb+2ImAtH4Cv4rPjV76xN3Zumqk3k3AhhQ==} + engines: {node: '>=16.17'} + + parseurl@1.3.3: + resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} + engines: {node: '>= 0.8'} + + path-key@3.1.1: + resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} + engines: {node: '>=8'} + + path-parse@1.0.7: + resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} + + path-to-regexp@8.4.2: + resolution: {integrity: sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==} + + pathe@1.1.2: + resolution: {integrity: sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==} + + pathe@2.0.3: + resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + + perfect-debounce@1.0.0: + resolution: {integrity: sha512-xCy9V055GLEqoFaHoC1SoLIaLmWctgCUaBaWxDZ7/Zx4CTyX7cJQLJOok/orfjZAh9kEYpjJa4d0KcJmCbctZA==} + + picocolors@1.1.1: + resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} + + picomatch@2.3.2: + resolution: {integrity: sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==} + engines: {node: '>=8.6'} + + picomatch@4.0.7: + resolution: {integrity: sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==} + engines: {node: '>=12'} + + pkce-challenge@5.0.1: + resolution: {integrity: sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==} + engines: {node: '>=16.20.0'} + + pkg-types@1.3.1: + resolution: {integrity: sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==} + + pkg-types@2.4.0: + resolution: {integrity: sha512-07Fys7iVx0X6TcviMOgogM1Lk33ZyqOezMnU79S+kffWqwolG/5W9V8EgQSeYCIfFTYVgssRbnH630Oa2+soBA==} + + postcss@8.5.31: + resolution: {integrity: sha512-YUbWYCgErwnMAVkjUnYa+KlMBzGpHwiIt/DE7f0G33AuX2LH9UcTk6KgWxd3zRU+pN3u/C+Xl3EnMLM4Hea7Sg==} + engines: {node: ^10 || ^12 || >=14} + + posthog-node@5.35.6: + resolution: {integrity: sha512-LwoXnR89A0l75jvFrXjtsAs0BbpyCjnwY8YIUkZT91rt1YnIUfBYiLj7qoUYApdNgesgWQQHqLXxLYX59a6ZYw==} + engines: {node: ^20.20.0 || >=22.22.0} + peerDependencies: + rxjs: ^7.0.0 + peerDependenciesMeta: + rxjs: + optional: true + + prettier@3.9.10: + resolution: {integrity: sha512-skGHTH1Yb9QvaTC4hWnRTzEYtCpApF0UMBPz+t/0ZPzP8ihefo/btKd+QA7pUE0fEwA1SbS/NmhyEcstdXOIdA==} + engines: {node: '>=14'} + hasBin: true + + prom-client@15.1.3: + resolution: {integrity: sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==} + engines: {node: ^16 || ^18 || >=20} + deprecated: prom-client has been replaced by @prometheus-io/client + + proxy-addr@2.0.8: + resolution: {integrity: sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ==} + engines: {node: '>= 0.10'} + + pure-rand@6.1.0: + resolution: {integrity: sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==} + + qs@6.16.1: + resolution: {integrity: sha512-5Px6J7rfHBAGUhNQ3es4YzcOdBBw0SDcVtThJml4RdnKw4tKs2epA0kPZ+K8X+WiHktT93RHVcQ/3ym5aTCgtg==} + engines: {node: '>=0.6'} + + range-parser@1.3.0: + resolution: {integrity: sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==} + engines: {node: '>= 0.6'} + + raw-body@3.0.2: + resolution: {integrity: sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==} + engines: {node: '>= 0.10'} + + rc9@2.1.2: + resolution: {integrity: sha512-btXCnMmRIBINM2LDZoEmOogIZU7Qe7zn4BpomSKZ/ykbLObuBdvG+mFq11DL6fjH1DRwHhrlgtYWG96bJiC7Cg==} + + react-dom@19.3.1: + resolution: {integrity: sha512-Y0bARz53lhEC1NMCASs0/j+faB/zqt3R8hpvg1e8+61ZVJ4UL0UdC4cBbXfZMcIuhrfgeILTLe+ZeBCavQPtKQ==} + peerDependencies: + react: ^19.3.1 + + react@19.3.1: + resolution: {integrity: sha512-QNhWdt7nrgVWWQG58QuGsAR3jv3Hbe4mb4vz7w2csnHavciZJgda1CPNaldF6eVxv90ud3jUXpKlHL69gO38yw==} + engines: {node: '>=0.10.0'} + + readdirp@3.6.0: + resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} + engines: {node: '>=8.10.0'} + + readdirp@4.1.2: + resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} + engines: {node: '>= 14.18.0'} + + require-from-string@2.0.2: + resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} + engines: {node: '>=0.10.0'} + + require-in-the-middle@8.0.1: + resolution: {integrity: sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ==} + engines: {node: '>=9.3.0 || >=8.10.0 <9.0.0'} + + resolve@1.22.13: + resolution: {integrity: sha512-Dj/cW9zBV2aRj6XzCxuGIs3ESZKv02jixne3q6jFZYZ3/Tg09gHHviSMDL4kObh9pqSahdRHEnDoaP2gn8YLhA==} + engines: {node: '>= 0.4'} + hasBin: true + + retry@0.13.1: + resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} + engines: {node: '>= 4'} + + rolldown@1.2.15: + resolution: {integrity: sha512-6odND+fel5zVw3I4Yc2e69gvguDnX0CAAGiNDZSq4ik5WUcaLrkD2GLuog0XcODbor31hZ5ayEMgVDno+KPV5g==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + + router@2.2.0: + resolution: {integrity: sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==} + engines: {node: '>= 18'} + + run-applescript@7.1.0: + resolution: {integrity: sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q==} + engines: {node: '>=18'} + + run-exclusive@2.2.19: + resolution: {integrity: sha512-K3mdoAi7tjJ/qT7Flj90L7QyPozwUaAG+CVhkdDje4HLKXUYC3N/Jzkau3flHVDLQVhiHBtcimVodMjN9egYbA==} + + safer-buffer@2.1.2: + resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + + scheduler@0.28.0: + resolution: {integrity: sha512-juorfCmIkIw8tT+p5BXSm6PJjQF/ycEYmKyzURCIt/RaZIhL+PulbQ9Yu2z1HdOJDdqDTlxA1+xKBmHXJsczAw==} + + semver@7.8.5: + resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} + engines: {node: '>=10'} + hasBin: true + + send@1.2.1: + resolution: {integrity: sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==} + engines: {node: '>= 18'} + + serve-static@2.2.1: + resolution: {integrity: sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==} + engines: {node: '>= 18'} + + set-function-length@1.2.2: + resolution: {integrity: sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==} + engines: {node: '>= 0.4'} + + setprototypeof@1.2.0: + resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} + + sharp@0.35.5: + resolution: {integrity: sha512-Ywn4OnzGukp7CDMrp08RQ50YKmuwG47brZgIVPTvBaaAfQlRlygrRqSrxdCiL9M+LlzLBiJ68IR1QqvzHyjC7g==} + engines: {node: '>=20.9.0'} + peerDependencies: + '@types/node': '*' + peerDependenciesMeta: + '@types/node': + optional: true + + shebang-command@2.0.0: + resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} + engines: {node: '>=8'} + + shebang-regex@3.0.0: + resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} + engines: {node: '>=8'} + + side-channel-list@1.0.1: + resolution: {integrity: sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==} + engines: {node: '>= 0.4'} + + side-channel-map@1.0.1: + resolution: {integrity: sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==} + engines: {node: '>= 0.4'} + + side-channel-weakmap@1.0.2: + resolution: {integrity: sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==} + engines: {node: '>= 0.4'} + + side-channel@1.1.1: + resolution: {integrity: sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==} + engines: {node: '>= 0.4'} + + siginfo@2.0.0: + resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + + signal-exit@4.1.0: + resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} + engines: {node: '>=14'} + + sisteransi@1.0.5: + resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==} + + socket.io-client@4.8.4: + resolution: {integrity: sha512-7wjKAvVXYOJsbmUgkWbnhPD4bFpkqrUR0++XOjpGFmqDqsqV1CI93VTraeExLxpjxEqrWgFoSRZu1ihjLWfkAA==} + engines: {node: '>=10.0.0'} + + socket.io-parser@4.2.7: + resolution: {integrity: sha512-IH/iSeO9T6gz1KkFleGDWkG9N3dl4jXVYUtMhIqH10Md0ttMer8nUNWiP1DKuNrybD2xBrixLJdCC9J6ECoYkg==} + engines: {node: '>=10.0.0'} + + source-map-js@1.2.2: + resolution: {integrity: sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==} + engines: {node: '>=0.10.0'} + + source-map-support@0.5.21: + resolution: {integrity: sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==} + + source-map@0.6.1: + resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} + engines: {node: '>=0.10.0'} + + stackback@0.0.2: + resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + + statuses@2.0.2: + resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==} + engines: {node: '>= 0.8'} + + std-env@3.10.0: + resolution: {integrity: sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==} + + std-env@4.3.0: + resolution: {integrity: sha512-OtU/EgQ1kIm5KwqQpBC6ZEMXrZRui11w8zgfTWp8cdO9B8OaPsbA8bTHO2P+HNo1VlUTGMVBwPhydu6poeXiag==} + + string-width@4.2.3: + resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} + engines: {node: '>=8'} + + strip-ansi@6.0.1: + resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} + engines: {node: '>=8'} + + strip-ansi@7.2.0: + resolution: {integrity: sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==} + engines: {node: '>=12'} + + supports-color@10.2.2: + resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} + engines: {node: '>=18'} + + supports-preserve-symlinks-flag@1.0.0: + resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==} + engines: {node: '>= 0.4'} + + swr@2.5.1: + resolution: {integrity: sha512-BRw55e8r0B7SpDN20CAzoQAHl7y1yP7/Zt7oqUjMv0vSt2u2Xnkm88Ws+VypbV9BXHQVuSuyVq7zMjO16wSExw==} + peerDependencies: + react: ^16.11.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + + systeminformation@5.33.16: + resolution: {integrity: sha512-R+Lh69IJXfJj6bfyH1In5cIR9FTr4hBS39YvlZHZjsK1p9cuELPq7hplZZIb56ZESMMb+DzdKqqE6Iomi01Mvg==} + engines: {node: '>=10.0.0'} + os: [darwin, linux, win32, freebsd, openbsd, netbsd, sunos, android] + hasBin: true + + tar@6.2.1: + resolution: {integrity: sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==} + engines: {node: '>=10'} + deprecated: Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me + + tar@7.5.22: + resolution: {integrity: sha512-MFO/QzvtAOmJbkhOaCTvbGcFN9L9b+JunIsDwaKljSOdcLMea3NJ1k9Usz/rjdfSXTq4dfzfeS7W4p4YOAAHeA==} + engines: {node: '>=18'} + + tdigest@0.1.3: + resolution: {integrity: sha512-zbRt+lT+/H4fRItHshczHErVCQnitJk8MfMT24MqFJf3YL7SJJPqGIGeuOdvxXxM/AHFzKBl7WoyaYwqO9s3Kw==} + + tinybench@2.9.0: + resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} + + tinyexec@0.3.2: + resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} + + tinyexec@1.3.1: + resolution: {integrity: sha512-GCvB3aoys96IuDFBMcTB46JOR6mdMtAToqwiW8JlWhsoh1mhHi/xn9ss/Dg7N555GiJyEt2qzoG/NHCwM6h1EA==} + engines: {node: '>=18'} + + tinyglobby@0.2.17: + resolution: {integrity: sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==} + engines: {node: '>=12.0.0'} + + tinyrainbow@3.2.0: + resolution: {integrity: sha512-LgO3D9yZJjApUiuUfl9iFAwrtaX4+lok3wJIqttGoKCHlWUqHqbQpnxCf82L8FjgKsh4iGo78hqJwgL8F6To2A==} + engines: {node: '>=14.0.0'} + + to-regex-range@5.0.1: + resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} + engines: {node: '>=8.0'} + + toidentifier@1.0.1: + resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} + engines: {node: '>=0.6'} + + trigger.dev@4.8.3: + resolution: {integrity: sha512-4Ut5OOnAs5C2z9ePQu7CpRi0QSODfi4stFDlmvI7DzkgsNNHf6mwc+tiTGzF0V/KMSBuhNvZPGNYEOX76mr0mA==} + engines: {node: '>=18.20.0'} + hasBin: true + + tsafe@1.8.12: + resolution: {integrity: sha512-nFRqW0ttu/2o6XTXsHiVZWJBCOaxhVqZLg7dgs3coZNsCMPXPfwz+zPHAQA+70fNnVJLAPg1EgGIqK9Q84tvAw==} + + tsconfck@3.1.3: + resolution: {integrity: sha512-ulNZP1SVpRDesxeMLON/LtWM8HIgAJEIVpVVhBM6gsmvQ8+Rh+ZG7FWGvHh7Ah3pRABwVJWklWCr/BTZSv0xnQ==} + engines: {node: ^18 || >=20} + deprecated: unmaintained + hasBin: true + peerDependencies: + typescript: ^5.0.0 + peerDependenciesMeta: + typescript: + optional: true + + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + + type-is@2.1.0: + resolution: {integrity: sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==} + engines: {node: '>= 18'} + + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + + ufo@1.6.4: + resolution: {integrity: sha512-JFNbkD1Svwe0KvGi8GOeLcP4kAWQ609twvCdcHxq1oSL8svv39ZuSvajcD8B+5D0eL4+s1Is2D/O6KN3qcTeRA==} + + uncrypto@0.1.3: + resolution: {integrity: sha512-Ql87qFHB3s/De2ClA9e0gsnS6zXG27SkTiSJwjCc9MebbfapQfuPzumMIUMi38ezPZVNFcHI9sUIepeQfw8J8Q==} + + undici-types@7.18.2: + resolution: {integrity: sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==} + + undici@7.30.1: + resolution: {integrity: sha512-03KrMbjIFDWQR5RNPrSw3bd1r12JpyFQl9/yxjUbJtqkjmllqC5lFjZh6ujeuJCKr8EOQvRW872Oy/az+nLYxg==} + engines: {node: '>=20.18.1'} + + unpipe@1.0.0: + resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} + engines: {node: '>= 0.8'} + + use-sync-external-store@1.7.0: + resolution: {integrity: sha512-6L+EeigHMQhdaIPNIFUKwfWJSwWFQ8gJbJ2DLOs5sDIegTwR9fRxvnM3uciHKjIZhFz+KAv2emhWMRvDmMcY8A==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + + vary@1.1.2: + resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} + engines: {node: '>= 0.8'} + + vite@8.3.5: + resolution: {integrity: sha512-z1YpjieJ9g0ebTmQpLBv/oq9E3F9EIlOe+7LahW8SrE3LTiG3udBRUXQOlYXP9b+IC2q2phHeVxj9xGlw34OEQ==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + peerDependencies: + '@types/node': ^20.19.0 || >=22.12.0 + '@vitejs/devtools': ^0.7.1 || ^0.8.0 + esbuild: ^0.27.0 || ^0.28.0 + jiti: '>=1.21.0' + less: ^4.0.0 + sass: ^1.70.0 + sass-embedded: ^1.70.0 + stylus: '>=0.54.8' + sugarss: ^5.0.0 + terser: ^5.16.0 + tsx: ^4.8.1 + yaml: ^2.4.2 + peerDependenciesMeta: + '@types/node': + optional: true + '@vitejs/devtools': + optional: true + esbuild: + optional: true + jiti: + optional: true + less: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + tsx: + optional: true + yaml: + optional: true + + vitest@4.1.11: + resolution: {integrity: sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw==} + engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@opentelemetry/api': ^1.9.0 + '@types/node': ^20.0.0 || ^22.0.0 || >=24.0.0 + '@vitest/browser-playwright': 4.1.11 + '@vitest/browser-preview': 4.1.11 + '@vitest/browser-webdriverio': 4.1.11 + '@vitest/coverage-istanbul': 4.1.11 + '@vitest/coverage-v8': 4.1.11 + '@vitest/ui': 4.1.11 + happy-dom: '*' + jsdom: '*' + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@opentelemetry/api': + optional: true + '@types/node': + optional: true + '@vitest/browser-playwright': + optional: true + '@vitest/browser-preview': + optional: true + '@vitest/browser-webdriverio': + optional: true + '@vitest/coverage-istanbul': + optional: true + '@vitest/coverage-v8': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + + which@2.0.2: + resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} + engines: {node: '>= 8'} + hasBin: true + + why-is-node-running@2.3.0: + resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} + engines: {node: '>=8'} + hasBin: true + + wrappy@1.0.2: + resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + + ws@8.21.0: + resolution: {integrity: sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + + ws@8.21.3: + resolution: {integrity: sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + + wsl-utils@0.1.0: + resolution: {integrity: sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw==} + engines: {node: '>=18'} + + xdg-app-paths@8.3.0: + resolution: {integrity: sha512-mgxlWVZw0TNWHoGmXq+NC3uhCIc55dDpAlDkMQUaIAcQzysb0kxctwv//fvuW61/nAAeUBJMQ8mnZjMmuYwOcQ==} + engines: {node: '>= 4.0'} + + xdg-portable@10.6.0: + resolution: {integrity: sha512-xrcqhWDvtZ7WLmt8G4f3hHy37iK7D2idtosRgkeiSPZEPmBShp0VfmRBLWAPC6zLF48APJ21yfea+RfQMF4/Aw==} + engines: {node: '>= 4.0'} + + xmlhttprequest-ssl@2.1.2: + resolution: {integrity: sha512-TEU+nJVUUnA4CYJFLvK5X9AOeH4KvDvhIfm0vV1GaQRtchnG0hgK5p8hw/xjv8cunWYCsiPCSDzObPyhEwq3KQ==} + engines: {node: '>=0.4.0'} + + yallist@4.0.0: + resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + + yallist@5.0.0: + resolution: {integrity: sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==} + engines: {node: '>=18'} + + yocto-queue@1.2.2: + resolution: {integrity: sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==} + engines: {node: '>=12.20'} + + zod-to-json-schema@3.25.2: + resolution: {integrity: sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==} + peerDependencies: + zod: ^3.25.28 || ^4 + + zod-validation-error@5.0.0: + resolution: {integrity: sha512-hmk+pkyKq7Q71PiWVSDUc3VfpzpvcRHZ3QPw9yEMVvmtCekaMeOHnbr3WbxfrgEnQTv6haGP4cmv0Ojmihzsxw==} + engines: {node: '>=18.0.0'} + peerDependencies: + zod: ^3.25.0 || ^4.0.0 + + zod@4.5.4: + resolution: {integrity: sha512-sC95tT5iHHH9gtpj6A81kh+NEaRAUFN+qlUPDUbRfOMvNf5QCBqsb3WgvnpVtK5Y+4UfA6KqufotuTvMGiTlsA==} + +snapshots: + + '@ai-sdk/gateway@4.0.113(zod@4.5.4)': + dependencies: + '@ai-sdk/provider': 4.0.27 + '@ai-sdk/provider-utils': 5.0.59(zod@4.5.4) + '@vercel/oidc': 3.2.0 + zod: 4.5.4 + + '@ai-sdk/provider-utils@5.0.59(zod@4.5.4)': + dependencies: + '@ai-sdk/provider': 4.0.27 + '@standard-schema/spec': 1.1.0 + '@workflow/serde': 4.1.0 + eventsource-parser: 3.1.1 + undici: 7.30.1 + zod: 4.5.4 + + '@ai-sdk/provider@4.0.27': + dependencies: + json-schema: 0.4.0 + + '@babel/helper-string-parser@7.29.11': {} + + '@babel/helper-validator-identifier@7.29.7': {} + + '@babel/parser@7.29.11': + dependencies: + '@babel/types': 7.29.11 + + '@babel/types@7.29.11': + dependencies: + '@babel/helper-string-parser': 7.29.11 + '@babel/helper-validator-identifier': 7.29.7 + + '@bugsnag/cuid@3.2.2': {} + + '@clack/core@0.5.0': + dependencies: + picocolors: 1.1.1 + sisteransi: 1.0.5 + + '@clack/prompts@0.11.0': + dependencies: + '@clack/core': 0.5.0 + picocolors: 1.1.1 + sisteransi: 1.0.5 + + '@colors/colors@1.5.0': + optional: true + + '@convex-dev/static-hosting@0.2.1(convex@1.48.0(react@19.3.1))(react@19.3.1)': + dependencies: + convex: 1.48.0(react@19.3.1) + react: 19.3.1 + + '@depot/cli-darwin-arm64@0.0.1-cli.2.80.0': + optional: true + + '@depot/cli-darwin-x64@0.0.1-cli.2.80.0': + optional: true + + '@depot/cli-linux-arm64@0.0.1-cli.2.80.0': + optional: true + + '@depot/cli-linux-arm@0.0.1-cli.2.80.0': + optional: true + + '@depot/cli-linux-ia32@0.0.1-cli.2.80.0': + optional: true + + '@depot/cli-linux-x64@0.0.1-cli.2.80.0': + optional: true + + '@depot/cli-win32-arm64@0.0.1-cli.2.80.0': + optional: true + + '@depot/cli-win32-ia32@0.0.1-cli.2.80.0': + optional: true + + '@depot/cli-win32-x64@0.0.1-cli.2.80.0': + optional: true + + '@depot/cli@0.0.1-cli.2.80.0': + optionalDependencies: + '@depot/cli-darwin-arm64': 0.0.1-cli.2.80.0 + '@depot/cli-darwin-x64': 0.0.1-cli.2.80.0 + '@depot/cli-linux-arm': 0.0.1-cli.2.80.0 + '@depot/cli-linux-arm64': 0.0.1-cli.2.80.0 + '@depot/cli-linux-ia32': 0.0.1-cli.2.80.0 + '@depot/cli-linux-x64': 0.0.1-cli.2.80.0 + '@depot/cli-win32-arm64': 0.0.1-cli.2.80.0 + '@depot/cli-win32-ia32': 0.0.1-cli.2.80.0 + '@depot/cli-win32-x64': 0.0.1-cli.2.80.0 + + '@electric-sql/client@1.0.14': + dependencies: + '@microsoft/fetch-event-source': 2.0.1 + optionalDependencies: + '@rollup/rollup-darwin-arm64': 4.65.1 + + '@emnapi/runtime@1.11.3': + dependencies: + tslib: 2.8.1 + optional: true + + '@esbuild/aix-ppc64@0.23.1': + optional: true + + '@esbuild/aix-ppc64@0.27.0': + optional: true + + '@esbuild/android-arm64@0.23.1': + optional: true + + '@esbuild/android-arm64@0.27.0': + optional: true + + '@esbuild/android-arm@0.23.1': + optional: true + + '@esbuild/android-arm@0.27.0': + optional: true + + '@esbuild/android-x64@0.23.1': + optional: true + + '@esbuild/android-x64@0.27.0': + optional: true + + '@esbuild/darwin-arm64@0.23.1': + optional: true + + '@esbuild/darwin-arm64@0.27.0': + optional: true + + '@esbuild/darwin-x64@0.23.1': + optional: true + + '@esbuild/darwin-x64@0.27.0': + optional: true + + '@esbuild/freebsd-arm64@0.23.1': + optional: true + + '@esbuild/freebsd-arm64@0.27.0': + optional: true + + '@esbuild/freebsd-x64@0.23.1': + optional: true + + '@esbuild/freebsd-x64@0.27.0': + optional: true + + '@esbuild/linux-arm64@0.23.1': + optional: true + + '@esbuild/linux-arm64@0.27.0': + optional: true + + '@esbuild/linux-arm@0.23.1': + optional: true + + '@esbuild/linux-arm@0.27.0': + optional: true + + '@esbuild/linux-ia32@0.23.1': + optional: true + + '@esbuild/linux-ia32@0.27.0': + optional: true + + '@esbuild/linux-loong64@0.23.1': + optional: true + + '@esbuild/linux-loong64@0.27.0': + optional: true + + '@esbuild/linux-mips64el@0.23.1': + optional: true + + '@esbuild/linux-mips64el@0.27.0': + optional: true + + '@esbuild/linux-ppc64@0.23.1': + optional: true + + '@esbuild/linux-ppc64@0.27.0': + optional: true + + '@esbuild/linux-riscv64@0.23.1': + optional: true + + '@esbuild/linux-riscv64@0.27.0': + optional: true + + '@esbuild/linux-s390x@0.23.1': + optional: true + + '@esbuild/linux-s390x@0.27.0': + optional: true + + '@esbuild/linux-x64@0.23.1': + optional: true + + '@esbuild/linux-x64@0.27.0': + optional: true + + '@esbuild/netbsd-arm64@0.27.0': + optional: true + + '@esbuild/netbsd-x64@0.23.1': + optional: true + + '@esbuild/netbsd-x64@0.27.0': + optional: true + + '@esbuild/openbsd-arm64@0.23.1': + optional: true + + '@esbuild/openbsd-arm64@0.27.0': + optional: true + + '@esbuild/openbsd-x64@0.23.1': + optional: true + + '@esbuild/openbsd-x64@0.27.0': + optional: true + + '@esbuild/openharmony-arm64@0.27.0': + optional: true + + '@esbuild/sunos-x64@0.23.1': + optional: true + + '@esbuild/sunos-x64@0.27.0': + optional: true + + '@esbuild/win32-arm64@0.23.1': + optional: true + + '@esbuild/win32-arm64@0.27.0': + optional: true + + '@esbuild/win32-ia32@0.23.1': + optional: true + + '@esbuild/win32-ia32@0.27.0': + optional: true + + '@esbuild/win32-x64@0.23.1': + optional: true + + '@esbuild/win32-x64@0.27.0': + optional: true + + '@google-cloud/precise-date@4.0.0': {} + + '@hono/node-server@2.1.5(hono@4.13.14)': + dependencies: + hono: 4.13.14 + + '@img/colour@1.1.0': {} + + '@img/sharp-darwin-arm64@0.35.5': + optionalDependencies: + '@img/sharp-libvips-darwin-arm64': 1.3.4 + optional: true + + '@img/sharp-darwin-x64@0.35.5': + optionalDependencies: + '@img/sharp-libvips-darwin-x64': 1.3.4 + optional: true + + '@img/sharp-freebsd-wasm32@0.35.5': + dependencies: + '@img/sharp-wasm32': 0.35.5 + optional: true + + '@img/sharp-libvips-darwin-arm64@1.3.4': + optional: true + + '@img/sharp-libvips-darwin-x64@1.3.4': + optional: true + + '@img/sharp-libvips-linux-arm64@1.3.4': + optional: true + + '@img/sharp-libvips-linux-arm@1.3.4': + optional: true + + '@img/sharp-libvips-linux-ppc64@1.3.4': + optional: true + + '@img/sharp-libvips-linux-riscv64@1.3.4': + optional: true + + '@img/sharp-libvips-linux-s390x@1.3.4': + optional: true + + '@img/sharp-libvips-linux-x64@1.3.4': + optional: true + + '@img/sharp-libvips-linuxmusl-arm64@1.3.4': + optional: true + + '@img/sharp-libvips-linuxmusl-x64@1.3.4': + optional: true + + '@img/sharp-linux-arm64@0.35.5': + optionalDependencies: + '@img/sharp-libvips-linux-arm64': 1.3.4 + optional: true + + '@img/sharp-linux-arm@0.35.5': + optionalDependencies: + '@img/sharp-libvips-linux-arm': 1.3.4 + optional: true + + '@img/sharp-linux-ppc64@0.35.5': + optionalDependencies: + '@img/sharp-libvips-linux-ppc64': 1.3.4 + optional: true + + '@img/sharp-linux-riscv64@0.35.5': + optionalDependencies: + '@img/sharp-libvips-linux-riscv64': 1.3.4 + optional: true + + '@img/sharp-linux-s390x@0.35.5': + optionalDependencies: + '@img/sharp-libvips-linux-s390x': 1.3.4 + optional: true + + '@img/sharp-linux-x64@0.35.5': + optionalDependencies: + '@img/sharp-libvips-linux-x64': 1.3.4 + optional: true + + '@img/sharp-linuxmusl-arm64@0.35.5': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-arm64': 1.3.4 + optional: true + + '@img/sharp-linuxmusl-x64@0.35.5': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-x64': 1.3.4 + optional: true + + '@img/sharp-wasm32@0.35.5': + dependencies: + '@emnapi/runtime': 1.11.3 + optional: true + + '@img/sharp-webcontainers-wasm32@0.35.5': + dependencies: + '@img/sharp-wasm32': 0.35.5 + optional: true + + '@img/sharp-win32-arm64@0.35.5': + optional: true + + '@img/sharp-win32-ia32@0.35.5': + optional: true + + '@img/sharp-win32-x64@0.35.5': + optional: true + + '@isaacs/fs-minipass@4.0.1': + dependencies: + minipass: 7.1.3 + + '@jridgewell/sourcemap-codec@1.6.0': {} + + '@jsonhero/path@1.0.21': {} + + '@microsoft/fetch-event-source@2.0.1': {} + + '@modelcontextprotocol/sdk@1.32.2(supports-color@10.2.2)(zod@4.5.4)': + dependencies: + '@hono/node-server': 2.1.5(hono@4.13.14) + ajv: 8.20.0 + ajv-formats: 3.0.1(ajv@8.20.0) + content-type: 1.0.5 + cors: 2.8.6 + cross-spawn: 7.0.6 + eventsource: 3.0.7 + eventsource-parser: 3.1.1 + express: 5.3.0(supports-color@10.2.2) + express-rate-limit: 8.7.1(express@5.3.0(supports-color@10.2.2))(supports-color@10.2.2) + hono: 4.13.14 + jose: 6.2.13 + json-schema-typed: 8.0.2 + pkce-challenge: 5.0.1 + raw-body: 3.0.2 + zod: 4.5.4 + zod-to-json-schema: 3.25.2(zod@4.5.4) + transitivePeerDependencies: + - supports-color + + '@opentelemetry/api-logs@0.219.0': + dependencies: + '@opentelemetry/api': 1.9.1 + + '@opentelemetry/api@1.9.1': {} + + '@opentelemetry/context-async-hooks@2.8.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + + '@opentelemetry/core@2.8.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/exporter-logs-otlp-http@0.219.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.219.0 + '@opentelemetry/core': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-exporter-base': 0.219.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.219.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-logs': 0.219.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/exporter-metrics-otlp-http@0.219.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-exporter-base': 0.219.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.219.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-metrics': 2.8.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/exporter-trace-otlp-http@0.219.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-exporter-base': 0.219.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.219.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-base': 2.8.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/host-metrics@0.38.3(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + systeminformation: 5.33.16 + + '@opentelemetry/instrumentation@0.219.0(@opentelemetry/api@1.9.1)(supports-color@10.2.2)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.219.0 + import-in-the-middle: 3.0.1 + require-in-the-middle: 8.0.1(supports-color@10.2.2) + transitivePeerDependencies: + - supports-color + + '@opentelemetry/otlp-exporter-base@0.219.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.219.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/otlp-transformer@0.219.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.219.0 + '@opentelemetry/core': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-logs': 0.219.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-metrics': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-base': 2.8.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/resources@2.8.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/sdk-logs@0.219.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.219.0 + '@opentelemetry/core': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/sdk-metrics@2.8.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.8.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/sdk-trace-base@2.8.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/sdk-trace-node@2.8.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/context-async-hooks': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/core': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-base': 2.8.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/semantic-conventions@1.41.1': {} + + '@opentelemetry/semantic-conventions@1.43.0': {} + + '@oxc-project/types@0.155.0': {} + + '@posthog/core@1.29.13': + dependencies: + '@posthog/types': 1.376.4 + + '@posthog/types@1.376.4': {} + + '@prisma/config@6.19.3(magicast@0.3.5)': + dependencies: + c12: 3.1.0(magicast@0.3.5) + deepmerge-ts: 7.1.5 + effect: 3.21.0 + empathic: 2.0.0 + transitivePeerDependencies: + - magicast + + '@protobuf-ts/runtime@2.11.1': {} + + '@rolldown/binding-android-arm-eabi@1.2.15': + optional: true + + '@rolldown/binding-android-arm64@1.2.15': + optional: true + + '@rolldown/binding-darwin-arm64@1.2.15': + optional: true + + '@rolldown/binding-darwin-x64@1.2.15': + optional: true + + '@rolldown/binding-freebsd-x64@1.2.15': + optional: true + + '@rolldown/binding-linux-arm-gnueabihf@1.2.15': + optional: true + + '@rolldown/binding-linux-arm64-gnu@1.2.15': + optional: true + + '@rolldown/binding-linux-arm64-musl@1.2.15': + optional: true + + '@rolldown/binding-linux-ppc64-gnu@1.2.15': + optional: true + + '@rolldown/binding-linux-s390x-gnu@1.2.15': + optional: true + + '@rolldown/binding-linux-x64-gnu@1.2.15': + optional: true + + '@rolldown/binding-linux-x64-musl@1.2.15': + optional: true + + '@rolldown/binding-openharmony-arm64@1.2.15': + optional: true + + '@rolldown/binding-win32-arm64-msvc@1.2.15': + optional: true + + '@rolldown/binding-win32-x64-msvc@1.2.15': + optional: true + + '@rolldown/pluginutils@1.0.1': {} + + '@rollup/rollup-darwin-arm64@4.65.1': + optional: true + + '@s2-dev/streamstore@0.25.0(supports-color@10.2.2)': + dependencies: + '@protobuf-ts/runtime': 2.11.1 + debug: 4.4.3(supports-color@10.2.2) + transitivePeerDependencies: + - supports-color + + '@s2-dev/streamstore@0.25.1(supports-color@10.2.2)': + dependencies: + '@protobuf-ts/runtime': 2.11.1 + debug: 4.4.3(supports-color@10.2.2) + transitivePeerDependencies: + - supports-color + + '@socket.io/component-emitter@3.1.2': {} + + '@sodaru/yup-to-json-schema@2.0.1': {} + + '@standard-schema/spec@1.1.0': {} + + '@trigger.dev/build@4.8.3(magicast@0.3.5)(supports-color@10.2.2)(typescript@5.9.3)(zod@4.5.4)': + dependencies: + '@prisma/config': 6.19.3(magicast@0.3.5) + '@trigger.dev/core': 4.8.3(supports-color@10.2.2)(zod@4.5.4) + mlly: 1.8.2 + pkg-types: 1.3.1 + tinyglobby: 0.2.17 + tsconfck: 3.1.3(typescript@5.9.3) + optionalDependencies: + typescript: 5.9.3 + transitivePeerDependencies: + - bufferutil + - magicast + - supports-color + - utf-8-validate + - zod + + '@trigger.dev/convex@4.8.3(convex@1.48.0(react@19.3.1))': + dependencies: + convex: 1.48.0(react@19.3.1) + + '@trigger.dev/core@4.8.3(supports-color@10.2.2)(zod@4.5.4)': + dependencies: + '@bugsnag/cuid': 3.2.2 + '@electric-sql/client': 1.0.14 + '@google-cloud/precise-date': 4.0.0 + '@jsonhero/path': 1.0.21 + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.219.0 + '@opentelemetry/core': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/exporter-logs-otlp-http': 0.219.0(@opentelemetry/api@1.9.1) + '@opentelemetry/exporter-metrics-otlp-http': 0.219.0(@opentelemetry/api@1.9.1) + '@opentelemetry/exporter-trace-otlp-http': 0.219.0(@opentelemetry/api@1.9.1) + '@opentelemetry/host-metrics': 0.38.3(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': 0.219.0(@opentelemetry/api@1.9.1)(supports-color@10.2.2) + '@opentelemetry/resources': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-logs': 0.219.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-metrics': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-base': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-node': 2.8.0(@opentelemetry/api@1.9.1) + '@s2-dev/streamstore': 0.25.0(supports-color@10.2.2) + dequal: 2.0.3 + eventsource: 3.0.7 + eventsource-parser: 3.1.1 + humanize-duration: 3.35.0 + jose: 5.10.0 + nanoid: 3.3.18 + prom-client: 15.1.3 + socket.io-client: 4.8.4(supports-color@10.2.2) + std-env: 3.10.0 + tinyexec: 0.3.2 + uncrypto: 0.1.3 + zod: 4.5.4 + zod-validation-error: 5.0.0(zod@4.5.4) + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + + '@trigger.dev/react-hooks@4.8.3(react-dom@19.3.1(react@19.3.1))(react@19.3.1)(zod@4.5.4)': + dependencies: + '@trigger.dev/core': 4.8.3(supports-color@10.2.2)(zod@4.5.4) + react: 19.3.1 + react-dom: 19.3.1(react@19.3.1) + swr: 2.5.1(react@19.3.1) + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + - zod + + '@trigger.dev/schema-to-json@4.8.3(supports-color@10.2.2)(zod@4.5.4)': + dependencies: + '@sodaru/yup-to-json-schema': 2.0.1 + '@trigger.dev/core': 4.8.3(supports-color@10.2.2)(zod@4.5.4) + effect: 3.22.4 + zod: 4.5.4 + zod-to-json-schema: 3.25.2(zod@4.5.4) + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + + '@trigger.dev/sdk@4.8.3(ai@7.0.140(zod@4.5.4))(react@19.3.1)(zod@4.5.4)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/semantic-conventions': 1.41.1 + '@trigger.dev/core': 4.8.3(supports-color@10.2.2)(zod@4.5.4) + uncrypto: 0.1.3 + zod: 4.5.4 + optionalDependencies: + ai: 7.0.140(zod@4.5.4) + react: 19.3.1 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + + '@types/chai@5.2.3': + dependencies: + '@types/deep-eql': 4.0.2 + assertion-error: 2.0.1 + + '@types/deep-eql@4.0.2': {} + + '@types/estree@1.0.9': {} + + '@types/node@24.13.3': + dependencies: + undici-types: 7.18.2 + + '@types/react-dom@19.2.3(@types/react@19.2.17)': + dependencies: + '@types/react': 19.2.17 + + '@types/react@19.2.17': + dependencies: + csstype: 3.2.3 + + '@types/retry@0.12.2': {} + + '@vercel/oidc@3.2.0': {} + + '@vitest/expect@4.1.11': + dependencies: + '@standard-schema/spec': 1.1.0 + '@types/chai': 5.2.3 + '@vitest/spy': 4.1.11 + '@vitest/utils': 4.1.11 + chai: 6.3.0 + tinyrainbow: 3.2.0 + + '@vitest/mocker@4.1.11(vite@8.3.5(@types/node@24.13.3)(esbuild@0.27.0)(jiti@2.7.0))': + dependencies: + '@vitest/spy': 4.1.11 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 8.3.5(@types/node@24.13.3)(esbuild@0.27.0)(jiti@2.7.0) + + '@vitest/pretty-format@4.1.11': + dependencies: + tinyrainbow: 3.2.0 + + '@vitest/runner@4.1.11': + dependencies: + '@vitest/utils': 4.1.11 + pathe: 2.0.3 + + '@vitest/snapshot@4.1.11': + dependencies: + '@vitest/pretty-format': 4.1.11 + '@vitest/utils': 4.1.11 + magic-string: 0.30.21 + pathe: 2.0.3 + + '@vitest/spy@4.1.11': {} + + '@vitest/utils@4.1.11': + dependencies: + '@vitest/pretty-format': 4.1.11 + convert-source-map: 2.0.0 + tinyrainbow: 3.2.0 + + '@workflow/serde@4.1.0': {} + + accepts@2.0.0: + dependencies: + mime-types: 3.0.2 + negotiator: 1.1.0 + + acorn-import-attributes@1.9.5(acorn@8.19.0): + dependencies: + acorn: 8.19.0 + + acorn@8.19.0: {} + + ai@7.0.140(zod@4.5.4): + dependencies: + '@ai-sdk/gateway': 4.0.113(zod@4.5.4) + '@ai-sdk/provider': 4.0.27 + '@ai-sdk/provider-utils': 5.0.59(zod@4.5.4) + zod: 4.5.4 + + ajv-formats@3.0.1(ajv@8.20.0): + optionalDependencies: + ajv: 8.20.0 + + ajv@8.20.0: + dependencies: + fast-deep-equal: 3.1.3 + fast-uri: 3.1.9 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + + ansi-escapes@7.3.0: + dependencies: + environment: 1.1.0 + + ansi-regex@5.0.1: {} + + ansi-regex@6.4.0: {} + + anymatch@3.1.3: + dependencies: + normalize-path: 3.0.0 + picomatch: 2.3.2 + + assertion-error@2.0.1: {} + + balanced-match@4.0.4: {} + + binary-extensions@2.3.0: {} + + bintrees@1.0.2: {} + + body-parser@2.4.0(supports-color@10.2.2): + dependencies: + bytes: 3.1.2 + content-type: 2.1.0 + debug: 4.4.3(supports-color@10.2.2) + http-errors: 2.0.1 + iconv-lite: 0.7.3 + on-finished: 2.4.2 + qs: 6.16.1 + raw-body: 3.0.2 + type-is: 2.1.0 + transitivePeerDependencies: + - supports-color + + brace-expansion@5.0.12: + dependencies: + balanced-match: 4.0.4 + + braces@3.0.3: + dependencies: + fill-range: 7.1.1 + + buffer-from@1.1.2: {} + + bundle-name@4.1.1: + dependencies: + run-applescript: 7.1.0 + + bytes@3.1.2: {} + + c12@1.11.2(magicast@0.3.5): + dependencies: + chokidar: 3.6.0 + confbox: 0.1.8 + defu: 6.1.7 + dotenv: 16.6.1 + giget: 1.2.5 + jiti: 1.21.7 + mlly: 1.8.2 + ohash: 1.1.6 + pathe: 1.1.2 + perfect-debounce: 1.0.0 + pkg-types: 1.3.1 + rc9: 2.1.2 + optionalDependencies: + magicast: 0.3.5 + + c12@3.1.0(magicast@0.3.5): + dependencies: + chokidar: 4.0.3 + confbox: 0.2.4 + defu: 6.1.7 + dotenv: 16.6.1 + exsolve: 1.1.3 + giget: 2.0.0 + jiti: 2.7.0 + ohash: 2.0.12 + pathe: 2.0.3 + perfect-debounce: 1.0.0 + pkg-types: 2.4.0 + rc9: 2.1.2 + optionalDependencies: + magicast: 0.3.5 + + call-bind-apply-helpers@1.0.2: + dependencies: + es-errors: 1.3.0 + function-bind: 1.1.2 + + call-bind@1.0.9: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-define-property: 1.0.1 + get-intrinsic: 1.3.0 + set-function-length: 1.2.2 + + call-bound@1.0.4: + dependencies: + call-bind-apply-helpers: 1.0.2 + get-intrinsic: 1.3.0 + + chai@6.3.0: {} + + chalk@5.6.2: {} + + chokidar@3.6.0: + dependencies: + anymatch: 3.1.3 + braces: 3.0.3 + glob-parent: 5.1.2 + is-binary-path: 2.1.0 + is-glob: 4.0.3 + normalize-path: 3.0.0 + readdirp: 3.6.0 + optionalDependencies: + fsevents: 2.3.3 + + chokidar@4.0.3: + dependencies: + readdirp: 4.1.2 + + chownr@2.0.0: {} + + chownr@3.0.0: {} + + citty@0.1.6: + dependencies: + consola: 3.4.2 + + citty@0.2.2: {} + + cjs-module-lexer@2.3.0: {} + + cli-table3@0.6.5: + dependencies: + string-width: 4.2.3 + optionalDependencies: + '@colors/colors': 1.5.0 + + commander@9.5.0: {} + + confbox@0.1.8: {} + + confbox@0.2.4: {} + + confbox@0.3.1: {} + + consola@3.4.2: {} + + content-disposition@2.0.1: {} + + content-type@1.0.5: {} + + content-type@2.1.0: {} + + convert-source-map@2.0.0: {} + + convex-test@0.0.65(convex@1.48.0(react@19.3.1)): + dependencies: + convex: 1.48.0(react@19.3.1) + + convex@1.48.0(react@19.3.1): + dependencies: + esbuild: 0.27.0 + prettier: 3.9.10 + ws: 8.21.0 + optionalDependencies: + react: 19.3.1 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + cookie-signature@1.2.2: {} + + cookie@0.7.2: {} + + cors@2.8.6: + dependencies: + object-assign: 4.1.1 + vary: 1.1.2 + + cross-spawn@7.0.6: + dependencies: + path-key: 3.1.1 + shebang-command: 2.0.0 + which: 2.0.2 + + csstype@3.2.3: {} + + debug@4.4.3(supports-color@10.2.2): + dependencies: + ms: 2.1.3 + optionalDependencies: + supports-color: 10.2.2 + + deepmerge-ts@7.1.5: {} + + default-browser-id@5.0.1: {} + + default-browser@5.5.1: + dependencies: + bundle-name: 4.1.1 + default-browser-id: 5.0.1 + + define-data-property@1.1.4: + dependencies: + es-define-property: 1.0.1 + es-errors: 1.3.0 + gopd: 1.2.0 + + define-lazy-prop@3.0.0: {} + + defu@6.1.7: {} + + depd@2.0.0: {} + + dequal@2.0.3: {} + + destr@2.0.5: {} + + detect-libc@2.1.2: {} + + dotenv@16.6.1: {} + + dunder-proto@1.0.1: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-errors: 1.3.0 + gopd: 1.2.0 + + ee-first@1.1.1: {} + + effect@3.21.0: + dependencies: + '@standard-schema/spec': 1.1.0 + fast-check: 3.23.2 + + effect@3.22.4: + dependencies: + '@standard-schema/spec': 1.1.0 + fast-check: 3.23.2 + + emoji-regex@8.0.0: {} + + empathic@2.0.0: {} + + encodeurl@2.0.0: {} + + engine.io-client@6.6.7(supports-color@10.2.2): + dependencies: + '@socket.io/component-emitter': 3.1.2 + debug: 4.4.3(supports-color@10.2.2) + engine.io-parser: 5.2.3 + ws: 8.21.3 + xmlhttprequest-ssl: 2.1.2 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + + engine.io-parser@5.2.3: {} + + environment@1.1.0: {} + + es-define-property@1.0.1: {} + + es-errors@1.3.0: {} + + es-module-lexer@2.3.2: {} + + es-object-atoms@1.1.2: + dependencies: + es-errors: 1.3.0 + + esbuild@0.23.1: + optionalDependencies: + '@esbuild/aix-ppc64': 0.23.1 + '@esbuild/android-arm': 0.23.1 + '@esbuild/android-arm64': 0.23.1 + '@esbuild/android-x64': 0.23.1 + '@esbuild/darwin-arm64': 0.23.1 + '@esbuild/darwin-x64': 0.23.1 + '@esbuild/freebsd-arm64': 0.23.1 + '@esbuild/freebsd-x64': 0.23.1 + '@esbuild/linux-arm': 0.23.1 + '@esbuild/linux-arm64': 0.23.1 + '@esbuild/linux-ia32': 0.23.1 + '@esbuild/linux-loong64': 0.23.1 + '@esbuild/linux-mips64el': 0.23.1 + '@esbuild/linux-ppc64': 0.23.1 + '@esbuild/linux-riscv64': 0.23.1 + '@esbuild/linux-s390x': 0.23.1 + '@esbuild/linux-x64': 0.23.1 + '@esbuild/netbsd-x64': 0.23.1 + '@esbuild/openbsd-arm64': 0.23.1 + '@esbuild/openbsd-x64': 0.23.1 + '@esbuild/sunos-x64': 0.23.1 + '@esbuild/win32-arm64': 0.23.1 + '@esbuild/win32-ia32': 0.23.1 + '@esbuild/win32-x64': 0.23.1 + + esbuild@0.27.0: + optionalDependencies: + '@esbuild/aix-ppc64': 0.27.0 + '@esbuild/android-arm': 0.27.0 + '@esbuild/android-arm64': 0.27.0 + '@esbuild/android-x64': 0.27.0 + '@esbuild/darwin-arm64': 0.27.0 + '@esbuild/darwin-x64': 0.27.0 + '@esbuild/freebsd-arm64': 0.27.0 + '@esbuild/freebsd-x64': 0.27.0 + '@esbuild/linux-arm': 0.27.0 + '@esbuild/linux-arm64': 0.27.0 + '@esbuild/linux-ia32': 0.27.0 + '@esbuild/linux-loong64': 0.27.0 + '@esbuild/linux-mips64el': 0.27.0 + '@esbuild/linux-ppc64': 0.27.0 + '@esbuild/linux-riscv64': 0.27.0 + '@esbuild/linux-s390x': 0.27.0 + '@esbuild/linux-x64': 0.27.0 + '@esbuild/netbsd-arm64': 0.27.0 + '@esbuild/netbsd-x64': 0.27.0 + '@esbuild/openbsd-arm64': 0.27.0 + '@esbuild/openbsd-x64': 0.27.0 + '@esbuild/openharmony-arm64': 0.27.0 + '@esbuild/sunos-x64': 0.27.0 + '@esbuild/win32-arm64': 0.27.0 + '@esbuild/win32-ia32': 0.27.0 + '@esbuild/win32-x64': 0.27.0 + + escape-html@1.0.3: {} + + estree-walker@3.0.3: + dependencies: + '@types/estree': 1.0.9 + + etag@1.8.1: {} + + eventsource-parser@3.1.1: {} + + eventsource@3.0.7: + dependencies: + eventsource-parser: 3.1.1 + + evt@2.5.9: + dependencies: + minimal-polyfills: 2.2.3 + run-exclusive: 2.2.19 + tsafe: 1.8.12 + + expect-type@1.4.0: {} + + express-rate-limit@8.7.1(express@5.3.0(supports-color@10.2.2))(supports-color@10.2.2): + dependencies: + debug: 4.4.3(supports-color@10.2.2) + express: 5.3.0(supports-color@10.2.2) + ip-address: 10.7.5 + transitivePeerDependencies: + - supports-color + + express@5.3.0(supports-color@10.2.2): + dependencies: + accepts: 2.0.0 + body-parser: 2.4.0(supports-color@10.2.2) + content-disposition: 2.0.1 + content-type: 2.1.0 + cookie: 0.7.2 + cookie-signature: 1.2.2 + debug: 4.4.3(supports-color@10.2.2) + depd: 2.0.0 + encodeurl: 2.0.0 + escape-html: 1.0.3 + etag: 1.8.1 + finalhandler: 2.1.1(supports-color@10.2.2) + fresh: 2.0.0 + http-errors: 2.0.1 + merge-descriptors: 2.0.0 + mime-types: 3.0.2 + on-finished: 2.4.2 + once: 1.4.0 + parseurl: 1.3.3 + proxy-addr: 2.0.8 + qs: 6.16.1 + range-parser: 1.3.0 + router: 2.2.0(supports-color@10.2.2) + send: 1.2.1(supports-color@10.2.2) + serve-static: 2.2.1(supports-color@10.2.2) + statuses: 2.0.2 + type-is: 2.1.0 + vary: 1.1.2 + transitivePeerDependencies: + - supports-color + + exsolve@1.1.3: {} + + fast-check@3.23.2: + dependencies: + pure-rand: 6.1.0 + + fast-deep-equal@3.1.3: {} + + fast-npm-meta@0.2.2: {} + + fast-uri@3.1.9: {} + + fdir@6.5.0(picomatch@4.0.7): + optionalDependencies: + picomatch: 4.0.7 + + fill-range@7.1.1: + dependencies: + to-regex-range: 5.0.1 + + finalhandler@2.1.1(supports-color@10.2.2): + dependencies: + debug: 4.4.3(supports-color@10.2.2) + encodeurl: 2.0.0 + escape-html: 1.0.3 + on-finished: 2.4.2 + parseurl: 1.3.3 + statuses: 2.0.2 + transitivePeerDependencies: + - supports-color + + forwarded@0.2.0: {} + + fresh@2.0.0: {} + + fs-minipass@2.1.0: + dependencies: + minipass: 3.3.6 + + fsevents@2.3.3: + optional: true + + function-bind@1.1.2: {} + + get-intrinsic@1.3.0: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-define-property: 1.0.1 + es-errors: 1.3.0 + es-object-atoms: 1.1.2 + function-bind: 1.1.2 + get-proto: 1.0.1 + gopd: 1.2.0 + has-symbols: 1.1.0 + hasown: 2.0.4 + math-intrinsics: 1.1.0 + + get-proto@1.0.1: + dependencies: + dunder-proto: 1.0.1 + es-object-atoms: 1.1.2 + + giget@1.2.5: + dependencies: + citty: 0.1.6 + consola: 3.4.2 + defu: 6.1.7 + node-fetch-native: 1.6.7 + nypm: 0.5.4 + pathe: 2.0.3 + tar: 6.2.1 + + giget@2.0.0: + dependencies: + citty: 0.1.6 + consola: 3.4.2 + defu: 6.1.7 + node-fetch-native: 1.6.7 + nypm: 0.6.12 + pathe: 2.0.3 + + git-last-commit@1.0.1: {} + + glob-parent@5.1.2: + dependencies: + is-glob: 4.0.3 + + gopd@1.2.0: {} + + has-flag@5.0.1: {} + + has-property-descriptors@1.0.2: + dependencies: + es-define-property: 1.0.1 + + has-symbols@1.1.0: {} + + hasown@2.0.4: + dependencies: + function-bind: 1.1.2 + + hono@4.13.14: {} + + http-errors@2.0.1: + dependencies: + depd: 2.0.0 + inherits: 2.0.4 + setprototypeof: 1.2.0 + statuses: 2.0.2 + toidentifier: 1.0.1 + + humanize-duration@3.35.0: {} + + iconv-lite@0.7.3: + dependencies: + safer-buffer: 2.1.2 + + ignore@7.0.14: {} + + import-in-the-middle@3.0.1: + dependencies: + acorn: 8.19.0 + acorn-import-attributes: 1.9.5(acorn@8.19.0) + cjs-module-lexer: 2.3.0 + module-details-from-path: 1.0.4 + + inherits@2.0.4: {} + + ini@5.0.0: {} + + ip-address@10.7.5: {} + + ipaddr.js@1.9.1: {} + + is-binary-path@2.1.0: + dependencies: + binary-extensions: 2.3.0 + + is-core-module@2.17.0: + dependencies: + hasown: 2.0.4 + + is-docker@3.0.0: {} + + is-extglob@2.1.1: {} + + is-fullwidth-code-point@3.0.0: {} + + is-glob@4.0.3: + dependencies: + is-extglob: 2.1.1 + + is-inside-container@1.0.0: + dependencies: + is-docker: 3.0.0 + + is-network-error@1.3.2: {} + + is-number@7.0.0: {} + + is-promise@4.0.0: {} + + is-wsl@3.1.1: + dependencies: + is-inside-container: 1.0.0 + + isarray@2.0.5: {} + + isexe@2.0.0: {} + + jiti@1.21.7: {} + + jiti@2.7.0: {} + + jose@5.10.0: {} + + jose@6.2.13: {} + + json-schema-traverse@1.0.0: {} + + json-schema-typed@8.0.2: {} + + json-schema@0.4.0: {} + + json-stable-stringify@1.3.0: + dependencies: + call-bind: 1.0.9 + call-bound: 1.0.4 + isarray: 2.0.5 + jsonify: 0.0.1 + object-keys: 1.1.1 + + jsonc-parser@3.2.1: {} + + jsonify@0.0.1: {} + + lightningcss-android-arm64@1.33.0: + optional: true + + lightningcss-darwin-arm64@1.33.0: + optional: true + + lightningcss-darwin-x64@1.33.0: + optional: true + + lightningcss-freebsd-x64@1.33.0: + optional: true + + lightningcss-linux-arm-gnueabihf@1.33.0: + optional: true + + lightningcss-linux-arm64-gnu@1.33.0: + optional: true + + lightningcss-linux-arm64-musl@1.33.0: + optional: true + + lightningcss-linux-x64-gnu@1.33.0: + optional: true + + lightningcss-linux-x64-musl@1.33.0: + optional: true + + lightningcss-win32-arm64-msvc@1.33.0: + optional: true + + lightningcss-win32-x64-msvc@1.33.0: + optional: true + + lightningcss@1.33.0: + dependencies: + detect-libc: 2.1.2 + optionalDependencies: + lightningcss-android-arm64: 1.33.0 + lightningcss-darwin-arm64: 1.33.0 + lightningcss-darwin-x64: 1.33.0 + lightningcss-freebsd-x64: 1.33.0 + lightningcss-linux-arm-gnueabihf: 1.33.0 + lightningcss-linux-arm64-gnu: 1.33.0 + lightningcss-linux-arm64-musl: 1.33.0 + lightningcss-linux-x64-gnu: 1.33.0 + lightningcss-linux-x64-musl: 1.33.0 + lightningcss-win32-arm64-msvc: 1.33.0 + lightningcss-win32-x64-msvc: 1.33.0 + + magic-string@0.30.21: + dependencies: + '@jridgewell/sourcemap-codec': 1.6.0 + + magicast@0.3.5: + dependencies: + '@babel/parser': 7.29.11 + '@babel/types': 7.29.11 + source-map-js: 1.2.2 + + math-intrinsics@1.1.0: {} + + media-typer@1.1.1: {} + + merge-descriptors@2.0.0: {} + + mime-db@1.54.0: {} + + mime-types@3.0.2: + dependencies: + mime-db: 1.54.0 + + minimal-polyfills@2.2.3: {} + + minimatch@10.2.6: + dependencies: + brace-expansion: 5.0.12 + + minipass@3.3.6: + dependencies: + yallist: 4.0.0 + + minipass@5.0.0: {} + + minipass@7.1.3: {} + + minizlib@2.1.2: + dependencies: + minipass: 3.3.6 + yallist: 4.0.0 + + minizlib@3.1.0: + dependencies: + minipass: 7.1.3 + + mkdirp@1.0.4: {} + + mlly@1.8.2: + dependencies: + acorn: 8.19.0 + pathe: 2.0.3 + pkg-types: 1.3.1 + ufo: 1.6.4 + + module-details-from-path@1.0.4: {} + + ms@2.1.3: {} + + nanoid@3.3.18: {} + + nanoid@3.3.20: {} + + negotiator@1.1.0: + dependencies: + content-type: 2.1.0 + + node-fetch-native@1.6.7: {} + + normalize-path@3.0.0: {} + + nypm@0.5.4: + dependencies: + citty: 0.1.6 + consola: 3.4.2 + pathe: 2.0.3 + pkg-types: 1.3.1 + tinyexec: 0.3.2 + ufo: 1.6.4 + + nypm@0.6.12: + dependencies: + citty: 0.2.2 + pathe: 2.0.3 + tinyexec: 1.3.1 + + object-assign@4.1.1: {} + + object-inspect@1.13.4: {} + + object-keys@1.1.1: {} + + obug@2.2.1: {} + + ohash@1.1.6: {} + + ohash@2.0.12: {} + + on-finished@2.4.2: + dependencies: + ee-first: 1.1.1 + + once@1.4.0: + dependencies: + wrappy: 1.0.2 + + open@10.2.0: + dependencies: + default-browser: 5.5.1 + define-lazy-prop: 3.0.0 + is-inside-container: 1.0.0 + wsl-utils: 0.1.0 + + os-paths@7.4.0: + optionalDependencies: + fsevents: 2.3.3 + + p-limit@6.2.0: + dependencies: + yocto-queue: 1.2.2 + + p-retry@6.2.1: + dependencies: + '@types/retry': 0.12.2 + is-network-error: 1.3.2 + retry: 0.13.1 + + parseurl@1.3.3: {} + + path-key@3.1.1: {} + + path-parse@1.0.7: {} + + path-to-regexp@8.4.2: {} + + pathe@1.1.2: {} + + pathe@2.0.3: {} + + perfect-debounce@1.0.0: {} + + picocolors@1.1.1: {} + + picomatch@2.3.2: {} + + picomatch@4.0.7: {} + + pkce-challenge@5.0.1: {} + + pkg-types@1.3.1: + dependencies: + confbox: 0.1.8 + mlly: 1.8.2 + pathe: 2.0.3 + + pkg-types@2.4.0: + dependencies: + confbox: 0.3.1 + exsolve: 1.1.3 + pathe: 2.0.3 + + postcss@8.5.31: + dependencies: + nanoid: 3.3.20 + picocolors: 1.1.1 + source-map-js: 1.2.2 + + posthog-node@5.35.6: + dependencies: + '@posthog/core': 1.29.13 + + prettier@3.9.10: {} + + prom-client@15.1.3: + dependencies: + '@opentelemetry/api': 1.9.1 + tdigest: 0.1.3 + + proxy-addr@2.0.8: + dependencies: + forwarded: 0.2.0 + ipaddr.js: 1.9.1 + + pure-rand@6.1.0: {} + + qs@6.16.1: + dependencies: + es-define-property: 1.0.1 + side-channel: 1.1.1 + + range-parser@1.3.0: {} + + raw-body@3.0.2: + dependencies: + bytes: 3.1.2 + http-errors: 2.0.1 + iconv-lite: 0.7.3 + unpipe: 1.0.0 + + rc9@2.1.2: + dependencies: + defu: 6.1.7 + destr: 2.0.5 + + react-dom@19.3.1(react@19.3.1): + dependencies: + react: 19.3.1 + scheduler: 0.28.0 + + react@19.3.1: {} + + readdirp@3.6.0: + dependencies: + picomatch: 2.3.2 + + readdirp@4.1.2: {} + + require-from-string@2.0.2: {} + + require-in-the-middle@8.0.1(supports-color@10.2.2): + dependencies: + debug: 4.4.3(supports-color@10.2.2) + module-details-from-path: 1.0.4 + transitivePeerDependencies: + - supports-color + + resolve@1.22.13: + dependencies: + es-errors: 1.3.0 + is-core-module: 2.17.0 + path-parse: 1.0.7 + supports-preserve-symlinks-flag: 1.0.0 + + retry@0.13.1: {} + + rolldown@1.2.15: + dependencies: + '@oxc-project/types': 0.155.0 + '@rolldown/pluginutils': 1.0.1 + optionalDependencies: + '@rolldown/binding-android-arm-eabi': 1.2.15 + '@rolldown/binding-android-arm64': 1.2.15 + '@rolldown/binding-darwin-arm64': 1.2.15 + '@rolldown/binding-darwin-x64': 1.2.15 + '@rolldown/binding-freebsd-x64': 1.2.15 + '@rolldown/binding-linux-arm-gnueabihf': 1.2.15 + '@rolldown/binding-linux-arm64-gnu': 1.2.15 + '@rolldown/binding-linux-arm64-musl': 1.2.15 + '@rolldown/binding-linux-ppc64-gnu': 1.2.15 + '@rolldown/binding-linux-s390x-gnu': 1.2.15 + '@rolldown/binding-linux-x64-gnu': 1.2.15 + '@rolldown/binding-linux-x64-musl': 1.2.15 + '@rolldown/binding-openharmony-arm64': 1.2.15 + '@rolldown/binding-win32-arm64-msvc': 1.2.15 + '@rolldown/binding-win32-x64-msvc': 1.2.15 + + router@2.2.0(supports-color@10.2.2): + dependencies: + debug: 4.4.3(supports-color@10.2.2) + depd: 2.0.0 + is-promise: 4.0.0 + parseurl: 1.3.3 + path-to-regexp: 8.4.2 + transitivePeerDependencies: + - supports-color + + run-applescript@7.1.0: {} + + run-exclusive@2.2.19: + dependencies: + minimal-polyfills: 2.2.3 + + safer-buffer@2.1.2: {} + + scheduler@0.28.0: {} + + semver@7.8.5: {} + + send@1.2.1(supports-color@10.2.2): + dependencies: + debug: 4.4.3(supports-color@10.2.2) + encodeurl: 2.0.0 + escape-html: 1.0.3 + etag: 1.8.1 + fresh: 2.0.0 + http-errors: 2.0.1 + mime-types: 3.0.2 + ms: 2.1.3 + on-finished: 2.4.2 + range-parser: 1.3.0 + statuses: 2.0.2 + transitivePeerDependencies: + - supports-color + + serve-static@2.2.1(supports-color@10.2.2): + dependencies: + encodeurl: 2.0.0 + escape-html: 1.0.3 + parseurl: 1.3.3 + send: 1.2.1(supports-color@10.2.2) + transitivePeerDependencies: + - supports-color + + set-function-length@1.2.2: + dependencies: + define-data-property: 1.1.4 + es-errors: 1.3.0 + function-bind: 1.1.2 + get-intrinsic: 1.3.0 + gopd: 1.2.0 + has-property-descriptors: 1.0.2 + + setprototypeof@1.2.0: {} + + sharp@0.35.5(@types/node@24.13.3): + dependencies: + '@img/colour': 1.1.0 + detect-libc: 2.1.2 + semver: 7.8.5 + optionalDependencies: + '@img/sharp-darwin-arm64': 0.35.5 + '@img/sharp-darwin-x64': 0.35.5 + '@img/sharp-freebsd-wasm32': 0.35.5 + '@img/sharp-libvips-darwin-arm64': 1.3.4 + '@img/sharp-libvips-darwin-x64': 1.3.4 + '@img/sharp-libvips-linux-arm': 1.3.4 + '@img/sharp-libvips-linux-arm64': 1.3.4 + '@img/sharp-libvips-linux-ppc64': 1.3.4 + '@img/sharp-libvips-linux-riscv64': 1.3.4 + '@img/sharp-libvips-linux-s390x': 1.3.4 + '@img/sharp-libvips-linux-x64': 1.3.4 + '@img/sharp-libvips-linuxmusl-arm64': 1.3.4 + '@img/sharp-libvips-linuxmusl-x64': 1.3.4 + '@img/sharp-linux-arm': 0.35.5 + '@img/sharp-linux-arm64': 0.35.5 + '@img/sharp-linux-ppc64': 0.35.5 + '@img/sharp-linux-riscv64': 0.35.5 + '@img/sharp-linux-s390x': 0.35.5 + '@img/sharp-linux-x64': 0.35.5 + '@img/sharp-linuxmusl-arm64': 0.35.5 + '@img/sharp-linuxmusl-x64': 0.35.5 + '@img/sharp-webcontainers-wasm32': 0.35.5 + '@img/sharp-win32-arm64': 0.35.5 + '@img/sharp-win32-ia32': 0.35.5 + '@img/sharp-win32-x64': 0.35.5 + '@types/node': 24.13.3 + + shebang-command@2.0.0: + dependencies: + shebang-regex: 3.0.0 + + shebang-regex@3.0.0: {} + + side-channel-list@1.0.1: + dependencies: + es-errors: 1.3.0 + object-inspect: 1.13.4 + + side-channel-map@1.0.1: + dependencies: + call-bound: 1.0.4 + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + object-inspect: 1.13.4 + + side-channel-weakmap@1.0.2: + dependencies: + call-bound: 1.0.4 + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + object-inspect: 1.13.4 + side-channel-map: 1.0.1 + + side-channel@1.1.1: + dependencies: + es-errors: 1.3.0 + object-inspect: 1.13.4 + side-channel-list: 1.0.1 + side-channel-map: 1.0.1 + side-channel-weakmap: 1.0.2 + + siginfo@2.0.0: {} + + signal-exit@4.1.0: {} + + sisteransi@1.0.5: {} + + socket.io-client@4.8.4(supports-color@10.2.2): + dependencies: + '@socket.io/component-emitter': 3.1.2 + debug: 4.4.3(supports-color@10.2.2) + engine.io-client: 6.6.7(supports-color@10.2.2) + socket.io-parser: 4.2.7(supports-color@10.2.2) + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + + socket.io-parser@4.2.7(supports-color@10.2.2): + dependencies: + '@socket.io/component-emitter': 3.1.2 + debug: 4.4.3(supports-color@10.2.2) + transitivePeerDependencies: + - supports-color + + source-map-js@1.2.2: {} + + source-map-support@0.5.21: + dependencies: + buffer-from: 1.1.2 + source-map: 0.6.1 + + source-map@0.6.1: {} + + stackback@0.0.2: {} + + statuses@2.0.2: {} + + std-env@3.10.0: {} + + std-env@4.3.0: {} + + string-width@4.2.3: + dependencies: + emoji-regex: 8.0.0 + is-fullwidth-code-point: 3.0.0 + strip-ansi: 6.0.1 + + strip-ansi@6.0.1: + dependencies: + ansi-regex: 5.0.1 + + strip-ansi@7.2.0: + dependencies: + ansi-regex: 6.4.0 + + supports-color@10.2.2: {} + + supports-preserve-symlinks-flag@1.0.0: {} + + swr@2.5.1(react@19.3.1): + dependencies: + dequal: 2.0.3 + react: 19.3.1 + use-sync-external-store: 1.7.0(react@19.3.1) + + systeminformation@5.33.16: {} + + tar@6.2.1: + dependencies: + chownr: 2.0.0 + fs-minipass: 2.1.0 + minipass: 5.0.0 + minizlib: 2.1.2 + mkdirp: 1.0.4 + yallist: 4.0.0 + + tar@7.5.22: + dependencies: + '@isaacs/fs-minipass': 4.0.1 + chownr: 3.0.0 + minipass: 7.1.3 + minizlib: 3.1.0 + yallist: 5.0.0 + + tdigest@0.1.3: + dependencies: + bintrees: 1.0.2 + + tinybench@2.9.0: {} + + tinyexec@0.3.2: {} + + tinyexec@1.3.1: {} + + tinyglobby@0.2.17: + dependencies: + fdir: 6.5.0(picomatch@4.0.7) + picomatch: 4.0.7 + + tinyrainbow@3.2.0: {} + + to-regex-range@5.0.1: + dependencies: + is-number: 7.0.0 + + toidentifier@1.0.1: {} + + trigger.dev@4.8.3(typescript@5.9.3): + dependencies: + '@babel/parser': 7.29.11 + '@clack/prompts': 0.11.0 + '@depot/cli': 0.0.1-cli.2.80.0 + '@modelcontextprotocol/sdk': 1.32.2(supports-color@10.2.2)(zod@4.5.4) + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.219.0 + '@opentelemetry/instrumentation': 0.219.0(@opentelemetry/api@1.9.1)(supports-color@10.2.2) + '@s2-dev/streamstore': 0.25.1(supports-color@10.2.2) + '@trigger.dev/build': 4.8.3(magicast@0.3.5)(supports-color@10.2.2)(typescript@5.9.3)(zod@4.5.4) + '@trigger.dev/core': 4.8.3(supports-color@10.2.2)(zod@4.5.4) + '@trigger.dev/schema-to-json': 4.8.3(supports-color@10.2.2)(zod@4.5.4) + ansi-escapes: 7.3.0 + braces: 3.0.3 + c12: 1.11.2(magicast@0.3.5) + chalk: 5.6.2 + chokidar: 3.6.0 + cli-table3: 0.6.5 + commander: 9.5.0 + confbox: 0.2.4 + defu: 6.1.7 + dotenv: 16.6.1 + esbuild: 0.23.1 + eventsource: 3.0.7 + evt: 2.5.9 + fast-npm-meta: 0.2.2 + git-last-commit: 1.0.1 + has-flag: 5.0.1 + ignore: 7.0.14 + import-in-the-middle: 3.0.1 + ini: 5.0.0 + json-stable-stringify: 1.3.0 + jsonc-parser: 3.2.1 + magicast: 0.3.5 + minimatch: 10.2.6 + mlly: 1.8.2 + nypm: 0.5.4 + open: 10.2.0 + p-limit: 6.2.0 + p-retry: 6.2.1 + pkg-types: 1.3.1 + posthog-node: 5.35.6 + resolve: 1.22.13 + semver: 7.8.5 + signal-exit: 4.1.0 + socket.io-client: 4.8.4(supports-color@10.2.2) + source-map-support: 0.5.21 + std-env: 3.10.0 + strip-ansi: 7.2.0 + supports-color: 10.2.2 + tar: 7.5.22 + tinyexec: 0.3.2 + tinyglobby: 0.2.17 + xdg-app-paths: 8.3.0 + zod: 4.5.4 + zod-validation-error: 5.0.0(zod@4.5.4) + transitivePeerDependencies: + - '@cfworker/json-schema' + - '@sinclair/typebox' + - '@typescript/typescript6' + - arktype + - bufferutil + - runtypes + - rxjs + - superstruct + - typescript + - utf-8-validate + - valibot + + tsafe@1.8.12: {} + + tsconfck@3.1.3(typescript@5.9.3): + optionalDependencies: + typescript: 5.9.3 + + tslib@2.8.1: + optional: true + + type-is@2.1.0: + dependencies: + content-type: 2.1.0 + media-typer: 1.1.1 + mime-types: 3.0.2 + + typescript@5.9.3: {} + + ufo@1.6.4: {} + + uncrypto@0.1.3: {} + + undici-types@7.18.2: {} + + undici@7.30.1: {} + + unpipe@1.0.0: {} + + use-sync-external-store@1.7.0(react@19.3.1): + dependencies: + react: 19.3.1 + + vary@1.1.2: {} + + vite@8.3.5(@types/node@24.13.3)(esbuild@0.27.0)(jiti@2.7.0): + dependencies: + lightningcss: 1.33.0 + picomatch: 4.0.7 + postcss: 8.5.31 + rolldown: 1.2.15 + tinyglobby: 0.2.17 + optionalDependencies: + '@types/node': 24.13.3 + esbuild: 0.27.0 + fsevents: 2.3.3 + jiti: 2.7.0 + + vitest@4.1.11(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(vite@8.3.5(@types/node@24.13.3)(esbuild@0.27.0)(jiti@2.7.0)): + dependencies: + '@vitest/expect': 4.1.11 + '@vitest/mocker': 4.1.11(vite@8.3.5(@types/node@24.13.3)(esbuild@0.27.0)(jiti@2.7.0)) + '@vitest/pretty-format': 4.1.11 + '@vitest/runner': 4.1.11 + '@vitest/snapshot': 4.1.11 + '@vitest/spy': 4.1.11 + '@vitest/utils': 4.1.11 + es-module-lexer: 2.3.2 + expect-type: 1.4.0 + magic-string: 0.30.21 + obug: 2.2.1 + pathe: 2.0.3 + picomatch: 4.0.7 + std-env: 4.3.0 + tinybench: 2.9.0 + tinyexec: 1.3.1 + tinyglobby: 0.2.17 + tinyrainbow: 3.2.0 + vite: 8.3.5(@types/node@24.13.3)(esbuild@0.27.0)(jiti@2.7.0) + why-is-node-running: 2.3.0 + optionalDependencies: + '@opentelemetry/api': 1.9.1 + '@types/node': 24.13.3 + transitivePeerDependencies: + - msw + + which@2.0.2: + dependencies: + isexe: 2.0.0 + + why-is-node-running@2.3.0: + dependencies: + siginfo: 2.0.0 + stackback: 0.0.2 + + wrappy@1.0.2: {} + + ws@8.21.0: {} + + ws@8.21.3: {} + + wsl-utils@0.1.0: + dependencies: + is-wsl: 3.1.1 + + xdg-app-paths@8.3.0: + dependencies: + xdg-portable: 10.6.0 + optionalDependencies: + fsevents: 2.3.3 + + xdg-portable@10.6.0: + dependencies: + os-paths: 7.4.0 + optionalDependencies: + fsevents: 2.3.3 + + xmlhttprequest-ssl@2.1.2: {} + + yallist@4.0.0: {} + + yallist@5.0.0: {} + + yocto-queue@1.2.2: {} + + zod-to-json-schema@3.25.2(zod@4.5.4): + dependencies: + zod: 4.5.4 + + zod-validation-error@5.0.0(zod@4.5.4): + dependencies: + zod: 4.5.4 + + zod@4.5.4: {} diff --git a/packages/convex/example/pnpm-workspace.yaml b/packages/convex/example/pnpm-workspace.yaml new file mode 100644 index 00000000000..2cd22800e79 --- /dev/null +++ b/packages/convex/example/pnpm-workspace.yaml @@ -0,0 +1,7 @@ +packages: + - "." + +minimumReleaseAge: 4320 +minimumReleaseAgeExclude: + - trigger.dev + - "@trigger.dev/*" diff --git a/packages/convex/example/public/samples/architecture.jpg b/packages/convex/example/public/samples/architecture.jpg new file mode 100644 index 00000000000..df4dbb223d6 Binary files /dev/null and b/packages/convex/example/public/samples/architecture.jpg differ diff --git a/packages/convex/example/public/samples/canyon.jpg b/packages/convex/example/public/samples/canyon.jpg new file mode 100644 index 00000000000..392c0f686cb Binary files /dev/null and b/packages/convex/example/public/samples/canyon.jpg differ diff --git a/packages/convex/example/public/samples/city.jpg b/packages/convex/example/public/samples/city.jpg new file mode 100644 index 00000000000..83d5c68cb5a Binary files /dev/null and b/packages/convex/example/public/samples/city.jpg differ diff --git a/packages/convex/example/public/samples/coast.jpg b/packages/convex/example/public/samples/coast.jpg new file mode 100644 index 00000000000..09166991d46 Binary files /dev/null and b/packages/convex/example/public/samples/coast.jpg differ diff --git a/packages/convex/example/public/samples/desert.jpg b/packages/convex/example/public/samples/desert.jpg new file mode 100644 index 00000000000..4043a6cd876 Binary files /dev/null and b/packages/convex/example/public/samples/desert.jpg differ diff --git a/packages/convex/example/public/samples/dog.jpg b/packages/convex/example/public/samples/dog.jpg new file mode 100644 index 00000000000..6792c82e081 Binary files /dev/null and b/packages/convex/example/public/samples/dog.jpg differ diff --git a/packages/convex/example/public/samples/flowers.jpg b/packages/convex/example/public/samples/flowers.jpg new file mode 100644 index 00000000000..080869bfca7 Binary files /dev/null and b/packages/convex/example/public/samples/flowers.jpg differ diff --git a/packages/convex/example/public/samples/forest.jpg b/packages/convex/example/public/samples/forest.jpg new file mode 100644 index 00000000000..45da95184c4 Binary files /dev/null and b/packages/convex/example/public/samples/forest.jpg differ diff --git a/packages/convex/example/public/samples/lake.jpg b/packages/convex/example/public/samples/lake.jpg new file mode 100644 index 00000000000..f9db4b75817 Binary files /dev/null and b/packages/convex/example/public/samples/lake.jpg differ diff --git a/packages/convex/example/public/samples/mountains.jpg b/packages/convex/example/public/samples/mountains.jpg new file mode 100644 index 00000000000..793f55c4034 Binary files /dev/null and b/packages/convex/example/public/samples/mountains.jpg differ diff --git a/packages/convex/example/scripts/benchmark.ts b/packages/convex/example/scripts/benchmark.ts new file mode 100644 index 00000000000..8e66e4ae6b4 --- /dev/null +++ b/packages/convex/example/scripts/benchmark.ts @@ -0,0 +1,16 @@ +import { createImageSet } from "../trigger/processor.ts"; + +const output = await createImageSet("forest", "balanced", async (stage, progress) => { + console.log(`${progress}% ${stage}`); +}); +console.log( + JSON.stringify( + { + inputBytes: output.inputBytes, + processingMs: output.processingMs, + variants: output.variants.map(({ data, ...variant }) => variant), + }, + null, + 2 + ) +); diff --git a/packages/convex/example/scripts/cloud.ts b/packages/convex/example/scripts/cloud.ts new file mode 100644 index 00000000000..31371edc28d --- /dev/null +++ b/packages/convex/example/scripts/cloud.ts @@ -0,0 +1,69 @@ +import { spawnSync } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import { dirname, join } from "node:path"; +import { createRequire } from "node:module"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const require = createRequire(import.meta.url); +const cliRoot = dirname(require.resolve("trigger.dev/package.json")); +const { getPatApiClient } = await import( + pathToFileURL(join(cliRoot, "dist/esm/commands/projects/common.js")).href +); +const root = dirname(dirname(fileURLToPath(import.meta.url))); +const client = await getPatApiClient({ profile: "test" }); +if (new URL(client.apiURL).hostname !== "test-cloud.trigger.dev") + throw new Error("Refusing non-test cloud"); +const environment = await client.getProjectEnv({ + projectRef: "proj_wcwcgcodbajkuocrovpq", + env: "prod", +}); +if (!environment.success) throw new Error(environment.error); +const { apiUrl, apiKey } = environment.data; +if (new URL(apiUrl).hostname !== "test-api.trigger.dev") throw new Error("Refusing non-test API"); + +if (process.argv[2] === "configure") { + const { CliApiClient } = await import(pathToFileURL(join(cliRoot, "dist/esm/apiClient.js")).href); + const project = new CliApiClient(apiUrl, apiKey); + const variables = await project.getEnvironmentVariables("proj_wcwcgcodbajkuocrovpq"); + if (!variables.success) + throw new Error("Could not inspect artifact authentication configuration"); + let artifactSecret = variables.data.variables.DEMO_ARTIFACT_SECRET; + if (artifactSecret !== undefined && !/^[a-f0-9]{64}$/.test(artifactSecret)) { + throw new Error("Existing artifact secret could not be reused; refusing to rotate it"); + } + if (!artifactSecret) { + artifactSecret = randomBytes(32).toString("hex"); + const imported = await project.importEnvVars("proj_wcwcgcodbajkuocrovpq", "prod", { + variables: { DEMO_ARTIFACT_SECRET: artifactSecret }, + override: true, + isSecret: true, + }); + if (!imported.success) throw new Error("Could not configure Trigger artifact authentication"); + console.log("Configured Trigger artifact authentication"); + } + for (const [name, value] of Object.entries({ + DEMO_ARTIFACT_SECRET: artifactSecret, + TRIGGER_SECRET_KEY: apiKey, + TRIGGER_API_URL: apiUrl, + DEMO_ENABLED: process.argv.includes("--enable") ? "true" : "false", + })) { + const child = spawnSync(`${root}/node_modules/.bin/convex`, ["env", "set", name], { + cwd: root, + input: value, + encoding: "utf8", + stdio: ["pipe", "pipe", "pipe"], + }); + if (child.status !== 0) + throw new Error(`Failed to set ${name}; check Convex login and deployment selection`); + console.log(`Configured ${name}`); + } +} else if (process.argv[2] === "status") { + const response = await fetch(`${apiUrl}/api/v1/deployments/current`, { + headers: { Authorization: `Bearer ${apiKey}` }, + }); + if (!response.ok) throw new Error(`Deployment lookup returned HTTP ${response.status}`); + const { status, shortCode, version } = await response.json(); + console.log(JSON.stringify({ status, shortCode, version })); +} else { + throw new Error("Use configure [--enable] or status"); +} diff --git a/packages/convex/example/scripts/download-samples.ts b/packages/convex/example/scripts/download-samples.ts new file mode 100644 index 00000000000..4b6ebbc7f4c --- /dev/null +++ b/packages/convex/example/scripts/download-samples.ts @@ -0,0 +1,36 @@ +import { mkdir, writeFile, readFile } from "node:fs/promises"; +import sharp from "sharp"; +import { SAMPLES } from "../shared/catalog.ts"; + +const MAX_SAMPLE_EDGE = 1600; +const resizeOnly = process.argv.includes("--resize-only"); +await mkdir(new URL("../public/samples/", import.meta.url), { recursive: true }); +for (const sample of SAMPLES) { + const file = new URL(`../public/samples/${sample.id}.jpg`, import.meta.url); + let bytes: Uint8Array; + if (resizeOnly) { + bytes = await readFile(file); + const metadata = await sharp(bytes).metadata(); + if (metadata.width! <= MAX_SAMPLE_EDGE && metadata.height! <= MAX_SAMPLE_EDGE) continue; + } else { + const response = await fetch(`${sample.source}?fit=max&w=1600&h=1600&q=85&fm=jpg`, { + signal: AbortSignal.timeout(30_000), + }); + if (!response.ok || !response.headers.get("content-type")?.startsWith("image/")) + throw new Error(`Could not fetch ${sample.id}`); + bytes = new Uint8Array(await response.arrayBuffer()); + } + if (bytes.byteLength > 5 * 1024 * 1024) throw new Error("Sample is too large"); + const normalized = await sharp(bytes, { limitInputPixels: 12_000_000 }) + .rotate() + .resize({ + width: MAX_SAMPLE_EDGE, + height: MAX_SAMPLE_EDGE, + fit: "inside", + withoutEnlargement: true, + }) + .jpeg({ quality: 85 }) + .toBuffer(); + await writeFile(file, normalized); + console.log(`Saved ${sample.id}: ${normalized.byteLength} bytes`); +} diff --git a/packages/convex/example/scripts/verify-hosting.ts b/packages/convex/example/scripts/verify-hosting.ts new file mode 100644 index 00000000000..cc70e0959eb --- /dev/null +++ b/packages/convex/example/scripts/verify-hosting.ts @@ -0,0 +1,41 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; + +const env = readFileSync(new URL("../.env.local", import.meta.url), "utf8"); +const site = env.match(/^VITE_CONVEX_SITE_URL=(.+)$/m)?.[1]?.replace(/^"|"$/g, ""); +assert(site, "Select a Convex deployment first"); +const root = await fetch(site, { signal: AbortSignal.timeout(20_000) }); +assert.equal(root.status, 200); +assert(root.headers.get("content-type")?.includes("text/html")); +const html = await root.text(); +assert(html.includes("Convex Ɨ Trigger.dev"), "Wrong app was served"); +const assets = [...html.matchAll(/(?:src|href)="(\/assets\/[^"\s]+\.(?:js|css))"/g)].map( + (match) => match[1]! +); +assert( + assets.some((asset) => asset.endsWith(".js")) && assets.some((asset) => asset.endsWith(".css")), + "Missing built assets" +); +for (const asset of assets) { + const response = await fetch(new URL(asset, site), { signal: AbortSignal.timeout(20_000) }); + assert.equal(response.status, 200, `Asset not served: ${asset}`); + assert( + !response.headers.get("content-type")?.includes("text/html"), + `Asset returned SPA HTML: ${asset}` + ); + assert((await response.arrayBuffer()).byteLength > 0); +} +const route = await fetch(new URL("/field-guide", site), { signal: AbortSignal.timeout(20_000) }); +assert.equal(route.status, 200); +assert.equal(await route.text(), html, "SPA fallback did not serve the current index"); +const missing = await fetch(new URL("/missing-asset.js", site), { + signal: AbortSignal.timeout(20_000), +}); +assert.equal(missing.status, 404); +console.log( + JSON.stringify( + { result: "PASS", site, assets, spaFallback: true, missingAsset404: true }, + null, + 2 + ) +); diff --git a/packages/convex/example/scripts/verify.ts b/packages/convex/example/scripts/verify.ts new file mode 100644 index 00000000000..803620b7044 --- /dev/null +++ b/packages/convex/example/scripts/verify.ts @@ -0,0 +1,262 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { createHash, randomBytes } from "node:crypto"; +import { ConvexClient, ConvexHttpClient } from "convex/browser"; +import { api } from "../convex/_generated/api.js"; +import { auth, runs } from "@trigger.dev/sdk"; +import type { Id } from "../convex/_generated/dataModel"; +import { PROCESSOR_VERSION, SAMPLES } from "../shared/catalog.ts"; + +const env = readFileSync(new URL("../.env.local", import.meta.url), "utf8"); +const url = env.match(/^VITE_CONVEX_URL=(.+)$/m)?.[1]?.replace(/^"|"$/g, ""); +assert(url, "Select a Convex deployment first"); +const directory = join(homedir(), ".cache", "convex-image-demo"); +const sessionPath = join(directory, "operator-session"); +await mkdir(directory, { recursive: true, mode: 0o700 }); +let sessionToken: string; +try { + sessionToken = await readFile(sessionPath, "utf8"); +} catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + sessionToken = randomBytes(32).toString("hex"); + await writeFile(sessionPath, sessionToken, { mode: 0o600, flag: "wx" }); +} +assert(/^[a-f0-9]{64}$/.test(sessionToken), "Invalid operator session file"); +const http = new ConvexHttpClient(url); +const initial = await http.query(api.jobs.board, { sessionToken }); +assert.equal( + (await http.query(api.jobs.board, { sessionToken: randomBytes(32).toString("hex") })).jobs.length, + 0 +); +const site = url.replace(".convex.cloud", ".convex.site"); +for (const sample of SAMPLES) { + const response = await fetch(site + sample.file); + assert.equal(response.status, 200); + assert(response.headers.get("content-type")?.startsWith("image/")); + const served = new Uint8Array(await response.arrayBuffer()); + const bundled = await readFile(new URL(`../public${sample.file}`, import.meta.url)); + assert.equal( + createHash("sha256").update(served).digest("hex"), + createHash("sha256").update(bundled).digest("hex"), + `Wrong hosted photograph: ${sample.id}` + ); +} +assert.equal((await fetch(site + "/api/image")).status, 401); +const args = { + sessionToken, + sample: "mountains" as const, + preset: "balanced" as const, + prompt: "Turn this mountain landscape into a snowy winter scene.", + requestId: `image-edit-verification-${PROCESSOR_VERSION}`, +}; +let jobId = initial.jobs.find( + (job) => job.sample === args.sample && job.preset === args.preset && job.prompt === args.prompt +)?.id; +if (!jobId && !process.argv.includes("--launch")) { + console.log( + JSON.stringify( + { + result: "READY", + samples: SAMPLES.length, + enabled: initial.quota.enabled, + budgetUsed: initial.quota.used, + modelCallTested: false, + note: "Use --launch to explicitly spend one generation after adding the Gateway key and enabling edits.", + }, + null, + 2 + ) + ); + process.exit(0); +} +if (!jobId) { + assert( + initial.quota.enabled, + "Edits are paused. Configure the Trigger Gateway key, then enable the demo." + ); + assert( + !initial.turnstileSiteKey, + "Use the browser to complete configured human verification. The CLI does not bypass it." + ); +} +const client = new ConvexClient(url); +const states: Array<{ status: string; stage: string; progress: number }> = []; +const realtimeStates: Array<{ + status: string; + phase: unknown; + progress: unknown; + completedExports: unknown; +}> = []; +const verifyRealtime = process.argv.includes("--realtime"); +const REALTIME_TIMEOUT_MS = 120_000; +let observing: Promise | undefined; +let realtimeFailed = false; +let terminalHintAt: number | undefined; +let stopRealtime: (() => void) | undefined; +async function observeRealtime(jobId: string) { + const credentials = await http.action(api.realtime.token, { + jobId: jobId as Id<"edits">, + sessionToken, + }); + assert(credentials, "No Realtime credentials issued"); + await assert.rejects( + http.action(api.realtime.token, { + jobId: jobId as Id<"edits">, + sessionToken: randomBytes(32).toString("hex"), + }) + ); + await assert.rejects( + http.mutation(api.jobs.refresh, { + jobId: jobId as Id<"edits">, + sessionToken: randomBytes(32).toString("hex"), + }) + ); + const headers = { Authorization: `Bearer ${credentials.accessToken}` }; + const own = await fetch(`${credentials.baseURL}/api/v3/runs/${credentials.runId}`, { headers }); + assert.equal(own.status, 200); + const data = await own.json(); + assert(data.payload && typeof data.payload === "object"); + assert(!("workerToken" in data.payload), "Worker credential exposed in readable payload"); + assert(!("secretKey" in data.payload), "Server credential exposed in readable payload"); + const foreign = await fetch(`${credentials.baseURL}/api/v3/runs/run_06gohij7b965itpig8b2bj6ca2`, { + headers, + }); + assert([403, 404].includes(foreign.status), "Realtime token read another run"); + assert.equal( + (await fetch(`${site}/api/artifacts?jobId=${jobId}&slot=prepared`, { headers })).status, + 401 + ); + await auth.withAuth( + { secretKey: credentials.accessToken, baseURL: credentials.baseURL }, + async () => { + const subscription = runs.subscribeToRun(credentials.runId); + stopRealtime = () => subscription.unsubscribe(); + const timeout = setTimeout(() => { + realtimeFailed = true; + subscription.unsubscribe(); + }, REALTIME_TIMEOUT_MS); + try { + for await (const run of subscription) { + assert.equal(run.id, credentials.runId); + const next = { + status: run.status, + phase: run.metadata?.phase, + progress: run.metadata?.progress, + completedExports: ["heroReady", "cardReady", "squareReady"].filter( + (key) => run.metadata?.[key] === true + ).length, + }; + if (JSON.stringify(realtimeStates.at(-1)) !== JSON.stringify(next)) { + realtimeStates.push(next); + console.log(JSON.stringify({ realtime: next })); + } + if (run.isCompleted || run.status === "EXPIRED" || run.status === "TIMED_OUT") { + terminalHintAt = Date.now(); + await http.mutation(api.jobs.refresh, { jobId: jobId as Id<"edits">, sessionToken }); + break; + } + } + } finally { + clearTimeout(timeout); + subscription.unsubscribe(); + } + } + ); +} +const unsubscribe = client.onUpdate(api.jobs.board, { sessionToken }, (board) => { + const job = board.jobs.find((job) => job.id === jobId); + if (!job) return; + const next = { status: job.status, stage: job.stage, progress: job.progress }; + if (JSON.stringify(states.at(-1)) !== JSON.stringify(next)) { + states.push(next); + console.log(JSON.stringify(next)); + } +}); +try { + const first = await http.action(api.jobs.process, args); + jobId = first.jobId; + const retry = await http.action(api.jobs.process, args); + assert.equal(retry.jobId, jobId); + assert.equal(retry.cached, true); + const deadline = Date.now() + 10 * 60_000; + let completed = false; + while (Date.now() < deadline) { + const state = await http.query(api.jobs.board, { sessionToken }); + const job = state.jobs.find((job) => job.id === jobId); + assert(job, "Job disappeared"); + if (job.error) throw new Error(job.error); + if (verifyRealtime && job.runId && !observing) { + observing = observeRealtime(job.id).catch((error) => { + realtimeFailed = true; + console.error( + JSON.stringify({ realtimeError: { name: error?.name, status: error?.status } }) + ); + }); + } + if (job.status === "COMPLETED") { + const deliveredAt = Date.now(); + assert.equal(job.syncStatus, "complete"); + assert.equal(job.results.length, 3); + assert.equal(state.quota.active, false); + for (const result of job.results) { + const response = await fetch(result.url, { + headers: { Authorization: `Bearer ${sessionToken}` }, + signal: AbortSignal.timeout(20_000), + }); + assert.equal(response.status, 200); + assert(response.headers.get("content-type")?.includes("image/avif")); + assert(response.headers.get("cache-control")?.includes("no-store")); + assert.equal((await response.arrayBuffer()).byteLength, result.bytes); + assert.equal((await fetch(result.url)).status, 401); + assert.equal( + ( + await fetch(result.url, { + headers: { Authorization: `Bearer ${randomBytes(32).toString("hex")}` }, + }) + ).status, + 404 + ); + } + if (verifyRealtime) { + assert(observing, "Realtime subscription was not started"); + await observing; + assert(!realtimeFailed, "Realtime verification failed"); + assert(realtimeStates.length > 0, "No Realtime frames received"); + assert( + terminalHintAt !== undefined, + "No authoritative refresh requested from a terminal frame" + ); + } + console.log( + JSON.stringify( + { + result: "PASS", + jobId, + runId: job.runId, + processingMs: job.processingMs, + budgetUsed: state.quota.used, + subscriptionUpdates: states.length, + realtimeUpdates: realtimeStates.length, + terminalHintToDeliveryMs: + terminalHintAt === undefined || deliveredAt < terminalHintAt + ? undefined + : deliveredAt - terminalHintAt, + }, + null, + 2 + ) + ); + completed = true; + break; + } + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + assert(completed, "Timed out waiting for image edit"); +} finally { + stopRealtime?.(); + unsubscribe(); + await client.close(); +} diff --git a/packages/convex/example/shared/catalog.ts b/packages/convex/example/shared/catalog.ts new file mode 100644 index 00000000000..963209a12ab --- /dev/null +++ b/packages/convex/example/shared/catalog.ts @@ -0,0 +1,88 @@ +import type { SampleId, PresetId } from "./types"; + +export const SAMPLES: { id: SampleId; name: string; file: string; source: string }[] = [ + { + id: "forest", + name: "Forest", + file: "/samples/forest.jpg", + source: "https://images.unsplash.com/photo-1441974231531-c6227db76b6e", + }, + { + id: "coast", + name: "Coast", + file: "/samples/coast.jpg", + source: "https://images.unsplash.com/photo-1518837695005-2083093ee35b", + }, + { + id: "mountains", + name: "Mountains", + file: "/samples/mountains.jpg", + source: "https://images.unsplash.com/photo-1470770841072-f978cf4d019e", + }, + { + id: "city", + name: "City", + file: "/samples/city.jpg", + source: "https://images.unsplash.com/photo-1519501025264-65ba15a82390", + }, + { + id: "desert", + name: "Desert", + file: "/samples/desert.jpg", + source: "https://images.unsplash.com/photo-1509316785289-025f5b846b35", + }, + { + id: "lake", + name: "Lake", + file: "/samples/lake.jpg", + source: "https://images.unsplash.com/photo-1501785888041-af3ef285b470", + }, + { + id: "dog", + name: "Dog", + file: "/samples/dog.jpg", + source: "https://images.unsplash.com/photo-1552053831-71594a27632d", + }, + { + id: "flowers", + name: "Flowers", + file: "/samples/flowers.jpg", + source: "https://images.unsplash.com/photo-1490750967868-88aa4486c946", + }, + { + id: "architecture", + name: "Architecture", + file: "/samples/architecture.jpg", + source: "https://images.unsplash.com/photo-1487958449943-2429e8be8625", + }, + { + id: "canyon", + name: "Canyon", + file: "/samples/canyon.jpg", + source: "https://images.unsplash.com/photo-1473580044384-7ba9967e16a0", + }, +]; +export const PRESETS: { id: PresetId; name: string; description: string; quality: number }[] = [ + { id: "balanced", name: "Balanced", description: "Good quality, smaller files", quality: 55 }, + { + id: "compact", + name: "Smallest files", + description: "More compression for faster delivery", + quality: 35, + }, + { id: "quality", name: "Higher quality", description: "More detail, larger files", quality: 75 }, +]; +export const VARIANTS = [ + { kind: "hero" as const, label: "Hero", width: 1024, height: 768 }, + { kind: "card" as const, label: "Card", width: 800, height: 600 }, + { kind: "square" as const, label: "Square", width: 400, height: 400 }, +]; +export const MAX_JOBS = 500; +export const DAILY_JOB_LIMIT = 50; +export const PROMPT_LIMIT = 500; +export const RESULT_TTL_MS = 60 * 60_000; +export const MODEL_ID = "meta/muse-image-1.0"; +export const MODEL_COST_USD = 0.01; +export const COOLDOWN_MS = 20_000; +export const PROCESSOR_VERSION = "muse-realtime-v2"; +export const AVIF_EFFORT = 4; diff --git a/packages/convex/example/shared/output.ts b/packages/convex/example/shared/output.ts new file mode 100644 index 00000000000..a3274300ddd --- /dev/null +++ b/packages/convex/example/shared/output.ts @@ -0,0 +1,30 @@ +import { z } from "zod"; +import { VARIANTS } from "./catalog"; + +export const outputSchema = z.object({ + inputBytes: z.number().int().positive(), + processingMs: z.number().nonnegative(), + variants: z + .array( + z.object({ + kind: z.enum(["hero", "card", "square"]), + width: z.number().int().positive(), + height: z.number().int().positive(), + bytes: z.number().int().positive(), + format: z.literal("avif"), + }) + ) + .length(3) + .refine( + (variants) => + VARIANTS.every((expected) => + variants.some( + (actual) => + actual.kind === expected.kind && + actual.width === expected.width && + actual.height === expected.height + ) + ), + "Unexpected image crop set" + ), +}); diff --git a/packages/convex/example/shared/types.ts b/packages/convex/example/shared/types.ts new file mode 100644 index 00000000000..444a6118305 --- /dev/null +++ b/packages/convex/example/shared/types.ts @@ -0,0 +1,54 @@ +export type SampleId = + | "forest" + | "coast" + | "mountains" + | "city" + | "desert" + | "lake" + | "dog" + | "flowers" + | "architecture" + | "canyon"; +export type PresetId = "balanced" | "compact" | "quality"; +export type VariantKind = "hero" | "card" | "square"; +export type ImageResult = { + kind: VariantKind; + width: number; + height: number; + bytes: number; + format: "avif"; + url: string; +}; +export type Job = { + id: string; + requestId?: string; + sample: SampleId; + prompt: string; + preset: PresetId; + createdAt: number; + completedAt?: number; + expiresAt: number; + status: string; + syncStatus: string; + progress: number; + stage: string; + runId: string | null; + realtimeEligible?: boolean; + error: string | null; + results: ImageResult[]; + inputBytes?: number; + processingMs?: number; +}; +export type Board = { + jobs: Job[]; + quota: { + used: number; + limit: number; + dailyUsed: number; + dailyLimit: number; + active: boolean; + enabled: boolean; + cooldownUntil: number; + }; + turnstileSiteKey: string | null; +}; diff --git a/packages/convex/example/shared/workerAuth.test.ts b/packages/convex/example/shared/workerAuth.test.ts new file mode 100644 index 00000000000..5010261c33e --- /dev/null +++ b/packages/convex/example/shared/workerAuth.test.ts @@ -0,0 +1,13 @@ +import { expect, test } from "vitest"; +import { jobWorkerToken } from "./workerAuth"; + +test("artifact credentials are deterministic and bound to both job and server secret", async () => { + const first = await jobWorkerToken("a".repeat(64), "job-one"); + expect(await jobWorkerToken("a".repeat(64), "job-one")).toBe(first); + expect(await jobWorkerToken("a".repeat(64), "job-two")).not.toBe(first); + expect(await jobWorkerToken("b".repeat(64), "job-one")).not.toBe(first); +}); + +test("artifact authentication fails closed without its server secret", async () => { + await expect(jobWorkerToken(undefined, "job-one")).rejects.toThrow("not configured"); +}); diff --git a/packages/convex/example/shared/workerAuth.ts b/packages/convex/example/shared/workerAuth.ts new file mode 100644 index 00000000000..012ee87fc6e --- /dev/null +++ b/packages/convex/example/shared/workerAuth.ts @@ -0,0 +1,22 @@ +const WORKER_SECRET_PATTERN = /^[a-f0-9]{64}$/; + +export async function jobWorkerToken(secret: string | undefined, jobId: string): Promise { + if (!secret || !WORKER_SECRET_PATTERN.test(secret)) + throw new Error("DEMO_ARTIFACT_SECRET is not configured"); + const encoder = new TextEncoder(); + const key = await crypto.subtle.importKey( + "raw", + encoder.encode(secret), + { name: "HMAC", hash: "SHA-256" }, + false, + ["sign"] + ); + const signature = await crypto.subtle.sign( + "HMAC", + key, + encoder.encode(`convex-image-artifact:${jobId}`) + ); + return Array.from(new Uint8Array(signature), (byte) => byte.toString(16).padStart(2, "0")).join( + "" + ); +} diff --git a/packages/convex/example/src/App.test.tsx b/packages/convex/example/src/App.test.tsx new file mode 100644 index 00000000000..30ca585dd1b --- /dev/null +++ b/packages/convex/example/src/App.test.tsx @@ -0,0 +1,112 @@ +import { afterEach, beforeEach, expect, test, vi } from "vitest"; +import { renderToStaticMarkup } from "react-dom/server"; +import App, { SUGGESTED_PROMPT } from "./App"; +import { draftFor, keyFor, writeEditor } from "./editorState"; +import type { Board, Job } from "../shared/types"; + +beforeEach(() => { + const values = new Map([["convex-image-processing-session", "a".repeat(64)]]); + vi.stubGlobal("localStorage", { + getItem: (key: string) => values.get(key) ?? null, + setItem: (key: string, value: string) => { + values.set(key, value); + }, + }); +}); +afterEach(() => vi.unstubAllGlobals()); + +function cachedJob(overrides: Partial = {}): Job { + return { + id: "previous-edit", + sample: "forest", + prompt: SUGGESTED_PROMPT, + preset: "balanced", + createdAt: 1, + expiresAt: Date.now() + 60_000, + status: "COMPLETED", + syncStatus: "complete", + progress: 100, + stage: "Images ready", + runId: null, + error: null, + results: [ + { + kind: "hero", + width: 1024, + height: 768, + bytes: 100, + format: "avif", + url: "blob:previous-result", + }, + ], + ...overrides, + }; +} +function render(job: Job, quota: Partial = {}) { + const board: Board = { + jobs: [job], + turnstileSiteKey: null, + quota: { + used: 1, + limit: 500, + dailyUsed: 1, + dailyLimit: 50, + active: false, + enabled: true, + cooldownUntil: 0, + ...quota, + }, + }; + return renderToStaticMarkup(); +} + +test.each([ + { sample: "architecture" as const }, + { prompt: "Make this a sunny desert" }, + { preset: "quality" as const }, +])("does not display a cached result for different inputs: %j", (mismatch) => { + expect(render(cachedJob(mismatch))).not.toContain("blob:previous-result"); +}); + +test("keeps the submit button disabled while the matching edit is running", () => { + const html = render(cachedJob({ status: "EXECUTING", progress: 25, results: [] })); + const submit = html.match(/]*type="submit"[^>]*>.*?<\/button>/)?.[0]; + expect(submit).toBeDefined(); + expect(submit).toContain('disabled=""'); + expect(submit).toContain(">Submit"); +}); + +test("explains why another visitor’s active edit disables submission", () => { + expect(render(cachedJob(), { active: true })).toContain( + "Another edit is running. Please try again shortly." + ); +}); + +test("does not silently select a previous result matching the current draft", () => { + writeEditor("a".repeat(64), { draft: draftFor(cachedJob()), selection: null }); + expect(render(cachedJob())).not.toContain('src="blob:previous-result"'); +}); + +test("restores an explicitly selected result and explains where it came from", () => { + const job = cachedJob(); + writeEditor("a".repeat(64), { + draft: draftFor(job), + selection: { key: keyFor(job), id: job.id }, + }); + const html = render(job); + expect(html).toContain('src="blob:previous-result"'); + expect(html).toContain("Restored your last edit."); +}); + +test("restores an owned active edit instead of the default photo and prompt", () => { + const job = cachedJob({ + sample: "coast", + prompt: "Make the waves golden", + status: "EXECUTING", + results: [], + }); + const html = render(job); + expect(html).toContain("Coast original photograph"); + expect(html).toContain("Make the waves golden"); + expect(html).toContain("Your edit is running. You can leave and return."); +}); diff --git a/packages/convex/example/src/App.tsx b/packages/convex/example/src/App.tsx new file mode 100644 index 00000000000..1f08669939f --- /dev/null +++ b/packages/convex/example/src/App.tsx @@ -0,0 +1,453 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { PROMPT_LIMIT, SAMPLES } from "../shared/catalog"; +import type { Board, PresetId, SampleId, Job } from "../shared/types"; +import { usePrivateHistory } from "./usePrivateHistory"; +import { ImagePicker } from "./ImagePicker"; +import { JobPanel, jobFinished } from "./JobPanel"; +import { + draftFor, + emptyEditor, + keyFor, + readEditor, + selectedJob, + writeEditor, + type Draft, + type EditorState, +} from "./editorState"; + +export type ProcessImages = (args: { + sample: SampleId; + preset: PresetId; + prompt: string; + requestId: string; + sessionToken: string; + challengeToken?: string; +}) => Promise<{ jobId: string; cached: boolean }>; +type AppProps = { + board?: Board; + processImages?: ProcessImages; + connected?: boolean; + preview?: boolean; + connectionError?: boolean; + JobView?: React.ComponentType<{ + job?: Job; + pending: boolean; + preview: boolean; + sessionToken: string; + onLoad: (job: Job) => Promise; + }>; +}; +const DEFAULT_PRESET = "balanced"; +export const SUGGESTED_PROMPT = + "Turn this into a cinematic winter scene with cool blue light and gentle falling snow."; + +type Turnstile = { + render: (element: HTMLElement, options: Record) => string; + reset: (id?: string) => void; + remove: (id: string) => void; +}; +declare global { + interface Window { + turnstile?: Turnstile; + } +} + +export function getSessionToken(): string { + const key = "convex-image-processing-session"; + let token = localStorage.getItem(key); + if (!token) { + token = Array.from(crypto.getRandomValues(new Uint8Array(32)), (byte) => + byte.toString(16).padStart(2, "0") + ).join(""); + localStorage.setItem(key, token); + } + return token; +} + +export default function App({ + board, + processImages, + connected = false, + preview = false, + connectionError = false, + JobView = JobPanel, +}: AppProps) { + const token = useMemo(() => (preview ? "" : getSessionToken()), [preview]); + const initialEditor = useMemo( + () => (preview ? emptyEditor() : readEditor(token)), + [preview, token] + ); + const [editor, setEditor] = useState(initialEditor); + const editorRef = useRef(editor); + editorRef.current = editor; + const [restored, setRestored] = useState( + !!initialEditor.selection || !!initialEditor.draft.prompt + ); + const [storageError, setStorageError] = useState(null); + const persistEditor = useCallback( + (next: EditorState) => { + editorRef.current = next; + setEditor(next); + if (!preview) + setStorageError( + writeEditor(token, next) + ? null + : "This browser could not save your draft. Running edits can still be restored from Convex." + ); + }, + [preview, token] + ); + const [pickerOpen, setPickerOpen] = useState(false); + const [pending, setPending] = useState(false); + const [error, setError] = useState(null); + const [challengeToken, setChallengeToken] = useState(); + const [now, setNow] = useState(Date.now); + const submitting = useRef(false); + const widget = useRef(null); + const resultSection = useRef(null); + const history = usePrivateHistory(token); + const { jobs: cachedJobs, download } = history; + const remoteJobs = board?.jobs ?? []; + const jobMap = new Map(remoteJobs.map((job) => [job.id, job])); + for (const local of history.jobs) jobMap.set(local.id, local); + const jobs = [...jobMap.values()].sort((a, b) => b.createdAt - a.createdAt); + const activeJob = remoteJobs.find((job) => !jobFinished(job)); + const job = activeJob ?? selectedJob(editor, jobs); + const draft = activeJob ? draftFor(activeJob) : editor.draft; + const reusable = job && (!jobFinished(job) || !!job.results.length); + const editing = !!job && !jobFinished(job); + const busy = pending || editing; + + useEffect(() => { + if ( + !activeJob || + (editorRef.current.selection?.id === activeJob.id && + keyFor(editorRef.current.draft) === keyFor(activeJob)) + ) + return; + persistEditor({ + draft: draftFor(activeJob), + selection: { key: keyFor(activeJob), id: activeJob.id, requestId: activeJob.requestId }, + }); + if (!submitting.current) setRestored(true); + }, [activeJob, persistEditor]); + const quota = board?.quota; + const cooldown = Math.max(0, Math.ceil(((quota?.cooldownUntil ?? 0) - now) / 1000)); + function submissionDisabledReason(): string | null { + if (pending) return "Submitting your edit…"; + if (editing) + return ( + job.error ?? + (restored + ? "Your in-progress edit was restored. You can leave and return." + : "Your edit is running. You can leave and return.") + ); + if (reusable) return null; + if (preview) return "This is a read-only preview."; + if (!connected || !board) return "Connecting to Convex…"; + if (!quota?.enabled) return "Image editing is paused."; + if (quota.used >= quota.limit) return "The demo has reached its generation limit."; + if (quota.dailyUsed >= quota.dailyLimit) + return "Today's demo allowance has been used. Try again tomorrow."; + if (quota.active) return "Another edit is running. Please try again shortly."; + if (cooldown > 0) return `Ready for another edit in ${cooldown}s.`; + if (!draft.prompt.trim()) return "Describe your edit before submitting."; + if (draft.prompt.length > PROMPT_LIMIT) + return `Keep your edit within ${PROMPT_LIMIT} characters.`; + if (board.turnstileSiteKey && !challengeToken) return "Complete human verification to submit."; + return null; + } + const disabledReason = submissionDisabledReason(); + const disabled = disabledReason !== null; + const photograph = SAMPLES.find((sample) => sample.id === draft.sample)!; + + useEffect(() => { + if (!quota?.cooldownUntil || quota.cooldownUntil <= Date.now()) return; + const timer = window.setInterval(() => setNow(Date.now()), 1000); + return () => window.clearInterval(timer); + }, [quota?.cooldownUntil]); + + useEffect(() => { + const siteKey = board?.turnstileSiteKey; + if (!siteKey || !widget.current) return; + let id: string | undefined; + const render = () => { + if (!window.turnstile || !widget.current || id) return; + id = window.turnstile.render(widget.current, { + sitekey: siteKey, + callback: (value: string) => setChallengeToken(value), + "expired-callback": () => setChallengeToken(undefined), + "error-callback": () => setChallengeToken(undefined), + }); + }; + let script = document.querySelector("script[data-turnstile]"); + if (!script) { + script = document.createElement("script"); + script.src = "https://challenges.cloudflare.com/turnstile/v0/api.js?render=explicit"; + script.dataset.turnstile = "true"; + script.async = true; + document.head.append(script); + } + script.addEventListener("load", render); + render(); + return () => { + script.removeEventListener("load", render); + if (id) window.turnstile?.remove(id); + }; + }, [board?.turnstileSiteKey]); + + useEffect(() => { + if (job?.results.length && !cachedJobs.some((local) => local.id === job.id)) void download(job); + }, [job, cachedJobs, download]); + + function selectDraft(next: Draft) { + persistEditor({ draft: next, selection: null }); + setRestored(false); + setError(null); + } + + function showResult() { + resultSection.current?.scrollIntoView({ + behavior: window.matchMedia("(prefers-reduced-motion: reduce)").matches ? "auto" : "smooth", + block: "start", + }); + } + + async function handleProcess(event: React.FormEvent) { + event.preventDefault(); + if (disabled || submitting.current) return; + setError(null); + if (reusable) { + showResult(); + return; + } + if (!processImages) return; + submitting.current = true; + setPending(true); + const submittedKey = keyFor(draft); + const requestId = + !job && editor.selection?.key === submittedKey && editor.selection.requestId + ? editor.selection.requestId + : crypto.randomUUID(); + persistEditor({ draft, selection: { key: submittedKey, requestId } }); + setRestored(false); + showResult(); + try { + const result = await processImages({ + ...draft, + prompt: draft.prompt.trim(), + requestId, + sessionToken: token, + ...(challengeToken ? { challengeToken } : {}), + }); + persistEditor({ draft, selection: { key: submittedKey, id: result.jobId, requestId } }); + } catch (cause) { + if (!editorRef.current.selection?.id) + setError( + cause instanceof Error + ? cause.message.replace(/^.*?Uncaught Error: /, "").split("\n")[0]! + : "Your edit could not be submitted." + ); + } finally { + submitting.current = false; + setPending(false); + setChallengeToken(undefined); + window.turnstile?.reset(); + } + } + + return ( +
+
+
+

A demo by Trigger.dev

+

Convex Ɨ Trigger.dev

+

Choose a photo. Describe a change. Let Trigger.dev do the background work.

+
+ {(preview || !connected) && ( +

+ {preview + ? connectionError + ? "Connection unavailable. This is a read-only preview." + : "Read-only preview. Connect Convex to try an edit." + : "Connecting to Convex. Your saved images are still available below."} +

+ )} + +
+
+

1. Choose an image

+ +
+ {`${photograph.name} +
+ +
+

2. Describe your edit

+
+ +