Merge pull request #529 from microsoft/copilot/fix-nightly-deploy-issue

fix(infra): fix container app activation failures

Author: MikeWedderburn-Clarke

apps/web/src/app/api/ready/route.ts

@@ -4,6 +4,23 @@ import { BlobServiceClient } from "@azure/storage-blob";
 import { DefaultAzureCredential } from "@azure/identity";
 import type { ReadinessResponse } from "@acroyoga/shared";
 
+const CHECK_TIMEOUT_MS = 5000;
+
+function withTimeout(promise: Promise<string>): Promise<string> {
+  return new Promise<string>((resolve) => {
+    const timer = setTimeout(
+      () => resolve("error: health check timeout"),
+      CHECK_TIMEOUT_MS,
+    );
+    promise
+      .then((value) => { clearTimeout(timer); resolve(value); })
+      .catch((err: unknown) => {
+        clearTimeout(timer);
+        resolve(`error: ${err instanceof Error ? err.message : "unknown"}`);
+      });
+  });
+}
+
 async function checkDatabase(): Promise<string> {
   try {
     await db().query("SELECT 1");
@@ -33,8 +50,8 @@ async function checkStorage(): Promise<string> {
 
 export async function GET(): Promise<NextResponse<ReadinessResponse>> {
   const [databaseStatus, storageStatus] = await Promise.all([
-    checkDatabase(),
-    checkStorage(),
+    withTimeout(checkDatabase()),
+    withTimeout(checkStorage()),
   ]);
 
   const allOk = databaseStatus === "ok" && storageStatus === "ok";

apps/web/src/lib/db/client.ts

@@ -42,6 +42,7 @@ export function getDb(): DbClient {
         password: getEntraPassword,
         ssl: { rejectUnauthorized: true },
         max: 10,
+        connectionTimeoutMillis: 5000,
       });
     }
     return pool;
@@ -68,6 +69,7 @@ export function getDb(): DbClient {
     pool = new pg.Pool({
       connectionString: dbUrl,
       max: 10,
+      connectionTimeoutMillis: 5000,
     });
   }
   return pool;

apps/web/tests/integration/health.test.ts

@@ -1,4 +1,4 @@
-import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { describe, it, expect, beforeAll, afterAll, vi } from "vitest";
 import { createTestDb } from "../helpers/db";
 import { setTestDb, clearTestDb } from "../../src/lib/db/client";
 import type { PGlite } from "@electric-sql/pglite";
@@ -69,5 +69,32 @@ describe("Health API endpoints", () => {
       }
       setTestDb(testDb);
     });
+
+    it("returns 503 with timeout error when database check hangs", async () => {
+      vi.useFakeTimers();
+
+      // Inject a DB client whose query never resolves (simulates a hanging connection)
+      setTestDb({
+        query: () => new Promise(() => {}),
+      } as unknown as PGlite);
+
+      try {
+        const { GET } = await readyModule();
+        const responsePromise = GET();
+
+        // Advance past the 5000 ms check timeout
+        await vi.advanceTimersByTimeAsync(5001);
+
+        const response = await responsePromise;
+        expect(response.status).toBe(503);
+
+        const body = await response.json();
+        expect(body).toHaveProperty("status", "not_ready");
+        expect(body.checks.database).toBe("error: health check timeout");
+      } finally {
+        vi.useRealTimers();
+        setTestDb(testDb);
+      }
+    });
   });
 });

infra/modules/container-apps.bicep

@@ -94,7 +94,7 @@ resource containerApp 'Microsoft.App/containerApps@2024-03-01' = {
   properties: {
     managedEnvironmentId: containerAppEnvironment.id
     configuration: {
-      activeRevisionsMode: 'Multiple'
+      activeRevisionsMode: 'Single'
       ingress: {
         external: true
         targetPort: 3000
@@ -208,14 +208,19 @@ resource containerApp 'Microsoft.App/containerApps@2024-03-01' = {
               failureThreshold: 10
             }
             {
+              // Startup probe window: initialDelaySeconds + failureThreshold × periodSeconds
+              // = 30 + 60 × 5 = 330 s total. Worst-case migration time is
+              // 3 attempts × 60 s + 2 × 10 s backoff = 200 s, leaving ~130 s of
+              // margin for Node.js to start and pass this probe.
               type: 'Startup'
               httpGet: {
                 path: '/api/health'
                 port: 3000
               }
+              initialDelaySeconds: 30
               periodSeconds: 5
               timeoutSeconds: 5
-              failureThreshold: 30
+              failureThreshold: 60
             }
           ]
         }

start.sh

@@ -1,6 +1,24 @@
 #!/bin/sh
 set -e
-# Run database migrations (idempotent — safe on every container start)
-MIGRATIONS_DIR=/app/migrations node /app/migrate.cjs
+
+# Run database migrations with retries to handle transient cold-start failures
+# (e.g. Managed Identity IMDS delays, PostgreSQL not yet accepting connections).
+MAX_RETRIES=3
+RETRY_DELAY=10
+MIGRATION_TIMEOUT=60
+RETRY=0
+while [ "$RETRY" -lt "$MAX_RETRIES" ]; do
+  if MIGRATIONS_DIR=/app/migrations timeout "$MIGRATION_TIMEOUT" node /app/migrate.cjs; then
+    break
+  fi
+  RETRY=$((RETRY + 1))
+  if [ "$RETRY" -eq "$MAX_RETRIES" ]; then
+    echo "DB migrations failed after $MAX_RETRIES attempts, exiting"
+    exit 1
+  fi
+  echo "Migration attempt $RETRY failed, retrying in ${RETRY_DELAY}s..."
+  sleep "$RETRY_DELAY"
+done
+
 # Start the Next.js server
 exec node apps/web/server.js