diff --git a/.github/workflows/e2e-tests-split.yml b/.github/workflows/e2e-tests-split.yml index 0cbd4f82..73eee00b 100644 --- a/.github/workflows/e2e-tests-split.yml +++ b/.github/workflows/e2e-tests-split.yml @@ -229,6 +229,7 @@ jobs: node-version: ${{ env.NODE_VERSION }} cache: 'npm' + - name: Log in to Docker Hub if: needs.build.outputs.image_source == 'registry' uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 @@ -430,6 +431,7 @@ jobs: node-version: ${{ env.NODE_VERSION }} cache: 'npm' + - name: Log in to Docker Hub if: needs.build.outputs.image_source == 'registry' uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 @@ -639,6 +641,7 @@ jobs: node-version: ${{ env.NODE_VERSION }} cache: 'npm' + - name: Log in to Docker Hub if: needs.build.outputs.image_source == 'registry' uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 @@ -860,6 +863,39 @@ jobs: node-version: ${{ env.NODE_VERSION }} cache: 'npm' + - name: Preflight disk diagnostics (before cleanup) + run: | + echo "Disk usage before cleanup" + df -h + docker system df || true + + - name: Preflight cleanup (best effort) + run: | + echo "Best-effort cleanup for CI runner" + docker system prune -af || true + rm -rf playwright-report playwright-output coverage/e2e test-results diagnostics || true + rm -f docker-logs-*.txt charon-e2e-image.tar || true + + - name: Preflight disk diagnostics and threshold gate + run: | + set -euo pipefail + MIN_FREE_BYTES=$((5 * 1024 * 1024 * 1024)) + echo "Disk usage after cleanup" + df -h + docker system df || true + + WORKSPACE_PATH="${GITHUB_WORKSPACE:-$PWD}" + FREE_ROOT_BYTES=$(df -PB1 / | awk 'NR==2 {print $4}') + FREE_WORKSPACE_BYTES=$(df -PB1 "$WORKSPACE_PATH" | awk 'NR==2 {print $4}') + + echo "Free bytes on /: $FREE_ROOT_BYTES" + echo "Free bytes on workspace ($WORKSPACE_PATH): $FREE_WORKSPACE_BYTES" + + if [ "$FREE_ROOT_BYTES" -lt "$MIN_FREE_BYTES" ] || [ "$FREE_WORKSPACE_BYTES" -lt "$MIN_FREE_BYTES" ]; then + echo "::error::[CI_DISK_PRESSURE] Insufficient free disk after cleanup. Required >= 5GiB on both / and workspace. root=${FREE_ROOT_BYTES}B workspace=${FREE_WORKSPACE_BYTES}B" + exit 42 + fi + - name: Log in to Docker Hub if: needs.build.outputs.image_source == 'registry' uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 @@ -1064,6 +1100,39 @@ jobs: node-version: ${{ env.NODE_VERSION }} cache: 'npm' + - name: Preflight disk diagnostics (before cleanup) + run: | + echo "Disk usage before cleanup" + df -h + docker system df || true + + - name: Preflight cleanup (best effort) + run: | + echo "Best-effort cleanup for CI runner" + docker system prune -af || true + rm -rf playwright-report playwright-output coverage/e2e test-results diagnostics || true + rm -f docker-logs-*.txt charon-e2e-image.tar || true + + - name: Preflight disk diagnostics and threshold gate + run: | + set -euo pipefail + MIN_FREE_BYTES=$((5 * 1024 * 1024 * 1024)) + echo "Disk usage after cleanup" + df -h + docker system df || true + + WORKSPACE_PATH="${GITHUB_WORKSPACE:-$PWD}" + FREE_ROOT_BYTES=$(df -PB1 / | awk 'NR==2 {print $4}') + FREE_WORKSPACE_BYTES=$(df -PB1 "$WORKSPACE_PATH" | awk 'NR==2 {print $4}') + + echo "Free bytes on /: $FREE_ROOT_BYTES" + echo "Free bytes on workspace ($WORKSPACE_PATH): $FREE_WORKSPACE_BYTES" + + if [ "$FREE_ROOT_BYTES" -lt "$MIN_FREE_BYTES" ] || [ "$FREE_WORKSPACE_BYTES" -lt "$MIN_FREE_BYTES" ]; then + echo "::error::[CI_DISK_PRESSURE] Insufficient free disk after cleanup. Required >= 5GiB on both / and workspace. root=${FREE_ROOT_BYTES}B workspace=${FREE_WORKSPACE_BYTES}B" + exit 42 + fi + - name: Log in to Docker Hub if: needs.build.outputs.image_source == 'registry' uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 @@ -1276,6 +1345,39 @@ jobs: node-version: ${{ env.NODE_VERSION }} cache: 'npm' + - name: Preflight disk diagnostics (before cleanup) + run: | + echo "Disk usage before cleanup" + df -h + docker system df || true + + - name: Preflight cleanup (best effort) + run: | + echo "Best-effort cleanup for CI runner" + docker system prune -af || true + rm -rf playwright-report playwright-output coverage/e2e test-results diagnostics || true + rm -f docker-logs-*.txt charon-e2e-image.tar || true + + - name: Preflight disk diagnostics and threshold gate + run: | + set -euo pipefail + MIN_FREE_BYTES=$((5 * 1024 * 1024 * 1024)) + echo "Disk usage after cleanup" + df -h + docker system df || true + + WORKSPACE_PATH="${GITHUB_WORKSPACE:-$PWD}" + FREE_ROOT_BYTES=$(df -PB1 / | awk 'NR==2 {print $4}') + FREE_WORKSPACE_BYTES=$(df -PB1 "$WORKSPACE_PATH" | awk 'NR==2 {print $4}') + + echo "Free bytes on /: $FREE_ROOT_BYTES" + echo "Free bytes on workspace ($WORKSPACE_PATH): $FREE_WORKSPACE_BYTES" + + if [ "$FREE_ROOT_BYTES" -lt "$MIN_FREE_BYTES" ] || [ "$FREE_WORKSPACE_BYTES" -lt "$MIN_FREE_BYTES" ]; then + echo "::error::[CI_DISK_PRESSURE] Insufficient free disk after cleanup. Required >= 5GiB on both / and workspace. root=${FREE_ROOT_BYTES}B workspace=${FREE_WORKSPACE_BYTES}B" + exit 42 + fi + - name: Log in to Docker Hub if: needs.build.outputs.image_source == 'registry' uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 diff --git a/tests/global-setup.ts b/tests/global-setup.ts index d876a359..bfe30570 100644 --- a/tests/global-setup.ts +++ b/tests/global-setup.ts @@ -14,6 +14,28 @@ import { dirname } from 'path'; import { TestDataManager } from './utils/TestDataManager'; import { STORAGE_STATE } from './constants'; +function isSqliteFullFailure(message: string): boolean { + const normalized = message.toLowerCase(); + return ( + normalized.includes('database or disk is full') || + normalized.includes('sqlite_full') || + (normalized.includes('(13)') && normalized.includes('sqlite')) + ); +} + +function buildSqliteFullInfrastructureError(context: string, details: string): Error { + const error = new Error( + `[INFRASTRUCTURE][SQLITE_FULL] ${context}\n` + + `Detected SQLite storage exhaustion during Playwright global setup.\n` + + `Action required:\n` + + `1. Free disk space and verify SQLite volume permissions.\n` + + `2. Rebuild/restart E2E environment before retry.\n` + + `Original error: ${details}` + ); + error.name = 'InfrastructureSQLiteFullError'; + return error; +} + // Singleton to prevent duplicate validation across workers let tokenValidated = false; @@ -433,6 +455,12 @@ async function emergencySecurityReset(requestContext: APIRequestContext): Promis const body = await response.text(); console.error(` ❌ Emergency reset failed: ${response.status()}`); console.error(` 📄 Response body: ${body}`); + if (isSqliteFullFailure(body)) { + throw buildSqliteFullInfrastructureError( + 'Emergency security reset returned non-OK status', + body + ); + } throw new Error(`Emergency reset returned ${response.status()}: ${body}`); } diff --git a/tests/utils/TestDataManager.ts b/tests/utils/TestDataManager.ts index c4c2fbb2..b00cb086 100644 --- a/tests/utils/TestDataManager.ts +++ b/tests/utils/TestDataManager.ts @@ -31,6 +31,38 @@ import { APIRequestContext, type APIResponse, request as playwrightRequest } from '@playwright/test'; import * as crypto from 'crypto'; +const SQLITE_FULL_PATTERN = { + fullText: 'database or disk is full', + sqliteCode: 'sqlite_full', + errno13: '(13)', +} as const; + +let sqliteInfraFailureMessage: string | null = null; + +function isSqliteFullFailure(message: string): boolean { + const normalized = message.toLowerCase(); + const hasDbFullText = normalized.includes(SQLITE_FULL_PATTERN.fullText); + const hasSqliteCode = normalized.includes(SQLITE_FULL_PATTERN.sqliteCode); + const hasErrno13InSqliteContext = + normalized.includes(SQLITE_FULL_PATTERN.errno13) && normalized.includes('sqlite'); + return hasDbFullText || hasSqliteCode || hasErrno13InSqliteContext; +} + +function buildSqliteFullInfrastructureError(context: string, details: string): Error { + const error = new Error( + `[INFRASTRUCTURE][SQLITE_FULL] ${context}\n` + + `Detected SQLite storage exhaustion while running Playwright test setup.\n` + + `Root cause indicators matched: \"database or disk is full\" | \"SQLITE_FULL\" | \"(13)\" in SQLite context.\n` + + `Action required:\n` + + `1. Free disk space on the test runner and ensure the SQLite volume is writable.\n` + + `2. Rebuild/restart the E2E test container to reset state.\n` + + `3. Re-run the failed shard after infrastructure recovery.\n` + + `Original error: ${details}` + ); + error.name = 'InfrastructureSQLiteFullError'; + return error; +} + /** * Represents a managed resource created during tests */ @@ -504,6 +536,10 @@ export class TestDataManager { data: UserData, options: { useNamespace?: boolean } = {} ): Promise { + if (sqliteInfraFailureMessage) { + throw new Error(sqliteInfraFailureMessage); + } + const useNamespace = options.useNamespace !== false; const namespacedEmail = useNamespace ? `${this.namespace}+${data.email}` : data.email; const namespaced = { @@ -513,14 +549,37 @@ export class TestDataManager { role: data.role, }; - const response = await this.postWithRetry('/api/v1/users', namespaced, { - maxAttempts: 4, - baseDelayMs: 300, - retryStatuses: [429], - }); + let response: APIResponse; + try { + response = await this.postWithRetry('/api/v1/users', namespaced, { + maxAttempts: 4, + baseDelayMs: 300, + retryStatuses: [429], + }); + } catch (error) { + const rawMessage = error instanceof Error ? error.message : String(error); + if (isSqliteFullFailure(rawMessage)) { + const infraError = buildSqliteFullInfrastructureError( + 'Failed to create user in TestDataManager.createUser()', + rawMessage + ); + sqliteInfraFailureMessage = infraError.message; + throw infraError; + } + throw error; + } if (!response.ok()) { - throw new Error(`Failed to create user: ${await response.text()}`); + const responseText = await response.text(); + if (isSqliteFullFailure(responseText)) { + const infraError = buildSqliteFullInfrastructureError( + 'Failed to create user in TestDataManager.createUser()', + responseText + ); + sqliteInfraFailureMessage = infraError.message; + throw infraError; + } + throw new Error(`Failed to create user: ${responseText}`); } const result = await response.json();