import { createRequire } from 'node:module';
import { readFileSync, writeFileSync } from 'node:fs';
* Assert that @openmaic/storage's PostgreSQL contract suites really exercised a
* real PostgreSQL, from evidence that cannot be produced by editing the tests.
*
* WHY THIS EXISTS AT ALL. The suites already refuse to skip when
* STORAGE_PG_CONTRACT_REQUIRED=1, but that refusal is a `throw` inside the test
* modules, so it cannot fire if vitest stops collecting them. Collection is
* decided by `packages/@openmaic/storage/vitest.config.ts`, and both that file
* and the whole `test/` directory are on the ignore list of publishable inputs
* in `check-package-version-bumps.mjs`. The entire surface those suites live on
* can therefore be rewritten with no version bump and no release gate noticing.
*
* WHY THE VITEST RESULTS ARE NOT ENOUGH ON THEIR OWN. `assertionResults` are
* test CASES, not `expect()` calls, so a file named `pg-document-store.pg.test.ts`
* containing nothing but `test('x', () => {})` satisfies every check in phase 1
* below. And because `test/setup.ts` is already wired as `setupFiles`, a single
* `vi.mock('pg', ...)` there makes both suites collect, run and pass green
* against an in-memory fake. Both edits are one line, in `test/`, and need no
* version bump. Phase 1 therefore proves only that two files with those names
* ran and reported passing cases — nothing whatsoever about a database.
*
* WHAT PHASE 2 ADDS. It connects to the contract database from OUTSIDE the
* vitest process and asks PostgreSQL itself what happened: the five tables
* these two backends own must exist, and each must have gained inserts DURING
* the run. Nothing inside `test/` can forge that, because producing it requires
* actually writing to the database this script independently connects to.
*
* Insert counters rather than surviving rows: the suites clean up after
* themselves, so counting rows would prove nothing, while `n_tup_ins` survives
* the cleanup. Counted as a delta against a baseline captured before the run
* rather than as an absolute, so a non-ephemeral database cannot satisfy the
* check forever on the strength of some earlier run. Both current workflows use
* a fresh per-job service container, but this check should not depend on that
* staying true.
*
* ── THREAT MODEL, STATED HONESTLY ────────────────────────────────────────────
*
* What this proves: during this run, rows were inserted into those five tables
* in a real PostgreSQL, and two files with the contract suites' names ran and
* passed.
*
* What it does NOT prove: that the built `PgDocumentStore` and `PgRuntimeStore`
* were the code that inserted them. The whole `test/` directory is on the
* publishable-input ignore list, so test code can create the schema and insert
* directly, and this audit would read the same either way. Closing that needs a
* harness living outside the ignored `test/` surface — separate work, not
* attempted here.
*
* That limit is acceptable because of who each threat is. This guard exists to
* catch ACCIDENTAL silencing: a vitest `include`/`exclude` change, a missing
* environment variable, a renamed suite file, a dropped workflow step. Those
* are the ways this coverage actually disappears, and they are all caught.
* It is not a defence against someone deliberately faking coverage from inside
* `test/` — and it does not need to be, because that person can merge changes
* to the production sources just as easily.
*/
const REQUIRED_SUITES = [
'packages/@openmaic/storage/test/pg-document-store.pg.test.ts',
'packages/@openmaic/storage/test/pg-runtime-store.pg.test.ts',
'packages/@openmaic/storage/test/pg-scene-revision.pg.test.ts',
];
* The tables created by `DOCUMENT_PG_SCHEMA` and `RUNTIME_PG_SCHEMA`. Kept
* explicit rather than parsed out of those sources: this list is the
* independent statement of what the contract must have touched, and deriving it
* from the code under test would let that code narrow its own audit.
*/
const REQUIRED_TABLES = [
'document_stages',
'document_scenes',
'document_outlines',
'document_stage_revision',
'document_scene_revision',
'runtime_sessions',
'runtime_records',
];
const usage = [
'Usage:',
' assert-pg-contract-suites.mjs --capture-baseline <file>',
' Record the current insert counters. Run BEFORE the vitest invocation.',
' assert-pg-contract-suites.mjs <vitest-json-results> --baseline <file>',
' Audit the run against that baseline. Run AFTER the vitest invocation.',
].join('\n');
const argv = process.argv.slice(2);
let capturingBaseline = false;
let baselinePath;
const positionals = [];
for (let i = 0; i < argv.length; i += 1) {
const arg = argv[i];
if (arg === '--baseline' || arg === '--capture-baseline') {
capturingBaseline ||= arg === '--capture-baseline';
i += 1;
baselinePath = argv[i];
if (baselinePath === undefined) {
console.error(`${arg} needs a file path.\n\n${usage}`);
process.exit(2);
}
continue;
}
if (arg.startsWith('--')) {
console.error(`Unknown option ${arg}.\n\n${usage}`);
process.exit(2);
}
positionals.push(arg);
}
const expectedPositionals = capturingBaseline ? 0 : 1;
if (!baselinePath || positionals.length !== expectedPositionals) {
console.error(usage);
process.exit(2);
}
const resultsPath = positionals[0];
const contractUrl = process.env.PG_CONTRACT_URL;
if (!contractUrl) {
console.error(
'PG_CONTRACT_URL is unset, so there is no database to audit and this check cannot ' +
'establish that the PostgreSQL contract ran. Invoke it in the same job, with the ' +
'same PG_CONTRACT_URL, as the vitest run it is auditing.',
);
process.exit(2);
}
const requireFromStorage = createRequire(
new URL('../packages/@openmaic/storage/package.json', import.meta.url),
);
const { Client } = requireFromStorage('pg');
async function collectInsertCounts(client) {
const { rows } = await client.query(
`SELECT t.relname AS table_name,
to_regclass('public.' || t.relname) IS NOT NULL AS present,
COALESCE(s.n_tup_ins, 0)::bigint AS inserts
FROM unnest($1::text[]) AS t(relname)
LEFT JOIN pg_stat_user_tables s
ON s.schemaname = 'public' AND s.relname = t.relname`,
[REQUIRED_TABLES],
);
return Object.fromEntries(
rows.map((row) => [row.table_name, { present: row.present, inserts: Number(row.inserts) }]),
);
}
* Read the counters, optionally waiting for them to move past `baseline`.
*
* Backends flush statistics at transaction end and on exit, so by the time
* vitest has returned they are normally already visible. Re-read a few times
* anyway rather than racing a slow flush, discarding the per-session snapshot
* each round because a backend caches it for the whole transaction. A flush
* that never arrives fails the check rather than passing it.
*/
async function readCounters({ waitFor } = {}) {
const client = new Client({ connectionString: contractUrl });
try {
await client.connect();
let counts = await collectInsertCounts(client);
if (!waitFor) return counts;
for (let attempt = 1; attempt < 5; attempt += 1) {
const unproven = REQUIRED_TABLES.filter(
(table) => !(counts[table]?.inserts > (waitFor[table]?.inserts ?? 0)),
);
if (unproven.length === 0) break;
await new Promise((resolve) => setTimeout(resolve, 500));
await client.query('SELECT pg_stat_clear_snapshot()');
counts = await collectInsertCounts(client);
}
return counts;
} catch (error) {
console.error(
`Cannot reach the contract database at PG_CONTRACT_URL: ${error.message}. ` +
'Without it there is no evidence the suites touched a real PostgreSQL.',
);
process.exit(2);
} finally {
await client.end().catch(() => {});
}
}
if (capturingBaseline) {
const baseline = await readCounters();
writeFileSync(baselinePath, `${JSON.stringify(baseline, null, 2)}\n`);
const summary = REQUIRED_TABLES.map((table) => `${table}=${baseline[table]?.inserts ?? 0}`);
console.log(`Captured contract-database baseline to ${baselinePath}: ${summary.join(' ')}.`);
process.exit(0);
}
let results;
try {
results = JSON.parse(readFileSync(resultsPath, 'utf8'));
} catch (error) {
console.error(`Cannot read the vitest results at ${resultsPath}: ${error.message}`);
process.exit(2);
}
const files = Array.isArray(results.testResults) ? results.testResults : undefined;
if (!files) {
console.error(
`${resultsPath} has no testResults array, so it cannot show which suites ran. ` +
'Was the run invoked with the json reporter?',
);
process.exit(2);
}
const failures = [];
for (const suite of REQUIRED_SUITES) {
const entry = files.find((file) => typeof file.name === 'string' && file.name.endsWith(suite));
if (!entry) {
failures.push(
`${suite} did not run, so the PostgreSQL contract is unverified. ` +
"Check vitest's `include`/`exclude` in packages/@openmaic/storage/vitest.config.ts.",
);
continue;
}
const cases = Array.isArray(entry.assertionResults) ? entry.assertionResults : [];
const passed = cases.filter((testCase) => testCase.status === 'passed').length;
const pending = cases.filter((testCase) => testCase.status !== 'passed');
if (entry.status !== 'passed') {
failures.push(`${suite} reported status "${entry.status}".`);
continue;
}
if (passed === 0) {
failures.push(`${suite} ran but reported no passing test cases, so it asserted nothing.`);
continue;
}
if (pending.length > 0) {
failures.push(
`${suite} left ${pending.length} test case(s) not passing ` +
`(${[...new Set(pending.map((testCase) => testCase.status))].join(', ')}).`,
);
continue;
}
console.log(`${suite}: ${passed} test cases ran and passed.`);
}
if (failures.length > 0) {
console.error(
[
'The PostgreSQL contract suites did not run as required:',
...failures.map((failure) => `- ${failure}`),
].join('\n'),
);
process.exit(1);
}
let baseline;
try {
baseline = JSON.parse(readFileSync(baselinePath, 'utf8'));
} catch (error) {
console.error(
`Cannot read the pre-run baseline at ${baselinePath}: ${error.message}. Capture it with ` +
'`--capture-baseline` before the vitest step; without it, counters left by an earlier ' +
'run against a non-ephemeral database would satisfy this check forever.',
);
process.exit(2);
}
const counts = await readCounters({ waitFor: baseline });
const databaseFailures = [];
for (const table of REQUIRED_TABLES) {
const observed = counts[table];
const before = baseline[table]?.inserts ?? 0;
if (!observed?.present) {
databaseFailures.push(
`${table} does not exist in the contract database, so the suites never created it ` +
'and did not run against this PostgreSQL.',
);
continue;
}
const gained = observed.inserts - before;
if (!(gained > 0)) {
databaseFailures.push(
`${table} exists but gained no inserts during this run (before ${before}, after ` +
`${observed.inserts}), so the suites did not write to this database. Check for a ` +
'mocked driver in packages/@openmaic/storage/test/setup.ts, or a stubbed-out suite body.',
);
continue;
}
console.log(`${table}: ${gained} inserts during this run (${before} -> ${observed.inserts}).`);
}
if (databaseFailures.length > 0) {
console.error(
[
'The contract database shows no evidence that these suites wrote to it during this run:',
...databaseFailures.map((failure) => `- ${failure}`),
].join('\n'),
);
process.exit(1);
}
console.log(
`Verified: the required contract suite files ran and passed, and all ${REQUIRED_TABLES.length} ` +
'tables the PostgreSQL backends own gained inserts in a real database during this run. ' +
'This does not attribute those inserts to the built PgDocumentStore / PgRuntimeStore ' +
'specifically — see the threat model at the top of this script.',
);