116 lines
4.8 KiB
JavaScript
116 lines
4.8 KiB
JavaScript
// Conformance harness — zero dependencies (node:test not even needed:
|
|
// entries are data, not code). See conformance/README notes in SURFACE.md.
|
|
//
|
|
// Modes:
|
|
// --all run every corpus entry; informational; exit 0 unless the
|
|
// harness itself errors (malformed entry, unreadable corpus)
|
|
// --ratified run only entries with "status": "ratified"; exit 0 iff all
|
|
// pass; exits 0 vacuously when zero entries are ratified.
|
|
// This is what CI gates on.
|
|
//
|
|
// Every case is executed twice and the observations compared; a mismatch is
|
|
// a determinism failure (NONDET) and fails the case.
|
|
//
|
|
// Expectations (exactly one per entry):
|
|
// expected.out — exact stdout, single trailing newline normalized
|
|
// expected.err — the interpreter's error KEY (e.g. err_div0), one per
|
|
// line; never localized message text
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { createRequire } from 'node:module';
|
|
|
|
// Same loader mechanism as js/test/load.mjs: the artifact is CommonJS-
|
|
// guarded; the loader adapts, the artifact is never modified.
|
|
const require = createRequire(import.meta.url);
|
|
const { runMPL } = require('../../js/mpl.js');
|
|
|
|
const CORPUS = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'corpus');
|
|
|
|
function harnessError(msg) {
|
|
console.error(`harness error: ${msg}`);
|
|
process.exit(2);
|
|
}
|
|
|
|
// One observation: printed lines plus the error key (null if none thrown).
|
|
// A keyless exception is recorded distinctly — it is a host-level crash,
|
|
// not an MPL error, and can never match an expected.err key.
|
|
function observe(src) {
|
|
const lines = [];
|
|
try {
|
|
runMPL(src, s => lines.push(s));
|
|
return { lines, key: null };
|
|
} catch (e) {
|
|
return { lines, key: e.key || `UNKEYED:${e.constructor.name}` };
|
|
}
|
|
}
|
|
|
|
// Exact stdout with a single trailing newline normalized on both sides.
|
|
const norm = s => (s === '' ? '' : s.replace(/\n*$/, '') + '\n');
|
|
|
|
function loadEntry(dir) {
|
|
const p = n => path.join(CORPUS, dir, n);
|
|
if (!fs.existsSync(p('program.mpl'))) harnessError(`${dir}: missing program.mpl`);
|
|
if (!fs.existsSync(p('meta.json'))) harnessError(`${dir}: missing meta.json`);
|
|
let meta;
|
|
try { meta = JSON.parse(fs.readFileSync(p('meta.json'), 'utf8')); }
|
|
catch (e) { harnessError(`${dir}: bad meta.json: ${e.message}`); }
|
|
if (meta.status !== 'unratified' && meta.status !== 'ratified') {
|
|
harnessError(`${dir}: meta.status must be "unratified" or "ratified"`);
|
|
}
|
|
const hasOut = fs.existsSync(p('expected.out'));
|
|
const hasErr = fs.existsSync(p('expected.err'));
|
|
if (hasOut === hasErr) harnessError(`${dir}: need exactly one of expected.out / expected.err`);
|
|
return {
|
|
dir,
|
|
meta,
|
|
program: fs.readFileSync(p('program.mpl'), 'utf8'),
|
|
out: hasOut ? fs.readFileSync(p('expected.out'), 'utf8') : null,
|
|
err: hasErr ? fs.readFileSync(p('expected.err'), 'utf8').split('\n').filter(Boolean) : null,
|
|
};
|
|
}
|
|
|
|
function runEntry(e) {
|
|
const first = observe(e.program);
|
|
const second = observe(e.program);
|
|
if (JSON.stringify(first) !== JSON.stringify(second)) {
|
|
return { verdict: 'NONDET', detail: 'two runs disagreed' };
|
|
}
|
|
if (e.out !== null) {
|
|
if (first.key !== null) return { verdict: 'FAIL', detail: `threw ${first.key}, expected output` };
|
|
const actual = first.lines.length ? first.lines.join('\n') + '\n' : '';
|
|
if (norm(actual) !== norm(e.out)) {
|
|
return { verdict: 'FAIL', detail: `output ${JSON.stringify(actual)} != expected ${JSON.stringify(norm(e.out))}` };
|
|
}
|
|
return { verdict: 'PASS' };
|
|
}
|
|
const keys = first.key === null ? [] : [first.key];
|
|
if (JSON.stringify(keys) !== JSON.stringify(e.err)) {
|
|
return { verdict: 'FAIL', detail: `error keys ${JSON.stringify(keys)} != expected ${JSON.stringify(e.err)}` };
|
|
}
|
|
return { verdict: 'PASS' };
|
|
}
|
|
|
|
const mode = process.argv[2];
|
|
if (mode !== '--all' && mode !== '--ratified') {
|
|
harnessError('usage: node conformance/harness/run.mjs --all | --ratified');
|
|
}
|
|
|
|
if (!fs.existsSync(CORPUS)) harnessError(`no corpus directory at ${CORPUS}`);
|
|
let dirs = fs.readdirSync(CORPUS).filter(d => fs.statSync(path.join(CORPUS, d)).isDirectory()).sort();
|
|
const entries = dirs.map(loadEntry);
|
|
const selected = mode === '--ratified' ? entries.filter(e => e.meta.status === 'ratified') : entries;
|
|
|
|
if (mode === '--ratified' && selected.length === 0) {
|
|
console.log('0 entries ratified — vacuously green (ratification is Stage 3).');
|
|
process.exit(0);
|
|
}
|
|
|
|
let pass = 0, fail = 0;
|
|
for (const e of selected) {
|
|
const r = runEntry(e);
|
|
if (r.verdict === 'PASS') pass++; else fail++;
|
|
console.log(`${r.verdict} ${e.dir}${r.detail ? ' — ' + r.detail : ''}`);
|
|
}
|
|
console.log(`${selected.length} cases: ${pass} pass, ${fail} fail (${entries.length - selected.length} not selected)`);
|
|
process.exit(mode === '--ratified' && fail > 0 ? 1 : 0);
|