diff --git a/.github/workflows/photo-boot-measurement.yml b/.github/workflows/photo-boot-measurement.yml new file mode 100644 index 0000000..54239be --- /dev/null +++ b/.github/workflows/photo-boot-measurement.yml @@ -0,0 +1,133 @@ +name: Photo FreeBSD browser bootstrap measurement +on: + pull_request: + paths: + - .github/workflows/photo-boot-measurement.yml + - scripts/compare_cargo_unit_closure.py + - tests/test_compare_cargo_unit_closure.py +permissions: + contents: read +jobs: + bootstrap: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a + with: + egress-policy: audit + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - run: python3 -m unittest discover -s tests -p test_compare_cargo_unit_closure.py -v + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + repository: storytold/photocraft + ref: 057297bb0b0872244ac0910e640db8d87e46572f + path: photo + persist-credentials: false + - name: Verify pinned checkout before VM transfer + run: | + test "$(git -C photo rev-parse HEAD)" = 057297bb0b0872244ac0910e640db8d87e46572f + python3 - <<'PYCODE' + import hashlib, json, os, pathlib, subprocess + names = subprocess.check_output(['git', '-C', 'photo', 'ls-files', '-z']).split(bytes([0])) + identity = {} + for raw_name in filter(None, names): + name = os.fsdecode(raw_name) + path = pathlib.Path('photo') / name + kind = 'symlink' if path.is_symlink() else 'file' + data = os.readlink(path).encode() if kind == 'symlink' else path.read_bytes() + identity[str(path)] = [kind, hashlib.sha256(data).hexdigest()] + assert identity, 'empty source inventory' + pathlib.Path('photo-source-identity.json').write_text(json.dumps(identity, sort_keys=True)) + PYCODE + - name: Prepare native FreeBSD host, without application compilation + uses: step-security/freebsd-vm@a2f5754275f7b0d5cea7f2762c9960727950f128 + with: + release: '15.1' + usesh: 'true' + cpu: '4' + mem: '12288' + copyback: 'false' + prepare: pkg install -y git curl python3 + - name: Install official pinned nightly planner and stable compiler + shell: freebsd {0} + run: | + set -eu + test "$(uname -s)" = FreeBSD + curl --proto '=https' --tlsv1.2 --retry 2 --connect-timeout 15 --max-time 60 -fsS https://sh.rustup.rs -o /tmp/photo-install-rustup.sh + timeout 180s sh /tmp/photo-install-rustup.sh -y --profile minimal --default-toolchain nightly-2026-11-18 --no-modify-path + "$HOME/.cargo/bin/cargo" +nightly-2026-11-18 -Vv + "$HOME/.cargo/bin/rustc" +nightly-2026-11-18 -Vv + timeout 180s "$HOME/.cargo/bin/rustup" toolchain install 1.100.0 --profile minimal + "$HOME/.cargo/bin/rustc" +1.100.0 -Vv + - name: Verify planned closure, then measure only native browser bootstrap + shell: freebsd {0} + run: | + set -eu + export PATH="$HOME/.cargo/bin:$PATH" + cd photo + python3 - <<'PYCODE' + import hashlib, json, os, pathlib + identity = json.loads(pathlib.Path('../photo-source-identity.json').read_text()) + for name, expected in identity.items(): + path = pathlib.Path('..') / name + kind = 'symlink' if path.is_symlink() else 'file' + data = os.readlink(path).encode() if kind == 'symlink' else path.read_bytes() + assert [kind, hashlib.sha256(data).hexdigest()] == expected, name + print('All tracked source files match host-verified pinned checkout') + PYCODE + sha256 Cargo.toml Cargo.lock + mkdir /tmp/photo-unit-graphs + timeout 60s cargo +nightly-2026-11-18 metadata --offline --no-deps --format-version 1 > /tmp/photo-unit-graphs/metadata.json + timeout 120s cargo +nightly-2026-11-18 test --locked --workspace --features heif --target x86_64-unknown-freebsd --unit-graph -Z unstable-options > /tmp/photo-unit-graphs/full.json + timeout 120s cargo +nightly-2026-11-18 test --locked --workspace --features heif --bin photocraft-web-boot --target x86_64-unknown-freebsd --unit-graph -Z unstable-options > /tmp/photo-unit-graphs/isolated.json + timeout 30s python3 ../scripts/compare_cargo_unit_closure.py /tmp/photo-unit-graphs/full.json /tmp/photo-unit-graphs/isolated.json /tmp/photo-unit-graphs/metadata.json photocraft-web-boot bin + echo 'Stable compile begins: browser bootstrap only; not the full application' + /usr/bin/time -l timeout 420s cargo +1.100.0 test --locked --workspace --features heif --bin photocraft-web-boot --target x86_64-unknown-freebsd --no-run --jobs 2 --message-format=json > /tmp/photo-boot-artifacts.jsonl + cat /tmp/photo-boot-artifacts.jsonl + python3 - <<'PYCODE' + import json, pathlib + metadata = json.loads(pathlib.Path('/tmp/photo-unit-graphs/metadata.json').read_text()) + package_ids = [p['id'] for p in metadata['packages'] if p['name'] == 'photocraft-web-boot'] + assert len(package_ids) == 1 + artifacts = [json.loads(line) for line in pathlib.Path('/tmp/photo-boot-artifacts.jsonl').read_text().splitlines()] + executables = [a['executable'] for a in artifacts if a.get('reason') == 'compiler-artifact' + and a.get('package_id') == package_ids[0] and a.get('profile', {}).get('test') + and a.get('target', {}).get('kind') == ['bin'] and a.get('executable')] + assert len(executables) == 1, executables + executable = pathlib.Path(executables[0]) + assert executable.is_file() + pathlib.Path('/tmp/photo-boot-executable').write_text(str(executable)) + PYCODE + executable="$(cat /tmp/photo-boot-executable)" + sha256 "$executable" + timeout 30s "$executable" --list --format pretty --color never > /tmp/photo-boot-list.txt + cat /tmp/photo-boot-list.txt + /usr/bin/time -l timeout 60s "$executable" --test-threads=1 --format pretty --color never > /tmp/photo-boot-result.txt + cat /tmp/photo-boot-result.txt + python3 - <<'PYCODE' + import pathlib, re + listed = pathlib.Path('/tmp/photo-boot-list.txt').read_text() + result = pathlib.Path('/tmp/photo-boot-result.txt').read_text() + totals = re.findall(r'(?m)^(\d+) tests?, (\d+) benchmarks?$', listed) + outcomes = re.findall(r'test result: ok\. (\d+) passed; (\d+) failed; (\d+) ignored; (\d+) measured; (\d+) filtered out;', result) + assert len(totals) == len(outcomes) == 1, 'missing or ambiguous libtest summary' + total, benches = map(int, totals[0]) + passed, failed, ignored, measured, filtered = map(int, outcomes[0]) + assert total > 0 and passed > 0 and total == passed + ignored + assert not (benches or failed or measured or filtered) + print(f'Validated nonzero native bootstrap coverage: {passed} passed, {ignored} ignored') + PYCODE + python3 - <<'PYCODE' + import hashlib, json, os, pathlib + identity = json.loads(pathlib.Path('../photo-source-identity.json').read_text()) + for name, expected in identity.items(): + path = pathlib.Path('..') / name + kind = 'symlink' if path.is_symlink() else 'file' + data = os.readlink(path).encode() if kind == 'symlink' else path.read_bytes() + assert [kind, hashlib.sha256(data).hexdigest()] == expected, name + print('Planning left all tracked source files unchanged') + PYCODE + echo 'Only browser-bootstrap binary libtest compiled/executed. No UI, application, doctest or full-gate coverage claim.' + echo 'Nightly closure equality is planned evidence only; stable measurement is limited to browser bootstrap.' diff --git a/scripts/compare_cargo_unit_closure.py b/scripts/compare_cargo_unit_closure.py new file mode 100644 index 0000000..662537c --- /dev/null +++ b/scripts/compare_cargo_unit_closure.py @@ -0,0 +1,65 @@ +"""Fail unless an isolated libtest plan matches its full-workspace planned closure. + +This is a nightly planning prerequisite for a narrow stable compile measurement, +not proof that nightly planning equals production stable compilation. +""" +import copy +import sys +from compare_cargo_units import canonical, compare, read_json + + +def closure(graph, package_id, kind='lib'): + if kind not in ('lib', 'bin'): + raise ValueError('unsupported target kind') + units = graph['units'] + if any(type(i) is not int or not 0 <= i < len(units) for i in graph['roots']): + raise ValueError('invalid root index') + roots = [i for i in graph['roots'] if units[i]['pkg_id'] == package_id + and units[i]['mode'] == 'test' and units[i]['target']['kind'] == [kind]] + if len(roots) != 1: + raise ValueError('expected exactly one matching libtest root') + reached, pending = set(), list(roots) + while pending: + index = pending.pop() + if type(index) is not int or not 0 <= index < len(units): + raise ValueError('invalid unit index') + if index in reached: + continue + reached.add(index) + pending.extend(d['index'] for d in units[index]['dependencies']) + indexes = sorted(reached) + remap = {old: new for new, old in enumerate(indexes)} + result = {k: copy.deepcopy(v) for k, v in graph.items() if k not in ('roots', 'units')} + result['roots'] = [remap[i] for i in roots] + result['units'] = [copy.deepcopy(units[i]) for i in indexes] + for unit in result['units']: + for dependency in unit['dependencies']: + dependency['index'] = remap[dependency['index']] + return result + + +def compare_closure(full, isolated, package_id, kind='lib'): + expected = closure(full, package_id, kind) + # The isolated invocation must itself contain only that one root/closure. + actual = closure(isolated, package_id, kind) + if len(isolated['roots']) != 1 or len(actual['units']) != len(isolated['units']): + raise ValueError('isolated graph contains unexpected roots or unused units') + result, _, _ = compare(expected, isolated, set()) + if not result['planned_graph_equal']: + raise ValueError('isolated planned closure differs: ' + canonical(result)) + return result + + +def main(): + full, isolated, metadata = map(read_json, sys.argv[1:4]) + package, kind = sys.argv[4:6] + matches = [p['id'] for p in metadata['packages'] if p['name'] == package] + if len(matches) != 1: + raise ValueError('missing or ambiguous package') + result = compare_closure(full, isolated, matches[0], kind) + print(canonical({'package': matches[0], 'planned_closure': result})) + print('Nightly closure only; subsequent stable compilation measures one target, not full gate.') + + +if __name__ == '__main__': + main() diff --git a/tests/test_compare_cargo_unit_closure.py b/tests/test_compare_cargo_unit_closure.py new file mode 100644 index 0000000..8c75cd5 --- /dev/null +++ b/tests/test_compare_cargo_unit_closure.py @@ -0,0 +1,70 @@ +import copy +import pathlib +import sys +import unittest +import tempfile +import contextlib +import io +sys.path.insert(0, str(pathlib.Path(__file__).parents[1] / 'scripts')) +from compare_cargo_unit_closure import compare_closure, closure + + +def graph(): + return {'version': 1, 'roots': [1, 2], 'units': [ + {'pkg_id': 'dep', 'mode': 'build', 'target': {'kind': ['lib']}, 'features': ['x'], 'dependencies': []}, + {'pkg_id': 'geom', 'mode': 'test', 'target': {'kind': ['lib']}, 'dependencies': [{'index': 0}]}, + {'pkg_id': 'other', 'mode': 'test', 'target': {'kind': ['bin']}, 'dependencies': [{'index': 0}]}, + ]} + + +class ClosureTests(unittest.TestCase): + def test_extract_and_compare(self): + full = graph(); isolated = closure(full, 'geom') + self.assertEqual(len(isolated['units']), 2) + self.assertTrue(compare_closure(full, isolated, 'geom')['planned_graph_equal']) + + def test_binary_target(self): + full = graph(); isolated = closure(full, 'other', 'bin') + self.assertTrue(compare_closure(full, isolated, 'other', 'bin')['planned_graph_equal']) + with self.assertRaises(ValueError): closure(full, 'other', 'lib') + + def test_feature_difference_rejected(self): + full = graph(); isolated = closure(full, 'geom'); isolated['units'][0]['features'] = ['other'] + with self.assertRaises(ValueError): compare_closure(full, isolated, 'geom') + + def test_unexpected_root_or_orphan_rejected(self): + full = graph() + with self.assertRaises(ValueError): compare_closure(full, full, 'geom') + isolated = closure(full, 'geom'); isolated['units'].append(copy.deepcopy(isolated['units'][0])) + with self.assertRaises(ValueError): compare_closure(full, isolated, 'geom') + + def test_invalid_root_rejected(self): + for index in [True, -1, 99]: + full = graph(); full['roots'] = [index] + with self.assertRaises(ValueError): closure(full, 'geom') + + def test_runtime_summary_rejects_zero_or_incomplete_coverage(self): + workflow = (pathlib.Path(__file__).parents[1] / '.github/workflows/photo-boot-measurement.yml').read_text() + blocks = workflow.split("<<'PYCODE'\n")[1:] + block = next(part for part in blocks if 'totals = re.findall' in part) + source = '\n'.join(line[10:] for line in block.split(' PYCODE', 1)[0].splitlines()) + with tempfile.TemporaryDirectory() as directory: + listing = pathlib.Path(directory) / 'list.txt' + result = pathlib.Path(directory) / 'result.txt' + source = source.replace('/tmp/photo-boot-list.txt', str(listing)).replace('/tmp/photo-boot-result.txt', str(result)) + listing.write_text('a: test\nb: test\n\n2 tests, 0 benchmarks\n') + result.write_text('test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s\n') + with contextlib.redirect_stdout(io.StringIO()): exec(source, {}) + for listed, outcome in [ + ('0 tests, 0 benchmarks\n', 'test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out;'), + ('2 tests, 0 benchmarks\n', 'test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 1 filtered out;'), + ('2 tests, 0 benchmarks\n', 'missing result'), + ]: + listing.write_text(listed); result.write_text(outcome) + with self.assertRaises(AssertionError): exec(source, {}) + + def test_missing_root_rejected(self): + with self.assertRaises(ValueError): closure(graph(), 'missing') + + +if __name__ == '__main__': unittest.main()