ci: measure native bootstrap while preserving workspace features (#284)

* ci: guard a bounded native geometry measurement with exact planned closure

* ci: preserve workspace features while measuring native browser bootstrap
This commit is contained in:
FredApps authored and GitHub committed 2026-11-19 12:20:14 +07:00
1 parent b5c6f5913e
commit 0be61baefd
3 files changed
+268

No files matched your search

@@ -0,0 +1,133 @@
name: Photo FreeBSD browser bootstrap measurement
on:
pull_request:
paths:
- .github/workflows/photo-boot-measurement.yml
- scripts/compare_cargo_unit_closure.py
- tests/test_compare_cargo_unit_closure.py
permissions:
contents: read
jobs:
bootstrap:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a
with:
egress-policy: audit
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
- run: python3 -m unittest discover -s tests -p test_compare_cargo_unit_closure.py -v
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
repository: storytold/photocraft
ref: 057297bb0b0872244ac0910e640db8d87e46572f
path: photo
persist-credentials: false
- name: Verify pinned checkout before VM transfer
run: |
test "$(git -C photo rev-parse HEAD)" = 057297bb0b0872244ac0910e640db8d87e46572f
python3 - <<'PYCODE'
import hashlib, json, os, pathlib, subprocess
names = subprocess.check_output(['git', '-C', 'photo', 'ls-files', '-z']).split(bytes([0]))
identity = {}
for raw_name in filter(None, names):
name = os.fsdecode(raw_name)
path = pathlib.Path('photo') / name
kind = 'symlink' if path.is_symlink() else 'file'
data = os.readlink(path).encode() if kind == 'symlink' else path.read_bytes()
identity[str(path)] = [kind, hashlib.sha256(data).hexdigest()]
assert identity, 'empty source inventory'
pathlib.Path('photo-source-identity.json').write_text(json.dumps(identity, sort_keys=True))
PYCODE
- name: Prepare native FreeBSD host, without application compilation
uses: step-security/freebsd-vm@a2f5754275f7b0d5cea7f2762c9960727950f128
with:
release: '15.1'
usesh: 'true'
cpu: '4'
mem: '12288'
copyback: 'false'
prepare: pkg install -y git curl python3
- name: Install official pinned nightly planner and stable compiler
shell: freebsd {0}
run: |
set -eu
test "$(uname -s)" = FreeBSD
curl --proto '=https' --tlsv1.2 --retry 2 --connect-timeout 15 --max-time 60 -fsS https://sh.rustup.rs -o /tmp/photo-install-rustup.sh
timeout 180s sh /tmp/photo-install-rustup.sh -y --profile minimal --default-toolchain nightly-2026-11-18 --no-modify-path
"$HOME/.cargo/bin/cargo" +nightly-2026-11-18 -Vv
"$HOME/.cargo/bin/rustc" +nightly-2026-11-18 -Vv
timeout 180s "$HOME/.cargo/bin/rustup" toolchain install 1.100.0 --profile minimal
"$HOME/.cargo/bin/rustc" +1.100.0 -Vv
- name: Verify planned closure, then measure only native browser bootstrap
shell: freebsd {0}
run: |
set -eu
export PATH="$HOME/.cargo/bin:$PATH"
cd photo
python3 - <<'PYCODE'
import hashlib, json, os, pathlib
identity = json.loads(pathlib.Path('../photo-source-identity.json').read_text())
for name, expected in identity.items():
path = pathlib.Path('..') / name
kind = 'symlink' if path.is_symlink() else 'file'
data = os.readlink(path).encode() if kind == 'symlink' else path.read_bytes()
assert [kind, hashlib.sha256(data).hexdigest()] == expected, name
print('All tracked source files match host-verified pinned checkout')
PYCODE
sha256 Cargo.toml Cargo.lock
mkdir /tmp/photo-unit-graphs
timeout 60s cargo +nightly-2026-11-18 metadata --offline --no-deps --format-version 1 > /tmp/photo-unit-graphs/metadata.json
timeout 120s cargo +nightly-2026-11-18 test --locked --workspace --features heif --target x86_64-unknown-freebsd --unit-graph -Z unstable-options > /tmp/photo-unit-graphs/full.json
timeout 120s cargo +nightly-2026-11-18 test --locked --workspace --features heif --bin photocraft-web-boot --target x86_64-unknown-freebsd --unit-graph -Z unstable-options > /tmp/photo-unit-graphs/isolated.json
timeout 30s python3 ../scripts/compare_cargo_unit_closure.py /tmp/photo-unit-graphs/full.json /tmp/photo-unit-graphs/isolated.json /tmp/photo-unit-graphs/metadata.json photocraft-web-boot bin
echo 'Stable compile begins: browser bootstrap only; not the full application'
/usr/bin/time -l timeout 420s cargo +1.100.0 test --locked --workspace --features heif --bin photocraft-web-boot --target x86_64-unknown-freebsd --no-run --jobs 2 --message-format=json > /tmp/photo-boot-artifacts.jsonl
cat /tmp/photo-boot-artifacts.jsonl
python3 - <<'PYCODE'
import json, pathlib
metadata = json.loads(pathlib.Path('/tmp/photo-unit-graphs/metadata.json').read_text())
package_ids = [p['id'] for p in metadata['packages'] if p['name'] == 'photocraft-web-boot']
assert len(package_ids) == 1
artifacts = [json.loads(line) for line in pathlib.Path('/tmp/photo-boot-artifacts.jsonl').read_text().splitlines()]
executables = [a['executable'] for a in artifacts if a.get('reason') == 'compiler-artifact'
and a.get('package_id') == package_ids[0] and a.get('profile', {}).get('test')
and a.get('target', {}).get('kind') == ['bin'] and a.get('executable')]
assert len(executables) == 1, executables
executable = pathlib.Path(executables[0])
assert executable.is_file()
pathlib.Path('/tmp/photo-boot-executable').write_text(str(executable))
PYCODE
executable="$(cat /tmp/photo-boot-executable)"
sha256 "$executable"
timeout 30s "$executable" --list --format pretty --color never > /tmp/photo-boot-list.txt
cat /tmp/photo-boot-list.txt
/usr/bin/time -l timeout 60s "$executable" --test-threads=1 --format pretty --color never > /tmp/photo-boot-result.txt
cat /tmp/photo-boot-result.txt
python3 - <<'PYCODE'
import pathlib, re
listed = pathlib.Path('/tmp/photo-boot-list.txt').read_text()
result = pathlib.Path('/tmp/photo-boot-result.txt').read_text()
totals = re.findall(r'(?m)^(\d+) tests?, (\d+) benchmarks?$', listed)
outcomes = re.findall(r'test result: ok\. (\d+) passed; (\d+) failed; (\d+) ignored; (\d+) measured; (\d+) filtered out;', result)
assert len(totals) == len(outcomes) == 1, 'missing or ambiguous libtest summary'
total, benches = map(int, totals[0])
passed, failed, ignored, measured, filtered = map(int, outcomes[0])
assert total > 0 and passed > 0 and total == passed + ignored
assert not (benches or failed or measured or filtered)
print(f'Validated nonzero native bootstrap coverage: {passed} passed, {ignored} ignored')
PYCODE
python3 - <<'PYCODE'
import hashlib, json, os, pathlib
identity = json.loads(pathlib.Path('../photo-source-identity.json').read_text())
for name, expected in identity.items():
path = pathlib.Path('..') / name
kind = 'symlink' if path.is_symlink() else 'file'
data = os.readlink(path).encode() if kind == 'symlink' else path.read_bytes()
assert [kind, hashlib.sha256(data).hexdigest()] == expected, name
print('Planning left all tracked source files unchanged')
PYCODE
echo 'Only browser-bootstrap binary libtest compiled/executed. No UI, application, doctest or full-gate coverage claim.'
echo 'Nightly closure equality is planned evidence only; stable measurement is limited to browser bootstrap.'
+65
View File
@@ -0,0 +1,65 @@
"""Fail unless an isolated libtest plan matches its full-workspace planned closure.
This is a nightly planning prerequisite for a narrow stable compile measurement,
not proof that nightly planning equals production stable compilation.
"""
import copy
import sys
from compare_cargo_units import canonical, compare, read_json
def closure(graph, package_id, kind='lib'):
if kind not in ('lib', 'bin'):
raise ValueError('unsupported target kind')
units = graph['units']
if any(type(i) is not int or not 0 <= i < len(units) for i in graph['roots']):
raise ValueError('invalid root index')
roots = [i for i in graph['roots'] if units[i]['pkg_id'] == package_id
and units[i]['mode'] == 'test' and units[i]['target']['kind'] == [kind]]
if len(roots) != 1:
raise ValueError('expected exactly one matching libtest root')
reached, pending = set(), list(roots)
while pending:
index = pending.pop()
if type(index) is not int or not 0 <= index < len(units):
raise ValueError('invalid unit index')
if index in reached:
continue
reached.add(index)
pending.extend(d['index'] for d in units[index]['dependencies'])
indexes = sorted(reached)
remap = {old: new for new, old in enumerate(indexes)}
result = {k: copy.deepcopy(v) for k, v in graph.items() if k not in ('roots', 'units')}
result['roots'] = [remap[i] for i in roots]
result['units'] = [copy.deepcopy(units[i]) for i in indexes]
for unit in result['units']:
for dependency in unit['dependencies']:
dependency['index'] = remap[dependency['index']]
return result
def compare_closure(full, isolated, package_id, kind='lib'):
expected = closure(full, package_id, kind)
# The isolated invocation must itself contain only that one root/closure.
actual = closure(isolated, package_id, kind)
if len(isolated['roots']) != 1 or len(actual['units']) != len(isolated['units']):
raise ValueError('isolated graph contains unexpected roots or unused units')
result, _, _ = compare(expected, isolated, set())
if not result['planned_graph_equal']:
raise ValueError('isolated planned closure differs: ' + canonical(result))
return result
def main():
full, isolated, metadata = map(read_json, sys.argv[1:4])
package, kind = sys.argv[4:6]
matches = [p['id'] for p in metadata['packages'] if p['name'] == package]
if len(matches) != 1:
raise ValueError('missing or ambiguous package')
result = compare_closure(full, isolated, matches[0], kind)
print(canonical({'package': matches[0], 'planned_closure': result}))
print('Nightly closure only; subsequent stable compilation measures one target, not full gate.')
if __name__ == '__main__':
main()
+70
View File
@@ -0,0 +1,70 @@
import copy
import pathlib
import sys
import unittest
import tempfile
import contextlib
import io
sys.path.insert(0, str(pathlib.Path(__file__).parents[1] / 'scripts'))
from compare_cargo_unit_closure import compare_closure, closure
def graph():
return {'version': 1, 'roots': [1, 2], 'units': [
{'pkg_id': 'dep', 'mode': 'build', 'target': {'kind': ['lib']}, 'features': ['x'], 'dependencies': []},
{'pkg_id': 'geom', 'mode': 'test', 'target': {'kind': ['lib']}, 'dependencies': [{'index': 0}]},
{'pkg_id': 'other', 'mode': 'test', 'target': {'kind': ['bin']}, 'dependencies': [{'index': 0}]},
]}
class ClosureTests(unittest.TestCase):
def test_extract_and_compare(self):
full = graph(); isolated = closure(full, 'geom')
self.assertEqual(len(isolated['units']), 2)
self.assertTrue(compare_closure(full, isolated, 'geom')['planned_graph_equal'])
def test_binary_target(self):
full = graph(); isolated = closure(full, 'other', 'bin')
self.assertTrue(compare_closure(full, isolated, 'other', 'bin')['planned_graph_equal'])
with self.assertRaises(ValueError): closure(full, 'other', 'lib')
def test_feature_difference_rejected(self):
full = graph(); isolated = closure(full, 'geom'); isolated['units'][0]['features'] = ['other']
with self.assertRaises(ValueError): compare_closure(full, isolated, 'geom')
def test_unexpected_root_or_orphan_rejected(self):
full = graph()
with self.assertRaises(ValueError): compare_closure(full, full, 'geom')
isolated = closure(full, 'geom'); isolated['units'].append(copy.deepcopy(isolated['units'][0]))
with self.assertRaises(ValueError): compare_closure(full, isolated, 'geom')
def test_invalid_root_rejected(self):
for index in [True, -1, 99]:
full = graph(); full['roots'] = [index]
with self.assertRaises(ValueError): closure(full, 'geom')
def test_runtime_summary_rejects_zero_or_incomplete_coverage(self):
workflow = (pathlib.Path(__file__).parents[1] / '.github/workflows/photo-boot-measurement.yml').read_text()
blocks = workflow.split("<<'PYCODE'\n")[1:]
block = next(part for part in blocks if 'totals = re.findall' in part)
source = '\n'.join(line[10:] for line in block.split(' PYCODE', 1)[0].splitlines())
with tempfile.TemporaryDirectory() as directory:
listing = pathlib.Path(directory) / 'list.txt'
result = pathlib.Path(directory) / 'result.txt'
source = source.replace('/tmp/photo-boot-list.txt', str(listing)).replace('/tmp/photo-boot-result.txt', str(result))
listing.write_text('a: test\nb: test\n\n2 tests, 0 benchmarks\n')
result.write_text('test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s\n')
with contextlib.redirect_stdout(io.StringIO()): exec(source, {})
for listed, outcome in [
('0 tests, 0 benchmarks\n', 'test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out;'),
('2 tests, 0 benchmarks\n', 'test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 1 filtered out;'),
('2 tests, 0 benchmarks\n', 'missing result'),
]:
listing.write_text(listed); result.write_text(outcome)
with self.assertRaises(AssertionError): exec(source, {})
def test_missing_root_rejected(self):
with self.assertRaises(ValueError): closure(graph(), 'missing')
if __name__ == '__main__': unittest.main()