mirror of
https://github.com/storytold/craft-common-actions.git
synced 2026-11-20 09:32:43 +01:00
ci: measure native bootstrap while preserving workspace features (#284)
* ci: guard a bounded native geometry measurement with exact planned closure * ci: preserve workspace features while measuring native browser bootstrap
This commit is contained in:
1 parent
b5c6f5913e
commit
0be61baefd
3 files changed
+268
No files matched your search
@@ -0,0 +1,133 @@
|
||||
name: Photo FreeBSD browser bootstrap measurement
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- .github/workflows/photo-boot-measurement.yml
|
||||
- scripts/compare_cargo_unit_closure.py
|
||||
- tests/test_compare_cargo_unit_closure.py
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
bootstrap:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a
|
||||
with:
|
||||
egress-policy: audit
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- run: python3 -m unittest discover -s tests -p test_compare_cargo_unit_closure.py -v
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
|
||||
with:
|
||||
repository: storytold/photocraft
|
||||
ref: 057297bb0b0872244ac0910e640db8d87e46572f
|
||||
path: photo
|
||||
persist-credentials: false
|
||||
- name: Verify pinned checkout before VM transfer
|
||||
run: |
|
||||
test "$(git -C photo rev-parse HEAD)" = 057297bb0b0872244ac0910e640db8d87e46572f
|
||||
python3 - <<'PYCODE'
|
||||
import hashlib, json, os, pathlib, subprocess
|
||||
names = subprocess.check_output(['git', '-C', 'photo', 'ls-files', '-z']).split(bytes([0]))
|
||||
identity = {}
|
||||
for raw_name in filter(None, names):
|
||||
name = os.fsdecode(raw_name)
|
||||
path = pathlib.Path('photo') / name
|
||||
kind = 'symlink' if path.is_symlink() else 'file'
|
||||
data = os.readlink(path).encode() if kind == 'symlink' else path.read_bytes()
|
||||
identity[str(path)] = [kind, hashlib.sha256(data).hexdigest()]
|
||||
assert identity, 'empty source inventory'
|
||||
pathlib.Path('photo-source-identity.json').write_text(json.dumps(identity, sort_keys=True))
|
||||
PYCODE
|
||||
- name: Prepare native FreeBSD host, without application compilation
|
||||
uses: step-security/freebsd-vm@a2f5754275f7b0d5cea7f2762c9960727950f128
|
||||
with:
|
||||
release: '15.1'
|
||||
usesh: 'true'
|
||||
cpu: '4'
|
||||
mem: '12288'
|
||||
copyback: 'false'
|
||||
prepare: pkg install -y git curl python3
|
||||
- name: Install official pinned nightly planner and stable compiler
|
||||
shell: freebsd {0}
|
||||
run: |
|
||||
set -eu
|
||||
test "$(uname -s)" = FreeBSD
|
||||
curl --proto '=https' --tlsv1.2 --retry 2 --connect-timeout 15 --max-time 60 -fsS https://sh.rustup.rs -o /tmp/photo-install-rustup.sh
|
||||
timeout 180s sh /tmp/photo-install-rustup.sh -y --profile minimal --default-toolchain nightly-2026-11-18 --no-modify-path
|
||||
"$HOME/.cargo/bin/cargo" +nightly-2026-11-18 -Vv
|
||||
"$HOME/.cargo/bin/rustc" +nightly-2026-11-18 -Vv
|
||||
timeout 180s "$HOME/.cargo/bin/rustup" toolchain install 1.100.0 --profile minimal
|
||||
"$HOME/.cargo/bin/rustc" +1.100.0 -Vv
|
||||
- name: Verify planned closure, then measure only native browser bootstrap
|
||||
shell: freebsd {0}
|
||||
run: |
|
||||
set -eu
|
||||
export PATH="$HOME/.cargo/bin:$PATH"
|
||||
cd photo
|
||||
python3 - <<'PYCODE'
|
||||
import hashlib, json, os, pathlib
|
||||
identity = json.loads(pathlib.Path('../photo-source-identity.json').read_text())
|
||||
for name, expected in identity.items():
|
||||
path = pathlib.Path('..') / name
|
||||
kind = 'symlink' if path.is_symlink() else 'file'
|
||||
data = os.readlink(path).encode() if kind == 'symlink' else path.read_bytes()
|
||||
assert [kind, hashlib.sha256(data).hexdigest()] == expected, name
|
||||
print('All tracked source files match host-verified pinned checkout')
|
||||
PYCODE
|
||||
sha256 Cargo.toml Cargo.lock
|
||||
mkdir /tmp/photo-unit-graphs
|
||||
timeout 60s cargo +nightly-2026-11-18 metadata --offline --no-deps --format-version 1 > /tmp/photo-unit-graphs/metadata.json
|
||||
timeout 120s cargo +nightly-2026-11-18 test --locked --workspace --features heif --target x86_64-unknown-freebsd --unit-graph -Z unstable-options > /tmp/photo-unit-graphs/full.json
|
||||
timeout 120s cargo +nightly-2026-11-18 test --locked --workspace --features heif --bin photocraft-web-boot --target x86_64-unknown-freebsd --unit-graph -Z unstable-options > /tmp/photo-unit-graphs/isolated.json
|
||||
timeout 30s python3 ../scripts/compare_cargo_unit_closure.py /tmp/photo-unit-graphs/full.json /tmp/photo-unit-graphs/isolated.json /tmp/photo-unit-graphs/metadata.json photocraft-web-boot bin
|
||||
echo 'Stable compile begins: browser bootstrap only; not the full application'
|
||||
/usr/bin/time -l timeout 420s cargo +1.100.0 test --locked --workspace --features heif --bin photocraft-web-boot --target x86_64-unknown-freebsd --no-run --jobs 2 --message-format=json > /tmp/photo-boot-artifacts.jsonl
|
||||
cat /tmp/photo-boot-artifacts.jsonl
|
||||
python3 - <<'PYCODE'
|
||||
import json, pathlib
|
||||
metadata = json.loads(pathlib.Path('/tmp/photo-unit-graphs/metadata.json').read_text())
|
||||
package_ids = [p['id'] for p in metadata['packages'] if p['name'] == 'photocraft-web-boot']
|
||||
assert len(package_ids) == 1
|
||||
artifacts = [json.loads(line) for line in pathlib.Path('/tmp/photo-boot-artifacts.jsonl').read_text().splitlines()]
|
||||
executables = [a['executable'] for a in artifacts if a.get('reason') == 'compiler-artifact'
|
||||
and a.get('package_id') == package_ids[0] and a.get('profile', {}).get('test')
|
||||
and a.get('target', {}).get('kind') == ['bin'] and a.get('executable')]
|
||||
assert len(executables) == 1, executables
|
||||
executable = pathlib.Path(executables[0])
|
||||
assert executable.is_file()
|
||||
pathlib.Path('/tmp/photo-boot-executable').write_text(str(executable))
|
||||
PYCODE
|
||||
executable="$(cat /tmp/photo-boot-executable)"
|
||||
sha256 "$executable"
|
||||
timeout 30s "$executable" --list --format pretty --color never > /tmp/photo-boot-list.txt
|
||||
cat /tmp/photo-boot-list.txt
|
||||
/usr/bin/time -l timeout 60s "$executable" --test-threads=1 --format pretty --color never > /tmp/photo-boot-result.txt
|
||||
cat /tmp/photo-boot-result.txt
|
||||
python3 - <<'PYCODE'
|
||||
import pathlib, re
|
||||
listed = pathlib.Path('/tmp/photo-boot-list.txt').read_text()
|
||||
result = pathlib.Path('/tmp/photo-boot-result.txt').read_text()
|
||||
totals = re.findall(r'(?m)^(\d+) tests?, (\d+) benchmarks?$', listed)
|
||||
outcomes = re.findall(r'test result: ok\. (\d+) passed; (\d+) failed; (\d+) ignored; (\d+) measured; (\d+) filtered out;', result)
|
||||
assert len(totals) == len(outcomes) == 1, 'missing or ambiguous libtest summary'
|
||||
total, benches = map(int, totals[0])
|
||||
passed, failed, ignored, measured, filtered = map(int, outcomes[0])
|
||||
assert total > 0 and passed > 0 and total == passed + ignored
|
||||
assert not (benches or failed or measured or filtered)
|
||||
print(f'Validated nonzero native bootstrap coverage: {passed} passed, {ignored} ignored')
|
||||
PYCODE
|
||||
python3 - <<'PYCODE'
|
||||
import hashlib, json, os, pathlib
|
||||
identity = json.loads(pathlib.Path('../photo-source-identity.json').read_text())
|
||||
for name, expected in identity.items():
|
||||
path = pathlib.Path('..') / name
|
||||
kind = 'symlink' if path.is_symlink() else 'file'
|
||||
data = os.readlink(path).encode() if kind == 'symlink' else path.read_bytes()
|
||||
assert [kind, hashlib.sha256(data).hexdigest()] == expected, name
|
||||
print('Planning left all tracked source files unchanged')
|
||||
PYCODE
|
||||
echo 'Only browser-bootstrap binary libtest compiled/executed. No UI, application, doctest or full-gate coverage claim.'
|
||||
echo 'Nightly closure equality is planned evidence only; stable measurement is limited to browser bootstrap.'
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Fail unless an isolated libtest plan matches its full-workspace planned closure.
|
||||
|
||||
This is a nightly planning prerequisite for a narrow stable compile measurement,
|
||||
not proof that nightly planning equals production stable compilation.
|
||||
"""
|
||||
import copy
|
||||
import sys
|
||||
from compare_cargo_units import canonical, compare, read_json
|
||||
|
||||
|
||||
def closure(graph, package_id, kind='lib'):
|
||||
if kind not in ('lib', 'bin'):
|
||||
raise ValueError('unsupported target kind')
|
||||
units = graph['units']
|
||||
if any(type(i) is not int or not 0 <= i < len(units) for i in graph['roots']):
|
||||
raise ValueError('invalid root index')
|
||||
roots = [i for i in graph['roots'] if units[i]['pkg_id'] == package_id
|
||||
and units[i]['mode'] == 'test' and units[i]['target']['kind'] == [kind]]
|
||||
if len(roots) != 1:
|
||||
raise ValueError('expected exactly one matching libtest root')
|
||||
reached, pending = set(), list(roots)
|
||||
while pending:
|
||||
index = pending.pop()
|
||||
if type(index) is not int or not 0 <= index < len(units):
|
||||
raise ValueError('invalid unit index')
|
||||
if index in reached:
|
||||
continue
|
||||
reached.add(index)
|
||||
pending.extend(d['index'] for d in units[index]['dependencies'])
|
||||
indexes = sorted(reached)
|
||||
remap = {old: new for new, old in enumerate(indexes)}
|
||||
result = {k: copy.deepcopy(v) for k, v in graph.items() if k not in ('roots', 'units')}
|
||||
result['roots'] = [remap[i] for i in roots]
|
||||
result['units'] = [copy.deepcopy(units[i]) for i in indexes]
|
||||
for unit in result['units']:
|
||||
for dependency in unit['dependencies']:
|
||||
dependency['index'] = remap[dependency['index']]
|
||||
return result
|
||||
|
||||
|
||||
def compare_closure(full, isolated, package_id, kind='lib'):
|
||||
expected = closure(full, package_id, kind)
|
||||
# The isolated invocation must itself contain only that one root/closure.
|
||||
actual = closure(isolated, package_id, kind)
|
||||
if len(isolated['roots']) != 1 or len(actual['units']) != len(isolated['units']):
|
||||
raise ValueError('isolated graph contains unexpected roots or unused units')
|
||||
result, _, _ = compare(expected, isolated, set())
|
||||
if not result['planned_graph_equal']:
|
||||
raise ValueError('isolated planned closure differs: ' + canonical(result))
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
full, isolated, metadata = map(read_json, sys.argv[1:4])
|
||||
package, kind = sys.argv[4:6]
|
||||
matches = [p['id'] for p in metadata['packages'] if p['name'] == package]
|
||||
if len(matches) != 1:
|
||||
raise ValueError('missing or ambiguous package')
|
||||
result = compare_closure(full, isolated, matches[0], kind)
|
||||
print(canonical({'package': matches[0], 'planned_closure': result}))
|
||||
print('Nightly closure only; subsequent stable compilation measures one target, not full gate.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,70 @@
|
||||
import copy
|
||||
import pathlib
|
||||
import sys
|
||||
import unittest
|
||||
import tempfile
|
||||
import contextlib
|
||||
import io
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).parents[1] / 'scripts'))
|
||||
from compare_cargo_unit_closure import compare_closure, closure
|
||||
|
||||
|
||||
def graph():
|
||||
return {'version': 1, 'roots': [1, 2], 'units': [
|
||||
{'pkg_id': 'dep', 'mode': 'build', 'target': {'kind': ['lib']}, 'features': ['x'], 'dependencies': []},
|
||||
{'pkg_id': 'geom', 'mode': 'test', 'target': {'kind': ['lib']}, 'dependencies': [{'index': 0}]},
|
||||
{'pkg_id': 'other', 'mode': 'test', 'target': {'kind': ['bin']}, 'dependencies': [{'index': 0}]},
|
||||
]}
|
||||
|
||||
|
||||
class ClosureTests(unittest.TestCase):
|
||||
def test_extract_and_compare(self):
|
||||
full = graph(); isolated = closure(full, 'geom')
|
||||
self.assertEqual(len(isolated['units']), 2)
|
||||
self.assertTrue(compare_closure(full, isolated, 'geom')['planned_graph_equal'])
|
||||
|
||||
def test_binary_target(self):
|
||||
full = graph(); isolated = closure(full, 'other', 'bin')
|
||||
self.assertTrue(compare_closure(full, isolated, 'other', 'bin')['planned_graph_equal'])
|
||||
with self.assertRaises(ValueError): closure(full, 'other', 'lib')
|
||||
|
||||
def test_feature_difference_rejected(self):
|
||||
full = graph(); isolated = closure(full, 'geom'); isolated['units'][0]['features'] = ['other']
|
||||
with self.assertRaises(ValueError): compare_closure(full, isolated, 'geom')
|
||||
|
||||
def test_unexpected_root_or_orphan_rejected(self):
|
||||
full = graph()
|
||||
with self.assertRaises(ValueError): compare_closure(full, full, 'geom')
|
||||
isolated = closure(full, 'geom'); isolated['units'].append(copy.deepcopy(isolated['units'][0]))
|
||||
with self.assertRaises(ValueError): compare_closure(full, isolated, 'geom')
|
||||
|
||||
def test_invalid_root_rejected(self):
|
||||
for index in [True, -1, 99]:
|
||||
full = graph(); full['roots'] = [index]
|
||||
with self.assertRaises(ValueError): closure(full, 'geom')
|
||||
|
||||
def test_runtime_summary_rejects_zero_or_incomplete_coverage(self):
|
||||
workflow = (pathlib.Path(__file__).parents[1] / '.github/workflows/photo-boot-measurement.yml').read_text()
|
||||
blocks = workflow.split("<<'PYCODE'\n")[1:]
|
||||
block = next(part for part in blocks if 'totals = re.findall' in part)
|
||||
source = '\n'.join(line[10:] for line in block.split(' PYCODE', 1)[0].splitlines())
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
listing = pathlib.Path(directory) / 'list.txt'
|
||||
result = pathlib.Path(directory) / 'result.txt'
|
||||
source = source.replace('/tmp/photo-boot-list.txt', str(listing)).replace('/tmp/photo-boot-result.txt', str(result))
|
||||
listing.write_text('a: test\nb: test\n\n2 tests, 0 benchmarks\n')
|
||||
result.write_text('test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s\n')
|
||||
with contextlib.redirect_stdout(io.StringIO()): exec(source, {})
|
||||
for listed, outcome in [
|
||||
('0 tests, 0 benchmarks\n', 'test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out;'),
|
||||
('2 tests, 0 benchmarks\n', 'test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 1 filtered out;'),
|
||||
('2 tests, 0 benchmarks\n', 'missing result'),
|
||||
]:
|
||||
listing.write_text(listed); result.write_text(outcome)
|
||||
with self.assertRaises(AssertionError): exec(source, {})
|
||||
|
||||
def test_missing_root_rejected(self):
|
||||
with self.assertRaises(ValueError): closure(graph(), 'missing')
|
||||
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
Reference in new issue
Block a user