From bff03f709cf2cbc8b0f60947422e5cdecf4d9c6b Mon Sep 17 00:00:00 2001 From: aiongg <67543387+aiongg@users.noreply.github.com> Date: Mon, 12 Oct 2026 22:09:46 +0000 Subject: [PATCH] tools: add check.py, a check run that reuses conversions, validations and renders compare.py --cache keeps conversions by content SHA-256 and file name, --valid-cache keeps schema validation results by part; render_compare.py takes conversions from the same cache and reuses verdicts and measurements with --reuse-results. Runs killed by a signal or a timeout and failed renders are not kept. --- README.md | 3 + docs/measurement.md | 64 ++++- tools/check.py | 491 +++++++++++++++++++++++++++++++++++++ tools/compare.py | 529 ++++++++++++++++++++++++++++++++-------- tools/render_compare.py | 145 +++++++++-- 5 files changed, 1107 insertions(+), 125 deletions(-) create mode 100644 tools/check.py diff --git a/README.md b/README.md index 175a789..030ab86 100644 --- a/README.md +++ b/README.md @@ -196,6 +196,9 @@ in nearly every sample, so that their INDD fields cannot be located. two revisions over the corpus; a refactoring must change none. - `python3 -I tools/readme_numbers.py --schemas DIR --jing DIR` runs the full measurement and updates [Current numbers](#current-numbers). +- `python3 -I tools/check.py LABEL [--base REV]` runs all of these checks + once for a batch of work, with the conversions, schema validations and + renders of earlier runs reused (`docs/measurement.md`, "Check runs"). - `python3 -I tools/inventory.py corpus/` lists each sample's InDesign version and whether it has a matching IDML file. diff --git a/docs/measurement.md b/docs/measurement.md index 801acc3..8917210 100644 --- a/docs/measurement.md +++ b/docs/measurement.md @@ -330,6 +330,17 @@ reference IDML get no exception: an endnote or anchored form field error there counts as a failure, and `compare.py --all` lists those files separately. +Jing validates each part of a package on its own, against the schema +that the part's path selects, so a part's errors depend only on that +schema and the part's bytes. `--valid-cache FILE` keeps the errors of +every validated part by schema and SHA-256 (keyed also by a hash of the +schemas, Jing and `tools/validate.sh`) and validates only the parts it +does not list, copied into reduced packages. The counts are the same as +validating every output; only the order of a package's messages can +differ. compare.py reads the part selection from `tools/validate.sh` +and stops with an error if it cannot. The file is appended one line at a +time; a line that cannot be read (from a killed run) is skipped. + ## Audit: what the converter does not read `indd audit ` needs no reference IDML. It converts the file while @@ -500,7 +511,9 @@ and the Python standard library only. Options: | `--link-fonts` | Give DesignCraft the font files a package keeps beside the IDML, in a `Document Fonts` folder; DesignCraft reads only that folder | | `--bin PATH` | Another converter binary | | `--gaps PATH` | compare.py's `gaps.tsv` to rank first-difference keys against (default `target/compare/gaps.tsv`) | -| `--reuse DIR` | Take A.pdf, B.pdf and DesignCraft's results from `work/` of an earlier run in DIR (which may be `--out` itself) for each triple whose converted IDML has the same bytes as in that run; render the others, which needs `--designcraft`. For measuring a change to the tool on the same renders: pass the earlier run's `--bin` | +| `--reuse DIR` | Take A.pdf, B.pdf and DesignCraft's results from `work/` of an earlier run in DIR (which may be `--out` itself) for each triple whose converted IDML has the same bytes as in that run and whose A and B renders succeeded there; render the others, which needs `--designcraft`. For measuring a change to the tool on the same renders: pass the earlier run's `--bin` | +| `--reuse-results DIR` | Take the verdict and measurement of each triple from `work/ID/result.pickle` of an earlier run in DIR (which may be `--out` itself) when they were made with the same key: the same converted IDML, converter stderr, `indd xmp` and `indd uids` output, tool sources, DesignCraft, `--max-pages` and `--link-fonts`. Only the other triples are measured. A triple whose A or B render failed or timed out gets no `result.pickle`, so the next run renders it again | +| `--cache DIR` | Take the conversions from compare.py's `--cache DIR` (converting into it what is missing, with the tool's conversion timeout) instead of converting each triple | | `--images ID:PAGE…` | Write images of pages of the last run (below) and nothing else | Run it after converter changes, like `diff_outputs.py`: a page whose A vs @@ -522,7 +535,7 @@ first 10 hexadecimal digits of the INDD's SHA-256; only `triples.tsv` and | `dropped.tsv` | Stories R shows and A does not, with frame height, point size, leading and fonts | | `issues.tsv` | DesignCraft issue kinds per document, without paths, for reporting to DesignCraft | | `summary.json` | Headline numbers: triples by verdict and reason; A vs B pixel-identical documents and pages, first-difference gap keys; DesignCraft renders, failures, times, issue kinds on font-complete documents and on all documents, and its warnings | -| `work/ID/` | The converter's IDML, `A.pdf`, `B.pdf`, the folders they were rendered from and DesignCraft's results (`run.json`) | +| `work/ID/` | The converter's IDML, `A.pdf`, `B.pdf`, the folders they were rendered from, DesignCraft's results (`run.json`), and the triple's verdict and measurement with their key (`result.pickle`) | | `images/` | With `--images ID:PAGE`: `ID-pPAGE.png` (R, A and B side by side) and `ID-pPAGE-ab.png` (A vs B: red where only A has ink, blue where only B has it) | ### Limits @@ -535,3 +548,50 @@ first 10 hexadecimal digits of the INDD's SHA-256; only `triples.tsv` and - The gap keys of the first differing page are candidates. A change on a page can also come from a story that starts on an earlier page or from a value outside the elements listed above. + +## Check runs + +`tools/check.py LABEL [--base REV]` runs every check of a batch of work +once: build, tests, clippy, the full measurement with schema validation, +the output diff against a base revision, the README numbers, the render +comparison, the privately held documents and the privacy checks. Most of +the work of the separate tools is repeated conversion and the validation +and rendering of packages that did not change; check.py does each once +and keeps the results. + +| Stage | What runs | What is kept | +|---|---|---| +| Build | `cargo build --release`; the binary is copied to `~/.cache/indd-check/bin/HASH` | The binary of each check run | +| Tests, clippy | `cargo test --release` (library, doc and fixture tests) and `cargo clippy --all-targets`, in the background | | +| Measurement | `compare.py --all --exclude own/` with schema validation, `--cache` and `--valid-cache` | Every conversion, under `out/HASH/` by cache key (below); validation results by part in `valid.tsv` | +| Base outputs | The base revision (default `origin/main`) is looked up in `revs.tsv` (commit → binary); if missing, it is built as `diff_outputs.py` builds it and converted, in the background | The base binary and its conversions | +| Diff | Exit status, stderr and package entries of both conversions of every file, as `diff_outputs.py` compares them | `diff.tsv` | +| README | `readme_numbers.py` from the measurement's `summary.json` | | +| Render | `render_compare.py --cache … --reuse-results …` into `~/.cache/indd-check/render/`: only triples whose conversion changed since the last check run are measured again | Renders and results per triple | +| Private, privacy | The privately held documents named in the local configuration are converted and validated (a failed conversion or a schema error fails the check run); `git grep` for the configured pattern; tracked binary files | | + +The cache is `~/.cache/indd-check/` (`INDD_CHECK_CACHE`). Each converter +binary gets its own output folder, and the folders are not deleted +automatically. A conversion's cache key is the INDD's SHA-256 and the +first 8 hexadecimal digits of the SHA-256 of its file name, because the +output contains the file name (designmap.xml's `Name`). The SHA-256 comes +from `corpus/inventory.tsv` when the inventory lists the file with its +present size, and is computed from the file otherwise. A conversion is +kept only when the converter exits by itself (status 0, 1, 2 or 101 for a +panic); a run killed by a signal or a timeout is converted again by the +next run, and so are `indd xmp` and `indd uids` if either was killed. A +commit enters `revs.tsv` when check.py runs on it with +no uncommitted change under `src/` or to the Cargo files, so the next +check run with that commit as its base reuses the outputs. The +configuration that names private documents, the privacy pattern, the +`--subtotal` prefixes and the DesignCraft paths is a local JSON file +(`--config`, default `.thoughts/tools/check.json`); the module docstring +lists its keys. The results equal those of the separate tools: +`compare.py` gives the same tables and `summary.json` with or without +`--cache` and `--valid-cache`, and `render_compare.py` gives the same +tables with or without `--reuse-results`. + +check.py prints each stage's time and a summary, and copies the binary, +the measurement's tables and `summary.json`, the render tables +(`render/`, only when the render comparison succeeded), the diff, the logs and the summary (`check.txt`) to +`~/.cache/indd-frozen-LABEL/`. diff --git a/tools/check.py b/tools/check.py new file mode 100644 index 0000000..8202d34 --- /dev/null +++ b/tools/check.py @@ -0,0 +1,491 @@ +#!/usr/bin/env python3 +"""Check the converter after a batch of work: build, tests, clippy, the +corpus measurement with schema validation, the output diff against a base +revision, the README numbers, the render comparison, the privately held +documents and the privacy checks, with each piece of work done once. + +Usage: python3 -I tools/check.py LABEL [--base REV] [--jobs N] [--config FILE] + [--schemas DIR] [--jing DIR] [--designcraft PATH] + [--render-seed DIR] [--no-render] [--show N] + +Stages (docs/measurement.md, "Check runs"): + +1. `cargo build --release`; the binary is copied to CACHE/bin/. + `cargo test --release` (library, doc and fixture tests; tests/corpus.rs + is left out, compare.py covers it) and `cargo clippy --all-targets` run + in the background (clippy in its own target directory, CACHE/target-clippy). +2. compare.py --all --exclude own/ with schema validation, conversions kept + in CACHE/out// (compare.py --cache) and validation results + in CACHE/valid.tsv (--valid-cache): a package is converted once per + binary and validated once per content. Tables and summary.json go to + target/compare/ as with compare.py; --subtotal prefixes come from the + configuration. +3. The outputs of the base revision (default origin/main): CACHE/revs.tsv + maps commits to binaries. A commit without an entry is built as + tools/diff_outputs.py builds it and converted into the cache, in the + background while stage 2 runs. The diff compares exit status, stderr and + package entries as diff_outputs.py does; the list goes to diff.tsv in + the frozen folder. +4. tools/readme_numbers.py rewrites README.md from the summary of stage 2. +5. tools/render_compare.py with --cache and --reuse-results in + CACHE/render: only triples whose conversion changed since the previous + check run are measured again (and rendered with --designcraft). +6. The privately held documents of the configuration are converted and + validated; `git grep` for the privacy pattern and a search for tracked + binary files. +7. The binary, the tables, summary.json and the logs are copied to + ~/.cache/indd-frozen-LABEL/. + +CACHE is ~/.cache/indd-check (INDD_CHECK_CACHE overrides it). Temporary +files go to ~/.cache/indd-tmp. The configuration (--config, default +.thoughts/tools/check.json, not in the repository) is a JSON object with +any of: "subtotal" (list of path prefixes under corpus/ for compare.py +--subtotal), "private" (paths under corpus/ of privately held documents), +"privacy_pattern" (extended regular expression that no committed file may +match), "designcraft", "render_seed" (defaults of the options of the same +name), "schemas", "jing". Prints each stage's time and a summary; exit +status 1 if a check fails. +""" + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from types import SimpleNamespace + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import compare # noqa: E402 +import diff_outputs # noqa: E402 + +ROOT = compare.ROOT +CACHE = Path(os.environ.get("INDD_CHECK_CACHE", Path.home() / ".cache" / "indd-check")) +BINARY_SUFFIXES = r"\.(indd|indt|indb|idml|pdf|psd|ai|zip|otf|ttf|jpg|png|tif)$" + +times = {} + + +class Stage: + """Times a stage into `times`.""" + + def __init__(self, name): + self.name = name + + def __enter__(self): + self.start = time.time() + return self + + def __exit__(self, *exc): + times[self.name] = time.time() - self.start + + +def run(cmd, log=None, env=None): + """Run a command in the repository; (exit status, output). With `log`, + the output also goes to that file.""" + p = subprocess.run(cmd, cwd=ROOT, env=env, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True, errors="replace") + if log: + Path(log).write_text(p.stdout) + return p.returncode, p.stdout + + +def git(*args): + return subprocess.run(["git", "-C", str(ROOT), *args], capture_output=True, + text=True).stdout.strip() + + +def keep_binary(path): + """Copy a converter binary to CACHE/bin/; its hash.""" + h = compare.file_sha256(path)[:16] + dest = CACHE / "bin" / h + if not dest.exists(): + dest.parent.mkdir(parents=True, exist_ok=True) + tmp = dest.with_suffix(".tmp") + shutil.copyfile(path, tmp) + tmp.chmod(0o755) + tmp.replace(dest) + return h + + +def revisions(): + """commit -> binary hash, from CACHE/revs.tsv.""" + try: + lines = (CACHE / "revs.tsv").read_text().splitlines() + except OSError: + return {} + return dict(line.split("\t") for line in lines if "\t" in line) + + +def record_revision(commit, h): + if revisions().get(commit) != h: + with open(CACHE / "revs.tsv", "a") as f: + f.write(f"{commit}\t{h}\n") + + +def corpus_items(): + """(path under corpus/, INDD, reference IDML or None, cache key) of every + file compare.py --all --exclude own/ converts, in its order.""" + seen = set() + known = compare.known_digests() + todo = compare.pairs(0, None, ["own/"], seen, known) + others = compare.unpaired(None, ["own/"], seen, known) + items = [(indd, None) for indd in others] + todo + return [(str(indd.relative_to(ROOT / "corpus")), indd, idml, + compare.cache_key(indd, compare.digest(indd, known))) for indd, idml in items] + + +def convert_all(binary, folder, items, jobs): + """Records of every item converted by `binary` into `folder`.""" + folder.mkdir(parents=True, exist_ok=True) + with ThreadPoolExecutor(jobs) as pool: + return list(pool.map(lambda it: compare.convert_cached(binary, it[1], it[3], folder, + False), items)) + + +def base_outputs(commit, items, jobs, current): + """(binary hash, records) of the base commit's conversions: cached ones + are reused; a commit without a binary is built first.""" + h = revisions().get(commit) + if not h or not (CACHE / "bin" / h).exists(): + h = keep_binary(diff_outputs.build(commit)) + record_revision(commit, h) + if h == current: + return h, None + return h, convert_all(CACHE / "bin" / h, CACHE / "out" / h, items, jobs) + + +def diff(items, old, new, show, dest): + """Compare the records of two conversions of `items` as diff_outputs.py + does; print the counts and write diff.tsv to `dest`. Paths whose output + or exit status differs.""" + results = [] + for (rel, _, _, _), a, b in zip(items, old, new): + r = {"path": rel, "status": (a["exit"], b["exit"]), + "warnings": a["stderr"].splitlines() != b["stderr"].splitlines(), "entries": []} + if a["exit"] == 0 and b["exit"] == 0 and a["package"] != b["package"]: + ea, eb = a["entries"], b["entries"] + if ea is None or eb is None: + r["entries"] = ["(unreadable package)"] + else: + if [n for n, _ in ea] != [n for n, _ in eb]: + r["entries"].append("(entry list)") + da, db = dict(map(tuple, ea)), dict(map(tuple, eb)) + r["entries"] += [n for n in da if n in db and da[n] != db[n]] + results.append(r) + differ = [r for r in results if r["entries"]] + status = [r for r in results if r["status"][0] != r["status"][1]] + failed = [r for r in results if r["status"] != (0, 0) and r["status"][0] == r["status"][1]] + warned = [r for r in results if r["warnings"]] + kinds = compare.Counter(diff_outputs.kind(n) for r in differ for n in r["entries"]) + with open(dest / "diff.tsv", "w") as out: + out.write("index\tpath\tstatus_old\tstatus_new\twarnings_differ\tentries\n") + for i, r in enumerate(results): + if r["entries"] or r["warnings"] or r["status"][0] != r["status"][1]: + out.write(f"{i}\t{r['path']}\t{r['status'][0]}\t{r['status'][1]}\t" + f"{r['warnings']}\t{' '.join(r['entries'])}\n") + print(f"files converted: {len(results)}") + print(f"outputs differ: {len(differ)}") + print(f"package entries differ: {sum(kinds.values())}") + print(f"failure status differs: {len(status)}") + print(f"warnings differ: {len(warned)}") + print(f"failed with both: {len(failed)}") + for k, n in kinds.most_common(): + print(f" {n:6d} {k}") + for r in (differ + status + warned)[:show]: + print(f" [{results.index(r)}] {r['path']}: status {r['status']}, " + f"warnings {'differ' if r['warnings'] else 'same'}, " + f"entries {' '.join(r['entries'][:5])}") + print(f"details: {dest / 'diff.tsv'}") + return {"files": len(results), "differ": len(differ), "entries": sum(kinds.values()), + "status": len(status), "warnings": len(warned), "failed": len(failed)} + + +def tests(dest): + """Library, doc and fixture tests in release mode; (ok, text).""" + passed = failed = 0 + ok = True + for i, args in enumerate((["--lib", "--test", "fixtures"], ["--doc"])): + code, out = run(["cargo", "test", "--release", "-q", *args], dest / f"tests{i}.txt") + ok = ok and code == 0 + for m in re.finditer(r"test result: \w+\. (\d+) passed; (\d+) failed", out): + passed, failed = passed + int(m[1]), failed + int(m[2]) + if code != 0: + bad = [x for x in out.splitlines() if re.search(r"FAILED|panicked|^error", x)] + print("tests:\n " + "\n ".join(bad[:10])) + return ok, f"{passed} passed, {failed} failed" + ("" if ok else " (FAILED)") + + +def clippy(dest): + env = dict(os.environ, CARGO_TARGET_DIR=str(CACHE / "target-clippy")) + code, out = run(["cargo", "clippy", "--all-targets", "-q"], dest / "clippy.txt", env) + lines = [x for x in out.splitlines() if x.strip()] + if code == 0 and not lines: + return True, "clean" + print("clippy:\n " + "\n ".join(lines[:10])) + return False, f"exit {code}, {len(lines)} lines of output" + + +def private(binary, folder, paths, vargs, vcache): + """Convert and validate the privately held documents as + .thoughts/tools/verify.sh does; (one text per document, whether any + failed to convert or has schema errors).""" + out = [] + bad = False + for i, path in enumerate(paths, 1): + indd = ROOT / "corpus" / path + key = compare.cache_key(indd, compare.file_sha256(indd)) + rec = compare.convert_cached(binary, indd, key, folder, False) + warnings = sum(1 for x in rec["stderr"].splitlines() if x.startswith("warning: ")) + if rec["exit"] != 0: + bad = True + out.append(f"{i}: conversion failed (exit {rec['exit']}): " + f"{compare.error_message(rec['stderr'])[:200]}") + continue + pkg = folder / f"{key}.idml" + errs = compare.validate_cached([(pkg, rec["entries"])], vargs, vcache).get(str(pkg), []) + bad = bad or bool(errs) + out.append(f"{i}: warnings={warnings} schema errors={len(errs)}") + return out, bad + + +def privacy(pattern): + """Committed lines that match the privacy pattern, and tracked binary files.""" + hits = [] + if pattern: + g = subprocess.run(["git", "-C", str(ROOT), "grep", "-n", "-i", "-E", pattern, "--", + ".", ":!corpus"], capture_output=True, text=True, errors="replace") + hits = [x for x in g.stdout.splitlines() if not x.startswith("Binary")] + tracked = [f for f in git("ls-files").splitlines() if re.search(BINARY_SUFFIXES, f, re.I)] + return hits, tracked + + +def grep(text, pattern): + m = re.search(pattern, text, re.M) + return m[1] if m else "?" + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("label", help="name of the run: ~/.cache/indd-frozen-LABEL/") + ap.add_argument("--base", default="origin/main", help="revision to diff against") + ap.add_argument("--jobs", type=int, default=os.cpu_count() or 1) + ap.add_argument("--config", default=str(ROOT / ".thoughts" / "tools" / "check.json")) + ap.add_argument("--schemas") + ap.add_argument("--jing") + ap.add_argument("--designcraft", help="DesignCraft command-line program") + ap.add_argument("--render-seed", metavar="DIR", + help="earlier render_compare.py run whose renders the first check run reuses") + ap.add_argument("--no-render", action="store_true", help="leave out the render comparison") + ap.add_argument("--show", type=int, default=10, help="differing files to list") + args = ap.parse_args() + try: + config = json.loads(Path(args.config).read_text()) + except OSError: + config = {} + schemas_home = Path.home() / ".cache" / "indd-schemas" + schemas = args.schemas or config.get("schemas") or str(schemas_home / "schemas") + jing = args.jing or config.get("jing") or str(schemas_home / "jing") + designcraft = args.designcraft or config.get("designcraft") or os.environ.get("DESIGNCRAFT_CLI") + seed = args.render_seed or config.get("render_seed") + expand = (lambda p: os.path.expanduser(p) if p else p) + schemas, jing, designcraft, seed = map(expand, (schemas, jing, designcraft, seed)) + os.environ["TMPDIR"] = str(Path.home() / ".cache" / "indd-tmp") + Path(os.environ["TMPDIR"]).mkdir(parents=True, exist_ok=True) + dest = Path.home() / ".cache" / f"indd-frozen-{args.label}" + dest.mkdir(parents=True, exist_ok=True) + CACHE.mkdir(parents=True, exist_ok=True) + start = time.time() + failed = [] + summary = [] + + head = git("rev-parse", "HEAD") + base = git("rev-parse", "--verify", args.base + "^{commit}") + if not base: + sys.exit(f"unknown revision {args.base}") + print(f"== {args.label}: {git('log', '--oneline', '-1')}; " + f"{git('status', '-sb').splitlines()[0]}") + changed = git("status", "--porcelain", "-uall", "--", "src", "Cargo.toml", "Cargo.lock", + "build.rs") + if changed: + print("uncommitted changes to the source:\n " + "\n ".join(changed.splitlines()[:5])) + + with Stage("build"): + code, out = run(["cargo", "build", "--release", "-q"]) + if code != 0: + print(out[-3000:]) + sys.exit("build failed") + h = keep_binary(ROOT / "target" / "release" / "indd") + binary = CACHE / "bin" / h + if not changed: + record_revision(head, h) + + background = {} + + def later(name, fn, *a): + """Run a stage in a thread; its result, or the exception it raised, + goes to `background`.""" + def go(): + with Stage(name): + try: + background[name] = fn(*a) + except Exception as e: # reported when the result is needed + background[name] = e + t = threading.Thread(target=go) + t.start() + return t + + def result(name): + r = background[name] + if isinstance(r, Exception): + sys.exit(f"{name} failed: {type(r).__name__}: {r}") + return r + + items = corpus_items() + jobs = [later("tests", tests, dest), later("clippy", clippy, dest), + later("base outputs", base_outputs, base, items, args.jobs, h)] + + with Stage("compare"): + cmd = [sys.executable, "-I", str(ROOT / "tools" / "compare.py"), "--all", + "--exclude", "own/", "--schemas", schemas, "--jing", jing, + "--bin", str(binary), "--cache", str(CACHE / "out"), + "--valid-cache", str(CACHE / "valid.tsv"), "--jobs", str(args.jobs)] + for prefix in config.get("subtotal", []): + cmd += ["--subtotal", prefix] + code, report = run(cmd, dest / "compare.txt") + if code != 0: + print(report[-3000:]) + sys.exit("compare.py failed") + print("\n".join(x for x in report.splitlines() + if re.match(r"(conversion failures|schema |files without|story text|" + r"trustworthy pairs|without )", x))) + st = json.loads((ROOT / "target" / "compare" / "summary.json").read_text()) + + render = None + if not args.no_render and designcraft: + out = CACHE / "render" + reuse = out if (out / "work").exists() or not seed else Path(seed) + rcmd = [sys.executable, "-I", str(ROOT / "tools" / "render_compare.py"), + "--designcraft", designcraft, "--bin", str(binary), "--cache", str(CACHE / "out"), + "--reuse", str(reuse), "--reuse-results", str(out), "--out", str(out), + "--jobs", str(args.jobs)] + render = later("render", run, rcmd, dest / "render.txt") + elif not args.no_render: + print("render comparison left out: no --designcraft") + + jobs[2].join() + with Stage("diff"): + base_hash, base_recs = result("base outputs") + print(f"\n== diff {args.base} ({base[:7]}, binary {base_hash}) .. working tree " + f"(binary {h})") + folder = CACHE / "out" / h + recs = [compare.convert_cached(binary, it[1], it[3], folder, False) for it in items] + d = diff(items, base_recs or recs, recs, args.show, dest) + + with Stage("readme"): + code, out = run([sys.executable, "-I", str(ROOT / "tools" / "readme_numbers.py")]) + readme = out.strip().splitlines()[-1] if out.strip() else f"exit {code}" + if code != 0: + failed.append("README numbers") + + with Stage("private"): + vargs = SimpleNamespace(schemas=schemas, jing=jing, jobs=args.jobs) + vcache = compare.ValidationCache(CACHE / "valid.tsv", schemas, jing) + docs, docs_bad = private(binary, CACHE / "out" / h, config.get("private", []), vargs, + vcache) + if docs_bad: + failed.append("private documents") + with Stage("privacy"): + hits, tracked = privacy(config.get("privacy_pattern")) + for x in hits[:5] + tracked[:5]: + print(f"privacy: {x}") + + for t in jobs[:2]: + t.join() + rtext = None + render_ok = False + if render: + render.join() + code, rtext = result("render") + render_ok = code == 0 + if not render_ok: + print(rtext[-3000:]) + failed.append("render comparison") + + with Stage("freeze"): + shutil.copyfile(binary, dest / "indd") + (dest / "indd").chmod(0o755) + for f in (ROOT / "target" / "compare").glob("*.tsv"): + shutil.copyfile(f, dest / f.name) + shutil.copyfile(ROOT / "target" / "compare" / "summary.json", dest / "summary.json") + if render_ok: + (dest / "render").mkdir(exist_ok=True) + for f in list((CACHE / "render").glob("*.tsv")) + [CACHE / "render" / "summary.json"]: + if f.exists(): + shutil.copyfile(f, dest / "render" / f.name) + + # Summary. + ok_tests, tests_text = result("tests") + ok_clippy, clippy_text = result("clippy") + paired, others = st["paired"], st["others"] + tr, ev = st["trustworthy"], st["all_pairs"] + + def stories(x): + s = x["stories"] + return (f"{s['exact']:,}/{sum(s.values()):,} exact, {s['differ']} differ, " + f"{s['missing']} missing") + + def cov(x): + return (f"{100 * x['reproduced'] / x['values']:.2f} % " + f"({x['reproduced']:,} of {x['values']:,})") + if not ok_tests: + failed.append("tests") + if not ok_clippy: + failed.append("clippy") + if paired["failures"] or others["failures"]: + failed.append("conversion failures") + if paired["invalid"]: + failed.append("schema validation (paired files)") + if hits or tracked: + failed.append("privacy") + summary.append(f"tests: {tests_text}; clippy: {clippy_text}") + summary.append(f"conversion failures: {len(paired['failures'])} of {paired['files']:,} paired, " + f"{len(others['failures'])} of {others['files']:,} other files; rejected " + f"{len(paired['rejected']) + len(others['rejected'])}") + summary.append(f"schema failures: {paired['invalid']} paired, {others['invalid']} other files; " + f"{grep(report, r'^schema errors accepted.*: (\d+) files')} accepted (endnote " + f"markup); {grep(report, r'^schema validation: (.*)$')}") + summary.append(f"story text: all pairs {stories(ev)}; trustworthy {stories(tr)}") + summary.append(f"value coverage: trustworthy {cov(tr)}; all pairs {cov(ev)}") + for prefix, sub in st.get("subtotals", {}).items(): + summary.append(f"without {prefix}: {sub['failures']} failures of {sub['files']} paired, " + f"{sub['invalid']} invalid; story text {stories(sub['all_pairs'])}; " + f"trustworthy {stories(sub['trustworthy'])}") + summary.append(f"diff vs {args.base}: {d['differ']} outputs differ ({d['entries']} entries), " + f"failure status differs {d['status']}, warnings differ {d['warnings']}, " + f"failed with both {d['failed']}") + summary.append(f"README: {readme}") + if rtext is not None: + summary.append("render: " + grep(rtext, r"^converter \(A vs B\): (.*?);") + "; " + + grep(rtext, r"^(verdicts and measurements reused: .*)$")) + summary.append(f"private documents: {'; '.join(docs) or 'none configured'}") + summary.append(f"privacy: {len(hits)} matches; tracked binary files: {len(tracked)}") + summary.append(f"frozen: {dest}") + times["total"] = time.time() - start + + print("\n== stage times (s; tests, clippy, base outputs and render run in the background)") + for name, secs in times.items(): + print(f" {name:14} {secs:7.1f}") + print(f"\n== {args.label}: {'FAILED: ' + ', '.join(failed) if failed else 'ok'}") + print("\n".join(summary)) + (dest / "check.txt").write_text("\n".join(summary) + "\n") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/compare.py b/tools/compare.py index d714add..c2e5457 100644 --- a/tools/compare.py +++ b/tools/compare.py @@ -13,11 +13,28 @@ Usage: python3 -I tools/compare.py [--limit N] [--detail TAG]... [--file SUBSTR] [--schemas DIR --jing DIR] [--bin PATH] [--all] [--exclude PREFIX]... [--jobs N] [--shortfalls N] [--trusted] [--stale N] - [--gaps N] [--out DIR] + [--gaps N] [--out DIR] [--cache DIR] + [--valid-cache FILE] [--subtotal PREFIX]... Run from the repository root after `cargo build --release`. With --schemas and --jing, also validates every output with tools/validate.sh, in batches of VALIDATE_BATCH files (one Jing run per schema for a whole batch). --bin runs another converter binary, for example a copy of the previous build. +Conversions run in --jobs threads and the comparisons of pairs in --jobs +worker processes. + +--cache DIR keeps every conversion in DIR//, named by cache_key (the SHA-256 of the INDD file and +a short hash of its file name): the package (.idml), a record of the exit +status, stderr and the package's entries (.json), and for pairs the output +of `indd xmp` and `indd uids`. A later run with the same binary reuses them +instead of converting. A run killed by a signal or a timeout is not kept. +--valid-cache FILE keeps schema validation results by part (schema and +SHA-256 of the part; for the same schemas, Jing and tools/validate.sh) and +validates only parts it does not list, in reduced packages; the results +are the same as validating every package. +--subtotal PREFIX also reports the pairs whose path does not start with +PREFIX (conversion failures, schema failures, story text, value coverage) +and adds them to summary.json under "subtotals". --all also converts every other INDD and INDT file under corpus/ (any version, either byte order, without a usable IDML), validates the output @@ -57,15 +74,18 @@ import fnmatch import functools import hashlib import json +import multiprocessing import os import re import subprocess import sys import tempfile +import threading +import time import xml.etree.ElementTree as ET import zipfile from collections import Counter, defaultdict -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor from datetime import datetime, timedelta, timezone from pathlib import Path @@ -112,12 +132,28 @@ def inventory(): return [dict(zip(cols, line.split("\t"))) for line in lines[1:]] +def known_digests(): + """Path under corpus/ -> (SHA-256, size in bytes) of the files the + inventory lists with a digest, for digest().""" + return {r["path"]: (r["sha256"], r.get("bytes")) for r in inventory() if r.get("sha256")} + + def digest(indd, known): - """SHA-256 of a corpus file, from the inventory if it lists the file.""" + """SHA-256 of a corpus file. The inventory's digest is taken when the + inventory lists the file with its present size (the inventory records no + modification time); otherwise the file is read.""" path = str(indd.relative_to(ROOT / "corpus")) - if path in known: - return known[path] - return hashlib.sha256(indd.read_bytes()).hexdigest() + sha, size = known.get(path, (None, None)) + if sha and size == str(indd.stat().st_size): + return sha + return file_sha256(indd) + + +def cache_key(indd, sha): + """The name of a conversion under a --cache folder: the INDD's SHA-256 + and a short hash of its file name, which the output contains + (designmap.xml's Name).""" + return f"{sha}-{hashlib.sha256(os.fsencode(indd.name)).hexdigest()[:8]}" def pairs(limit, substr, exclude, seen, known): @@ -855,6 +891,243 @@ def value_coverage(p, ref, ours): p.extra_docs.update(k for k, v in extra.items() if v) +def file_sha256(path): + h = hashlib.sha256() + with open(path, "rb") as f: + while chunk := f.read(1 << 20): + h.update(chunk) + return h.hexdigest() + + +def cache_dir(binary, root): + """The folder of a converter binary's outputs under a --cache root: + the first 16 hex digits of the binary's SHA-256.""" + return Path(root) / file_sha256(binary)[:16] + + +def package_entries(path): + """[name, SHA-256 of the content] of each entry of a package, in order; + None if it is not a readable ZIP file.""" + try: + with zipfile.ZipFile(path) as z: + return [[i.filename, hashlib.sha256(z.read(i)).hexdigest()] for i in z.infolist()] + except (OSError, zipfile.BadZipFile): + return None + + +# Exit statuses of the converter itself (src/main.rs; 101 is a panic). Any +# other status means the process was killed, so the run is not kept. +OWN_EXITS = (0, 1, 2, 101) + + +def run_killable(cmd, timeout, text): + """subprocess.run with capture; (exit status or "timeout", stdout, stderr).""" + try: + r = subprocess.run(cmd, capture_output=True, text=text, timeout=timeout) + except subprocess.TimeoutExpired: + return "timeout", "" if text else b"", "" + return r.returncode, r.stdout, r.stderr + + +def convert_cached(binary, indd, key, folder, pair, timeout=None): + """Convert `indd` to folder/KEY.idml and return the conversion's record, + or the record of an earlier conversion into the same folder. The record + (folder/KEY.json) has the exit status, stderr, seconds and, for an output, the + package's SHA-256 and entries (package_entries). With `pair`, also runs + `indd xmp` and `indd uids`, whose output goes to folder/KEY.xmp and + folder/KEY.uids (left out when the command fails). A conversion that + times out (exit status "timeout") or is killed by a signal is returned + but not recorded, and neither is the xmp and uids step if one of them is.""" + folder = Path(folder) + rec_path, out = folder / f"{key}.json", folder / f"{key}.idml" + tmp = folder / f"{key}.{os.getpid()}.{threading.get_ident()}.tmp" + try: + rec = json.loads(rec_path.read_text()) + except (OSError, ValueError): + rec = None + changed = False + if rec is None or rec["exit"] == 0 and not out.exists(): + part = tmp.with_suffix(".idml") + t = time.time() + code, _, stderr = run_killable([binary, "convert", indd, part], timeout, True) + rec = {"exit": code, "stderr": stderr, "secs": round(time.time() - t, 2)} + if code == 0: + rec["package"] = file_sha256(part) + rec["entries"] = package_entries(part) + part.replace(out) + else: + part.unlink(missing_ok=True) + if code not in OWN_EXITS: + return rec + changed = True + if pair and "meta" not in rec: + killed = False + for cmd in ("xmp", "uids"): + code, stdout, _ = run_killable([binary, cmd, indd], timeout, False) + if code == 0: + tmp.write_bytes(stdout) + tmp.replace(folder / f"{key}.{cmd}") + killed = killed or code not in OWN_EXITS + if not killed: + rec["meta"] = True + changed = True + if changed: + tmp.write_text(json.dumps(rec)) + tmp.replace(rec_path) + return rec + + +def read_uids(path): + """UID -> class (None for a slot without a class) from the output of + `indd uids`, or None if there is none.""" + try: + text = Path(path).read_text() + except OSError: + return None + uids = {} + for line in text.splitlines(): + uid, cls = line.split("\t") + uids[int(uid)] = None if cls == "-" else int(cls, 16) + return uids + + +def compare_job(rel, name, idml, folder, key, show): + """Compare one converted pair (convert_cached); runs in a worker + process. (Stats, stale reasons, examples).""" + folder = Path(folder) + xmp = folder / f"{key}.xmp" + indd_date = modify_date(xmp.read_bytes()) if xmp.exists() else None + uids = read_uids(folder / f"{key}.uids") + ref, ours = load(idml), load(folder / f"{key}.idml") + reasons = stale_reasons(indd_date, idml_modify_date(idml), uids, ref[0], ref[1], ours[1]) + examples = defaultdict(list) + p = compare_pair(name, ref, ours, examples, show) + p.pairs = [(rel, p.reproduced, p.values)] + return p, reasons, examples + + +@functools.cache +def schema_selection(): + """The parts tools/validate.sh validates and their schemas, read from + the script: ({part: schema}, {folder: schema of its direct .xml + children}).""" + text = (ROOT / "tools" / "validate.sh").read_text() + folders = re.search(r"\bfor d in ([^;]+); do", text) + parts = re.search(r"\bfor p in ((?:[^;\\]|\\\n)+); do", text) + if not folders or not parts or "-name designmap.xml" not in text: + raise SystemExit("tools/validate.sh: part selection not found; " + "update compare.schema_selection") + single = {"designmap.xml": "designmap.rnc"} + single.update({p + ".xml": p + ".rnc" for p in parts[1].replace("\\", " ").split()}) + by_folder = {f: f"{f}/{n}.rnc" for f, n in (d.split(":") for d in folders[1].split())} + return single, by_folder + + +def part_schema(name): + """The schema tools/validate.sh validates a package entry with, or None.""" + single, by_folder = schema_selection() + if name in single: + return single[name] + folder, _, rest = name.partition("/") + if folder in by_folder and rest.endswith(".xml") and "/" not in rest: + return by_folder[folder] + return None + + +class ValidationCache: + """Schema errors of package parts (--valid-cache). Jing validates each + part on its own against the schema its path selects, so the errors of a + part depend only on the schema and the part's bytes. The file is a TSV + of a key of the schemas, Jing and tools/validate.sh; the schema; the + part's SHA-256; and its errors as a JSON list, without the part name. + Results under another key are ignored.""" + + def __init__(self, path, schemas, jing): + h = hashlib.sha256((ROOT / "tools" / "validate.sh").read_bytes()) + for d in (Path(schemas), Path(jing)): + for f in sorted(d.rglob("*")): + if f.is_file(): + h.update(str(f.relative_to(d)).encode() + b"\0" + f.read_bytes()) + self.key = h.hexdigest()[:16] + self.path = Path(path) + self.known = {} + # Parts taken from the cache and validated, in this run. + self.hits = self.validated = 0 + # A run killed while appending leaves at most a broken last line, + # which is skipped. + if self.path.exists(): + for line in self.path.read_text(errors="replace").splitlines(): + try: + key, schema, sha, errs = line.split("\t", 3) + if key == self.key: + self.known[(schema, sha)] = json.loads(errs) + except ValueError: + continue + + def add(self, results): + """Record {(schema, part SHA-256): errors}, one write per line.""" + new = {k: e for k, e in results.items() if k not in self.known} + self.known.update(new) + if new: + self.path.parent.mkdir(parents=True, exist_ok=True) + with open(self.path, "ab", buffering=0) as f: + if f.tell(): + with open(self.path, "rb") as r: + r.seek(-1, os.SEEK_END) + if r.read(1) != b"\n": + f.write(b"\n") + for k, e in new.items(): + f.write(f"{self.key}\t{k[0]}\t{k[1]}\t{json.dumps(e)}\n".encode()) + + +def validate_cached(outs, args, cache): + """Schema errors of packages given as (path, entries or None; see + package_entries), as {path: [messages]} like validate(). With a + ValidationCache, only parts it does not know are validated: they are + copied into reduced packages for one tools/validate.sh run.""" + if cache is None: + return validate([p for p, _ in outs], args) + listed = [] + todo = {} # (schema, part SHA-256) -> (package, part name) + for path, entries in outs: + if entries is None: + entries = package_entries(path) + parts = [(n, (part_schema(n), s)) for n, s in entries or [] if part_schema(n)] + listed.append((path, parts)) + for n, key in parts: + if key in cache.known: + cache.hits += 1 + elif key not in todo: + todo[key] = (path, n) + if todo: + by_package = defaultdict(dict) + for key, (path, n) in todo.items(): + by_package[path][n] = key + found = {key: [] for key in todo} + with tempfile.TemporaryDirectory() as tmp: + reduced = {} + for i, (path, names) in enumerate(by_package.items()): + r = Path(tmp) / f"{i}.idml" + with zipfile.ZipFile(path) as z, zipfile.ZipFile(r, "w") as w: + for n in names: + w.writestr(n, z.read(n)) + reduced[str(r)] = names + for r, msgs in validate(list(reduced), args).items(): + for m in msgs: + part, _, rest = m.partition(":") + if part not in reduced[r]: + raise SystemExit(f"tools/validate.sh: unexpected message {m}") + found[reduced[r][part]].append(rest) + cache.add(found) + cache.validated += len(todo) + errs = {} + for path, parts in listed: + msgs = [f"{n}:{m}" for n, key in parts for m in cache.known[key]] + if msgs: + errs[str(path)] = msgs + return errs + + def main(): ap = argparse.ArgumentParser() ap.add_argument("--limit", type=int, default=0) @@ -882,7 +1155,13 @@ def main(): ap.add_argument("--out", default=str(ROOT / "target" / "compare"), help="directory for pairs.tsv and gaps*.tsv (default target/compare)") ap.add_argument("--jobs", type=int, default=os.cpu_count() or 1, - help="conversions and schema validations run in parallel") + help="conversions, comparisons and schema validations run in parallel") + ap.add_argument("--cache", metavar="DIR", + help="keep the conversions in DIR// and reuse them") + ap.add_argument("--valid-cache", metavar="FILE", + help="schema validation results by part SHA-256, reused and extended") + ap.add_argument("--subtotal", action="append", default=[], metavar="PREFIX", + help="also report the pairs whose path does not start with PREFIX") args = ap.parse_args() check = bool(args.schemas and args.jing) @@ -895,34 +1174,20 @@ def main(): warnings = Counter() warned_files = Counter() seen = set() - known = {r["path"]: r["sha256"] for r in inventory() if r.get("sha256")} + known = known_digests() todo = pairs(args.limit, args.file, args.exclude, seen, known) others = unpaired(args.file, args.exclude, seen, known) if args.all else [] other_failures = [] - pool = ThreadPoolExecutor(max(1, args.jobs)) + # Paths under corpus/ of the paired files that fail, and of those whose + # output is invalid, for --subtotal. + failed_paths = {} + invalid_paths = set() + vcache = ValidationCache(args.valid_cache, args.schemas, args.jing) \ + if check and args.valid_cache else None - def convert(indd, out): - """Convert one file; runs in the pool.""" - return subprocess.run([args.bin, "convert", indd, out], capture_output=True, text=True) - - def convert_pair(indd, idml, out): - """Convert one paired file and read what stale_reasons needs: the - INDD's UIDs and the ModifyDate of both files; runs in the pool.""" - r = convert(indd, out) - x = subprocess.run([args.bin, "xmp", indd], capture_output=True) - u = subprocess.run([args.bin, "uids", indd], capture_output=True, text=True) - uids = None - if u.returncode == 0: - uids = {} - for line in u.stdout.splitlines(): - uid, cls = line.split("\t") - uids[int(uid)] = None if cls == "-" else int(cls, 16) - dates = (modify_date(x.stdout) if x.returncode == 0 else None, idml_modify_date(idml)) - return r, dates, uids - - def count_warnings(r): + def count_warnings(stderr): kinds = set() - for line in r.stderr.splitlines(): + for line in stderr.splitlines(): if line.startswith("warning: "): kind = warning_kind(line[len("warning: "):]) warnings[kind] += 1 @@ -932,82 +1197,104 @@ def main(): other_invalid = [] invalid = [] + # Paired outputs whose only schema errors are InDesign's endnote + # markup: (name, errors, normalised errors, reference IDML, path). They + # pass if the reference has the same errors. + endnote_pending = [] with tempfile.TemporaryDirectory() as tmp: - jobs = [pool.submit(convert, indd, Path(tmp) / f"other{n}.idml") - for n, indd in enumerate(others)] - jobs += [pool.submit(convert_pair, indd, idml, Path(tmp) / f"pair{n}.idml") - for n, (indd, idml) in enumerate(todo)] - # Converted files waiting for validation: (output, name, list of - # invalid files it belongs to). They are validated in batches, which - # bounds the space the unpacked outputs take. - to_check = [] + folder = cache_dir(args.bin, args.cache) if args.cache else Path(tmp) + folder.mkdir(parents=True, exist_ok=True) + # (INDD, reference IDML or None, cache key of the INDD) + items = [(indd, None) for indd in others] + todo + items = [(indd, idml, cache_key(indd, digest(indd, known))) for indd, idml in items] + threads = ThreadPoolExecutor(max(1, args.jobs)) + procs = ProcessPoolExecutor(max(1, args.jobs), + mp_context=multiprocessing.get_context("forkserver")) + # A pair is compared in a worker process as soon as it is converted. + jobs = {} + lock = threading.Lock() - # Paired outputs whose only schema errors are InDesign's endnote - # markup: (name, errors, normalised errors, reference IDML). They - # pass if the reference has the same errors. - endnote_pending = [] + def comparison(n): + indd, idml, key = items[n] + with lock: + if n not in jobs: + jobs[n] = procs.submit(compare_job, str(indd.relative_to(ROOT / "corpus")), + indd.name, idml, folder, key, args.show) + return jobs[n] - def flush(at_least): - if len(to_check) < at_least: - return - errs = validate([out for out, _, _, _ in to_check], args) - for out, name, bad, ref in to_check: + def compare_when_converted(n): + def done(f): + if not f.exception() and f.result()["exit"] == 0: + comparison(n) + return done + + # Pairs are converted first, so that their comparisons start early. + order = sorted(range(len(items)), key=lambda n: items[n][1] is None) + conversions = {} + for n in order: + indd, idml, key = items[n] + conversions[n] = threads.submit(convert_cached, args.bin, indd, key, folder, + idml is not None) + if idml is not None: + conversions[n].add_done_callback(compare_when_converted(n)) + # Files are taken in batches of VALIDATE_BATCH: one tools/validate.sh + # run per batch. Without --cache, the batch's outputs are deleted + # after it, which bounds the space they take. + for start in range(0, len(items), VALIDATE_BATCH): + batch = range(start, min(start + VALIDATE_BATCH, len(items))) + recs = {n: conversions[n].result() for n in batch} + errs = validate_cached([(folder / f"{items[n][2]}.idml", recs[n]["entries"]) + for n in batch if recs[n]["exit"] == 0], + args, vcache) if check else {} + for n in batch: + indd, idml, key = items[n] + rec, out = recs[n], folder / f"{key}.idml" + rel = str(indd.relative_to(ROOT / "corpus")) + count_warnings(rec["stderr"]) + if rec["exit"] != 0: + if idml is None: + other_failures.append((indd.name, rec["stderr"].strip())) + else: + failures.append((indd.name, rec["stderr"].strip())) + failed_paths[rel] = rec["stderr"].strip() + continue e = errs.get(str(out)) if e: - known = endnote_errors(out, e) if ref else None - if known: - endnote_pending.append((name, e, known, ref)) + endnotes = endnote_errors(out, e) if idml else None + if endnotes: + endnote_pending.append((indd.name, e, endnotes, idml, rel)) + elif idml: + invalid.append((indd.name, e)) + invalid_paths.add(rel) else: - bad.append((name, e)) - out.unlink(missing_ok=True) - to_check.clear() - for n, indd in enumerate(others): - r = jobs[n].result() - count_warnings(r) - out = Path(tmp) / f"other{n}.idml" - if r.returncode != 0: - other_failures.append((indd.name, r.stderr.strip())) - elif check: - to_check.append((out, indd.name, other_invalid, None)) - flush(VALIDATE_BATCH) - continue - out.unlink(missing_ok=True) - for n, (indd, idml) in enumerate(todo): - out = Path(tmp) / f"pair{n}.idml" - r, (indd_date, idml_date), indd_uids = jobs[len(others) + n].result() - count_warnings(r) - if r.returncode != 0: - failures.append((indd.name, r.stderr.strip())) - continue - if check: - to_check.append((out, indd.name, invalid, idml)) - ref = load(idml) - ours = load(out) - reasons = stale_reasons(indd_date, idml_date, indd_uids, ref[0], ref[1], ours[1]) - p = compare_pair(indd.name, ref, ours, examples, args.show) - rel = str(indd.relative_to(ROOT / "corpus")) - p.pairs = [(rel, p.reproduced, p.values)] - pair_rows.append((rel, p, reasons)) - every.add(p) - if reasons: - stale.append((rel, reasons)) - else: - trusted.add(p) - if check: - flush(VALIDATE_BATCH) - else: - out.unlink(missing_ok=True) - pool.shutdown() - flush(1) + other_invalid.append((indd.name, e)) + if idml is not None: + p, reasons, ex = comparison(n).result() + for k, found in ex.items(): + room = (max(args.show, 3) if k == ("Story", "text") else args.show) \ + - len(examples[k]) + examples[k] += found[:max(room, 0)] + pair_rows.append((rel, p, reasons)) + every.add(p) + if reasons: + stale.append((rel, reasons)) + else: + trusted.add(p) + if not args.cache: + out.unlink(missing_ok=True) + threads.shutdown() + procs.shutdown() endnote_passed = [] if endnote_pending: - ref_errs = validate([ref for _, _, _, ref in endnote_pending], args) - for name, e, known, ref in endnote_pending: + refs = [(ref, None) for _, _, _, ref, _ in endnote_pending] + ref_errs = validate_cached(refs, args, vcache) + for name, e, endnotes, ref, rel in endnote_pending: theirs = endnote_errors(ref, ref_errs.get(str(ref), [])) or set() - if known <= theirs: + if endnotes <= theirs: endnote_passed.append(name) else: invalid.append((name, e)) + invalid_paths.add(rel) rejected, failures = split_rejected(failures) other_rejected, other_failures = split_rejected(other_failures) @@ -1023,6 +1310,9 @@ def main(): print(f" {name}: {len(errs)} errors, {errs[:3]}") print(f"schema errors accepted (endnote and anchored form field markup, as in " f"the reference): {len(endnote_passed)} files") + if vcache: + print(f"schema validation: {vcache.validated} parts validated, " + f"{vcache.hits} known from {args.valid_cache}") if args.all: print(f"files without a reference: {len(others)}, rejected: {len(other_rejected)}" + (f" ({rejection_kinds(other_rejected)})" if other_rejected else "") @@ -1084,13 +1374,49 @@ def main(): for ex in examples[("Story", "text")]: print(" story mismatch:", ex) headline(args, every, trusted, pair_rows) + subtotals = {prefix: subtotal(prefix, check, todo, pair_rows, failed_paths, invalid_paths) + for prefix in args.subtotal} write_summary(Path(args.out) / "summary.json", args, check, (todo, failures, rejected, invalid), (others, other_failures, other_rejected, other_invalid), - every, trusted) + every, trusted, subtotals) -def write_summary(path, args, check, paired, unpaired_files, every, trusted): +def subtotal(prefix, check, todo, pair_rows, failed_paths, invalid_paths): + """Print and return the numbers of the pairs whose path under corpus/ + does not start with `prefix` (--subtotal).""" + def inside(rel): + return not rel.startswith(prefix) + files = sum(1 for indd, _ in todo if inside(str(indd.relative_to(ROOT / "corpus")))) + every, trusted = Stats(), Stats() + for rel, p, reasons in pair_rows: + if inside(rel): + every.add(p) + if not reasons: + trusted.add(p) + failed = [(rel, err) for rel, err in failed_paths.items() if inside(rel)] + failures = split_rejected(failed)[1] + invalid = sum(1 for rel in invalid_paths if inside(rel)) if check else None + print(f"\nwithout {prefix}: conversion failures: {len(failures)} of {files} paired files") + if check: + print(f"without {prefix}: schema validation failures: {invalid}") + print(f"without {prefix}: story text: {every.story_line()}") + print(f"without {prefix}: trustworthy pairs: {trusted.docs} of {every.docs}; " + f"story text: {trusted.story_line()}") + print(f"without {prefix}: value coverage: trustworthy pairs {trusted.coverage():.2%} " + f"({trusted.reproduced} of {trusted.values} values), all pairs {every.coverage():.2%}") + return {"files": files, "failures": len(failures), "invalid": invalid, + "trustworthy": pair_numbers(trusted), "all_pairs": pair_numbers(every)} + + +def pair_numbers(st): + """The numbers of a set of pairs in summary.json.""" + return {"pairs": st.docs, "values": st.values, "reproduced": st.reproduced, + "stories": {"exact": st.story_ok["ok"], "differ": st.story_ok["wrong"], + "missing": st.story_ok["missing"]}} + + +def write_summary(path, args, check, paired, unpaired_files, every, trusted, subtotals): """Write the numbers of the README's "Current numbers" section, and the options they were measured with, as JSON (tools/readme_numbers.py). `paired` and `unpaired_files` are (files, failures, rejected, @@ -1105,19 +1431,16 @@ def write_summary(path, args, check, paired, unpaired_files, every, trusted): "rejected": [error_message(err) for _, err in rejected], "invalid": len(invalid) if check else None} - def pairs(st): - return {"pairs": st.docs, "values": st.values, "reproduced": st.reproduced, - "stories": {"exact": st.story_ok["ok"], "differ": st.story_ok["wrong"], - "missing": st.story_ok["missing"]}} - summary = { "options": {"all": args.all, "validated": check, "exclude": args.exclude, "limit": args.limit, "file": args.file}, "paired": files(paired, True), "others": files(unpaired_files, args.all), - "trustworthy": pairs(trusted), - "all_pairs": pairs(every), + "trustworthy": pair_numbers(trusted), + "all_pairs": pair_numbers(every), } + if subtotals: + summary["subtotals"] = subtotals path.write_text(json.dumps(summary, indent=1) + "\n") diff --git a/tools/render_compare.py b/tools/render_compare.py index a007f49..a5a1aba 100644 --- a/tools/render_compare.py +++ b/tools/render_compare.py @@ -18,7 +18,7 @@ variable). Usage: python3 -I tools/render_compare.py --designcraft PATH [--bin PATH] [--jobs N] [--limit N] [--file SUBSTR] [--exclude PREFIX]... [--versions 13-21] [--max-pages N] [--link-fonts] [--out DIR] - [--gaps GAPS_TSV] [--reuse DIR] + [--gaps GAPS_TSV] [--reuse DIR] [--reuse-results DIR] [--cache DIR] python3 -I tools/render_compare.py --images ID:PAGE [ID:PAGE...] [--out DIR] Run from the repository root after `cargo build --release`. Needs @@ -37,8 +37,19 @@ reads only that folder). --reuse DIR takes A.pdf, B.pdf and the renderer's results from the work/ folder of an earlier run in DIR (which may be --out itself) instead of rendering, for each triple whose converted IDML has the same bytes as in -that run; other triples are rendered, which needs --designcraft. Use it -to measure a change to this tool on the same renders. +that run and whose renders succeeded there; other triples are rendered, +which needs --designcraft. Use it to measure a change to this tool on the +same renders. +--reuse-results DIR takes the verdict and measurement of a triple from +work/ID/result.pickle of an earlier run in DIR when everything they depend +on besides the corpus files is the same: this tool and compare.py, the +DesignCraft binary, --max-pages, --link-fonts, the converted IDML, the +converter's stderr and the output of `indd xmp` and `indd uids`. A triple +whose A or B render failed or timed out gets no result.pickle, so it is +rendered again. +--cache DIR takes the conversions (and the output of `indd xmp` and `indd +uids`) from compare.py's --cache DIR and adds to it, with the conversion +timeout of this tool. Outputs go to --out (default target/render-compare): triples.tsv, docs.tsv, pages.tsv, lines.tsv, dropped.tsv, issues.tsv, summary.json, and @@ -54,6 +65,7 @@ import argparse import hashlib import json import os +import pickle import re import statistics import struct @@ -109,6 +121,7 @@ def major(version): def candidates(args): """Inventory rows with an IDML and a PDF, without duplicates.""" lo, hi = (int(x) for x in args.versions.split("-")) + known = compare.known_digests() out = [] for r in compare.inventory(): path = r["path"] @@ -126,9 +139,9 @@ def candidates(args): skip.append(f"version outside {args.versions}") if v != dom: skip.append("IDML of another major version") - sha = r.get("sha256") or hashlib.sha256((CORPUS / path).read_bytes()).hexdigest() - out.append({"id": sha[:10], "path": path, "idml": r["idml"], "pdf": r["pdf"], - "version": r["version"], "skip": skip}) + sha = compare.digest(CORPUS / path, known) + out.append({"id": sha[:10], "key": compare.cache_key(CORPUS / path, sha), "path": path, + "idml": r["idml"], "pdf": r["pdf"], "version": r["version"], "skip": skip}) return out[:args.limit] if args.limit else out @@ -157,9 +170,15 @@ def judge(cfg, t): indd, idml, pdf = CORPUS / t["path"], CORPUS / t["idml"], CORPUS / t["pdf"] pages, creator = pdf_info(pdf) _, _, pdf_xmp, _ = run(["pdfinfo", "-meta", str(pdf)], 120) - code, _, indd_xmp, _ = run([cfg["bin"], "xmp", str(indd)], 120) - if code != 0: - indd_xmp = b"" + if cfg["cache"] and not t["skip"]: + rec = compare.convert_cached(cfg["bin"], indd, t["key"], cfg["cache"], True, + CONVERT_TIMEOUT) + xmp = Path(cfg["cache"]) / f"{t['key']}.xmp" + indd_xmp = xmp.read_bytes() if xmp.exists() else b"" + else: + code, _, indd_xmp, _ = run([cfg["bin"], "xmp", str(indd)], 120) + if code != 0: + indd_xmp = b"" n_idml = idml_pages(idml) reasons = compare.pdf_stale_reasons(indd_xmp, pdf_xmp, pages, n_idml, creator) res = dict(t, pdf_pages=pages, idml_pages=n_idml, creator=creator, reasons=reasons) @@ -169,20 +188,44 @@ def judge(cfg, t): work.mkdir(parents=True, exist_ok=True) ours = work / "ours.idml" ours.unlink(missing_ok=True) - code, secs, _, err = run([cfg["bin"], "convert", str(indd), str(ours)], CONVERT_TIMEOUT) + if cfg["cache"]: + code, secs, err = rec["exit"], rec.get("secs"), rec["stderr"] + if code == 0: + try: + os.link(Path(cfg["cache"]) / f"{t['key']}.idml", ours) + except OSError: + ours.write_bytes((Path(cfg["cache"]) / f"{t['key']}.idml").read_bytes()) + else: + code, secs, _, err = run([cfg["bin"], "convert", str(indd), str(ours)], CONVERT_TIMEOUT) res["convert"] = {"exit": code, "secs": secs, "warnings": sum(1 for x in err.splitlines() if x.startswith("warning: ")), "error": compare.error_message(err) if code != 0 else ""} if code != 0: res["reasons"] = reasons + ["conversion failed"] return res - u = subprocess.run([cfg["bin"], "uids", str(indd)], capture_output=True, text=True) + if cfg["cache"]: + uids_path = Path(cfg["cache"]) / f"{t['key']}.uids" + uids_out = uids_path.read_bytes() if uids_path.exists() else None + else: + u = subprocess.run([cfg["bin"], "uids", str(indd)], capture_output=True) + uids_out = u.stdout if u.returncode == 0 else None uids = None - if u.returncode == 0: + if uids_out is not None: uids = {} - for line in u.stdout.splitlines(): + for line in uids_out.decode().splitlines(): uid, cls = line.split("\t") uids[int(uid)] = None if cls == "-" else int(cls, 16) + # Everything the verdict and the measurement depend on besides the + # corpus files; with --reuse-results, an earlier result with the same + # key is taken as it is. + res["key"] = dict(cfg["result_key"], ours=sha256(ours), + stderr=hashlib.sha256(err.encode()).hexdigest(), + xmp=hashlib.sha256(indd_xmp).hexdigest(), + uids=hashlib.sha256(uids_out).hexdigest() if uids_out is not None else None) + prev = earlier_result(cfg, t, res["key"]) + if prev: + res["reused_result"] = prev + return res try: ref, mine = compare.load(idml), compare.load(ours) except SystemExit as e: @@ -903,7 +946,8 @@ def sha256(path): def reuse(cfg, work, res): """With --reuse, take A.pdf, B.pdf and the renderer's results from the - earlier run when its converted IDML has the same bytes; True if taken.""" + earlier run when its converted IDML has the same bytes and both renders + succeeded; True if taken.""" if not cfg["reuse"]: return False src = Path(cfg["reuse"]) / "work" / work.name @@ -916,7 +960,7 @@ def reuse(cfg, work, res): return False pdfs = [src / "A.pdf", src / "B.pdf"] if same != res["ours_sha256"] or not all(f.exists() for f in pdfs) \ - or "A" not in prev or "B" not in prev: + or "A" not in prev or "B" not in prev or not renders_ok(prev): return False if src.resolve() != work.resolve(): for f in pdfs: @@ -1041,15 +1085,58 @@ def measure(cfg, t): return res +def renders_ok(m): + """Whether a measurement (or none) has both renders, A and B, without a + failure or a timeout.""" + return m is None or m["A"]["exit"] == 0 and m["B"]["exit"] == 0 + + +def earlier_result(cfg, t, key): + """With --reuse-results, the (verdict, measurement) of the triple in the + earlier run if it was judged with the same key and its renders + succeeded; its PDFs are linked into this run's work folder.""" + if not cfg["reuse_results"]: + return None + src = Path(cfg["reuse_results"]) / "work" / t["id"] + try: + prev = pickle.loads((src / "result.pickle").read_bytes()) + except (OSError, pickle.UnpicklingError, EOFError): + return None + if prev.get("key") != key or not renders_ok(prev["measured"]): + return None + work = Path(cfg["out"]) / "work" / t["id"] + if src.resolve() != work.resolve(): + for name in ("A.pdf", "B.pdf"): + if (src / name).exists(): + link = work / name + link.unlink(missing_ok=True) + link.symlink_to((src / name).resolve()) + (work / "result.pickle").write_bytes(pickle.dumps(prev)) + prev["judged"]["result_reused"] = True + return prev["judged"], prev["measured"] + + def process(cfg, t): - """Judge one triple and, if it is a same-save triple to measure, measure it.""" + """Judge one triple and, if it is a same-save triple to measure, measure + it. A judged triple's verdict and measurement go to work/ID/result.pickle.""" try: res = judge(cfg, t) + if "reused_result" in res: + return tuple(res["reused_result"]) + work = Path(cfg["out"]) / "work" / t["id"] + (work / "result.pickle").unlink(missing_ok=True) + m = None if res["skip"] or res["reasons"]: - work = Path(cfg["out"]) / "work" / t["id"] / "ours.idml" - work.unlink(missing_ok=True) - return res, None - return res, measure(cfg, t) + (work / "ours.idml").unlink(missing_ok=True) + else: + m = measure(cfg, t) + if "key" in res and renders_ok(m): + # Pickled: the tables need the values' types back (tuples). A + # failed or timed-out render is not kept, so that the next run + # renders the triple again. + (work / "result.pickle").write_bytes(pickle.dumps( + {"key": res["key"], "judged": res, "measured": m})) + return res, m except Exception as e: # report the triple and go on with the others import traceback return dict(t, reasons=[f"tool error: {type(e).__name__}: {e}"[:300]], @@ -1401,6 +1488,11 @@ def main(): help="compare.py's gaps.tsv, to rank first-difference keys against") ap.add_argument("--reuse", metavar="DIR", help="take the renders of an earlier run in DIR where the IDML is the same") + ap.add_argument("--reuse-results", metavar="DIR", + help="take the verdicts and measurements of an earlier run in DIR where " + "the IDML and the converter's other output are the same") + ap.add_argument("--cache", metavar="DIR", + help="take the conversions from compare.py's --cache DIR (and add to it)") ap.add_argument("--images", nargs="+", metavar="ID:PAGE", help="write images of these pages of an earlier run, and nothing else") args = ap.parse_args() @@ -1421,6 +1513,9 @@ def main(): "out": str(out.resolve()), "tmp": str(tmp.resolve()), "link_fonts": args.link_fonts, "max_pages": args.max_pages, "reuse": str(Path(args.reuse).resolve()) if args.reuse else None, + "reuse_results": str(Path(args.reuse_results).resolve()) if args.reuse_results + else None, + "cache": str(compare.cache_dir(args.bin, args.cache).resolve()) if args.cache else None, "tools": {"converter": args.bin, "repository": subprocess.run(["git", "-C", str(ROOT), "describe", "--always", "--dirty"], capture_output=True, @@ -1428,6 +1523,12 @@ def main(): "designcraft": version, "designcraft_sha256": sha256(dc)[:16] if dc else None, "renders_reused_from": str(Path(args.reuse).resolve()) if args.reuse else None}} + tool = hashlib.sha256(Path(__file__).read_bytes() + Path(compare.__file__).read_bytes()) + cfg["result_key"] = {"tool": tool.hexdigest()[:16], + "designcraft": cfg["tools"]["designcraft_sha256"], + "max_pages": args.max_pages, "link_fonts": args.link_fonts} + if cfg["cache"]: + Path(cfg["cache"]).mkdir(parents=True, exist_ok=True) start = time.time() todo = candidates(args) results = {} @@ -1443,6 +1544,10 @@ def main(): if n % 25 == 0: print(f" {n}/{len(todo)} triples", file=sys.stderr) judged = [results[t["id"]][0] for t in todo] + if cfg["reuse_results"]: + print(f"verdicts and measurements reused: " + f"{sum(1 for r in judged if r.get('result_reused'))} of " + f"{sum(1 for r in judged if not r.get('skip'))} judged triples", file=sys.stderr) measured = [results[t["id"]][1] for t in todo if results[t["id"]][1] is not None] report(args, cfg, judged, measured, time.time() - start)