From 4ce9073dfa958183fd3b3e4075f65aa7cdcd92f8 Mon Sep 17 00:00:00 2001 From: aiongg <67543387+aiongg@users.noreply.github.com> Date: Thu, 8 Oct 2026 03:57:04 +0000 Subject: [PATCH] chore: refresh the README numbers from compare.py's summary compare.py now also writes target/compare/summary.json: the options of the run (--all, schema validation, --exclude, --limit, --file), the files, conversion failures (error messages) and schema validation failures of the paired and the other files, and the values, reproduced values and story counts of the trustworthy and of all pairs. tools/readme_numbers.py rewrites the "Current numbers" table of the README, now between `` and ``, from that summary. It refuses a summary from a run that included own/, covered part of the corpus, or ran without --all or schema validation. With --schemas and --jing it first builds the converter and runs `compare.py --all --exclude own/` with validation, so one command refreshes the README; --check only reports whether it is current. The numbers in the README are unchanged. --- CLAUDE.md | 12 ++- README.md | 6 ++ tools/compare.py | 41 +++++++++- tools/readme_numbers.py | 166 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 222 insertions(+), 3 deletions(-) create mode 100755 tools/readme_numbers.py diff --git a/CLAUDE.md b/CLAUDE.md index e6df6a0..1ab4d40 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -87,8 +87,16 @@ separately (rule and evidence: `docs/measurement.md`). - Headline numbers (value coverage, extra values, document scores, ranked gaps; defined in `docs/measurement.md`): `cargo build --release && python3 -I tools/compare.py --exclude own/`. It prints them last and - writes `pairs.tsv`, `gaps*.tsv`, `values*.tsv` and `extras*.tsv` to - `target/compare/`. + writes `pairs.tsv`, `gaps*.tsv`, `values*.tsv`, `extras*.tsv` and + `summary.json` to `target/compare/`. +- README numbers: `python3 -I tools/readme_numbers.py --schemas + --jing ` builds the converter, runs `compare.py --all --exclude + own/` with schema validation and rewrites the README's "Current + numbers" table (between `` and ``) from `target/compare/summary.json`. Without + `--schemas`/`--jing` it uses the last run's summary, which must be from + such a full run; `--check` only reports whether the README is current. + - `python3 -I tools/diff_outputs.py [OLD [NEW]]`: build two revisions (default `HEAD` and the working tree `.`) under `~/.cache/indd-diff/`, convert every corpus file with both and report which outputs differ diff --git a/README.md b/README.md index 7ef710c..050250f 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,7 @@ version. In 489 of these pairs the IDML shows the same save as the INDD ## Current numbers + From `tools/compare.py --all` with schema validation, without the privately held samples: @@ -158,6 +159,8 @@ privately held samples: | Value coverage, trustworthy pairs | 88.72 % (9,885,604 of 11,142,835 values) | | Value coverage, all pairs | 86.81 % | | Story text, trustworthy pairs | 18,509 of 18,610 stories exact, 100 differ, 1 missing | + + The biggest remaining gaps are elements the converter does not write yet (hyperlinks in some documents, page references, footnote options of @@ -179,6 +182,9 @@ in nearly every sample, so that their INDD fields cannot be located. are not part of the repository; see `tools/validate.sh`. - `python3 -I tools/diff_outputs.py [OLD [NEW]]` compares the output of two revisions over the corpus; a refactoring must change none. +- `python3 -I tools/readme_numbers.py --schemas DIR --jing DIR` runs the + full measurement and updates [Current numbers](#current-numbers). + - `python3 -I tools/inventory.py corpus/` lists each sample's InDesign version and whether it has a matching IDML file. diff --git a/tools/compare.py b/tools/compare.py index 8b59665..4fbc1cd 100644 --- a/tools/compare.py +++ b/tools/compare.py @@ -37,7 +37,9 @@ extra values (values the output has and the reference does not); and the biggest gaps (--gaps N). pairs.tsv, gaps.tsv and gaps-all.tsv go to --out (default target/compare), with values.tsv and values-all.tsv, which list every key with its values and how many are reproduced, and extras.tsv and -extras-all.tsv, which list the extra values by key. +extras-all.tsv, which list the extra values by key. summary.json holds the +numbers of the README's "Current numbers" section, which +tools/readme_numbers.py writes into the README. --shortfalls N lists the N element types and attributes that fall short (missing elements; wrong or missing attribute values; differing story @@ -51,6 +53,7 @@ import argparse import fnmatch import functools import hashlib +import json import os import re import subprocess @@ -935,9 +938,45 @@ def main(): for ex in examples[("Story", "text")]: print(" story mismatch:", ex) headline(args, every, trusted, pair_rows) + write_summary(Path(args.out) / "summary.json", args, check, + (todo, failures, invalid), (others, other_failures, other_invalid), + every, trusted) + + +def write_summary(path, args, check, paired, unpaired_files, every, trusted): + """Write the numbers of the README's "Current numbers" section, and the + options they were measured with, as JSON (tools/readme_numbers.py). + `paired` and `unpaired_files` are (files, failures, invalid).""" + def error(stderr): + lines = [x for x in stderr.splitlines() if not x.startswith("warning: ")] + return lines[-1] if lines else "" + + def files(group, converted): + todo, failures, invalid = group + if not converted: + return None + return {"files": len(todo), + "failures": [error(err) for _, err in failures], + "invalid": len(invalid) if check else None} + + def pairs(st): + return {"pairs": st.docs, "values": st.values, "reproduced": st.reproduced, + "stories": {"exact": st.story_ok["ok"], "differ": st.story_ok["wrong"], + "missing": st.story_ok["missing"]}} + + summary = { + "options": {"all": args.all, "validated": check, "exclude": args.exclude, + "limit": args.limit, "file": args.file}, + "paired": files(paired, True), + "others": files(unpaired_files, args.all), + "trustworthy": pairs(trusted), + "all_pairs": pairs(every), + } + path.write_text(json.dumps(summary, indent=1) + "\n") def gap_name(key): + tag, k = key return f"{tag} {k}" diff --git a/tools/readme_numbers.py b/tools/readme_numbers.py new file mode 100755 index 0000000..c934f76 --- /dev/null +++ b/tools/readme_numbers.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""Write the numbers of a compare.py run into README.md. + +Usage: python3 -I tools/readme_numbers.py [--schemas DIR --jing DIR] + [--summary PATH] [--readme PATH] [--check] + +With --schemas and --jing, first builds the converter (cargo build +--release) and runs the full measurement the README reports: + + python3 -I tools/compare.py --all --exclude own/ --schemas DIR --jing DIR + +Without them, uses the summary of the last run. Either way it reads +target/compare/summary.json (--summary), which compare.py writes, and +replaces the text between the markers `` and +`` in README.md (--readme) with the table of numbers. + +The summary must come from a run over the whole corpus (no --limit or +--file) with --all and schema validation, and without the privately held +samples (--exclude own/); otherwise nothing is written and the exit +status is 1. With --check, nothing is written either; the exit status is +1 if the README's numbers differ from the summary's. +""" + +import argparse +import json +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +START = "" +END = "" + +# Conversion failures of files without a reference IDML, by the start of +# the error message: how the table names them. +FAILURE_KINDS = [ + ("not an INDD file", "not INDD files"), + ("file truncated", "truncated"), + ("not supported yet: InDesign 1.x", "InDesign 1.x files"), + ("no object database", "without an object database"), +] + + +def run_compare(schemas, jing): + """Build the converter and run the full measurement.""" + subprocess.run(["cargo", "build", "--release", "-q"], cwd=ROOT, check=True) + target = Path(os.environ.get("CARGO_TARGET_DIR", ROOT / "target")) + if not target.is_absolute(): + target = ROOT / target + subprocess.run([sys.executable, "-I", str(ROOT / "tools" / "compare.py"), "--all", + "--exclude", "own/", "--schemas", schemas, "--jing", jing, + "--bin", str(target / "release" / "indd")], + cwd=ROOT, check=True) + + +def problems(summary): + """Why the summary cannot give the README's numbers.""" + o = summary["options"] + out = [] + if not any(e.rstrip("/") == "own" for e in o["exclude"]): + out.append("the run included the privately held samples (use --exclude own/)") + if o["limit"] or o["file"]: + out.append("the run covered part of the corpus (--limit or --file)") + if not o["all"]: + out.append("the run left out the files without a reference IDML (use --all)") + if not o["validated"]: + out.append("the run did not validate the output (use --schemas and --jing)") + return out + + +def failures(group, label): + """The conversion failures of a group of files (`label`), by kind.""" + text = f"{len(group['failures']):,} of {group['files']:,} {label}" + if not group["failures"]: + return text + kinds = {} + for message in group["failures"]: + name = next((n for start, n in FAILURE_KINDS if message.startswith(start)), + "other errors") + kinds[name] = kinds.get(name, 0) + 1 + ranked = sorted(kinds.items(), key=lambda kv: -kv[1]) + return text + " (" + ", ".join(f"{n:,} {name}" for name, n in ranked) + ")" + + +def coverage(st): + return 100 * st["reproduced"] / st["values"] if st["values"] else 100.0 + + +def table(summary): + """The text between the markers.""" + paired, others = summary["paired"], summary["others"] + trusted, every = summary["trustworthy"], summary["all_pairs"] + invalid = paired["invalid"] + others["invalid"] + if invalid: + invalid_text = (f"{paired['invalid']:,} paired files, " + f"{others['invalid']:,} other files") + else: + invalid_text = "0" + stories = trusted["stories"] + rows = [ + ("Conversion failures", + f"{failures(paired, 'paired files')}; {failures(others, 'other files')}"), + + ("Schema validation failures", invalid_text), + ("Value coverage, trustworthy pairs", + f"{coverage(trusted):.2f} % ({trusted['reproduced']:,} of " + f"{trusted['values']:,} values)"), + ("Value coverage, all pairs", f"{coverage(every):.2f} %"), + ("Story text, trustworthy pairs", + f"{stories['exact']:,} of {sum(stories.values()):,} stories exact, " + f"{stories['differ']:,} differ, {stories['missing']:,} missing"), + ] + lines = [ + "From `tools/compare.py --all` with schema validation, without the", + "privately held samples:", + "", + "| Measure | Result |", + "|---|---|", + ] + lines += [f"| {k} | {v} |" for k, v in rows] + return "\n".join(lines) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--schemas", help="IDML RelaxNG schema directory: run compare.py first") + ap.add_argument("--jing", help="directory with jing.jar, isorelax.jar, saxon.jar") + ap.add_argument("--summary", default=str(ROOT / "target" / "compare" / "summary.json")) + ap.add_argument("--readme", default=str(ROOT / "README.md")) + ap.add_argument("--check", action="store_true", + help="change nothing; exit 1 if the README's numbers differ") + args = ap.parse_args() + if bool(args.schemas) != bool(args.jing): + ap.error("--schemas and --jing go together") + if args.schemas: + run_compare(args.schemas, args.jing) + + summary = json.loads(Path(args.summary).read_text()) + why = problems(summary) + if why: + print(f"{args.summary}: not used for the README:", file=sys.stderr) + for w in why: + print(f" {w}", file=sys.stderr) + return 1 + + readme = Path(args.readme) + text = readme.read_text() + start, end = text.find(START), text.find(END) + if start < 0 or end < start: + print(f"{readme}: no {START} ... {END} section", file=sys.stderr) + return 1 + new = text[:start + len(START)] + "\n" + table(summary) + "\n" + text[end:] + if new == text: + print(f"{readme}: numbers up to date") + return 0 + if args.check: + print(f"{readme}: numbers differ from {args.summary}") + return 1 + readme.write_text(new) + print(f"{readme}: numbers updated") + return 0 + + +if __name__ == "__main__": + sys.exit(main())