diff --git a/CLAUDE.md b/CLAUDE.md index e6df6a0..1ab4d40 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -87,8 +87,16 @@ separately (rule and evidence: `docs/measurement.md`). - Headline numbers (value coverage, extra values, document scores, ranked gaps; defined in `docs/measurement.md`): `cargo build --release && python3 -I tools/compare.py --exclude own/`. It prints them last and - writes `pairs.tsv`, `gaps*.tsv`, `values*.tsv` and `extras*.tsv` to - `target/compare/`. + writes `pairs.tsv`, `gaps*.tsv`, `values*.tsv`, `extras*.tsv` and + `summary.json` to `target/compare/`. +- README numbers: `python3 -I tools/readme_numbers.py --schemas + --jing ` builds the converter, runs `compare.py --all --exclude + own/` with schema validation and rewrites the README's "Current + numbers" table (between `` and ``) from `target/compare/summary.json`. Without + `--schemas`/`--jing` it uses the last run's summary, which must be from + such a full run; `--check` only reports whether the README is current. + - `python3 -I tools/diff_outputs.py [OLD [NEW]]`: build two revisions (default `HEAD` and the working tree `.`) under `~/.cache/indd-diff/`, convert every corpus file with both and report which outputs differ diff --git a/README.md b/README.md index 7ef710c..050250f 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,7 @@ version. In 489 of these pairs the IDML shows the same save as the INDD ## Current numbers + From `tools/compare.py --all` with schema validation, without the privately held samples: @@ -158,6 +159,8 @@ privately held samples: | Value coverage, trustworthy pairs | 88.72 % (9,885,604 of 11,142,835 values) | | Value coverage, all pairs | 86.81 % | | Story text, trustworthy pairs | 18,509 of 18,610 stories exact, 100 differ, 1 missing | + + The biggest remaining gaps are elements the converter does not write yet (hyperlinks in some documents, page references, footnote options of @@ -179,6 +182,9 @@ in nearly every sample, so that their INDD fields cannot be located. are not part of the repository; see `tools/validate.sh`. - `python3 -I tools/diff_outputs.py [OLD [NEW]]` compares the output of two revisions over the corpus; a refactoring must change none. +- `python3 -I tools/readme_numbers.py --schemas DIR --jing DIR` runs the + full measurement and updates [Current numbers](#current-numbers). + - `python3 -I tools/inventory.py corpus/` lists each sample's InDesign version and whether it has a matching IDML file. diff --git a/tools/compare.py b/tools/compare.py index 8b59665..4fbc1cd 100644 --- a/tools/compare.py +++ b/tools/compare.py @@ -37,7 +37,9 @@ extra values (values the output has and the reference does not); and the biggest gaps (--gaps N). pairs.tsv, gaps.tsv and gaps-all.tsv go to --out (default target/compare), with values.tsv and values-all.tsv, which list every key with its values and how many are reproduced, and extras.tsv and -extras-all.tsv, which list the extra values by key. +extras-all.tsv, which list the extra values by key. summary.json holds the +numbers of the README's "Current numbers" section, which +tools/readme_numbers.py writes into the README. --shortfalls N lists the N element types and attributes that fall short (missing elements; wrong or missing attribute values; differing story @@ -51,6 +53,7 @@ import argparse import fnmatch import functools import hashlib +import json import os import re import subprocess @@ -935,9 +938,45 @@ def main(): for ex in examples[("Story", "text")]: print(" story mismatch:", ex) headline(args, every, trusted, pair_rows) + write_summary(Path(args.out) / "summary.json", args, check, + (todo, failures, invalid), (others, other_failures, other_invalid), + every, trusted) + + +def write_summary(path, args, check, paired, unpaired_files, every, trusted): + """Write the numbers of the README's "Current numbers" section, and the + options they were measured with, as JSON (tools/readme_numbers.py). + `paired` and `unpaired_files` are (files, failures, invalid).""" + def error(stderr): + lines = [x for x in stderr.splitlines() if not x.startswith("warning: ")] + return lines[-1] if lines else "" + + def files(group, converted): + todo, failures, invalid = group + if not converted: + return None + return {"files": len(todo), + "failures": [error(err) for _, err in failures], + "invalid": len(invalid) if check else None} + + def pairs(st): + return {"pairs": st.docs, "values": st.values, "reproduced": st.reproduced, + "stories": {"exact": st.story_ok["ok"], "differ": st.story_ok["wrong"], + "missing": st.story_ok["missing"]}} + + summary = { + "options": {"all": args.all, "validated": check, "exclude": args.exclude, + "limit": args.limit, "file": args.file}, + "paired": files(paired, True), + "others": files(unpaired_files, args.all), + "trustworthy": pairs(trusted), + "all_pairs": pairs(every), + } + path.write_text(json.dumps(summary, indent=1) + "\n") def gap_name(key): + tag, k = key return f"{tag} {k}" diff --git a/tools/readme_numbers.py b/tools/readme_numbers.py new file mode 100755 index 0000000..c934f76 --- /dev/null +++ b/tools/readme_numbers.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""Write the numbers of a compare.py run into README.md. + +Usage: python3 -I tools/readme_numbers.py [--schemas DIR --jing DIR] + [--summary PATH] [--readme PATH] [--check] + +With --schemas and --jing, first builds the converter (cargo build +--release) and runs the full measurement the README reports: + + python3 -I tools/compare.py --all --exclude own/ --schemas DIR --jing DIR + +Without them, uses the summary of the last run. Either way it reads +target/compare/summary.json (--summary), which compare.py writes, and +replaces the text between the markers `` and +`` in README.md (--readme) with the table of numbers. + +The summary must come from a run over the whole corpus (no --limit or +--file) with --all and schema validation, and without the privately held +samples (--exclude own/); otherwise nothing is written and the exit +status is 1. With --check, nothing is written either; the exit status is +1 if the README's numbers differ from the summary's. +""" + +import argparse +import json +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +START = "" +END = "" + +# Conversion failures of files without a reference IDML, by the start of +# the error message: how the table names them. +FAILURE_KINDS = [ + ("not an INDD file", "not INDD files"), + ("file truncated", "truncated"), + ("not supported yet: InDesign 1.x", "InDesign 1.x files"), + ("no object database", "without an object database"), +] + + +def run_compare(schemas, jing): + """Build the converter and run the full measurement.""" + subprocess.run(["cargo", "build", "--release", "-q"], cwd=ROOT, check=True) + target = Path(os.environ.get("CARGO_TARGET_DIR", ROOT / "target")) + if not target.is_absolute(): + target = ROOT / target + subprocess.run([sys.executable, "-I", str(ROOT / "tools" / "compare.py"), "--all", + "--exclude", "own/", "--schemas", schemas, "--jing", jing, + "--bin", str(target / "release" / "indd")], + cwd=ROOT, check=True) + + +def problems(summary): + """Why the summary cannot give the README's numbers.""" + o = summary["options"] + out = [] + if not any(e.rstrip("/") == "own" for e in o["exclude"]): + out.append("the run included the privately held samples (use --exclude own/)") + if o["limit"] or o["file"]: + out.append("the run covered part of the corpus (--limit or --file)") + if not o["all"]: + out.append("the run left out the files without a reference IDML (use --all)") + if not o["validated"]: + out.append("the run did not validate the output (use --schemas and --jing)") + return out + + +def failures(group, label): + """The conversion failures of a group of files (`label`), by kind.""" + text = f"{len(group['failures']):,} of {group['files']:,} {label}" + if not group["failures"]: + return text + kinds = {} + for message in group["failures"]: + name = next((n for start, n in FAILURE_KINDS if message.startswith(start)), + "other errors") + kinds[name] = kinds.get(name, 0) + 1 + ranked = sorted(kinds.items(), key=lambda kv: -kv[1]) + return text + " (" + ", ".join(f"{n:,} {name}" for name, n in ranked) + ")" + + +def coverage(st): + return 100 * st["reproduced"] / st["values"] if st["values"] else 100.0 + + +def table(summary): + """The text between the markers.""" + paired, others = summary["paired"], summary["others"] + trusted, every = summary["trustworthy"], summary["all_pairs"] + invalid = paired["invalid"] + others["invalid"] + if invalid: + invalid_text = (f"{paired['invalid']:,} paired files, " + f"{others['invalid']:,} other files") + else: + invalid_text = "0" + stories = trusted["stories"] + rows = [ + ("Conversion failures", + f"{failures(paired, 'paired files')}; {failures(others, 'other files')}"), + + ("Schema validation failures", invalid_text), + ("Value coverage, trustworthy pairs", + f"{coverage(trusted):.2f} % ({trusted['reproduced']:,} of " + f"{trusted['values']:,} values)"), + ("Value coverage, all pairs", f"{coverage(every):.2f} %"), + ("Story text, trustworthy pairs", + f"{stories['exact']:,} of {sum(stories.values()):,} stories exact, " + f"{stories['differ']:,} differ, {stories['missing']:,} missing"), + ] + lines = [ + "From `tools/compare.py --all` with schema validation, without the", + "privately held samples:", + "", + "| Measure | Result |", + "|---|---|", + ] + lines += [f"| {k} | {v} |" for k, v in rows] + return "\n".join(lines) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--schemas", help="IDML RelaxNG schema directory: run compare.py first") + ap.add_argument("--jing", help="directory with jing.jar, isorelax.jar, saxon.jar") + ap.add_argument("--summary", default=str(ROOT / "target" / "compare" / "summary.json")) + ap.add_argument("--readme", default=str(ROOT / "README.md")) + ap.add_argument("--check", action="store_true", + help="change nothing; exit 1 if the README's numbers differ") + args = ap.parse_args() + if bool(args.schemas) != bool(args.jing): + ap.error("--schemas and --jing go together") + if args.schemas: + run_compare(args.schemas, args.jing) + + summary = json.loads(Path(args.summary).read_text()) + why = problems(summary) + if why: + print(f"{args.summary}: not used for the README:", file=sys.stderr) + for w in why: + print(f" {w}", file=sys.stderr) + return 1 + + readme = Path(args.readme) + text = readme.read_text() + start, end = text.find(START), text.find(END) + if start < 0 or end < start: + print(f"{readme}: no {START} ... {END} section", file=sys.stderr) + return 1 + new = text[:start + len(START)] + "\n" + table(summary) + "\n" + text[end:] + if new == text: + print(f"{readme}: numbers up to date") + return 0 + if args.check: + print(f"{readme}: numbers differ from {args.summary}") + return 1 + readme.write_text(new) + print(f"{readme}: numbers updated") + return 0 + + +if __name__ == "__main__": + sys.exit(main())