Files
indd-utils/tools/compare.py
T
aiongg 722fddfe97 feat: write languages, document users and document settings
designmap.xml gets the document's languages from their INDD objects,
with the quotes every IDML gives each language, the document users from
the document object, and the settings elements every IDML of the version
has (watermark, linked story and page item options, footnote and
endnote options and others) with their common values. User colours are
set by the exporting InDesign, not the INDD, and are left out of value
coverage.

Value coverage on trustworthy pairs: 81.65 % to 82.03 %.
2026-10-07 11:08:03 +00:00

800 lines
33 KiB
Python

#!/usr/bin/env python3
"""Compare converter output with reference IDML files.
For every INDD file in the corpus that has a reference IDML from the same
major version (the `idml` column of corpus/inventory.tsv, written by
tools/inventory.py), convert it with `target/release/indd` and compare the
result with the reference. Files with the same SHA-256 are converted
once. Reports, per element type: how many referenced elements we produce
(matched by Self), and per attribute how often our value matches. Also
reports story text agreement.
Usage: python3 -I tools/compare.py [--limit N] [--detail TAG]... [--file SUBSTR]
[--schemas DIR --jing DIR] [--bin PATH]
[--all] [--exclude PREFIX]... [--jobs N]
[--shortfalls N] [--trusted] [--stale N]
[--gaps N] [--out DIR]
Run from the repository root after `cargo build --release`. With --schemas
and --jing, also validates every output with tools/validate.sh, in batches
of VALIDATE_BATCH files (one Jing run per schema for a whole batch). --bin runs
another converter binary, for example a copy of the previous build.
--all also converts every other INDD and INDT file under corpus/ (any
version, either byte order, without a usable IDML), validates the output
if schemas are given, and reports failures for those files separately.
--exclude leaves out files whose path under corpus/ starts with PREFIX.
Converter warnings are counted by kind over all converted files.
Pairs are trustworthy or stale (the IDML shows a different save than the
INDD; rule and evidence in docs/measurement.md). Totals are printed for all
pairs and for trustworthy pairs; --trusted limits the element tables,
shortfalls and details to trustworthy pairs, --stale N lists stale pairs.
Last comes the headline (docs/measurement.md): value coverage, the share
of reference values the conversion reproduces, over trustworthy pairs and
over all pairs; document scores (pairs with coverage >= 99 % and >= 99.9 %);
and the biggest gaps (--gaps N). pairs.tsv, gaps.tsv and gaps-all.tsv go to
--out (default target/compare), with values.tsv and values-all.tsv, which
list every key with its values and how many are reproduced.
--shortfalls N lists the N element types and attributes that fall short
(missing elements; wrong or missing attribute values; differing story
text) in the most documents.
Embedded file data (`Contents`) is compared by digest, so it is reported as
`md5:<hex> <length>` rather than as the full text.
"""
import argparse
import fnmatch
import functools
import hashlib
import os
import re
import subprocess
import sys
import tempfile
import xml.etree.ElementTree as ET
import zipfile
from collections import Counter, defaultdict
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timedelta, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
BIN = ROOT / "target" / "release" / "indd"
# Converted files validated together in one tools/validate.sh run.
VALIDATE_BATCH = 600
def inventory():
"""Rows of corpus/inventory.tsv (tools/inventory.py), as dicts."""
lines = (ROOT / "corpus" / "inventory.tsv").read_text().splitlines()
cols = lines[0].split("\t")
return [dict(zip(cols, line.split("\t"))) for line in lines[1:]]
def digest(indd, known):
"""SHA-256 of a corpus file, from the inventory if it lists the file."""
path = str(indd.relative_to(ROOT / "corpus"))
if path in known:
return known[path]
return hashlib.sha256(indd.read_bytes()).hexdigest()
def pairs(limit, substr, exclude, seen, known):
"""Corpus files with a same-version reference IDML, without duplicates.
Adds the digest of each file to `seen`."""
out = []
for row in inventory():
path, order, ver, dom = row["path"], row["order"], row["version"], row["idml_dom"]
if order != "LE" or not dom or ver.split(".")[0] != dom.split(".")[0]:
continue
if substr and substr not in path or any(path.startswith(e) for e in exclude):
continue
indd = ROOT / "corpus" / path
d = digest(indd, known)
if d in seen:
continue
seen.add(d)
idml = row.get("idml") or str(Path(path).with_suffix(".idml"))
out.append((indd, ROOT / "corpus" / idml))
return out[:limit] if limit else out
def unpaired(substr, exclude, seen, known):
"""Every other INDD and INDT file under corpus/, without duplicates."""
out = []
for indd in sorted((ROOT / "corpus").rglob("*")):
if indd.suffix.lower() not in (".indd", ".indt") or not indd.is_file():
continue
path = str(indd.relative_to(ROOT / "corpus"))
if substr and substr not in path or any(path.startswith(e) for e in exclude):
continue
d = digest(indd, known)
if d in seen:
continue
seen.add(d)
out.append(indd)
return out
def warning_kind(msg):
"""A warning with its numbers and quoted names replaced, for counting."""
msg = re.sub(r'"[^"]*"', '"…"', msg)
msg = re.sub(r"\b0x[0-9a-fA-F]+\b", "#", msg)
return re.sub(r"\b\d+(\.\d+)?\b", "#", msg)
XMP_NS = {"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"xmp": "http://ns.adobe.com/xap/1.0/"}
# A pair is stale if the INDD was saved more than this long after the
# IDML's metadata date (docs/measurement.md).
STALE_AFTER = timedelta(hours=1)
def modify_date(packet):
"""xmp:ModifyDate of the document in an XMP packet, or None."""
if not packet:
return None
start = packet.find(b"<x:xmpmeta")
end = packet.rfind(b"</x:xmpmeta>")
if start < 0 or end < 0:
return None
try:
meta = ET.fromstring(packet[start:end + len(b"</x:xmpmeta>")])
except ET.ParseError:
return None
key = "{%s}ModifyDate" % XMP_NS["xmp"]
for desc in meta.findall("rdf:RDF/rdf:Description", XMP_NS):
text = desc.get(key) or desc.findtext(key)
if text:
try:
d = datetime.fromisoformat(text.strip().replace("Z", "+00:00"))
except ValueError:
return None
# Without a time zone, compare clock times.
return d.replace(tzinfo=d.tzinfo or timezone.utc)
return None
def idml_modify_date(idml):
"""xmp:ModifyDate in an IDML package's META-INF/metadata.xml, or None."""
try:
with zipfile.ZipFile(idml) as z:
return modify_date(z.read("META-INF/metadata.xml"))
except (KeyError, zipfile.BadZipFile, OSError):
return None
def stale_reasons(indd_date, idml_date, indd_uids, ref_el, ref_st, our_st):
"""Why a pair's IDML may not show the same save as its INDD (empty if
there is no sign of that). See docs/measurement.md for the evidence."""
out = []
if indd_date is None or idml_date is None:
out.append("no ModifyDate")
elif indd_date < idml_date:
out.append("IDML modified after the INDD")
elif indd_date - idml_date > STALE_AFTER:
out.append("INDD saved over an hour after the IDML")
ref_uids = {int(s[1:], 16) for _, s in ref_el if re.fullmatch(r"u[0-9a-f]+", s)}
if indd_uids is not None and ref_uids - indd_uids:
out.append("IDML objects not in the INDD")
if set(our_st) - set(ref_st):
out.append("INDD stories not in the IDML")
return out
def validate(outs, args):
"""Schema errors of outputs, as {path: [messages]} (valid outputs are
left out). Runs tools/validate.sh once over all of them; it batches the
Jing runs."""
errs = defaultdict(list)
if not outs:
return errs
env = dict(os.environ, VALIDATE_JOBS=str(max(1, args.jobs)))
v = subprocess.run(
[ROOT / "tools" / "validate.sh", *map(str, outs), args.schemas, args.jing],
capture_output=True, text=True, env=env)
for line in v.stdout.splitlines():
path, _, msg = line.partition("\t")
errs[path].append(msg)
if v.returncode not in (0, 1) or v.returncode == 1 and not errs or "jing" in errs:
raise SystemExit(f"tools/validate.sh failed: {v.stderr.strip()} {errs.get('jing')}")
return errs
def text_ranges(story):
"""Map text offset -> attributes of the paragraph and character range
starting there, with keys prefixed PSR. and CSR."""
out = {}
pos = 0
for psr in story.findall("ParagraphStyleRange"):
pa = {"PSR." + k: v for k, v in props(psr).items()}
for csr in psr.findall("CharacterStyleRange"):
start = pos
for el in csr:
if el.tag == "Content":
pos += len((el.text or "").encode("utf-16-le")) // 2
elif el.tag != "Properties":
pos += 1
if pos > start:
ca = {"CSR." + k: v for k, v in props(csr).items()}
out[start] = {**pa, **ca}
return out
def load(path):
"""Map Self -> element, plus story texts, from an IDML package."""
z = zipfile.ZipFile(path)
elements = {}
stories = {}
ranges = {}
for name in z.namelist():
if not name.endswith(".xml"):
continue
try:
root = ET.fromstring(z.read(name))
except ET.ParseError as e:
raise SystemExit(f"{path}: {name}: {e}")
for el in root.iter():
s = el.get("Self")
if s is not None:
elements.setdefault((el.tag, s), el)
# Children without Self (TextFramePreference, ...) are
# compared as "<parent tag>/<child tag>" of the parent.
for ch in el:
if ch.get("Self") is not None or ch.tag == "Properties":
continue
if len(ch.attrib):
elements.setdefault((f"{el.tag}/{ch.tag}", s), ch)
else:
# A container without attributes (TransparencySetting):
# compare its children as "<parent>/<child>/<grandchild>".
for g in ch:
if g.get("Self") is None and len(g.attrib):
elements.setdefault((f"{el.tag}/{ch.tag}/{g.tag}", s), g)
if name == "Resources/Preferences.xml":
for el in root:
elements.setdefault((el.tag, "Preferences"), el)
if name.startswith("Stories/"):
for st in root.iter("Story"):
stories[st.get("Self")] = story_text(st)
ranges[st.get("Self")] = text_ranges(st)
return elements, stories, ranges
def story_text(story):
parts = []
for el in story.iter():
if el.tag == "Content":
parts.append(el.text or "")
elif el.tag == "Br":
parts.append("\n")
return "".join(parts)
def norm(v):
"""Normalise numbers so 1 == 1.0 and tiny float noise is ignored."""
def fix(tok):
try:
f = float(tok)
except ValueError:
return tok
return f"{f:.6g}"
return " ".join(fix(t) for t in v.split(" "))
def canon(el):
"""A structured property (TabList, AllNestedStyles, ...) as one string:
each leaf as tag=text with numbers normalised, in document order."""
if len(el) == 0 and el.get("type") != "list":
return f"{el.tag}={norm(el.text or '')}"
return el.tag + "[" + ",".join(canon(c) for c in el) + "]"
def props(el):
"""Attributes plus <Properties> children (as P.Name)."""
a = dict(el.attrib)
p = el.find("Properties")
if p is not None:
for c in p:
if c.get("type") == "list":
a["P." + c.tag] = canon(c)
elif len(c) == 0:
text = c.text or ""
extra = sorted((k, v) for k, v in c.attrib.items() if k != "type")
if extra:
text = " ".join(f"{k}={v}" for k, v in extra)
if c.tag == "Contents":
n = norm(text)
text = f"md5:{hashlib.md5(n.encode()).hexdigest()} {len(n)}"
a["P." + c.tag] = text
return a
# Values left out of value coverage because they cannot be derived from an
# INDD file: (tag pattern, attribute or P.<Properties child>, reason), tag
# patterns as in fnmatch. docs/measurement.md explains each entry.
EXCLUDED = [
("Font", "Status", "whether the font is installed on the exporting computer"),
("Document/{http://ns.adobe.com/AdobeInDesign/idml/1.0/packaging}*", "src",
"file names of the package parts, chosen by the IDML writer"),
("DocumentUser", "P.UserColor",
"colour the exporting InDesign gives each user, not the colour in the INDD"),
]
@functools.cache
def excluded(tag, key):
return any(k == key and fnmatch.fnmatchcase(tag, t) for t, k, _ in EXCLUDED)
def tree(el):
"""An element with its attributes, text and descendants as one string,
numbers normalised: the value of a structured <Properties> child such as
PathGeometry."""
attrs = ",".join(f"{k}={norm(v)}" for k, v in sorted(el.attrib.items()) if k != "type")
kids = ",".join(tree(c) for c in el)
return f"{el.tag}({attrs})[{kids}]{norm((el.text or '').strip())}"
def values(tag, el):
"""The values of an element that value coverage counts: attributes and
<Properties> children, without Self and without EXCLUDED values."""
a = props(el)
p = el.find("Properties")
if p is not None:
for c in p:
if c.get("type") != "list" and len(c):
a["P." + c.tag] = tree(c)
return {k: v for k, v in a.items() if k != "Self" and not excluded(tag, k)}
# Gap keys for values that could not be compared one by one.
NO_ELEMENT = "(element not produced)"
NO_TEXT = "(story text differs)"
ELEMENT = "(element)"
class Stats:
"""Comparison counts over a set of pairs."""
def __init__(self):
self.docs = 0
# Value coverage: reference values, and those reproduced exactly.
self.values = 0
self.reproduced = 0
# (tag, key) -> values wrong / missing, and documents affected.
self.gap_wrong = Counter()
self.gap_missing = Counter()
self.gap_docs = Counter()
self.gap_total = Counter()
# (pair, reproduced, values) for each pair.
self.pairs = []
self.found = Counter()
self.total = Counter()
self.attr_ok = defaultdict(Counter)
self.story_ok = Counter()
self.short_docs = Counter() # (kind, tag, attribute) -> documents affected
def add(self, other):
self.docs += other.docs
self.values += other.values
self.reproduced += other.reproduced
for c in ("gap_wrong", "gap_missing", "gap_docs", "gap_total"):
getattr(self, c).update(getattr(other, c))
self.pairs += other.pairs
self.found.update(other.found)
self.total.update(other.total)
for k, c in other.attr_ok.items():
self.attr_ok[k].update(c)
self.story_ok.update(other.story_ok)
self.short_docs.update(other.short_docs)
def coverage(self):
return self.reproduced / self.values if self.values else 1.0
def quantile(self, q):
"""Value coverage of the pair at quantile q (0 = lowest)."""
cov = sorted(ok / n if n else 1.0 for _, ok, n in self.pairs)
return cov[min(len(cov) - 1, int(q * len(cov)))] if cov else 1.0
def document_score(self, at_least):
"""Pairs whose value coverage is at least `at_least`."""
return sum(1 for _, ok, n in self.pairs if n == 0 or ok / n >= at_least)
def gaps(self):
"""(tag, key), documents, values wrong, values missing; most
documents first, then most values."""
rows = [(k, d, self.gap_wrong[k], self.gap_missing[k]) for k, d in self.gap_docs.items()]
return sorted(rows, key=lambda r: (-r[1], -(r[2] + r[3]), r[0]))
def story_line(self):
st = sum(self.story_ok.values())
return (f"{self.story_ok['ok']}/{st} exact, {self.story_ok['wrong']} differ, "
f"{self.story_ok['missing']} missing")
def compare_pair(name, ref, ours, examples, show):
"""Stats of one pair: `ref` and `ours` are load() results. Adds up to
`show` mismatches per attribute to `examples`."""
ref_el, ref_st, ref_rg = ref
our_el, our_st, our_rg = ours
p = Stats()
p.docs = 1
short = set()
for (tag, s), el in ref_el.items():
p.total[tag] += 1
mine = our_el.get((tag, s))
if mine is None:
short.add(("element", tag, ""))
continue
p.found[tag] += 1
mine = props(mine)
for k, v in props(el).items():
if k == "Self":
continue
if k not in mine:
p.attr_ok[(tag, k)]["missing"] += 1
short.add(("attribute", tag, k))
elif norm(mine[k]) == norm(v):
p.attr_ok[(tag, k)]["ok"] += 1
else:
p.attr_ok[(tag, k)]["wrong"] += 1
short.add(("attribute", tag, k))
if len(examples[(tag, k)]) < show:
examples[(tag, k)].append((name, s, v, mine[k]))
for sid, rgs in ref_rg.items():
if our_st.get(sid) != ref_st.get(sid):
continue
mine = our_rg.get(sid, {})
for start, attrs in rgs.items():
p.total["TextRange"] += 1
if start not in mine:
short.add(("element", "TextRange", ""))
continue
p.found["TextRange"] += 1
for k, v in attrs.items():
o = mine[start].get(k)
if o is None:
p.attr_ok[("TextRange", k)]["missing"] += 1
short.add(("attribute", "TextRange", k))
elif norm(o) == norm(v):
p.attr_ok[("TextRange", k)]["ok"] += 1
else:
p.attr_ok[("TextRange", k)]["wrong"] += 1
short.add(("attribute", "TextRange", k))
if len(examples[("TextRange", k)]) < show:
examples[("TextRange", k)].append((name, sid, start, v, o))
for sid, text in ref_st.items():
if sid not in our_st:
p.story_ok["missing"] += 1
short.add(("element", "Story", ""))
elif our_st[sid] == text:
p.story_ok["ok"] += 1
else:
p.story_ok["wrong"] += 1
short.add(("text", "Story", ""))
if len(examples[("Story", "text")]) < max(show, 3):
examples[("Story", "text")].append((name, sid, text[:80], our_st[sid][:80]))
p.short_docs.update(short)
value_coverage(p, ref, ours)
return p
def value_coverage(p, ref, ours):
"""Count reference values and the reproduced ones into `p`: element
presence, attribute and <Properties> values, story text, and the start
and attributes of each text range."""
ref_el, ref_st, ref_rg = ref
our_el, our_st, our_rg = ours
total = Counter()
wrong = Counter()
missing = Counter()
def compare(tag, theirs, mine):
for k, v in theirs.items():
key = (tag, k)
total[key] += 1
o = mine.get(k)
if o is None:
missing[key] += 1
elif norm(o) != norm(v):
wrong[key] += 1
for (tag, s), el in ref_el.items():
theirs = values(tag, el)
mine = our_el.get((tag, s))
if mine is None:
n = 1 + len(theirs)
total[(tag, NO_ELEMENT)] += n
missing[(tag, NO_ELEMENT)] += n
continue
total[(tag, ELEMENT)] += 1
compare(tag, theirs, values(tag, mine))
for sid, text in ref_st.items():
key = ("Story", "(text)")
total[key] += 1
if sid not in our_st:
missing[key] += 1
elif our_st[sid] != text:
wrong[key] += 1
for sid, rgs in ref_rg.items():
if our_st.get(sid) != ref_st.get(sid):
n = sum(1 + len(a) for a in rgs.values())
total[("TextRange", NO_TEXT)] += n
missing[("TextRange", NO_TEXT)] += n
continue
mine = our_rg.get(sid, {})
for start, attrs in rgs.items():
if start not in mine:
n = 1 + len(attrs)
total[("TextRange", NO_ELEMENT)] += n
missing[("TextRange", NO_ELEMENT)] += n
continue
total[("TextRange", ELEMENT)] += 1
compare("TextRange", attrs, mine[start])
n = sum(total.values())
bad = sum(wrong.values()) + sum(missing.values())
p.values += n
p.reproduced += n - bad
p.gap_total.update(total)
p.gap_wrong.update(wrong)
p.gap_missing.update(missing)
p.gap_docs.update(k for k in total if wrong[k] or missing[k])
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--limit", type=int, default=0)
ap.add_argument("--detail", action="append", default=[],
help="print attribute table for this tag (repeatable)")
ap.add_argument("--file", help="only pairs whose path contains this")
ap.add_argument("--show", type=int, default=0, help="show N mismatches per attribute")
ap.add_argument("--schemas", help="IDML RelaxNG schema directory (validate output)")
ap.add_argument("--jing", help="directory with jing.jar, isorelax.jar, saxon.jar")
ap.add_argument("--bin", default=str(BIN), help="converter binary")
ap.add_argument("--all", action="store_true",
help="also convert and validate files without a reference IDML")
ap.add_argument("--exclude", action="append", default=[],
help="leave out files whose path under corpus/ starts with this (repeatable)")
ap.add_argument("--warnings", type=int, default=15,
help="number of warning kinds to list")
ap.add_argument("--shortfalls", type=int, default=0,
help="rank the N biggest shortfalls by documents affected")
ap.add_argument("--trusted", action="store_true",
help="element tables, shortfalls and details over trustworthy pairs only")
ap.add_argument("--stale", type=int, default=0,
help="list N stale pairs with their reasons")
ap.add_argument("--gaps", type=int, default=30,
help="print the N biggest value coverage gaps")
ap.add_argument("--out", default=str(ROOT / "target" / "compare"),
help="directory for pairs.tsv and gaps*.tsv (default target/compare)")
ap.add_argument("--jobs", type=int, default=os.cpu_count() or 1,
help="conversions and schema validations run in parallel")
args = ap.parse_args()
check = bool(args.schemas and args.jing)
every = Stats() # all pairs
trusted = Stats() # pairs without a sign of staleness
stale = [] # (name, reasons)
pair_rows = [] # (name, Stats, stale reasons) per pair
examples = defaultdict(list)
failures = []
warnings = Counter()
warned_files = Counter()
seen = set()
known = {r["path"]: r["sha256"] for r in inventory() if r.get("sha256")}
todo = pairs(args.limit, args.file, args.exclude, seen, known)
others = unpaired(args.file, args.exclude, seen, known) if args.all else []
other_failures = []
pool = ThreadPoolExecutor(max(1, args.jobs))
def convert(indd, out):
"""Convert one file; runs in the pool."""
return subprocess.run([args.bin, "convert", indd, out], capture_output=True, text=True)
def convert_pair(indd, idml, out):
"""Convert one paired file and read what stale_reasons needs: the
INDD's UIDs and the ModifyDate of both files; runs in the pool."""
r = convert(indd, out)
x = subprocess.run([args.bin, "xmp", indd], capture_output=True)
u = subprocess.run([args.bin, "uids", indd], capture_output=True, text=True)
uids = ({int(line.split("\t")[0]) for line in u.stdout.splitlines()}
if u.returncode == 0 else None)
dates = (modify_date(x.stdout) if x.returncode == 0 else None, idml_modify_date(idml))
return r, dates, uids
def count_warnings(r):
kinds = set()
for line in r.stderr.splitlines():
if line.startswith("warning: "):
kind = warning_kind(line[len("warning: "):])
warnings[kind] += 1
kinds.add(kind)
for kind in kinds:
warned_files[kind] += 1
other_invalid = []
invalid = []
with tempfile.TemporaryDirectory() as tmp:
jobs = [pool.submit(convert, indd, Path(tmp) / f"other{n}.idml")
for n, indd in enumerate(others)]
jobs += [pool.submit(convert_pair, indd, idml, Path(tmp) / f"pair{n}.idml")
for n, (indd, idml) in enumerate(todo)]
# Converted files waiting for validation: (output, name, list of
# invalid files it belongs to). They are validated in batches, which
# bounds the space the unpacked outputs take.
to_check = []
def flush(at_least):
if len(to_check) < at_least:
return
errs = validate([out for out, _, _ in to_check], args)
for out, name, bad in to_check:
if errs.get(str(out)):
bad.append((name, errs[str(out)]))
out.unlink(missing_ok=True)
to_check.clear()
for n, indd in enumerate(others):
r = jobs[n].result()
count_warnings(r)
out = Path(tmp) / f"other{n}.idml"
if r.returncode != 0:
other_failures.append((indd.name, r.stderr.strip()))
elif check:
to_check.append((out, indd.name, other_invalid))
flush(VALIDATE_BATCH)
continue
out.unlink(missing_ok=True)
for n, (indd, idml) in enumerate(todo):
out = Path(tmp) / f"pair{n}.idml"
r, (indd_date, idml_date), indd_uids = jobs[len(others) + n].result()
count_warnings(r)
if r.returncode != 0:
failures.append((indd.name, r.stderr.strip()))
continue
if check:
to_check.append((out, indd.name, invalid))
ref = load(idml)
ours = load(out)
reasons = stale_reasons(indd_date, idml_date, indd_uids, ref[0], ref[1], ours[1])
p = compare_pair(indd.name, ref, ours, examples, args.show)
rel = str(indd.relative_to(ROOT / "corpus"))
p.pairs = [(rel, p.reproduced, p.values)]
pair_rows.append((rel, p, reasons))
every.add(p)
if reasons:
stale.append((rel, reasons))
else:
trusted.add(p)
if check:
flush(VALIDATE_BATCH)
else:
out.unlink(missing_ok=True)
pool.shutdown()
flush(1)
print(f"conversion failures: {len(failures)} of {len(todo)} paired files")
for name, err in failures[:10]:
print(f" {name}: {err}")
if check:
print(f"schema validation failures: {len(invalid)}")
for name, errs in invalid[:10]:
print(f" {name}: {len(errs)} errors, {errs[:3]}")
if args.all:
print(f"files without a reference: {len(others)}, "
f"conversion failures: {len(other_failures)}")
for name, err in other_failures[:10]:
print(f" {name}: {err}")
if check:
print(f"files without a reference: schema validation failures: {len(other_invalid)}")
for name, errs in other_invalid[:10]:
print(f" {name}: {len(errs)} errors, {errs[:3]}")
print(f"warnings: {sum(warnings.values())} (count, files, kind)")
for kind, count in warnings.most_common(args.warnings):
print(f" {count:6} {warned_files[kind]:4} {kind[:150]}")
print(f"story text: {every.story_line()}")
reasons = Counter(r for _, rs in stale for r in rs)
print(f"trustworthy pairs: {trusted.docs} of {every.docs}; stale: {len(stale)} "
f"({', '.join(f'{r}: {n}' for r, n in reasons.most_common())})")
print(f"trustworthy pairs: story text: {trusted.story_line()}")
for name, rs in stale[:args.stale]:
print(f" stale: {name}: {'; '.join(rs)}")
st = trusted if args.trusted else every
which = "trustworthy pairs" if args.trusted else "all pairs"
print(f"\nelements, {which} (produced / in reference):")
for tag, n in st.total.most_common():
print(f" {tag:34} {st.found[tag]:6} / {n:6}")
if args.shortfalls:
print(f"\nshortfalls by documents affected (of {st.docs} {which}):")
def instances(kind, tag, k):
"""How many elements, values or stories fall short."""
if kind == "attribute":
return st.attr_ok[(tag, k)]["wrong"] + st.attr_ok[(tag, k)]["missing"]
if kind == "element":
return st.total[tag] - st.found[tag]
return st.story_ok["wrong"]
ranked = sorted(st.short_docs.items(), key=lambda kv: (-kv[1], -instances(*kv[0]), kv[0]))
for (kind, tag, k), n in ranked[:args.shortfalls]:
what = f"{st.story_ok['wrong']} stories differ"
if kind == "attribute":
c = st.attr_ok[(tag, k)]
what = f"{c['wrong']} wrong, {c['missing']} missing of {sum(c.values())}"
elif kind == "element":
what = f"{st.total[tag] - st.found[tag]} of {st.total[tag]} not produced"
print(f" {n:5} {tag + (' ' + k if k else ''):56} {what}")
for tag in args.detail:
print(f"\nattributes of {tag}, {which} (ok / wrong / missing):")
rows = [(k, c) for (t, k), c in st.attr_ok.items() if t == tag]
for k, c in sorted(rows, key=lambda kc: -sum(kc[1].values())):
print(f" {k:40} {c['ok']:6} {c['wrong']:6} {c['missing']:6}")
for ex in examples[(tag, k)]:
print(f" {ex}")
for ex in examples[("Story", "text")]:
print(" story mismatch:", ex)
headline(args, every, trusted, pair_rows)
def gap_name(key):
tag, k = key
return f"{tag} {k}"
def write_gaps(path, st):
with open(path, "w") as f:
f.write("tag\tkey\tdocuments\twrong\tmissing\tvalues\n")
for key, docs, w, m in st.gaps():
f.write(f"{key[0]}\t{key[1]}\t{docs}\t{w}\t{m}\t{st.gap_total[key]}\n")
def write_values(path, st):
"""Every key with its values and how many are reproduced, for checking
that a change reproduces no fewer values of any key."""
with open(path, "w") as f:
f.write("tag\tkey\tvalues\treproduced\twrong\tmissing\n")
for key in sorted(st.gap_total):
w, m = st.gap_wrong[key], st.gap_missing[key]
f.write(f"{key[0]}\t{key[1]}\t{st.gap_total[key]}\t{st.gap_total[key] - w - m}"
f"\t{w}\t{m}\n")
def headline(args, every, trusted, pair_rows):
"""Print value coverage, document scores and the biggest gaps; write
pairs.tsv, gaps.tsv (trustworthy pairs) and gaps-all.tsv to --out."""
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
with open(out / "pairs.tsv", "w") as f:
f.write("path\ttrusted\tcoverage\treproduced\tvalues\tstale reasons\n")
for name, p, reasons in sorted(pair_rows, key=lambda r: r[1].coverage()):
f.write(f"{name}\t{not reasons}\t{p.coverage():.5f}\t{p.reproduced}\t{p.values}"
f"\t{'; '.join(reasons)}\n")
write_gaps(out / "gaps.tsv", trusted)
write_gaps(out / "gaps-all.tsv", every)
write_values(out / "values.tsv", trusted)
write_values(out / "values-all.tsv", every)
print("\nheadline (docs/measurement.md):")
for label, st in (("trustworthy pairs", trusted), ("all pairs", every)):
n = st.docs or 1
print(f" {label}: value coverage {st.coverage():.2%} "
f"({st.reproduced} of {st.values} values); documents at >= 99 %: "
f"{st.document_score(0.99)}/{st.docs} ({st.document_score(0.99) / n:.1%}), "
f">= 99.9 %: {st.document_score(0.999)}/{st.docs} "
f"({st.document_score(0.999) / n:.1%})")
print(f" per pair: lowest {st.quantile(0):.1%}, 10th percentile {st.quantile(0.1):.1%}, "
f"median {st.quantile(0.5):.1%}, 90th percentile {st.quantile(0.9):.1%}, "
f"highest {st.quantile(1):.1%}")
rows = trusted.gaps()
print(f"\nvalue gaps, trustworthy pairs, by documents affected "
f"(documents, values wrong, missing, of values; all in {out / 'gaps.tsv'}):")
for key, docs, w, m in rows[:args.gaps]:
print(f" {docs:5} {w:7} {m:7} {trusted.gap_total[key]:8} {gap_name(key)[:90]}")
print("\nvalue gaps, trustworthy pairs, by values (documents, values wrong, missing, of values):")
for key, docs, w, m in sorted(rows, key=lambda r: -(r[2] + r[3]))[:args.gaps // 2]:
print(f" {docs:5} {w:7} {m:7} {trusted.gap_total[key]:8} {gap_name(key)[:90]}")
if __name__ == "__main__":
main()