mirror of
https://github.com/storytold/indd-utils.git
synced 2026-11-19 16:50:48 +01:00
text_ranges read only ParagraphStyleRange children of the story and their CharacterStyleRange children. Ranges inside HyperlinkTextSource, CrossReferenceSource, XMLElement, Change and EndnoteRange were skipped, and their text was left out of the offsets, so every later range of the story started earlier in the reference than in the output. It now walks through those wrappers at any depth. Trustworthy pairs (compare.py --exclude own/): - hyperlink and cross-reference sources alone: reference values 11,142,835 -> 11,158,348, reproduced 10,501,458 -> 10,505,081, extra values 30,237 -> 26,634 - with XMLElement, Change and EndnoteRange (counted as the headline must include everything InDesign writes): reference values 11,508,691, coverage 94.24 % -> 91.28 %; 348,306 of the 350,343 new values are in one document - story text unchanged (18,517 of 18,610); documents >= 99 %: 18 - three keys reproduce one value fewer (CSR.Underline, PSR.Justification, PSR.P.TabList): ranges that matched at a shifted offset before
935 lines
39 KiB
Python
935 lines
39 KiB
Python
#!/usr/bin/env python3
|
|
"""Compare converter output with reference IDML files.
|
|
|
|
For every INDD file in the corpus that has a reference IDML from the same
|
|
major version (the `idml` column of corpus/inventory.tsv, written by
|
|
tools/inventory.py), convert it with `target/release/indd` and compare the
|
|
result with the reference. Files with the same SHA-256 are converted
|
|
once. Reports, per element type: how many referenced elements we produce
|
|
(matched by Self), and per attribute how often our value matches. Also
|
|
reports story text agreement.
|
|
|
|
Usage: python3 -I tools/compare.py [--limit N] [--detail TAG]... [--file SUBSTR]
|
|
[--schemas DIR --jing DIR] [--bin PATH]
|
|
[--all] [--exclude PREFIX]... [--jobs N]
|
|
[--shortfalls N] [--trusted] [--stale N]
|
|
[--gaps N] [--out DIR]
|
|
Run from the repository root after `cargo build --release`. With --schemas
|
|
and --jing, also validates every output with tools/validate.sh, in batches
|
|
of VALIDATE_BATCH files (one Jing run per schema for a whole batch). --bin runs
|
|
another converter binary, for example a copy of the previous build.
|
|
|
|
--all also converts every other INDD and INDT file under corpus/ (any
|
|
version, either byte order, without a usable IDML), validates the output
|
|
if schemas are given, and reports failures for those files separately.
|
|
--exclude leaves out files whose path under corpus/ starts with PREFIX.
|
|
Converter warnings are counted by kind over all converted files.
|
|
|
|
Pairs are trustworthy or stale (the IDML shows a different save than the
|
|
INDD; rule and evidence in docs/measurement.md). Totals are printed for all
|
|
pairs and for trustworthy pairs; --trusted limits the element tables,
|
|
shortfalls and details to trustworthy pairs, --stale N lists stale pairs.
|
|
|
|
Last comes the headline (docs/measurement.md): value coverage, the share
|
|
of reference values the conversion reproduces, over trustworthy pairs and
|
|
over all pairs; document scores (pairs with coverage >= 99 % and >= 99.9 %);
|
|
extra values (values the output has and the reference does not); and the
|
|
biggest gaps (--gaps N). pairs.tsv, gaps.tsv and gaps-all.tsv go to --out
|
|
(default target/compare), with values.tsv and values-all.tsv, which list
|
|
every key with its values and how many are reproduced, and extras.tsv and
|
|
extras-all.tsv, which list the extra values by key.
|
|
|
|
--shortfalls N lists the N element types and attributes that fall short
|
|
(missing elements; wrong or missing attribute values; differing story
|
|
text) in the most documents.
|
|
|
|
Embedded file data (`Contents`) is compared by digest, so it is reported as
|
|
`md5:<hex> <length>` rather than as the full text.
|
|
"""
|
|
|
|
import argparse
|
|
import fnmatch
|
|
import functools
|
|
import hashlib
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import xml.etree.ElementTree as ET
|
|
import zipfile
|
|
from collections import Counter, defaultdict
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
BIN = ROOT / "target" / "release" / "indd"
|
|
# Converted files validated together in one tools/validate.sh run.
|
|
VALIDATE_BATCH = 600
|
|
|
|
|
|
def inventory():
|
|
"""Rows of corpus/inventory.tsv (tools/inventory.py), as dicts."""
|
|
lines = (ROOT / "corpus" / "inventory.tsv").read_text().splitlines()
|
|
cols = lines[0].split("\t")
|
|
return [dict(zip(cols, line.split("\t"))) for line in lines[1:]]
|
|
|
|
|
|
def digest(indd, known):
|
|
"""SHA-256 of a corpus file, from the inventory if it lists the file."""
|
|
path = str(indd.relative_to(ROOT / "corpus"))
|
|
if path in known:
|
|
return known[path]
|
|
return hashlib.sha256(indd.read_bytes()).hexdigest()
|
|
|
|
|
|
def pairs(limit, substr, exclude, seen, known):
|
|
"""Corpus files with a same-version reference IDML, without duplicates.
|
|
Adds the digest of each file to `seen`."""
|
|
out = []
|
|
for row in inventory():
|
|
path, order, ver, dom = row["path"], row["order"], row["version"], row["idml_dom"]
|
|
if order != "LE" or not dom or ver.split(".")[0] != dom.split(".")[0]:
|
|
continue
|
|
if substr and substr not in path or any(path.startswith(e) for e in exclude):
|
|
continue
|
|
indd = ROOT / "corpus" / path
|
|
d = digest(indd, known)
|
|
if d in seen:
|
|
continue
|
|
seen.add(d)
|
|
idml = row.get("idml") or str(Path(path).with_suffix(".idml"))
|
|
out.append((indd, ROOT / "corpus" / idml))
|
|
return out[:limit] if limit else out
|
|
|
|
|
|
def unpaired(substr, exclude, seen, known):
|
|
"""Every other INDD and INDT file under corpus/, without duplicates."""
|
|
out = []
|
|
for indd in sorted((ROOT / "corpus").rglob("*")):
|
|
if indd.suffix.lower() not in (".indd", ".indt") or not indd.is_file():
|
|
continue
|
|
path = str(indd.relative_to(ROOT / "corpus"))
|
|
if substr and substr not in path or any(path.startswith(e) for e in exclude):
|
|
continue
|
|
d = digest(indd, known)
|
|
if d in seen:
|
|
continue
|
|
seen.add(d)
|
|
out.append(indd)
|
|
return out
|
|
|
|
|
|
def warning_kind(msg):
|
|
"""A warning with its numbers and quoted names replaced, for counting."""
|
|
msg = re.sub(r'"[^"]*"', '"…"', msg)
|
|
msg = re.sub(r"\b0x[0-9a-fA-F]+\b", "#", msg)
|
|
return re.sub(r"\b\d+(\.\d+)?\b", "#", msg)
|
|
|
|
|
|
XMP_NS = {"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
|
|
"xmp": "http://ns.adobe.com/xap/1.0/"}
|
|
# A pair is stale if the INDD was saved more than this long after the
|
|
# IDML's metadata date (docs/measurement.md).
|
|
STALE_AFTER = timedelta(hours=1)
|
|
|
|
|
|
def modify_date(packet):
|
|
"""xmp:ModifyDate of the document in an XMP packet, or None."""
|
|
if not packet:
|
|
return None
|
|
start = packet.find(b"<x:xmpmeta")
|
|
end = packet.rfind(b"</x:xmpmeta>")
|
|
if start < 0 or end < 0:
|
|
return None
|
|
try:
|
|
meta = ET.fromstring(packet[start:end + len(b"</x:xmpmeta>")])
|
|
except ET.ParseError:
|
|
return None
|
|
key = "{%s}ModifyDate" % XMP_NS["xmp"]
|
|
for desc in meta.findall("rdf:RDF/rdf:Description", XMP_NS):
|
|
text = desc.get(key) or desc.findtext(key)
|
|
if text:
|
|
try:
|
|
d = datetime.fromisoformat(text.strip().replace("Z", "+00:00"))
|
|
except ValueError:
|
|
return None
|
|
# Without a time zone, compare clock times.
|
|
return d.replace(tzinfo=d.tzinfo or timezone.utc)
|
|
return None
|
|
|
|
|
|
def idml_modify_date(idml):
|
|
"""xmp:ModifyDate in an IDML package's META-INF/metadata.xml, or None."""
|
|
try:
|
|
with zipfile.ZipFile(idml) as z:
|
|
return modify_date(z.read("META-INF/metadata.xml"))
|
|
except (KeyError, zipfile.BadZipFile, OSError):
|
|
return None
|
|
|
|
|
|
def stale_reasons(indd_date, idml_date, indd_uids, ref_el, ref_st, our_st):
|
|
"""Why a pair's IDML may not show the same save as its INDD (empty if
|
|
there is no sign of that). `indd_uids` maps each UID in the INDD
|
|
database to its class (None for a slot without a class). See
|
|
docs/measurement.md for the evidence."""
|
|
out = []
|
|
if indd_date is None or idml_date is None:
|
|
out.append("no ModifyDate")
|
|
elif indd_date < idml_date:
|
|
out.append("IDML modified after the INDD")
|
|
elif indd_date - idml_date > STALE_AFTER:
|
|
out.append("INDD saved over an hour after the IDML")
|
|
ref_uids = {int(s[1:], 16) for _, s in ref_el if re.fullmatch(r"u[0-9a-f]+", s)}
|
|
if indd_uids is not None and ref_uids - indd_uids.keys():
|
|
out.append("IDML objects not in the INDD")
|
|
if indd_uids is not None and any(indd_uids.get(u, 0) is None for u in ref_uids):
|
|
out.append("IDML objects deleted in the INDD")
|
|
if set(our_st) - set(ref_st):
|
|
out.append("INDD stories not in the IDML")
|
|
return out
|
|
|
|
|
|
def validate(outs, args):
|
|
"""Schema errors of outputs, as {path: [messages]} (valid outputs are
|
|
left out). Runs tools/validate.sh once over all of them; it batches the
|
|
Jing runs."""
|
|
errs = defaultdict(list)
|
|
if not outs:
|
|
return errs
|
|
env = dict(os.environ, VALIDATE_JOBS=str(max(1, args.jobs)))
|
|
v = subprocess.run(
|
|
[ROOT / "tools" / "validate.sh", *map(str, outs), args.schemas, args.jing],
|
|
capture_output=True, text=True, env=env)
|
|
for line in v.stdout.splitlines():
|
|
path, _, msg = line.partition("\t")
|
|
errs[path].append(msg)
|
|
if v.returncode not in (0, 1) or v.returncode == 1 and not errs or "jing" in errs:
|
|
raise SystemExit(f"tools/validate.sh failed: {v.stderr.strip()} {errs.get('jing')}")
|
|
return errs
|
|
|
|
|
|
# Elements that wrap part of a story's text without being text themselves.
|
|
# IDML puts them around ranges (ParagraphStyleRange > HyperlinkTextSource >
|
|
# CharacterStyleRange, XMLElement > ParagraphStyleRange) or inside a range
|
|
# around text; text_ranges looks through them (docs/measurement.md).
|
|
WRAPPERS = {"XMLElement", "HyperlinkTextSource", "CrossReferenceSource", "Change",
|
|
"EndnoteRange"}
|
|
# Children that are neither text nor a character position.
|
|
NOT_TEXT = {"Properties", "XMLAttribute"}
|
|
|
|
|
|
def text_ranges(story):
|
|
"""Map text offset -> attributes of the paragraph and character range
|
|
starting there, with keys prefixed PSR. and CSR. Ranges inside wrapper
|
|
elements count, at whatever depth. Offsets count Content in UTF-16 code
|
|
units and every other element (Br, Table, Footnote, an anchored frame) as
|
|
one position."""
|
|
out = {}
|
|
pos = 0
|
|
|
|
def walk(el, pa):
|
|
"""Walk the children of `el` inside the paragraph attributes `pa`."""
|
|
nonlocal pos
|
|
for ch in el:
|
|
if ch.tag in NOT_TEXT:
|
|
continue
|
|
if ch.tag == "ParagraphStyleRange":
|
|
walk(ch, {"PSR." + k: v for k, v in props(ch).items()})
|
|
elif ch.tag == "CharacterStyleRange":
|
|
start = pos
|
|
walk(ch, pa)
|
|
if pos > start:
|
|
ca = {"CSR." + k: v for k, v in props(ch).items()}
|
|
# A range nested at the same offset (Change > range at
|
|
# the start of this one) keeps the offset.
|
|
out.setdefault(start, {**pa, **ca})
|
|
elif ch.tag in WRAPPERS:
|
|
walk(ch, pa)
|
|
elif ch.tag == "Content":
|
|
pos += len((ch.text or "").encode("utf-16-le")) // 2
|
|
else:
|
|
pos += 1
|
|
|
|
walk(story, {})
|
|
return out
|
|
|
|
|
|
def load(path):
|
|
"""Map Self -> element, plus story texts, from an IDML package."""
|
|
z = zipfile.ZipFile(path)
|
|
elements = {}
|
|
stories = {}
|
|
ranges = {}
|
|
for name in z.namelist():
|
|
if not name.endswith(".xml"):
|
|
continue
|
|
try:
|
|
root = ET.fromstring(z.read(name))
|
|
except ET.ParseError as e:
|
|
raise SystemExit(f"{path}: {name}: {e}")
|
|
for el in root.iter():
|
|
s = el.get("Self")
|
|
if s is not None:
|
|
if el.tag == "Font":
|
|
s = font_self(el, s)
|
|
elements.setdefault((el.tag, s), el)
|
|
# Children without Self (TextFramePreference, ...) are
|
|
# compared as "<parent tag>/<child tag>" of the parent.
|
|
for ch in el:
|
|
if ch.get("Self") is not None or ch.tag == "Properties":
|
|
continue
|
|
if len(ch.attrib):
|
|
elements.setdefault((f"{el.tag}/{ch.tag}", s), ch)
|
|
else:
|
|
# A container without attributes (TransparencySetting):
|
|
# compare its children as "<parent>/<child>/<grandchild>".
|
|
for g in ch:
|
|
if g.get("Self") is None and len(g.attrib):
|
|
elements.setdefault((f"{el.tag}/{ch.tag}/{g.tag}", s), g)
|
|
if name == "Resources/Preferences.xml":
|
|
for el in root:
|
|
elements.setdefault((el.tag, "Preferences"), el)
|
|
if name.startswith("Stories/"):
|
|
for st in root.iter("Story"):
|
|
stories[st.get("Self")] = story_text(st)
|
|
ranges[st.get("Self")] = text_ranges(st)
|
|
return elements, stories, ranges
|
|
|
|
|
|
def story_text(story):
|
|
parts = []
|
|
for el in story.iter():
|
|
if el.tag == "Content":
|
|
parts.append(el.text or "")
|
|
elif el.tag == "Br":
|
|
parts.append("\n")
|
|
return "".join(parts)
|
|
|
|
|
|
def norm(v):
|
|
"""Normalise numbers so 1 == 1.0 and tiny float noise is ignored."""
|
|
def fix(tok):
|
|
try:
|
|
f = float(tok)
|
|
except ValueError:
|
|
return tok
|
|
return f"{f:.6g}"
|
|
return " ".join(fix(t) for t in v.split(" "))
|
|
|
|
|
|
def canon(el):
|
|
"""A structured property (TabList, AllNestedStyles, ...) as one string:
|
|
each leaf as tag=text with numbers normalised, in document order."""
|
|
if len(el) == 0 and el.get("type") != "list":
|
|
return f"{el.tag}={norm(el.text or '')}"
|
|
return el.tag + "[" + ",".join(canon(c) for c in el) + "]"
|
|
|
|
|
|
def props(el):
|
|
"""Attributes plus <Properties> children (as P.Name)."""
|
|
a = dict(el.attrib)
|
|
p = el.find("Properties")
|
|
if p is not None:
|
|
for c in p:
|
|
if c.get("type") == "list":
|
|
a["P." + c.tag] = canon(c)
|
|
elif len(c) == 0:
|
|
text = c.text or ""
|
|
extra = sorted((k, v) for k, v in c.attrib.items() if k != "type")
|
|
if extra:
|
|
text = " ".join(f"{k}={v}" for k, v in extra)
|
|
if c.tag == "Contents":
|
|
n = norm(text)
|
|
text = f"md5:{hashlib.md5(n.encode()).hexdigest()} {len(n)}"
|
|
a["P." + c.tag] = text
|
|
return a
|
|
|
|
|
|
# Values left out of value coverage because they cannot be derived from an
|
|
# INDD file: (tag pattern, attribute or P.<Properties child>, reason), tag
|
|
# patterns as in fnmatch. docs/measurement.md explains each entry.
|
|
EXCLUDED = [
|
|
("Font", "Status", "whether the font is installed on the exporting computer"),
|
|
("Document/{http://ns.adobe.com/AdobeInDesign/idml/1.0/packaging}*", "src",
|
|
"file names of the package parts, chosen by the IDML writer"),
|
|
("DocumentUser", "P.UserColor",
|
|
"colour the exporting InDesign gives each user, not the colour in the INDD"),
|
|
("Assignment", "Name",
|
|
"the exporting InDesign's name for unassigned content, which depends on its language"),
|
|
("Link", "LinkImportTime", "local time of the exporting computer; the INDD stores UTC"),
|
|
("Link", "LinkImportModificationTime",
|
|
"local time of the exporting computer; the INDD stores UTC"),
|
|
("*", "BitmapPrinting", "evaluated by the exporting computer; not stored in the INDD"),
|
|
]
|
|
|
|
# Attributes of a Link that describe the linked file as InDesign found it
|
|
# when it exported the IDML. When the IDML's LinkResourceURI differs from
|
|
# the converter's (which comes from the INDD), the link was resolved to
|
|
# another file on the exporting computer, and these values of that link
|
|
# are left out (docs/measurement.md).
|
|
RELINKED = re.compile(r"LinkResource.*|LinkImport.*")
|
|
|
|
# Font technology suffix that IDML adds to some family names, depending on
|
|
# the fonts installed on the exporting computer (docs/measurement.md).
|
|
FONT_SUFFIX = re.compile(r" \((OTF|TT|T1)\)$")
|
|
|
|
|
|
def font_name(tag, key, v, family=None):
|
|
"""A value with the font technology suffix removed if it names a font
|
|
family: FontFamily Name, Font FontFamily, AppliedFont and BulletsFont.
|
|
A Font's Name starts with its family (passed as family)."""
|
|
if (tag, key) in (("FontFamily", "Name"), ("Font", "FontFamily")) or \
|
|
key.endswith(("P.AppliedFont", "P.BulletsFont")):
|
|
return FONT_SUFFIX.sub("", v)
|
|
if (tag, key) == ("Font", "Name") and family and v.startswith(family):
|
|
return FONT_SUFFIX.sub("", family) + v[len(family):]
|
|
return v
|
|
|
|
|
|
def font_self(el, s):
|
|
"""A Font's Self with the technology suffix removed from the family
|
|
name it contains, so that fonts match whether or not the exporting
|
|
computer added the suffix."""
|
|
family = el.get("FontFamily") or ""
|
|
base = FONT_SUFFIX.sub("", family)
|
|
if base != family:
|
|
return s.replace("Fontn" + family, "Fontn" + base, 1)
|
|
return s
|
|
|
|
|
|
@functools.cache
|
|
def excluded(tag, key):
|
|
return any(k == key and fnmatch.fnmatchcase(tag, t) for t, k, _ in EXCLUDED)
|
|
|
|
|
|
def tree(el):
|
|
"""An element with its attributes, text and descendants as one string,
|
|
numbers normalised: the value of a structured <Properties> child such as
|
|
PathGeometry."""
|
|
attrs = ",".join(f"{k}={norm(v)}" for k, v in sorted(el.attrib.items()) if k != "type")
|
|
kids = ",".join(tree(c) for c in el)
|
|
return f"{el.tag}({attrs})[{kids}]{norm((el.text or '').strip())}"
|
|
|
|
|
|
def values(tag, el):
|
|
"""The values of an element that value coverage counts: attributes and
|
|
<Properties> children, without Self and without EXCLUDED values."""
|
|
a = props(el)
|
|
p = el.find("Properties")
|
|
if p is not None:
|
|
for c in p:
|
|
if c.get("type") != "list" and len(c):
|
|
a["P." + c.tag] = tree(c)
|
|
return {k: v for k, v in a.items() if k != "Self" and not excluded(tag, k)}
|
|
|
|
|
|
# Gap keys for values that could not be compared one by one.
|
|
NO_ELEMENT = "(element not produced)"
|
|
NO_TEXT = "(story text differs)"
|
|
NOT_IN_REFERENCE = "(element not in reference)"
|
|
ELEMENT = "(element)"
|
|
|
|
|
|
class Stats:
|
|
"""Comparison counts over a set of pairs."""
|
|
|
|
def __init__(self):
|
|
self.docs = 0
|
|
# Value coverage: reference values, and those reproduced exactly.
|
|
self.values = 0
|
|
self.reproduced = 0
|
|
# Links whose resource values are left out (RELINKED).
|
|
self.relinked = 0
|
|
# (tag, key) -> values wrong / missing, and documents affected.
|
|
self.gap_wrong = Counter()
|
|
self.gap_missing = Counter()
|
|
self.gap_docs = Counter()
|
|
self.gap_total = Counter()
|
|
# Values the output has and the reference does not, by (tag, key),
|
|
# and documents affected.
|
|
self.extra = 0
|
|
self.extra_values = Counter()
|
|
self.extra_docs = Counter()
|
|
# (pair, reproduced, values) for each pair.
|
|
self.pairs = []
|
|
self.found = Counter()
|
|
self.total = Counter()
|
|
self.attr_ok = defaultdict(Counter)
|
|
self.story_ok = Counter()
|
|
self.short_docs = Counter() # (kind, tag, attribute) -> documents affected
|
|
|
|
def add(self, other):
|
|
self.docs += other.docs
|
|
self.values += other.values
|
|
self.reproduced += other.reproduced
|
|
self.extra += other.extra
|
|
self.relinked += other.relinked
|
|
for c in ("gap_wrong", "gap_missing", "gap_docs", "gap_total", "extra_values",
|
|
"extra_docs"):
|
|
getattr(self, c).update(getattr(other, c))
|
|
self.pairs += other.pairs
|
|
self.found.update(other.found)
|
|
self.total.update(other.total)
|
|
for k, c in other.attr_ok.items():
|
|
self.attr_ok[k].update(c)
|
|
self.story_ok.update(other.story_ok)
|
|
self.short_docs.update(other.short_docs)
|
|
|
|
def coverage(self):
|
|
return self.reproduced / self.values if self.values else 1.0
|
|
|
|
def quantile(self, q):
|
|
"""Value coverage of the pair at quantile q (0 = lowest)."""
|
|
cov = sorted(ok / n if n else 1.0 for _, ok, n in self.pairs)
|
|
return cov[min(len(cov) - 1, int(q * len(cov)))] if cov else 1.0
|
|
|
|
def document_score(self, at_least):
|
|
"""Pairs whose value coverage is at least `at_least`."""
|
|
return sum(1 for _, ok, n in self.pairs if n == 0 or ok / n >= at_least)
|
|
|
|
def gaps(self):
|
|
"""(tag, key), documents, values wrong, values missing; most
|
|
documents first, then most values."""
|
|
rows = [(k, d, self.gap_wrong[k], self.gap_missing[k]) for k, d in self.gap_docs.items()]
|
|
return sorted(rows, key=lambda r: (-r[1], -(r[2] + r[3]), r[0]))
|
|
|
|
def extras(self):
|
|
"""(tag, key), documents, extra values; most documents first."""
|
|
rows = [(k, d, self.extra_values[k]) for k, d in self.extra_docs.items()]
|
|
return sorted(rows, key=lambda r: (-r[1], -r[2], r[0]))
|
|
|
|
def story_line(self):
|
|
st = sum(self.story_ok.values())
|
|
return (f"{self.story_ok['ok']}/{st} exact, {self.story_ok['wrong']} differ, "
|
|
f"{self.story_ok['missing']} missing")
|
|
|
|
|
|
def compare_pair(name, ref, ours, examples, show):
|
|
"""Stats of one pair: `ref` and `ours` are load() results. Adds up to
|
|
`show` mismatches per attribute to `examples`."""
|
|
ref_el, ref_st, ref_rg = ref
|
|
our_el, our_st, our_rg = ours
|
|
p = Stats()
|
|
p.docs = 1
|
|
short = set()
|
|
for (tag, s), el in ref_el.items():
|
|
p.total[tag] += 1
|
|
mine = our_el.get((tag, s))
|
|
if mine is None:
|
|
short.add(("element", tag, ""))
|
|
continue
|
|
p.found[tag] += 1
|
|
mine = props(mine)
|
|
for k, v in props(el).items():
|
|
if k == "Self":
|
|
continue
|
|
if k not in mine:
|
|
p.attr_ok[(tag, k)]["missing"] += 1
|
|
short.add(("attribute", tag, k))
|
|
elif norm(mine[k]) == norm(v):
|
|
p.attr_ok[(tag, k)]["ok"] += 1
|
|
else:
|
|
p.attr_ok[(tag, k)]["wrong"] += 1
|
|
short.add(("attribute", tag, k))
|
|
if len(examples[(tag, k)]) < show:
|
|
examples[(tag, k)].append((name, s, v, mine[k]))
|
|
for sid, rgs in ref_rg.items():
|
|
if our_st.get(sid) != ref_st.get(sid):
|
|
continue
|
|
mine = our_rg.get(sid, {})
|
|
for start, attrs in rgs.items():
|
|
p.total["TextRange"] += 1
|
|
if start not in mine:
|
|
short.add(("element", "TextRange", ""))
|
|
continue
|
|
p.found["TextRange"] += 1
|
|
for k, v in attrs.items():
|
|
o = mine[start].get(k)
|
|
if o is None:
|
|
p.attr_ok[("TextRange", k)]["missing"] += 1
|
|
short.add(("attribute", "TextRange", k))
|
|
elif norm(o) == norm(v):
|
|
p.attr_ok[("TextRange", k)]["ok"] += 1
|
|
else:
|
|
p.attr_ok[("TextRange", k)]["wrong"] += 1
|
|
short.add(("attribute", "TextRange", k))
|
|
if len(examples[("TextRange", k)]) < show:
|
|
examples[("TextRange", k)].append((name, sid, start, v, o))
|
|
for sid, text in ref_st.items():
|
|
if sid not in our_st:
|
|
p.story_ok["missing"] += 1
|
|
short.add(("element", "Story", ""))
|
|
elif our_st[sid] == text:
|
|
p.story_ok["ok"] += 1
|
|
else:
|
|
p.story_ok["wrong"] += 1
|
|
short.add(("text", "Story", ""))
|
|
if len(examples[("Story", "text")]) < max(show, 3):
|
|
examples[("Story", "text")].append((name, sid, text[:80], our_st[sid][:80]))
|
|
p.short_docs.update(short)
|
|
value_coverage(p, ref, ours)
|
|
return p
|
|
|
|
|
|
def value_coverage(p, ref, ours):
|
|
"""Count reference values and the reproduced ones into `p`: element
|
|
presence, attribute and <Properties> values, story text, and the start
|
|
and attributes of each text range. Also count the values the output has
|
|
and the reference does not (extra values), with the same keys."""
|
|
ref_el, ref_st, ref_rg = ref
|
|
our_el, our_st, our_rg = ours
|
|
total = Counter()
|
|
wrong = Counter()
|
|
missing = Counter()
|
|
extra = Counter()
|
|
|
|
def compare(tag, theirs, mine):
|
|
for k, v in theirs.items():
|
|
key = (tag, k)
|
|
total[key] += 1
|
|
o = mine.get(k)
|
|
if o is None:
|
|
missing[key] += 1
|
|
elif norm(font_name(tag, k, o, mine.get("FontFamily"))) != \
|
|
norm(font_name(tag, k, v, theirs.get("FontFamily"))):
|
|
wrong[key] += 1
|
|
for k in mine.keys() - theirs.keys():
|
|
extra[(tag, k)] += 1
|
|
|
|
for (tag, s), el in ref_el.items():
|
|
theirs = values(tag, el)
|
|
mine = our_el.get((tag, s))
|
|
if mine is None:
|
|
n = 1 + len(theirs)
|
|
total[(tag, NO_ELEMENT)] += n
|
|
missing[(tag, NO_ELEMENT)] += n
|
|
continue
|
|
total[(tag, ELEMENT)] += 1
|
|
mine = values(tag, mine)
|
|
if tag == "Link" and mine.get("LinkResourceURI") not in (None, theirs.get("LinkResourceURI")):
|
|
theirs = {k: v for k, v in theirs.items() if not RELINKED.fullmatch(k)}
|
|
mine = {k: v for k, v in mine.items() if not RELINKED.fullmatch(k)}
|
|
p.relinked += 1
|
|
compare(tag, theirs, mine)
|
|
for (tag, s), el in our_el.items():
|
|
if (tag, s) not in ref_el:
|
|
extra[(tag, NOT_IN_REFERENCE)] += 1 + len(values(tag, el))
|
|
for sid, text in ref_st.items():
|
|
key = ("Story", "(text)")
|
|
total[key] += 1
|
|
if sid not in our_st:
|
|
missing[key] += 1
|
|
elif our_st[sid] != text:
|
|
wrong[key] += 1
|
|
for sid in our_st.keys() - ref_st.keys():
|
|
extra[("Story", "(text)")] += 1
|
|
extra[("TextRange", NOT_IN_REFERENCE)] += sum(1 + len(a) for a in
|
|
our_rg.get(sid, {}).values())
|
|
for sid, rgs in ref_rg.items():
|
|
if our_st.get(sid) != ref_st.get(sid):
|
|
n = sum(1 + len(a) for a in rgs.values())
|
|
total[("TextRange", NO_TEXT)] += n
|
|
missing[("TextRange", NO_TEXT)] += n
|
|
continue
|
|
mine = our_rg.get(sid, {})
|
|
for start, attrs in rgs.items():
|
|
if start not in mine:
|
|
n = 1 + len(attrs)
|
|
total[("TextRange", NO_ELEMENT)] += n
|
|
missing[("TextRange", NO_ELEMENT)] += n
|
|
continue
|
|
total[("TextRange", ELEMENT)] += 1
|
|
compare("TextRange", attrs, mine[start])
|
|
for start in mine.keys() - rgs.keys():
|
|
extra[("TextRange", NOT_IN_REFERENCE)] += 1 + len(mine[start])
|
|
n = sum(total.values())
|
|
bad = sum(wrong.values()) + sum(missing.values())
|
|
p.values += n
|
|
p.reproduced += n - bad
|
|
p.gap_total.update(total)
|
|
p.gap_wrong.update(wrong)
|
|
p.gap_missing.update(missing)
|
|
p.gap_docs.update(k for k in total if wrong[k] or missing[k])
|
|
p.extra += sum(extra.values())
|
|
p.extra_values.update(extra)
|
|
p.extra_docs.update(k for k, v in extra.items() if v)
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--limit", type=int, default=0)
|
|
ap.add_argument("--detail", action="append", default=[],
|
|
help="print attribute table for this tag (repeatable)")
|
|
ap.add_argument("--file", help="only pairs whose path contains this")
|
|
ap.add_argument("--show", type=int, default=0, help="show N mismatches per attribute")
|
|
ap.add_argument("--schemas", help="IDML RelaxNG schema directory (validate output)")
|
|
ap.add_argument("--jing", help="directory with jing.jar, isorelax.jar, saxon.jar")
|
|
ap.add_argument("--bin", default=str(BIN), help="converter binary")
|
|
ap.add_argument("--all", action="store_true",
|
|
help="also convert and validate files without a reference IDML")
|
|
ap.add_argument("--exclude", action="append", default=[],
|
|
help="leave out files whose path under corpus/ starts with this (repeatable)")
|
|
ap.add_argument("--warnings", type=int, default=15,
|
|
help="number of warning kinds to list")
|
|
ap.add_argument("--shortfalls", type=int, default=0,
|
|
help="rank the N biggest shortfalls by documents affected")
|
|
ap.add_argument("--trusted", action="store_true",
|
|
help="element tables, shortfalls and details over trustworthy pairs only")
|
|
ap.add_argument("--stale", type=int, default=0,
|
|
help="list N stale pairs with their reasons")
|
|
ap.add_argument("--gaps", type=int, default=30,
|
|
help="print the N biggest value coverage gaps")
|
|
ap.add_argument("--out", default=str(ROOT / "target" / "compare"),
|
|
help="directory for pairs.tsv and gaps*.tsv (default target/compare)")
|
|
ap.add_argument("--jobs", type=int, default=os.cpu_count() or 1,
|
|
help="conversions and schema validations run in parallel")
|
|
args = ap.parse_args()
|
|
check = bool(args.schemas and args.jing)
|
|
|
|
every = Stats() # all pairs
|
|
trusted = Stats() # pairs without a sign of staleness
|
|
stale = [] # (name, reasons)
|
|
pair_rows = [] # (name, Stats, stale reasons) per pair
|
|
examples = defaultdict(list)
|
|
failures = []
|
|
warnings = Counter()
|
|
warned_files = Counter()
|
|
seen = set()
|
|
known = {r["path"]: r["sha256"] for r in inventory() if r.get("sha256")}
|
|
todo = pairs(args.limit, args.file, args.exclude, seen, known)
|
|
others = unpaired(args.file, args.exclude, seen, known) if args.all else []
|
|
other_failures = []
|
|
pool = ThreadPoolExecutor(max(1, args.jobs))
|
|
|
|
def convert(indd, out):
|
|
"""Convert one file; runs in the pool."""
|
|
return subprocess.run([args.bin, "convert", indd, out], capture_output=True, text=True)
|
|
|
|
def convert_pair(indd, idml, out):
|
|
"""Convert one paired file and read what stale_reasons needs: the
|
|
INDD's UIDs and the ModifyDate of both files; runs in the pool."""
|
|
r = convert(indd, out)
|
|
x = subprocess.run([args.bin, "xmp", indd], capture_output=True)
|
|
u = subprocess.run([args.bin, "uids", indd], capture_output=True, text=True)
|
|
uids = None
|
|
if u.returncode == 0:
|
|
uids = {}
|
|
for line in u.stdout.splitlines():
|
|
uid, cls = line.split("\t")
|
|
uids[int(uid)] = None if cls == "-" else int(cls, 16)
|
|
dates = (modify_date(x.stdout) if x.returncode == 0 else None, idml_modify_date(idml))
|
|
return r, dates, uids
|
|
|
|
def count_warnings(r):
|
|
kinds = set()
|
|
for line in r.stderr.splitlines():
|
|
if line.startswith("warning: "):
|
|
kind = warning_kind(line[len("warning: "):])
|
|
warnings[kind] += 1
|
|
kinds.add(kind)
|
|
for kind in kinds:
|
|
warned_files[kind] += 1
|
|
|
|
other_invalid = []
|
|
invalid = []
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
jobs = [pool.submit(convert, indd, Path(tmp) / f"other{n}.idml")
|
|
for n, indd in enumerate(others)]
|
|
jobs += [pool.submit(convert_pair, indd, idml, Path(tmp) / f"pair{n}.idml")
|
|
for n, (indd, idml) in enumerate(todo)]
|
|
# Converted files waiting for validation: (output, name, list of
|
|
# invalid files it belongs to). They are validated in batches, which
|
|
# bounds the space the unpacked outputs take.
|
|
to_check = []
|
|
|
|
def flush(at_least):
|
|
if len(to_check) < at_least:
|
|
return
|
|
errs = validate([out for out, _, _ in to_check], args)
|
|
for out, name, bad in to_check:
|
|
if errs.get(str(out)):
|
|
bad.append((name, errs[str(out)]))
|
|
out.unlink(missing_ok=True)
|
|
to_check.clear()
|
|
for n, indd in enumerate(others):
|
|
r = jobs[n].result()
|
|
count_warnings(r)
|
|
out = Path(tmp) / f"other{n}.idml"
|
|
if r.returncode != 0:
|
|
other_failures.append((indd.name, r.stderr.strip()))
|
|
elif check:
|
|
to_check.append((out, indd.name, other_invalid))
|
|
flush(VALIDATE_BATCH)
|
|
continue
|
|
out.unlink(missing_ok=True)
|
|
for n, (indd, idml) in enumerate(todo):
|
|
out = Path(tmp) / f"pair{n}.idml"
|
|
r, (indd_date, idml_date), indd_uids = jobs[len(others) + n].result()
|
|
count_warnings(r)
|
|
if r.returncode != 0:
|
|
failures.append((indd.name, r.stderr.strip()))
|
|
continue
|
|
if check:
|
|
to_check.append((out, indd.name, invalid))
|
|
ref = load(idml)
|
|
ours = load(out)
|
|
reasons = stale_reasons(indd_date, idml_date, indd_uids, ref[0], ref[1], ours[1])
|
|
p = compare_pair(indd.name, ref, ours, examples, args.show)
|
|
rel = str(indd.relative_to(ROOT / "corpus"))
|
|
p.pairs = [(rel, p.reproduced, p.values)]
|
|
pair_rows.append((rel, p, reasons))
|
|
every.add(p)
|
|
if reasons:
|
|
stale.append((rel, reasons))
|
|
else:
|
|
trusted.add(p)
|
|
if check:
|
|
flush(VALIDATE_BATCH)
|
|
else:
|
|
out.unlink(missing_ok=True)
|
|
pool.shutdown()
|
|
flush(1)
|
|
|
|
print(f"conversion failures: {len(failures)} of {len(todo)} paired files")
|
|
for name, err in failures[:10]:
|
|
print(f" {name}: {err}")
|
|
if check:
|
|
print(f"schema validation failures: {len(invalid)}")
|
|
for name, errs in invalid[:10]:
|
|
print(f" {name}: {len(errs)} errors, {errs[:3]}")
|
|
if args.all:
|
|
print(f"files without a reference: {len(others)}, "
|
|
f"conversion failures: {len(other_failures)}")
|
|
for name, err in other_failures[:10]:
|
|
print(f" {name}: {err}")
|
|
if check:
|
|
print(f"files without a reference: schema validation failures: {len(other_invalid)}")
|
|
for name, errs in other_invalid[:10]:
|
|
print(f" {name}: {len(errs)} errors, {errs[:3]}")
|
|
print(f"warnings: {sum(warnings.values())} (count, files, kind)")
|
|
for kind, count in warnings.most_common(args.warnings):
|
|
print(f" {count:6} {warned_files[kind]:4} {kind[:150]}")
|
|
print(f"story text: {every.story_line()}")
|
|
reasons = Counter(r for _, rs in stale for r in rs)
|
|
print(f"trustworthy pairs: {trusted.docs} of {every.docs}; stale: {len(stale)} "
|
|
f"({', '.join(f'{r}: {n}' for r, n in reasons.most_common())})")
|
|
print(f"trustworthy pairs: story text: {trusted.story_line()}")
|
|
for name, rs in stale[:args.stale]:
|
|
print(f" stale: {name}: {'; '.join(rs)}")
|
|
st = trusted if args.trusted else every
|
|
which = "trustworthy pairs" if args.trusted else "all pairs"
|
|
print(f"\nelements, {which} (produced / in reference):")
|
|
for tag, n in st.total.most_common():
|
|
print(f" {tag:34} {st.found[tag]:6} / {n:6}")
|
|
if args.shortfalls:
|
|
print(f"\nshortfalls by documents affected (of {st.docs} {which}):")
|
|
def instances(kind, tag, k):
|
|
"""How many elements, values or stories fall short."""
|
|
if kind == "attribute":
|
|
return st.attr_ok[(tag, k)]["wrong"] + st.attr_ok[(tag, k)]["missing"]
|
|
if kind == "element":
|
|
return st.total[tag] - st.found[tag]
|
|
return st.story_ok["wrong"]
|
|
ranked = sorted(st.short_docs.items(), key=lambda kv: (-kv[1], -instances(*kv[0]), kv[0]))
|
|
for (kind, tag, k), n in ranked[:args.shortfalls]:
|
|
what = f"{st.story_ok['wrong']} stories differ"
|
|
if kind == "attribute":
|
|
c = st.attr_ok[(tag, k)]
|
|
what = f"{c['wrong']} wrong, {c['missing']} missing of {sum(c.values())}"
|
|
elif kind == "element":
|
|
what = f"{st.total[tag] - st.found[tag]} of {st.total[tag]} not produced"
|
|
print(f" {n:5} {tag + (' ' + k if k else ''):56} {what}")
|
|
for tag in args.detail:
|
|
print(f"\nattributes of {tag}, {which} (ok / wrong / missing):")
|
|
rows = [(k, c) for (t, k), c in st.attr_ok.items() if t == tag]
|
|
for k, c in sorted(rows, key=lambda kc: -sum(kc[1].values())):
|
|
print(f" {k:40} {c['ok']:6} {c['wrong']:6} {c['missing']:6}")
|
|
for ex in examples[(tag, k)]:
|
|
print(f" {ex}")
|
|
for ex in examples[("Story", "text")]:
|
|
print(" story mismatch:", ex)
|
|
headline(args, every, trusted, pair_rows)
|
|
|
|
|
|
def gap_name(key):
|
|
tag, k = key
|
|
return f"{tag} {k}"
|
|
|
|
|
|
def write_gaps(path, st):
|
|
with open(path, "w") as f:
|
|
f.write("tag\tkey\tdocuments\twrong\tmissing\tvalues\n")
|
|
for key, docs, w, m in st.gaps():
|
|
f.write(f"{key[0]}\t{key[1]}\t{docs}\t{w}\t{m}\t{st.gap_total[key]}\n")
|
|
|
|
|
|
def write_extras(path, st):
|
|
with open(path, "w") as f:
|
|
f.write("tag\tkey\tdocuments\textra\n")
|
|
for key, docs, n in st.extras():
|
|
f.write(f"{key[0]}\t{key[1]}\t{docs}\t{n}\n")
|
|
|
|
|
|
def write_values(path, st):
|
|
"""Every key with its values and how many are reproduced, for checking
|
|
that a change reproduces no fewer values of any key."""
|
|
with open(path, "w") as f:
|
|
f.write("tag\tkey\tvalues\treproduced\twrong\tmissing\n")
|
|
for key in sorted(st.gap_total):
|
|
w, m = st.gap_wrong[key], st.gap_missing[key]
|
|
f.write(f"{key[0]}\t{key[1]}\t{st.gap_total[key]}\t{st.gap_total[key] - w - m}"
|
|
f"\t{w}\t{m}\n")
|
|
|
|
|
|
def headline(args, every, trusted, pair_rows):
|
|
"""Print value coverage, document scores and the biggest gaps; write
|
|
pairs.tsv, gaps.tsv (trustworthy pairs) and gaps-all.tsv to --out."""
|
|
out = Path(args.out)
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
with open(out / "pairs.tsv", "w") as f:
|
|
f.write("path\ttrusted\tcoverage\treproduced\tvalues\textra\tstories"
|
|
"\tstories_identical\tstale reasons\n")
|
|
for name, p, reasons in sorted(pair_rows, key=lambda r: r[1].coverage()):
|
|
f.write(f"{name}\t{not reasons}\t{p.coverage():.5f}\t{p.reproduced}\t{p.values}"
|
|
f"\t{p.extra}\t{sum(p.story_ok.values())}\t{p.story_ok['ok']}"
|
|
f"\t{'; '.join(reasons)}\n")
|
|
write_gaps(out / "gaps.tsv", trusted)
|
|
write_gaps(out / "gaps-all.tsv", every)
|
|
write_extras(out / "extras.tsv", trusted)
|
|
write_extras(out / "extras-all.tsv", every)
|
|
write_values(out / "values.tsv", trusted)
|
|
write_values(out / "values-all.tsv", every)
|
|
print("\nheadline (docs/measurement.md):")
|
|
for label, st in (("trustworthy pairs", trusted), ("all pairs", every)):
|
|
n = st.docs or 1
|
|
print(f" {label}: value coverage {st.coverage():.2%} "
|
|
f"({st.reproduced} of {st.values} values); documents at >= 99 %: "
|
|
f"{st.document_score(0.99)}/{st.docs} ({st.document_score(0.99) / n:.1%}), "
|
|
f">= 99.9 %: {st.document_score(0.999)}/{st.docs} "
|
|
f"({st.document_score(0.999) / n:.1%})")
|
|
print(f" extra values (in the output, not in the reference): {st.extra} "
|
|
f"({st.extra / (st.values or 1):.2%} of reference values); "
|
|
f"links resolved to another file at export: {st.relinked}")
|
|
print(f" per pair: lowest {st.quantile(0):.1%}, 10th percentile {st.quantile(0.1):.1%}, "
|
|
f"median {st.quantile(0.5):.1%}, 90th percentile {st.quantile(0.9):.1%}, "
|
|
f"highest {st.quantile(1):.1%}")
|
|
rows = trusted.gaps()
|
|
print(f"\nvalue gaps, trustworthy pairs, by documents affected "
|
|
f"(documents, values wrong, missing, of values; all in {out / 'gaps.tsv'}):")
|
|
for key, docs, w, m in rows[:args.gaps]:
|
|
print(f" {docs:5} {w:7} {m:7} {trusted.gap_total[key]:8} {gap_name(key)[:90]}")
|
|
print("\nvalue gaps, trustworthy pairs, by values (documents, values wrong, missing, of values):")
|
|
for key, docs, w, m in sorted(rows, key=lambda r: -(r[2] + r[3]))[:args.gaps // 2]:
|
|
print(f" {docs:5} {w:7} {m:7} {trusted.gap_total[key]:8} {gap_name(key)[:90]}")
|
|
rows = trusted.extras()
|
|
print(f"\nextra values, trustworthy pairs, by documents affected "
|
|
f"(documents, extra values; all in {out / 'extras.tsv'}):")
|
|
for key, docs, n in rows[:args.gaps // 2]:
|
|
print(f" {docs:5} {n:7} {gap_name(key)[:90]}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|