mirror of
https://github.com/storytold/indd-utils.git
synced 2026-11-20 17:20:49 +01:00
InDesign writes some empty Properties children open and closed on two lines, so their text is a line feed and tabs (XMLImportPreference/Properties/TransformParameters). compare.py now treats such text as empty. measurement.md says so. Effect (measurement only, same output): P.TransformParameters reproduced in 489 of 489 trustworthy pairs. Trustworthy value coverage 97.12% -> 97.12% (+489 values); all pairs +654.
1062 lines
45 KiB
Python
1062 lines
45 KiB
Python
#!/usr/bin/env python3
|
|
"""Compare converter output with reference IDML files.
|
|
|
|
For every INDD file in the corpus that has a reference IDML from the same
|
|
major version (the `idml` column of corpus/inventory.tsv, written by
|
|
tools/inventory.py), convert it with `target/release/indd` and compare the
|
|
result with the reference. Files with the same SHA-256 are converted
|
|
once. Reports, per element type: how many referenced elements we produce
|
|
(matched by Self), and per attribute how often our value matches. Also
|
|
reports story text agreement.
|
|
|
|
Usage: python3 -I tools/compare.py [--limit N] [--detail TAG]... [--file SUBSTR]
|
|
[--schemas DIR --jing DIR] [--bin PATH]
|
|
[--all] [--exclude PREFIX]... [--jobs N]
|
|
[--shortfalls N] [--trusted] [--stale N]
|
|
[--gaps N] [--out DIR]
|
|
Run from the repository root after `cargo build --release`. With --schemas
|
|
and --jing, also validates every output with tools/validate.sh, in batches
|
|
of VALIDATE_BATCH files (one Jing run per schema for a whole batch). --bin runs
|
|
another converter binary, for example a copy of the previous build.
|
|
|
|
--all also converts every other INDD and INDT file under corpus/ (any
|
|
version, either byte order, without a usable IDML), validates the output
|
|
if schemas are given, and reports failures for those files separately.
|
|
--exclude leaves out files whose path under corpus/ starts with PREFIX.
|
|
Converter warnings are counted by kind over all converted files.
|
|
|
|
Pairs are trustworthy or stale (the IDML shows a different save than the
|
|
INDD; rule and evidence in docs/measurement.md). Totals are printed for all
|
|
pairs and for trustworthy pairs; --trusted limits the element tables,
|
|
shortfalls and details to trustworthy pairs, --stale N lists stale pairs.
|
|
|
|
Last comes the headline (docs/measurement.md): value coverage, the share
|
|
of reference values the conversion reproduces, over trustworthy pairs and
|
|
over all pairs; document scores (pairs with coverage >= 99 % and >= 99.9 %);
|
|
extra values (values the output has and the reference does not); and the
|
|
biggest gaps (--gaps N). pairs.tsv, gaps.tsv and gaps-all.tsv go to --out
|
|
(default target/compare), with values.tsv and values-all.tsv, which list
|
|
every key with its values and how many are reproduced, and extras.tsv and
|
|
extras-all.tsv, which list the extra values by key. summary.json holds the
|
|
numbers of the README's "Current numbers" section, which
|
|
tools/readme_numbers.py writes into the README.
|
|
|
|
--shortfalls N lists the N element types and attributes that fall short
|
|
(missing elements; wrong or missing attribute values; differing story
|
|
text) in the most documents.
|
|
|
|
Embedded file data (`Contents`) is compared by digest, so it is reported as
|
|
`md5:<hex> <length>` rather than as the full text.
|
|
"""
|
|
|
|
import argparse
|
|
import fnmatch
|
|
import functools
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import xml.etree.ElementTree as ET
|
|
import zipfile
|
|
from collections import Counter, defaultdict
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
BIN = ROOT / "target" / "release" / "indd"
|
|
# Converted files validated together in one tools/validate.sh run.
|
|
VALIDATE_BATCH = 600
|
|
|
|
|
|
def inventory():
|
|
"""Rows of corpus/inventory.tsv (tools/inventory.py), as dicts."""
|
|
lines = (ROOT / "corpus" / "inventory.tsv").read_text().splitlines()
|
|
cols = lines[0].split("\t")
|
|
return [dict(zip(cols, line.split("\t"))) for line in lines[1:]]
|
|
|
|
|
|
def digest(indd, known):
|
|
"""SHA-256 of a corpus file, from the inventory if it lists the file."""
|
|
path = str(indd.relative_to(ROOT / "corpus"))
|
|
if path in known:
|
|
return known[path]
|
|
return hashlib.sha256(indd.read_bytes()).hexdigest()
|
|
|
|
|
|
def pairs(limit, substr, exclude, seen, known):
|
|
"""Corpus files with a same-version reference IDML, without duplicates.
|
|
Adds the digest of each file to `seen`."""
|
|
out = []
|
|
for row in inventory():
|
|
path, order, ver, dom = row["path"], row["order"], row["version"], row["idml_dom"]
|
|
if order != "LE" or not dom or ver.split(".")[0] != dom.split(".")[0]:
|
|
continue
|
|
if substr and substr not in path or any(path.startswith(e) for e in exclude):
|
|
continue
|
|
indd = ROOT / "corpus" / path
|
|
d = digest(indd, known)
|
|
if d in seen:
|
|
continue
|
|
seen.add(d)
|
|
idml = row.get("idml") or str(Path(path).with_suffix(".idml"))
|
|
out.append((indd, ROOT / "corpus" / idml))
|
|
return out[:limit] if limit else out
|
|
|
|
|
|
def unpaired(substr, exclude, seen, known):
|
|
"""Every other INDD and INDT file under corpus/, without duplicates."""
|
|
out = []
|
|
for indd in sorted((ROOT / "corpus").rglob("*")):
|
|
if indd.suffix.lower() not in (".indd", ".indt") or not indd.is_file():
|
|
continue
|
|
path = str(indd.relative_to(ROOT / "corpus"))
|
|
if substr and substr not in path or any(path.startswith(e) for e in exclude):
|
|
continue
|
|
d = digest(indd, known)
|
|
if d in seen:
|
|
continue
|
|
seen.add(d)
|
|
out.append(indd)
|
|
return out
|
|
|
|
|
|
def warning_kind(msg):
|
|
"""A warning with its numbers and quoted names replaced, for counting."""
|
|
msg = re.sub(r'"[^"]*"', '"…"', msg)
|
|
msg = re.sub(r"\b0x[0-9a-fA-F]+\b", "#", msg)
|
|
return re.sub(r"\b\d+(\.\d+)?\b", "#", msg)
|
|
|
|
|
|
XMP_NS = {"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
|
|
"xmp": "http://ns.adobe.com/xap/1.0/"}
|
|
# A pair is stale if the INDD was saved more than this long after the
|
|
# IDML's metadata date (docs/measurement.md).
|
|
STALE_AFTER = timedelta(hours=1)
|
|
|
|
|
|
def modify_date(packet):
|
|
"""xmp:ModifyDate of the document in an XMP packet, or None."""
|
|
if not packet:
|
|
return None
|
|
start = packet.find(b"<x:xmpmeta")
|
|
end = packet.rfind(b"</x:xmpmeta>")
|
|
if start < 0 or end < 0:
|
|
return None
|
|
try:
|
|
meta = ET.fromstring(packet[start:end + len(b"</x:xmpmeta>")])
|
|
except ET.ParseError:
|
|
return None
|
|
key = "{%s}ModifyDate" % XMP_NS["xmp"]
|
|
for desc in meta.findall("rdf:RDF/rdf:Description", XMP_NS):
|
|
text = desc.get(key) or desc.findtext(key)
|
|
if text:
|
|
try:
|
|
d = datetime.fromisoformat(text.strip().replace("Z", "+00:00"))
|
|
except ValueError:
|
|
return None
|
|
# Without a time zone, compare clock times.
|
|
return d.replace(tzinfo=d.tzinfo or timezone.utc)
|
|
return None
|
|
|
|
|
|
def idml_modify_date(idml):
|
|
"""xmp:ModifyDate in an IDML package's META-INF/metadata.xml, or None."""
|
|
try:
|
|
with zipfile.ZipFile(idml) as z:
|
|
return modify_date(z.read("META-INF/metadata.xml"))
|
|
except (KeyError, zipfile.BadZipFile, OSError):
|
|
return None
|
|
|
|
|
|
def stale_reasons(indd_date, idml_date, indd_uids, ref_el, ref_st, our_st):
|
|
"""Why a pair's IDML may not show the same save as its INDD (empty if
|
|
there is no sign of that). `indd_uids` maps each UID in the INDD
|
|
database to its class (None for a slot without a class). See
|
|
docs/measurement.md for the evidence."""
|
|
out = []
|
|
if indd_date is None or idml_date is None:
|
|
out.append("no ModifyDate")
|
|
elif indd_date < idml_date:
|
|
out.append("IDML modified after the INDD")
|
|
elif indd_date - idml_date > STALE_AFTER:
|
|
out.append("INDD saved over an hour after the IDML")
|
|
ref_uids = {int(s[1:], 16) for _, s in ref_el if re.fullmatch(r"u[0-9a-f]+", s)}
|
|
if indd_uids is not None and ref_uids - indd_uids.keys():
|
|
out.append("IDML objects not in the INDD")
|
|
if indd_uids is not None and any(indd_uids.get(u, 0) is None for u in ref_uids):
|
|
out.append("IDML objects deleted in the INDD")
|
|
if set(our_st) - set(ref_st):
|
|
out.append("INDD stories not in the IDML")
|
|
return out
|
|
|
|
|
|
def validate(outs, args):
|
|
"""Schema errors of outputs, as {path: [messages]} (valid outputs are
|
|
left out). Runs tools/validate.sh once over all of them; it batches the
|
|
Jing runs."""
|
|
errs = defaultdict(list)
|
|
if not outs:
|
|
return errs
|
|
env = dict(os.environ, VALIDATE_JOBS=str(max(1, args.jobs)))
|
|
v = subprocess.run(
|
|
[ROOT / "tools" / "validate.sh", *map(str, outs), args.schemas, args.jing],
|
|
capture_output=True, text=True, env=env)
|
|
for line in v.stdout.splitlines():
|
|
path, _, msg = line.partition("\t")
|
|
errs[path].append(msg)
|
|
if v.returncode not in (0, 1) or v.returncode == 1 and not errs or "jing" in errs:
|
|
raise SystemExit(f"tools/validate.sh failed: {v.stderr.strip()} {errs.get('jing')}")
|
|
return errs
|
|
|
|
|
|
# Elements that wrap part of a story's text without being text themselves.
|
|
# IDML puts them around ranges (ParagraphStyleRange > HyperlinkTextSource >
|
|
# CharacterStyleRange, XMLElement > ParagraphStyleRange) or inside a range
|
|
# around text; text_ranges looks through them (docs/measurement.md).
|
|
WRAPPERS = {"XMLElement", "HyperlinkTextSource", "CrossReferenceSource", "Change",
|
|
"EndnoteRange"}
|
|
# Children that are neither text nor a character position.
|
|
NOT_TEXT = {"Properties", "XMLAttribute"}
|
|
# Elements that hold text of their own inside a story: their ranges are
|
|
# counted in a text flow of their own (docs/measurement.md).
|
|
NESTED = {"Footnote", "Cell", "Note"}
|
|
|
|
|
|
ERROR = re.compile(r"(?P<part>[^:]+):(?P<line>\d+):(?P<col>\d+): error: (?P<msg>.*)")
|
|
|
|
|
|
def endnote_errors(idml, errs):
|
|
"""The schema errors of `idml` as (part, message) without positions,
|
|
if every one is a deviation of InDesign's endnote markup from the 21.5
|
|
schema: an `Endnote` in a character range, or an element inside an
|
|
`EndnoteRange`. None otherwise (docs/measurement.md)."""
|
|
out = set()
|
|
z = zipfile.ZipFile(idml)
|
|
for e in errs:
|
|
m = ERROR.match(e)
|
|
if not m:
|
|
return None
|
|
part, line, msg = m["part"], int(m["line"]), m["msg"]
|
|
if not msg.startswith('element "Endnote" not allowed here') and \
|
|
endnote_parent(z.read(part), line) != "EndnoteRange":
|
|
return None
|
|
out.add((part, msg))
|
|
return out
|
|
|
|
|
|
def endnote_parent(data, line):
|
|
"""The tag of the parent of the first element that starts on `line`."""
|
|
import xml.parsers.expat
|
|
stack, found = [], []
|
|
p = xml.parsers.expat.ParserCreate()
|
|
|
|
def start(tag, attrs):
|
|
if not found and p.CurrentLineNumber == line:
|
|
found.append(stack[-1] if stack else None)
|
|
stack.append(tag)
|
|
p.StartElementHandler = start
|
|
p.EndElementHandler = lambda tag: stack.pop()
|
|
p.Parse(data, True)
|
|
return found[0] if found else None
|
|
|
|
|
|
def text_ranges(story):
|
|
"""Map (text flow, offset) -> attributes of the paragraph and character
|
|
range starting there, with keys prefixed PSR. and CSR. Flow 0 is the
|
|
story's own text; each footnote, table cell and note is a flow of its
|
|
own, numbered in document order. Ranges inside wrapper elements count,
|
|
at whatever depth. Offsets count Content in UTF-16 code units and every
|
|
other element inside a character range (Br, Table, Footnote, an
|
|
anchored frame) as one position; elements outside character ranges
|
|
(StoryPreference, ...) are not text."""
|
|
out = {}
|
|
flows = 0
|
|
|
|
def walk(el, pa, in_csr, flow, pos):
|
|
"""Walk the children of `el` inside the paragraph attributes `pa`;
|
|
`in_csr` tells whether `el` is inside a character range. Returns
|
|
the offset after them."""
|
|
for ch in el:
|
|
if ch.tag in NOT_TEXT:
|
|
continue
|
|
if ch.tag == "ParagraphStyleRange":
|
|
pos = walk(ch, {"PSR." + k: v for k, v in props(ch).items()}, False, flow, pos)
|
|
elif ch.tag == "CharacterStyleRange":
|
|
start = pos
|
|
pos = walk(ch, pa, True, flow, pos)
|
|
if pos > start:
|
|
ca = {"CSR." + k: v for k, v in props(ch).items()}
|
|
# A range nested at the same offset (Change > range at
|
|
# the start of this one) keeps the offset.
|
|
out.setdefault((flow, start), {**pa, **ca})
|
|
elif ch.tag in WRAPPERS:
|
|
pos = walk(ch, pa, in_csr, flow, pos)
|
|
elif ch.tag == "Content":
|
|
pos += len((ch.text or "").encode("utf-16-le")) // 2
|
|
else:
|
|
if in_csr:
|
|
pos += 1
|
|
nested(ch)
|
|
return pos
|
|
|
|
def nested(el):
|
|
"""Walk the text flows held in `el` (a table's cells, a footnote)."""
|
|
nonlocal flows
|
|
if el.tag in NESTED:
|
|
flows += 1
|
|
walk(el, {}, False, flows, 0)
|
|
return
|
|
for ch in el:
|
|
if ch.tag != "Properties":
|
|
nested(ch)
|
|
|
|
walk(story, {}, False, 0, 0)
|
|
return out
|
|
|
|
|
|
def load(path):
|
|
"""Map Self -> element, plus story texts, from an IDML package."""
|
|
z = zipfile.ZipFile(path)
|
|
elements = {}
|
|
stories = {}
|
|
ranges = {}
|
|
for name in z.namelist():
|
|
if not name.endswith(".xml"):
|
|
continue
|
|
try:
|
|
root = ET.fromstring(z.read(name))
|
|
except ET.ParseError as e:
|
|
raise SystemExit(f"{path}: {name}: {e}")
|
|
for el in root.iter():
|
|
s = el.get("Self")
|
|
if s is not None:
|
|
if el.tag == "Font":
|
|
s = font_self(el, s)
|
|
elements.setdefault((el.tag, s), el)
|
|
# Children without Self (TextFramePreference, ...) are
|
|
# compared as "<parent tag>/<child tag>" of the parent.
|
|
for ch in el:
|
|
if ch.get("Self") is not None or ch.tag == "Properties":
|
|
continue
|
|
if len(ch.attrib):
|
|
elements.setdefault((f"{el.tag}/{ch.tag}", s), ch)
|
|
else:
|
|
# A container without attributes (TransparencySetting):
|
|
# compare its children as "<parent>/<child>/<grandchild>".
|
|
for g in ch:
|
|
if g.get("Self") is None and len(g.attrib):
|
|
elements.setdefault((f"{el.tag}/{ch.tag}/{g.tag}", s), g)
|
|
if name == "Resources/Preferences.xml":
|
|
for el in root:
|
|
elements.setdefault((el.tag, "Preferences"), el)
|
|
if name.startswith("Stories/"):
|
|
for st in root.iter("Story"):
|
|
stories[st.get("Self")] = story_text(st)
|
|
ranges[st.get("Self")] = text_ranges(st)
|
|
return elements, stories, ranges
|
|
|
|
|
|
def story_text(story):
|
|
parts = []
|
|
for el in story.iter():
|
|
if el.tag == "Content":
|
|
parts.append(el.text or "")
|
|
elif el.tag == "Br":
|
|
parts.append("\n")
|
|
return "".join(parts)
|
|
|
|
|
|
def norm(v):
|
|
"""Normalise numbers so 1 == 1.0 and tiny float noise is ignored."""
|
|
def fix(tok):
|
|
try:
|
|
f = float(tok)
|
|
except ValueError:
|
|
return tok
|
|
return f"{f:.6g}"
|
|
return " ".join(fix(t) for t in v.split(" "))
|
|
|
|
|
|
def canon(el):
|
|
"""A structured property (TabList, AllNestedStyles, ...) as one string:
|
|
each leaf as tag=text with numbers normalised, in document order."""
|
|
if len(el) == 0 and el.get("type") != "list":
|
|
return f"{el.tag}={norm(el.text or '')}"
|
|
return el.tag + "[" + ",".join(canon(c) for c in el) + "]"
|
|
|
|
|
|
def props(el):
|
|
"""Attributes plus <Properties> children (as P.Name)."""
|
|
a = dict(el.attrib)
|
|
p = el.find("Properties")
|
|
if p is not None:
|
|
for c in p:
|
|
if c.get("type") == "list":
|
|
a["P." + c.tag] = canon(c)
|
|
elif len(c) == 0:
|
|
text = c.text or ""
|
|
# A line feed and tabs alone are the indentation of an
|
|
# element written open and closed (docs/measurement.md).
|
|
if re.fullmatch(r"\n\t*", text):
|
|
text = ""
|
|
extra = sorted((k, v) for k, v in c.attrib.items() if k != "type")
|
|
if extra:
|
|
text = " ".join(f"{k}={v}" for k, v in extra)
|
|
if c.tag == "Contents":
|
|
n = norm(text)
|
|
text = f"md5:{hashlib.md5(n.encode()).hexdigest()} {len(n)}"
|
|
a["P." + c.tag] = text
|
|
return a
|
|
|
|
|
|
# Values left out of value coverage because they cannot be derived from an
|
|
# INDD file: (tag pattern, attribute or P.<Properties child>, reason), tag
|
|
# patterns as in fnmatch. docs/measurement.md explains each entry.
|
|
EXCLUDED = [
|
|
("Font", "Status", "whether the font is installed on the exporting computer"),
|
|
("Document/{http://ns.adobe.com/AdobeInDesign/idml/1.0/packaging}*", "src",
|
|
"file names of the package parts, chosen by the IDML writer"),
|
|
("DocumentUser", "P.UserColor",
|
|
"colour the exporting InDesign gives each user, not the colour in the INDD"),
|
|
("Assignment", "Name",
|
|
"the exporting InDesign's name for unassigned content, which depends on its language"),
|
|
("Link", "LinkImportTime", "local time of the exporting computer; the INDD stores UTC"),
|
|
("Link", "LinkImportModificationTime",
|
|
"local time of the exporting computer; the INDD stores UTC"),
|
|
("*", "BitmapPrinting", "evaluated by the exporting computer; not stored in the INDD"),
|
|
]
|
|
|
|
# Attributes of a Link that describe the linked file as InDesign found it
|
|
# when it exported the IDML. When the IDML's LinkResourceURI differs from
|
|
# the converter's (which comes from the INDD), the link was resolved to
|
|
# another file on the exporting computer, and these values of that link
|
|
# are left out (docs/measurement.md).
|
|
RELINKED = re.compile(r"LinkResource.*|LinkImport.*")
|
|
|
|
# Font technology suffix that IDML adds to some family names, depending on
|
|
# the fonts installed on the exporting computer (docs/measurement.md).
|
|
FONT_SUFFIX = re.compile(r" \((OTF|TT|T1)\)$")
|
|
|
|
|
|
def font_name(tag, key, v, family=None):
|
|
"""A value with the font technology suffix removed if it names a font
|
|
family: FontFamily Name, Font FontFamily, AppliedFont and BulletsFont.
|
|
A Font's Name starts with its family (passed as family)."""
|
|
if (tag, key) in (("FontFamily", "Name"), ("Font", "FontFamily")) or \
|
|
key.endswith(("P.AppliedFont", "P.BulletsFont")):
|
|
return FONT_SUFFIX.sub("", v)
|
|
if (tag, key) == ("Font", "Name") and family and v.startswith(family):
|
|
return FONT_SUFFIX.sub("", family) + v[len(family):]
|
|
return v
|
|
|
|
|
|
def font_self(el, s):
|
|
"""A Font's Self with the technology suffix removed from the family
|
|
name it contains, so that fonts match whether or not the exporting
|
|
computer added the suffix."""
|
|
family = el.get("FontFamily") or ""
|
|
base = FONT_SUFFIX.sub("", family)
|
|
if base != family:
|
|
return s.replace("Fontn" + family, "Fontn" + base, 1)
|
|
return s
|
|
|
|
|
|
@functools.cache
|
|
def excluded(tag, key):
|
|
return any(k == key and fnmatch.fnmatchcase(tag, t) for t, k, _ in EXCLUDED)
|
|
|
|
|
|
def tree(el):
|
|
"""An element with its attributes, text and descendants as one string,
|
|
numbers normalised: the value of a structured <Properties> child such as
|
|
PathGeometry."""
|
|
attrs = ",".join(f"{k}={norm(v)}" for k, v in sorted(el.attrib.items()) if k != "type")
|
|
kids = ",".join(tree(c) for c in el)
|
|
return f"{el.tag}({attrs})[{kids}]{norm((el.text or '').strip())}"
|
|
|
|
|
|
def values(tag, el):
|
|
"""The values of an element that value coverage counts: attributes and
|
|
<Properties> children, without Self and without EXCLUDED values."""
|
|
a = props(el)
|
|
p = el.find("Properties")
|
|
if p is not None:
|
|
for c in p:
|
|
if c.get("type") != "list" and len(c):
|
|
a["P." + c.tag] = tree(c)
|
|
return {k: v for k, v in a.items() if k != "Self" and not excluded(tag, k)}
|
|
|
|
|
|
# Gap keys for values that could not be compared one by one.
|
|
NO_ELEMENT = "(element not produced)"
|
|
NO_TEXT = "(story text differs)"
|
|
NOT_IN_REFERENCE = "(element not in reference)"
|
|
ELEMENT = "(element)"
|
|
|
|
|
|
class Stats:
|
|
"""Comparison counts over a set of pairs."""
|
|
|
|
def __init__(self):
|
|
self.docs = 0
|
|
# Value coverage: reference values, and those reproduced exactly.
|
|
self.values = 0
|
|
self.reproduced = 0
|
|
# Links whose resource values are left out (RELINKED).
|
|
self.relinked = 0
|
|
# (tag, key) -> values wrong / missing, and documents affected.
|
|
self.gap_wrong = Counter()
|
|
self.gap_missing = Counter()
|
|
self.gap_docs = Counter()
|
|
self.gap_total = Counter()
|
|
# Values the output has and the reference does not, by (tag, key),
|
|
# and documents affected.
|
|
self.extra = 0
|
|
self.extra_values = Counter()
|
|
self.extra_docs = Counter()
|
|
# (pair, reproduced, values) for each pair.
|
|
self.pairs = []
|
|
self.found = Counter()
|
|
self.total = Counter()
|
|
self.attr_ok = defaultdict(Counter)
|
|
self.story_ok = Counter()
|
|
self.short_docs = Counter() # (kind, tag, attribute) -> documents affected
|
|
|
|
def add(self, other):
|
|
self.docs += other.docs
|
|
self.values += other.values
|
|
self.reproduced += other.reproduced
|
|
self.extra += other.extra
|
|
self.relinked += other.relinked
|
|
for c in ("gap_wrong", "gap_missing", "gap_docs", "gap_total", "extra_values",
|
|
"extra_docs"):
|
|
getattr(self, c).update(getattr(other, c))
|
|
self.pairs += other.pairs
|
|
self.found.update(other.found)
|
|
self.total.update(other.total)
|
|
for k, c in other.attr_ok.items():
|
|
self.attr_ok[k].update(c)
|
|
self.story_ok.update(other.story_ok)
|
|
self.short_docs.update(other.short_docs)
|
|
|
|
def coverage(self):
|
|
return self.reproduced / self.values if self.values else 1.0
|
|
|
|
def quantile(self, q):
|
|
"""Value coverage of the pair at quantile q (0 = lowest)."""
|
|
cov = sorted(ok / n if n else 1.0 for _, ok, n in self.pairs)
|
|
return cov[min(len(cov) - 1, int(q * len(cov)))] if cov else 1.0
|
|
|
|
def document_score(self, at_least):
|
|
"""Pairs whose value coverage is at least `at_least`."""
|
|
return sum(1 for _, ok, n in self.pairs if n == 0 or ok / n >= at_least)
|
|
|
|
def gaps(self):
|
|
"""(tag, key), documents, values wrong, values missing; most
|
|
documents first, then most values."""
|
|
rows = [(k, d, self.gap_wrong[k], self.gap_missing[k]) for k, d in self.gap_docs.items()]
|
|
return sorted(rows, key=lambda r: (-r[1], -(r[2] + r[3]), r[0]))
|
|
|
|
def extras(self):
|
|
"""(tag, key), documents, extra values; most documents first."""
|
|
rows = [(k, d, self.extra_values[k]) for k, d in self.extra_docs.items()]
|
|
return sorted(rows, key=lambda r: (-r[1], -r[2], r[0]))
|
|
|
|
def story_line(self):
|
|
st = sum(self.story_ok.values())
|
|
return (f"{self.story_ok['ok']}/{st} exact, {self.story_ok['wrong']} differ, "
|
|
f"{self.story_ok['missing']} missing")
|
|
|
|
|
|
def compare_pair(name, ref, ours, examples, show):
|
|
"""Stats of one pair: `ref` and `ours` are load() results. Adds up to
|
|
`show` mismatches per attribute to `examples`."""
|
|
ref_el, ref_st, ref_rg = ref
|
|
our_el, our_st, our_rg = ours
|
|
p = Stats()
|
|
p.docs = 1
|
|
short = set()
|
|
for (tag, s), el in ref_el.items():
|
|
p.total[tag] += 1
|
|
mine = our_el.get((tag, s))
|
|
if mine is None:
|
|
short.add(("element", tag, ""))
|
|
continue
|
|
p.found[tag] += 1
|
|
mine = props(mine)
|
|
for k, v in props(el).items():
|
|
if k == "Self":
|
|
continue
|
|
if k not in mine:
|
|
p.attr_ok[(tag, k)]["missing"] += 1
|
|
short.add(("attribute", tag, k))
|
|
elif norm(mine[k]) == norm(v):
|
|
p.attr_ok[(tag, k)]["ok"] += 1
|
|
else:
|
|
p.attr_ok[(tag, k)]["wrong"] += 1
|
|
short.add(("attribute", tag, k))
|
|
if len(examples[(tag, k)]) < show:
|
|
examples[(tag, k)].append((name, s, v, mine[k]))
|
|
for sid, rgs in ref_rg.items():
|
|
if our_st.get(sid) != ref_st.get(sid):
|
|
continue
|
|
mine = our_rg.get(sid, {})
|
|
for start, attrs in rgs.items():
|
|
p.total["TextRange"] += 1
|
|
if start not in mine:
|
|
short.add(("element", "TextRange", ""))
|
|
continue
|
|
p.found["TextRange"] += 1
|
|
for k, v in attrs.items():
|
|
o = mine[start].get(k)
|
|
if o is None:
|
|
p.attr_ok[("TextRange", k)]["missing"] += 1
|
|
short.add(("attribute", "TextRange", k))
|
|
elif norm(o) == norm(v):
|
|
p.attr_ok[("TextRange", k)]["ok"] += 1
|
|
else:
|
|
p.attr_ok[("TextRange", k)]["wrong"] += 1
|
|
short.add(("attribute", "TextRange", k))
|
|
if len(examples[("TextRange", k)]) < show:
|
|
examples[("TextRange", k)].append((name, sid, start, v, o))
|
|
for sid, text in ref_st.items():
|
|
if sid not in our_st:
|
|
p.story_ok["missing"] += 1
|
|
short.add(("element", "Story", ""))
|
|
elif our_st[sid] == text:
|
|
p.story_ok["ok"] += 1
|
|
else:
|
|
p.story_ok["wrong"] += 1
|
|
short.add(("text", "Story", ""))
|
|
if len(examples[("Story", "text")]) < max(show, 3):
|
|
examples[("Story", "text")].append((name, sid, text[:80], our_st[sid][:80]))
|
|
p.short_docs.update(short)
|
|
value_coverage(p, ref, ours)
|
|
return p
|
|
|
|
|
|
def value_coverage(p, ref, ours):
|
|
"""Count reference values and the reproduced ones into `p`: element
|
|
presence, attribute and <Properties> values, story text, and the start
|
|
and attributes of each text range. Also count the values the output has
|
|
and the reference does not (extra values), with the same keys."""
|
|
ref_el, ref_st, ref_rg = ref
|
|
our_el, our_st, our_rg = ours
|
|
total = Counter()
|
|
wrong = Counter()
|
|
missing = Counter()
|
|
extra = Counter()
|
|
|
|
def compare(tag, theirs, mine):
|
|
for k, v in theirs.items():
|
|
key = (tag, k)
|
|
total[key] += 1
|
|
o = mine.get(k)
|
|
if o is None:
|
|
missing[key] += 1
|
|
elif norm(font_name(tag, k, o, mine.get("FontFamily"))) != \
|
|
norm(font_name(tag, k, v, theirs.get("FontFamily"))):
|
|
wrong[key] += 1
|
|
for k in mine.keys() - theirs.keys():
|
|
extra[(tag, k)] += 1
|
|
|
|
for (tag, s), el in ref_el.items():
|
|
theirs = values(tag, el)
|
|
mine = our_el.get((tag, s))
|
|
if mine is None:
|
|
n = 1 + len(theirs)
|
|
total[(tag, NO_ELEMENT)] += n
|
|
missing[(tag, NO_ELEMENT)] += n
|
|
continue
|
|
total[(tag, ELEMENT)] += 1
|
|
mine = values(tag, mine)
|
|
if tag == "Link" and mine.get("LinkResourceURI") not in (None, theirs.get("LinkResourceURI")):
|
|
theirs = {k: v for k, v in theirs.items() if not RELINKED.fullmatch(k)}
|
|
mine = {k: v for k, v in mine.items() if not RELINKED.fullmatch(k)}
|
|
p.relinked += 1
|
|
compare(tag, theirs, mine)
|
|
for (tag, s), el in our_el.items():
|
|
if (tag, s) not in ref_el:
|
|
extra[(tag, NOT_IN_REFERENCE)] += 1 + len(values(tag, el))
|
|
for sid, text in ref_st.items():
|
|
key = ("Story", "(text)")
|
|
total[key] += 1
|
|
if sid not in our_st:
|
|
missing[key] += 1
|
|
elif our_st[sid] != text:
|
|
wrong[key] += 1
|
|
for sid in our_st.keys() - ref_st.keys():
|
|
extra[("Story", "(text)")] += 1
|
|
extra[("TextRange", NOT_IN_REFERENCE)] += sum(1 + len(a) for a in
|
|
our_rg.get(sid, {}).values())
|
|
for sid, rgs in ref_rg.items():
|
|
if our_st.get(sid) != ref_st.get(sid):
|
|
n = sum(1 + len(a) for a in rgs.values())
|
|
total[("TextRange", NO_TEXT)] += n
|
|
missing[("TextRange", NO_TEXT)] += n
|
|
continue
|
|
mine = our_rg.get(sid, {})
|
|
for start, attrs in rgs.items():
|
|
if start not in mine:
|
|
n = 1 + len(attrs)
|
|
total[("TextRange", NO_ELEMENT)] += n
|
|
missing[("TextRange", NO_ELEMENT)] += n
|
|
continue
|
|
total[("TextRange", ELEMENT)] += 1
|
|
compare("TextRange", attrs, mine[start])
|
|
for start in mine.keys() - rgs.keys():
|
|
extra[("TextRange", NOT_IN_REFERENCE)] += 1 + len(mine[start])
|
|
n = sum(total.values())
|
|
bad = sum(wrong.values()) + sum(missing.values())
|
|
p.values += n
|
|
p.reproduced += n - bad
|
|
p.gap_total.update(total)
|
|
p.gap_wrong.update(wrong)
|
|
p.gap_missing.update(missing)
|
|
p.gap_docs.update(k for k in total if wrong[k] or missing[k])
|
|
p.extra += sum(extra.values())
|
|
p.extra_values.update(extra)
|
|
p.extra_docs.update(k for k, v in extra.items() if v)
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--limit", type=int, default=0)
|
|
ap.add_argument("--detail", action="append", default=[],
|
|
help="print attribute table for this tag (repeatable)")
|
|
ap.add_argument("--file", help="only pairs whose path contains this")
|
|
ap.add_argument("--show", type=int, default=0, help="show N mismatches per attribute")
|
|
ap.add_argument("--schemas", help="IDML RelaxNG schema directory (validate output)")
|
|
ap.add_argument("--jing", help="directory with jing.jar, isorelax.jar, saxon.jar")
|
|
ap.add_argument("--bin", default=str(BIN), help="converter binary")
|
|
ap.add_argument("--all", action="store_true",
|
|
help="also convert and validate files without a reference IDML")
|
|
ap.add_argument("--exclude", action="append", default=[],
|
|
help="leave out files whose path under corpus/ starts with this (repeatable)")
|
|
ap.add_argument("--warnings", type=int, default=15,
|
|
help="number of warning kinds to list")
|
|
ap.add_argument("--shortfalls", type=int, default=0,
|
|
help="rank the N biggest shortfalls by documents affected")
|
|
ap.add_argument("--trusted", action="store_true",
|
|
help="element tables, shortfalls and details over trustworthy pairs only")
|
|
ap.add_argument("--stale", type=int, default=0,
|
|
help="list N stale pairs with their reasons")
|
|
ap.add_argument("--gaps", type=int, default=30,
|
|
help="print the N biggest value coverage gaps")
|
|
ap.add_argument("--out", default=str(ROOT / "target" / "compare"),
|
|
help="directory for pairs.tsv and gaps*.tsv (default target/compare)")
|
|
ap.add_argument("--jobs", type=int, default=os.cpu_count() or 1,
|
|
help="conversions and schema validations run in parallel")
|
|
args = ap.parse_args()
|
|
check = bool(args.schemas and args.jing)
|
|
|
|
every = Stats() # all pairs
|
|
trusted = Stats() # pairs without a sign of staleness
|
|
stale = [] # (name, reasons)
|
|
pair_rows = [] # (name, Stats, stale reasons) per pair
|
|
examples = defaultdict(list)
|
|
failures = []
|
|
warnings = Counter()
|
|
warned_files = Counter()
|
|
seen = set()
|
|
known = {r["path"]: r["sha256"] for r in inventory() if r.get("sha256")}
|
|
todo = pairs(args.limit, args.file, args.exclude, seen, known)
|
|
others = unpaired(args.file, args.exclude, seen, known) if args.all else []
|
|
other_failures = []
|
|
pool = ThreadPoolExecutor(max(1, args.jobs))
|
|
|
|
def convert(indd, out):
|
|
"""Convert one file; runs in the pool."""
|
|
return subprocess.run([args.bin, "convert", indd, out], capture_output=True, text=True)
|
|
|
|
def convert_pair(indd, idml, out):
|
|
"""Convert one paired file and read what stale_reasons needs: the
|
|
INDD's UIDs and the ModifyDate of both files; runs in the pool."""
|
|
r = convert(indd, out)
|
|
x = subprocess.run([args.bin, "xmp", indd], capture_output=True)
|
|
u = subprocess.run([args.bin, "uids", indd], capture_output=True, text=True)
|
|
uids = None
|
|
if u.returncode == 0:
|
|
uids = {}
|
|
for line in u.stdout.splitlines():
|
|
uid, cls = line.split("\t")
|
|
uids[int(uid)] = None if cls == "-" else int(cls, 16)
|
|
dates = (modify_date(x.stdout) if x.returncode == 0 else None, idml_modify_date(idml))
|
|
return r, dates, uids
|
|
|
|
def count_warnings(r):
|
|
kinds = set()
|
|
for line in r.stderr.splitlines():
|
|
if line.startswith("warning: "):
|
|
kind = warning_kind(line[len("warning: "):])
|
|
warnings[kind] += 1
|
|
kinds.add(kind)
|
|
for kind in kinds:
|
|
warned_files[kind] += 1
|
|
|
|
other_invalid = []
|
|
invalid = []
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
jobs = [pool.submit(convert, indd, Path(tmp) / f"other{n}.idml")
|
|
for n, indd in enumerate(others)]
|
|
jobs += [pool.submit(convert_pair, indd, idml, Path(tmp) / f"pair{n}.idml")
|
|
for n, (indd, idml) in enumerate(todo)]
|
|
# Converted files waiting for validation: (output, name, list of
|
|
# invalid files it belongs to). They are validated in batches, which
|
|
# bounds the space the unpacked outputs take.
|
|
to_check = []
|
|
|
|
# Paired outputs whose only schema errors are InDesign's endnote
|
|
# markup: (name, errors, normalised errors, reference IDML). They
|
|
# pass if the reference has the same errors.
|
|
endnote_pending = []
|
|
|
|
def flush(at_least):
|
|
if len(to_check) < at_least:
|
|
return
|
|
errs = validate([out for out, _, _, _ in to_check], args)
|
|
for out, name, bad, ref in to_check:
|
|
e = errs.get(str(out))
|
|
if e:
|
|
known = endnote_errors(out, e) if ref else None
|
|
if known:
|
|
endnote_pending.append((name, e, known, ref))
|
|
else:
|
|
bad.append((name, e))
|
|
out.unlink(missing_ok=True)
|
|
to_check.clear()
|
|
for n, indd in enumerate(others):
|
|
r = jobs[n].result()
|
|
count_warnings(r)
|
|
out = Path(tmp) / f"other{n}.idml"
|
|
if r.returncode != 0:
|
|
other_failures.append((indd.name, r.stderr.strip()))
|
|
elif check:
|
|
to_check.append((out, indd.name, other_invalid, None))
|
|
flush(VALIDATE_BATCH)
|
|
continue
|
|
out.unlink(missing_ok=True)
|
|
for n, (indd, idml) in enumerate(todo):
|
|
out = Path(tmp) / f"pair{n}.idml"
|
|
r, (indd_date, idml_date), indd_uids = jobs[len(others) + n].result()
|
|
count_warnings(r)
|
|
if r.returncode != 0:
|
|
failures.append((indd.name, r.stderr.strip()))
|
|
continue
|
|
if check:
|
|
to_check.append((out, indd.name, invalid, idml))
|
|
ref = load(idml)
|
|
ours = load(out)
|
|
reasons = stale_reasons(indd_date, idml_date, indd_uids, ref[0], ref[1], ours[1])
|
|
p = compare_pair(indd.name, ref, ours, examples, args.show)
|
|
rel = str(indd.relative_to(ROOT / "corpus"))
|
|
p.pairs = [(rel, p.reproduced, p.values)]
|
|
pair_rows.append((rel, p, reasons))
|
|
every.add(p)
|
|
if reasons:
|
|
stale.append((rel, reasons))
|
|
else:
|
|
trusted.add(p)
|
|
if check:
|
|
flush(VALIDATE_BATCH)
|
|
else:
|
|
out.unlink(missing_ok=True)
|
|
pool.shutdown()
|
|
flush(1)
|
|
endnote_passed = []
|
|
if endnote_pending:
|
|
ref_errs = validate([ref for _, _, _, ref in endnote_pending], args)
|
|
for name, e, known, ref in endnote_pending:
|
|
theirs = endnote_errors(ref, ref_errs.get(str(ref), [])) or set()
|
|
if known <= theirs:
|
|
endnote_passed.append(name)
|
|
else:
|
|
invalid.append((name, e))
|
|
|
|
print(f"conversion failures: {len(failures)} of {len(todo)} paired files")
|
|
for name, err in failures[:10]:
|
|
print(f" {name}: {err}")
|
|
if check:
|
|
print(f"schema validation failures: {len(invalid)}")
|
|
for name, errs in invalid[:10]:
|
|
print(f" {name}: {len(errs)} errors, {errs[:3]}")
|
|
print(f"schema errors accepted (endnote markup, as in the reference): "
|
|
f"{len(endnote_passed)} files")
|
|
if args.all:
|
|
print(f"files without a reference: {len(others)}, "
|
|
f"conversion failures: {len(other_failures)}")
|
|
for name, err in other_failures[:10]:
|
|
print(f" {name}: {err}")
|
|
if check:
|
|
print(f"files without a reference: schema validation failures: {len(other_invalid)}")
|
|
with_endnotes = [n for n, errs in other_invalid if any(
|
|
'element "Endnote"' in e or "EndnoteRange" in e for e in errs)]
|
|
if with_endnotes:
|
|
print(f" with endnote markup errors: {len(with_endnotes)}: {with_endnotes[:10]}")
|
|
for name, errs in other_invalid[:10]:
|
|
print(f" {name}: {len(errs)} errors, {errs[:3]}")
|
|
print(f"warnings: {sum(warnings.values())} (count, files, kind)")
|
|
for kind, count in warnings.most_common(args.warnings):
|
|
print(f" {count:6} {warned_files[kind]:4} {kind[:150]}")
|
|
print(f"story text: {every.story_line()}")
|
|
reasons = Counter(r for _, rs in stale for r in rs)
|
|
print(f"trustworthy pairs: {trusted.docs} of {every.docs}; stale: {len(stale)} "
|
|
f"({', '.join(f'{r}: {n}' for r, n in reasons.most_common())})")
|
|
print(f"trustworthy pairs: story text: {trusted.story_line()}")
|
|
for name, rs in stale[:args.stale]:
|
|
print(f" stale: {name}: {'; '.join(rs)}")
|
|
st = trusted if args.trusted else every
|
|
which = "trustworthy pairs" if args.trusted else "all pairs"
|
|
print(f"\nelements, {which} (produced / in reference):")
|
|
for tag, n in st.total.most_common():
|
|
print(f" {tag:34} {st.found[tag]:6} / {n:6}")
|
|
if args.shortfalls:
|
|
print(f"\nshortfalls by documents affected (of {st.docs} {which}):")
|
|
def instances(kind, tag, k):
|
|
"""How many elements, values or stories fall short."""
|
|
if kind == "attribute":
|
|
return st.attr_ok[(tag, k)]["wrong"] + st.attr_ok[(tag, k)]["missing"]
|
|
if kind == "element":
|
|
return st.total[tag] - st.found[tag]
|
|
return st.story_ok["wrong"]
|
|
ranked = sorted(st.short_docs.items(), key=lambda kv: (-kv[1], -instances(*kv[0]), kv[0]))
|
|
for (kind, tag, k), n in ranked[:args.shortfalls]:
|
|
what = f"{st.story_ok['wrong']} stories differ"
|
|
if kind == "attribute":
|
|
c = st.attr_ok[(tag, k)]
|
|
what = f"{c['wrong']} wrong, {c['missing']} missing of {sum(c.values())}"
|
|
elif kind == "element":
|
|
what = f"{st.total[tag] - st.found[tag]} of {st.total[tag]} not produced"
|
|
print(f" {n:5} {tag + (' ' + k if k else ''):56} {what}")
|
|
for tag in args.detail:
|
|
print(f"\nattributes of {tag}, {which} (ok / wrong / missing):")
|
|
rows = [(k, c) for (t, k), c in st.attr_ok.items() if t == tag]
|
|
for k, c in sorted(rows, key=lambda kc: -sum(kc[1].values())):
|
|
print(f" {k:40} {c['ok']:6} {c['wrong']:6} {c['missing']:6}")
|
|
for ex in examples[(tag, k)]:
|
|
print(f" {ex}")
|
|
for ex in examples[("Story", "text")]:
|
|
print(" story mismatch:", ex)
|
|
headline(args, every, trusted, pair_rows)
|
|
write_summary(Path(args.out) / "summary.json", args, check,
|
|
(todo, failures, invalid), (others, other_failures, other_invalid),
|
|
every, trusted)
|
|
|
|
|
|
def write_summary(path, args, check, paired, unpaired_files, every, trusted):
|
|
"""Write the numbers of the README's "Current numbers" section, and the
|
|
options they were measured with, as JSON (tools/readme_numbers.py).
|
|
`paired` and `unpaired_files` are (files, failures, invalid)."""
|
|
def error(stderr):
|
|
lines = [x for x in stderr.splitlines() if not x.startswith("warning: ")]
|
|
return lines[-1] if lines else ""
|
|
|
|
def files(group, converted):
|
|
todo, failures, invalid = group
|
|
if not converted:
|
|
return None
|
|
return {"files": len(todo),
|
|
"failures": [error(err) for _, err in failures],
|
|
"invalid": len(invalid) if check else None}
|
|
|
|
def pairs(st):
|
|
return {"pairs": st.docs, "values": st.values, "reproduced": st.reproduced,
|
|
"stories": {"exact": st.story_ok["ok"], "differ": st.story_ok["wrong"],
|
|
"missing": st.story_ok["missing"]}}
|
|
|
|
summary = {
|
|
"options": {"all": args.all, "validated": check, "exclude": args.exclude,
|
|
"limit": args.limit, "file": args.file},
|
|
"paired": files(paired, True),
|
|
"others": files(unpaired_files, args.all),
|
|
"trustworthy": pairs(trusted),
|
|
"all_pairs": pairs(every),
|
|
}
|
|
path.write_text(json.dumps(summary, indent=1) + "\n")
|
|
|
|
|
|
def gap_name(key):
|
|
|
|
tag, k = key
|
|
return f"{tag} {k}"
|
|
|
|
|
|
def write_gaps(path, st):
|
|
with open(path, "w") as f:
|
|
f.write("tag\tkey\tdocuments\twrong\tmissing\tvalues\n")
|
|
for key, docs, w, m in st.gaps():
|
|
f.write(f"{key[0]}\t{key[1]}\t{docs}\t{w}\t{m}\t{st.gap_total[key]}\n")
|
|
|
|
|
|
def write_extras(path, st):
|
|
with open(path, "w") as f:
|
|
f.write("tag\tkey\tdocuments\textra\n")
|
|
for key, docs, n in st.extras():
|
|
f.write(f"{key[0]}\t{key[1]}\t{docs}\t{n}\n")
|
|
|
|
|
|
def write_values(path, st):
|
|
"""Every key with its values and how many are reproduced, for checking
|
|
that a change reproduces no fewer values of any key."""
|
|
with open(path, "w") as f:
|
|
f.write("tag\tkey\tvalues\treproduced\twrong\tmissing\n")
|
|
for key in sorted(st.gap_total):
|
|
w, m = st.gap_wrong[key], st.gap_missing[key]
|
|
f.write(f"{key[0]}\t{key[1]}\t{st.gap_total[key]}\t{st.gap_total[key] - w - m}"
|
|
f"\t{w}\t{m}\n")
|
|
|
|
|
|
def headline(args, every, trusted, pair_rows):
|
|
"""Print value coverage, document scores and the biggest gaps; write
|
|
pairs.tsv, gaps.tsv (trustworthy pairs) and gaps-all.tsv to --out."""
|
|
out = Path(args.out)
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
with open(out / "pairs.tsv", "w") as f:
|
|
f.write("path\ttrusted\tcoverage\treproduced\tvalues\textra\tstories"
|
|
"\tstories_identical\tstale reasons\n")
|
|
for name, p, reasons in sorted(pair_rows, key=lambda r: r[1].coverage()):
|
|
f.write(f"{name}\t{not reasons}\t{p.coverage():.5f}\t{p.reproduced}\t{p.values}"
|
|
f"\t{p.extra}\t{sum(p.story_ok.values())}\t{p.story_ok['ok']}"
|
|
f"\t{'; '.join(reasons)}\n")
|
|
write_gaps(out / "gaps.tsv", trusted)
|
|
write_gaps(out / "gaps-all.tsv", every)
|
|
write_extras(out / "extras.tsv", trusted)
|
|
write_extras(out / "extras-all.tsv", every)
|
|
write_values(out / "values.tsv", trusted)
|
|
write_values(out / "values-all.tsv", every)
|
|
print("\nheadline (docs/measurement.md):")
|
|
for label, st in (("trustworthy pairs", trusted), ("all pairs", every)):
|
|
n = st.docs or 1
|
|
print(f" {label}: value coverage {st.coverage():.2%} "
|
|
f"({st.reproduced} of {st.values} values); documents at >= 99 %: "
|
|
f"{st.document_score(0.99)}/{st.docs} ({st.document_score(0.99) / n:.1%}), "
|
|
f">= 99.9 %: {st.document_score(0.999)}/{st.docs} "
|
|
f"({st.document_score(0.999) / n:.1%})")
|
|
print(f" extra values (in the output, not in the reference): {st.extra} "
|
|
f"({st.extra / (st.values or 1):.2%} of reference values); "
|
|
f"links resolved to another file at export: {st.relinked}")
|
|
print(f" per pair: lowest {st.quantile(0):.1%}, 10th percentile {st.quantile(0.1):.1%}, "
|
|
f"median {st.quantile(0.5):.1%}, 90th percentile {st.quantile(0.9):.1%}, "
|
|
f"highest {st.quantile(1):.1%}")
|
|
rows = trusted.gaps()
|
|
print(f"\nvalue gaps, trustworthy pairs, by documents affected "
|
|
f"(documents, values wrong, missing, of values; all in {out / 'gaps.tsv'}):")
|
|
for key, docs, w, m in rows[:args.gaps]:
|
|
print(f" {docs:5} {w:7} {m:7} {trusted.gap_total[key]:8} {gap_name(key)[:90]}")
|
|
print("\nvalue gaps, trustworthy pairs, by values (documents, values wrong, missing, of values):")
|
|
for key, docs, w, m in sorted(rows, key=lambda r: -(r[2] + r[3]))[:args.gaps // 2]:
|
|
print(f" {docs:5} {w:7} {m:7} {trusted.gap_total[key]:8} {gap_name(key)[:90]}")
|
|
rows = trusted.extras()
|
|
print(f"\nextra values, trustworthy pairs, by documents affected "
|
|
f"(documents, extra values; all in {out / 'extras.tsv'}):")
|
|
for key, docs, n in rows[:args.gaps // 2]:
|
|
print(f" {docs:5} {n:7} {gap_name(key)[:90]}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|