mirror of
https://github.com/storytold/indd-utils.git
synced 2026-11-20 01:00:46 +01:00
tools/root_values.py collects the root styles from the pairs that compare.py compares (little-endian, same major version, without the privately held samples): 794 distinct IDML files, DOM 7 to 21, instead of the 88 the corpus had first. The pair rule leaves out an IDML with no root styles beside a newer INDD and a big-endian pair. The other object styles and the preferences stay pinned to SOURCE; their files are unchanged. The root table style writes every mapped attribute of its list, since the stroke weights, spacing and end tints now vary between files and are no longer in the values file. The root object style's varying values (named grid, optical margin size, baseline frame grid colour, corner radii, gutter, first baseline offset) are read from the INDD. Evidence in idml-values.md and tables.md. compare.py --exclude own/, trustworthy pairs: coverage 98.53 % -> 98.56 % (+2,942 values whose first DOM version is now earlier: IncludeClass, EpubAriaRole, paragraph border options, ParagraphKashidaWidth, shading overprint, TransformAttributeOption and FlexLayoutAttributeOption), documents >= 99 % 278 -> 313; all pairs 96.65 % -> 96.67 %. No pair loses a reproduced value; extras unchanged (8,968 trustworthy, 278,333 all). No new failures.
437 lines
18 KiB
Python
437 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""Find the values that every corpus IDML has on its root styles, on its
|
|
other object styles and in its preferences.
|
|
|
|
Reads Resources/Styles.xml and Resources/Preferences.xml of the corpus
|
|
IDML files (distinct files only: the pairs compare.py compares for the
|
|
root styles, the pairs under SOURCE for the other object styles and the
|
|
preferences) and collects each attribute,
|
|
Properties child and child element attribute of the root styles and of
|
|
the top-level preference elements. A value is kept when it is the same in
|
|
every IDML from some DOM version on and present in all of them; the
|
|
earliest such version is its minimum version. Attributes the converter
|
|
reads from the INDD (the TEXT_ATTRS table in src/idml/attrs.rs) are left
|
|
out for the paragraph and character root styles, and the document setup
|
|
it reads is left out of the preferences. Preference values that refer to
|
|
a style other than a root style are left out, since the package need not
|
|
have that style.
|
|
|
|
Object styles other than the root style `[None]` are collected the same
|
|
way, all styles of a file together: a value is kept when every such style
|
|
in every IDML from some DOM version on has it, with the same value.
|
|
|
|
Usage: python3 -I tools/root_values.py [--write]
|
|
Prints the evidence counts. With --write, regenerates
|
|
src/idml/root_values.xml, src/idml/object_style_values.xml and
|
|
src/idml/preference_values.xml. See docs/format/idml-values.md.
|
|
"""
|
|
|
|
import hashlib
|
|
import re
|
|
import sys
|
|
import xml.etree.ElementTree as ET
|
|
import zipfile
|
|
from collections import OrderedDict, defaultdict
|
|
from pathlib import Path
|
|
from xml.sax.saxutils import quoteattr
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
OUT = ROOT / "src" / "idml" / "root_values.xml"
|
|
PREF_OUT = ROOT / "src" / "idml" / "preference_values.xml"
|
|
OBJECT_OUT = ROOT / "src" / "idml" / "object_style_values.xml"
|
|
|
|
# The root styles are collected from the pairs that tools/compare.py
|
|
# compares, without the files whose path under corpus/ starts with one of
|
|
# these (privately held samples).
|
|
EXCLUDE = ("own/",)
|
|
# The object styles other than [None] and the preferences were collected
|
|
# from the corpus pairs under this directory, the whole corpus at the
|
|
# time. Later pairs are evidence for tools/element_values.py;
|
|
# regenerating these files from them would drop values that the larger
|
|
# corpus shows to vary (docs/format/idml-values.md).
|
|
SOURCE = "third-party/"
|
|
|
|
# Preference attributes the converter reads from the INDD.
|
|
PREF_SKIP = {
|
|
("DocumentPreference", "PageHeight"), ("DocumentPreference", "PageWidth"),
|
|
("DocumentPreference", "FacingPages"), ("DocumentPreference", "Intent"),
|
|
("DocumentPreference", "DocumentBleedTopOffset"),
|
|
("DocumentPreference", "DocumentBleedBottomOffset"),
|
|
("DocumentPreference", "DocumentBleedInsideOrLeftOffset"),
|
|
("DocumentPreference", "DocumentBleedOutsideOrRightOffset"),
|
|
("ViewPreference", "RulerOrigin"),
|
|
}
|
|
# Styles a preference value may refer to: those every package has.
|
|
ROOT_STYLES = {
|
|
"ParagraphStyle/$ID/[No paragraph style]",
|
|
"CharacterStyle/$ID/[No character style]",
|
|
"ObjectStyle/$ID/[None]",
|
|
}
|
|
# A preference value first seen in a later version is kept only when at
|
|
# least this many files show it.
|
|
MIN_FILES = 10
|
|
STYLE_REF = re.compile(r"(?:>|^)(Paragraph|Character|Object|Cell|Table)Style/[^<]*")
|
|
|
|
PREF_HEADER = """<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
Values that every IDML exported by InDesign in the corpus has in
|
|
Resources/Preferences.xml. The converter writes them; values it reads
|
|
from the INDD take precedence. Values in a block with MinimumVersion are
|
|
written only for documents of that InDesign version or later. Generated
|
|
by tools/root_values.py; evidence in docs/format/idml-values.md.
|
|
-->
|
|
"""
|
|
|
|
TARGETS = OrderedDict([
|
|
("ParagraphStyle", "ParagraphStyle/$ID/[No paragraph style]"),
|
|
("CharacterStyle", "CharacterStyle/$ID/[No character style]"),
|
|
("ObjectStyle", "ObjectStyle/$ID/[None]"),
|
|
("CellStyle", "CellStyle/$ID/[None]"),
|
|
("TableStyle", "TableStyle/$ID/[No table style]"),
|
|
])
|
|
# Written by the converter itself.
|
|
SKIP = {"Self", "Name"}
|
|
|
|
HEADER = """<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
Values that every IDML exported by InDesign in the corpus has on the root
|
|
styles. The converter writes them on its root styles; values it reads from
|
|
the INDD take precedence. Values in a block with MinimumVersion are written
|
|
only for documents of that InDesign version or later. Generated by
|
|
tools/root_values.py; evidence in docs/format/idml-values.md.
|
|
-->
|
|
"""
|
|
|
|
|
|
def decoded_names():
|
|
src = (ROOT / "src" / "idml" / "attrs.rs").read_text()
|
|
table = src[src.index("const TEXT_ATTRS"):src.index("/// An attribute written as a")]
|
|
return set(re.findall(r'0x[0-9A-F]+,\s*"(\w+)"', table))
|
|
|
|
|
|
def corpus_idmls(source=None):
|
|
"""Without `source`, the reference IDML of every pair that
|
|
tools/compare.py compares: a little-endian INDD whose IDML has the
|
|
same major version, not in EXCLUDE. With `source`, the IDML of every
|
|
pair whose path starts with it. Each distinct IDML counts once."""
|
|
lines = (ROOT / "corpus" / "inventory.tsv").read_text().splitlines()
|
|
cols = lines[0].split("\t")
|
|
seen, out, pairs = set(), [], 0
|
|
for line in lines[1:]:
|
|
row = dict(zip(cols, line.split("\t")))
|
|
path, order, ver, dom = row["path"], row["order"], row["version"], row["idml_dom"]
|
|
if source is not None:
|
|
if not dom or not dom[0].isdigit() or not path.startswith(source):
|
|
continue
|
|
elif order != "LE" or not dom or ver.split(".")[0] != dom.split(".")[0]:
|
|
continue
|
|
elif path.startswith(EXCLUDE):
|
|
continue
|
|
pairs += 1
|
|
idml = ROOT / "corpus" / (row.get("idml") or str(Path(path).with_suffix(".idml")))
|
|
digest = hashlib.md5(idml.read_bytes()).hexdigest()
|
|
if digest not in seen:
|
|
seen.add(digest)
|
|
out.append((int(dom.split(".")[0]), idml))
|
|
return pairs, out
|
|
|
|
|
|
def leaves(el, path, out, order):
|
|
"""Flatten an element: (path, '@attr') and (path, 'P/tag') keys."""
|
|
for k, v in el.attrib.items():
|
|
out[(path, "@" + k)] = v
|
|
order.setdefault((path, "@" + k), len(order))
|
|
for c in el:
|
|
if c.tag == "Properties":
|
|
for p in c:
|
|
key = (path, "P/" + p.tag)
|
|
out[key] = re.sub(r">\s+<", "><", ET.tostring(p, encoding="unicode").strip())
|
|
order.setdefault(key, len(order))
|
|
else:
|
|
order.setdefault((path + (c.tag,), ""), len(order))
|
|
leaves(c, path + (c.tag,), out, order)
|
|
|
|
|
|
def analyse():
|
|
decoded = decoded_names()
|
|
pairs, idmls = corpus_idmls()
|
|
doms = [d for d, _ in idmls]
|
|
values = defaultdict(dict)
|
|
order = defaultdict(dict)
|
|
for i, (_dom, idml) in enumerate(idmls):
|
|
root = ET.fromstring(zipfile.ZipFile(idml).read("Resources/Styles.xml"))
|
|
for el in root.iter():
|
|
if el.tag in TARGETS and TARGETS[el.tag] == el.get("Self"):
|
|
out = {}
|
|
leaves(el, (), out, order[el.tag])
|
|
for k, v in out.items():
|
|
values[(el.tag, k)][i] = v
|
|
kept = defaultdict(list)
|
|
left_out = defaultdict(lambda: defaultdict(list))
|
|
for (tag, key), by_file in values.items():
|
|
path, name = key
|
|
if not path and name[1:] in SKIP:
|
|
continue
|
|
plain = name.split("/", 1)[1] if name.startswith("P/") else name[1:]
|
|
if not path and tag in ("ParagraphStyle", "CharacterStyle") and plain in decoded:
|
|
left_out[tag]["read from the INDD"].append(plain)
|
|
continue
|
|
since = None
|
|
for m in sorted({doms[i] for i in by_file}):
|
|
files = [i for i, d in enumerate(doms) if d >= m]
|
|
if all(i in by_file for i in files):
|
|
if len({by_file[i] for i in files}) == 1:
|
|
since = (m, len(files))
|
|
break
|
|
if since is None:
|
|
left_out[tag]["varies"].append("/".join(path + (plain,)))
|
|
continue
|
|
kept[tag].append((key, next(iter(by_file.values())), since))
|
|
return pairs, idmls, doms, kept, left_out, order
|
|
|
|
|
|
def since_of(by_file, doms):
|
|
"""The first DOM version from which every file has the same value, and
|
|
the number of those files; None if there is none."""
|
|
for m in sorted({doms[i] for i in by_file}):
|
|
files = [i for i, d in enumerate(doms) if d >= m]
|
|
if all(i in by_file for i in files):
|
|
if len({by_file[i] for i in files}) == 1:
|
|
return (m, len(files))
|
|
return None
|
|
return None
|
|
|
|
|
|
def style_ref(v):
|
|
m = STYLE_REF.search(v)
|
|
return m and m.group(0).lstrip(">") not in ROOT_STYLES
|
|
|
|
|
|
def analyse_preferences(idmls, doms):
|
|
values = defaultdict(dict)
|
|
order = defaultdict(dict)
|
|
tags = OrderedDict()
|
|
for i, (_dom, idml) in enumerate(idmls):
|
|
root = ET.fromstring(zipfile.ZipFile(idml).read("Resources/Preferences.xml"))
|
|
for el in root:
|
|
tags.setdefault(el.tag, len(tags))
|
|
out = {}
|
|
leaves(el, (), out, order[el.tag])
|
|
for k, v in out.items():
|
|
values[(el.tag, k)][i] = v
|
|
kept = defaultdict(list)
|
|
left_out = defaultdict(lambda: defaultdict(list))
|
|
for (tag, key), by_file in values.items():
|
|
path, name = key
|
|
plain = name.split("/", 1)[1] if name.startswith("P/") else name[1:]
|
|
if not path and (tag, plain) in PREF_SKIP:
|
|
left_out[tag]["read from the INDD"].append(plain)
|
|
continue
|
|
since = since_of(by_file, doms)
|
|
if since is None:
|
|
left_out[tag]["varies"].append("/".join(path + (plain,)))
|
|
continue
|
|
if since[1] < MIN_FILES:
|
|
left_out[tag][f"in fewer than {MIN_FILES} files"].append("/".join(path + (plain,)))
|
|
continue
|
|
value = next(iter(by_file.values()))
|
|
if style_ref(value):
|
|
left_out[tag]["refers to a style"].append("/".join(path + (plain,)))
|
|
continue
|
|
kept[tag].append((key, value, since))
|
|
return tags, kept, left_out, order
|
|
|
|
|
|
OBJECT_HEADER = """<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
Values that every object style other than [None] has in every IDML
|
|
exported by InDesign in the corpus. The converter writes them on those
|
|
styles; values it reads from the INDD take precedence. Values in a block
|
|
with MinimumVersion are written only for documents of that InDesign
|
|
version or later. Generated by tools/root_values.py; evidence in
|
|
docs/format/idml-values.md.
|
|
-->
|
|
"""
|
|
|
|
|
|
def analyse_object_styles(idmls, doms):
|
|
"""Values of the object styles other than the root style. A file
|
|
contributes a value for a key when all its such styles have the key
|
|
with one value; the key is kept when every file with such styles from
|
|
some DOM version on has the same value."""
|
|
values = defaultdict(dict)
|
|
order = {}
|
|
files = []
|
|
styles = 0
|
|
for i, (_dom, idml) in enumerate(idmls):
|
|
root = ET.fromstring(zipfile.ZipFile(idml).read("Resources/Styles.xml"))
|
|
found = [el for el in root.iter("ObjectStyle")
|
|
if el.get("Self") != TARGETS["ObjectStyle"]]
|
|
if not found:
|
|
continue
|
|
files.append(i)
|
|
styles += len(found)
|
|
per_key = defaultdict(list)
|
|
for el in found:
|
|
out = {}
|
|
leaves(el, (), out, order)
|
|
for k, v in out.items():
|
|
per_key[k].append(v)
|
|
for k, vs in per_key.items():
|
|
if len(vs) == len(found) and len(set(vs)) == 1:
|
|
values[k][i] = vs[0]
|
|
sub_doms = [doms[i] if i in files else -1 for i in range(len(doms))]
|
|
kept, left_out = [], []
|
|
for key, by_file in values.items():
|
|
path, name = key
|
|
plain = name.split("/", 1)[1] if name.startswith("P/") else name[1:]
|
|
if not path and (plain in SKIP or plain == "BasedOn"):
|
|
continue
|
|
since = None
|
|
for m in sorted({sub_doms[i] for i in by_file}):
|
|
within = [i for i in files if sub_doms[i] >= m]
|
|
if all(i in by_file for i in within):
|
|
if len({by_file[i] for i in within}) == 1:
|
|
since = (m, len(within))
|
|
break
|
|
if since is None or since[1] < MIN_FILES:
|
|
left_out.append("/".join(path + (plain,)))
|
|
continue
|
|
kept.append((key, by_file[next(iter(by_file))], since))
|
|
return files, styles, kept, left_out, order
|
|
|
|
|
|
def write_object_styles(kept, order, first):
|
|
blocks = []
|
|
for since in sorted({s for _, _, (s, _) in kept}):
|
|
gate = "" if since == first else f' MinimumVersion="{since}"'
|
|
items = sorted((it for it in kept if it[2][0] == since), key=lambda it: order[it[0]])
|
|
tree = {"attrs": [], "props": [], "children": OrderedDict()}
|
|
for (path, name), v, _ in items:
|
|
node = tree
|
|
for p in path:
|
|
node = node["children"].setdefault(
|
|
p, {"attrs": [], "props": [], "children": OrderedDict()})
|
|
if name.startswith("@"):
|
|
node["attrs"].append((name[1:], v))
|
|
else:
|
|
node["props"].append(v)
|
|
blocks.append(f"\t<Values{gate}>\n" + element("ObjectStyle", tree, "\t\t") + "\t</Values>\n")
|
|
OBJECT_OUT.write_text(OBJECT_HEADER + "<ObjectStyleValues>\n" + "".join(blocks)
|
|
+ "</ObjectStyleValues>\n")
|
|
|
|
|
|
def write_preferences(tags, kept, order, first):
|
|
blocks = []
|
|
for since in sorted({s for items in kept.values() for _, _, (s, _) in items}):
|
|
gate = "" if since == first else f' MinimumVersion="{since}"'
|
|
s = f"\t<Values{gate}>\n"
|
|
for tag in tags:
|
|
items = sorted((it for it in kept[tag] if it[2][0] == since),
|
|
key=lambda it: order[tag][it[0]])
|
|
if not items:
|
|
continue
|
|
tree = {"attrs": [], "props": [], "children": OrderedDict()}
|
|
for (path, name), v, _ in items:
|
|
node = tree
|
|
for p in path:
|
|
node = node["children"].setdefault(
|
|
p, {"attrs": [], "props": [], "children": OrderedDict()})
|
|
if name.startswith("@"):
|
|
node["attrs"].append((name[1:], v))
|
|
else:
|
|
node["props"].append(v)
|
|
s += element(tag, tree, "\t\t")
|
|
blocks.append(s + "\t</Values>\n")
|
|
PREF_OUT.write_text(PREF_HEADER + "<PreferenceValues>\n" + "".join(blocks) + "</PreferenceValues>\n")
|
|
|
|
|
|
def element(tag, node, indent):
|
|
attrs = "".join(f" {k}={quoteattr(v)}" for k, v in node["attrs"])
|
|
if not node["props"] and not node["children"]:
|
|
return f"{indent}<{tag}{attrs} />\n"
|
|
s = f"{indent}<{tag}{attrs}>\n"
|
|
if node["props"]:
|
|
s += f"{indent}\t<Properties>\n"
|
|
s += "".join(f"{indent}\t\t{p}\n" for p in node["props"])
|
|
s += f"{indent}\t</Properties>\n"
|
|
for t, c in node["children"].items():
|
|
s += element(t, c, indent + "\t")
|
|
return s + f"{indent}</{tag}>\n"
|
|
|
|
|
|
def write(kept, order, first):
|
|
blocks = []
|
|
for since in sorted({s for items in kept.values() for _, _, (s, _) in items}):
|
|
gate = "" if since == first else f' MinimumVersion="{since}"'
|
|
s = f"\t<Values{gate}>\n"
|
|
for tag in TARGETS:
|
|
items = sorted((it for it in kept[tag] if it[2][0] == since),
|
|
key=lambda it: order[tag][it[0]])
|
|
if not items:
|
|
continue
|
|
tree = {"attrs": [], "props": [], "children": OrderedDict()}
|
|
for (path, name), v, _ in items:
|
|
node = tree
|
|
for p in path:
|
|
node = node["children"].setdefault(
|
|
p, {"attrs": [], "props": [], "children": OrderedDict()})
|
|
if name.startswith("@"):
|
|
node["attrs"].append((name[1:], v))
|
|
else:
|
|
node["props"].append(v)
|
|
s += element(tag, tree, "\t\t")
|
|
blocks.append(s + "\t</Values>\n")
|
|
OUT.write_text(HEADER + "<RootStyleValues>\n" + "".join(blocks) + "</RootStyleValues>\n")
|
|
|
|
|
|
def main():
|
|
pairs, idmls, doms, kept, left_out, order = analyse()
|
|
print(f"{pairs} pairs, {len(idmls)} distinct IDML files, DOM versions {sorted(set(doms))}")
|
|
for tag in TARGETS:
|
|
counts = defaultdict(int)
|
|
for _, _, since in kept[tag]:
|
|
counts[since] += 1
|
|
print(f"\n{tag}:")
|
|
for (since, n), k in sorted(counts.items()):
|
|
print(f" {k:4} values, the same in {n} of {n} IDML files with DOM >= {since}")
|
|
for why, names in left_out[tag].items():
|
|
print(f" {len(names):4} left out ({why}): {', '.join(sorted(names))}")
|
|
pinned_pairs, pinned = corpus_idmls(SOURCE)
|
|
pinned_doms = [d for d, _ in pinned]
|
|
print(f"\nPreferences and object styles: {pinned_pairs} pairs under {SOURCE}, "
|
|
f"{len(pinned)} distinct IDML files")
|
|
tags, pkept, pleft, porder = analyse_preferences(pinned, pinned_doms)
|
|
print("\nPreferences:")
|
|
total = defaultdict(int)
|
|
for tag in tags:
|
|
for _, _, since in pkept[tag]:
|
|
total[since] += 1
|
|
for (since, n), k in sorted(total.items()):
|
|
print(f" {k:4} values, the same in {n} of {n} IDML files with DOM >= {since}")
|
|
for tag in tags:
|
|
counts = {why: len(names) for why, names in pleft[tag].items()}
|
|
print(f" {tag}: {len(pkept[tag])} kept, left out {counts}")
|
|
if pleft[tag].get("refers to a style") or pleft[tag].get("read from the INDD"):
|
|
print(f" {pleft[tag].get('refers to a style', [])} {pleft[tag].get('read from the INDD', [])}")
|
|
files, styles, okept, oleft, oorder = analyse_object_styles(pinned, pinned_doms)
|
|
print(f"\nObject styles other than [None]: {styles} in {len(files)} IDML files")
|
|
counts = defaultdict(int)
|
|
for _, _, since in okept:
|
|
counts[since] += 1
|
|
for (since, n), k in sorted(counts.items()):
|
|
print(f" {k:4} values, the same in {n} of {n} IDML files with DOM >= {since}")
|
|
print(f" {len(oleft)} left out (vary, not in every style, or in fewer than {MIN_FILES} files):")
|
|
print(f" {', '.join(sorted(oleft))}")
|
|
if "--write" in sys.argv:
|
|
write_object_styles(okept, oorder, min(pinned_doms))
|
|
print(f"wrote {OBJECT_OUT.relative_to(ROOT)}")
|
|
write(kept, order, min(doms))
|
|
print(f"\nwrote {OUT.relative_to(ROOT)}")
|
|
write_preferences(tags, pkept, porder, min(pinned_doms))
|
|
print(f"wrote {PREF_OUT.relative_to(ROOT)}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|