Files
indd-utils/tools/root_values.py
T
aiongg c9528b10ff feat(tools): list reproduced values per key and dump whole chunks
compare.py writes values.tsv and values-all.tsv with every key's values and
how many are reproduced, so a change can be checked for keys that lose
reproduced values. indd dump --full prints every byte of each chunk.
root_values.py reads the current inventory layout and keeps to the pairs
its generated files were made from.
2026-10-07 09:39:02 +00:00

417 lines
17 KiB
Python

#!/usr/bin/env python3
"""Find the values that every corpus IDML has on its root styles, on its
other object styles and in its preferences.
Reads Resources/Styles.xml and Resources/Preferences.xml of every IDML in
the corpus pairs (distinct files only) and collects each attribute,
Properties child and child element attribute of the root styles and of
the top-level preference elements. A value is kept when it is the same in
every IDML from some DOM version on and present in all of them; the
earliest such version is its minimum version. Attributes the converter
reads from the INDD (the TEXT_ATTRS table in src/idml/mod.rs) are left
out for the paragraph and character root styles, and the document setup
it reads is left out of the preferences. Preference values that refer to
a style other than a root style are left out, since the package need not
have that style.
Object styles other than the root style `[None]` are collected the same
way, all styles of a file together: a value is kept when every such style
in every IDML from some DOM version on has it, with the same value.
Usage: python3 -I tools/root_values.py [--write]
Prints the evidence counts. With --write, regenerates
src/idml/root_values.xml, src/idml/object_style_values.xml and
src/idml/preference_values.xml. See docs/format/idml-values.md.
"""
import hashlib
import re
import sys
import xml.etree.ElementTree as ET
import zipfile
from collections import OrderedDict, defaultdict
from pathlib import Path
from xml.sax.saxutils import quoteattr
ROOT = Path(__file__).resolve().parent.parent
OUT = ROOT / "src" / "idml" / "root_values.xml"
PREF_OUT = ROOT / "src" / "idml" / "preference_values.xml"
OBJECT_OUT = ROOT / "src" / "idml" / "object_style_values.xml"
# The generated files were made from the corpus pairs under this
# directory, the whole corpus at the time. Later pairs are evidence for
# tools/element_values.py; regenerating these files from them would drop
# values that the larger corpus shows to vary (docs/format/idml-values.md).
SOURCE = "third-party/"
# Preference attributes the converter reads from the INDD.
PREF_SKIP = {
("DocumentPreference", "PageHeight"), ("DocumentPreference", "PageWidth"),
("DocumentPreference", "FacingPages"), ("DocumentPreference", "Intent"),
("DocumentPreference", "DocumentBleedTopOffset"),
("DocumentPreference", "DocumentBleedBottomOffset"),
("DocumentPreference", "DocumentBleedInsideOrLeftOffset"),
("DocumentPreference", "DocumentBleedOutsideOrRightOffset"),
("ViewPreference", "RulerOrigin"),
}
# Styles a preference value may refer to: those every package has.
ROOT_STYLES = {
"ParagraphStyle/$ID/[No paragraph style]",
"CharacterStyle/$ID/[No character style]",
"ObjectStyle/$ID/[None]",
}
# A preference value first seen in a later version is kept only when at
# least this many files show it.
MIN_FILES = 10
STYLE_REF = re.compile(r"(?:>|^)(Paragraph|Character|Object|Cell|Table)Style/[^<]*")
PREF_HEADER = """<?xml version="1.0" encoding="UTF-8"?>
<!--
Values that every IDML exported by InDesign in the corpus has in
Resources/Preferences.xml. The converter writes them; values it reads
from the INDD take precedence. Values in a block with MinimumVersion are
written only for documents of that InDesign version or later. Generated
by tools/root_values.py; evidence in docs/format/idml-values.md.
-->
"""
TARGETS = OrderedDict([
("ParagraphStyle", "ParagraphStyle/$ID/[No paragraph style]"),
("CharacterStyle", "CharacterStyle/$ID/[No character style]"),
("ObjectStyle", "ObjectStyle/$ID/[None]"),
("CellStyle", "CellStyle/$ID/[None]"),
("TableStyle", "TableStyle/$ID/[No table style]"),
])
# Written by the converter itself.
SKIP = {"Self", "Name"}
HEADER = """<?xml version="1.0" encoding="UTF-8"?>
<!--
Values that every IDML exported by InDesign in the corpus has on the root
styles. The converter writes them on its root styles; values it reads from
the INDD take precedence. Values in a block with MinimumVersion are written
only for documents of that InDesign version or later. Generated by
tools/root_values.py; evidence in docs/format/idml-values.md.
-->
"""
def decoded_names():
src = (ROOT / "src" / "idml" / "mod.rs").read_text()
table = src[src.index("const TEXT_ATTRS"):src.index("/// An attribute written as a")]
return set(re.findall(r'0x[0-9A-F]+,\s*"(\w+)"', table))
def corpus_idmls():
lines = (ROOT / "corpus" / "inventory.tsv").read_text().splitlines()
cols = lines[0].split("\t")
seen, out, pairs = set(), [], 0
for line in lines[1:]:
row = dict(zip(cols, line.split("\t")))
path, dom = row["path"], row["idml_dom"]
if not dom or not dom[0].isdigit() or not path.startswith(SOURCE):
continue
pairs += 1
idml = ROOT / "corpus" / (row.get("idml") or str(Path(path).with_suffix(".idml")))
digest = hashlib.md5(idml.read_bytes()).hexdigest()
if digest not in seen:
seen.add(digest)
out.append((int(dom.split(".")[0]), idml))
return pairs, out
def leaves(el, path, out, order):
"""Flatten an element: (path, '@attr') and (path, 'P/tag') keys."""
for k, v in el.attrib.items():
out[(path, "@" + k)] = v
order.setdefault((path, "@" + k), len(order))
for c in el:
if c.tag == "Properties":
for p in c:
key = (path, "P/" + p.tag)
out[key] = re.sub(r">\s+<", "><", ET.tostring(p, encoding="unicode").strip())
order.setdefault(key, len(order))
else:
order.setdefault((path + (c.tag,), ""), len(order))
leaves(c, path + (c.tag,), out, order)
def analyse():
decoded = decoded_names()
pairs, idmls = corpus_idmls()
doms = [d for d, _ in idmls]
values = defaultdict(dict)
order = defaultdict(dict)
for i, (_dom, idml) in enumerate(idmls):
root = ET.fromstring(zipfile.ZipFile(idml).read("Resources/Styles.xml"))
for el in root.iter():
if el.tag in TARGETS and TARGETS[el.tag] == el.get("Self"):
out = {}
leaves(el, (), out, order[el.tag])
for k, v in out.items():
values[(el.tag, k)][i] = v
kept = defaultdict(list)
left_out = defaultdict(lambda: defaultdict(list))
for (tag, key), by_file in values.items():
path, name = key
if not path and name[1:] in SKIP:
continue
plain = name.split("/", 1)[1] if name.startswith("P/") else name[1:]
if not path and tag in ("ParagraphStyle", "CharacterStyle") and plain in decoded:
left_out[tag]["read from the INDD"].append(plain)
continue
since = None
for m in sorted({doms[i] for i in by_file}):
files = [i for i, d in enumerate(doms) if d >= m]
if all(i in by_file for i in files):
if len({by_file[i] for i in files}) == 1:
since = (m, len(files))
break
if since is None:
left_out[tag]["varies"].append("/".join(path + (plain,)))
continue
kept[tag].append((key, next(iter(by_file.values())), since))
return pairs, idmls, doms, kept, left_out, order
def since_of(by_file, doms):
"""The first DOM version from which every file has the same value, and
the number of those files; None if there is none."""
for m in sorted({doms[i] for i in by_file}):
files = [i for i, d in enumerate(doms) if d >= m]
if all(i in by_file for i in files):
if len({by_file[i] for i in files}) == 1:
return (m, len(files))
return None
return None
def style_ref(v):
m = STYLE_REF.search(v)
return m and m.group(0).lstrip(">") not in ROOT_STYLES
def analyse_preferences(idmls, doms):
values = defaultdict(dict)
order = defaultdict(dict)
tags = OrderedDict()
for i, (_dom, idml) in enumerate(idmls):
root = ET.fromstring(zipfile.ZipFile(idml).read("Resources/Preferences.xml"))
for el in root:
tags.setdefault(el.tag, len(tags))
out = {}
leaves(el, (), out, order[el.tag])
for k, v in out.items():
values[(el.tag, k)][i] = v
kept = defaultdict(list)
left_out = defaultdict(lambda: defaultdict(list))
for (tag, key), by_file in values.items():
path, name = key
plain = name.split("/", 1)[1] if name.startswith("P/") else name[1:]
if not path and (tag, plain) in PREF_SKIP:
left_out[tag]["read from the INDD"].append(plain)
continue
since = since_of(by_file, doms)
if since is None:
left_out[tag]["varies"].append("/".join(path + (plain,)))
continue
if since[1] < MIN_FILES:
left_out[tag][f"in fewer than {MIN_FILES} files"].append("/".join(path + (plain,)))
continue
value = next(iter(by_file.values()))
if style_ref(value):
left_out[tag]["refers to a style"].append("/".join(path + (plain,)))
continue
kept[tag].append((key, value, since))
return tags, kept, left_out, order
OBJECT_HEADER = """<?xml version="1.0" encoding="UTF-8"?>
<!--
Values that every object style other than [None] has in every IDML
exported by InDesign in the corpus. The converter writes them on those
styles; values it reads from the INDD take precedence. Values in a block
with MinimumVersion are written only for documents of that InDesign
version or later. Generated by tools/root_values.py; evidence in
docs/format/idml-values.md.
-->
"""
def analyse_object_styles(idmls, doms):
"""Values of the object styles other than the root style. A file
contributes a value for a key when all its such styles have the key
with one value; the key is kept when every file with such styles from
some DOM version on has the same value."""
values = defaultdict(dict)
order = {}
files = []
styles = 0
for i, (_dom, idml) in enumerate(idmls):
root = ET.fromstring(zipfile.ZipFile(idml).read("Resources/Styles.xml"))
found = [el for el in root.iter("ObjectStyle")
if el.get("Self") != TARGETS["ObjectStyle"]]
if not found:
continue
files.append(i)
styles += len(found)
per_key = defaultdict(list)
for el in found:
out = {}
leaves(el, (), out, order)
for k, v in out.items():
per_key[k].append(v)
for k, vs in per_key.items():
if len(vs) == len(found) and len(set(vs)) == 1:
values[k][i] = vs[0]
sub_doms = [doms[i] if i in files else -1 for i in range(len(doms))]
kept, left_out = [], []
for key, by_file in values.items():
path, name = key
plain = name.split("/", 1)[1] if name.startswith("P/") else name[1:]
if not path and (plain in SKIP or plain == "BasedOn"):
continue
since = None
for m in sorted({sub_doms[i] for i in by_file}):
within = [i for i in files if sub_doms[i] >= m]
if all(i in by_file for i in within):
if len({by_file[i] for i in within}) == 1:
since = (m, len(within))
break
if since is None or since[1] < MIN_FILES:
left_out.append("/".join(path + (plain,)))
continue
kept.append((key, by_file[next(iter(by_file))], since))
return files, styles, kept, left_out, order
def write_object_styles(kept, order, first):
blocks = []
for since in sorted({s for _, _, (s, _) in kept}):
gate = "" if since == first else f' MinimumVersion="{since}"'
items = sorted((it for it in kept if it[2][0] == since), key=lambda it: order[it[0]])
tree = {"attrs": [], "props": [], "children": OrderedDict()}
for (path, name), v, _ in items:
node = tree
for p in path:
node = node["children"].setdefault(
p, {"attrs": [], "props": [], "children": OrderedDict()})
if name.startswith("@"):
node["attrs"].append((name[1:], v))
else:
node["props"].append(v)
blocks.append(f"\t<Values{gate}>\n" + element("ObjectStyle", tree, "\t\t") + "\t</Values>\n")
OBJECT_OUT.write_text(OBJECT_HEADER + "<ObjectStyleValues>\n" + "".join(blocks)
+ "</ObjectStyleValues>\n")
def write_preferences(tags, kept, order, first):
blocks = []
for since in sorted({s for items in kept.values() for _, _, (s, _) in items}):
gate = "" if since == first else f' MinimumVersion="{since}"'
s = f"\t<Values{gate}>\n"
for tag in tags:
items = sorted((it for it in kept[tag] if it[2][0] == since),
key=lambda it: order[tag][it[0]])
if not items:
continue
tree = {"attrs": [], "props": [], "children": OrderedDict()}
for (path, name), v, _ in items:
node = tree
for p in path:
node = node["children"].setdefault(
p, {"attrs": [], "props": [], "children": OrderedDict()})
if name.startswith("@"):
node["attrs"].append((name[1:], v))
else:
node["props"].append(v)
s += element(tag, tree, "\t\t")
blocks.append(s + "\t</Values>\n")
PREF_OUT.write_text(PREF_HEADER + "<PreferenceValues>\n" + "".join(blocks) + "</PreferenceValues>\n")
def element(tag, node, indent):
attrs = "".join(f" {k}={quoteattr(v)}" for k, v in node["attrs"])
if not node["props"] and not node["children"]:
return f"{indent}<{tag}{attrs} />\n"
s = f"{indent}<{tag}{attrs}>\n"
if node["props"]:
s += f"{indent}\t<Properties>\n"
s += "".join(f"{indent}\t\t{p}\n" for p in node["props"])
s += f"{indent}\t</Properties>\n"
for t, c in node["children"].items():
s += element(t, c, indent + "\t")
return s + f"{indent}</{tag}>\n"
def write(kept, order, first):
blocks = []
for since in sorted({s for items in kept.values() for _, _, (s, _) in items}):
gate = "" if since == first else f' MinimumVersion="{since}"'
s = f"\t<Values{gate}>\n"
for tag in TARGETS:
items = sorted((it for it in kept[tag] if it[2][0] == since),
key=lambda it: order[tag][it[0]])
if not items:
continue
tree = {"attrs": [], "props": [], "children": OrderedDict()}
for (path, name), v, _ in items:
node = tree
for p in path:
node = node["children"].setdefault(
p, {"attrs": [], "props": [], "children": OrderedDict()})
if name.startswith("@"):
node["attrs"].append((name[1:], v))
else:
node["props"].append(v)
s += element(tag, tree, "\t\t")
blocks.append(s + "\t</Values>\n")
OUT.write_text(HEADER + "<RootStyleValues>\n" + "".join(blocks) + "</RootStyleValues>\n")
def main():
pairs, idmls, doms, kept, left_out, order = analyse()
print(f"{pairs} pairs, {len(idmls)} distinct IDML files, DOM versions {sorted(set(doms))}")
for tag in TARGETS:
counts = defaultdict(int)
for _, _, since in kept[tag]:
counts[since] += 1
print(f"\n{tag}:")
for (since, n), k in sorted(counts.items()):
print(f" {k:4} values, the same in {n} of {n} IDML files with DOM >= {since}")
for why, names in left_out[tag].items():
print(f" {len(names):4} left out ({why}): {', '.join(sorted(names))}")
tags, pkept, pleft, porder = analyse_preferences(idmls, doms)
print("\nPreferences:")
total = defaultdict(int)
for tag in tags:
for _, _, since in pkept[tag]:
total[since] += 1
for (since, n), k in sorted(total.items()):
print(f" {k:4} values, the same in {n} of {n} IDML files with DOM >= {since}")
for tag in tags:
counts = {why: len(names) for why, names in pleft[tag].items()}
print(f" {tag}: {len(pkept[tag])} kept, left out {counts}")
if pleft[tag].get("refers to a style") or pleft[tag].get("read from the INDD"):
print(f" {pleft[tag].get('refers to a style', [])} {pleft[tag].get('read from the INDD', [])}")
files, styles, okept, oleft, oorder = analyse_object_styles(idmls, doms)
print(f"\nObject styles other than [None]: {styles} in {len(files)} IDML files")
counts = defaultdict(int)
for _, _, since in okept:
counts[since] += 1
for (since, n), k in sorted(counts.items()):
print(f" {k:4} values, the same in {n} of {n} IDML files with DOM >= {since}")
print(f" {len(oleft)} left out (vary, not in every style, or in fewer than {MIN_FILES} files):")
print(f" {', '.join(sorted(oleft))}")
if "--write" in sys.argv:
write_object_styles(okept, oorder, min(doms))
print(f"wrote {OBJECT_OUT.relative_to(ROOT)}")
write(kept, order, min(doms))
print(f"\nwrote {OUT.relative_to(ROOT)}")
write_preferences(tags, pkept, porder, min(doms))
print(f"wrote {PREF_OUT.relative_to(ROOT)}")
if __name__ == "__main__":
main()