mirror of
https://github.com/storytold/indd-utils.git
synced 2026-11-19 16:50:48 +01:00
compare.py writes values.tsv and values-all.tsv with every key's values and how many are reproduced, so a change can be checked for keys that lose reproduced values. indd dump --full prints every byte of each chunk. root_values.py reads the current inventory layout and keeps to the pairs its generated files were made from.
417 lines
17 KiB
Python
417 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""Find the values that every corpus IDML has on its root styles, on its
|
|
other object styles and in its preferences.
|
|
|
|
Reads Resources/Styles.xml and Resources/Preferences.xml of every IDML in
|
|
the corpus pairs (distinct files only) and collects each attribute,
|
|
Properties child and child element attribute of the root styles and of
|
|
the top-level preference elements. A value is kept when it is the same in
|
|
every IDML from some DOM version on and present in all of them; the
|
|
earliest such version is its minimum version. Attributes the converter
|
|
reads from the INDD (the TEXT_ATTRS table in src/idml/mod.rs) are left
|
|
out for the paragraph and character root styles, and the document setup
|
|
it reads is left out of the preferences. Preference values that refer to
|
|
a style other than a root style are left out, since the package need not
|
|
have that style.
|
|
|
|
Object styles other than the root style `[None]` are collected the same
|
|
way, all styles of a file together: a value is kept when every such style
|
|
in every IDML from some DOM version on has it, with the same value.
|
|
|
|
Usage: python3 -I tools/root_values.py [--write]
|
|
Prints the evidence counts. With --write, regenerates
|
|
src/idml/root_values.xml, src/idml/object_style_values.xml and
|
|
src/idml/preference_values.xml. See docs/format/idml-values.md.
|
|
"""
|
|
|
|
import hashlib
|
|
import re
|
|
import sys
|
|
import xml.etree.ElementTree as ET
|
|
import zipfile
|
|
from collections import OrderedDict, defaultdict
|
|
from pathlib import Path
|
|
from xml.sax.saxutils import quoteattr
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
OUT = ROOT / "src" / "idml" / "root_values.xml"
|
|
PREF_OUT = ROOT / "src" / "idml" / "preference_values.xml"
|
|
OBJECT_OUT = ROOT / "src" / "idml" / "object_style_values.xml"
|
|
|
|
# The generated files were made from the corpus pairs under this
|
|
# directory, the whole corpus at the time. Later pairs are evidence for
|
|
# tools/element_values.py; regenerating these files from them would drop
|
|
# values that the larger corpus shows to vary (docs/format/idml-values.md).
|
|
SOURCE = "third-party/"
|
|
|
|
# Preference attributes the converter reads from the INDD.
|
|
PREF_SKIP = {
|
|
("DocumentPreference", "PageHeight"), ("DocumentPreference", "PageWidth"),
|
|
("DocumentPreference", "FacingPages"), ("DocumentPreference", "Intent"),
|
|
("DocumentPreference", "DocumentBleedTopOffset"),
|
|
("DocumentPreference", "DocumentBleedBottomOffset"),
|
|
("DocumentPreference", "DocumentBleedInsideOrLeftOffset"),
|
|
("DocumentPreference", "DocumentBleedOutsideOrRightOffset"),
|
|
("ViewPreference", "RulerOrigin"),
|
|
}
|
|
# Styles a preference value may refer to: those every package has.
|
|
ROOT_STYLES = {
|
|
"ParagraphStyle/$ID/[No paragraph style]",
|
|
"CharacterStyle/$ID/[No character style]",
|
|
"ObjectStyle/$ID/[None]",
|
|
}
|
|
# A preference value first seen in a later version is kept only when at
|
|
# least this many files show it.
|
|
MIN_FILES = 10
|
|
STYLE_REF = re.compile(r"(?:>|^)(Paragraph|Character|Object|Cell|Table)Style/[^<]*")
|
|
|
|
PREF_HEADER = """<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
Values that every IDML exported by InDesign in the corpus has in
|
|
Resources/Preferences.xml. The converter writes them; values it reads
|
|
from the INDD take precedence. Values in a block with MinimumVersion are
|
|
written only for documents of that InDesign version or later. Generated
|
|
by tools/root_values.py; evidence in docs/format/idml-values.md.
|
|
-->
|
|
"""
|
|
|
|
TARGETS = OrderedDict([
|
|
("ParagraphStyle", "ParagraphStyle/$ID/[No paragraph style]"),
|
|
("CharacterStyle", "CharacterStyle/$ID/[No character style]"),
|
|
("ObjectStyle", "ObjectStyle/$ID/[None]"),
|
|
("CellStyle", "CellStyle/$ID/[None]"),
|
|
("TableStyle", "TableStyle/$ID/[No table style]"),
|
|
])
|
|
# Written by the converter itself.
|
|
SKIP = {"Self", "Name"}
|
|
|
|
HEADER = """<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
Values that every IDML exported by InDesign in the corpus has on the root
|
|
styles. The converter writes them on its root styles; values it reads from
|
|
the INDD take precedence. Values in a block with MinimumVersion are written
|
|
only for documents of that InDesign version or later. Generated by
|
|
tools/root_values.py; evidence in docs/format/idml-values.md.
|
|
-->
|
|
"""
|
|
|
|
|
|
def decoded_names():
|
|
src = (ROOT / "src" / "idml" / "mod.rs").read_text()
|
|
table = src[src.index("const TEXT_ATTRS"):src.index("/// An attribute written as a")]
|
|
return set(re.findall(r'0x[0-9A-F]+,\s*"(\w+)"', table))
|
|
|
|
|
|
def corpus_idmls():
|
|
lines = (ROOT / "corpus" / "inventory.tsv").read_text().splitlines()
|
|
cols = lines[0].split("\t")
|
|
seen, out, pairs = set(), [], 0
|
|
for line in lines[1:]:
|
|
row = dict(zip(cols, line.split("\t")))
|
|
path, dom = row["path"], row["idml_dom"]
|
|
if not dom or not dom[0].isdigit() or not path.startswith(SOURCE):
|
|
continue
|
|
pairs += 1
|
|
idml = ROOT / "corpus" / (row.get("idml") or str(Path(path).with_suffix(".idml")))
|
|
digest = hashlib.md5(idml.read_bytes()).hexdigest()
|
|
if digest not in seen:
|
|
seen.add(digest)
|
|
out.append((int(dom.split(".")[0]), idml))
|
|
return pairs, out
|
|
|
|
|
|
def leaves(el, path, out, order):
|
|
"""Flatten an element: (path, '@attr') and (path, 'P/tag') keys."""
|
|
for k, v in el.attrib.items():
|
|
out[(path, "@" + k)] = v
|
|
order.setdefault((path, "@" + k), len(order))
|
|
for c in el:
|
|
if c.tag == "Properties":
|
|
for p in c:
|
|
key = (path, "P/" + p.tag)
|
|
out[key] = re.sub(r">\s+<", "><", ET.tostring(p, encoding="unicode").strip())
|
|
order.setdefault(key, len(order))
|
|
else:
|
|
order.setdefault((path + (c.tag,), ""), len(order))
|
|
leaves(c, path + (c.tag,), out, order)
|
|
|
|
|
|
def analyse():
|
|
decoded = decoded_names()
|
|
pairs, idmls = corpus_idmls()
|
|
doms = [d for d, _ in idmls]
|
|
values = defaultdict(dict)
|
|
order = defaultdict(dict)
|
|
for i, (_dom, idml) in enumerate(idmls):
|
|
root = ET.fromstring(zipfile.ZipFile(idml).read("Resources/Styles.xml"))
|
|
for el in root.iter():
|
|
if el.tag in TARGETS and TARGETS[el.tag] == el.get("Self"):
|
|
out = {}
|
|
leaves(el, (), out, order[el.tag])
|
|
for k, v in out.items():
|
|
values[(el.tag, k)][i] = v
|
|
kept = defaultdict(list)
|
|
left_out = defaultdict(lambda: defaultdict(list))
|
|
for (tag, key), by_file in values.items():
|
|
path, name = key
|
|
if not path and name[1:] in SKIP:
|
|
continue
|
|
plain = name.split("/", 1)[1] if name.startswith("P/") else name[1:]
|
|
if not path and tag in ("ParagraphStyle", "CharacterStyle") and plain in decoded:
|
|
left_out[tag]["read from the INDD"].append(plain)
|
|
continue
|
|
since = None
|
|
for m in sorted({doms[i] for i in by_file}):
|
|
files = [i for i, d in enumerate(doms) if d >= m]
|
|
if all(i in by_file for i in files):
|
|
if len({by_file[i] for i in files}) == 1:
|
|
since = (m, len(files))
|
|
break
|
|
if since is None:
|
|
left_out[tag]["varies"].append("/".join(path + (plain,)))
|
|
continue
|
|
kept[tag].append((key, next(iter(by_file.values())), since))
|
|
return pairs, idmls, doms, kept, left_out, order
|
|
|
|
|
|
def since_of(by_file, doms):
|
|
"""The first DOM version from which every file has the same value, and
|
|
the number of those files; None if there is none."""
|
|
for m in sorted({doms[i] for i in by_file}):
|
|
files = [i for i, d in enumerate(doms) if d >= m]
|
|
if all(i in by_file for i in files):
|
|
if len({by_file[i] for i in files}) == 1:
|
|
return (m, len(files))
|
|
return None
|
|
return None
|
|
|
|
|
|
def style_ref(v):
|
|
m = STYLE_REF.search(v)
|
|
return m and m.group(0).lstrip(">") not in ROOT_STYLES
|
|
|
|
|
|
def analyse_preferences(idmls, doms):
|
|
values = defaultdict(dict)
|
|
order = defaultdict(dict)
|
|
tags = OrderedDict()
|
|
for i, (_dom, idml) in enumerate(idmls):
|
|
root = ET.fromstring(zipfile.ZipFile(idml).read("Resources/Preferences.xml"))
|
|
for el in root:
|
|
tags.setdefault(el.tag, len(tags))
|
|
out = {}
|
|
leaves(el, (), out, order[el.tag])
|
|
for k, v in out.items():
|
|
values[(el.tag, k)][i] = v
|
|
kept = defaultdict(list)
|
|
left_out = defaultdict(lambda: defaultdict(list))
|
|
for (tag, key), by_file in values.items():
|
|
path, name = key
|
|
plain = name.split("/", 1)[1] if name.startswith("P/") else name[1:]
|
|
if not path and (tag, plain) in PREF_SKIP:
|
|
left_out[tag]["read from the INDD"].append(plain)
|
|
continue
|
|
since = since_of(by_file, doms)
|
|
if since is None:
|
|
left_out[tag]["varies"].append("/".join(path + (plain,)))
|
|
continue
|
|
if since[1] < MIN_FILES:
|
|
left_out[tag][f"in fewer than {MIN_FILES} files"].append("/".join(path + (plain,)))
|
|
continue
|
|
value = next(iter(by_file.values()))
|
|
if style_ref(value):
|
|
left_out[tag]["refers to a style"].append("/".join(path + (plain,)))
|
|
continue
|
|
kept[tag].append((key, value, since))
|
|
return tags, kept, left_out, order
|
|
|
|
|
|
OBJECT_HEADER = """<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
Values that every object style other than [None] has in every IDML
|
|
exported by InDesign in the corpus. The converter writes them on those
|
|
styles; values it reads from the INDD take precedence. Values in a block
|
|
with MinimumVersion are written only for documents of that InDesign
|
|
version or later. Generated by tools/root_values.py; evidence in
|
|
docs/format/idml-values.md.
|
|
-->
|
|
"""
|
|
|
|
|
|
def analyse_object_styles(idmls, doms):
|
|
"""Values of the object styles other than the root style. A file
|
|
contributes a value for a key when all its such styles have the key
|
|
with one value; the key is kept when every file with such styles from
|
|
some DOM version on has the same value."""
|
|
values = defaultdict(dict)
|
|
order = {}
|
|
files = []
|
|
styles = 0
|
|
for i, (_dom, idml) in enumerate(idmls):
|
|
root = ET.fromstring(zipfile.ZipFile(idml).read("Resources/Styles.xml"))
|
|
found = [el for el in root.iter("ObjectStyle")
|
|
if el.get("Self") != TARGETS["ObjectStyle"]]
|
|
if not found:
|
|
continue
|
|
files.append(i)
|
|
styles += len(found)
|
|
per_key = defaultdict(list)
|
|
for el in found:
|
|
out = {}
|
|
leaves(el, (), out, order)
|
|
for k, v in out.items():
|
|
per_key[k].append(v)
|
|
for k, vs in per_key.items():
|
|
if len(vs) == len(found) and len(set(vs)) == 1:
|
|
values[k][i] = vs[0]
|
|
sub_doms = [doms[i] if i in files else -1 for i in range(len(doms))]
|
|
kept, left_out = [], []
|
|
for key, by_file in values.items():
|
|
path, name = key
|
|
plain = name.split("/", 1)[1] if name.startswith("P/") else name[1:]
|
|
if not path and (plain in SKIP or plain == "BasedOn"):
|
|
continue
|
|
since = None
|
|
for m in sorted({sub_doms[i] for i in by_file}):
|
|
within = [i for i in files if sub_doms[i] >= m]
|
|
if all(i in by_file for i in within):
|
|
if len({by_file[i] for i in within}) == 1:
|
|
since = (m, len(within))
|
|
break
|
|
if since is None or since[1] < MIN_FILES:
|
|
left_out.append("/".join(path + (plain,)))
|
|
continue
|
|
kept.append((key, by_file[next(iter(by_file))], since))
|
|
return files, styles, kept, left_out, order
|
|
|
|
|
|
def write_object_styles(kept, order, first):
|
|
blocks = []
|
|
for since in sorted({s for _, _, (s, _) in kept}):
|
|
gate = "" if since == first else f' MinimumVersion="{since}"'
|
|
items = sorted((it for it in kept if it[2][0] == since), key=lambda it: order[it[0]])
|
|
tree = {"attrs": [], "props": [], "children": OrderedDict()}
|
|
for (path, name), v, _ in items:
|
|
node = tree
|
|
for p in path:
|
|
node = node["children"].setdefault(
|
|
p, {"attrs": [], "props": [], "children": OrderedDict()})
|
|
if name.startswith("@"):
|
|
node["attrs"].append((name[1:], v))
|
|
else:
|
|
node["props"].append(v)
|
|
blocks.append(f"\t<Values{gate}>\n" + element("ObjectStyle", tree, "\t\t") + "\t</Values>\n")
|
|
OBJECT_OUT.write_text(OBJECT_HEADER + "<ObjectStyleValues>\n" + "".join(blocks)
|
|
+ "</ObjectStyleValues>\n")
|
|
|
|
|
|
def write_preferences(tags, kept, order, first):
|
|
blocks = []
|
|
for since in sorted({s for items in kept.values() for _, _, (s, _) in items}):
|
|
gate = "" if since == first else f' MinimumVersion="{since}"'
|
|
s = f"\t<Values{gate}>\n"
|
|
for tag in tags:
|
|
items = sorted((it for it in kept[tag] if it[2][0] == since),
|
|
key=lambda it: order[tag][it[0]])
|
|
if not items:
|
|
continue
|
|
tree = {"attrs": [], "props": [], "children": OrderedDict()}
|
|
for (path, name), v, _ in items:
|
|
node = tree
|
|
for p in path:
|
|
node = node["children"].setdefault(
|
|
p, {"attrs": [], "props": [], "children": OrderedDict()})
|
|
if name.startswith("@"):
|
|
node["attrs"].append((name[1:], v))
|
|
else:
|
|
node["props"].append(v)
|
|
s += element(tag, tree, "\t\t")
|
|
blocks.append(s + "\t</Values>\n")
|
|
PREF_OUT.write_text(PREF_HEADER + "<PreferenceValues>\n" + "".join(blocks) + "</PreferenceValues>\n")
|
|
|
|
|
|
def element(tag, node, indent):
|
|
attrs = "".join(f" {k}={quoteattr(v)}" for k, v in node["attrs"])
|
|
if not node["props"] and not node["children"]:
|
|
return f"{indent}<{tag}{attrs} />\n"
|
|
s = f"{indent}<{tag}{attrs}>\n"
|
|
if node["props"]:
|
|
s += f"{indent}\t<Properties>\n"
|
|
s += "".join(f"{indent}\t\t{p}\n" for p in node["props"])
|
|
s += f"{indent}\t</Properties>\n"
|
|
for t, c in node["children"].items():
|
|
s += element(t, c, indent + "\t")
|
|
return s + f"{indent}</{tag}>\n"
|
|
|
|
|
|
def write(kept, order, first):
|
|
blocks = []
|
|
for since in sorted({s for items in kept.values() for _, _, (s, _) in items}):
|
|
gate = "" if since == first else f' MinimumVersion="{since}"'
|
|
s = f"\t<Values{gate}>\n"
|
|
for tag in TARGETS:
|
|
items = sorted((it for it in kept[tag] if it[2][0] == since),
|
|
key=lambda it: order[tag][it[0]])
|
|
if not items:
|
|
continue
|
|
tree = {"attrs": [], "props": [], "children": OrderedDict()}
|
|
for (path, name), v, _ in items:
|
|
node = tree
|
|
for p in path:
|
|
node = node["children"].setdefault(
|
|
p, {"attrs": [], "props": [], "children": OrderedDict()})
|
|
if name.startswith("@"):
|
|
node["attrs"].append((name[1:], v))
|
|
else:
|
|
node["props"].append(v)
|
|
s += element(tag, tree, "\t\t")
|
|
blocks.append(s + "\t</Values>\n")
|
|
OUT.write_text(HEADER + "<RootStyleValues>\n" + "".join(blocks) + "</RootStyleValues>\n")
|
|
|
|
|
|
def main():
|
|
pairs, idmls, doms, kept, left_out, order = analyse()
|
|
print(f"{pairs} pairs, {len(idmls)} distinct IDML files, DOM versions {sorted(set(doms))}")
|
|
for tag in TARGETS:
|
|
counts = defaultdict(int)
|
|
for _, _, since in kept[tag]:
|
|
counts[since] += 1
|
|
print(f"\n{tag}:")
|
|
for (since, n), k in sorted(counts.items()):
|
|
print(f" {k:4} values, the same in {n} of {n} IDML files with DOM >= {since}")
|
|
for why, names in left_out[tag].items():
|
|
print(f" {len(names):4} left out ({why}): {', '.join(sorted(names))}")
|
|
tags, pkept, pleft, porder = analyse_preferences(idmls, doms)
|
|
print("\nPreferences:")
|
|
total = defaultdict(int)
|
|
for tag in tags:
|
|
for _, _, since in pkept[tag]:
|
|
total[since] += 1
|
|
for (since, n), k in sorted(total.items()):
|
|
print(f" {k:4} values, the same in {n} of {n} IDML files with DOM >= {since}")
|
|
for tag in tags:
|
|
counts = {why: len(names) for why, names in pleft[tag].items()}
|
|
print(f" {tag}: {len(pkept[tag])} kept, left out {counts}")
|
|
if pleft[tag].get("refers to a style") or pleft[tag].get("read from the INDD"):
|
|
print(f" {pleft[tag].get('refers to a style', [])} {pleft[tag].get('read from the INDD', [])}")
|
|
files, styles, okept, oleft, oorder = analyse_object_styles(idmls, doms)
|
|
print(f"\nObject styles other than [None]: {styles} in {len(files)} IDML files")
|
|
counts = defaultdict(int)
|
|
for _, _, since in okept:
|
|
counts[since] += 1
|
|
for (since, n), k in sorted(counts.items()):
|
|
print(f" {k:4} values, the same in {n} of {n} IDML files with DOM >= {since}")
|
|
print(f" {len(oleft)} left out (vary, not in every style, or in fewer than {MIN_FILES} files):")
|
|
print(f" {', '.join(sorted(oleft))}")
|
|
if "--write" in sys.argv:
|
|
write_object_styles(okept, oorder, min(doms))
|
|
print(f"wrote {OBJECT_OUT.relative_to(ROOT)}")
|
|
write(kept, order, min(doms))
|
|
print(f"\nwrote {OUT.relative_to(ROOT)}")
|
|
write_preferences(tags, pkept, porder, min(doms))
|
|
print(f"wrote {PREF_OUT.relative_to(ROOT)}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|