#!/usr/bin/env python3
"""Find the values that every corpus IDML has on its root styles, on its
other object styles and in its preferences.
Reads Resources/Styles.xml and Resources/Preferences.xml of the
reference IDML of every pair that compare.py compares (distinct files
only) and collects each attribute,
Properties child and child element attribute of the root styles and of
the top-level preference elements. A value is kept when it is the same in
every IDML from some DOM version on and present in all of them; the
earliest such version is its minimum version. Attributes the converter
reads from the INDD (the TEXT_ATTRS table in src/idml/attrs.rs) are left
out for the paragraph and character root styles, and the document setup
it reads is left out of the preferences. Preference values that refer to
a style other than a root style are left out, since the package need not
have that style.
Object styles other than the root style `[None]` are collected the same
way, all styles of a file together: a value is kept when every such style
in every IDML from some DOM version on has it, with the same value.
Usage: python3 -I tools/root_values.py [--write]
Prints the evidence counts. With --write, regenerates
src/idml/root_values.xml, src/idml/object_style_values.xml and
src/idml/preference_values.xml. See docs/format/idml-values.md.
"""
import hashlib
import re
import sys
import xml.etree.ElementTree as ET
import zipfile
from collections import OrderedDict, defaultdict
from pathlib import Path
from xml.sax.saxutils import quoteattr
ROOT = Path(__file__).resolve().parent.parent
OUT = ROOT / "src" / "idml" / "root_values.xml"
PREF_OUT = ROOT / "src" / "idml" / "preference_values.xml"
OBJECT_OUT = ROOT / "src" / "idml" / "object_style_values.xml"
# The values are collected from the pairs that tools/compare.py compares,
# without the files whose path under corpus/ starts with one of these
# (privately held samples).
EXCLUDE = ("own/",)
# Preference attributes the converter reads from the INDD.
PREF_SKIP = {
("DocumentPreference", "PageHeight"), ("DocumentPreference", "PageWidth"),
("DocumentPreference", "FacingPages"), ("DocumentPreference", "Intent"),
("DocumentPreference", "DocumentBleedTopOffset"),
("DocumentPreference", "DocumentBleedBottomOffset"),
("DocumentPreference", "DocumentBleedInsideOrLeftOffset"),
("DocumentPreference", "DocumentBleedOutsideOrRightOffset"),
("ViewPreference", "RulerOrigin"),
}
# Styles a preference value may refer to: those every package has.
ROOT_STYLES = {
"ParagraphStyle/$ID/[No paragraph style]",
"CharacterStyle/$ID/[No character style]",
"ObjectStyle/$ID/[None]",
}
# A preference value first seen in a later version is kept only when at
# least this many files show it.
MIN_FILES = 10
STYLE_REF = re.compile(r"(?:>|^)(Paragraph|Character|Object|Cell|Table)Style/[^<]*")
PREF_HEADER = """
"""
TARGETS = OrderedDict([
("ParagraphStyle", "ParagraphStyle/$ID/[No paragraph style]"),
("CharacterStyle", "CharacterStyle/$ID/[No character style]"),
("ObjectStyle", "ObjectStyle/$ID/[None]"),
("CellStyle", "CellStyle/$ID/[None]"),
("TableStyle", "TableStyle/$ID/[No table style]"),
])
# Written by the converter itself.
SKIP = {"Self", "Name"}
HEADER = """
"""
def decoded_names():
src = (ROOT / "src" / "idml" / "attrs.rs").read_text()
table = src[src.index("const TEXT_ATTRS"):src.index("/// An attribute written as a")]
return set(re.findall(r'0x[0-9A-F]+,\s*"(\w+)"', table))
def corpus_idmls():
"""The reference IDML of every pair that tools/compare.py compares: a
little-endian INDD whose IDML has the same major version, not in
EXCLUDE. Each distinct IDML counts once."""
lines = (ROOT / "corpus" / "inventory.tsv").read_text().splitlines()
cols = lines[0].split("\t")
seen, out, pairs = set(), [], 0
for line in lines[1:]:
row = dict(zip(cols, line.split("\t")))
path, order, ver, dom = row["path"], row["order"], row["version"], row["idml_dom"]
if order != "LE" or not dom or ver.split(".")[0] != dom.split(".")[0]:
continue
elif path.startswith(EXCLUDE):
continue
pairs += 1
idml = ROOT / "corpus" / (row.get("idml") or str(Path(path).with_suffix(".idml")))
digest = hashlib.md5(idml.read_bytes()).hexdigest()
if digest not in seen:
seen.add(digest)
out.append((int(dom.split(".")[0]), idml))
return pairs, out
def leaves(el, path, out, order):
"""Flatten an element: (path, '@attr') and (path, 'P/tag') keys."""
for k, v in el.attrib.items():
out[(path, "@" + k)] = v
order.setdefault((path, "@" + k), len(order))
for c in el:
if c.tag == "Properties":
for p in c:
key = (path, "P/" + p.tag)
out[key] = re.sub(r">\s+<", "><", ET.tostring(p, encoding="unicode").strip())
order.setdefault(key, len(order))
else:
order.setdefault((path + (c.tag,), ""), len(order))
leaves(c, path + (c.tag,), out, order)
def analyse():
decoded = decoded_names()
pairs, idmls = corpus_idmls()
doms = [d for d, _ in idmls]
values = defaultdict(dict)
order = defaultdict(dict)
for i, (_dom, idml) in enumerate(idmls):
root = ET.fromstring(zipfile.ZipFile(idml).read("Resources/Styles.xml"))
for el in root.iter():
if el.tag in TARGETS and TARGETS[el.tag] == el.get("Self"):
out = {}
leaves(el, (), out, order[el.tag])
for k, v in out.items():
values[(el.tag, k)][i] = v
kept = defaultdict(list)
left_out = defaultdict(lambda: defaultdict(list))
for (tag, key), by_file in values.items():
path, name = key
if not path and name[1:] in SKIP:
continue
plain = name.split("/", 1)[1] if name.startswith("P/") else name[1:]
if not path and tag in ("ParagraphStyle", "CharacterStyle") and plain in decoded:
left_out[tag]["read from the INDD"].append(plain)
continue
since = None
for m in sorted({doms[i] for i in by_file}):
files = [i for i, d in enumerate(doms) if d >= m]
if all(i in by_file for i in files):
if len({by_file[i] for i in files}) == 1:
since = (m, len(files))
break
if since is None:
left_out[tag]["varies"].append("/".join(path + (plain,)))
continue
kept[tag].append((key, next(iter(by_file.values())), since))
return pairs, idmls, doms, kept, left_out, order
def since_of(by_file, doms):
"""The first DOM version from which every file has the same value, and
the number of those files; None if there is none."""
for m in sorted({doms[i] for i in by_file}):
files = [i for i, d in enumerate(doms) if d >= m]
if all(i in by_file for i in files):
if len({by_file[i] for i in files}) == 1:
return (m, len(files))
return None
return None
def style_ref(v):
m = STYLE_REF.search(v)
return m and m.group(0).lstrip(">") not in ROOT_STYLES
def analyse_preferences(idmls, doms):
values = defaultdict(dict)
order = defaultdict(dict)
tags = OrderedDict()
for i, (_dom, idml) in enumerate(idmls):
root = ET.fromstring(zipfile.ZipFile(idml).read("Resources/Preferences.xml"))
for el in root:
tags.setdefault(el.tag, len(tags))
out = {}
leaves(el, (), out, order[el.tag])
for k, v in out.items():
values[(el.tag, k)][i] = v
kept = defaultdict(list)
left_out = defaultdict(lambda: defaultdict(list))
for (tag, key), by_file in values.items():
path, name = key
plain = name.split("/", 1)[1] if name.startswith("P/") else name[1:]
if not path and (tag, plain) in PREF_SKIP:
left_out[tag]["read from the INDD"].append(plain)
continue
since = since_of(by_file, doms)
if since is None:
left_out[tag]["varies"].append("/".join(path + (plain,)))
continue
if since[1] < MIN_FILES:
left_out[tag][f"in fewer than {MIN_FILES} files"].append("/".join(path + (plain,)))
continue
value = next(iter(by_file.values()))
if style_ref(value):
left_out[tag]["refers to a style"].append("/".join(path + (plain,)))
continue
kept[tag].append((key, value, since))
return tags, kept, left_out, order
OBJECT_HEADER = """
"""
def analyse_object_styles(idmls, doms):
"""Values of the object styles other than the root style. A file
contributes a value for a key when all its such styles have the key
with one value; the key is kept when every file with such styles from
some DOM version on has the same value."""
values = defaultdict(dict)
order = {}
files = []
styles = 0
for i, (_dom, idml) in enumerate(idmls):
root = ET.fromstring(zipfile.ZipFile(idml).read("Resources/Styles.xml"))
found = [el for el in root.iter("ObjectStyle")
if el.get("Self") != TARGETS["ObjectStyle"]]
if not found:
continue
files.append(i)
styles += len(found)
per_key = defaultdict(list)
for el in found:
out = {}
leaves(el, (), out, order)
for k, v in out.items():
per_key[k].append(v)
for k, vs in per_key.items():
if len(vs) == len(found) and len(set(vs)) == 1:
values[k][i] = vs[0]
sub_doms = [doms[i] if i in files else -1 for i in range(len(doms))]
kept, left_out = [], []
for key, by_file in values.items():
path, name = key
plain = name.split("/", 1)[1] if name.startswith("P/") else name[1:]
if not path and (plain in SKIP or plain == "BasedOn"):
continue
since = None
for m in sorted({sub_doms[i] for i in by_file}):
within = [i for i in files if sub_doms[i] >= m]
if all(i in by_file for i in within):
if len({by_file[i] for i in within}) == 1:
since = (m, len(within))
break
if since is None or since[1] < MIN_FILES:
left_out.append("/".join(path + (plain,)))
continue
kept.append((key, by_file[next(iter(by_file))], since))
return files, styles, kept, left_out, order
def write_object_styles(kept, order, first):
blocks = []
for since in sorted({s for _, _, (s, _) in kept}):
gate = "" if since == first else f' MinimumVersion="{since}"'
items = sorted((it for it in kept if it[2][0] == since), key=lambda it: order[it[0]])
tree = {"attrs": [], "props": [], "children": OrderedDict()}
for (path, name), v, _ in items:
node = tree
for p in path:
node = node["children"].setdefault(
p, {"attrs": [], "props": [], "children": OrderedDict()})
if name.startswith("@"):
node["attrs"].append((name[1:], v))
else:
node["props"].append(v)
blocks.append(f"\t\n" + element("ObjectStyle", tree, "\t\t") + "\t\n")
OBJECT_OUT.write_text(OBJECT_HEADER + "\n" + "".join(blocks)
+ "\n")
def write_preferences(tags, kept, order, first):
blocks = []
for since in sorted({s for items in kept.values() for _, _, (s, _) in items}):
gate = "" if since == first else f' MinimumVersion="{since}"'
s = f"\t\n"
for tag in tags:
items = sorted((it for it in kept[tag] if it[2][0] == since),
key=lambda it: order[tag][it[0]])
if not items:
continue
tree = {"attrs": [], "props": [], "children": OrderedDict()}
for (path, name), v, _ in items:
node = tree
for p in path:
node = node["children"].setdefault(
p, {"attrs": [], "props": [], "children": OrderedDict()})
if name.startswith("@"):
node["attrs"].append((name[1:], v))
else:
node["props"].append(v)
s += element(tag, tree, "\t\t")
blocks.append(s + "\t\n")
PREF_OUT.write_text(PREF_HEADER + "\n" + "".join(blocks) + "\n")
def element(tag, node, indent):
attrs = "".join(f" {k}={quoteattr(v)}" for k, v in node["attrs"])
if not node["props"] and not node["children"]:
return f"{indent}<{tag}{attrs} />\n"
s = f"{indent}<{tag}{attrs}>\n"
if node["props"]:
s += f"{indent}\t\n"
s += "".join(f"{indent}\t\t{p}\n" for p in node["props"])
s += f"{indent}\t\n"
for t, c in node["children"].items():
s += element(t, c, indent + "\t")
return s + f"{indent}{tag}>\n"
def write(kept, order, first):
blocks = []
for since in sorted({s for items in kept.values() for _, _, (s, _) in items}):
gate = "" if since == first else f' MinimumVersion="{since}"'
s = f"\t\n"
for tag in TARGETS:
items = sorted((it for it in kept[tag] if it[2][0] == since),
key=lambda it: order[tag][it[0]])
if not items:
continue
tree = {"attrs": [], "props": [], "children": OrderedDict()}
for (path, name), v, _ in items:
node = tree
for p in path:
node = node["children"].setdefault(
p, {"attrs": [], "props": [], "children": OrderedDict()})
if name.startswith("@"):
node["attrs"].append((name[1:], v))
else:
node["props"].append(v)
s += element(tag, tree, "\t\t")
blocks.append(s + "\t\n")
OUT.write_text(HEADER + "\n" + "".join(blocks) + "\n")
def main():
pairs, idmls, doms, kept, left_out, order = analyse()
print(f"{pairs} pairs, {len(idmls)} distinct IDML files, DOM versions {sorted(set(doms))}")
for tag in TARGETS:
counts = defaultdict(int)
for _, _, since in kept[tag]:
counts[since] += 1
print(f"\n{tag}:")
for (since, n), k in sorted(counts.items()):
print(f" {k:4} values, the same in {n} of {n} IDML files with DOM >= {since}")
for why, names in left_out[tag].items():
print(f" {len(names):4} left out ({why}): {', '.join(sorted(names))}")
tags, pkept, pleft, porder = analyse_preferences(idmls, doms)
print("\nPreferences:")
total = defaultdict(int)
for tag in tags:
for _, _, since in pkept[tag]:
total[since] += 1
for (since, n), k in sorted(total.items()):
print(f" {k:4} values, the same in {n} of {n} IDML files with DOM >= {since}")
for tag in tags:
counts = {why: len(names) for why, names in pleft[tag].items()}
print(f" {tag}: {len(pkept[tag])} kept, left out {counts}")
if pleft[tag].get("refers to a style") or pleft[tag].get("read from the INDD"):
print(f" {pleft[tag].get('refers to a style', [])} {pleft[tag].get('read from the INDD', [])}")
files, styles, okept, oleft, oorder = analyse_object_styles(idmls, doms)
print(f"\nObject styles other than [None]: {styles} in {len(files)} IDML files")
counts = defaultdict(int)
for _, _, since in okept:
counts[since] += 1
for (since, n), k in sorted(counts.items()):
print(f" {k:4} values, the same in {n} of {n} IDML files with DOM >= {since}")
print(f" {len(oleft)} left out (vary, not in every style, or in fewer than {MIN_FILES} files):")
print(f" {', '.join(sorted(oleft))}")
if "--write" in sys.argv:
write_object_styles(okept, oorder, min(doms))
print(f"wrote {OBJECT_OUT.relative_to(ROOT)}")
write(kept, order, min(doms))
print(f"\nwrote {OUT.relative_to(ROOT)}")
write_preferences(tags, pkept, porder, min(doms))
print(f"wrote {PREF_OUT.relative_to(ROOT)}")
if __name__ == "__main__":
main()