#!/usr/bin/env python3 """Find the values that every corpus IDML has on its root styles. Reads Resources/Styles.xml of every IDML in the corpus pairs (distinct files only) and collects each attribute, Properties child and child element attribute of the root styles. A value is kept when it is the same in every IDML from some DOM version on and present in all of them; the earliest such version is its minimum version. Attributes the converter reads from the INDD (the TEXT_ATTRS table in src/idml/mod.rs) are left out for the paragraph and character root styles. Usage: python3 -I tools/root_values.py [--write] Prints the evidence counts. With --write, regenerates src/idml/root_values.xml. See docs/format/idml-values.md. """ import hashlib import re import sys import xml.etree.ElementTree as ET import zipfile from collections import OrderedDict, defaultdict from pathlib import Path from xml.sax.saxutils import quoteattr ROOT = Path(__file__).resolve().parent.parent OUT = ROOT / "src" / "idml" / "root_values.xml" TARGETS = OrderedDict([ ("ParagraphStyle", "ParagraphStyle/$ID/[No paragraph style]"), ("CharacterStyle", "CharacterStyle/$ID/[No character style]"), ("ObjectStyle", "ObjectStyle/$ID/[None]"), ("CellStyle", "CellStyle/$ID/[None]"), ("TableStyle", "TableStyle/$ID/[No table style]"), ]) # Written by the converter itself. SKIP = {"Self", "Name"} HEADER = """ """ def decoded_names(): src = (ROOT / "src" / "idml" / "mod.rs").read_text() table = src[src.index("const TEXT_ATTRS"):src.index("/// An attribute written as a")] return set(re.findall(r'0x[0-9A-F]+,\s*"(\w+)"', table)) def corpus_idmls(): rows = (ROOT / "corpus" / "inventory.tsv").read_text().splitlines()[1:] seen, out, pairs = set(), [], 0 for row in rows: path, _size, _valid, _order, _ver, _creator, dom = row.split("\t") if not dom: continue pairs += 1 idml = (ROOT / "corpus" / path).with_suffix(".idml") digest = hashlib.md5(idml.read_bytes()).hexdigest() if digest not in seen: seen.add(digest) out.append((int(dom.split(".")[0]), idml)) return pairs, out def leaves(el, path, out, order): """Flatten an element: (path, '@attr') and (path, 'P/tag') keys.""" for k, v in el.attrib.items(): out[(path, "@" + k)] = v order.setdefault((path, "@" + k), len(order)) for c in el: if c.tag == "Properties": for p in c: key = (path, "P/" + p.tag) out[key] = re.sub(r">\s+<", "><", ET.tostring(p, encoding="unicode").strip()) order.setdefault(key, len(order)) else: order.setdefault((path + (c.tag,), ""), len(order)) leaves(c, path + (c.tag,), out, order) def analyse(): decoded = decoded_names() pairs, idmls = corpus_idmls() doms = [d for d, _ in idmls] values = defaultdict(dict) order = defaultdict(dict) for i, (_dom, idml) in enumerate(idmls): root = ET.fromstring(zipfile.ZipFile(idml).read("Resources/Styles.xml")) for el in root.iter(): if el.tag in TARGETS and TARGETS[el.tag] == el.get("Self"): out = {} leaves(el, (), out, order[el.tag]) for k, v in out.items(): values[(el.tag, k)][i] = v kept = defaultdict(list) left_out = defaultdict(lambda: defaultdict(list)) for (tag, key), by_file in values.items(): path, name = key if not path and name[1:] in SKIP: continue plain = name.split("/", 1)[1] if name.startswith("P/") else name[1:] if not path and tag in ("ParagraphStyle", "CharacterStyle") and plain in decoded: left_out[tag]["read from the INDD"].append(plain) continue since = None for m in sorted({doms[i] for i in by_file}): files = [i for i, d in enumerate(doms) if d >= m] if all(i in by_file for i in files): if len({by_file[i] for i in files}) == 1: since = (m, len(files)) break if since is None: left_out[tag]["varies"].append("/".join(path + (plain,))) continue kept[tag].append((key, next(iter(by_file.values())), since)) return pairs, idmls, doms, kept, left_out, order def element(tag, node, indent): attrs = "".join(f" {k}={quoteattr(v)}" for k, v in node["attrs"]) if not node["props"] and not node["children"]: return f"{indent}<{tag}{attrs} />\n" s = f"{indent}<{tag}{attrs}>\n" if node["props"]: s += f"{indent}\t\n" s += "".join(f"{indent}\t\t{p}\n" for p in node["props"]) s += f"{indent}\t\n" for t, c in node["children"].items(): s += element(t, c, indent + "\t") return s + f"{indent}\n" def write(kept, order, first): blocks = [] for since in sorted({s for items in kept.values() for _, _, (s, _) in items}): gate = "" if since == first else f' MinimumVersion="{since}"' s = f"\t\n" for tag in TARGETS: items = sorted((it for it in kept[tag] if it[2][0] == since), key=lambda it: order[tag][it[0]]) if not items: continue tree = {"attrs": [], "props": [], "children": OrderedDict()} for (path, name), v, _ in items: node = tree for p in path: node = node["children"].setdefault( p, {"attrs": [], "props": [], "children": OrderedDict()}) if name.startswith("@"): node["attrs"].append((name[1:], v)) else: node["props"].append(v) s += element(tag, tree, "\t\t") blocks.append(s + "\t\n") OUT.write_text(HEADER + "\n" + "".join(blocks) + "\n") def main(): pairs, idmls, doms, kept, left_out, order = analyse() print(f"{pairs} pairs, {len(idmls)} distinct IDML files, DOM versions {sorted(set(doms))}") for tag in TARGETS: counts = defaultdict(int) for _, _, since in kept[tag]: counts[since] += 1 print(f"\n{tag}:") for (since, n), k in sorted(counts.items()): print(f" {k:4} values, the same in {n} of {n} IDML files with DOM >= {since}") for why, names in left_out[tag].items(): print(f" {len(names):4} left out ({why}): {', '.join(sorted(names))}") if "--write" in sys.argv: write(kept, order, min(doms)) print(f"\nwrote {OUT.relative_to(ROOT)}") if __name__ == "__main__": main()