#!/usr/bin/env python3 """Find the values that every corpus IDML has on its root styles, on its other object styles and in its preferences. Reads Resources/Styles.xml and Resources/Preferences.xml of the reference IDML of every pair that compare.py compares (distinct files only) and collects each attribute, Properties child and child element attribute of the root styles and of the top-level preference elements. A value is kept when it is the same in every IDML from some DOM version on and present in all of them; the earliest such version is its minimum version. Attributes the converter reads from the INDD (the TEXT_ATTRS table in src/idml/attrs.rs) are left out for the paragraph and character root styles, and the document setup it reads is left out of the preferences. Preference values that refer to a style other than a root style are left out, since the package need not have that style. Object styles other than the root style `[None]` are collected the same way, all styles of a file together: a value is kept when every such style in every IDML from some DOM version on has it, with the same value. Usage: python3 -I tools/root_values.py [--write] Prints the evidence counts. With --write, regenerates src/idml/root_values.xml, src/idml/object_style_values.xml and src/idml/preference_values.xml. See docs/format/idml-values.md. """ import hashlib import re import sys import xml.etree.ElementTree as ET import zipfile from collections import OrderedDict, defaultdict from pathlib import Path from xml.sax.saxutils import quoteattr ROOT = Path(__file__).resolve().parent.parent OUT = ROOT / "src" / "idml" / "root_values.xml" PREF_OUT = ROOT / "src" / "idml" / "preference_values.xml" OBJECT_OUT = ROOT / "src" / "idml" / "object_style_values.xml" # The values are collected from the pairs that tools/compare.py compares, # without the files whose path under corpus/ starts with one of these # (privately held samples). EXCLUDE = ("own/",) # Preference attributes the converter reads from the INDD. PREF_SKIP = { ("DocumentPreference", "PageHeight"), ("DocumentPreference", "PageWidth"), ("DocumentPreference", "FacingPages"), ("DocumentPreference", "Intent"), ("DocumentPreference", "DocumentBleedTopOffset"), ("DocumentPreference", "DocumentBleedBottomOffset"), ("DocumentPreference", "DocumentBleedInsideOrLeftOffset"), ("DocumentPreference", "DocumentBleedOutsideOrRightOffset"), ("ViewPreference", "RulerOrigin"), } # Styles a preference value may refer to: those every package has. ROOT_STYLES = { "ParagraphStyle/$ID/[No paragraph style]", "CharacterStyle/$ID/[No character style]", "ObjectStyle/$ID/[None]", } # A preference value first seen in a later version is kept only when at # least this many files show it. MIN_FILES = 10 STYLE_REF = re.compile(r"(?:>|^)(Paragraph|Character|Object|Cell|Table)Style/[^<]*") PREF_HEADER = """ """ TARGETS = OrderedDict([ ("ParagraphStyle", "ParagraphStyle/$ID/[No paragraph style]"), ("CharacterStyle", "CharacterStyle/$ID/[No character style]"), ("ObjectStyle", "ObjectStyle/$ID/[None]"), ("CellStyle", "CellStyle/$ID/[None]"), ("TableStyle", "TableStyle/$ID/[No table style]"), ]) # Written by the converter itself. SKIP = {"Self", "Name"} HEADER = """ """ def decoded_names(): src = (ROOT / "src" / "idml" / "attrs.rs").read_text() table = src[src.index("const TEXT_ATTRS"):src.index("/// An attribute written as a")] return set(re.findall(r'0x[0-9A-F]+,\s*"(\w+)"', table)) def corpus_idmls(): """The reference IDML of every pair that tools/compare.py compares: a little-endian INDD whose IDML has the same major version, not in EXCLUDE. Each distinct IDML counts once.""" lines = (ROOT / "corpus" / "inventory.tsv").read_text().splitlines() cols = lines[0].split("\t") seen, out, pairs = set(), [], 0 for line in lines[1:]: row = dict(zip(cols, line.split("\t"))) path, order, ver, dom = row["path"], row["order"], row["version"], row["idml_dom"] if order != "LE" or not dom or ver.split(".")[0] != dom.split(".")[0]: continue elif path.startswith(EXCLUDE): continue pairs += 1 idml = ROOT / "corpus" / (row.get("idml") or str(Path(path).with_suffix(".idml"))) digest = hashlib.md5(idml.read_bytes()).hexdigest() if digest not in seen: seen.add(digest) out.append((int(dom.split(".")[0]), idml)) return pairs, out def leaves(el, path, out, order): """Flatten an element: (path, '@attr') and (path, 'P/tag') keys.""" for k, v in el.attrib.items(): out[(path, "@" + k)] = v order.setdefault((path, "@" + k), len(order)) for c in el: if c.tag == "Properties": for p in c: key = (path, "P/" + p.tag) out[key] = re.sub(r">\s+<", "><", ET.tostring(p, encoding="unicode").strip()) order.setdefault(key, len(order)) else: order.setdefault((path + (c.tag,), ""), len(order)) leaves(c, path + (c.tag,), out, order) def analyse(): decoded = decoded_names() pairs, idmls = corpus_idmls() doms = [d for d, _ in idmls] values = defaultdict(dict) order = defaultdict(dict) for i, (_dom, idml) in enumerate(idmls): root = ET.fromstring(zipfile.ZipFile(idml).read("Resources/Styles.xml")) for el in root.iter(): if el.tag in TARGETS and TARGETS[el.tag] == el.get("Self"): out = {} leaves(el, (), out, order[el.tag]) for k, v in out.items(): values[(el.tag, k)][i] = v kept = defaultdict(list) left_out = defaultdict(lambda: defaultdict(list)) for (tag, key), by_file in values.items(): path, name = key if not path and name[1:] in SKIP: continue plain = name.split("/", 1)[1] if name.startswith("P/") else name[1:] if not path and tag in ("ParagraphStyle", "CharacterStyle") and plain in decoded: left_out[tag]["read from the INDD"].append(plain) continue since = None for m in sorted({doms[i] for i in by_file}): files = [i for i, d in enumerate(doms) if d >= m] if all(i in by_file for i in files): if len({by_file[i] for i in files}) == 1: since = (m, len(files)) break if since is None: left_out[tag]["varies"].append("/".join(path + (plain,))) continue kept[tag].append((key, next(iter(by_file.values())), since)) return pairs, idmls, doms, kept, left_out, order def since_of(by_file, doms): """The first DOM version from which every file has the same value, and the number of those files; None if there is none.""" for m in sorted({doms[i] for i in by_file}): files = [i for i, d in enumerate(doms) if d >= m] if all(i in by_file for i in files): if len({by_file[i] for i in files}) == 1: return (m, len(files)) return None return None def style_ref(v): m = STYLE_REF.search(v) return m and m.group(0).lstrip(">") not in ROOT_STYLES def analyse_preferences(idmls, doms): values = defaultdict(dict) order = defaultdict(dict) tags = OrderedDict() for i, (_dom, idml) in enumerate(idmls): root = ET.fromstring(zipfile.ZipFile(idml).read("Resources/Preferences.xml")) for el in root: tags.setdefault(el.tag, len(tags)) out = {} leaves(el, (), out, order[el.tag]) for k, v in out.items(): values[(el.tag, k)][i] = v kept = defaultdict(list) left_out = defaultdict(lambda: defaultdict(list)) for (tag, key), by_file in values.items(): path, name = key plain = name.split("/", 1)[1] if name.startswith("P/") else name[1:] if not path and (tag, plain) in PREF_SKIP: left_out[tag]["read from the INDD"].append(plain) continue since = since_of(by_file, doms) if since is None: left_out[tag]["varies"].append("/".join(path + (plain,))) continue if since[1] < MIN_FILES: left_out[tag][f"in fewer than {MIN_FILES} files"].append("/".join(path + (plain,))) continue value = next(iter(by_file.values())) if style_ref(value): left_out[tag]["refers to a style"].append("/".join(path + (plain,))) continue kept[tag].append((key, value, since)) return tags, kept, left_out, order OBJECT_HEADER = """ """ def analyse_object_styles(idmls, doms): """Values of the object styles other than the root style. A file contributes a value for a key when all its such styles have the key with one value; the key is kept when every file with such styles from some DOM version on has the same value.""" values = defaultdict(dict) order = {} files = [] styles = 0 for i, (_dom, idml) in enumerate(idmls): root = ET.fromstring(zipfile.ZipFile(idml).read("Resources/Styles.xml")) found = [el for el in root.iter("ObjectStyle") if el.get("Self") != TARGETS["ObjectStyle"]] if not found: continue files.append(i) styles += len(found) per_key = defaultdict(list) for el in found: out = {} leaves(el, (), out, order) for k, v in out.items(): per_key[k].append(v) for k, vs in per_key.items(): if len(vs) == len(found) and len(set(vs)) == 1: values[k][i] = vs[0] sub_doms = [doms[i] if i in files else -1 for i in range(len(doms))] kept, left_out = [], [] for key, by_file in values.items(): path, name = key plain = name.split("/", 1)[1] if name.startswith("P/") else name[1:] if not path and (plain in SKIP or plain == "BasedOn"): continue since = None for m in sorted({sub_doms[i] for i in by_file}): within = [i for i in files if sub_doms[i] >= m] if all(i in by_file for i in within): if len({by_file[i] for i in within}) == 1: since = (m, len(within)) break if since is None or since[1] < MIN_FILES: left_out.append("/".join(path + (plain,))) continue kept.append((key, by_file[next(iter(by_file))], since)) return files, styles, kept, left_out, order def write_object_styles(kept, order, first): blocks = [] for since in sorted({s for _, _, (s, _) in kept}): gate = "" if since == first else f' MinimumVersion="{since}"' items = sorted((it for it in kept if it[2][0] == since), key=lambda it: order[it[0]]) tree = {"attrs": [], "props": [], "children": OrderedDict()} for (path, name), v, _ in items: node = tree for p in path: node = node["children"].setdefault( p, {"attrs": [], "props": [], "children": OrderedDict()}) if name.startswith("@"): node["attrs"].append((name[1:], v)) else: node["props"].append(v) blocks.append(f"\t\n" + element("ObjectStyle", tree, "\t\t") + "\t\n") OBJECT_OUT.write_text(OBJECT_HEADER + "\n" + "".join(blocks) + "\n") def write_preferences(tags, kept, order, first): blocks = [] for since in sorted({s for items in kept.values() for _, _, (s, _) in items}): gate = "" if since == first else f' MinimumVersion="{since}"' s = f"\t\n" for tag in tags: items = sorted((it for it in kept[tag] if it[2][0] == since), key=lambda it: order[tag][it[0]]) if not items: continue tree = {"attrs": [], "props": [], "children": OrderedDict()} for (path, name), v, _ in items: node = tree for p in path: node = node["children"].setdefault( p, {"attrs": [], "props": [], "children": OrderedDict()}) if name.startswith("@"): node["attrs"].append((name[1:], v)) else: node["props"].append(v) s += element(tag, tree, "\t\t") blocks.append(s + "\t\n") PREF_OUT.write_text(PREF_HEADER + "\n" + "".join(blocks) + "\n") def element(tag, node, indent): attrs = "".join(f" {k}={quoteattr(v)}" for k, v in node["attrs"]) if not node["props"] and not node["children"]: return f"{indent}<{tag}{attrs} />\n" s = f"{indent}<{tag}{attrs}>\n" if node["props"]: s += f"{indent}\t\n" s += "".join(f"{indent}\t\t{p}\n" for p in node["props"]) s += f"{indent}\t\n" for t, c in node["children"].items(): s += element(t, c, indent + "\t") return s + f"{indent}\n" def write(kept, order, first): blocks = [] for since in sorted({s for items in kept.values() for _, _, (s, _) in items}): gate = "" if since == first else f' MinimumVersion="{since}"' s = f"\t\n" for tag in TARGETS: items = sorted((it for it in kept[tag] if it[2][0] == since), key=lambda it: order[tag][it[0]]) if not items: continue tree = {"attrs": [], "props": [], "children": OrderedDict()} for (path, name), v, _ in items: node = tree for p in path: node = node["children"].setdefault( p, {"attrs": [], "props": [], "children": OrderedDict()}) if name.startswith("@"): node["attrs"].append((name[1:], v)) else: node["props"].append(v) s += element(tag, tree, "\t\t") blocks.append(s + "\t\n") OUT.write_text(HEADER + "\n" + "".join(blocks) + "\n") def main(): pairs, idmls, doms, kept, left_out, order = analyse() print(f"{pairs} pairs, {len(idmls)} distinct IDML files, DOM versions {sorted(set(doms))}") for tag in TARGETS: counts = defaultdict(int) for _, _, since in kept[tag]: counts[since] += 1 print(f"\n{tag}:") for (since, n), k in sorted(counts.items()): print(f" {k:4} values, the same in {n} of {n} IDML files with DOM >= {since}") for why, names in left_out[tag].items(): print(f" {len(names):4} left out ({why}): {', '.join(sorted(names))}") tags, pkept, pleft, porder = analyse_preferences(idmls, doms) print("\nPreferences:") total = defaultdict(int) for tag in tags: for _, _, since in pkept[tag]: total[since] += 1 for (since, n), k in sorted(total.items()): print(f" {k:4} values, the same in {n} of {n} IDML files with DOM >= {since}") for tag in tags: counts = {why: len(names) for why, names in pleft[tag].items()} print(f" {tag}: {len(pkept[tag])} kept, left out {counts}") if pleft[tag].get("refers to a style") or pleft[tag].get("read from the INDD"): print(f" {pleft[tag].get('refers to a style', [])} {pleft[tag].get('read from the INDD', [])}") files, styles, okept, oleft, oorder = analyse_object_styles(idmls, doms) print(f"\nObject styles other than [None]: {styles} in {len(files)} IDML files") counts = defaultdict(int) for _, _, since in okept: counts[since] += 1 for (since, n), k in sorted(counts.items()): print(f" {k:4} values, the same in {n} of {n} IDML files with DOM >= {since}") print(f" {len(oleft)} left out (vary, not in every style, or in fewer than {MIN_FILES} files):") print(f" {', '.join(sorted(oleft))}") if "--write" in sys.argv: write_object_styles(okept, oorder, min(doms)) print(f"wrote {OBJECT_OUT.relative_to(ROOT)}") write(kept, order, min(doms)) print(f"\nwrote {OUT.relative_to(ROOT)}") write_preferences(tags, pkept, porder, min(doms)) print(f"wrote {PREF_OUT.relative_to(ROOT)}") if __name__ == "__main__": main()