#!/usr/bin/env python3 """Find the values that every corpus IDML has on its root styles, on its other object styles and in its preferences. Reads Resources/Styles.xml and Resources/Preferences.xml of every IDML in the corpus pairs (distinct files only) and collects each attribute, Properties child and child element attribute of the root styles and of the top-level preference elements. A value is kept when it is the same in every IDML from some DOM version on and present in all of them; the earliest such version is its minimum version. Attributes the converter reads from the INDD (the TEXT_ATTRS table in src/idml/mod.rs) are left out for the paragraph and character root styles, and the document setup it reads is left out of the preferences. Preference values that refer to a style other than a root style are left out, since the package need not have that style. Object styles other than the root style `[None]` are collected the same way, all styles of a file together: a value is kept when every such style in every IDML from some DOM version on has it, with the same value. Usage: python3 -I tools/root_values.py [--write] Prints the evidence counts. With --write, regenerates src/idml/root_values.xml, src/idml/object_style_values.xml and src/idml/preference_values.xml. See docs/format/idml-values.md. """ import hashlib import re import sys import xml.etree.ElementTree as ET import zipfile from collections import OrderedDict, defaultdict from pathlib import Path from xml.sax.saxutils import quoteattr ROOT = Path(__file__).resolve().parent.parent OUT = ROOT / "src" / "idml" / "root_values.xml" PREF_OUT = ROOT / "src" / "idml" / "preference_values.xml" OBJECT_OUT = ROOT / "src" / "idml" / "object_style_values.xml" # The generated files were made from the corpus pairs under this # directory, the whole corpus at the time. Later pairs are evidence for # tools/element_values.py; regenerating these files from them would drop # values that the larger corpus shows to vary (docs/format/idml-values.md). SOURCE = "third-party/" # Preference attributes the converter reads from the INDD. PREF_SKIP = { ("DocumentPreference", "PageHeight"), ("DocumentPreference", "PageWidth"), ("DocumentPreference", "FacingPages"), ("DocumentPreference", "Intent"), ("DocumentPreference", "DocumentBleedTopOffset"), ("DocumentPreference", "DocumentBleedBottomOffset"), ("DocumentPreference", "DocumentBleedInsideOrLeftOffset"), ("DocumentPreference", "DocumentBleedOutsideOrRightOffset"), ("ViewPreference", "RulerOrigin"), } # Styles a preference value may refer to: those every package has. ROOT_STYLES = { "ParagraphStyle/$ID/[No paragraph style]", "CharacterStyle/$ID/[No character style]", "ObjectStyle/$ID/[None]", } # A preference value first seen in a later version is kept only when at # least this many files show it. MIN_FILES = 10 STYLE_REF = re.compile(r"(?:>|^)(Paragraph|Character|Object|Cell|Table)Style/[^<]*") PREF_HEADER = """ """ TARGETS = OrderedDict([ ("ParagraphStyle", "ParagraphStyle/$ID/[No paragraph style]"), ("CharacterStyle", "CharacterStyle/$ID/[No character style]"), ("ObjectStyle", "ObjectStyle/$ID/[None]"), ("CellStyle", "CellStyle/$ID/[None]"), ("TableStyle", "TableStyle/$ID/[No table style]"), ]) # Written by the converter itself. SKIP = {"Self", "Name"} HEADER = """ """ def decoded_names(): src = (ROOT / "src" / "idml" / "mod.rs").read_text() table = src[src.index("const TEXT_ATTRS"):src.index("/// An attribute written as a")] return set(re.findall(r'0x[0-9A-F]+,\s*"(\w+)"', table)) def corpus_idmls(): lines = (ROOT / "corpus" / "inventory.tsv").read_text().splitlines() cols = lines[0].split("\t") seen, out, pairs = set(), [], 0 for line in lines[1:]: row = dict(zip(cols, line.split("\t"))) path, dom = row["path"], row["idml_dom"] if not dom or not dom[0].isdigit() or not path.startswith(SOURCE): continue pairs += 1 idml = ROOT / "corpus" / (row.get("idml") or str(Path(path).with_suffix(".idml"))) digest = hashlib.md5(idml.read_bytes()).hexdigest() if digest not in seen: seen.add(digest) out.append((int(dom.split(".")[0]), idml)) return pairs, out def leaves(el, path, out, order): """Flatten an element: (path, '@attr') and (path, 'P/tag') keys.""" for k, v in el.attrib.items(): out[(path, "@" + k)] = v order.setdefault((path, "@" + k), len(order)) for c in el: if c.tag == "Properties": for p in c: key = (path, "P/" + p.tag) out[key] = re.sub(r">\s+<", "><", ET.tostring(p, encoding="unicode").strip()) order.setdefault(key, len(order)) else: order.setdefault((path + (c.tag,), ""), len(order)) leaves(c, path + (c.tag,), out, order) def analyse(): decoded = decoded_names() pairs, idmls = corpus_idmls() doms = [d for d, _ in idmls] values = defaultdict(dict) order = defaultdict(dict) for i, (_dom, idml) in enumerate(idmls): root = ET.fromstring(zipfile.ZipFile(idml).read("Resources/Styles.xml")) for el in root.iter(): if el.tag in TARGETS and TARGETS[el.tag] == el.get("Self"): out = {} leaves(el, (), out, order[el.tag]) for k, v in out.items(): values[(el.tag, k)][i] = v kept = defaultdict(list) left_out = defaultdict(lambda: defaultdict(list)) for (tag, key), by_file in values.items(): path, name = key if not path and name[1:] in SKIP: continue plain = name.split("/", 1)[1] if name.startswith("P/") else name[1:] if not path and tag in ("ParagraphStyle", "CharacterStyle") and plain in decoded: left_out[tag]["read from the INDD"].append(plain) continue since = None for m in sorted({doms[i] for i in by_file}): files = [i for i, d in enumerate(doms) if d >= m] if all(i in by_file for i in files): if len({by_file[i] for i in files}) == 1: since = (m, len(files)) break if since is None: left_out[tag]["varies"].append("/".join(path + (plain,))) continue kept[tag].append((key, next(iter(by_file.values())), since)) return pairs, idmls, doms, kept, left_out, order def since_of(by_file, doms): """The first DOM version from which every file has the same value, and the number of those files; None if there is none.""" for m in sorted({doms[i] for i in by_file}): files = [i for i, d in enumerate(doms) if d >= m] if all(i in by_file for i in files): if len({by_file[i] for i in files}) == 1: return (m, len(files)) return None return None def style_ref(v): m = STYLE_REF.search(v) return m and m.group(0).lstrip(">") not in ROOT_STYLES def analyse_preferences(idmls, doms): values = defaultdict(dict) order = defaultdict(dict) tags = OrderedDict() for i, (_dom, idml) in enumerate(idmls): root = ET.fromstring(zipfile.ZipFile(idml).read("Resources/Preferences.xml")) for el in root: tags.setdefault(el.tag, len(tags)) out = {} leaves(el, (), out, order[el.tag]) for k, v in out.items(): values[(el.tag, k)][i] = v kept = defaultdict(list) left_out = defaultdict(lambda: defaultdict(list)) for (tag, key), by_file in values.items(): path, name = key plain = name.split("/", 1)[1] if name.startswith("P/") else name[1:] if not path and (tag, plain) in PREF_SKIP: left_out[tag]["read from the INDD"].append(plain) continue since = since_of(by_file, doms) if since is None: left_out[tag]["varies"].append("/".join(path + (plain,))) continue if since[1] < MIN_FILES: left_out[tag][f"in fewer than {MIN_FILES} files"].append("/".join(path + (plain,))) continue value = next(iter(by_file.values())) if style_ref(value): left_out[tag]["refers to a style"].append("/".join(path + (plain,))) continue kept[tag].append((key, value, since)) return tags, kept, left_out, order OBJECT_HEADER = """ """ def analyse_object_styles(idmls, doms): """Values of the object styles other than the root style. A file contributes a value for a key when all its such styles have the key with one value; the key is kept when every file with such styles from some DOM version on has the same value.""" values = defaultdict(dict) order = {} files = [] styles = 0 for i, (_dom, idml) in enumerate(idmls): root = ET.fromstring(zipfile.ZipFile(idml).read("Resources/Styles.xml")) found = [el for el in root.iter("ObjectStyle") if el.get("Self") != TARGETS["ObjectStyle"]] if not found: continue files.append(i) styles += len(found) per_key = defaultdict(list) for el in found: out = {} leaves(el, (), out, order) for k, v in out.items(): per_key[k].append(v) for k, vs in per_key.items(): if len(vs) == len(found) and len(set(vs)) == 1: values[k][i] = vs[0] sub_doms = [doms[i] if i in files else -1 for i in range(len(doms))] kept, left_out = [], [] for key, by_file in values.items(): path, name = key plain = name.split("/", 1)[1] if name.startswith("P/") else name[1:] if not path and (plain in SKIP or plain == "BasedOn"): continue since = None for m in sorted({sub_doms[i] for i in by_file}): within = [i for i in files if sub_doms[i] >= m] if all(i in by_file for i in within): if len({by_file[i] for i in within}) == 1: since = (m, len(within)) break if since is None or since[1] < MIN_FILES: left_out.append("/".join(path + (plain,))) continue kept.append((key, by_file[next(iter(by_file))], since)) return files, styles, kept, left_out, order def write_object_styles(kept, order, first): blocks = [] for since in sorted({s for _, _, (s, _) in kept}): gate = "" if since == first else f' MinimumVersion="{since}"' items = sorted((it for it in kept if it[2][0] == since), key=lambda it: order[it[0]]) tree = {"attrs": [], "props": [], "children": OrderedDict()} for (path, name), v, _ in items: node = tree for p in path: node = node["children"].setdefault( p, {"attrs": [], "props": [], "children": OrderedDict()}) if name.startswith("@"): node["attrs"].append((name[1:], v)) else: node["props"].append(v) blocks.append(f"\t\n" + element("ObjectStyle", tree, "\t\t") + "\t\n") OBJECT_OUT.write_text(OBJECT_HEADER + "\n" + "".join(blocks) + "\n") def write_preferences(tags, kept, order, first): blocks = [] for since in sorted({s for items in kept.values() for _, _, (s, _) in items}): gate = "" if since == first else f' MinimumVersion="{since}"' s = f"\t\n" for tag in tags: items = sorted((it for it in kept[tag] if it[2][0] == since), key=lambda it: order[tag][it[0]]) if not items: continue tree = {"attrs": [], "props": [], "children": OrderedDict()} for (path, name), v, _ in items: node = tree for p in path: node = node["children"].setdefault( p, {"attrs": [], "props": [], "children": OrderedDict()}) if name.startswith("@"): node["attrs"].append((name[1:], v)) else: node["props"].append(v) s += element(tag, tree, "\t\t") blocks.append(s + "\t\n") PREF_OUT.write_text(PREF_HEADER + "\n" + "".join(blocks) + "\n") def element(tag, node, indent): attrs = "".join(f" {k}={quoteattr(v)}" for k, v in node["attrs"]) if not node["props"] and not node["children"]: return f"{indent}<{tag}{attrs} />\n" s = f"{indent}<{tag}{attrs}>\n" if node["props"]: s += f"{indent}\t\n" s += "".join(f"{indent}\t\t{p}\n" for p in node["props"]) s += f"{indent}\t\n" for t, c in node["children"].items(): s += element(t, c, indent + "\t") return s + f"{indent}\n" def write(kept, order, first): blocks = [] for since in sorted({s for items in kept.values() for _, _, (s, _) in items}): gate = "" if since == first else f' MinimumVersion="{since}"' s = f"\t\n" for tag in TARGETS: items = sorted((it for it in kept[tag] if it[2][0] == since), key=lambda it: order[tag][it[0]]) if not items: continue tree = {"attrs": [], "props": [], "children": OrderedDict()} for (path, name), v, _ in items: node = tree for p in path: node = node["children"].setdefault( p, {"attrs": [], "props": [], "children": OrderedDict()}) if name.startswith("@"): node["attrs"].append((name[1:], v)) else: node["props"].append(v) s += element(tag, tree, "\t\t") blocks.append(s + "\t\n") OUT.write_text(HEADER + "\n" + "".join(blocks) + "\n") def main(): pairs, idmls, doms, kept, left_out, order = analyse() print(f"{pairs} pairs, {len(idmls)} distinct IDML files, DOM versions {sorted(set(doms))}") for tag in TARGETS: counts = defaultdict(int) for _, _, since in kept[tag]: counts[since] += 1 print(f"\n{tag}:") for (since, n), k in sorted(counts.items()): print(f" {k:4} values, the same in {n} of {n} IDML files with DOM >= {since}") for why, names in left_out[tag].items(): print(f" {len(names):4} left out ({why}): {', '.join(sorted(names))}") tags, pkept, pleft, porder = analyse_preferences(idmls, doms) print("\nPreferences:") total = defaultdict(int) for tag in tags: for _, _, since in pkept[tag]: total[since] += 1 for (since, n), k in sorted(total.items()): print(f" {k:4} values, the same in {n} of {n} IDML files with DOM >= {since}") for tag in tags: counts = {why: len(names) for why, names in pleft[tag].items()} print(f" {tag}: {len(pkept[tag])} kept, left out {counts}") if pleft[tag].get("refers to a style") or pleft[tag].get("read from the INDD"): print(f" {pleft[tag].get('refers to a style', [])} {pleft[tag].get('read from the INDD', [])}") files, styles, okept, oleft, oorder = analyse_object_styles(idmls, doms) print(f"\nObject styles other than [None]: {styles} in {len(files)} IDML files") counts = defaultdict(int) for _, _, since in okept: counts[since] += 1 for (since, n), k in sorted(counts.items()): print(f" {k:4} values, the same in {n} of {n} IDML files with DOM >= {since}") print(f" {len(oleft)} left out (vary, not in every style, or in fewer than {MIN_FILES} files):") print(f" {', '.join(sorted(oleft))}") if "--write" in sys.argv: write_object_styles(okept, oorder, min(doms)) print(f"wrote {OBJECT_OUT.relative_to(ROOT)}") write(kept, order, min(doms)) print(f"\nwrote {OUT.relative_to(ROOT)}") write_preferences(tags, pkept, porder, min(doms)) print(f"wrote {PREF_OUT.relative_to(ROOT)}") if __name__ == "__main__": main()