From c9528b10ff56f063d3a33bbef4200712bb333a55 Mon Sep 17 00:00:00 2001 From: aiongg <67543387+aiongg@users.noreply.github.com> Date: Wed, 7 Oct 2026 09:39:02 +0000 Subject: [PATCH] feat(tools): list reproduced values per key and dump whole chunks compare.py writes values.tsv and values-all.tsv with every key's values and how many are reproduced, so a change can be checked for keys that lose reproduced values. indd dump --full prints every byte of each chunk. root_values.py reads the current inventory layout and keeps to the pairs its generated files were made from. --- docs/format/idml-values.md | 8 ++++++-- docs/measurement.md | 5 +++++ src/main.rs | 18 +++++++++++++----- tools/compare.py | 16 +++++++++++++++- tools/root_values.py | 18 +++++++++++++----- 5 files changed, 52 insertions(+), 13 deletions(-) diff --git a/docs/format/idml-values.md b/docs/format/idml-values.md index acdd69c..aae539f 100644 --- a/docs/format/idml-values.md +++ b/docs/format/idml-values.md @@ -33,11 +33,15 @@ character root styles, attributes in the text attribute table (`attributes.md`) are left out of the file, and any attribute the INDD supplies is written from the INDD. -The corpus has 240 INDD files with an IDML, which are 88 distinct IDML +The files were generated from the 240 INDD files with an IDML that the +corpus had then (`SOURCE` in the tool), which are 88 distinct IDML files. Their DOM versions are 7 (1 file), 12 (7 files) and 15 to 20 (80 files). Values that first appear after DOM 7 are written only for documents of that InDesign version or later (`MinimumVersion` in the -file). +file). The corpus has grown since; the later pairs are evidence for the +values of other elements (below). Regenerating these files from all pairs +would drop some values that the larger corpus shows to vary between +documents, such as table style stroke weights. | Root style | Values | Same in | Written for version | |---|---|---|---| diff --git a/docs/measurement.md b/docs/measurement.md index f389b39..3a1c2fd 100644 --- a/docs/measurement.md +++ b/docs/measurement.md @@ -155,6 +155,11 @@ number of documents affected and the number of values wrong and missing. `compare.py` prints the top 30 keys by documents affected (`--gaps N`) and the top 15 by values. +`values.tsv` and `values-all.tsv` list every key with its number of +values and how many are reproduced, wrong and missing. Comparing them +between two runs shows whether a change reproduces fewer values of any +key. + ## Audit: what the converter does not read `indd audit ` needs no reference IDML. It converts the file while diff --git a/src/main.rs b/src/main.rs index dd9ba7d..19fb979 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,7 +7,9 @@ const USAGE: &str = "usage: indd objects one line per object: UID, class, length, first bytes indd uids one line per object: UID, class (no object data read) indd object write one object's bytes to stdout - indd dump ... print objects' chunks in hex + indd dump [--full] ... + print objects' chunks in hex (the first + 120 bytes of each, or all with --full) indd xmp write the document's XMP packet to stdout indd audit [--tsv] what the converter does not read in a document"; @@ -21,11 +23,16 @@ fn main() -> ExitCode { (Some("uids"), 2) => uids(&args[1]), (Some("audit"), 2) => audit(&args[1], false), (Some("audit"), 3) if args[1] == "--tsv" => audit(&args[2], true), + (Some("dump"), n) if n > 3 && args[1] == "--full" => args[3..] + .iter() + .map(|a| parse_uid(a)) + .collect::, _>>() + .and_then(|uids| dump(&args[2], &uids, usize::MAX)), (Some("dump"), n) if n > 2 => args[2..] .iter() .map(|a| parse_uid(a)) .collect::, _>>() - .and_then(|uids| dump(&args[1], &uids)), + .and_then(|uids| dump(&args[1], &uids, 120)), (Some("object"), 3) => match args[2].parse() { Ok(uid) => object(&args[1], uid), Err(_) => Err(format!("not a UID: {}", args[2]).into()), @@ -260,7 +267,8 @@ fn parse_uid(s: &str) -> Result> { parsed.map_err(|_| format!("not a UID: {s}").into()) } -fn dump(path: &str, uids: &[u32]) -> CliResult { +/// Print the chunks of objects, at most `limit` bytes of each. +fn dump(path: &str, uids: &[u32], limit: usize) -> CliResult { let bytes = std::fs::read(path)?; let c = indd::Container::parse(&bytes)?; let _order = indd::object::use_byte_order(c.header.byte_order); @@ -284,10 +292,10 @@ fn dump(path: &str, uids: &[u32]) -> CliResult { let hex: Vec = ch .data .iter() - .take(120) + .take(limit) .map(|b| format!("{b:02x}")) .collect(); - let more = if ch.data.len() > 120 { " ..." } else { "" }; + let more = if ch.data.len() > limit { " ..." } else { "" }; writeln!( out, " {:#07x} {:5}: {}{more}", diff --git a/tools/compare.py b/tools/compare.py index 9103e40..f2dc87d 100644 --- a/tools/compare.py +++ b/tools/compare.py @@ -34,7 +34,8 @@ Last comes the headline (docs/measurement.md): value coverage, the share of reference values the conversion reproduces, over trustworthy pairs and over all pairs; document scores (pairs with coverage >= 99 % and >= 99.9 %); and the biggest gaps (--gaps N). pairs.tsv, gaps.tsv and gaps-all.tsv go to ---out (default target/compare). +--out (default target/compare), with values.tsv and values-all.tsv, which +list every key with its values and how many are reproduced. --shortfalls N lists the N element types and attributes that fall short (missing elements; wrong or missing attribute values; differing story @@ -746,6 +747,17 @@ def write_gaps(path, st): f.write(f"{key[0]}\t{key[1]}\t{docs}\t{w}\t{m}\t{st.gap_total[key]}\n") +def write_values(path, st): + """Every key with its values and how many are reproduced, for checking + that a change reproduces no fewer values of any key.""" + with open(path, "w") as f: + f.write("tag\tkey\tvalues\treproduced\twrong\tmissing\n") + for key in sorted(st.gap_total): + w, m = st.gap_wrong[key], st.gap_missing[key] + f.write(f"{key[0]}\t{key[1]}\t{st.gap_total[key]}\t{st.gap_total[key] - w - m}" + f"\t{w}\t{m}\n") + + def headline(args, every, trusted, pair_rows): """Print value coverage, document scores and the biggest gaps; write pairs.tsv, gaps.tsv (trustworthy pairs) and gaps-all.tsv to --out.""" @@ -758,6 +770,8 @@ def headline(args, every, trusted, pair_rows): f"\t{'; '.join(reasons)}\n") write_gaps(out / "gaps.tsv", trusted) write_gaps(out / "gaps-all.tsv", every) + write_values(out / "values.tsv", trusted) + write_values(out / "values-all.tsv", every) print("\nheadline (docs/measurement.md):") for label, st in (("trustworthy pairs", trusted), ("all pairs", every)): n = st.docs or 1 diff --git a/tools/root_values.py b/tools/root_values.py index c9b6a00..6b7b7bf 100644 --- a/tools/root_values.py +++ b/tools/root_values.py @@ -38,6 +38,12 @@ OUT = ROOT / "src" / "idml" / "root_values.xml" PREF_OUT = ROOT / "src" / "idml" / "preference_values.xml" OBJECT_OUT = ROOT / "src" / "idml" / "object_style_values.xml" +# The generated files were made from the corpus pairs under this +# directory, the whole corpus at the time. Later pairs are evidence for +# tools/element_values.py; regenerating these files from them would drop +# values that the larger corpus shows to vary (docs/format/idml-values.md). +SOURCE = "third-party/" + # Preference attributes the converter reads from the INDD. PREF_SKIP = { ("DocumentPreference", "PageHeight"), ("DocumentPreference", "PageWidth"), @@ -97,14 +103,16 @@ def decoded_names(): def corpus_idmls(): - rows = (ROOT / "corpus" / "inventory.tsv").read_text().splitlines()[1:] + lines = (ROOT / "corpus" / "inventory.tsv").read_text().splitlines() + cols = lines[0].split("\t") seen, out, pairs = set(), [], 0 - for row in rows: - path, _size, _valid, _order, _ver, _creator, dom = row.split("\t") - if not dom: + for line in lines[1:]: + row = dict(zip(cols, line.split("\t"))) + path, dom = row["path"], row["idml_dom"] + if not dom or not dom[0].isdigit() or not path.startswith(SOURCE): continue pairs += 1 - idml = (ROOT / "corpus" / path).with_suffix(".idml") + idml = ROOT / "corpus" / (row.get("idml") or str(Path(path).with_suffix(".idml"))) digest = hashlib.md5(idml.read_bytes()).hexdigest() if digest not in seen: seen.add(digest)