feat(tools): list reproduced values per key and dump whole chunks

compare.py writes values.tsv and values-all.tsv with every key's values and
how many are reproduced, so a change can be checked for keys that lose
reproduced values. indd dump --full prints every byte of each chunk.
root_values.py reads the current inventory layout and keeps to the pairs
its generated files were made from.
This commit is contained in:
aiongg committed 2026-10-07 09:39:02 +00:00
1 parent 5543af5867
commit c9528b10ff
5 files changed
+52 -13

No files matched your search

+6 -2
View File
@@ -33,11 +33,15 @@ character root styles, attributes in the text attribute table
(`attributes.md`) are left out of the file, and any attribute the INDD
supplies is written from the INDD.
The corpus has 240 INDD files with an IDML, which are 88 distinct IDML
The files were generated from the 240 INDD files with an IDML that the
corpus had then (`SOURCE` in the tool), which are 88 distinct IDML
files. Their DOM versions are 7 (1 file), 12 (7 files) and 15 to 20 (80
files). Values that first appear after DOM 7 are written only for
documents of that InDesign version or later (`MinimumVersion` in the
file).
file). The corpus has grown since; the later pairs are evidence for the
values of other elements (below). Regenerating these files from all pairs
would drop some values that the larger corpus shows to vary between
documents, such as table style stroke weights.
| Root style | Values | Same in | Written for version |
|---|---|---|---|
+5
View File
@@ -155,6 +155,11 @@ number of documents affected and the number of values wrong and missing.
`compare.py` prints the top 30 keys by documents affected (`--gaps N`) and
the top 15 by values.
`values.tsv` and `values-all.tsv` list every key with its number of
values and how many are reproduced, wrong and missing. Comparing them
between two runs shows whether a change reproduces fewer values of any
key.
## Audit: what the converter does not read
`indd audit <file>` needs no reference IDML. It converts the file while
+13 -5
View File
@@ -7,7 +7,9 @@ const USAGE: &str = "usage:
indd objects <file.indd> one line per object: UID, class, length, first bytes
indd uids <file.indd> one line per object: UID, class (no object data read)
indd object <file.indd> <uid> write one object's bytes to stdout
indd dump <file.indd> <uid>... print objects' chunks in hex
indd dump [--full] <file.indd> <uid>...
print objects' chunks in hex (the first
120 bytes of each, or all with --full)
indd xmp <file.indd> write the document's XMP packet to stdout
indd audit [--tsv] <file.indd> what the converter does not read in a document";
@@ -21,11 +23,16 @@ fn main() -> ExitCode {
(Some("uids"), 2) => uids(&args[1]),
(Some("audit"), 2) => audit(&args[1], false),
(Some("audit"), 3) if args[1] == "--tsv" => audit(&args[2], true),
(Some("dump"), n) if n > 3 && args[1] == "--full" => args[3..]
.iter()
.map(|a| parse_uid(a))
.collect::<Result<Vec<_>, _>>()
.and_then(|uids| dump(&args[2], &uids, usize::MAX)),
(Some("dump"), n) if n > 2 => args[2..]
.iter()
.map(|a| parse_uid(a))
.collect::<Result<Vec<_>, _>>()
.and_then(|uids| dump(&args[1], &uids)),
.and_then(|uids| dump(&args[1], &uids, 120)),
(Some("object"), 3) => match args[2].parse() {
Ok(uid) => object(&args[1], uid),
Err(_) => Err(format!("not a UID: {}", args[2]).into()),
@@ -260,7 +267,8 @@ fn parse_uid(s: &str) -> Result<u32, Box<dyn std::error::Error>> {
parsed.map_err(|_| format!("not a UID: {s}").into())
}
fn dump(path: &str, uids: &[u32]) -> CliResult {
/// Print the chunks of objects, at most `limit` bytes of each.
fn dump(path: &str, uids: &[u32], limit: usize) -> CliResult {
let bytes = std::fs::read(path)?;
let c = indd::Container::parse(&bytes)?;
let _order = indd::object::use_byte_order(c.header.byte_order);
@@ -284,10 +292,10 @@ fn dump(path: &str, uids: &[u32]) -> CliResult {
let hex: Vec<String> = ch
.data
.iter()
.take(120)
.take(limit)
.map(|b| format!("{b:02x}"))
.collect();
let more = if ch.data.len() > 120 { " ..." } else { "" };
let more = if ch.data.len() > limit { " ..." } else { "" };
writeln!(
out,
" {:#07x} {:5}: {}{more}",
+15 -1
View File
@@ -34,7 +34,8 @@ Last comes the headline (docs/measurement.md): value coverage, the share
of reference values the conversion reproduces, over trustworthy pairs and
over all pairs; document scores (pairs with coverage >= 99 % and >= 99.9 %);
and the biggest gaps (--gaps N). pairs.tsv, gaps.tsv and gaps-all.tsv go to
--out (default target/compare).
--out (default target/compare), with values.tsv and values-all.tsv, which
list every key with its values and how many are reproduced.
--shortfalls N lists the N element types and attributes that fall short
(missing elements; wrong or missing attribute values; differing story
@@ -746,6 +747,17 @@ def write_gaps(path, st):
f.write(f"{key[0]}\t{key[1]}\t{docs}\t{w}\t{m}\t{st.gap_total[key]}\n")
def write_values(path, st):
"""Every key with its values and how many are reproduced, for checking
that a change reproduces no fewer values of any key."""
with open(path, "w") as f:
f.write("tag\tkey\tvalues\treproduced\twrong\tmissing\n")
for key in sorted(st.gap_total):
w, m = st.gap_wrong[key], st.gap_missing[key]
f.write(f"{key[0]}\t{key[1]}\t{st.gap_total[key]}\t{st.gap_total[key] - w - m}"
f"\t{w}\t{m}\n")
def headline(args, every, trusted, pair_rows):
"""Print value coverage, document scores and the biggest gaps; write
pairs.tsv, gaps.tsv (trustworthy pairs) and gaps-all.tsv to --out."""
@@ -758,6 +770,8 @@ def headline(args, every, trusted, pair_rows):
f"\t{'; '.join(reasons)}\n")
write_gaps(out / "gaps.tsv", trusted)
write_gaps(out / "gaps-all.tsv", every)
write_values(out / "values.tsv", trusted)
write_values(out / "values-all.tsv", every)
print("\nheadline (docs/measurement.md):")
for label, st in (("trustworthy pairs", trusted), ("all pairs", every)):
n = st.docs or 1
+13 -5
View File
@@ -38,6 +38,12 @@ OUT = ROOT / "src" / "idml" / "root_values.xml"
PREF_OUT = ROOT / "src" / "idml" / "preference_values.xml"
OBJECT_OUT = ROOT / "src" / "idml" / "object_style_values.xml"
# The generated files were made from the corpus pairs under this
# directory, the whole corpus at the time. Later pairs are evidence for
# tools/element_values.py; regenerating these files from them would drop
# values that the larger corpus shows to vary (docs/format/idml-values.md).
SOURCE = "third-party/"
# Preference attributes the converter reads from the INDD.
PREF_SKIP = {
("DocumentPreference", "PageHeight"), ("DocumentPreference", "PageWidth"),
@@ -97,14 +103,16 @@ def decoded_names():
def corpus_idmls():
rows = (ROOT / "corpus" / "inventory.tsv").read_text().splitlines()[1:]
lines = (ROOT / "corpus" / "inventory.tsv").read_text().splitlines()
cols = lines[0].split("\t")
seen, out, pairs = set(), [], 0
for row in rows:
path, _size, _valid, _order, _ver, _creator, dom = row.split("\t")
if not dom:
for line in lines[1:]:
row = dict(zip(cols, line.split("\t")))
path, dom = row["path"], row["idml_dom"]
if not dom or not dom[0].isdigit() or not path.startswith(SOURCE):
continue
pairs += 1
idml = (ROOT / "corpus" / path).with_suffix(".idml")
idml = ROOT / "corpus" / (row.get("idml") or str(Path(path).with_suffix(".idml")))
digest = hashlib.md5(idml.read_bytes()).hexdigest()
if digest not in seen:
seen.add(digest)