mirror of
https://github.com/storytold/indd-utils.git
synced 2026-11-20 09:10:47 +01:00
feat(tools): list reproduced values per key and dump whole chunks
compare.py writes values.tsv and values-all.tsv with every key's values and how many are reproduced, so a change can be checked for keys that lose reproduced values. indd dump --full prints every byte of each chunk. root_values.py reads the current inventory layout and keeps to the pairs its generated files were made from.
This commit is contained in:
1 parent
5543af5867
commit
c9528b10ff
5 files changed
+52
-13
No files matched your search
@@ -33,11 +33,15 @@ character root styles, attributes in the text attribute table
|
||||
(`attributes.md`) are left out of the file, and any attribute the INDD
|
||||
supplies is written from the INDD.
|
||||
|
||||
The corpus has 240 INDD files with an IDML, which are 88 distinct IDML
|
||||
The files were generated from the 240 INDD files with an IDML that the
|
||||
corpus had then (`SOURCE` in the tool), which are 88 distinct IDML
|
||||
files. Their DOM versions are 7 (1 file), 12 (7 files) and 15 to 20 (80
|
||||
files). Values that first appear after DOM 7 are written only for
|
||||
documents of that InDesign version or later (`MinimumVersion` in the
|
||||
file).
|
||||
file). The corpus has grown since; the later pairs are evidence for the
|
||||
values of other elements (below). Regenerating these files from all pairs
|
||||
would drop some values that the larger corpus shows to vary between
|
||||
documents, such as table style stroke weights.
|
||||
|
||||
| Root style | Values | Same in | Written for version |
|
||||
|---|---|---|---|
|
||||
|
||||
@@ -155,6 +155,11 @@ number of documents affected and the number of values wrong and missing.
|
||||
`compare.py` prints the top 30 keys by documents affected (`--gaps N`) and
|
||||
the top 15 by values.
|
||||
|
||||
`values.tsv` and `values-all.tsv` list every key with its number of
|
||||
values and how many are reproduced, wrong and missing. Comparing them
|
||||
between two runs shows whether a change reproduces fewer values of any
|
||||
key.
|
||||
|
||||
## Audit: what the converter does not read
|
||||
|
||||
`indd audit <file>` needs no reference IDML. It converts the file while
|
||||
|
||||
+13
-5
@@ -7,7 +7,9 @@ const USAGE: &str = "usage:
|
||||
indd objects <file.indd> one line per object: UID, class, length, first bytes
|
||||
indd uids <file.indd> one line per object: UID, class (no object data read)
|
||||
indd object <file.indd> <uid> write one object's bytes to stdout
|
||||
indd dump <file.indd> <uid>... print objects' chunks in hex
|
||||
indd dump [--full] <file.indd> <uid>...
|
||||
print objects' chunks in hex (the first
|
||||
120 bytes of each, or all with --full)
|
||||
indd xmp <file.indd> write the document's XMP packet to stdout
|
||||
indd audit [--tsv] <file.indd> what the converter does not read in a document";
|
||||
|
||||
@@ -21,11 +23,16 @@ fn main() -> ExitCode {
|
||||
(Some("uids"), 2) => uids(&args[1]),
|
||||
(Some("audit"), 2) => audit(&args[1], false),
|
||||
(Some("audit"), 3) if args[1] == "--tsv" => audit(&args[2], true),
|
||||
(Some("dump"), n) if n > 3 && args[1] == "--full" => args[3..]
|
||||
.iter()
|
||||
.map(|a| parse_uid(a))
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.and_then(|uids| dump(&args[2], &uids, usize::MAX)),
|
||||
(Some("dump"), n) if n > 2 => args[2..]
|
||||
.iter()
|
||||
.map(|a| parse_uid(a))
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.and_then(|uids| dump(&args[1], &uids)),
|
||||
.and_then(|uids| dump(&args[1], &uids, 120)),
|
||||
(Some("object"), 3) => match args[2].parse() {
|
||||
Ok(uid) => object(&args[1], uid),
|
||||
Err(_) => Err(format!("not a UID: {}", args[2]).into()),
|
||||
@@ -260,7 +267,8 @@ fn parse_uid(s: &str) -> Result<u32, Box<dyn std::error::Error>> {
|
||||
parsed.map_err(|_| format!("not a UID: {s}").into())
|
||||
}
|
||||
|
||||
fn dump(path: &str, uids: &[u32]) -> CliResult {
|
||||
/// Print the chunks of objects, at most `limit` bytes of each.
|
||||
fn dump(path: &str, uids: &[u32], limit: usize) -> CliResult {
|
||||
let bytes = std::fs::read(path)?;
|
||||
let c = indd::Container::parse(&bytes)?;
|
||||
let _order = indd::object::use_byte_order(c.header.byte_order);
|
||||
@@ -284,10 +292,10 @@ fn dump(path: &str, uids: &[u32]) -> CliResult {
|
||||
let hex: Vec<String> = ch
|
||||
.data
|
||||
.iter()
|
||||
.take(120)
|
||||
.take(limit)
|
||||
.map(|b| format!("{b:02x}"))
|
||||
.collect();
|
||||
let more = if ch.data.len() > 120 { " ..." } else { "" };
|
||||
let more = if ch.data.len() > limit { " ..." } else { "" };
|
||||
writeln!(
|
||||
out,
|
||||
" {:#07x} {:5}: {}{more}",
|
||||
|
||||
+15
-1
@@ -34,7 +34,8 @@ Last comes the headline (docs/measurement.md): value coverage, the share
|
||||
of reference values the conversion reproduces, over trustworthy pairs and
|
||||
over all pairs; document scores (pairs with coverage >= 99 % and >= 99.9 %);
|
||||
and the biggest gaps (--gaps N). pairs.tsv, gaps.tsv and gaps-all.tsv go to
|
||||
--out (default target/compare).
|
||||
--out (default target/compare), with values.tsv and values-all.tsv, which
|
||||
list every key with its values and how many are reproduced.
|
||||
|
||||
--shortfalls N lists the N element types and attributes that fall short
|
||||
(missing elements; wrong or missing attribute values; differing story
|
||||
@@ -746,6 +747,17 @@ def write_gaps(path, st):
|
||||
f.write(f"{key[0]}\t{key[1]}\t{docs}\t{w}\t{m}\t{st.gap_total[key]}\n")
|
||||
|
||||
|
||||
def write_values(path, st):
|
||||
"""Every key with its values and how many are reproduced, for checking
|
||||
that a change reproduces no fewer values of any key."""
|
||||
with open(path, "w") as f:
|
||||
f.write("tag\tkey\tvalues\treproduced\twrong\tmissing\n")
|
||||
for key in sorted(st.gap_total):
|
||||
w, m = st.gap_wrong[key], st.gap_missing[key]
|
||||
f.write(f"{key[0]}\t{key[1]}\t{st.gap_total[key]}\t{st.gap_total[key] - w - m}"
|
||||
f"\t{w}\t{m}\n")
|
||||
|
||||
|
||||
def headline(args, every, trusted, pair_rows):
|
||||
"""Print value coverage, document scores and the biggest gaps; write
|
||||
pairs.tsv, gaps.tsv (trustworthy pairs) and gaps-all.tsv to --out."""
|
||||
@@ -758,6 +770,8 @@ def headline(args, every, trusted, pair_rows):
|
||||
f"\t{'; '.join(reasons)}\n")
|
||||
write_gaps(out / "gaps.tsv", trusted)
|
||||
write_gaps(out / "gaps-all.tsv", every)
|
||||
write_values(out / "values.tsv", trusted)
|
||||
write_values(out / "values-all.tsv", every)
|
||||
print("\nheadline (docs/measurement.md):")
|
||||
for label, st in (("trustworthy pairs", trusted), ("all pairs", every)):
|
||||
n = st.docs or 1
|
||||
|
||||
+13
-5
@@ -38,6 +38,12 @@ OUT = ROOT / "src" / "idml" / "root_values.xml"
|
||||
PREF_OUT = ROOT / "src" / "idml" / "preference_values.xml"
|
||||
OBJECT_OUT = ROOT / "src" / "idml" / "object_style_values.xml"
|
||||
|
||||
# The generated files were made from the corpus pairs under this
|
||||
# directory, the whole corpus at the time. Later pairs are evidence for
|
||||
# tools/element_values.py; regenerating these files from them would drop
|
||||
# values that the larger corpus shows to vary (docs/format/idml-values.md).
|
||||
SOURCE = "third-party/"
|
||||
|
||||
# Preference attributes the converter reads from the INDD.
|
||||
PREF_SKIP = {
|
||||
("DocumentPreference", "PageHeight"), ("DocumentPreference", "PageWidth"),
|
||||
@@ -97,14 +103,16 @@ def decoded_names():
|
||||
|
||||
|
||||
def corpus_idmls():
|
||||
rows = (ROOT / "corpus" / "inventory.tsv").read_text().splitlines()[1:]
|
||||
lines = (ROOT / "corpus" / "inventory.tsv").read_text().splitlines()
|
||||
cols = lines[0].split("\t")
|
||||
seen, out, pairs = set(), [], 0
|
||||
for row in rows:
|
||||
path, _size, _valid, _order, _ver, _creator, dom = row.split("\t")
|
||||
if not dom:
|
||||
for line in lines[1:]:
|
||||
row = dict(zip(cols, line.split("\t")))
|
||||
path, dom = row["path"], row["idml_dom"]
|
||||
if not dom or not dom[0].isdigit() or not path.startswith(SOURCE):
|
||||
continue
|
||||
pairs += 1
|
||||
idml = (ROOT / "corpus" / path).with_suffix(".idml")
|
||||
idml = ROOT / "corpus" / (row.get("idml") or str(Path(path).with_suffix(".idml")))
|
||||
digest = hashlib.md5(idml.read_bytes()).hexdigest()
|
||||
if digest not in seen:
|
||||
seen.add(digest)
|
||||
|
||||
Reference in new issue
Block a user