From f7fe41e65de817ccf79a741e4cd1d024b3dac047 Mon Sep 17 00:00:00 2001 From: aiongg <67543387+aiongg@users.noreply.github.com> Date: Thu, 8 Oct 2026 06:09:17 +0000 Subject: [PATCH] feat: write the index, its topics and the page reference topics The index object (class 0x13004) lists its sections (class 0x13005); chunk 0x13007 of a section holds its topic records depth first, with their level, name, sort order and page references. The converter writes the Index element in designmap.xml with its nested Topic elements, and gives each PageReference its ReferencedTopic and PageReferenceType. Evidence: docs/format/index.md. compare.py --exclude own/: - trustworthy value coverage 97.17% -> 97.36% (+21,617 values); all pairs 95.21% -> 95.35% (+21,808) - Topic 2,185 of 2,185 with Name and SortOrder; Index 16 of 16; PageReference ReferencedTopic and PageReferenceType 7,523 of 7,523 - extra values +15: the 3 topics and 3 page references one INDD has and its earlier IDML lacks - compare.py --all with schemas: 0 schema failures on pairs; files without a reference as before. One of them (DOM 19) has topic records of another length; its sections are left out with a warning (21 warnings). --- docs/format/index.md | 95 +++++++++++++++++++++ docs/format/objects.md | 9 +- src/idml/designmap.rs | 55 ++++++++++++ src/idml/mod.rs | 4 + src/idml/story.rs | 14 ++- src/model/document.rs | 7 ++ src/model/index.rs | 190 +++++++++++++++++++++++++++++++++++++++++ src/model/mod.rs | 1 + src/model/story.rs | 42 +++++++-- 9 files changed, 399 insertions(+), 18 deletions(-) create mode 100644 docs/format/index.md create mode 100644 src/model/index.rs diff --git a/docs/format/index.md b/docs/format/index.md new file mode 100644 index 0000000..2a02bbf --- /dev/null +++ b/docs/format/index.md @@ -0,0 +1,95 @@ +# Index + +The index of a document: the index object, its sections and topics, and +the page references in the text. Little-endian files; no big-endian +sample has an index. A "u32 string" is a u32 length in UTF-16 units, +then text segments. + +## Index object (class 0x13004) + +A document with an index has one object of class 0x13004. IDML writes +an `Index` element in `designmap.xml` with `Self` `u` and the UID in +hexadecimal, after the `CrossReferenceFormat` elements and before +`idPkg:BackingStory`. Evidence: 17 of 17 pairs with an index (16 +trustworthy); no other pair has either. + +Chunk 0x13005: u16 0, u16 *n*, *n* UIDs of section objects (class +0x13005), u16 0. *n* is 27 in most pairs and 60 in a pair with a +Cyrillic group. Chunk 0x1300E holds the index's own copy of the header +groups; IDML takes the header list from the preferences +(`preferences.md`, index header setting). + +## Sections (class 0x13005) and topics + +Chunk 0x13006 of a section is a u32, the index UID. Chunk 0x13007 holds +the section's topics; sections without topics do not have it: + +| Field | Contents | +|---|---| +| u32 | Number of level-1 topics | +| u16 | 0 | +| u32 | Number of topic records that follow (all levels) | +| records | Depth first: each topic is followed by its subtopics | + +Topic record: + +| Field | Contents | IDML `Topic` | +|---|---|---| +| u8, u8 | 0, 1 | | +| u16 | Level: 1 for a top topic, 2 for its subtopic, … | nesting | +| u32 | Number of direct subtopics | | +| u32 string | Topic text | `Name` | +| u32 string | Sort order (empty in every sample) | `SortOrder` | +| u32 *m*, *m* UIDs | The page references (class 0x13006) of this topic | their `ReferencedTopic` | +| 8 bytes | 0 in every sample | | + +- IDML writes the topics as `Topic` children of `Index`, a subtopic as a + child of its topic: sections in the order of chunk 0x13005, records in + stored order. +- `Self` is the parent's `Self`, `Topicn` and the topic text, so a + level-2 topic has `uTopicnTopicn`. +- Evidence: every chunk of the pairs is read to its end. The topics + (`Self`, `Name`, `SortOrder`, nesting) equal the IDML in 2,185 of + 2,185 IDML topics of the trustworthy pairs (2,223 in all pairs). In + one trustworthy pair the INDD has 3 more topics, and 3 more page + references, than its IDML (below). + +## Page references (class 0x13006) + +A page reference is owned by a U+FEFF in the story text (owned items, +`objects.md`). IDML writes a `PageReference` in place of the character. + +| Chunk | Contents | +|---|---| +| 0x13053 | u32: the owned-items strand that holds the marker | +| 0x13008 | u16 1, u32 the section that holds the topic | +| 0x13009 | 30 bytes (26 in InDesign 8.0): below | + +Chunk 0x13009: + +| Offset | Contents | IDML | +|---|---|---| +| 0–13 | u32 1, u16 0, u32 1, u32 0 in every sample | `PageReferenceType="CurrentPage"` (every IDML page reference) | +| 14, 18 | two equal u32 (0 to 8, or 0xFFFFFFFF) | not written | +| 22 | u32 (1 or 9) | not written | +| 26 | u32, only in a 30-byte chunk | `Id` | + +- `Self` is `u` and the UID in hexadecimal. +- `ReferencedTopic` is the `Self` of the topic whose record lists the + page reference. +- The 3 page references of the one InDesign 8.0 pair have the 26-byte + chunk, and their IDML has no `Id`. + +Evidence: all four attributes equal the IDML for 7,523 of 7,523 page +references of the trustworthy pairs (7,561 in all pairs; `Id` for 7,520 +and 7,558, the others have the 26-byte chunk). The converter writes +`PageReferenceType` only when the first 14 bytes are those of the +samples. + +## Topics missing from the IDML + +In one trustworthy pair (DOM 14) the INDD has 3 topics and 3 page +references that its IDML lacks; the IDML text also lacks their 3 +U+FEFF characters. Their `Id`s are the 3 highest of the document. The +IDML was most likely exported before these entries were made. The +converter writes them. diff --git a/docs/format/objects.md b/docs/format/objects.md index 7f1a00c..30aa225 100644 --- a/docs/format/objects.md +++ b/docs/format/objects.md @@ -1227,14 +1227,7 @@ only. **Index markers (class 0x13006).** An index marker is owned by a U+FEFF, which is not text in IDML: IDML writes a `PageReference` in its place. -`Self` is `u` and the UID in hexadecimal (7,495 of 7,495 in the -research run). `Id` is the u32 at offset 26 of chunk 0x13009 when the -chunk has 30 bytes (7,558 of 7,558 over all pairs); the IDML has no -`Id` for the 3 markers whose chunk has 26 bytes. `PageReferenceType` -(`CurrentPage` in all) has no located field, and `ReferencedTopic` names -an index topic, which the converter does not write yet; both are left -out (the schema allows that). Chunk 0x13008 is u16 1 and the topic UID -(class 0x13005). +Its layout and attributes are in `index.md`. **Anchored object settings (chunk 0x2800).** The anchor object (class 0x262) and object styles have a 62-byte chunk 0x2800: diff --git a/src/idml/designmap.rs b/src/idml/designmap.rs index 42e912f..d020cbb 100644 --- a/src/idml/designmap.rs +++ b/src/idml/designmap.rs @@ -264,6 +264,9 @@ impl Writer<'_> { ); } self.cross_reference_formats(&mut x); + if let Some(index) = &doc.index { + index_xml(&mut x, index); + } x.empty( "idPkg:BackingStory", &[("src", "XML/BackingStory.xml".into())], @@ -922,3 +925,55 @@ impl Writer<'_> { x.end(); } } + +/// The `Self` of a topic: its parent's `Self`, `Topicn` and its name +/// (`docs/format/index.md`). +fn topic_self(parent: &str, t: &crate::model::index::Topic) -> String { + format!("{parent}Topicn{}", t.name) +} + +/// The `Index` element and its topics. +fn index_xml(x: &mut Xml, index: &crate::model::index::Index) { + fn topic(x: &mut Xml, parent: &str, t: &crate::model::index::Topic) { + let me = topic_self(parent, t); + x.start("Topic") + .attr("Self", &me) + .attr("SortOrder", &t.sort_order) + .attr("Name", &t.name); + for c in &t.children { + topic(x, &me, c); + } + x.end(); + } + let me = uref(Some(index.uid)); + x.start("Index").attr("Self", &me); + for t in &index.topics { + topic(x, &me, t); + } + x.end(); +} + +/// The `Self` of the topic of each page reference, by page reference UID. +pub(super) fn topic_refs(doc: &Document) -> std::collections::HashMap { + fn walk( + out: &mut std::collections::HashMap, + parent: &str, + t: &crate::model::index::Topic, + ) { + let me = topic_self(parent, t); + for &r in &t.refs { + out.insert(r, me.clone()); + } + for c in &t.children { + walk(out, &me, c); + } + } + let mut out = std::collections::HashMap::new(); + if let Some(index) = &doc.index { + let me = uref(Some(index.uid)); + for t in &index.topics { + walk(&mut out, &me, t); + } + } + out +} diff --git a/src/idml/mod.rs b/src/idml/mod.rs index 140c580..1cb3881 100644 --- a/src/idml/mod.rs +++ b/src/idml/mod.rs @@ -74,6 +74,8 @@ struct Writer<'a> { /// Effective values of object styles, by UID (`applied`). style_values: std::cell::RefCell>>, + /// The `Self` of the index topic of each page reference UID. + topics: std::collections::HashMap, } /// Attributes observed on an element path (`values::element_attrs`). @@ -146,6 +148,7 @@ pub fn write(doc: &Document, name: &str, out: impl std::io::Write) -> std::io::R style_values: Default::default(), page_layouts: alternate_layouts(doc).1, page_sections: page_sections(doc), + topics: designmap::topic_refs(doc), }; let mut files: BTreeMap = BTreeMap::new(); files.insert( @@ -225,6 +228,7 @@ impl<'a> Writer<'a> { style_values: Default::default(), page_layouts: Vec::new(), page_sections: Vec::new(), + topics: Default::default(), } } } diff --git a/src/idml/story.rs b/src/idml/story.rs index 321a1a5..88734b4 100644 --- a/src/idml/story.rs +++ b/src/idml/story.rs @@ -369,15 +369,21 @@ impl Writer<'_> { } } // An index marker is written as a `PageReference` in place of - // its character (objects.md, notes and index markers). + // its character (index.md, page references). if ch == '\u{FEFF}' && !story.xml_markers.contains_key(&pos) - && let Some(&(uid, id)) = story.index_markers.get(&pos) + && let Some(m) = story.index_markers.get(&pos) { flush(x, &mut buf); st.open_csr(self, x, run); - x.start("PageReference").attr("Self", uref(Some(uid))); - if let Some(id) = id { + x.start("PageReference").attr("Self", uref(Some(m.uid))); + if m.current_page { + x.attr("PageReferenceType", "CurrentPage"); + } + if let Some(t) = self.topics.get(&m.uid) { + x.attr("ReferencedTopic", t); + } + if let Some(id) = m.id { x.attr("Id", id.to_string()); } x.end(); diff --git a/src/model/document.rs b/src/model/document.rs index a570a13..deace02 100644 --- a/src/model/document.rs +++ b/src/model/document.rs @@ -23,6 +23,8 @@ pub struct Document { /// Index sort groups (preferences chunk 0x1307E), in order: name, /// include flag and header variant. pub index_groups: Vec<(String, bool, u16)>, + /// The index (class 0x13004) and its topics. + pub index: Option, /// The constant shade the document lists (the class 0x5533 object of /// lowest UID with a constant shade in chunk 0x5532): its UID and its /// u32 and three f64; and its name (chunk 0x5531: a flag byte, 1 for @@ -171,6 +173,10 @@ impl<'a> Reader<'a> { let toc_styles = self.toc_styles(); let named_grids = self.named_grids(); let index_groups = self.index_groups(); + let index = self.index().unwrap_or_else(|e| { + self.warn(format!("index left out: {e}")); + None + }); let constant_shade = self.constant_shade(); let assignments = self.assignments(); let users = self.users(DOC).unwrap_or_else(|e| { @@ -222,6 +228,7 @@ impl<'a> Reader<'a> { toc_styles, named_grids, index_groups, + index, constant_shade, assignments, style_groups: objects.style_groups, diff --git a/src/model/index.rs b/src/model/index.rs new file mode 100644 index 0000000..8e6f68b --- /dev/null +++ b/src/model/index.rs @@ -0,0 +1,190 @@ +//! The index: one object of class 0x13004, its sections (class 0x13005) +//! and the topic records inside them. See `docs/format/index.md`. + +use super::Reader; +use crate::Error; +use crate::object::Cursor; + +/// Index object, section and page reference classes. +pub mod class { + pub const INDEX: u32 = 0x13004; + pub const SECTION: u32 = 0x13005; + pub const PAGE_REFERENCE: u32 = 0x13006; +} + +mod chunk { + /// Index: its sections. + pub const SECTIONS: u32 = 0x13005; + /// Section: its topic records. + pub const TOPICS: u32 = 0x13007; +} + +/// The document's index. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct Index { + pub uid: u32, + /// The level-1 topics, section by section. + pub topics: Vec, +} + +/// An index topic. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct Topic { + pub name: String, + pub sort_order: String, + /// The page references (class 0x13006) of this topic. + pub refs: Vec, + pub children: Vec, +} + +/// A u32 length in UTF-16 units, then text segments. +fn counted(c: &mut Cursor) -> Result { + let n = c.u32()? as usize; + if n == 0 { + Ok(String::new()) + } else { + c.segments(n) + } +} + +/// The topic records of a section (chunk 0x13007), as a tree. The +/// records are depth first; each has its level (1 for a top topic). +fn read_topics(c: &mut Cursor) -> Result, Error> { + let top = c.u32()? as usize; + c.u16()?; + let n = c.u32()? as usize; + // The open topics, one per level. + let mut stack: Vec = Vec::new(); + let mut out = Vec::new(); + // Close open topics until `depth` remain. + fn close(stack: &mut Vec, out: &mut Vec, depth: usize) { + while stack.len() > depth { + let Some(t) = stack.pop() else { return }; + match stack.last_mut() { + Some(p) => p.children.push(t), + None => out.push(t), + } + } + } + for _ in 0..n { + c.u16()?; + let level = c.u16()? as usize; + if level == 0 || level > stack.len() + 1 { + return Err(Error::Corrupt(format!("index topic of level {level}"))); + } + c.u32()?; + let name = counted(c)?; + let sort_order = counted(c)?; + let m = c.u32()? as usize; + if m > c.remaining() / 4 { + return Err(Error::Corrupt(format!("{m} page references"))); + } + let refs = (0..m).map(|_| c.u32()).collect::>()?; + c.skip(8)?; + close(&mut stack, &mut out, level - 1); + stack.push(Topic { + name, + sort_order, + refs, + children: Vec::new(), + }); + } + close(&mut stack, &mut out, 0); + if out.len() != top || c.remaining() != 0 { + return Err(Error::Corrupt(format!( + "{} top topics, {top} expected, {} bytes left", + out.len(), + c.remaining() + ))); + } + Ok(out) +} + +impl Reader<'_> { + /// The document's index, if it has one. A section whose topics cannot + /// be read is left out with a warning. + pub(super) fn index(&self) -> Result, Error> { + if self.enc().big_endian() { + return Ok(None); + } + let mut found = self + .db + .classes() + .iter() + .filter(|(_, c)| *c == class::INDEX) + .map(|&(u, _)| u); + let Some(uid) = found.next() else { + return Ok(None); + }; + if found.next().is_some() { + self.warn("more than one index; only the first is written".into()); + } + let Some(d) = self.chunk(uid, chunk::SECTIONS)? else { + return Ok(Some(Index { + uid, + topics: Vec::new(), + })); + }; + let mut c = self.cursor(&d); + c.u16()?; + let n = c.u16()? as usize; + let sections: Vec = (0..n).map(|_| c.u32()).collect::>()?; + let mut topics = Vec::new(); + for s in sections { + if self.class(s) != Some(class::SECTION) { + continue; + } + if let Some(d) = self.chunk(s, chunk::TOPICS)? { + match read_topics(&mut self.cursor(&d)) { + Ok(t) => topics.extend(t), + Err(e) => self.warn(format!("index section {s} left out: {e}")), + } + } + } + Ok(Some(Index { uid, topics })) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::object::Encoding; + + fn record(d: &mut Vec, level: u16, subs: u32, name: &str, refs: &[u32]) { + d.extend([0, 1]); + d.extend(level.to_le_bytes()); + d.extend(subs.to_le_bytes()); + d.extend((name.len() as u32).to_le_bytes()); + d.extend((0x4000 | name.len() as u16).to_le_bytes()); + d.extend(name.as_bytes()); + d.extend(0u32.to_le_bytes()); + d.extend((refs.len() as u32).to_le_bytes()); + for r in refs { + d.extend(r.to_le_bytes()); + } + d.extend([0; 8]); + } + + #[test] + fn builds_the_topic_tree() { + let mut d = Vec::new(); + d.extend(2u32.to_le_bytes()); + d.extend(0u16.to_le_bytes()); + d.extend(4u32.to_le_bytes()); + record(&mut d, 1, 1, "a", &[7]); + record(&mut d, 2, 1, "b", &[]); + record(&mut d, 3, 0, "c", &[8, 9]); + record(&mut d, 1, 0, "d", &[]); + let enc = Encoding::default(); + let t = read_topics(&mut enc.cursor(&d)).unwrap(); + assert_eq!(t.len(), 2); + assert_eq!((t[0].name.as_str(), &t[0].refs[..]), ("a", &[7][..])); + assert_eq!(t[0].children[0].children[0].refs, vec![8, 9]); + assert_eq!(t[1].name, "d"); + // A level that skips one is an error. + let mut bad = d[..10].to_vec(); + bad[6..10].copy_from_slice(&1u32.to_le_bytes()); + record(&mut bad, 2, 0, "x", &[]); + assert!(read_topics(&mut enc.cursor(&bad)).is_err()); + } +} diff --git a/src/model/mod.rs b/src/model/mod.rs index cdbd458..41e22ac 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -14,6 +14,7 @@ mod document; pub mod font; pub mod hyperlink; mod ids; +pub mod index; mod item; pub mod prefs; mod reader; diff --git a/src/model/story.rs b/src/model/story.rs index 944ba25..21d46a1 100644 --- a/src/model/story.rs +++ b/src/model/story.rs @@ -20,6 +20,18 @@ pub struct TextRun { pub character_attrs: Attrs, } +/// An index marker: a page reference (class 0x13006) owned by a U+FEFF. +/// See `docs/format/index.md`. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct IndexMarker { + pub uid: u32, + /// `Id`: the u32 at 26 of a 30-byte chunk 0x13009. + pub id: Option, + /// The first 14 bytes of chunk 0x13009 are those of every sample, + /// whose IDML has `PageReferenceType="CurrentPage"`. + pub current_page: bool, +} + #[derive(Debug, Clone, PartialEq)] pub struct Story { pub uid: u32, @@ -45,8 +57,8 @@ pub struct Story { /// sorted by start. pub changes: Vec, /// Index markers (`PageReference`), by UTF-16 offset of their - /// U+FEFF: UID and `Id`. - pub index_markers: BTreeMap)>, + /// U+FEFF. + pub index_markers: BTreeMap, /// Endnote references, by UTF-16 offset of their U+0004: the endnote /// UID and the UID of its range. pub endnotes: BTreeMap, @@ -502,11 +514,29 @@ impl<'a> Reader<'a> { class::INDEX_MARKER => { // `Id` is the last u32 of chunk 0x13009 when the chunk // has 30 bytes; IDML has no `Id` for a 26-byte chunk. - let id = self - .chunk(item, chunk::INDEX_MARKER)? + // Its first 14 bytes give the type (index.md). + let d = self.chunk(item, chunk::INDEX_MARKER)?; + let id = d + .as_deref() .filter(|d| d.len() == 30) - .and_then(|d| self.enc().u32_at(&d, 26)); - index_markers.insert(pos, (item, id)); + .and_then(|d| self.enc().u32_at(d, 26)); + let current_page = d.as_deref().is_some_and(|d| { + let e = self.enc(); + ( + e.u32_at(d, 0), + e.u16_at(d, 4), + e.u32_at(d, 6), + e.u32_at(d, 10), + ) == (Some(1), Some(0), Some(1), Some(0)) + }); + index_markers.insert( + pos, + IndexMarker { + uid: item, + id, + current_page, + }, + ); } class::FOOTNOTE => { footnotes.insert(