diff --git a/docs/format/index.md b/docs/format/index.md new file mode 100644 index 0000000..2a02bbf --- /dev/null +++ b/docs/format/index.md @@ -0,0 +1,95 @@ +# Index + +The index of a document: the index object, its sections and topics, and +the page references in the text. Little-endian files; no big-endian +sample has an index. A "u32 string" is a u32 length in UTF-16 units, +then text segments. + +## Index object (class 0x13004) + +A document with an index has one object of class 0x13004. IDML writes +an `Index` element in `designmap.xml` with `Self` `u` and the UID in +hexadecimal, after the `CrossReferenceFormat` elements and before +`idPkg:BackingStory`. Evidence: 17 of 17 pairs with an index (16 +trustworthy); no other pair has either. + +Chunk 0x13005: u16 0, u16 *n*, *n* UIDs of section objects (class +0x13005), u16 0. *n* is 27 in most pairs and 60 in a pair with a +Cyrillic group. Chunk 0x1300E holds the index's own copy of the header +groups; IDML takes the header list from the preferences +(`preferences.md`, index header setting). + +## Sections (class 0x13005) and topics + +Chunk 0x13006 of a section is a u32, the index UID. Chunk 0x13007 holds +the section's topics; sections without topics do not have it: + +| Field | Contents | +|---|---| +| u32 | Number of level-1 topics | +| u16 | 0 | +| u32 | Number of topic records that follow (all levels) | +| records | Depth first: each topic is followed by its subtopics | + +Topic record: + +| Field | Contents | IDML `Topic` | +|---|---|---| +| u8, u8 | 0, 1 | | +| u16 | Level: 1 for a top topic, 2 for its subtopic, … | nesting | +| u32 | Number of direct subtopics | | +| u32 string | Topic text | `Name` | +| u32 string | Sort order (empty in every sample) | `SortOrder` | +| u32 *m*, *m* UIDs | The page references (class 0x13006) of this topic | their `ReferencedTopic` | +| 8 bytes | 0 in every sample | | + +- IDML writes the topics as `Topic` children of `Index`, a subtopic as a + child of its topic: sections in the order of chunk 0x13005, records in + stored order. +- `Self` is the parent's `Self`, `Topicn` and the topic text, so a + level-2 topic has `uTopicnTopicn`. +- Evidence: every chunk of the pairs is read to its end. The topics + (`Self`, `Name`, `SortOrder`, nesting) equal the IDML in 2,185 of + 2,185 IDML topics of the trustworthy pairs (2,223 in all pairs). In + one trustworthy pair the INDD has 3 more topics, and 3 more page + references, than its IDML (below). + +## Page references (class 0x13006) + +A page reference is owned by a U+FEFF in the story text (owned items, +`objects.md`). IDML writes a `PageReference` in place of the character. + +| Chunk | Contents | +|---|---| +| 0x13053 | u32: the owned-items strand that holds the marker | +| 0x13008 | u16 1, u32 the section that holds the topic | +| 0x13009 | 30 bytes (26 in InDesign 8.0): below | + +Chunk 0x13009: + +| Offset | Contents | IDML | +|---|---|---| +| 0–13 | u32 1, u16 0, u32 1, u32 0 in every sample | `PageReferenceType="CurrentPage"` (every IDML page reference) | +| 14, 18 | two equal u32 (0 to 8, or 0xFFFFFFFF) | not written | +| 22 | u32 (1 or 9) | not written | +| 26 | u32, only in a 30-byte chunk | `Id` | + +- `Self` is `u` and the UID in hexadecimal. +- `ReferencedTopic` is the `Self` of the topic whose record lists the + page reference. +- The 3 page references of the one InDesign 8.0 pair have the 26-byte + chunk, and their IDML has no `Id`. + +Evidence: all four attributes equal the IDML for 7,523 of 7,523 page +references of the trustworthy pairs (7,561 in all pairs; `Id` for 7,520 +and 7,558, the others have the 26-byte chunk). The converter writes +`PageReferenceType` only when the first 14 bytes are those of the +samples. + +## Topics missing from the IDML + +In one trustworthy pair (DOM 14) the INDD has 3 topics and 3 page +references that its IDML lacks; the IDML text also lacks their 3 +U+FEFF characters. Their `Id`s are the 3 highest of the document. The +IDML was most likely exported before these entries were made. The +converter writes them. diff --git a/docs/format/objects.md b/docs/format/objects.md index 7f1a00c..30aa225 100644 --- a/docs/format/objects.md +++ b/docs/format/objects.md @@ -1227,14 +1227,7 @@ only. **Index markers (class 0x13006).** An index marker is owned by a U+FEFF, which is not text in IDML: IDML writes a `PageReference` in its place. -`Self` is `u` and the UID in hexadecimal (7,495 of 7,495 in the -research run). `Id` is the u32 at offset 26 of chunk 0x13009 when the -chunk has 30 bytes (7,558 of 7,558 over all pairs); the IDML has no -`Id` for the 3 markers whose chunk has 26 bytes. `PageReferenceType` -(`CurrentPage` in all) has no located field, and `ReferencedTopic` names -an index topic, which the converter does not write yet; both are left -out (the schema allows that). Chunk 0x13008 is u16 1 and the topic UID -(class 0x13005). +Its layout and attributes are in `index.md`. **Anchored object settings (chunk 0x2800).** The anchor object (class 0x262) and object styles have a 62-byte chunk 0x2800: diff --git a/src/idml/designmap.rs b/src/idml/designmap.rs index 42e912f..d020cbb 100644 --- a/src/idml/designmap.rs +++ b/src/idml/designmap.rs @@ -264,6 +264,9 @@ impl Writer<'_> { ); } self.cross_reference_formats(&mut x); + if let Some(index) = &doc.index { + index_xml(&mut x, index); + } x.empty( "idPkg:BackingStory", &[("src", "XML/BackingStory.xml".into())], @@ -922,3 +925,55 @@ impl Writer<'_> { x.end(); } } + +/// The `Self` of a topic: its parent's `Self`, `Topicn` and its name +/// (`docs/format/index.md`). +fn topic_self(parent: &str, t: &crate::model::index::Topic) -> String { + format!("{parent}Topicn{}", t.name) +} + +/// The `Index` element and its topics. +fn index_xml(x: &mut Xml, index: &crate::model::index::Index) { + fn topic(x: &mut Xml, parent: &str, t: &crate::model::index::Topic) { + let me = topic_self(parent, t); + x.start("Topic") + .attr("Self", &me) + .attr("SortOrder", &t.sort_order) + .attr("Name", &t.name); + for c in &t.children { + topic(x, &me, c); + } + x.end(); + } + let me = uref(Some(index.uid)); + x.start("Index").attr("Self", &me); + for t in &index.topics { + topic(x, &me, t); + } + x.end(); +} + +/// The `Self` of the topic of each page reference, by page reference UID. +pub(super) fn topic_refs(doc: &Document) -> std::collections::HashMap { + fn walk( + out: &mut std::collections::HashMap, + parent: &str, + t: &crate::model::index::Topic, + ) { + let me = topic_self(parent, t); + for &r in &t.refs { + out.insert(r, me.clone()); + } + for c in &t.children { + walk(out, &me, c); + } + } + let mut out = std::collections::HashMap::new(); + if let Some(index) = &doc.index { + let me = uref(Some(index.uid)); + for t in &index.topics { + walk(&mut out, &me, t); + } + } + out +} diff --git a/src/idml/mod.rs b/src/idml/mod.rs index 140c580..1cb3881 100644 --- a/src/idml/mod.rs +++ b/src/idml/mod.rs @@ -74,6 +74,8 @@ struct Writer<'a> { /// Effective values of object styles, by UID (`applied`). style_values: std::cell::RefCell>>, + /// The `Self` of the index topic of each page reference UID. + topics: std::collections::HashMap, } /// Attributes observed on an element path (`values::element_attrs`). @@ -146,6 +148,7 @@ pub fn write(doc: &Document, name: &str, out: impl std::io::Write) -> std::io::R style_values: Default::default(), page_layouts: alternate_layouts(doc).1, page_sections: page_sections(doc), + topics: designmap::topic_refs(doc), }; let mut files: BTreeMap = BTreeMap::new(); files.insert( @@ -225,6 +228,7 @@ impl<'a> Writer<'a> { style_values: Default::default(), page_layouts: Vec::new(), page_sections: Vec::new(), + topics: Default::default(), } } } diff --git a/src/idml/story.rs b/src/idml/story.rs index 321a1a5..88734b4 100644 --- a/src/idml/story.rs +++ b/src/idml/story.rs @@ -369,15 +369,21 @@ impl Writer<'_> { } } // An index marker is written as a `PageReference` in place of - // its character (objects.md, notes and index markers). + // its character (index.md, page references). if ch == '\u{FEFF}' && !story.xml_markers.contains_key(&pos) - && let Some(&(uid, id)) = story.index_markers.get(&pos) + && let Some(m) = story.index_markers.get(&pos) { flush(x, &mut buf); st.open_csr(self, x, run); - x.start("PageReference").attr("Self", uref(Some(uid))); - if let Some(id) = id { + x.start("PageReference").attr("Self", uref(Some(m.uid))); + if m.current_page { + x.attr("PageReferenceType", "CurrentPage"); + } + if let Some(t) = self.topics.get(&m.uid) { + x.attr("ReferencedTopic", t); + } + if let Some(id) = m.id { x.attr("Id", id.to_string()); } x.end(); diff --git a/src/model/document.rs b/src/model/document.rs index a570a13..deace02 100644 --- a/src/model/document.rs +++ b/src/model/document.rs @@ -23,6 +23,8 @@ pub struct Document { /// Index sort groups (preferences chunk 0x1307E), in order: name, /// include flag and header variant. pub index_groups: Vec<(String, bool, u16)>, + /// The index (class 0x13004) and its topics. + pub index: Option, /// The constant shade the document lists (the class 0x5533 object of /// lowest UID with a constant shade in chunk 0x5532): its UID and its /// u32 and three f64; and its name (chunk 0x5531: a flag byte, 1 for @@ -171,6 +173,10 @@ impl<'a> Reader<'a> { let toc_styles = self.toc_styles(); let named_grids = self.named_grids(); let index_groups = self.index_groups(); + let index = self.index().unwrap_or_else(|e| { + self.warn(format!("index left out: {e}")); + None + }); let constant_shade = self.constant_shade(); let assignments = self.assignments(); let users = self.users(DOC).unwrap_or_else(|e| { @@ -222,6 +228,7 @@ impl<'a> Reader<'a> { toc_styles, named_grids, index_groups, + index, constant_shade, assignments, style_groups: objects.style_groups, diff --git a/src/model/index.rs b/src/model/index.rs new file mode 100644 index 0000000..8e6f68b --- /dev/null +++ b/src/model/index.rs @@ -0,0 +1,190 @@ +//! The index: one object of class 0x13004, its sections (class 0x13005) +//! and the topic records inside them. See `docs/format/index.md`. + +use super::Reader; +use crate::Error; +use crate::object::Cursor; + +/// Index object, section and page reference classes. +pub mod class { + pub const INDEX: u32 = 0x13004; + pub const SECTION: u32 = 0x13005; + pub const PAGE_REFERENCE: u32 = 0x13006; +} + +mod chunk { + /// Index: its sections. + pub const SECTIONS: u32 = 0x13005; + /// Section: its topic records. + pub const TOPICS: u32 = 0x13007; +} + +/// The document's index. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct Index { + pub uid: u32, + /// The level-1 topics, section by section. + pub topics: Vec, +} + +/// An index topic. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct Topic { + pub name: String, + pub sort_order: String, + /// The page references (class 0x13006) of this topic. + pub refs: Vec, + pub children: Vec, +} + +/// A u32 length in UTF-16 units, then text segments. +fn counted(c: &mut Cursor) -> Result { + let n = c.u32()? as usize; + if n == 0 { + Ok(String::new()) + } else { + c.segments(n) + } +} + +/// The topic records of a section (chunk 0x13007), as a tree. The +/// records are depth first; each has its level (1 for a top topic). +fn read_topics(c: &mut Cursor) -> Result, Error> { + let top = c.u32()? as usize; + c.u16()?; + let n = c.u32()? as usize; + // The open topics, one per level. + let mut stack: Vec = Vec::new(); + let mut out = Vec::new(); + // Close open topics until `depth` remain. + fn close(stack: &mut Vec, out: &mut Vec, depth: usize) { + while stack.len() > depth { + let Some(t) = stack.pop() else { return }; + match stack.last_mut() { + Some(p) => p.children.push(t), + None => out.push(t), + } + } + } + for _ in 0..n { + c.u16()?; + let level = c.u16()? as usize; + if level == 0 || level > stack.len() + 1 { + return Err(Error::Corrupt(format!("index topic of level {level}"))); + } + c.u32()?; + let name = counted(c)?; + let sort_order = counted(c)?; + let m = c.u32()? as usize; + if m > c.remaining() / 4 { + return Err(Error::Corrupt(format!("{m} page references"))); + } + let refs = (0..m).map(|_| c.u32()).collect::>()?; + c.skip(8)?; + close(&mut stack, &mut out, level - 1); + stack.push(Topic { + name, + sort_order, + refs, + children: Vec::new(), + }); + } + close(&mut stack, &mut out, 0); + if out.len() != top || c.remaining() != 0 { + return Err(Error::Corrupt(format!( + "{} top topics, {top} expected, {} bytes left", + out.len(), + c.remaining() + ))); + } + Ok(out) +} + +impl Reader<'_> { + /// The document's index, if it has one. A section whose topics cannot + /// be read is left out with a warning. + pub(super) fn index(&self) -> Result, Error> { + if self.enc().big_endian() { + return Ok(None); + } + let mut found = self + .db + .classes() + .iter() + .filter(|(_, c)| *c == class::INDEX) + .map(|&(u, _)| u); + let Some(uid) = found.next() else { + return Ok(None); + }; + if found.next().is_some() { + self.warn("more than one index; only the first is written".into()); + } + let Some(d) = self.chunk(uid, chunk::SECTIONS)? else { + return Ok(Some(Index { + uid, + topics: Vec::new(), + })); + }; + let mut c = self.cursor(&d); + c.u16()?; + let n = c.u16()? as usize; + let sections: Vec = (0..n).map(|_| c.u32()).collect::>()?; + let mut topics = Vec::new(); + for s in sections { + if self.class(s) != Some(class::SECTION) { + continue; + } + if let Some(d) = self.chunk(s, chunk::TOPICS)? { + match read_topics(&mut self.cursor(&d)) { + Ok(t) => topics.extend(t), + Err(e) => self.warn(format!("index section {s} left out: {e}")), + } + } + } + Ok(Some(Index { uid, topics })) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::object::Encoding; + + fn record(d: &mut Vec, level: u16, subs: u32, name: &str, refs: &[u32]) { + d.extend([0, 1]); + d.extend(level.to_le_bytes()); + d.extend(subs.to_le_bytes()); + d.extend((name.len() as u32).to_le_bytes()); + d.extend((0x4000 | name.len() as u16).to_le_bytes()); + d.extend(name.as_bytes()); + d.extend(0u32.to_le_bytes()); + d.extend((refs.len() as u32).to_le_bytes()); + for r in refs { + d.extend(r.to_le_bytes()); + } + d.extend([0; 8]); + } + + #[test] + fn builds_the_topic_tree() { + let mut d = Vec::new(); + d.extend(2u32.to_le_bytes()); + d.extend(0u16.to_le_bytes()); + d.extend(4u32.to_le_bytes()); + record(&mut d, 1, 1, "a", &[7]); + record(&mut d, 2, 1, "b", &[]); + record(&mut d, 3, 0, "c", &[8, 9]); + record(&mut d, 1, 0, "d", &[]); + let enc = Encoding::default(); + let t = read_topics(&mut enc.cursor(&d)).unwrap(); + assert_eq!(t.len(), 2); + assert_eq!((t[0].name.as_str(), &t[0].refs[..]), ("a", &[7][..])); + assert_eq!(t[0].children[0].children[0].refs, vec![8, 9]); + assert_eq!(t[1].name, "d"); + // A level that skips one is an error. + let mut bad = d[..10].to_vec(); + bad[6..10].copy_from_slice(&1u32.to_le_bytes()); + record(&mut bad, 2, 0, "x", &[]); + assert!(read_topics(&mut enc.cursor(&bad)).is_err()); + } +} diff --git a/src/model/mod.rs b/src/model/mod.rs index cdbd458..41e22ac 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -14,6 +14,7 @@ mod document; pub mod font; pub mod hyperlink; mod ids; +pub mod index; mod item; pub mod prefs; mod reader; diff --git a/src/model/story.rs b/src/model/story.rs index 944ba25..21d46a1 100644 --- a/src/model/story.rs +++ b/src/model/story.rs @@ -20,6 +20,18 @@ pub struct TextRun { pub character_attrs: Attrs, } +/// An index marker: a page reference (class 0x13006) owned by a U+FEFF. +/// See `docs/format/index.md`. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct IndexMarker { + pub uid: u32, + /// `Id`: the u32 at 26 of a 30-byte chunk 0x13009. + pub id: Option, + /// The first 14 bytes of chunk 0x13009 are those of every sample, + /// whose IDML has `PageReferenceType="CurrentPage"`. + pub current_page: bool, +} + #[derive(Debug, Clone, PartialEq)] pub struct Story { pub uid: u32, @@ -45,8 +57,8 @@ pub struct Story { /// sorted by start. pub changes: Vec, /// Index markers (`PageReference`), by UTF-16 offset of their - /// U+FEFF: UID and `Id`. - pub index_markers: BTreeMap)>, + /// U+FEFF. + pub index_markers: BTreeMap, /// Endnote references, by UTF-16 offset of their U+0004: the endnote /// UID and the UID of its range. pub endnotes: BTreeMap, @@ -502,11 +514,29 @@ impl<'a> Reader<'a> { class::INDEX_MARKER => { // `Id` is the last u32 of chunk 0x13009 when the chunk // has 30 bytes; IDML has no `Id` for a 26-byte chunk. - let id = self - .chunk(item, chunk::INDEX_MARKER)? + // Its first 14 bytes give the type (index.md). + let d = self.chunk(item, chunk::INDEX_MARKER)?; + let id = d + .as_deref() .filter(|d| d.len() == 30) - .and_then(|d| self.enc().u32_at(&d, 26)); - index_markers.insert(pos, (item, id)); + .and_then(|d| self.enc().u32_at(d, 26)); + let current_page = d.as_deref().is_some_and(|d| { + let e = self.enc(); + ( + e.u32_at(d, 0), + e.u16_at(d, 4), + e.u32_at(d, 6), + e.u32_at(d, 10), + ) == (Some(1), Some(0), Some(1), Some(0)) + }); + index_markers.insert( + pos, + IndexMarker { + uid: item, + id, + current_page, + }, + ); } class::FOOTNOTE => { footnotes.insert(