diff --git a/docs/format/objects.md b/docs/format/objects.md index d96c7d3..e0be0fd 100644 --- a/docs/format/objects.md +++ b/docs/format/objects.md @@ -25,8 +25,9 @@ of chunks: | 4 | Data length *n* | | *n* | Data | -The rest are plain byte streams: embedded image files (TIFF, PNG, SVG) and -the XMP packet (UID 0x80000001). Chunk IDs are in the same number space as +The rest are plain byte streams: objects of class 0x129 (raw data, below) +and the XMP packet (UID 0x80000001). In the 245 distinct little-endian +files, all 1,171 class 0x129 objects are plain byte streams. Chunk IDs are in the same number space as class IDs; a chunk often lists objects of the class with the same number (for example chunk 0x501 of the document lists spreads, class 0x501). @@ -131,9 +132,64 @@ segments, then two 8-byte timestamps. **Link resources (0x8C41)**: chunk 0x8C92 is a flag byte, u32 length, then the URI as bytes (`LinkResourceURI`; 335/340 match, the rest were relinked after export). -Not yet identified: what marks a link as embedded (`StoredState= -"Embedded"`, 12 of 340 links). The second u32 of the resource's chunk -0x15A09 is 0 for all embedded links but also for 23 normal ones. +After the URI, chunk 0x8C92 continues: + +| Offset after the URI | Field | +|---|---| +| 0 | u8 1, u8 1, u16 0 (all resources) | +| 4 | u32 0 or 1, not identified | +| 8 | u32 0, 2 or 3 | +| 12 | u32 UID of a raw data object (class 0x129), or 0 | + +**Embedded links.** A link is embedded (`StoredState="Embedded"`) when its +resource names a raw data object at offset 12. That object holds the +linked file's bytes, which IDML writes as the graphic's `Contents`. + +Evidence: + +- In the 245 distinct little-endian files there are 883 link resources. + The field at offset 8 is 2 in 123 of them, and exactly those 123 name a + raw data object at offset 12. The other 760 have 0 at offset 12 (742 + with 0 at offset 8, 18 with 3; no pair shows what 3 means). +- In the corpus pairs, all 10 resources of embedded links (12 links) name + a raw data object, and none of the 369 resources of normal links do. +- For all 12 embedded links, the raw data object's bytes equal the + base64-decoded `Contents` of the IDML graphic (JPEG, PNG, PDF, EPS and + SVG files, 2 KB to 11 MB). + +## Graphics pasted without a link + +A graphic with no link (its chunk 0x8CBC names no link) can hold its file +itself: + +| Class | Chunk | Contents | +|---|---|---| +| Image (0x1702) | 0x8C23 | u32 raw data object (0 = none) | +| PDF (0x2501) | 0x2521 | u32 raw data object (0 = none) | + +Evidence from the corpus pairs: the raw data object equals the IDML +`Contents` for 124 of 124 PDFs and 2 of 2 images without a link. One +more image without a link has 0x8C23 = 0; its IDML `Contents` equals the +preview data described below, which the converter does not use. In the +whole corpus, two linked and embedded PDFs also have a 0x2521 object, +different from the link resource's; with no pair to compare, the +converter uses the link resource's. + +## Graphic previews + +Placed graphics also have chunk 0x170D, the UID of an object of class +0x1708, whose chunk 0x119 names a raw data object. In the 245 distinct +little-endian files, these objects are TIFF (613), JPEG (411), PNG (307), +GIF (27) and one other. In the same-version pairs they equal the IDML +`Contents` for only 3 of 109 embedded or pasted graphics, so they are +most likely screen previews. The converter does not use them. + +## Embedded data in IDML + +The reference IDML files write `Contents` inside the graphic's +`Properties` as base64 (with `=` padding) in lines of 76 characters joined +by a line feed, with no line feed at the end. The text is split into +CDATA sections of 262,144 characters. The converter writes the same form. ## Stories (0x201) diff --git a/src/idml/mod.rs b/src/idml/mod.rs index 4883bab..696be5f 100644 --- a/src/idml/mod.rs +++ b/src/idml/mod.rs @@ -162,6 +162,42 @@ fn matrix(m: &Matrix) -> String { nums(&m.0) } +/// Characters per CDATA section of embedded file data, as in InDesign's +/// IDML export. +const CDATA_SECTION: usize = 262_144; + +/// Base64 (RFC 4648, with padding) in lines of 76 characters separated by +/// line feeds, as IDML writes embedded file data. +fn base64_lines(data: &[u8]) -> String { + const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + const LINE: usize = 76; + let encoded_len = data.len().div_ceil(3) * 4; + let mut out = String::with_capacity(encoded_len + encoded_len / LINE); + let mut column = 0; + for group in data.chunks(3) { + let b = [ + group[0], + group.get(1).copied().unwrap_or(0), + group.get(2).copied().unwrap_or(0), + ]; + let n = (b[0] as u32) << 16 | (b[1] as u32) << 8 | b[2] as u32; + if column == LINE { + out.push('\n'); + column = 0; + } + for i in 0..4 { + let c = if i <= group.len() { + ALPHABET[(n >> (18 - 6 * i) & 0x3F) as usize] as char + } else { + '=' + }; + out.push(c); + } + column += 4; + } + out +} + fn uref(uid: Option) -> String { uid.map_or("n".into(), |u| format!("u{u:x}")) } @@ -723,6 +759,11 @@ impl Writer<'_> { .attr("Self", uref(Some(g.uid))) .attr("ItemTransform", matrix(&g.transform)); x.start("Properties"); + if let Some(data) = &g.contents { + x.start("Contents") + .cdata(&base64_lines(data), CDATA_SECTION) + .end(); + } x.empty( "GraphicBounds", &[ @@ -739,7 +780,10 @@ impl Writer<'_> { &[ ("Self", uref(Some(link.uid))), ("LinkResourceURI", link.uri.clone()), - ("StoredState", "Normal".into()), + ( + "StoredState", + if link.embedded { "Embedded" } else { "Normal" }.into(), + ), ], ); } @@ -1081,4 +1125,25 @@ mod tests { assert_eq!(num(1.0), "1"); assert_eq!(num(-89.99999999999999), "-89.99999999999999"); } + + #[test] + fn encodes_base64_in_lines() { + assert_eq!(base64_lines(b""), ""); + assert_eq!(base64_lines(b"f"), "Zg=="); + assert_eq!(base64_lines(b"fo"), "Zm8="); + assert_eq!(base64_lines(b"foobar"), "Zm9vYmFy"); + let lines = base64_lines(&[0xFF; 58]); + assert_eq!(lines.split('\n').map(str::len).collect::>(), [76, 4]); + assert!(base64_lines(&[0xFF; 57]).find('\n').is_none()); + } + + #[test] + fn splits_cdata_sections() { + let mut x = Xml::new(); + x.start("Contents").cdata("abcde", 2).end(); + assert!( + x.finish() + .ends_with("\n") + ); + } } diff --git a/src/idml/xml.rs b/src/idml/xml.rs index 9d1ac5c..6aa69ac 100644 --- a/src/idml/xml.rs +++ b/src/idml/xml.rs @@ -109,6 +109,23 @@ impl Xml { self } + /// Text content as CDATA sections of at most `section` characters + /// each. `s` must not contain `]]>`. + pub fn cdata(&mut self, s: &str, section: usize) -> &mut Self { + self.close_start(); + self.inline = true; + let mut rest = s; + while !rest.is_empty() { + let mut n = section.min(rest.len()); + while !rest.is_char_boundary(n) { + n -= 1; + } + let _ = write!(self.out, "", &rest[..n]); + rest = &rest[n..]; + } + self + } + pub fn end(&mut self) -> &mut Self { let name = self.stack.pop().expect("end without start"); if self.open { diff --git a/src/model/mod.rs b/src/model/mod.rs index cf94aad..66755b5 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -34,6 +34,8 @@ pub mod class { pub const PDF: u32 = 0x2501; pub const EPS: u32 = 0x6601; pub const SVG: u32 = 0x6639; + /// A plain byte stream, such as the file of an embedded graphic. + pub const RAW_DATA: u32 = 0x129; pub const FONT_FAMILY: u32 = 0x3E03; /// Holds an item anchored in text. pub const ANCHOR: u32 = 0x262; @@ -94,6 +96,10 @@ pub mod chunk { pub const GRAPHIC_LINK: u32 = 0x8CBC; pub const LINK_INFO: u32 = 0x8C9B; pub const LINK_RESOURCE_URI: u32 = 0x8C92; + /// Pasted image without a link: u32 raw data object. + pub const IMAGE_DATA: u32 = 0x8C23; + /// Pasted PDF without a link: u32 raw data object. + pub const PDF_DATA: u32 = 0x2521; pub const SWATCH_NAME: u32 = 0x1F30; } @@ -208,8 +214,13 @@ pub struct Link { pub uid: u32, /// `LinkResourceURI`, for example `file:/Users/me/image.jpg`. pub uri: String, + /// The linked file is stored in the document (`StoredState="Embedded"`). + pub embedded: bool, } +/// The bytes of a file stored in the document, if there is one. +type EmbeddedFile = Option>; + #[derive(Debug, Clone, PartialEq)] pub struct Graphic { pub uid: u32, @@ -218,6 +229,9 @@ pub struct Graphic { /// Left, top, right, bottom in the graphic's own coordinates. pub bounds: [f64; 4], pub link: Option, + /// The graphic's file, when the document holds it: an embedded link or + /// a graphic pasted without a link. IDML writes it as `Contents`. + pub contents: Option>, } #[derive(Debug, Clone, PartialEq)] @@ -871,17 +885,45 @@ impl<'a> Reader<'a> { Some(d) if d.len() >= 12 => self.link(Cursor::new(&d[8..]).u32()?)?, _ => None, }; + let (link, contents) = match link { + Some((link, data)) => (Some(link), data), + None => { + let id = match kind { + GraphicKind::Image => Some(chunk::IMAGE_DATA), + GraphicKind::Pdf => Some(chunk::PDF_DATA), + _ => None, + }; + let data = match id { + Some(id) => match self.chunk(uid, id)? { + Some(d) if d.len() >= 4 => self.raw_data(Cursor::new(&d).u32()?)?, + _ => None, + }, + None => None, + }; + (None, data) + } + }; Ok(Some(Graphic { uid, kind, transform, bounds, link, + contents, })) } - /// A link and the URI of its resource. - fn link(&self, uid: u32) -> Result, Error> { + /// The bytes of raw data object `uid` (`None` if `uid` is not one). + fn raw_data(&self, uid: u32) -> Result { + if uid == 0 || self.class(uid) != Some(class::RAW_DATA) { + return Ok(None); + } + self.db.object(uid) + } + + /// A link, the URI of its resource and, for an embedded link, the + /// embedded file. + fn link(&self, uid: u32) -> Result, Error> { if uid == 0 || self.db.object(uid)?.is_none() { return Ok(None); } @@ -892,16 +934,30 @@ impl<'a> Reader<'a> { return Ok(None); } let resource = Cursor::new(&info[8..]).u32()?; - let uri = match self.chunk(resource, chunk::LINK_RESOURCE_URI)? { + let (uri, data) = match self.chunk(resource, chunk::LINK_RESOURCE_URI)? { Some(d) if d.len() >= 5 => { let mut c = Cursor::new(&d); c.u8()?; let n = c.u32()? as usize; - String::from_utf8_lossy(c.bytes(n.min(c.remaining()))?).into_owned() + let uri = String::from_utf8_lossy(c.bytes(n.min(c.remaining()))?).into_owned(); + // After the URI: 12 bytes, then the UID of the embedded + // file's raw data object (0 for a normal link). + let data = if c.remaining() >= 16 { + c.skip(12)?; + self.raw_data(c.u32()?)? + } else { + None + }; + (uri, data) } - _ => String::new(), + _ => (String::new(), None), }; - Ok(Some(Link { uid, uri })) + let link = Link { + uid, + uri, + embedded: data.is_some(), + }; + Ok(Some((link, data))) } /// Story and threading of the text frame owning `column`.