diff --git a/docs/format/idml-values.md b/docs/format/idml-values.md index aae539f..1e36cbb 100644 --- a/docs/format/idml-values.md +++ b/docs/format/idml-values.md @@ -187,15 +187,69 @@ All 973 `Ink` elements of the 240 corpus IDML files have No INDD field for them is identified; the converter writes them on every ink (`objects.md`). -## Export options of page items +## Values of other elements -All 6,151 `ObjectExportOption` elements of page items (text frames, -rectangles, ovals, polygons, lines and groups) in the corpus IDML files -with DOM 12 or later have the same values, which are also the values -every object style other than `[None]` has (above). The converter writes -that element on every page item of documents from InDesign 12 on. The 2 -elements in the DOM 7 file differ and that file has the element on few -items, so nothing is written for older documents. +`tools/element_values.py` collects the values of other elements the +converter writes from every distinct IDML of the corpus (683 files +exported by InDesign; one more IDML in the corpus lacks the `product` of +InDesign's `aid` processing instruction and was written by a script, so +it is left out). A value is kept for a range of DOM versions when every +element of that kind in every file of those versions has it with the +same value, and it is left out entirely when some version shows two +values or elements without it. The values may differ between versions; +each range needs 10 files. The kept values are in +`src/idml/element_values.xml`, in blocks with `MinimumVersion` and +`MaximumVersion`. The converter writes them after the values it reads +from the INDD and only where the element lacks the attribute. + +Files per DOM version: 7: 9, 8: 67, 9: 8, 10: 52, 11: 48, 12: 39, 13: 79, +14: 74, 15: 66, 16: 42, 17: 17, 18: 48, 19: 70, 20: 31, 21: 33. + +### Page items + +| Element | Elements | Values kept | +|---|---|---| +| `TextFrame` | 32,002 | `LocalDisplaySetting="Default"`, `GradientStrokeAngle`, `GradientStrokeHiliteLength`, `GradientStrokeHiliteAngle` (`0`) from DOM 7; `FlexItemWidthMode` and `FlexItemHeightMode` (`FlexFixed`) from 21 | +| `Rectangle` | 24,314 | `LocalDisplaySetting`, `StoryTitle="$ID/"` from 7 | +| `Oval` | 3,356 | `LocalDisplaySetting`, `StoryTitle` and the gradient highlights from 8 | +| `Polygon` | 52,774 | `LocalDisplaySetting`, `StoryTitle` and the gradient highlights from 7 | +| `GraphicLine` | 4,351 | `LocalDisplaySetting`, `LockState="None"` and the gradient highlights from 7 | +| `Group` | 9,010 | `LocalDisplaySetting`, `GradientStrokeAngle` and the gradient highlights from 7 | + +Each kind also has `FlexItemWidthMode` and `FlexItemHeightMode` from +DOM 21, and values that the converter reads from the INDD +(`objects.md`, page item settings), which take precedence. Values such +as `Name`, `Visible`, `Locked`, the gradient angles and the layout +constraints are not kept: they differ in some elements, and those +values come from the INDD. + +### Export options of page items + +`ObjectExportOption` of page items has different attributes in +different versions. Kept for each kind of item: + +| Values | Versions | +|---|---| +| Image conversion, GIF, JPEG, spacing, page break, layout and metadata options (13 to 18 values, 22 for ovals) | 8 on (7 on for rectangles) | +| `EpubType="$ID/"` | 10 on | +| `EpubAriaRole="$ID/"` (ovals: also two ARIA label values) | 21 on | + +`CustomImageSizeOption="SizeRelativeToPageWidth"`, which every page item +of DOM 8 and 9 has, is left out: the IDML schema the output is checked +against does not allow that value. The elements are 31,857 text frames, 23,981 rectangles, 3,356 ovals, +52,694 polygons, 4,332 lines and 8,949 groups. The alternative text and +tagging values (`AltTextSourceType`, `ActualTextSourceType`, +`CustomAltText`, `CustomActualText`, `ApplyTagType`) are kept only for +ovals: in one DOM 20 document, text frames, rectangles, polygons and +groups are tagged as artifacts with custom text. Their INDD objects hold +these settings in chunk 0x1E206 (its first u32 is 5 in items with the +usual values, 8 or 0 in those items), which is not decoded. + +The converter writes these values on every page item. From DOM 12 on it +adds the values that every object style other than `[None]` has in the +files the object style values were made from (above); all 6,151 +`ObjectExportOption` elements of page items in those files with DOM 12 +or later have them. All 3,412 `InCopyExportOption` elements of rectangles, ovals and polygons have `IncludeGraphicProxies="true"` and `IncludeAllResources="false"`, diff --git a/docs/format/objects.md b/docs/format/objects.md index 848d12d..4bd357b 100644 --- a/docs/format/objects.md +++ b/docs/format/objects.md @@ -259,6 +259,75 @@ frame list (class 0x228). Its chunk 0x205: u32 story, then a UID list of the columns of all threaded frames in order, which gives `PreviousTextFrame` and `NextTextFrame`. +## Page item settings + +Every page item element in IDML (`TextFrame`, `Rectangle`, `Oval`, +`Polygon`, `GraphicLine`, `Group`) has the attributes below. Each is +stored in a chunk of the item (class 0x6201 or 0x401), or in its +attribute list (chunk 0x6E03, `attributes.md`); without the chunk the +IDML value is the default in the table. + +| Chunk | Contents | IDML | Without the chunk | +|---|---|---|---| +| 0x2C10 (groups: 0x418) | u8 1 if a built-in key, then an in-object string | `Name`: the string, or `$ID/` and the key | `$ID/` | +| 0x2C32 | u16, 0 = hidden | `Visible` | `true` | +| 0x2C2D | u32, 1 = locked | `Locked` | `false` | +| 0x21D4E | u32 count *n*, *n* pairs of u32 | `ParentInterfaceChangeCount`: the numbers | empty | +| 0x21D50 | same | `TargetInterfaceChangeCount` | empty | +| 0x21D53 | same | `LastUpdatedInterfaceChangeCount` | empty | +| 0x1424 | u32 master page item, UID list | `OverriddenPageItemProps`: the list in decimal; empty if the master item is 0 | empty | +| 0x22228 | u8 flags | `HorizontalLayoutConstraints` (bits 4–6), `VerticalLayoutConstraints` (bits 0–2): per bit `FixedDimension` if set, else `FlexibleDimension` | as 0x22 | + +| Attribute | IDML | Without the attribute | +|---|---|---| +| 0x5520, 0x551F, 0x551E | `GradientFillStart`, `GradientFillLength`, `GradientFillAngle` | `0 0`, `0`, `0` | +| 0x5526, 0x5525, 0x5524 | `GradientStrokeStart`, `GradientStrokeLength`, `GradientStrokeAngle` | same | +| 0x5522, 0x5523 | `GradientFillHiliteLength`, `GradientFillHiliteAngle` | `0` | +| 0x5528, 0x5529 | `GradientStrokeHiliteLength`, `GradientStrokeHiliteAngle` | `0` | + +Rules the IDML follows, over the 92,409 page item elements of the 495 +trustworthy pairs: + +- `Locked` is on the 36,817 items that are children of a spread or + anchored in text, and on none of the 55,592 items inside a group or + another page item. +- The change counts, `OverriddenPageItemProps` and the layout + constraints are in IDML from DOM 8 on: every page item of the + distinct corpus IDML files from DOM 8 on has them, none of the DOM 7 + files does (145 text frames, 335 rectangles, 61 groups). +- A group has no gradient attributes of its own (no group in the pairs + has 0x551E to 0x5529). IDML gives it, for each gradient attribute, the + value all its child items have, and leaves the attribute out when they + differ: 58,068 of the 58,080 gradient values of the 5,808 groups. + +Evidence: the INDD values, decoded as above, against the IDML of the +same item, over the page items of the trustworthy pairs that have an +INDD object: + +| IDML attribute | Equal | +|---|---| +| `Name` | 92,264 of 92,402 | +| `Visible` | 92,402 of 92,402 | +| `Locked` | 36,810 of 36,810 | +| The three change counts and `OverriddenPageItemProps` | 91,812 of 91,812 each | +| Both layout constraints | 91,812 of 91,812 each | +| Gradient start and length, fill and stroke (four attributes), items other than groups | 86,594 of 86,594 each | +| Gradient angles and highlights (six attributes) | all, 92,396 to 92,402 per attribute (groups by the rule above) | + +The 138 names that differ belong to groups that are states of a button +(their parent has class 0x1450D); they have no name chunk, and IDML +names them `$ID/$$$/StateType/...`. The converter does not write such +groups. + +Values other than the default: 206 items with a name given by the +user, 176 hidden, 1,241 with change counts, 1,113 with overridden +properties. Flags 0x22 are in 66,820 items, 0x55 (`FixedDimension +FlexibleDimension FixedDimension` both ways) in 643 and 0x02 +(`FlexibleDimension` three times horizontally) in 13; with another bit +set the converter leaves both attributes out. In 14,918 items with +chunk 0x1424 the master item is 0; in 197 of them the list still holds +IDs, and IDML writes the attribute empty. + ## Text wrap Page items (class 0x6201 and groups, 0x401) and placed graphics have diff --git a/src/idml/element_values.xml b/src/idml/element_values.xml new file mode 100644 index 0000000..063052c --- /dev/null +++ b/src/idml/element_values.xml @@ -0,0 +1,112 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/idml/mod.rs b/src/idml/mod.rs index 37236b5..75ac49a 100644 --- a/src/idml/mod.rs +++ b/src/idml/mod.rs @@ -457,6 +457,50 @@ const ITEM_ATTRS: &[(u32, &str, AttrKind)] = &[ AttrKind::Enum(&[(0, "CenterAlignment"), (1, "InsideAlignment")]), ), ]; +/// Gradient attributes every page item has: attribute-list ID (0 for +/// none, `GradientStroke...` of groups), IDML name, value without the +/// attribute. Fill and stroke start and length are in `ITEM_ATTRS`. See +/// `docs/format/objects.md`, page item settings. +const GRADIENT_ATTRS: &[(u32, &str, &str)] = &[ + (0x5520, "GradientFillStart", "0 0"), + (0x551F, "GradientFillLength", "0"), + (0x551E, "GradientFillAngle", "0"), + (0x5526, "GradientStrokeStart", "0 0"), + (0x5525, "GradientStrokeLength", "0"), + (0x5524, "GradientStrokeAngle", "0"), + (0x5522, "GradientFillHiliteLength", "0"), + (0x5523, "GradientFillHiliteAngle", "0"), + (0x5528, "GradientStrokeHiliteLength", "0"), + (0x5529, "GradientStrokeHiliteAngle", "0"), +]; + +/// IDML `HorizontalLayoutConstraints` and `VerticalLayoutConstraints` of +/// layout constraint flags (chunk 0x22228): bits 4 to 6 and 0 to 2, each +/// set for `FixedDimension`. `None` if another bit is set. +fn layout_constraints(flags: u8) -> Option<(String, String)> { + if flags & 0x88 != 0 { + return None; + } + let side = |bits: u8| { + (0..3) + .map(|i| { + if bits & (1 << i) != 0 { + "FixedDimension" + } else { + "FlexibleDimension" + } + }) + .collect::>() + .join(" ") + }; + Some((side(flags >> 4), side(flags & 7))) +} + +/// Numbers separated by spaces. +fn join_numbers(v: &[u32]) -> String { + v.iter().map(u32::to_string).collect::>().join(" ") +} + /// Kinds of cell attribute values. #[derive(Clone, Copy)] enum CellKind { @@ -900,8 +944,13 @@ struct Writer<'a> { group_path: std::collections::HashMap>, /// Values left out while writing, reported with the model's warnings. warnings: std::cell::RefCell>, + /// Observed attributes by element path (`values::element_attrs`). + observed: std::cell::RefCell>, } +/// Attributes observed on an element path (`values::element_attrs`). +type Observed = std::rc::Rc>; + fn group_paths(doc: &Document) -> std::collections::HashMap> { fn walk( doc: &Document, @@ -927,6 +976,18 @@ fn group_paths(doc: &Document) -> std::collections::HashMap> { } impl Writer<'_> { + /// The attributes every IDML of the document's version has on the + /// elements on `path` (`docs/format/idml-values.md`). + fn observed(&self, path: &str) -> Observed { + self.observed + .borrow_mut() + .entry(path.to_string()) + .or_insert_with(|| { + std::rc::Rc::new(values::element_attrs(path, self.doc.version.major)) + }) + .clone() + } + fn package_root(&self, x: &mut Xml, kind: &str) { x.start(&format!("idPkg:{kind}")) .attr("xmlns:idPkg", PACKAGING_NS) @@ -2735,7 +2796,82 @@ impl Writer<'_> { x.end(); } - fn page_item(&self, x: &mut Xml, item: &PageItem) { + /// The gradient attributes of a page item: its own, or, for a group, + /// the values all its children have. See `docs/format/objects.md`. + fn gradients(item: &PageItem) -> Vec<(&'static str, String)> { + if item.kind == ItemKind::Group { + let kids: Vec<_> = item.children.iter().map(Self::gradients).collect(); + return GRADIENT_ATTRS + .iter() + .filter_map(|&(_, name, _)| { + let mut values = kids + .iter() + .map(|k| k.iter().find(|(n, _)| *n == name).map(|(_, v)| v)); + let first = values.next()??; + values + .all(|v| v == Some(first)) + .then(|| (name, first.clone())) + }) + .collect(); + } + GRADIENT_ATTRS + .iter() + .map(|&(id, name, default)| { + let v = match item.attrs.get(id) { + Some(Value::Point(x, y)) => Some(nums(&[*x, *y])), + Some(v) => v.as_f64().map(num), + None => None, + }; + (name, v.unwrap_or_else(|| default.to_string())) + }) + .collect() + } + + /// The name, visibility, lock and the other settings every page item + /// has; `nested` for an item inside another page item, which IDML + /// writes without `Locked`. See `docs/format/objects.md`. + fn item_settings(&self, x: &mut Xml, item: &PageItem, nested: bool) { + let p = &item.props; + let name = match &p.name { + Some(n) if n.builtin => format!("$ID/{}", n.name), + Some(n) => n.name.clone(), + None => "$ID/".into(), + }; + x.attr("Name", name) + .attr("Visible", (!p.hidden).to_string()); + if !nested { + x.attr("Locked", p.locked.to_string()); + } + if self.doc.version.major >= 8 { + for (name, counts) in [ + "ParentInterfaceChangeCount", + "TargetInterfaceChangeCount", + "LastUpdatedInterfaceChangeCount", + ] + .into_iter() + .zip(&p.change_counts) + { + x.attr(name, join_numbers(counts)); + } + let overridden = match &p.overridden { + Some((master, ids)) if *master != 0 => join_numbers(ids), + _ => String::new(), + }; + x.attr("OverriddenPageItemProps", overridden); + if let Some((h, v)) = layout_constraints(p.layout_constraints.unwrap_or(0x22)) { + x.attr("HorizontalLayoutConstraints", h) + .attr("VerticalLayoutConstraints", v); + } + } + for (name, v) in Self::gradients(item) { + if !x.has_attr(name) { + x.attr(name, v); + } + } + } + + /// `nested` for an item inside another page item. + fn page_item(&self, x: &mut Xml, item: &PageItem, nested: bool) { let tag = match &item.kind { ItemKind::TextFrame { .. } => "TextFrame", ItemKind::Group => "Group", @@ -2783,6 +2919,8 @@ impl Writer<'_> { x.attr("ItemLayer", uref(Some(layer))); } x.attr("ItemTransform", matrix(&item.transform)); + self.item_settings(x, item, nested); + x.attrs_missing(self.observed(tag).iter()); Self::path_geometry(x, &item.paths); if let ItemKind::TextFrame { preferences: Some(p), @@ -2798,12 +2936,20 @@ impl Writer<'_> { if frame { self.frame_fitting(x, item); } - // Every page item of IDML from DOM 12 on has the same export - // options as the object styles (idml-values.md). - if self.doc.version.major >= 12 - && let Some(n) = - values::object_style(self.doc.version.major).child("ObjectExportOption") + // Export options every IDML of the version has on this kind of + // item; from DOM 12 on, also those of the object styles + // (idml-values.md). + let major = self.doc.version.major; + let mut export = values::element(&format!("{tag}/ObjectExportOption"), major); + if major >= 12 + && let Some(n) = values::object_style(major).child("ObjectExportOption") { + match &mut export { + Some(e) => e.merge(n), + None => export = Some(n.clone()), + } + } + if let Some(n) = export { n.write(x); } if let Some(d) = &item.anchor { @@ -2838,7 +2984,7 @@ impl Writer<'_> { )); } for child in &item.children { - self.page_item(x, child); + self.page_item(x, child, true); } for g in &item.graphics { Self::placed_graphic(x, g); @@ -3026,7 +3172,7 @@ impl Writer<'_> { x.end(); } for item in &s.items { - self.page_item(&mut x, item); + self.page_item(&mut x, item, false); } x.end(); x.finish() @@ -3291,7 +3437,7 @@ impl Writer<'_> { '\u{FFFC}' if story.anchors.contains_key(&pos) => { flush(x, &mut buf); for item in &story.anchors[&pos] { - self.page_item(x, item); + self.page_item(x, item, false); } } '\u{16}' if story.tables.contains_key(&pos) => { @@ -3457,6 +3603,7 @@ pub fn write(doc: &Document, name: &str, out: impl std::io::Write) -> std::io::R name: name.to_string(), group_path: group_paths(doc), warnings: Default::default(), + observed: Default::default(), }; let mut files: BTreeMap = BTreeMap::new(); files.insert( @@ -3559,6 +3706,7 @@ mod tests { name: String::new(), group_path: Default::default(), warnings: Default::default(), + observed: Default::default(), }; let out: String = w .backing_story() diff --git a/src/idml/values.rs b/src/idml/values.rs index e29de47..2b0b523 100644 --- a/src/idml/values.rs +++ b/src/idml/values.rs @@ -1,7 +1,9 @@ //! Values written from IDML observation: what every InDesign-exported IDML //! in the corpus has on the root styles (`root_values.xml`) and in its //! preferences (`preference_values.xml`), both generated by -//! `tools/root_values.py`. See `docs/format/idml-values.md`. +//! `tools/root_values.py`, and on the other elements the converter writes +//! (`element_values.xml`, generated by `tools/element_values.py`). See +//! `docs/format/idml-values.md`. use super::xml::Xml; @@ -60,7 +62,7 @@ impl Node { /// Add the attributes and children of `other` that `self` lacks. /// Children with the same tag are merged. - fn merge(&mut self, other: &Node) { + pub fn merge(&mut self, other: &Node) { for (k, v) in &other.attrs { if self.attr(k).is_none() { self.attrs.push((k.clone(), v.clone())); @@ -169,6 +171,77 @@ pub fn preferences(major: u32) -> Vec { out } +const ELEMENT_VALUES: &str = include_str!("element_values.xml"); + +fn element_values() -> &'static Node { + static PARSED: std::sync::OnceLock = std::sync::OnceLock::new(); + PARSED.get_or_init(|| parse(ELEMENT_VALUES).expect("element_values.xml is well-formed")) +} + +/// Whether a block of `element_values.xml` applies to InDesign version +/// `major`. +fn applies(block: &Node, major: u32) -> bool { + let bound = |name: &str| { + block + .attr(name) + .map(|v| v.parse::().expect("version bounds are numbers")) + }; + bound("MinimumVersion").is_none_or(|v| v <= major) + && bound("MaximumVersion").is_none_or(|v| major <= v) +} + +/// The observed values of the elements on `path` (a tag, or a tag and a +/// child tag such as `Spread/FlattenerPreference`) for a document of +/// InDesign version `major`, from `element_values.xml`. `None` if the +/// file has no values for it. +pub fn element(path: &str, major: u32) -> Option { + let (parent, child) = match path.split_once('/') { + Some((p, c)) => (p, Some(c)), + None => (path, None), + }; + let mut out: Option = None; + for block in element_values() + .children + .iter() + .filter(|b| b.tag == "Values") + { + if !applies(block, major) { + continue; + } + let Some(mut n) = block.child(parent) else { + continue; + }; + if let Some(c) = child { + match n.child(c) { + Some(cn) => n = cn, + None => continue, + } + } + let mut n = n.clone(); + if child.is_none() { + // Children are other paths. + n.children.retain(|c| c.tag == "Properties"); + } + match &mut out { + Some(o) => o.merge(&n), + None => out = Some(n), + } + } + out +} + +/// The observed attributes of the elements on `path`, without `Present`. +pub fn element_attrs(path: &str, major: u32) -> Vec<(String, String)> { + element(path, major) + .map(|n| { + n.attrs + .into_iter() + .filter(|(k, _)| k != "Present") + .collect() + }) + .unwrap_or_default() +} + fn unescape(s: &str) -> String { s.replace("<", "<") .replace(">", ">") @@ -294,6 +367,24 @@ mod tests { assert_eq!(wrap.children[0].tag, "Properties"); } + #[test] + fn gates_element_values_by_version() { + let old = element("TextFrame", 7).unwrap(); + assert_eq!(old.attr("LocalDisplaySetting"), Some("Default")); + // Values of child paths are not attributes of the parent. + assert!(old.child("ObjectExportOption").is_none()); + let new = element("Polygon/ObjectExportOption", 21).unwrap(); + assert!(new.child("Properties").is_some()); + // Export options differ between versions. + assert!(new.attr("EpubType").is_some()); + assert!( + element("Polygon/ObjectExportOption", 8) + .unwrap() + .attr("EpubType") + .is_none() + ); + } + #[test] fn gates_preferences_by_version() { let old = preferences(7); diff --git a/src/idml/xml.rs b/src/idml/xml.rs index cbbc5ce..b25dcfa 100644 --- a/src/idml/xml.rs +++ b/src/idml/xml.rs @@ -9,6 +9,8 @@ pub struct Xml { open: bool, /// The current element has text content, so its end tag is not indented. inline: bool, + /// Names of the attributes written in the open start tag. + written: Vec, } /// Whether XML 1.0 allows `c` (control characters aside): not U+FFFE or @@ -48,6 +50,7 @@ impl Xml { stack: Vec::new(), open: false, inline: false, + written: Vec::new(), } } @@ -85,12 +88,32 @@ impl Xml { self.stack.push(name.to_string()); self.open = true; self.inline = false; + self.written.clear(); self } pub fn attr(&mut self, name: &str, value: impl AsRef) -> &mut Self { debug_assert!(self.open, "attribute outside a start tag"); let _ = write!(self.out, " {name}=\"{}\"", escape_attr(value.as_ref())); + self.written.push(name.to_string()); + self + } + + /// Whether the open start tag has attribute `name`. + pub fn has_attr(&self, name: &str) -> bool { + self.open && self.written.iter().any(|w| w == name) + } + + /// Write the attributes the open start tag does not have yet. + pub fn attrs_missing<'a>( + &mut self, + attrs: impl IntoIterator, + ) -> &mut Self { + for (k, v) in attrs { + if !self.has_attr(k) { + self.attr(k, v); + } + } self } @@ -181,6 +204,18 @@ mod tests { ); } + #[test] + fn adds_only_missing_attributes() { + let mut x = Xml::new(); + x.start("a").attr("k", "1"); + let more = [ + ("k".to_string(), "2".to_string()), + ("m".to_string(), "3".to_string()), + ]; + x.attrs_missing(&more).end(); + assert!(x.finish().ends_with("")); + } + #[test] fn control_characters_become_processing_instructions() { let mut x = Xml::new(); diff --git a/src/model/mod.rs b/src/model/mod.rs index a2c9802..76cd1a6 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -112,6 +112,23 @@ pub mod chunk { pub const OBJECT_STYLE_INFO: u32 = 0x1B907; pub const OBJECT_STYLE_ROOT_CHILDREN: u32 = 0x1B95A; pub const ITEM_OBJECT_STYLE: u32 = 0x1B916; + /// Page item name: u8 1 if a built-in key, then an in-object string. + pub const ITEM_NAME: u32 = 0x2C10; + /// The same for a group. + pub const GROUP_NAME: u32 = 0x418; + /// u32 1 if the page item is locked. + pub const ITEM_LOCKED: u32 = 0x2C2D; + /// u16 0 if the page item is hidden. + pub const ITEM_VISIBLE: u32 = 0x2C32; + /// Override of a master page item: u32 master item, then a list of + /// attribute IDs. + pub const ITEM_OVERRIDE: u32 = 0x1424; + /// Interface change counts: u32 count *n*, then *n* pairs of u32. + pub const ITEM_PARENT_CHANGES: u32 = 0x21D4E; + pub const ITEM_TARGET_CHANGES: u32 = 0x21D50; + pub const ITEM_UPDATED_CHANGES: u32 = 0x21D53; + /// u8 layout constraint flags. + pub const ITEM_LAYOUT_CONSTRAINTS: u32 = 0x22228; pub const ROOT_GROUP_KIND: u32 = 0x28C2; pub const SECTION_INFO: u32 = 0x4C02; pub const DOCUMENT_PREFERENCES: u32 = 0x533; @@ -291,6 +308,35 @@ pub struct PageItem { /// Anchored object settings (chunk 0x2800 of the anchor), for an item /// anchored in text. pub anchor: Option>, + pub props: ItemProps, +} + +/// Settings every page item has. See `docs/format/objects.md`, page item +/// settings. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct ItemProps { + /// The name (chunk 0x2C10, of a group 0x418); `None` without the + /// chunk. + pub name: Option, + /// Chunk 0x2C32 is 0. + pub hidden: bool, + /// Chunk 0x2C2D is 1. + pub locked: bool, + /// Layout constraint flags (chunk 0x22228). + pub layout_constraints: Option, + /// The parent, target and last updated interface change counts + /// (chunks 0x21D4E, 0x21D50, 0x21D53), each a list of numbers. + pub change_counts: [Vec; 3], + /// The overridden master page item (0 for none) and the IDs of the + /// attributes overridden (chunk 0x1424). + pub overridden: Option<(u32, Vec)>, +} + +/// A page item name: a built-in key or a name given by the user. +#[derive(Debug, Clone, PartialEq)] +pub struct ItemName { + pub builtin: bool, + pub name: String, } /// Text wrap settings of a page item or graphic (chunk 0x3703). @@ -1583,6 +1629,63 @@ impl<'a> Reader<'a> { }) } + /// The settings every page item has (`ItemProps`). + fn item_props(&self, uid: u32) -> Result { + let name_chunk = if self.class(uid) == Some(class::GROUP) { + chunk::GROUP_NAME + } else { + chunk::ITEM_NAME + }; + let name = match self.chunk(uid, name_chunk)? { + Some(d) => { + let mut c = Cursor::new(&d); + let builtin = c.flag()? == 1; + Some(ItemName { + builtin, + name: c.string()?, + }) + } + None => None, + }; + let counts = |id: u32| -> Result, Error> { + let Some(d) = self.chunk(uid, id)? else { + return Ok(Vec::new()); + }; + let mut c = Cursor::new(&d); + let n = c.u32()? as usize; + if n > d.len() / 8 { + return Err(Error::Corrupt(format!("item {uid}: {n} change counts"))); + } + (0..2 * n).map(|_| c.u32()).collect() + }; + let overridden = match self.chunk(uid, chunk::ITEM_OVERRIDE)? { + Some(d) => { + let mut c = Cursor::new(&d); + let master = c.u32()?; + Some((master, c.u32_list()?)) + } + None => None, + }; + Ok(ItemProps { + name, + hidden: self + .chunk(uid, chunk::ITEM_VISIBLE)? + .is_some_and(|d| d.len() >= 2 && Cursor::new(&d).u16().ok() == Some(0)), + locked: self + .chunk(uid, chunk::ITEM_LOCKED)? + .is_some_and(|d| d.len() >= 4 && Cursor::new(&d).u32().ok() == Some(1)), + layout_constraints: self + .chunk(uid, chunk::ITEM_LAYOUT_CONSTRAINTS)? + .and_then(|d| d.first().copied()), + change_counts: [ + counts(chunk::ITEM_PARENT_CHANGES)?, + counts(chunk::ITEM_TARGET_CHANGES)?, + counts(chunk::ITEM_UPDATED_CHANGES)?, + ], + overridden, + }) + } + /// Text wrap of a page item or graphic, from chunk 0x3703: u32 mode, /// u32 contour path object, four f64 offsets, u32 flags. fn text_wrap(&self, uid: u32) -> Result, Error> { @@ -1735,6 +1838,10 @@ impl<'a> Reader<'a> { children, text_wrap: self.text_wrap(uid)?, anchor: None, + props: self.item_props(uid).unwrap_or_else(|e| { + self.warn(format!("item {uid}: settings left out: {e}")); + ItemProps::default() + }), })) } diff --git a/tools/element_values.py b/tools/element_values.py new file mode 100644 index 0000000..8ada1b4 --- /dev/null +++ b/tools/element_values.py @@ -0,0 +1,368 @@ +#!/usr/bin/env python3 +"""Find the attribute values that every corpus IDML has on the elements +the converter writes: page items, spreads, pages, layers, swatches, styles +and the singleton elements of designmap.xml. + +Reads every IDML in the corpus inventory (distinct files only, the +privately held samples left out, and only files exported by InDesign) and collects, for each element path in +ELEMENTS, each attribute and each `Properties` child of every element on +that path. An element path is a tag (`TextFrame`), or a tag and the tag +of a child without `Self` (`Spread/FlattenerPreference`), as +tools/compare.py counts them. + +A value is kept for a range of DOM versions when, in every file of those +versions that has elements on the path, every such element has the value. +Ranges of consecutive versions with the same value are joined; a range +needs at least MIN_FILES files. A range that reaches the newest version +in the corpus is written without an upper bound, so later versions get +its values too. + +Paths in SINGLETONS are elements of which every IDML from some version on +has one (`Document/WatermarkPreference`); for them the presence of the +element is kept the same way, and the converter writes the element. +LISTS are elements with `Self` whose whole list (every element, with all +attributes and children, in order) is the same in every file of a range; +the converter writes the list. + +EXPLAINED lists values that are the same in nearly every element; the +others are values the converter reads from the INDD (the reason says +which). For them a value is kept when its share is at least +EXPLAINED_SHARE of the elements of each version, and the exceptions are +counted in the evidence. + +Usage: python3 -I tools/element_values.py [--write] [--key PATH ATTR] [--why PATH] +Prints the evidence counts. With --write, regenerates +src/idml/element_values.xml. --key prints the values of one key by +version; --why lists the values of a path that are left out, with the +share of the commonest value (None: element without it) and the +exceptions by version. See docs/format/idml-values.md. +""" + +import hashlib +import re +import sys +import xml.etree.ElementTree as ET +import zipfile +from collections import Counter, OrderedDict, defaultdict +from concurrent.futures import ProcessPoolExecutor +from pathlib import Path +from xml.sax.saxutils import quoteattr + +ROOT = Path(__file__).resolve().parent.parent +OUT = ROOT / "src" / "idml" / "element_values.xml" +# A range of versions is kept only when at least this many files show it. +MIN_FILES = 10 +# Paths under corpus/ left out: privately held samples. +EXCLUDE = ("own/",) + +# Element paths whose observed values the converter writes. +ELEMENTS = [ + "TextFrame", "Rectangle", "Oval", "Polygon", "GraphicLine", "Group", + "TextFrame/ObjectExportOption", "Rectangle/ObjectExportOption", + "Oval/ObjectExportOption", "Polygon/ObjectExportOption", + "GraphicLine/ObjectExportOption", "Group/ObjectExportOption", +] +# Of those, the elements every IDML has one of (from some version on). +SINGLETONS = {p for p in ELEMENTS if p.startswith("Document/")} +# Elements with Self written as a whole list. +LISTS = [] +# Values that are nearly constant; the exceptions are read from the INDD. +# (path, key) -> reason, as recorded in docs/format/idml-values.md. +EXPLAINED = {} +EXPLAINED_SHARE = 0.995 +# Attributes that are names or references of the element itself, or +# written by the converter from the document structure. +SKIP = {"Self", "Name"} +# Values the IDML schema the output is checked against does not allow +# (InDesign 8 and 9 wrote them): (tag, attribute). +SCHEMA_INVALID = {("ObjectExportOption", "CustomImageSizeOption")} + + +def exported_by_indesign(idml): + """Whether designmap.xml has the `product` that InDesign writes in its + `aid` processing instruction. One corpus IDML lacks it and differs + from InDesign's exports in other ways (it was written by a script).""" + head = zipfile.ZipFile(idml).read("designmap.xml")[:400] + return re.search(rb'<\?aid [^>]*product="', head) is not None + + +def corpus_idmls(): + """(DOM major, IDML path) of every distinct corpus IDML.""" + lines = (ROOT / "corpus" / "inventory.tsv").read_text().splitlines() + cols = lines[0].split("\t") + seen, out = set(), [] + for line in lines[1:]: + row = dict(zip(cols, line.split("\t"))) + dom = row.get("idml_dom", "") + if not dom or not dom[0].isdigit() or row["path"].startswith(EXCLUDE): + continue + idml = ROOT / "corpus" / (row.get("idml") or str(Path(row["path"]).with_suffix(".idml"))) + digest = hashlib.md5(idml.read_bytes()).hexdigest() + if digest not in seen and exported_by_indesign(idml): + seen.add(digest) + out.append((int(dom.split(".")[0]), str(idml))) + return out + + +def prop_text(p): + return re.sub(r">\s+<", "><", ET.tostring(p, encoding="unicode").strip()) + + +def flatten(el): + """Attributes as '@name' and Properties children as 'P/tag'.""" + out = {"@" + k: v for k, v in el.attrib.items()} + for c in el: + if c.tag == "Properties": + for p in c: + out["P/" + p.tag] = prop_text(p) + return out + + +def scan(arg): + """Per path: the flattened elements of one IDML; per list tag: the + serialized list.""" + dom, idml = arg + z = zipfile.ZipFile(idml) + found = defaultdict(list) + lists = defaultdict(list) + wanted = set(ELEMENTS) + for name in z.namelist(): + if not name.endswith(".xml"): + continue + root = ET.fromstring(z.read(name)) + for el in root.iter(): + if el.tag in LISTS: + lists[el.tag].append(prop_text(el)) + if el.get("Self") is None: + continue + if el.tag in wanted: + found[el.tag].append(flatten(el)) + for ch in el: + path = f"{el.tag}/{ch.tag}" + if ch.get("Self") is None and path in wanted: + found[path].append(flatten(ch)) + return dom, idml, dict(found), dict(lists) + + +def ranges(per_dom, files_per_dom, newest): + """Join consecutive versions with the same value. per_dom: {dom: + value or None (varies)}; versions without files are skipped. Returns + [(first, last or None, value, files)].""" + out = [] + for d in sorted(per_dom): + v = per_dom[d] + if out and out[-1][2] == v and v is not None: + first, _, _, n = out[-1] + out[-1] = (first, d, v, n + files_per_dom[d]) + else: + out.append((d, d, v, files_per_dom[d])) + kept = [] + for first, last, v, n in out: + if v is None or n < MIN_FILES: + continue + kept.append((first, None if last == newest else last, v, n)) + return kept + + +def analyse(scans): + doms = sorted({d for d, _, _, _ in scans}) + newest = doms[-1] + # (path, key) -> dom -> Counter of per-file status + status = defaultdict(lambda: defaultdict(Counter)) + elements = defaultdict(lambda: defaultdict(Counter)) # (path,key)->dom->value counts + present = defaultdict(lambda: defaultdict(int)) # path -> dom -> files with it + files = Counter(d for d, _, _, _ in scans) + counts = Counter() # path -> elements + order = defaultdict(dict) + for dom, _idml, found, _lists in scans: + for path, els in found.items(): + present[path][dom] += 1 + counts[path] += len(els) + keys = {} + for e in els: + for k in e: + keys.setdefault(k, None) + order[path].setdefault(k, len(order[path])) + for k in keys: + vals = [e.get(k) for e in els] + elements[(path, k)][dom].update(vals) + if None in vals: + status[(path, k)][dom]["partial"] += 1 + elif len(set(vals)) > 1: + status[(path, k)][dom]["varies"] += 1 + else: + status[(path, k)][dom]["=" + vals[0]] += 1 + kept = defaultdict(list) # path -> [(key, first, last, value, files)] + left_out = defaultdict(list) + for (path, key), by_dom in status.items(): + if key[1:] in SKIP and key.startswith("@"): + continue + if (path.split("/")[-1], key[1:]) in SCHEMA_INVALID: + continue + per_dom = {} + varies = False + for d in doms: + n = present[path].get(d, 0) + s = by_dom.get(d, Counter()) + if not n or not s: + continue + if (path, key) in EXPLAINED: + c = elements[(path, key)][d] + total = sum(c.values()) + v, k = c.most_common(1)[0] + if v is None or k / total < EXPLAINED_SHARE or sum(s.values()) != n: + varies = True + per_dom[d] = v + elif len(s) == 1 and sum(s.values()) == n and next(iter(s)).startswith("="): + per_dom[d] = next(iter(s))[1:] + else: + varies = True + if varies: + left_out[path].append(key) + continue + rs = ranges(per_dom, {d: present[path].get(d, 0) for d in doms}, newest) + for first, last, v, n in rs: + kept[path].append((key, first, last, v, n)) + presence = {} + for path in SINGLETONS: + per_dom = {d: (True if present[path].get(d, 0) == files[d] else None) for d in doms} + presence[path] = ranges(per_dom, files, newest) + return doms, files, kept, left_out, presence, order, elements, counts + + +def analyse_lists(scans, doms): + newest = doms[-1] + files = Counter(d for d, _, _, _ in scans) + out = {} + for tag in LISTS: + per_dom = {} + for d in doms: + lists = {tuple(ls.get(tag, ())) for dd, _, _, ls in scans if dd == d} + per_dom[d] = next(iter(lists)) if len(lists) == 1 and () not in lists else None + out[tag] = ranges(per_dom, files, newest) + return out + + +def node_xml(path_tag, attrs, props, indent): + a = "".join(f" {k}={quoteattr(v)}" for k, v in attrs) + if not props: + return f"{indent}<{path_tag}{a} />\n" + s = f"{indent}<{path_tag}{a}>\n{indent}\t\n" + s += "".join(f"{indent}\t\t{p}\n" for p in props) + return s + f"{indent}\t\n{indent}\n" + + +HEADER = """ + +""" + + +def write(kept, presence, lists, order): + blocks = defaultdict(lambda: defaultdict(list)) # (first,last) -> path -> [(key,v)] + for path, items in kept.items(): + for key, first, last, v, _n in items: + blocks[(first, last)][path].append((key, v)) + for path, rs in presence.items(): + for first, last, _v, _n in rs: + blocks[(first, last)].setdefault(path, []) + s = HEADER + "\n" + for (first, last) in sorted(blocks, key=lambda r: (r[0], r[1] or 999)): + gate = f' MinimumVersion="{first}"' + (f' MaximumVersion="{last}"' if last else "") + s += f"\t\n" + by_parent = OrderedDict() + for path in ELEMENTS: + if path not in blocks[(first, last)]: + continue + items = sorted(blocks[(first, last)][path], key=lambda kv: order[path].get(kv[0], 0)) + parent, _, child = path.partition("/") + by_parent.setdefault(parent, {"self": None, "children": []}) + pres = path in presence and any(f == first and l == last for f, l, _, _ in presence[path]) + attrs = [(k[1:], v) for k, v in items if k.startswith("@")] + if pres: + attrs = [("Present", "true")] + attrs + props = [v for k, v in items if k.startswith("P/")] + if child: + by_parent[parent]["children"].append((child, attrs, props)) + else: + by_parent[parent]["self"] = (attrs, props) + for parent, n in by_parent.items(): + attrs, props = n["self"] or ([], []) + if not n["children"]: + s += node_xml(parent, attrs, props, "\t\t") + continue + a = "".join(f" {k}={quoteattr(v)}" for k, v in attrs) + s += f"\t\t<{parent}{a}>\n" + if props: + s += "\t\t\t\n" + "".join(f"\t\t\t\t{p}\n" for p in props) + s += "\t\t\t\n" + for child, ca, cp in n["children"]: + s += node_xml(child, ca, cp, "\t\t\t") + s += f"\t\t\n" + s += "\t\n" + for tag, rs in lists.items(): + for first, last, lst, _n in rs: + gate = f' MinimumVersion="{first}"' + (f' MaximumVersion="{last}"' if last else "") + s += f'\t\n' + s += "".join(f"\t\t{e}\n" for e in lst) + s += "\t\n" + OUT.write_text(s + "\n") + + +def main(): + idmls = corpus_idmls() + with ProcessPoolExecutor() as pool: + scans = list(pool.map(scan, idmls, chunksize=4)) + doms, files, kept, left_out, presence, order, elements, counts = analyse(scans) + print(f"{len(idmls)} distinct IDML files; files per DOM version: " + + ", ".join(f"{d}: {files[d]}" for d in doms)) + if "--key" in sys.argv: + i = sys.argv.index("--key") + path, key = sys.argv[i + 1], sys.argv[i + 2] + for d in doms: + print(d, elements[(path, key)].get(d, Counter()).most_common(4)) + return + if "--why" in sys.argv: + path = sys.argv[sys.argv.index("--why") + 1] + for key in sorted(left_out.get(path, [])): + total = Counter() + for c in elements[(path, key)].values(): + total.update(c) + n = sum(total.values()) + v, k = total.most_common(1)[0] + odd = [f"{d}:{sum(c.values()) - c.most_common(1)[0][1]}/{sum(c.values())}" + for d, c in sorted(elements[(path, key)].items()) + if len(c) > 1] + print(f" {key[1:] if key[0] == '@' else key}: {k}/{n} {k / n:.4f} {str(v)[:40]!r} {' '.join(odd)}") + return + for path in ELEMENTS: + items = kept.get(path, []) + by_range = Counter((f, l, n) for _, f, l, _, n in items) + line = "; ".join(f"{k} from {f}{'' if l is None else f' to {l}'} ({n} files)" + for (f, l, n), k in sorted(by_range.items(), key=lambda x: (x[0][0], x[0][1] or 99))) + extra = "" + if path in presence: + extra = " present: " + ", ".join( + f"{f}{'+' if l is None else f'-{l}'} ({n} files)" for f, l, _, n in presence[path]) + print(f"{path} ({counts[path]} elements): kept {line}{extra}") + if left_out.get(path): + print(f" left out ({len(left_out[path])}): {', '.join(sorted(k[1:] if k[0] == '@' else k for k in left_out[path]))}") + lists = analyse_lists(scans, doms) + for tag, rs in lists.items(): + print(f"list {tag}: " + ", ".join( + f"{len(v)} elements, {f}{'+' if l is None else f'-{l}'} ({n} files)" for f, l, v, n in rs)) + if "--write" in sys.argv: + write(kept, presence, lists, order) + print(f"wrote {OUT.relative_to(ROOT)}") + + +if __name__ == "__main__": + main()