fix: count story positions in characters, not UTF-16 units

Text records count UTF-16 code units, but style runs, owned items and
text owners count a surrogate pair as one position. Convert those
positions to UTF-16 offsets so characters outside the Basic Multilingual
Plane keep their formatting and are not split into two replacement
characters.
This commit is contained in:
aiongg committed 2026-10-07 02:27:40 +00:00
1 parent 201697c779
commit 56b59e1319
2 files changed
+69 -1

No files matched your search

+12
View File
@@ -532,6 +532,18 @@ converter does the same. The other fields of the chunk are not
identified: `PinPosition` and `AnchorPoint` change together with three
other u16 fields in the object styles.
**Positions count characters.** A text record's run length counts
UTF-16 code units, but every other position counts characters, a
surrogate pair being one: the lengths in a strand's chunk 0x261 (also
for the text strand), and the run lengths of style runs, owned items and
text owners. Evidence: the 6 stories of the little-endian corpus whose
text has a character outside the Basic Multilingual Plane (emoji, all in
one pair). In all 6 the text records add up to the number of UTF-16
units, and the other strands and all chunk 0x261 lengths to the number
of characters; with that reading, each emoji gets the character style
range IDML gives it (6 of 6). The converter applies the same reading to
hyperlink range trees, for which no sample with such a character exists.
INDD stores a forced line break as U+000A; IDML writes it as U+2028. The
last paragraph return of a story is not written to IDML.
+57 -1
View File
@@ -1777,6 +1777,39 @@ impl<'a> Reader<'a> {
}
}
}
// Text records count UTF-16 code units, every other strand counts
// characters (a surrogate pair is one position). Convert the
// positions to UTF-16 offsets.
let offsets = char_offsets(&text);
let unit = |p: usize| match offsets.get(p) {
Some(&u) => u,
None => text.len() + (p + 1 - offsets.len()),
};
let to_units = |list: Vec<StyleRun>| -> Vec<StyleRun> {
let mut at = 0;
list.into_iter()
.map(|(len, style, attrs)| {
let n = unit(at + len) - unit(at);
at += len;
(n, style, attrs)
})
.collect()
};
let para = to_units(para);
let chars = to_units(chars);
let owned: Vec<_> = owned
.into_iter()
.map(|(p, cls, item)| (unit(p), cls, item))
.collect();
let owners: Vec<_> = owners
.into_iter()
.map(|(p, len, owner, cell)| (unit(p), unit(p + len) - unit(p), owner, cell))
.collect();
for r in &mut sources {
let start = unit(r.start);
r.len = unit(r.start + r.len) - start;
r.start = start;
}
let mut anchors: BTreeMap<usize, Vec<PageItem>> = BTreeMap::new();
let mut tables: BTreeMap<usize, Table> = BTreeMap::new();
let mut text_variables = BTreeMap::new();
@@ -1865,7 +1898,23 @@ impl<'a> Reader<'a> {
}
}
/// Find the first in-object string at or after `from`.
/// The UTF-16 offset of each character of `text`, where a surrogate pair is
/// one character, followed by the length of `text`.
fn char_offsets(text: &[u16]) -> Vec<usize> {
let mut out = Vec::with_capacity(text.len() + 1);
let mut i = 0;
while i < text.len() {
out.push(i);
let pair = (0xD800..0xDC00).contains(&text[i])
&& text
.get(i + 1)
.is_some_and(|u| (0xDC00..0xE000).contains(u));
i += if pair { 2 } else { 1 };
}
out.push(text.len());
out
}
/// A master page's applied master and its own margins and columns.
type MasterPageLayout = (Option<u32>, Option<Margins>, Option<Columns>);
@@ -1934,6 +1983,7 @@ fn resolve_page_layout(spreads: &mut [Spread], masters: &mut [Spread]) {
}
}
/// Find the first in-object string at or after `from`.
fn find_string(data: &[u8], from: usize) -> Result<String, Error> {
for i in from..data.len().saturating_sub(4) {
if data[i] == 2 && data[i + 1] == 0 {
@@ -2093,6 +2143,12 @@ mod tests {
assert_eq!(tops, [30.0, 20.0]);
}
#[test]
fn counts_a_surrogate_pair_as_one_position() {
let text: Vec<u16> = "a\u{1F93F}b".encode_utf16().collect();
assert_eq!(char_offsets(&text), [0, 1, 3, 4]);
}
#[test]
fn splits_runs_at_style_boundaries() {
let text: Vec<u16> = "abcdef".encode_utf16().collect();