mirror of
https://github.com/storytold/indd-utils.git
synced 2026-11-19 16:50:48 +01:00
fix: count story positions in characters, not UTF-16 units
Text records count UTF-16 code units, but style runs, owned items and text owners count a surrogate pair as one position. Convert those positions to UTF-16 offsets so characters outside the Basic Multilingual Plane keep their formatting and are not split into two replacement characters.
This commit is contained in:
1 parent
201697c779
commit
56b59e1319
2 files changed
+69
-1
No files matched your search
@@ -532,6 +532,18 @@ converter does the same. The other fields of the chunk are not
|
||||
identified: `PinPosition` and `AnchorPoint` change together with three
|
||||
other u16 fields in the object styles.
|
||||
|
||||
**Positions count characters.** A text record's run length counts
|
||||
UTF-16 code units, but every other position counts characters, a
|
||||
surrogate pair being one: the lengths in a strand's chunk 0x261 (also
|
||||
for the text strand), and the run lengths of style runs, owned items and
|
||||
text owners. Evidence: the 6 stories of the little-endian corpus whose
|
||||
text has a character outside the Basic Multilingual Plane (emoji, all in
|
||||
one pair). In all 6 the text records add up to the number of UTF-16
|
||||
units, and the other strands and all chunk 0x261 lengths to the number
|
||||
of characters; with that reading, each emoji gets the character style
|
||||
range IDML gives it (6 of 6). The converter applies the same reading to
|
||||
hyperlink range trees, for which no sample with such a character exists.
|
||||
|
||||
INDD stores a forced line break as U+000A; IDML writes it as U+2028. The
|
||||
last paragraph return of a story is not written to IDML.
|
||||
|
||||
|
||||
+57
-1
@@ -1777,6 +1777,39 @@ impl<'a> Reader<'a> {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Text records count UTF-16 code units, every other strand counts
|
||||
// characters (a surrogate pair is one position). Convert the
|
||||
// positions to UTF-16 offsets.
|
||||
let offsets = char_offsets(&text);
|
||||
let unit = |p: usize| match offsets.get(p) {
|
||||
Some(&u) => u,
|
||||
None => text.len() + (p + 1 - offsets.len()),
|
||||
};
|
||||
let to_units = |list: Vec<StyleRun>| -> Vec<StyleRun> {
|
||||
let mut at = 0;
|
||||
list.into_iter()
|
||||
.map(|(len, style, attrs)| {
|
||||
let n = unit(at + len) - unit(at);
|
||||
at += len;
|
||||
(n, style, attrs)
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
let para = to_units(para);
|
||||
let chars = to_units(chars);
|
||||
let owned: Vec<_> = owned
|
||||
.into_iter()
|
||||
.map(|(p, cls, item)| (unit(p), cls, item))
|
||||
.collect();
|
||||
let owners: Vec<_> = owners
|
||||
.into_iter()
|
||||
.map(|(p, len, owner, cell)| (unit(p), unit(p + len) - unit(p), owner, cell))
|
||||
.collect();
|
||||
for r in &mut sources {
|
||||
let start = unit(r.start);
|
||||
r.len = unit(r.start + r.len) - start;
|
||||
r.start = start;
|
||||
}
|
||||
let mut anchors: BTreeMap<usize, Vec<PageItem>> = BTreeMap::new();
|
||||
let mut tables: BTreeMap<usize, Table> = BTreeMap::new();
|
||||
let mut text_variables = BTreeMap::new();
|
||||
@@ -1865,7 +1898,23 @@ impl<'a> Reader<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Find the first in-object string at or after `from`.
|
||||
/// The UTF-16 offset of each character of `text`, where a surrogate pair is
|
||||
/// one character, followed by the length of `text`.
|
||||
fn char_offsets(text: &[u16]) -> Vec<usize> {
|
||||
let mut out = Vec::with_capacity(text.len() + 1);
|
||||
let mut i = 0;
|
||||
while i < text.len() {
|
||||
out.push(i);
|
||||
let pair = (0xD800..0xDC00).contains(&text[i])
|
||||
&& text
|
||||
.get(i + 1)
|
||||
.is_some_and(|u| (0xDC00..0xE000).contains(u));
|
||||
i += if pair { 2 } else { 1 };
|
||||
}
|
||||
out.push(text.len());
|
||||
out
|
||||
}
|
||||
|
||||
/// A master page's applied master and its own margins and columns.
|
||||
type MasterPageLayout = (Option<u32>, Option<Margins>, Option<Columns>);
|
||||
|
||||
@@ -1934,6 +1983,7 @@ fn resolve_page_layout(spreads: &mut [Spread], masters: &mut [Spread]) {
|
||||
}
|
||||
}
|
||||
|
||||
/// Find the first in-object string at or after `from`.
|
||||
fn find_string(data: &[u8], from: usize) -> Result<String, Error> {
|
||||
for i in from..data.len().saturating_sub(4) {
|
||||
if data[i] == 2 && data[i + 1] == 0 {
|
||||
@@ -2093,6 +2143,12 @@ mod tests {
|
||||
assert_eq!(tops, [30.0, 20.0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn counts_a_surrogate_pair_as_one_position() {
|
||||
let text: Vec<u16> = "a\u{1F93F}b".encode_utf16().collect();
|
||||
assert_eq!(char_offsets(&text), [0, 1, 3, 4]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn splits_runs_at_style_boundaries() {
|
||||
let text: Vec<u16> = "abcdef".encode_utf16().collect();
|
||||
|
||||
Reference in new issue
Block a user