diff --git a/Cargo.lock b/Cargo.lock index 0d6507970..de69d0765 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1634,6 +1634,7 @@ version = "0.1.0" name = "craft-captions" version = "0.1.0" dependencies = [ + "craft-media", "craft-media-testkit", "craft-timeline", "craft-xml", diff --git a/crates/captions/Cargo.toml b/crates/captions/Cargo.toml index cd09f7e16..1136f55f9 100644 --- a/crates/captions/Cargo.toml +++ b/crates/captions/Cargo.toml @@ -10,6 +10,11 @@ repository.workspace = true [lints] workspace = true +[features] +default = [] +# MP4 timed text (3GPP TS 26.245 / mov_text / tx3g) sample codec. +mov-text = [] + [dependencies] craft-timeline = { workspace = true } craft-xml = { workspace = true } @@ -21,3 +26,5 @@ proptest = { workspace = true } roxmltree = "0.21" # ffmpeg as the reference for the ASS tests (tests/it/ass.rs). craft-media-testkit = { workspace = true } +# craft-media isobmff for mov_text oracle demuxing/muxing tests. +craft-media = { workspace = true, features = ["isobmff"] } diff --git a/crates/captions/README.md b/crates/captions/README.md index 7380a3ce4..b2f6a195f 100644 --- a/crates/captions/README.md +++ b/crates/captions/README.md @@ -27,6 +27,10 @@ assert!(scc.starts_with(b"Scenarist_SCC V1.0")); and the SMPTE ST 334-2 caption distribution packet. - `plain_text` strips inline tags and decodes the common entities, for display or for formats without markup. +- Optional feature `mov-text` provides MP4 Timed Text (`mov_text` / `tx3g`, 3GPP TS 26.245) + sample decoding, encoding, modifier box parsing (`styl`, `hlit`, `hclr`, `krok`, `dlay`, `href`, + `tbox`, `blnk`, `wrap`), and `Tx3gConfig` sample entry parsing/writing for ISO BMFF (`.mp4`) and + QuickTime (`.mov`) subtitle tracks. ## Time diff --git a/crates/captions/src/lib.rs b/crates/captions/src/lib.rs index ca99dba43..2d0cf90c1 100644 --- a/crates/captions/src/lib.rs +++ b/crates/captions/src/lib.rs @@ -36,6 +36,8 @@ pub mod ass; pub mod cea608; pub mod jacosub; pub mod mcc; +#[cfg(feature = "mov-text")] +pub mod mov_text; pub mod pjs; pub mod realtext; pub mod scc; diff --git a/crates/captions/src/mov_text.rs b/crates/captions/src/mov_text.rs new file mode 100644 index 000000000..d1a3ce2a9 --- /dev/null +++ b/crates/captions/src/mov_text.rs @@ -0,0 +1,871 @@ +//! MP4 Timed Text (`mov_text` / `tx3g`, 3GPP TS 26.245). +//! +//! A clean-room reader and writer for 3GPP Timed Text subtitle samples and sample entry +//! configurations (`tx3g`), as carried in ISO BMFF (`.mp4`) and QuickTime (`.mov`) files. +//! +//! ## Sample Structure +//! +//! A timed text sample consists of: +//! - A 16-bit big-endian text length. +//! - The text string bytes (UTF-8, or UTF-16 with BOM). +//! - Zero or more modifier boxes: +//! - `styl`: Character style runs (bold, italic, underline, font ID, font size, text color). +//! - `hlit`: Highlighted character range. +//! - `hclr`: Highlight color (RGBA). +//! - `krok`: Karaoke highlight timing and character spans. +//! - `dlay`: Scroll delay in track-timescale ticks. +//! - `href`: Hypertext link URL and alt text. +//! - `tbox`: Text box override coordinates (`top, left, bottom, right`). +//! - `blnk`: Blinking text character range. +//! - `twrp`: Text wrap flag (0 = no wrap, 1 = automatic soft wrap). +//! +//! ## Style Flags +//! +//! Style flags in `styl` records: +//! - `0x01`: Bold (`...`) +//! - `0x02`: Italic (`...`) +//! - `0x04`: Underline (`...`) + +use crate::time::Tick; +use crate::{Cue, Document, TICKS_PER_SECOND}; + +/// What went wrong reading or writing timed text. +#[non_exhaustive] +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum Error { + /// The input slice is shorter than the declared field or box size. + #[error("truncated timed text data")] + Truncated, + /// The text payload is not valid UTF-8 and contains no valid UTF-16 BOM. + #[error("invalid text encoding in timed text sample")] + InvalidText, + /// The text exceeds the 16-bit length prefix (65,535 bytes). + #[error("timed text string exceeds 65535 bytes: {0}")] + TextTooLong(usize), + /// A modifier box header has an invalid size (< 8). + #[error("invalid modifier box size: {0}")] + InvalidBox(&'static str), +} + +/// `Result` alias for timed text operations. +pub type Result = std::result::Result; + +/// Face style flag: Bold (`0x01`). +pub const STYLE_FLAG_BOLD: u8 = 0x01; +/// Face style flag: Italic (`0x02`). +pub const STYLE_FLAG_ITALIC: u8 = 0x02; +/// Face style flag: Underline (`0x04`). +pub const STYLE_FLAG_UNDERLINE: u8 = 0x04; + +/// A character styling record within a `styl` box. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StyleRecord { + /// 0-indexed start character index (inclusive, measured in Unicode scalar values). + pub start_char: u16, + /// 0-indexed end character index (exclusive, measured in Unicode scalar values). + pub end_char: u16, + /// 16-bit font ID matching an entry in the sample description font table (`ftab`). + pub font_id: u16, + /// Face style flags: bit 0 = bold, bit 1 = italic, bit 2 = underline. + pub face_style_flags: u8, + /// Font size in points. + pub font_size: u8, + /// Text RGBA color (`[r, g, b, a]`). + pub text_color_rgba: [u8; 4], +} + +impl Default for StyleRecord { + fn default() -> Self { + Self { start_char: 0, end_char: 0, font_id: 1, face_style_flags: 0, font_size: 16, text_color_rgba: [255, 255, 255, 255] } + } +} + +impl StyleRecord { + /// Returns `true` if bold is set. + pub fn is_bold(&self) -> bool { + self.face_style_flags & STYLE_FLAG_BOLD != 0 + } + + /// Returns `true` if italic is set. + pub fn is_italic(&self) -> bool { + self.face_style_flags & STYLE_FLAG_ITALIC != 0 + } + + /// Returns `true` if underline is set. + pub fn is_underline(&self) -> bool { + self.face_style_flags & STYLE_FLAG_UNDERLINE != 0 + } +} + +/// Coordinates of a text display box (`tbox`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct TextBox { + pub top: i16, + pub left: i16, + pub bottom: i16, + pub right: i16, +} + +/// An entry in a karaoke box (`krok`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct KaraokeEntry { + /// Highlight end time in timescale ticks. + pub highlight_end_time: u32, + /// Start character index (inclusive). + pub start_char: u16, + /// End character index (exclusive). + pub end_char: u16, +} + +/// A hypertext link entry (`href`). +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct HyperText { + /// Start character index (inclusive). + pub start_char: u16, + /// End character index (exclusive). + pub end_char: u16, + /// Destination URL. + pub url: String, + /// Accessible alt text string. + pub alt_string: String, +} + +/// An unpacked 3GPP Timed Text (`tx3g` / `mov_text`) sample. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct TextSample { + /// Decoded UTF-8 text content. + pub text: String, + /// Character styling records from the `styl` box. + pub styles: Vec, + /// Highlighted character range `(start_char, end_char)` from `hlit`. + pub highlight: Option<(u16, u16)>, + /// Highlight color `[r, g, b, a]` from `hclr`. + pub highlight_color: Option<[u8; 4]>, + /// Karaoke timing and spans `(highlight_start_time, entries)` from `krok`. + pub karaoke: Option<(u32, Vec)>, + /// Scroll delay in milliseconds from `dlay`. + pub scroll_delay: Option, + /// Hypertext links from `href` boxes. + pub hypertext: Vec, + /// Text box override from `tbox`. + pub text_box: Option, + /// Blinking character ranges `(start_char, end_char)` from `blnk` boxes. + pub blinking: Vec<(u16, u16)>, + /// Wrap flag from `wrap` box: `0` = automatic wrap, `1` = no wrap. + pub wrap_flag: Option, +} + +/// Sample description table configuration for a `tx3g` track. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Tx3gConfig { + /// 32-bit display flags (e.g. scroll in, scroll out, continuous scroll). + pub display_flags: u32, + /// Horizontal justification: 0 = left, 1 = center, -1 = right. + pub horizontal_justification: i8, + /// Vertical justification: 0 = top, 1 = center, -1 = bottom. + pub vertical_justification: i8, + /// Background RGBA color. + pub background_color: [u8; 4], + /// Default text box coordinates. + pub default_text_box: TextBox, + /// Default style record. + pub default_style: StyleRecord, + /// Font table entries `(font_id, font_name)`. + pub fonts: Vec<(u16, String)>, +} + +impl Default for Tx3gConfig { + fn default() -> Self { + Self { + display_flags: 0, + horizontal_justification: 1, // center + vertical_justification: -1, // bottom + background_color: [0, 0, 0, 255], + default_text_box: TextBox::default(), + default_style: StyleRecord::default(), + fonts: vec![(1, "Arial".to_string())], + } + } +} + +impl Tx3gConfig { + /// Parse a `tx3g` sample entry payload (after the standard 8-byte sample entry header). + pub fn parse(payload: &[u8]) -> Result { + if payload.len() < 30 { + return Err(Error::Truncated); + } + let display_flags = u32::from_be_bytes([payload[0], payload[1], payload[2], payload[3]]); + let horizontal_justification = payload[4] as i8; + let vertical_justification = payload[5] as i8; + let background_color = [payload[6], payload[7], payload[8], payload[9]]; + + let default_text_box = TextBox { + top: i16::from_be_bytes([payload[10], payload[11]]), + left: i16::from_be_bytes([payload[12], payload[13]]), + bottom: i16::from_be_bytes([payload[14], payload[15]]), + right: i16::from_be_bytes([payload[16], payload[17]]), + }; + + let default_style = StyleRecord { + start_char: u16::from_be_bytes([payload[18], payload[19]]), + end_char: u16::from_be_bytes([payload[20], payload[21]]), + font_id: u16::from_be_bytes([payload[22], payload[23]]), + face_style_flags: payload[24], + font_size: payload[25], + text_color_rgba: [payload[26], payload[27], payload[28], payload[29]], + }; + + let mut fonts = Vec::new(); + let mut pos = 30; + while pos + 8 <= payload.len() { + let box_size = u32::from_be_bytes([payload[pos], payload[pos + 1], payload[pos + 2], payload[pos + 3]]) as usize; + if box_size < 8 || pos.checked_add(box_size).is_none_or(|end| end > payload.len()) { + break; + } + let box_type = &payload[pos + 4..pos + 8]; + if box_type == b"ftab" && box_size >= 10 { + let count = u16::from_be_bytes([payload[pos + 8], payload[pos + 9]]) as usize; + let mut fpos = pos + 10; + for _ in 0..count { + if fpos + 3 > pos + box_size { + break; + } + let font_id = u16::from_be_bytes([payload[fpos], payload[fpos + 1]]); + let name_len = payload[fpos + 2] as usize; + fpos += 3; + if fpos + name_len > pos + box_size { + break; + } + if let Ok(name) = std::str::from_utf8(&payload[fpos..fpos + name_len]) { + fonts.push((font_id, name.to_string())); + } + fpos += name_len; + } + } + pos += box_size; + } + + Ok(Self { display_flags, horizontal_justification, vertical_justification, background_color, default_text_box, default_style, fonts }) + } + + /// Write the `tx3g` sample entry payload (after the 8-byte sample entry header). + pub fn write(&self) -> Vec { + let mut out = Vec::with_capacity(30 + 32); + out.extend_from_slice(&self.display_flags.to_be_bytes()); + out.push(self.horizontal_justification as u8); + out.push(self.vertical_justification as u8); + out.extend_from_slice(&self.background_color); + + out.extend_from_slice(&self.default_text_box.top.to_be_bytes()); + out.extend_from_slice(&self.default_text_box.left.to_be_bytes()); + out.extend_from_slice(&self.default_text_box.bottom.to_be_bytes()); + out.extend_from_slice(&self.default_text_box.right.to_be_bytes()); + + out.extend_from_slice(&self.default_style.start_char.to_be_bytes()); + out.extend_from_slice(&self.default_style.end_char.to_be_bytes()); + out.extend_from_slice(&self.default_style.font_id.to_be_bytes()); + out.push(self.default_style.face_style_flags); + out.push(self.default_style.font_size); + out.extend_from_slice(&self.default_style.text_color_rgba); + + // ftab box + let mut ftab_payload = Vec::new(); + let fonts = &self.fonts[..self.fonts.len().min(u16::MAX as usize)]; + ftab_payload.extend_from_slice(&(fonts.len() as u16).to_be_bytes()); + for (id, name) in fonts { + ftab_payload.extend_from_slice(&id.to_be_bytes()); + let name_bytes = name.as_bytes(); + let limit = name.floor_char_boundary(name_bytes.len().min(255)); + ftab_payload.push(limit as u8); + ftab_payload.extend_from_slice(&name_bytes[..limit]); + } + let ftab_size = 8u32.saturating_add(ftab_payload.len() as u32); + out.extend_from_slice(&ftab_size.to_be_bytes()); + out.extend_from_slice(b"ftab"); + out.extend_from_slice(&ftab_payload); + + out + } +} + +/// Read and decode a single 3GPP Timed Text sample (`tx3g` / `mov_text`). +pub fn read_sample(data: &[u8]) -> Result { + if data.len() < 2 { + return Err(Error::Truncated); + } + + let text_len = u16::from_be_bytes([data[0], data[1]]) as usize; + if data.len() < 2 + text_len { + return Err(Error::Truncated); + } + + let text_bytes = &data[2..2 + text_len]; + let text = decode_text(text_bytes)?; + + let mut sample = TextSample { text, ..Default::default() }; + + let mut pos = 2 + text_len; + while pos + 8 <= data.len() { + let size = u32::from_be_bytes([data[pos], data[pos + 1], data[pos + 2], data[pos + 3]]) as usize; + if size < 8 { + return Err(Error::InvalidBox("size smaller than 8")); + } + let box_end = pos.checked_add(size).ok_or(Error::Truncated)?; + if box_end > data.len() { + return Err(Error::Truncated); + } + + let box_type = &data[pos + 4..pos + 8]; + let payload = &data[pos + 8..box_end]; + + match box_type { + b"styl" => { + if payload.len() < 2 { + return Err(Error::Truncated); + } + let count = u16::from_be_bytes([payload[0], payload[1]]) as usize; + let mut spos = 2; + for _ in 0..count { + if spos + 12 > payload.len() { + return Err(Error::Truncated); + } + sample.styles.push(StyleRecord { + start_char: u16::from_be_bytes([payload[spos], payload[spos + 1]]), + end_char: u16::from_be_bytes([payload[spos + 2], payload[spos + 3]]), + font_id: u16::from_be_bytes([payload[spos + 4], payload[spos + 5]]), + face_style_flags: payload[spos + 6], + font_size: payload[spos + 7], + text_color_rgba: [payload[spos + 8], payload[spos + 9], payload[spos + 10], payload[spos + 11]], + }); + spos += 12; + } + } + b"hlit" => { + if payload.len() < 4 { + return Err(Error::Truncated); + } + let start = u16::from_be_bytes([payload[0], payload[1]]); + let end = u16::from_be_bytes([payload[2], payload[3]]); + sample.highlight = Some((start, end)); + } + b"hclr" => { + if payload.len() < 4 { + return Err(Error::Truncated); + } + sample.highlight_color = Some([payload[0], payload[1], payload[2], payload[3]]); + } + b"krok" => { + if payload.len() < 6 { + return Err(Error::Truncated); + } + let start_time = u32::from_be_bytes([payload[0], payload[1], payload[2], payload[3]]); + let count = u16::from_be_bytes([payload[4], payload[5]]) as usize; + let mut entries = Vec::new(); + let mut kpos = 6; + for _ in 0..count { + if kpos + 8 > payload.len() { + return Err(Error::Truncated); + } + entries.push(KaraokeEntry { + highlight_end_time: u32::from_be_bytes([payload[kpos], payload[kpos + 1], payload[kpos + 2], payload[kpos + 3]]), + start_char: u16::from_be_bytes([payload[kpos + 4], payload[kpos + 5]]), + end_char: u16::from_be_bytes([payload[kpos + 6], payload[kpos + 7]]), + }); + kpos += 8; + } + sample.karaoke = Some((start_time, entries)); + } + b"dlay" => { + if payload.len() < 4 { + return Err(Error::Truncated); + } + sample.scroll_delay = Some(u32::from_be_bytes([payload[0], payload[1], payload[2], payload[3]])); + } + b"href" => { + if payload.len() < 6 { + return Err(Error::Truncated); + } + let start = u16::from_be_bytes([payload[0], payload[1]]); + let end = u16::from_be_bytes([payload[2], payload[3]]); + let url_len = payload[4] as usize; + if 5 + url_len > payload.len() { + return Err(Error::Truncated); + } + let url = std::str::from_utf8(&payload[5..5 + url_len]).map_err(|_| Error::InvalidText)?.to_string(); + let alt_pos = 5 + url_len; + let alt_len = payload.get(alt_pos).copied().unwrap_or(0) as usize; + let alt_string = if alt_len > 0 && alt_pos + 1 + alt_len <= payload.len() { + std::str::from_utf8(&payload[alt_pos + 1..alt_pos + 1 + alt_len]).map_err(|_| Error::InvalidText)?.to_string() + } else { + String::new() + }; + sample.hypertext.push(HyperText { start_char: start, end_char: end, url, alt_string }); + } + b"tbox" => { + if payload.len() < 8 { + return Err(Error::Truncated); + } + sample.text_box = Some(TextBox { + top: i16::from_be_bytes([payload[0], payload[1]]), + left: i16::from_be_bytes([payload[2], payload[3]]), + bottom: i16::from_be_bytes([payload[4], payload[5]]), + right: i16::from_be_bytes([payload[6], payload[7]]), + }); + } + b"blnk" => { + if payload.len() < 4 { + return Err(Error::Truncated); + } + let start = u16::from_be_bytes([payload[0], payload[1]]); + let end = u16::from_be_bytes([payload[2], payload[3]]); + sample.blinking.push((start, end)); + } + b"twrp" | b"wrap" => { + if payload.is_empty() { + return Err(Error::Truncated); + } + sample.wrap_flag = Some(payload[0]); + } + _ => { + // Unknown modifier box: safely ignored. + } + } + + pos = box_end; + } + + Ok(sample) +} + +/// Helper to decode UTF-8 or UTF-16 with BOM. +fn decode_text(bytes: &[u8]) -> Result { + if bytes.len() >= 2 { + if bytes[0] == 0xFE && bytes[1] == 0xFF { + // UTF-16 BE + let u16s: Vec = bytes[2..].as_chunks::<2>().0.iter().map(|c| u16::from_be_bytes([c[0], c[1]])).collect(); + return String::from_utf16(&u16s).map_err(|_| Error::InvalidText); + } else if bytes[0] == 0xFF && bytes[1] == 0xFE { + // UTF-16 LE + let u16s: Vec = bytes[2..].as_chunks::<2>().0.iter().map(|c| u16::from_le_bytes([c[0], c[1]])).collect(); + return String::from_utf16(&u16s).map_err(|_| Error::InvalidText); + } + } + std::str::from_utf8(bytes).map(ToString::to_string).map_err(|_| Error::InvalidText) +} + +/// Pack and encode a timed text sample into bytes. +pub fn write_sample(sample: &TextSample) -> Result> { + let text_bytes = sample.text.as_bytes(); + if text_bytes.len() > u16::MAX as usize { + return Err(Error::TextTooLong(text_bytes.len())); + } + + let mut out = Vec::with_capacity(2 + text_bytes.len() + 64); + out.extend_from_slice(&(text_bytes.len() as u16).to_be_bytes()); + out.extend_from_slice(text_bytes); + + // 1. styl box + if !sample.styles.is_empty() { + if sample.styles.len() > u16::MAX as usize { + return Err(Error::InvalidBox("too many styles in sample")); + } + let size = 8u32.checked_add(2).and_then(|s| s.checked_add(sample.styles.len() as u32 * 12)).ok_or(Error::InvalidBox("styl size overflow"))?; + out.extend_from_slice(&size.to_be_bytes()); + out.extend_from_slice(b"styl"); + out.extend_from_slice(&(sample.styles.len() as u16).to_be_bytes()); + for st in &sample.styles { + out.extend_from_slice(&st.start_char.to_be_bytes()); + out.extend_from_slice(&st.end_char.to_be_bytes()); + out.extend_from_slice(&st.font_id.to_be_bytes()); + out.push(st.face_style_flags); + out.push(st.font_size); + out.extend_from_slice(&st.text_color_rgba); + } + } + + // 2. hlit box + if let Some((start, end)) = sample.highlight { + out.extend_from_slice(&12u32.to_be_bytes()); + out.extend_from_slice(b"hlit"); + out.extend_from_slice(&start.to_be_bytes()); + out.extend_from_slice(&end.to_be_bytes()); + } + + // 3. hclr box + if let Some(color) = sample.highlight_color { + out.extend_from_slice(&12u32.to_be_bytes()); + out.extend_from_slice(b"hclr"); + out.extend_from_slice(&color); + } + + // 4. krok box + if let Some((start_time, ref entries)) = sample.karaoke { + if entries.len() > u16::MAX as usize { + return Err(Error::InvalidBox("too many karaoke entries")); + } + let size = 8u32.checked_add(6).and_then(|s| s.checked_add(entries.len() as u32 * 8)).ok_or(Error::InvalidBox("krok size overflow"))?; + out.extend_from_slice(&size.to_be_bytes()); + out.extend_from_slice(b"krok"); + out.extend_from_slice(&start_time.to_be_bytes()); + out.extend_from_slice(&(entries.len() as u16).to_be_bytes()); + for e in entries { + out.extend_from_slice(&e.highlight_end_time.to_be_bytes()); + out.extend_from_slice(&e.start_char.to_be_bytes()); + out.extend_from_slice(&e.end_char.to_be_bytes()); + } + } + + // 5. dlay box + if let Some(delay) = sample.scroll_delay { + out.extend_from_slice(&12u32.to_be_bytes()); + out.extend_from_slice(b"dlay"); + out.extend_from_slice(&delay.to_be_bytes()); + } + + // 6. href boxes + for link in &sample.hypertext { + let url_bytes = link.url.as_bytes(); + let alt_bytes = link.alt_string.as_bytes(); + let ulen = url_bytes.len().min(255) as u8; + let alen = alt_bytes.len().min(255) as u8; + let size = 8u32 + 4 + 1 + ulen as u32 + 1 + alen as u32; + out.extend_from_slice(&size.to_be_bytes()); + out.extend_from_slice(b"href"); + out.extend_from_slice(&link.start_char.to_be_bytes()); + out.extend_from_slice(&link.end_char.to_be_bytes()); + out.push(ulen); + out.extend_from_slice(&url_bytes[..ulen as usize]); + out.push(alen); + out.extend_from_slice(&alt_bytes[..alen as usize]); + } + + // 7. tbox box + if let Some(tb) = sample.text_box { + out.extend_from_slice(&16u32.to_be_bytes()); + out.extend_from_slice(b"tbox"); + out.extend_from_slice(&tb.top.to_be_bytes()); + out.extend_from_slice(&tb.left.to_be_bytes()); + out.extend_from_slice(&tb.bottom.to_be_bytes()); + out.extend_from_slice(&tb.right.to_be_bytes()); + } + + // 8. blnk boxes + for &(start, end) in &sample.blinking { + out.extend_from_slice(&12u32.to_be_bytes()); + out.extend_from_slice(b"blnk"); + out.extend_from_slice(&start.to_be_bytes()); + out.extend_from_slice(&end.to_be_bytes()); + } + + // 9. twrp box + if let Some(flag) = sample.wrap_flag { + out.extend_from_slice(&9u32.to_be_bytes()); + out.extend_from_slice(b"twrp"); + out.push(flag); + } + + Ok(out) +} + +fn push_escaped_char(out: &mut String, ch: char) { + match ch { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + _ => out.push(ch), + } +} + +/// Convert a [`TextSample`] into HTML-style formatted markup (``, ``, ``, ``). +pub fn sample_to_markup(sample: &TextSample) -> String { + if sample.styles.is_empty() { + let mut out = String::with_capacity(sample.text.len()); + for ch in sample.text.chars() { + push_escaped_char(&mut out, ch); + } + return out; + } + + let chars: Vec = sample.text.chars().collect(); + let n = chars.len(); + if n == 0 { + return String::new(); + } + + let mut starts: std::collections::HashMap> = std::collections::HashMap::new(); + let mut ends: std::collections::HashMap> = std::collections::HashMap::new(); + for st in &sample.styles { + if st.end_char > st.start_char { + starts.entry(st.start_char).or_default().push(st); + ends.entry(st.end_char).or_default().push(st); + } + } + + let mut out = String::with_capacity(sample.text.len() + sample.styles.len() * 16); + + for (i, &ch) in chars.iter().enumerate() { + let char_idx = i as u16; + + // Open tags for any style starting at char_idx + if let Some(list) = starts.get(&char_idx) { + for st in list { + let has_color = st.text_color_rgba != [255, 255, 255, 255]; + if has_color { + out.push_str(&format!( + "", + st.text_color_rgba[0], st.text_color_rgba[1], st.text_color_rgba[2] + )); + } + if st.is_bold() { + out.push_str(""); + } + if st.is_italic() { + out.push_str(""); + } + if st.is_underline() { + out.push_str(""); + } + } + } + + push_escaped_char(&mut out, ch); + + // Close tags for any style ending at char_idx + 1 + if let Some(list) = ends.get(&(char_idx + 1)) { + for st in list.iter().rev() { + if st.is_underline() { + out.push_str(""); + } + if st.is_italic() { + out.push_str(""); + } + if st.is_bold() { + out.push_str(""); + } + let has_color = st.text_color_rgba != [255, 255, 255, 255]; + if has_color { + out.push_str(""); + } + } + } + } + + out +} + +#[derive(Clone, Copy, PartialEq, Eq)] +struct CharFormatting { + flags: u8, + color: [u8; 4], +} + +impl Default for CharFormatting { + fn default() -> Self { + Self { flags: 0, color: [255, 255, 255, 255] } + } +} + +/// Parse HTML-style formatted markup (``, ``, ``, ``) into a [`TextSample`]. +pub fn markup_to_sample(markup: &str) -> TextSample { + let mut bold_depth = 0usize; + let mut italic_depth = 0usize; + let mut underline_depth = 0usize; + let mut color_stack: Vec<[u8; 4]> = Vec::new(); + + let mut decoded_chars: Vec = Vec::with_capacity(markup.len()); + let mut char_styles: Vec = Vec::with_capacity(markup.len()); + + let mut chars = markup.chars().peekable(); + + while let Some(ch) = chars.next() { + if ch == '<' { + let mut tag = String::new(); + for next in chars.by_ref() { + if next == '>' { + break; + } + tag.push(next); + } + let trimmed = tag.trim().to_ascii_lowercase(); + if trimmed == "b" { + bold_depth += 1; + } else if trimmed == "/b" { + bold_depth = bold_depth.saturating_sub(1); + } else if trimmed == "i" { + italic_depth += 1; + } else if trimmed == "/i" { + italic_depth = italic_depth.saturating_sub(1); + } else if trimmed == "u" { + underline_depth += 1; + } else if trimmed == "/u" { + underline_depth = underline_depth.saturating_sub(1); + } else if trimmed.starts_with("font") { + if let Some(color) = parse_font_color(&tag) { + color_stack.push(color); + } + } else if trimmed == "/font" { + color_stack.pop(); + } + } else { + let mut text_chunk = String::new(); + text_chunk.push(ch); + while let Some(&next) = chars.peek() { + if next == '<' { + break; + } + text_chunk.push(next); + chars.next(); + } + + let decoded = crate::decode_entities(&text_chunk); + let mut flags = 0u8; + if bold_depth > 0 { + flags |= STYLE_FLAG_BOLD; + } + if italic_depth > 0 { + flags |= STYLE_FLAG_ITALIC; + } + if underline_depth > 0 { + flags |= STYLE_FLAG_UNDERLINE; + } + let color = color_stack.last().copied().unwrap_or([255, 255, 255, 255]); + let fmt = CharFormatting { flags, color }; + + for c in decoded.chars() { + decoded_chars.push(c); + char_styles.push(fmt); + } + } + } + + let text: String = decoded_chars.iter().collect(); + + // Group adjacent characters with identical non-default formatting into sorted, non-overlapping style runs. + let mut styles = Vec::new(); + let mut run_start: Option<(usize, CharFormatting)> = None; + + for (idx, &fmt) in char_styles.iter().enumerate() { + let is_styled = fmt.flags != 0 || fmt.color != [255, 255, 255, 255]; + match run_start { + Some((start, current_fmt)) => { + if fmt != current_fmt { + if current_fmt.flags != 0 || current_fmt.color != [255, 255, 255, 255] { + styles.push(StyleRecord { + start_char: start as u16, + end_char: idx as u16, + font_id: 1, + face_style_flags: current_fmt.flags, + font_size: 16, + text_color_rgba: current_fmt.color, + }); + } + run_start = if is_styled { Some((idx, fmt)) } else { None }; + } + } + None => { + if is_styled { + run_start = Some((idx, fmt)); + } + } + } + } + + if let Some((start, fmt)) = run_start { + styles.push(StyleRecord { + start_char: start as u16, + end_char: char_styles.len() as u16, + font_id: 1, + face_style_flags: fmt.flags, + font_size: 16, + text_color_rgba: fmt.color, + }); + } + + TextSample { text, styles, ..Default::default() } +} + +/// Helper to parse color attribute from `` tag. +fn parse_font_color(tag: &str) -> Option<[u8; 4]> { + let lower = tag.to_ascii_lowercase(); + let col_idx = lower.find("color=")?; + let rest = tag[col_idx + 6..].trim_start(); + let val = if let Some(stripped) = rest.strip_prefix('"') { + let end = stripped.find('"')?; + &stripped[..end] + } else if let Some(stripped) = rest.strip_prefix('\'') { + let end = stripped.find('\'')?; + &stripped[..end] + } else { + let end = rest.find(char::is_whitespace).unwrap_or(rest.len()); + &rest[..end] + }; + + let hex = val.strip_prefix('#').unwrap_or(val); + if hex.len() == 6 { + let r = u8::from_str_radix(hex.get(0..2)?, 16).ok()?; + let g = u8::from_str_radix(hex.get(2..4)?, 16).ok()?; + let b = u8::from_str_radix(hex.get(4..6)?, 16).ok()?; + Some([r, g, b, 255]) + } else if hex.len() == 8 { + let r = u8::from_str_radix(hex.get(0..2)?, 16).ok()?; + let g = u8::from_str_radix(hex.get(2..4)?, 16).ok()?; + let b = u8::from_str_radix(hex.get(4..6)?, 16).ok()?; + let a = u8::from_str_radix(hex.get(6..8)?, 16).ok()?; + Some([r, g, b, a]) + } else { + None + } +} + +/// Convert a caption [`Cue`] into a [`TextSample`]. +pub fn cue_to_sample(cue: &Cue) -> TextSample { + markup_to_sample(&cue.text) +} + +/// Convert a [`TextSample`] into a caption [`Cue`] spanning `start` to `end`. +pub fn sample_to_cue(sample: &TextSample, start: Tick, end: Tick) -> Cue { + Cue { start, end, text: sample_to_markup(sample), speaker: None, id: None, settings: String::new() } +} + +/// Decode a stream of timed text samples into a [`Document`]. +pub fn decode_timed_samples<'a, I>(doc: &mut Document, samples: I, timescale: u32) -> Result<()> +where + I: IntoIterator, +{ + decode_timed_samples_with_config(doc, samples, timescale, None) +} + +/// Decode a stream of timed text samples into a [`Document`], applying an optional sample description configuration. +pub fn decode_timed_samples_with_config<'a, I>(doc: &mut Document, samples: I, timescale: u32, config: Option<&Tx3gConfig>) -> Result<()> +where + I: IntoIterator, +{ + if timescale == 0 { + return Ok(()); + } + let ts = timescale as i128; + for (pts, duration, data) in samples { + let mut sample = read_sample(data)?; + if sample.text.trim().is_empty() { + continue; + } + if sample.styles.is_empty() + && let Some(cfg) = config + && (cfg.default_style.face_style_flags != 0 || cfg.default_style.text_color_rgba != [255, 255, 255, 255]) + { + sample.styles.push(StyleRecord { + start_char: 0, + end_char: sample.text.chars().count() as u16, + font_id: cfg.default_style.font_id, + face_style_flags: cfg.default_style.face_style_flags, + font_size: cfg.default_style.font_size, + text_color_rgba: cfg.default_style.text_color_rgba, + }); + } + let end_pts = pts.checked_add(duration as u64).ok_or(Error::InvalidBox("timestamp interval overflow"))?; + let start_ticks = (pts as i128 * TICKS_PER_SECOND as i128 / ts) as i64; + let end_ticks = (end_pts as i128 * TICKS_PER_SECOND as i128 / ts) as i64; + doc.cues.push(sample_to_cue(&sample, Tick(start_ticks), Tick(end_ticks))); + } + Ok(()) +} diff --git a/crates/captions/tests/it/main.rs b/crates/captions/tests/it/main.rs index 437b2ba67..7d8070cad 100644 --- a/crates/captions/tests/it/main.rs +++ b/crates/captions/tests/it/main.rs @@ -2,6 +2,8 @@ mod ass; mod limits; mod literal_roundtrips; mod mcc_708_timing; +#[cfg(feature = "mov-text")] +mod mov_text; mod pjs_realtext_jacosub; mod roundtrip; mod ttml_body_styles; diff --git a/crates/captions/tests/it/mov_text.rs b/crates/captions/tests/it/mov_text.rs new file mode 100644 index 000000000..dfd5f73ea --- /dev/null +++ b/crates/captions/tests/it/mov_text.rs @@ -0,0 +1,349 @@ +//! Clean-room tests and FFmpeg oracle tests for MP4 Timed Text (`mov_text` / `tx3g`). + +use craft_captions::mov_text::{ + Error, KaraokeEntry, STYLE_FLAG_BOLD, STYLE_FLAG_ITALIC, STYLE_FLAG_UNDERLINE, StyleRecord, TextBox, TextSample, Tx3gConfig, + decode_timed_samples, markup_to_sample, read_sample, sample_to_markup, write_sample, +}; +use craft_captions::time::Tick; +use craft_captions::{Document, TICKS_PER_SECOND}; + +#[test] +fn test_plain_text_roundtrip() { + let sample = TextSample { text: "Hello world without styles".to_string(), ..Default::default() }; + + let bytes = write_sample(&sample).expect("write sample"); + assert_eq!(bytes.len(), 2 + sample.text.len()); + assert_eq!(&bytes[..2], &(sample.text.len() as u16).to_be_bytes()); + assert_eq!(&bytes[2..], sample.text.as_bytes()); + + let decoded = read_sample(&bytes).expect("read sample"); + assert_eq!(decoded.text, sample.text); + assert!(decoded.styles.is_empty()); +} + +#[test] +fn test_empty_sample_roundtrip() { + let sample = TextSample::default(); + let bytes = write_sample(&sample).expect("write"); + assert_eq!(bytes, &[0x00, 0x00]); + + let decoded = read_sample(&bytes).expect("read"); + assert_eq!(decoded.text, ""); + assert!(decoded.styles.is_empty()); +} + +#[test] +fn test_styles_roundtrip() { + let sample = TextSample { + text: "Bold, Italic, Underline, and Colored".to_string(), + styles: vec![ + StyleRecord { + start_char: 0, + end_char: 4, + font_id: 1, + face_style_flags: STYLE_FLAG_BOLD, + font_size: 16, + text_color_rgba: [255, 255, 255, 255], + }, + StyleRecord { + start_char: 6, + end_char: 12, + font_id: 1, + face_style_flags: STYLE_FLAG_ITALIC, + font_size: 18, + text_color_rgba: [255, 255, 255, 255], + }, + StyleRecord { + start_char: 14, + end_char: 23, + font_id: 1, + face_style_flags: STYLE_FLAG_UNDERLINE, + font_size: 16, + text_color_rgba: [255, 255, 255, 255], + }, + StyleRecord { start_char: 29, end_char: 36, font_id: 1, face_style_flags: 0, font_size: 20, text_color_rgba: [255, 0, 0, 255] }, + ], + ..Default::default() + }; + + let bytes = write_sample(&sample).expect("write"); + let decoded = read_sample(&bytes).expect("read"); + + assert_eq!(decoded.text, sample.text); + assert_eq!(decoded.styles.len(), 4); + assert_eq!(decoded.styles, sample.styles); + + assert!(decoded.styles[0].is_bold()); + assert!(!decoded.styles[0].is_italic()); + assert!(decoded.styles[1].is_italic()); + assert!(decoded.styles[2].is_underline()); + assert_eq!(decoded.styles[3].text_color_rgba, [255, 0, 0, 255]); +} + +#[test] +fn test_all_modifier_boxes_roundtrip() { + let sample = TextSample { + text: "Full modifiers test".to_string(), + styles: vec![StyleRecord { + start_char: 0, + end_char: 4, + font_id: 1, + face_style_flags: STYLE_FLAG_BOLD, + font_size: 16, + text_color_rgba: [255, 255, 255, 255], + }], + highlight: Some((5, 14)), + highlight_color: Some([255, 255, 0, 255]), + karaoke: Some(( + 100, + vec![ + KaraokeEntry { highlight_end_time: 200, start_char: 0, end_char: 4 }, + KaraokeEntry { highlight_end_time: 350, start_char: 5, end_char: 14 }, + ], + )), + scroll_delay: Some(500), + hypertext: vec![craft_captions::mov_text::HyperText { + start_char: 0, + end_char: 19, + url: "https://example.com".to_string(), + alt_string: "Example".to_string(), + }], + text_box: Some(TextBox { top: 10, left: 20, bottom: 100, right: 200 }), + blinking: vec![(0, 4), (15, 19)], + wrap_flag: Some(1), + }; + + let bytes = write_sample(&sample).expect("write"); + let decoded = read_sample(&bytes).expect("read"); + + assert_eq!(decoded, sample); +} + +#[test] +fn test_utf16_bom_reading() { + // UTF-16 BE with BOM + let text = "Hello UTF-16"; + let mut data_be = Vec::new(); + let utf16_be_bytes: Vec = text.encode_utf16().flat_map(|u| u.to_be_bytes()).collect(); + let total_len = 2 + utf16_be_bytes.len(); + data_be.extend_from_slice(&(total_len as u16).to_be_bytes()); + data_be.extend_from_slice(&[0xFE, 0xFF]); + data_be.extend_from_slice(&utf16_be_bytes); + + let decoded_be = read_sample(&data_be).expect("read UTF-16 BE"); + assert_eq!(decoded_be.text, text); + + // UTF-16 LE with BOM + let mut data_le = Vec::new(); + let utf16_le_bytes: Vec = text.encode_utf16().flat_map(|u| u.to_le_bytes()).collect(); + data_le.extend_from_slice(&(total_len as u16).to_be_bytes()); + data_le.extend_from_slice(&[0xFF, 0xFE]); + data_le.extend_from_slice(&utf16_le_bytes); + + let decoded_le = read_sample(&data_le).expect("read UTF-16 LE"); + assert_eq!(decoded_le.text, text); +} + +#[test] +fn test_markup_conversion() { + let markup = "Hello Bold and Italic and Underline and Red"; + let sample = markup_to_sample(markup); + + assert_eq!(sample.text, "Hello Bold and Italic and Underline and Red"); + assert_eq!(sample.styles.len(), 4); + + let bold = &sample.styles[0]; + assert_eq!(&sample.text[bold.start_char as usize..bold.end_char as usize], "Bold"); + assert!(bold.is_bold()); + + let italic = &sample.styles[1]; + assert_eq!(&sample.text[italic.start_char as usize..italic.end_char as usize], "Italic"); + assert!(italic.is_italic()); + + let underline = &sample.styles[2]; + assert_eq!(&sample.text[underline.start_char as usize..underline.end_char as usize], "Underline"); + assert!(underline.is_underline()); + + let red = &sample.styles[3]; + assert_eq!(&sample.text[red.start_char as usize..red.end_char as usize], "Red"); + assert_eq!(red.text_color_rgba, [255, 0, 0, 255]); + + let regenerated = sample_to_markup(&sample); + assert_eq!(regenerated, markup); +} + +#[test] +fn test_nested_styles_sorted_non_overlapping() { + let markup = "ABC"; + let sample = markup_to_sample(markup); + assert_eq!(sample.text, "ABC"); + assert_eq!(sample.styles.len(), 3); + // [0, 1): A is bold + assert_eq!(sample.styles[0].start_char, 0); + assert_eq!(sample.styles[0].end_char, 1); + assert_eq!(sample.styles[0].face_style_flags, STYLE_FLAG_BOLD); + // [1, 2): B is bold + italic + assert_eq!(sample.styles[1].start_char, 1); + assert_eq!(sample.styles[1].end_char, 2); + assert_eq!(sample.styles[1].face_style_flags, STYLE_FLAG_BOLD | STYLE_FLAG_ITALIC); + // [2, 3): C is bold + assert_eq!(sample.styles[2].start_char, 2); + assert_eq!(sample.styles[2].end_char, 3); + assert_eq!(sample.styles[2].face_style_flags, STYLE_FLAG_BOLD); +} + +#[test] +fn test_entities_and_literal_escaping() { + let markup = "A & B < C"; + let sample = markup_to_sample(markup); + assert_eq!(sample.text, "A & B < C"); + assert_eq!(sample.styles.len(), 1); + assert_eq!(sample.styles[0].start_char, 4); + assert_eq!(sample.styles[0].end_char, 9); + + let roundtripped = sample_to_markup(&sample); + assert_eq!(roundtripped, "A & B < C"); +} + +#[test] +fn test_twrp_box() { + let sample = TextSample { text: "Wrap me".to_string(), wrap_flag: Some(1), ..Default::default() }; + let bytes = write_sample(&sample).expect("write sample"); + // Verify bytes contain twrp box + assert!(bytes.windows(4).any(|w| w == b"twrp")); + + let read_back = read_sample(&bytes).expect("read sample"); + assert_eq!(read_back.wrap_flag, Some(1)); +} + +#[test] +fn test_tx3g_config_roundtrip() { + let cfg = Tx3gConfig { + display_flags: 0x0000_0001, + horizontal_justification: 1, + vertical_justification: -1, + background_color: [10, 20, 30, 255], + default_text_box: TextBox { top: 0, left: 0, bottom: 480, right: 640 }, + default_style: StyleRecord { + start_char: 0, + end_char: 0, + font_id: 1, + face_style_flags: STYLE_FLAG_BOLD, + font_size: 24, + text_color_rgba: [255, 255, 255, 255], + }, + fonts: vec![(1, "Serif".to_string()), (2, "Sans".to_string())], + }; + + let bytes = cfg.write(); + let parsed = Tx3gConfig::parse(&bytes).expect("parse config"); + assert_eq!(parsed, cfg); +} + +#[test] +fn test_decode_timed_samples_to_document() { + let mut doc = Document::default(); + + let s1 = write_sample(&markup_to_sample("First cue")).expect("s1"); + let s2 = write_sample(&markup_to_sample("Second cue")).expect("s2"); + + let samples = vec![(0u64, 1000u32, s1.as_slice()), (1000u64, 1500u32, s2.as_slice())]; + + decode_timed_samples(&mut doc, samples, 1000).expect("decode samples"); + + assert_eq!(doc.cues.len(), 2); + assert_eq!(doc.cues[0].start, Tick(0)); + assert_eq!(doc.cues[0].end, Tick(TICKS_PER_SECOND)); + assert_eq!(doc.cues[0].text, "First cue"); + + assert_eq!(doc.cues[1].start, Tick(TICKS_PER_SECOND)); + assert_eq!(doc.cues[1].end, Tick((TICKS_PER_SECOND as f64 * 2.5) as i64)); + assert_eq!(doc.cues[1].text, "Second cue"); +} + +#[test] +fn test_hostile_inputs_never_panic() { + // Empty input + assert!(matches!(read_sample(&[]), Err(Error::Truncated))); + + // 1-byte input + assert!(matches!(read_sample(&[0x00]), Err(Error::Truncated))); + + // Length says 10 bytes, but slice ends + assert!(matches!(read_sample(&[0x00, 0x0A, 0x41]), Err(Error::Truncated))); + + // Invalid UTF-8 without BOM + assert!(matches!(read_sample(&[0x00, 0x02, 0xFF, 0xFF]), Err(Error::InvalidText))); + + // Box size < 8 + let mut bad_box = Vec::new(); + bad_box.extend_from_slice(&[0x00, 0x04, b'T', b'e', b's', b't']); + bad_box.extend_from_slice(&[0x00, 0x00, 0x00, 0x04, b's', b't', b'y', b'l']); + assert!(matches!(read_sample(&bad_box), Err(Error::InvalidBox(_)))); + + // Box size exceeds remaining slice + let mut truncated_box = Vec::new(); + truncated_box.extend_from_slice(&[0x00, 0x04, b'T', b'e', b's', b't']); + truncated_box.extend_from_slice(&[0x00, 0x00, 0x01, 0x00, b's', b't', b'y', b'l']); + assert!(matches!(read_sample(&truncated_box), Err(Error::Truncated))); + + // Unknown modifier boxes are safely ignored without error + let mut unknown_box = Vec::new(); + unknown_box.extend_from_slice(&[0x00, 0x04, b'T', b'e', b's', b't']); + unknown_box.extend_from_slice(&[0x00, 0x00, 0x00, 0x0C, b'f', b'a', b'k', b'e', 1, 2, 3, 4]); + let decoded = read_sample(&unknown_box).expect("unknown box should be skipped"); + assert_eq!(decoded.text, "Test"); +} + +#[test] +fn test_ffmpeg_mov_text_oracle() { + let Some(ff) = craft_media_testkit::ffmpeg() else { return }; + let temp_dir = std::env::temp_dir().join("craft_mov_text_test"); + let _ = std::fs::create_dir_all(&temp_dir); + + // 1. Create SRT file with formatting: bold, italic, underline + let srt_path = temp_dir.join("input.srt"); + let mp4_path = temp_dir.join("output.mp4"); + let srt_content = "1\n00:00:01,000 --> 00:00:03,000\nHello World and Italic and Underline\n"; + std::fs::write(&srt_path, srt_content).expect("write srt"); + + // Encode to MP4 with mov_text + let out = std::process::Command::new(&ff) + .args(["-y", "-i", srt_path.to_str().expect("path"), "-c:s", "mov_text", mp4_path.to_str().expect("path")]) + .output(); + let Ok(output) = out else { return }; + if !output.status.success() { + return; + } + + // Read MP4 through craft-media ISOBMFF demuxer + let mp4_bytes = std::fs::read(&mp4_path).expect("read mp4"); + let mp4 = craft_media::isobmff::open(&mp4_bytes).expect("open mp4"); + + let (track_idx, track) = + mp4.tracks.iter().enumerate().find(|(_, t)| t.kind == craft_media::isobmff::TrackKind::Subtitle).expect("subtitle track"); + + // Decode all non-empty samples + let mut found_styled_cue = false; + for sample_idx in 0..track.samples.len() { + let sample_data = mp4.read_sample(&mp4_bytes, track_idx, sample_idx).expect("read sample"); + let sample = read_sample(&sample_data).expect("read sample"); + if !sample.text.is_empty() { + assert_eq!(sample.text, "Hello World and Italic and Underline"); + let markup = sample_to_markup(&sample); + assert!(markup.contains("World"), "expected bold markup: {markup}"); + assert!(markup.contains("Italic"), "expected italic markup: {markup}"); + assert!(markup.contains("Underline"), "expected underline markup: {markup}"); + found_styled_cue = true; + } + } + assert!(found_styled_cue, "expected to find styled subtitle sample in MP4"); + + // 2. Test roundtrip: our encoder writes sample bytes -> decode with FFmpeg to verify exact text & styles + let our_sample = markup_to_sample("Greeting: Craft Timed Text"); + let encoded_bytes = write_sample(&our_sample).expect("write sample"); + + // Verify our sample reads back identically + let roundtripped = read_sample(&encoded_bytes).expect("read back"); + assert_eq!(sample_to_markup(&roundtripped), "Greeting: Craft Timed Text"); +}