From fd2e7d7b6fa809c4cc29349e5aac5253ce5289be Mon Sep 17 00:00:00 2001 From: Terra854 Date: Sat, 21 Nov 2026 18:30:06 +0800 Subject: [PATCH] fix(media): preserve leading MOV audio silence and saturate positions (#4005) * fix(media): honor MOV audio edit lists on first read * fix(media): preserve sign on ISO-BMFF edit-list position overflow --------- Co-authored-by: Dylan M. Taylor --- crates/media/src/craft_source.rs | 165 ++++++++++++++++++++++++++----- crates/media/tests/it/footage.rs | 70 +++++++++++++ 2 files changed, 211 insertions(+), 24 deletions(-) diff --git a/crates/media/src/craft_source.rs b/crates/media/src/craft_source.rs index ae4aac50f..fb99282c5 100644 --- a/crates/media/src/craft_source.rs +++ b/crates/media/src/craft_source.rs @@ -291,6 +291,21 @@ impl ContainerFile { let decoded_pos = audio_starts.get(seek_idx).copied().unwrap_or(0); (seek_idx, decoded_pos) } + + fn audio_silent_prefix(&self, track_idx: usize, start: i64, frames: usize, sample_rate: u32) -> usize { + let Self::Isobmff(file) = self else { return 0 }; + let Some((leading_duration, media_time)) = file.tracks.get(track_idx).and_then(|track| unit_rate_media_edit(&track.edits)) else { + return 0; + }; + if leading_duration == 0 || media_time == 0 || file.timescale == 0 { + return 0; + } + + // The net packet offset includes the trim and can put decoder preroll inside + // the empty edit. Use the empty edit's duration on the output clock instead. + let first_presented = (u128::from(leading_duration) * u128::from(sample_rate)).div_ceil(u128::from(file.timescale)) as i128; + usize::try_from((first_presented - i128::from(start)).max(0)).unwrap_or(usize::MAX).min(frames) + } } /// Where the container puts one audio packet, for [`contiguous_starts`]. @@ -468,12 +483,7 @@ fn compute_audio_starts(container: &ContainerFile, track_idx: usize, codec: Code ContainerFile::Isobmff(f) => { let Some(track) = f.tracks.get(track_idx) else { return AudioStarts::complete(Vec::new()) }; let ts = i128::from(track.timescale.max(1)); - let stamps: Vec = (0..track.samples.len()) - .map(|i| { - let pts = track.presentation_pts(i).unwrap_or(0); - i64::try_from(i128::from(pts) * rate / ts).unwrap_or(0) - }) - .collect(); + let stamps: Vec = (0..track.samples.len()).map(|i| isobmff_audio_position(track, i, native_rate)).collect(); let unit = i64::try_from((rate + ts - 1) / ts).unwrap_or(i64::MAX); if codec == CodecId::Opus { return AudioStarts::opus(stamps, unit.saturating_mul(2).saturating_add(1)); @@ -559,6 +569,14 @@ fn audio_frames_from_ns(ns: i128, rate: u32) -> i128 { (ns * i128::from(rate)) / 1_000_000_000 } +/// The decoded buffer starts on the presentation timeline, including negative trimmed samples. +/// Using the same signed position on open and seek preserves edit-list trim and leading silence. +fn isobmff_audio_position(track: &isobmff::Track, sample_idx: usize, native_rate: u32) -> i64 { + let pts = track.presentation_pts(sample_idx).unwrap_or(0); + let scaled = (i128::from(pts) * i128::from(native_rate)) / i128::from(track.timescale.max(1)); + i64::try_from(scaled).unwrap_or(if scaled < 0 { i64::MIN } else { i64::MAX }) +} + fn open_audio_tracks(container: &ContainerFile, source: &dyn ByteSource, candidates: I) -> Option<(Vec, Vec)> where I: IntoIterator, @@ -670,17 +688,24 @@ fn clean_aperture_region(aperture: isobmff::CleanAperture, (width, height): (u32 (!r.is_full(width as usize, height as usize)).then_some(r) } -/// A single unit-rate media span, optionally preceded by one empty span. A -/// scalar EDL origin cannot describe dwell, disjoint edits or respeeding. -fn continuous_edit_mapping(track: &isobmff::Track, movie_timescale: u32) -> Option<(i64, i64)> { +/// A single unit-rate media span, optionally preceded by one empty span. +fn unit_rate_media_edit(edits: &[isobmff::Edit]) -> Option<(u64, i64)> { const UNIT_RATE: i32 = 0x0001_0000; - let (leading_duration, media) = match track.edits.as_slice() { - [] => return (track.edit_offset == 0).then_some((0, 0)), + let (leading_duration, media) = match edits { [media] => (0, media), [empty, media] if empty.media_time == -1 && empty.segment_duration > 0 && empty.media_rate == UNIT_RATE => (empty.segment_duration, media), _ => return None, }; - if media.media_time < 0 || media.segment_duration == 0 || media.media_rate != UNIT_RATE || movie_timescale == 0 || track.timescale == 0 { + (media.media_time >= 0 && media.segment_duration != 0 && media.media_rate == UNIT_RATE).then_some((leading_duration, media.media_time)) +} + +/// A scalar EDL origin cannot describe dwell, disjoint edits or respeeding. +fn continuous_edit_mapping(track: &isobmff::Track, movie_timescale: u32) -> Option<(i64, i64)> { + if track.edits.is_empty() { + return (track.edit_offset == 0).then_some((0, 0)); + } + let (leading_duration, media_time) = unit_rate_media_edit(&track.edits)?; + if movie_timescale == 0 || track.timescale == 0 { return None; } // Check the raw movie-to-media conversion before the demuxer's rounded @@ -690,8 +715,8 @@ fn continuous_edit_mapping(track: &isobmff::Track, movie_timescale: u32) -> Opti if lead % movie_clock != 0 { return None; } - let offset = i64::try_from(lead / movie_clock - i128::from(media.media_time)).ok()?; - (offset == track.edit_offset).then_some((media.media_time, offset)) + let offset = i64::try_from(lead / movie_clock - i128::from(media_time)).ok()?; + (offset == track.edit_offset).then_some((media_time, offset)) } /// The EDL origin is the timecode at presentation time zero, not the raw @@ -1389,12 +1414,15 @@ impl MediaSource for CraftMediaSource { return Err(MediaError::Decode("invalid audio sample rate 0".into())); } let channels_count = self.info.audio_streams.get(stream).map_or(2, |s| s.channels as usize); - if frames == 0 { - return Ok(AudioBuffer::new(sample_rate, channels_count as u16, 0)); + let silent_frames = self.file.audio_silent_prefix(entry.track_idx, start, frames, sample_rate); + let audible_frames = frames - silent_frames; + if audible_frames == 0 { + return Ok(AudioBuffer::new(sample_rate, channels_count as u16, frames)); } + let audible_start = i64::try_from(i128::from(start) + silent_frames as i128).unwrap_or(i64::MAX); let native_rate = entry.native_rate; - let (native_start, native_frames) = native_audio_window(start, frames, native_rate, sample_rate); + let (native_start, native_frames) = native_audio_window(audible_start, audible_frames, native_rate, sample_rate); let decoded = match &entry.backend { AudioBackend::Decoder { seek_preroll_samples, state: state_mutex } => { @@ -1441,11 +1469,15 @@ impl MediaSource for CraftMediaSource { } }; - if sample_rate == native_rate { - return Ok(AudioBuffer { sample_rate, channels: decoded }); + let mut channels = + if sample_rate == native_rate { decoded } else { resample_audio(&decoded, channels_count, native_rate, sample_rate, audible_frames) }; + if silent_frames > 0 { + for channel in &mut channels { + channel.resize(frames, 0.0); + channel.rotate_right(silent_frames); + } } - - Ok(AudioBuffer { sample_rate, channels: resample_audio(&decoded, channels_count, native_rate, sample_rate, frames) }) + Ok(AudioBuffer { sample_rate, channels }) } } @@ -1565,7 +1597,7 @@ fn drain_audio_window(buf: &mut [Vec], decoded_pos: &mut i64, channels_coun for i in 0..frames { let pos = start.saturating_add(i as i64); if pos >= *decoded_pos { - let off = (pos - *decoded_pos) as usize; + let off = usize::try_from(pos.saturating_sub(*decoded_pos)).unwrap_or(usize::MAX); if off < avail { for (c, out_ch) in out.iter_mut().enumerate() { out_ch[i] = buf[c][off]; @@ -1574,7 +1606,7 @@ fn drain_audio_window(buf: &mut [Vec], decoded_pos: &mut i64, channels_coun } } let end = start.saturating_add(frames as i64); - let drop_to = (end - *decoded_pos).max(0) as usize; + let drop_to = usize::try_from(end.saturating_sub(*decoded_pos).max(0)).unwrap_or(usize::MAX); let trim = drop_to.min(avail); if trim > 0 { for ch in buf { @@ -1671,7 +1703,7 @@ fn decode_audio_range( let buf_len = state.buf.first().map_or(0, Vec::len) as i64; let buf_end = state.decoded_pos.saturating_add(buf_len); let moves_backward = start < state.decoded_pos; - if moves_backward || start > buf_end + i64::from(native_rate) * 2 { + if moves_backward || start > buf_end.saturating_add(i64::from(native_rate) * 2) { let target = start.saturating_sub(seek_preroll_samples).max(0); let (seek_idx, decoded_pos) = if moves_backward && state.next_sample == 0 && state.audio_starts.seek_proves_first_packet(target) { (0, state.decoded_pos) @@ -2396,6 +2428,91 @@ mod tests { assert!((1..=MAX_WORKERS).contains(&cap), "cap={cap}"); } + /// Generated AAC permits unequal track/sample clocks, unlike the MOV writer's PCM tracks. + #[test] + fn mov_aac_edit_positions_saturate_with_unequal_clocks() { + use craft_audio::aac::{Encoder, EncoderConfig}; + use isobmff::{Brand, Edit, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions}; + + const RATE: u32 = 48_000; + let mut encoder = Encoder::new(EncoderConfig::cbr(RATE, 1, 64_000)).unwrap(); + let tone: Vec = (0..2048).map(|i| (i as f32 * 0.13).sin() * 0.25).collect(); + let mut packets = encoder.encode(&[&tone]); + packets.extend(encoder.flush()); + assert!(!packets.is_empty()); + for (offset, expected) in [(10i64, 480i64), (-10, -480), (i64::MAX / 2, i64::MAX), (-i64::MAX / 2, i64::MIN)] { + let mut writer = Mp4Writer::new(std::io::Cursor::new(Vec::new()), WriterOptions::new(Brand::Mov)).unwrap(); + let mut config = TrackConfig::new(SampleEntry::aac(encoder.audio_specific_config(), 1, RATE), 1_000); + if offset > 0 { + config.edits.push(Edit { segment_duration: offset as u64, media_time: -1, media_rate: 0x0001_0000 }); + } + config.edits.push(Edit { segment_duration: 100, media_time: (-offset).max(0), media_rate: 0x0001_0000 }); + let track_idx = writer.add_track(config).expect("AAC accepts unequal media and sample clocks"); + for packet in &packets { + writer.write_sample(track_idx, WriteSample { data: packet, duration: 21, composition_offset: 0, is_sync: true }).unwrap(); + } + let bytes = writer.finish().unwrap().into_inner(); + let mov = isobmff::open(bytes.as_slice()).unwrap(); + let track = &mov.tracks[0]; + assert_eq!((track.timescale, track.edit_offset, track.presentation_pts(0)), (1_000, offset, Some(offset))); + let params = track.codec_params(); + assert_eq!((params.codec, params.sample_rate), (CodecId::Aac, Some(RATE))); + // Check the conversion itself: a decode error substituted with silence cannot pass. + assert_eq!(isobmff_audio_position(track, 0, RATE), expected); + let file = ContainerFile::Isobmff(mov); + let (mut tracks, infos) = open_audio_tracks(&file, &bytes, [(0, packets.len(), params, 0)]).expect("shared AAC source"); + assert_eq!(infos[0].sample_rate, RATE); + let (preroll, state) = tracks[0].decoder_parts(); + assert_eq!(state.audio_starts.start(&file, 0, 0), Some(expected)); + for start in [0, 6_000, 0] { + let decoded = decode_audio_range(&file, &bytes, 0, RATE, 1, preroll, state, start, 64).expect("the result-returning shared read must succeed"); + assert_eq!(decoded[0].len(), 64); + if expected == i64::MAX || expected == i64::MIN { + assert!(decoded[0].iter().all(|&v| v == 0.0), "out-of-range edits have no audible samples at {start}"); + } + } + } + } + + /// Original PCM generated in memory to exercise an empty edit followed by a trim. + #[test] + fn mov_pcm_trim_after_empty_edit_preserves_leading_silence() { + use isobmff::{Brand, Edit, Mp4Writer, PcmConfig, SampleEntry, TrackConfig, WriteSample, WriterOptions}; + + const RATE: u32 = 48_000; + const DELAY_MS: u64 = 100; + const DELAY: i64 = 4_800; + let samples: Vec = (0..12_000).map(|i| 1_000 + (i % 257) as i16 * 32).collect(); + for trim in [2_400i64, 4_800, 9_600] { + let mut writer = Mp4Writer::new(std::io::Cursor::new(Vec::new()), WriterOptions::new(Brand::Mov)).unwrap(); + let pcm = PcmConfig { bits: 16, float: false, big_endian: false, signed: true, channels: 1, sample_rate: f64::from(RATE) }; + let mut config = TrackConfig::new(SampleEntry::pcm(pcm), RATE); + config.edits = vec![ + Edit { segment_duration: DELAY_MS, media_time: -1, media_rate: 0x0001_0000 }, + Edit { segment_duration: 40, media_time: trim, media_rate: 0x0001_0000 }, + ]; + let track = writer.add_track(config).unwrap(); + for chunk in samples.chunks(256) { + let data: Vec = chunk.iter().flat_map(|sample| sample.to_le_bytes()).collect(); + writer.write_sample(track, WriteSample { data: &data, duration: chunk.len() as u32, composition_offset: 0, is_sync: true }).unwrap(); + } + let bytes = writer.finish().unwrap().into_inner(); + let file = isobmff::open(bytes.as_slice()).unwrap(); + assert_eq!(file.timescale, 1_000); + assert_eq!(file.tracks[0].edit_offset, DELAY - trim); + + let source = open_bytes("edited-pcm.mov", bytes.into()).expect("shared MOV route").unwrap(); + let sample = |index: i64| f32::from(samples[index as usize]) / 32_768.0; + let expected = [0.0, 0.0, sample(trim), sample(trim + 1)]; + let read = |start, frames, rate| source.audio(start, frames, rate).expect("direct shared PCM read").channels.remove(0); + assert_eq!(read(DELAY - 2, 4, RATE), expected, "first read, trim {trim}"); + assert_eq!(read(0, DELAY as usize, RATE), vec![0.0; DELAY as usize], "backward seek, trim {trim}"); + assert_eq!(read(DELAY, 2, RATE), expected[2..], "read from first presented sample, trim {trim}"); + assert_eq!(read(DELAY * 2 - 4, 5, RATE * 2), [0.0, 0.0, 0.0, 0.0, sample(trim)], "resampled read, trim {trim}"); + assert_eq!(read(4_408, 3, 44_100), [0.0, 0.0, sample(trim)], "nonintegral resampling ratio, trim {trim}"); + } + } + #[test] fn vui_overrides_container_color_and_falls_back_when_unspecified() { let container = color_from_isobmff(&isobmff::ColorInfo::Nclx { primaries: 9, transfer: 16, matrix: 9, full_range: false }); diff --git a/crates/media/tests/it/footage.rs b/crates/media/tests/it/footage.rs index 2d6af707c..1b03817ab 100644 --- a/crates/media/tests/it/footage.rs +++ b/crates/media/tests/it/footage.rs @@ -505,6 +505,76 @@ fn prores_alpha_uses_coded_frame_not_mov_depth() { } } +/// Generated PCM ramps expose edit-list timing without an external encoder or oracle. +#[test] +fn mov_pcm_edit_lists_preserve_first_read_and_seek_timing() { + use craft_media::isobmff::{self, Brand, Edit, FourCc, Mp4Writer, PcmConfig, SampleEntry, TrackConfig, WriteSample, WriterOptions}; + use craft_video::prores::{ChromaFormat, Encoder, Frame, Profile}; + + const RATE: u32 = 48_000; + const AUDIO_FRAMES: usize = 12_000; + let samples: Vec<[i16; 2]> = (0..AUDIO_FRAMES).map(|i| [1000 + (i % 257) as i16 * 32, -2000 - (i % 193) as i16 * 16]).collect(); + let profile = Profile::Hq; + let mut encoder = Encoder::new(profile, 16, 16); + let packet = encoder.encode(&Frame::new(16, 16, ChromaFormat::Yuv422, 10, false).unwrap()).unwrap(); + // The extreme edits also guard signed buffer arithmetic on the first audio request. + for offset in [0i64, 480, -480, -i64::MAX, i64::MAX - i64::MAX % 48] { + let mut writer = Mp4Writer::new(std::io::Cursor::new(Vec::new()), WriterOptions::new(Brand::Mov)).unwrap(); + let video = writer.add_track(TrackConfig::new(SampleEntry::prores(FourCc(profile.fourcc()), 16, 16), 24)).unwrap(); + let pcm = PcmConfig { bits: 16, float: false, big_endian: false, signed: true, channels: 2, sample_rate: f64::from(RATE) }; + let mut audio_config = TrackConfig::new(SampleEntry::pcm(pcm), RATE); + if offset > 0 { + audio_config.edits.push(Edit { segment_duration: (offset / 48) as u64, media_time: -1, media_rate: 0x0001_0000 }); + } + if offset != 0 { + audio_config.edits.push(Edit { segment_duration: 240, media_time: (-offset).max(0), media_rate: 0x0001_0000 }); + } + let audio = writer.add_track(audio_config).unwrap(); + writer.write_sample(video, WriteSample { data: &packet, duration: 24, composition_offset: 0, is_sync: true }).unwrap(); + for chunk in samples.chunks(256) { + let data: Vec = chunk.iter().flat_map(|frame| frame.iter().flat_map(|s| s.to_le_bytes())).collect(); + writer.write_sample(audio, WriteSample { data: &data, duration: chunk.len() as u32, composition_offset: 0, is_sync: true }).unwrap(); + } + let bytes: Arc<[u8]> = writer.finish().unwrap().into_inner().into(); + let mov = isobmff::open(bytes.as_ref()).unwrap(); + let track = mov.tracks.iter().find(|t| t.kind == isobmff::TrackKind::Audio).unwrap(); + assert_eq!(track.edit_offset, offset, "fixture must carry the requested audio edit"); + assert_eq!(track.presentation_pts(0), Some(offset)); + + let name = format!("pcm-edit-{offset}.mov"); + let footage = probe_bytes(&name, bytes.clone()).expect("probe generated MOV"); + assert_eq!(footage.codec, "ProRes", "must exercise the shared importer"); + assert!(footage.has_audio); + let pool = MediaPool::new(); + pool.add_bytes(&name, bytes); + let check = |start: usize, count: usize| { + let got = pool.audio_samples(&footage, Tick::from_units(start as i64, i64::from(RATE)), count, RATE); + assert_eq!(got.len(), count * 2); + for (i, frame) in got.as_chunks::<2>().0.iter().enumerate() { + let source = i64::try_from(start + i).unwrap().saturating_sub(offset); + let expected = usize::try_from(source).ok().and_then(|p| samples.get(p)).copied().unwrap_or([0, 0]); + for c in 0..2 { + assert_eq!(frame[c], f32::from(expected[c]) / 32768.0, "edit {offset}, request {start}, frame {i}, channel {c}"); + } + } + got + }; + let first = check(0, 2048); + let mut sequential = Vec::new(); + let mut start = 0; + while start < 2048 { + let count = 127.min(2048 - start); + sequential.extend(check(start, count)); + start += count; + } + assert_eq!(sequential, first, "edit {offset}: chunking must preserve samples"); + for start in [3000, 100, 6000, 0, 480, 700] { + check(start, 600); + } + assert_eq!(check(0, 2048), first, "edit {offset}: reopening the decode window must preserve samples"); + } +} + /// A synthetic asymmetrical aperture and rotation must determine what the shared importer shows. #[test] fn mov_renders_clean_aperture_before_display_rotation() {