From 1901762de2fbfcfda5b12410648fa68d970459b6 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Thu, 19 Nov 2026 18:49:30 -0500 Subject: [PATCH] engine: read Lightroom backup zip via craft-zip (#4555) * engine: read Lightroom backup zip via craft-zip * engine: pin Lightroom ZIP reader to merged shared revision * engine: add mid-stream cancellation test for Lightroom backup extraction --- Cargo.lock | 11 + Cargo.toml | 1 + crates/engine/Cargo.toml | 4 +- crates/engine/src/lightroom_backup.rs | 279 ++++++++------------------ 4 files changed, 94 insertions(+), 201 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cd78730948..4410738317 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1517,6 +1517,16 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "craft-zip" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=5c496f6e962fcccd8ba91704994c4c06c2fc6a6f#5c496f6e962fcccd8ba91704994c4c06c2fc6a6f" +dependencies = [ + "flate2", + "thiserror 2.0.21", + "zip 9.1.3", +] + [[package]] name = "crc32fast" version = "1.5.3" @@ -4030,6 +4040,7 @@ dependencies = [ "craft-plist", "craft-tiff 0.1.0 (git+https://github.com/storytold/craft-libs?rev=23acc704933a9a5619a5cd1cddc9bb1fb5a9b2d3)", "craft-tiff 0.1.0 (git+https://github.com/storytold/craft-libs?rev=f589ff485707007e814671229d0f4e77bc06968a)", + "craft-zip", "crc32fast", "fs4", "harfrust 0.13.3", diff --git a/Cargo.toml b/Cargo.toml index 176796d4c3..f8269a4e37 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ craft-window = { git = "https://github.com/storytold/craft-libs", rev = "23acc70 craft-gainmap = { git = "https://github.com/storytold/craft-libs", rev = "23acc704933a9a5619a5cd1cddc9bb1fb5a9b2d3" } # PSD/PSB: the header, colour mode data and image resources (craft-libs#885). craft-psd = { git = "https://github.com/storytold/craft-libs", rev = "010702ee9b030844c852939095b1da0d0656a890" } +craft-zip = { git = "https://github.com/storytold/craft-libs", rev = "5c496f6e962fcccd8ba91704994c4c06c2fc6a6f" } jpeg-encoder = { version = "0.7", default-features = false, features = ["std"] } # internal lightcraft-denoise-core = { path = "crates/denoise-core" } diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 0fa54516b2..bab4d20094 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -32,8 +32,7 @@ lightcraft-meta = { workspace = true } lightcraft-lensdb = { workspace = true } lightcraft-faces = { workspace = true } miniz_oxide = { workspace = true } -# CRC-32 of a catalog unpacked from a Lightroom backup zip (lightroom_backup.rs). -crc32fast = { workspace = true } +craft-zip = { workspace = true } lightcraft-merge = { workspace = true } lightcraft-geocode = { workspace = true } lightcraft-slideshow = { workspace = true } @@ -74,3 +73,4 @@ craft-jxl = { workspace = true } craft-heif = { workspace = true, features = ["testdata"] } libc = { workspace = true } craft-tiff = { workspace = true } +crc32fast = { workspace = true } diff --git a/crates/engine/src/lightroom_backup.rs b/crates/engine/src/lightroom_backup.rs index ab51ffd3de..f4fa40eb8c 100644 --- a/crates/engine/src/lightroom_backup.rs +++ b/crates/engine/src/lightroom_backup.rs @@ -10,16 +10,16 @@ //! else under [`std::env::temp_dir`]; a folder left by a crash (`lc-lrbackup-*`, older than a day) //! is removed by the next extraction there. //! -//! Written from the ZIP application note (PKWARE APPNOTE.TXT 6.3: end of central directory, -//! ZIP64 end record and locator, central directory headers, local headers, the ZIP64 extra field -//! 0x0001) and RFC 1951 (deflate, inflated by `miniz_oxide`); no third-party zip code was used. -//! Every size and offset comes from the file and is checked before use. +//! ZIP/ZIP64 parsing, metadata bounds, size-cap enforcement, and CRC-32 verification are handled +//! by [`craft_zip::SeekableArchive`]. use std::fs::File; -use std::io::{Read, Seek, SeekFrom, Write}; +use std::io::{Read, Write}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; +use craft_zip::{SeekableArchive, SeekableLimits}; + /// The largest catalog a backup may unpack to: the SQLite reader's cap on a database file. pub const MAX_CATALOG_BYTES: u64 = 16 << 30; /// The most central directory entries looked at (a Lightroom backup has one to four). @@ -74,135 +74,39 @@ impl Drop for Extracted { } } -/// One file in the zip's central directory. -#[derive(Clone, Debug, PartialEq, Eq)] -struct Entry { - name: String, - method: u16, - crc: u32, - compressed: u64, - size: u64, - local: u64, +/// Opens `path` as a bounded seekable ZIP archive without buffering payloads in memory. +fn open_archive(path: &Path) -> Result, String> { + let file = File::open(path).map_err(|e| format!("{}: {e}", path.display()))?; + SeekableArchive::open( + file, + SeekableLimits { max_entries: MAX_ENTRIES, max_metadata_bytes: MAX_DIRECTORY_BYTES + 65_535 + 128, max_total_bytes: MAX_CATALOG_BYTES }, + ) + .map_err(|e| e.to_string()) } -/// The little-endian `u16` at `at` in `b`, or `None` past its end. -fn u16_at(b: &[u8], at: usize) -> Option { - let s = b.get(at..at.checked_add(2)?)?; - Some(u16::from_le_bytes([s[0], s[1]])) -} - -/// The little-endian `u32` at `at` in `b`, or `None` past its end. -fn u32_at(b: &[u8], at: usize) -> Option { - let s = b.get(at..at.checked_add(4)?)?; - Some(u32::from_le_bytes([s[0], s[1], s[2], s[3]])) -} - -/// The little-endian `u64` at `at` in `b`, or `None` past its end. -fn u64_at(b: &[u8], at: usize) -> Option { - let s = b.get(at..at.checked_add(8)?)?; - let mut a = [0u8; 8]; - a.copy_from_slice(s); - Some(u64::from_le_bytes(a)) -} - -/// `len` bytes of `f` from offset `at`; a length that doesn't fit memory or a short read is an error. -fn read_at(f: &mut File, at: u64, len: u64) -> Result, String> { - let len = usize::try_from(len).map_err(|_| "zip record too large".to_string())?; - f.seek(SeekFrom::Start(at)).map_err(|e| e.to_string())?; - let mut buf = vec![0u8; len]; - f.read_exact(&mut buf).map_err(|e| format!("truncated zip: {e}"))?; - Ok(buf) -} - -/// The central directory's entries (offset and count from the end record, or its ZIP64 record). -fn entries(f: &mut File) -> Result, String> { - let len = f.metadata().map_err(|e| e.to_string())?.len(); - // end of central directory: 22 bytes plus a comment of up to 64 KiB - let tail_len = len.min(22 + 65_535); - let tail_at = len - tail_len; - let tail = read_at(f, tail_at, tail_len)?; - let eocd = (0..tail.len().saturating_sub(21)).rev().find(|&i| tail.get(i..).is_some_and(|b| b.starts_with(b"PK\x05\x06"))); - let Some(eocd) = eocd else { return Err("not a zip file (no end of central directory)".into()) }; - let bad = || "damaged zip (end of central directory)".to_string(); - let mut count = u64::from(u16_at(&tail, eocd + 10).ok_or_else(bad)?); - let mut dir_len = u64::from(u32_at(&tail, eocd + 12).ok_or_else(bad)?); - let mut dir_at = u64::from(u32_at(&tail, eocd + 16).ok_or_else(bad)?); - // ZIP64: a locator just before the end record points at the ZIP64 end record - if (count == 0xFFFF || dir_len == 0xFFFF_FFFF || dir_at == 0xFFFF_FFFF) - && let Some(loc) = eocd.checked_sub(20) - && tail.get(loc..).is_some_and(|b| b.starts_with(b"PK\x06\x07")) - { - let rec_at = u64_at(&tail, loc + 8).ok_or_else(bad)?; - let rec = read_at(f, rec_at, 56)?; - if !rec.starts_with(b"PK\x06\x06") { - return Err("damaged zip (ZIP64 end record)".into()); - } - count = u64_at(&rec, 32).ok_or_else(bad)?; - dir_len = u64_at(&rec, 40).ok_or_else(bad)?; - dir_at = u64_at(&rec, 48).ok_or_else(bad)?; - } - if count > MAX_ENTRIES || dir_len > MAX_DIRECTORY_BYTES || dir_at.checked_add(dir_len).is_none_or(|end| end > len) { - return Err("damaged zip (central directory out of range)".into()); - } - let dir = read_at(f, dir_at, dir_len)?; - let mut out = Vec::new(); - let mut at = 0usize; - let bad = || "damaged zip (central directory)".to_string(); - for _ in 0..count { - if !dir.get(at..).is_some_and(|b| b.starts_with(b"PK\x01\x02")) { - return Err(bad()); - } - let method = u16_at(&dir, at + 10).ok_or_else(bad)?; - let crc = u32_at(&dir, at + 16).ok_or_else(bad)?; - let mut compressed = u64::from(u32_at(&dir, at + 20).ok_or_else(bad)?); - let mut size = u64::from(u32_at(&dir, at + 24).ok_or_else(bad)?); - let name_len = usize::from(u16_at(&dir, at + 28).ok_or_else(bad)?); - let extra_len = usize::from(u16_at(&dir, at + 30).ok_or_else(bad)?); - let comment_len = usize::from(u16_at(&dir, at + 32).ok_or_else(bad)?); - let mut local = u64::from(u32_at(&dir, at + 42).ok_or_else(bad)?); - let name_at = at + 46; - let name_end = name_at.checked_add(name_len).ok_or_else(bad)?; - let name = String::from_utf8_lossy(dir.get(name_at..name_end).ok_or_else(bad)?).replace('\\', "/"); - let extra = dir.get(name_end..name_end.checked_add(extra_len).ok_or_else(bad)?).ok_or_else(bad)?; - // ZIP64 extra field: the 64-bit values, in this order, of those set to 0xFFFFFFFF - let mut e = 0usize; - while e + 4 <= extra.len() { - let (id, len) = (u16_at(extra, e).ok_or_else(bad)?, usize::from(u16_at(extra, e + 2).ok_or_else(bad)?)); - let body = extra.get(e + 4..e + 4 + len).ok_or_else(bad)?; - if id == 0x0001 { - let mut v = 0usize; - for field in [&mut size, &mut compressed, &mut local] { - if *field == 0xFFFF_FFFF { - *field = u64_at(body, v).ok_or_else(bad)?; - v += 8; - } - } +/// The backup's catalog entry: `(index, normalized_name, declared_size)` for the one `*.lrcat` +/// file in the zip, outside macOS resource forks. +fn catalog_entry(archive: &mut SeekableArchive) -> Result<(usize, String, u64), String> { + let mut found: Option<(usize, String, u64)> = None; + for index in 0..archive.len() { + let Some(name) = archive.entry_name(index).map_err(|e| e.to_string())? else { continue }; + let normalized = name.replace('\\', "/"); + let lower = normalized.to_lowercase(); + if lower.ends_with(".lrcat") && !lower.split('/').any(|p| p == "__macosx" || p.starts_with("._")) { + if found.is_some() { + return Err("this zip holds more than one Lightroom catalog (.lrcat); unzip it and import one".into()); } - e += 4 + len; + let (size, _) = archive.entry_sizes(index).map_err(|e| e.to_string())?.ok_or_else(|| "damaged zip".to_string())?; + found = Some((index, normalized, size)); } - at = name_end.checked_add(extra_len).and_then(|x| x.checked_add(comment_len)).ok_or_else(bad)?; - out.push(Entry { name, method, crc, compressed, size, local }); - } - Ok(out) -} - -/// The backup's catalog entry: the one `*.lrcat` file in the zip, outside macOS resource forks. -fn catalog_entry(entries: &[Entry]) -> Result<&Entry, String> { - let mut found = entries.iter().filter(|e| { - let lower = e.name.to_lowercase(); - lower.ends_with(".lrcat") && !lower.split('/').any(|p| p == "__macosx" || p.starts_with("._")) - }); - match (found.next(), found.next()) { - (Some(e), None) => Ok(e), - (None, _) => Err("this zip holds no Lightroom catalog (.lrcat)".into()), - (Some(_), Some(_)) => Err("this zip holds more than one Lightroom catalog (.lrcat); unzip it and import one".into()), } + found.ok_or_else(|| "this zip holds no Lightroom catalog (.lrcat)".into()) } /// Whether the zip at `path` holds exactly one Lightroom catalog, by its central directory only /// (nothing is unpacked). Any read error means no. pub fn holds_catalog(path: &Path) -> bool { - File::open(path).ok().and_then(|mut f| entries(&mut f).ok()).is_some_and(|all| catalog_entry(&all).is_ok()) + open_archive(path).ok().is_some_and(|mut archive| catalog_entry(&mut archive).is_ok()) } /// Remove `lc-lrbackup-*` folders in `root` left by an earlier run that ended without cleaning up. @@ -239,14 +143,19 @@ fn check_space(root: &Path, needed: u64, available: Option) -> Result<(), S /// Unpack the catalog of the backup `zip` into a new private folder under `root`, checking its /// size and CRC-32. Stops with an error when `cancel` is set; the folder is removed then too. pub fn extract(zip: &Path, root: &Path, cancel: &AtomicBool) -> Result { - let mut f = File::open(zip).map_err(|e| format!("{}: {e}", zip.display()))?; - let all = entries(&mut f)?; - let entry = catalog_entry(&all)?.clone(); - if entry.size > MAX_CATALOG_BYTES { - return Err(format!("the catalog in this backup unpacks to {} GB, more than LightCraft reads", entry.size >> 30)); + extract_inner(zip, root, cancel, || {}) +} + +/// Core extraction loop; `on_chunk` runs after each written chunk (a no-op in production, used +/// in tests to trigger mid-stream cancellation deterministically). +fn extract_inner(zip: &Path, root: &Path, cancel: &AtomicBool, mut on_chunk: impl FnMut()) -> Result { + let mut archive = open_archive(zip)?; + let (index, entry_name, entry_size) = catalog_entry(&mut archive)?; + if entry_size > MAX_CATALOG_BYTES { + return Err(format!("the catalog in this backup unpacks to {} GB, more than LightCraft reads", entry_size >> 30)); } std::fs::create_dir_all(root).map_err(|e| format!("{}: {e}", root.display()))?; - check_space(root, entry.size, crate::disk_space::read(root).map(|space| space.available))?; + check_space(root, entry_size, crate::disk_space::read(root).map(|space| space.available))?; sweep(root); let stamp = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_nanos()); let dir = root.join(format!("{PREFIX}{}-{stamp}", std::process::id())); @@ -263,82 +172,27 @@ pub fn extract(zip: &Path, root: &Path, cancel: &AtomicBool) -> Result Result<(), String> { - *written = written.saturating_add(bytes.len() as u64); - if *written > entry.size { - return Err("damaged zip (the catalog is longer than the zip says)".into()); - } - crc.update(bytes); - writer.write_all(bytes).map_err(|e| format!("writing {}: {e}", out.catalog.display())) - }; let mut buf = vec![0u8; CHUNK]; - match entry.method { - 0 => loop { - if cancel.load(Ordering::Relaxed) { - return Err("cancelled".into()); - } - let n = input.read(&mut buf).map_err(|e| e.to_string())?; - if n == 0 { - break; - } - write(buf.get(..n).unwrap_or_default(), &mut written)?; - }, - 8 => { - use miniz_oxide::inflate::stream::{InflateState, inflate}; - use miniz_oxide::{DataFormat, MZFlush, MZStatus}; - let mut state = InflateState::new_boxed(DataFormat::Raw); - let mut out_buf = vec![0u8; CHUNK]; - let (mut start, mut end) = (0usize, 0usize); - loop { - if cancel.load(Ordering::Relaxed) { - return Err("cancelled".into()); - } - if start == end { - end = input.read(&mut buf).map_err(|e| e.to_string())?; - start = 0; - } - let flush = if end == 0 { MZFlush::Finish } else { MZFlush::None }; - let r = inflate(&mut state, buf.get(start..end).unwrap_or_default(), &mut out_buf, flush); - start = start.saturating_add(r.bytes_consumed).min(end); - write(out_buf.get(..r.bytes_written).unwrap_or_default(), &mut written)?; - match r.status { - Ok(MZStatus::StreamEnd) => break, - Ok(_) => {} - // no progress with all input given: the stream is cut short - Err(miniz_oxide::MZError::Buf) if end == 0 && r.bytes_written == 0 => return Err("damaged zip (the catalog is cut short)".into()), - Err(miniz_oxide::MZError::Buf) => {} - Err(e) => return Err(format!("damaged zip (deflate: {e:?})")), - } - if r.bytes_consumed == 0 && r.bytes_written == 0 && start < end { - return Err("damaged zip (the catalog's data does not inflate)".into()); - } - } + loop { + if cancel.load(Ordering::Relaxed) { + return Err("cancelled".into()); } - m => return Err(format!("this zip uses compression method {m}, which LightCraft does not read; unzip it and import the .lrcat")), + let n = reader.read(&mut buf).map_err(|e| format!("reading {}: {e}", zip.display()))?; + if n == 0 { + break; + } + writer.write_all(buf.get(..n).unwrap_or_default()).map_err(|e| format!("writing {}: {e}", out.catalog.display()))?; + on_chunk(); } writer.flush().map_err(|e| format!("writing {}: {e}", out.catalog.display()))?; - drop(writer); - if written != entry.size { - return Err("damaged zip (the catalog is shorter than the zip says)".into()); - } - if crc.finalize() != entry.crc { - return Err("damaged zip (the catalog's checksum does not match)".into()); - } Ok(out) } @@ -496,6 +350,33 @@ pub(crate) mod tests { std::fs::remove_dir_all(&dir).unwrap(); } + /// Setting the cancel flag mid-stream stops extraction cleanly and removes the partial temp folder. + #[test] + fn mid_stream_cancellation_stops_extraction_and_cleans_up() { + let dir = temp("cancel-mid-stream"); + let root = dir.join("tmp"); + let path = dir.join("Cat.zip"); + // Larger than two CHUNK reads so the loop must iterate multiple times. + let catalog = vec![0x5Au8; CHUNK * 2 + 123]; + std::fs::write(&path, zip(&[("Cat.lrcat", &catalog)], false, false)).unwrap(); + + let cancel = AtomicBool::new(false); + let mut chunks = 0usize; + let err = extract_inner(&path, &root, &cancel, || { + chunks += 1; + if chunks == 1 { + assert_eq!(leftovers(&root).len(), 1, "temp folder must exist while extracting"); + cancel.store(true, Ordering::Relaxed); + } + }) + .unwrap_err(); + + assert_eq!(err, "cancelled"); + assert_eq!(chunks, 1, "extraction must stop before reading remaining chunks"); + assert!(leftovers(&root).is_empty(), "mid-stream cancel must clean up temp files"); + std::fs::remove_dir_all(&dir).unwrap(); + } + /// A ZIP64 size of 1 TB is refused before anything is written. #[test] fn a_zip_that_lies_about_the_catalog_size_is_refused() {