From 399b55eb615b200af5a94eec0ab03e851e68548b Mon Sep 17 00:00:00 2001 From: francoisgn <32835330+francoisgn@users.noreply.github.com> Date: Thu, 22 Oct 2026 20:18:18 +0300 Subject: [PATCH 01/55] LRC-LIB-CATALOG-IMPORT: import Lightroom catalog backup zips directly (#2592) Someone who has removed Lightroom and its catalog may have only the backups Lightroom wrote (Backups//.zip) and the photos. library.importLightroom / inspectLightroom and File > Import Lightroom Catalog now take the zip: the .lrcat entry is streamed (stored or deflated, ZIP64 included) into a private folder in the library's Interop/ (the system temp folder when the library isn't on disk), checked against its size and CRC-32, read, and removed on success, error and cancel alike. The zip is only read. Paths relative to the catalog resolve from where it was before the backup. Open Library recognises Backups/ (newest backup) and its dated folders even with the catalog gone. Collections are keyed on the catalog's Adobe_storeProviderID, which every backup keeps, so a backup of an imported catalog no longer adds each collection twice; path keys from earlier imports are still matched. Checked on a real catalog (2,740 photos, 96 collections) and its four backups from Lightroom 7 to Classic 13: each zip imports like its unpacked catalog, leaves nothing behind and is unchanged. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + crates/engine/Cargo.toml | 2 + crates/engine/src/cmd/library.rs | 11 +- crates/engine/src/lib.rs | 1 + crates/engine/src/lightroom_archive.rs | 2 + crates/engine/src/lightroom_backup.rs | 483 +++++++++++++++++++ crates/engine/src/lightroom_catalog.rs | 167 ++++++- crates/engine/src/lightroom_folder.rs | 32 +- crates/engine/src/lightroom_job.rs | 12 +- crates/engine/src/tests_lightroom_folder.rs | 22 + crates/ui-egui/src/lightroom_import.rs | 7 +- crates/ui-egui/src/menus.rs | 7 +- docs/gaps.md | 5 +- docs/lightroom-catalog-import.md | 30 ++ docs/parity-checklist.md | 5 +- docs/user/guides/import-lightroom-catalog.md | 3 +- docs/whats-new.md | 10 + 17 files changed, 772 insertions(+), 28 deletions(-) create mode 100644 crates/engine/src/lightroom_backup.rs diff --git a/Cargo.lock b/Cargo.lock index 216fdeca4e..9b15950459 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3236,6 +3236,7 @@ dependencies = [ "ab_glyph", "chrono", "craft-docpath", + "crc32fast", "harfrust", "jxl-oxide", "libc", diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 9caa8cdb0e..1ec6131063 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -29,6 +29,8 @@ lightcraft-meta = { workspace = true } lightcraft-lensdb = { workspace = true } lightcraft-faces = { workspace = true } miniz_oxide = { workspace = true } +# CRC-32 of a catalog unpacked from a Lightroom backup zip (lightroom_backup.rs). +crc32fast = { workspace = true } lightcraft-merge = { workspace = true } lightcraft-geocode = { workspace = true } lightcraft-slideshow = { workspace = true } diff --git a/crates/engine/src/cmd/library.rs b/crates/engine/src/cmd/library.rs index e437dccc23..4a03e851c4 100644 --- a/crates/engine/src/cmd/library.rs +++ b/crates/engine/src/cmd/library.rs @@ -1484,10 +1484,13 @@ pub fn specs() -> Vec { }), // ---- import cmd!(query "library.inspectLightroom", "Inspect Lightroom Catalog", [], None, - "{path: .lrcat} → {photos, collections, missing, warnings}; read-only, no Lightroom required", - always, |_, p| { + "{path: .lrcat, or a Lightroom catalog backup .zip} → {photos, collections, missing, warnings}; read-only, no Lightroom required", + always, |s, p| { let path = str_param(p,"path").ok_or_else(|| bad("library.inspectLightroom","missing path"))?; - let data = crate::lightroom_catalog::read(std::path::Path::new(path)).map_err(|e| bad("library.inspectLightroom",e))?; + let (cancel, total, done) = Default::default(); + let temp = crate::lightroom_catalog::backup_temp_root(s); + let data = crate::lightroom_catalog::read_source(std::path::Path::new(path), temp.as_deref(), &cancel, &total, &done) + .map_err(|e| bad("library.inspectLightroom",e))?; Ok(json!({"photos":data.photos.len(),"collections":data.collections.iter().filter(|r|r.get("systemOnly").and_then(Value::as_f64).unwrap_or(0.0)==0.0).count(),"missing":data.photos.iter().filter(|p| !p.original_is_file()).map(|p|&p.path).collect::>(),"warnings":data.warnings,"skipped":data.skipped})) }), cmd!( @@ -1495,7 +1498,7 @@ pub fn specs() -> Vec { "Import Lightroom Catalog", [], None, - "{path: .lrcat, updateExisting?: false}; direct import of paths, ratings, flags, metadata, keywords, collections, virtual copies & mapped edits. Archives source settings/history first. Existing edited photos are preserved unless updateExisting=true; source catalog is read-only.", + "{path: .lrcat, or a Lightroom catalog backup .zip (unpacked to a temporary folder in the library, removed after reading; the zip is not changed), updateExisting?: false}; direct import of paths, ratings, flags, metadata, keywords, collections, virtual copies & mapped edits. Archives source settings/history first. Existing edited photos are preserved unless updateExisting=true; source catalog is read-only.", always, |s, p| { let path = str_param(p, "path").ok_or_else(|| bad("library.importLightroom", "missing path"))?; diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 323ed90ef4..96aedee27e 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -48,6 +48,7 @@ mod import_move; pub mod lens_profiles; pub mod library; mod lightroom_archive; +pub mod lightroom_backup; pub mod lightroom_catalog; pub mod lightroom_folder; pub mod lightroom_job; diff --git a/crates/engine/src/lightroom_archive.rs b/crates/engine/src/lightroom_archive.rs index 9c9979d970..df6ba5d5aa 100644 --- a/crates/engine/src/lightroom_archive.rs +++ b/crates/engine/src/lightroom_archive.rs @@ -519,6 +519,7 @@ mod tests { fn data(source: String) -> CatalogImport { CatalogImport { source, + catalog_id: None, photos: Vec::new(), collections: Vec::new(), members: Vec::new(), @@ -543,6 +544,7 @@ mod tests { }; CatalogImport { source: "source.lrcat".into(), + catalog_id: None, photos: vec![photo], collections: vec![BTreeMap::from([("name".into(), serde_json::json!("Keepers"))])], members: vec![BTreeMap::from([("image".into(), serde_json::json!(7))])], diff --git a/crates/engine/src/lightroom_backup.rs b/crates/engine/src/lightroom_backup.rs new file mode 100644 index 0000000000..78e4dd1474 --- /dev/null +++ b/crates/engine/src/lightroom_backup.rs @@ -0,0 +1,483 @@ +//! Lightroom Classic catalog backups (issue #2592): `Backups/YYYY-MM-DD HHMM/.zip`, which +//! holds the catalog `.lrcat` (and, since Classic 12, a small `.lrcat-data/` folder that +//! the import does not need). Someone who has left Lightroom may have nothing else: the import takes +//! the zip as it is. +//! +//! The catalog entry is inflated by streaming into a private folder ([`Extracted`]), read from +//! there like any `.lrcat`, and the folder is removed when the [`Extracted`] guard is dropped (after +//! the read, on an error and on cancel alike). The zip itself is only read. The folder is made under +//! the open library's `Interop/` when there is one (the library's disk, not a RAM-backed `/tmp`), +//! else under [`std::env::temp_dir`]; a folder left by a crash (`lc-lrbackup-*`, older than a day) +//! is removed by the next extraction there. +//! +//! Written from the ZIP application note (PKWARE APPNOTE.TXT 6.3: end of central directory, +//! ZIP64 end record and locator, central directory headers, local headers, the ZIP64 extra field +//! 0x0001) and RFC 1951 (deflate, inflated by `miniz_oxide`); no third-party zip code was used. +//! Every size and offset comes from the file and is checked before use. + +use std::fs::File; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; + +/// The largest catalog a backup may unpack to: the SQLite reader's cap on a database file. +pub const MAX_CATALOG_BYTES: u64 = 16 << 30; +/// The most central directory entries looked at (a Lightroom backup has one to four). +const MAX_ENTRIES: u64 = 4096; +/// The largest central directory read (names of `MAX_ENTRIES` entries fit easily). +const MAX_DIRECTORY_BYTES: u64 = 4 << 20; +/// Prefix of the private extraction folders, so leftovers can be found. +const PREFIX: &str = "lc-lrbackup-"; +/// Leftover folders older than this are removed (one still being read is never this old). +const STALE_SECS: u64 = 24 * 60 * 60; +/// Bytes read from the zip and written to the catalog per step. +const CHUNK: usize = 1 << 20; + +/// Whether `path` names a Lightroom catalog backup: a `.zip` file (any case). +pub fn is_backup(path: &Path) -> bool { + path.extension().is_some_and(|e| e.eq_ignore_ascii_case("zip")) +} + +/// Where the catalog in a backup lived, for resolving the paths it records relative to itself: +/// `/Backups//.zip` → `/.lrcat`; a zip +/// elsewhere → the `.lrcat` next to it. +pub fn original_catalog(zip: &Path, entry: &str) -> PathBuf { + let name = Path::new(entry).file_name().map_or_else(|| PathBuf::from("catalog.lrcat"), PathBuf::from); + let dated = zip.parent(); + let backups = dated.and_then(Path::parent); + match backups { + Some(b) if b.file_name().is_some_and(|n| n.eq_ignore_ascii_case("backups")) => b.parent().unwrap_or(b).join(name), + _ => dated.map_or_else(|| name.clone(), |d| d.join(&name)), + } +} + +/// A catalog unpacked from a backup; its private folder is removed when this is dropped. +#[derive(Debug)] +pub struct Extracted { + dir: PathBuf, + /// The unpacked `.lrcat`. + pub catalog: PathBuf, + /// The catalog's name inside the zip. + pub entry: String, +} + +impl Drop for Extracted { + fn drop(&mut self) { + if let Err(e) = std::fs::remove_dir_all(&self.dir) + && e.kind() != std::io::ErrorKind::NotFound + { + log::warn!("Lightroom backup: could not remove {}: {e}", self.dir.display()); + } + } +} + +/// One file in the zip's central directory. +#[derive(Clone, Debug, PartialEq, Eq)] +struct Entry { + name: String, + method: u16, + crc: u32, + compressed: u64, + size: u64, + local: u64, +} + +fn u16_at(b: &[u8], at: usize) -> Option { + let s = b.get(at..at.checked_add(2)?)?; + Some(u16::from_le_bytes([s[0], s[1]])) +} + +fn u32_at(b: &[u8], at: usize) -> Option { + let s = b.get(at..at.checked_add(4)?)?; + Some(u32::from_le_bytes([s[0], s[1], s[2], s[3]])) +} + +fn u64_at(b: &[u8], at: usize) -> Option { + let s = b.get(at..at.checked_add(8)?)?; + let mut a = [0u8; 8]; + a.copy_from_slice(s); + Some(u64::from_le_bytes(a)) +} + +fn read_at(f: &mut File, at: u64, len: u64) -> Result, String> { + let len = usize::try_from(len).map_err(|_| "zip record too large".to_string())?; + f.seek(SeekFrom::Start(at)).map_err(|e| e.to_string())?; + let mut buf = vec![0u8; len]; + f.read_exact(&mut buf).map_err(|e| format!("truncated zip: {e}"))?; + Ok(buf) +} + +/// The central directory's entries (offset and count from the end record, or its ZIP64 record). +fn entries(f: &mut File) -> Result, String> { + let len = f.metadata().map_err(|e| e.to_string())?.len(); + // end of central directory: 22 bytes plus a comment of up to 64 KiB + let tail_len = len.min(22 + 65_535); + let tail_at = len - tail_len; + let tail = read_at(f, tail_at, tail_len)?; + let eocd = (0..tail.len().saturating_sub(21)).rev().find(|&i| tail.get(i..).is_some_and(|b| b.starts_with(b"PK\x05\x06"))); + let Some(eocd) = eocd else { return Err("not a zip file (no end of central directory)".into()) }; + let bad = || "damaged zip (end of central directory)".to_string(); + let mut count = u64::from(u16_at(&tail, eocd + 10).ok_or_else(bad)?); + let mut dir_len = u64::from(u32_at(&tail, eocd + 12).ok_or_else(bad)?); + let mut dir_at = u64::from(u32_at(&tail, eocd + 16).ok_or_else(bad)?); + // ZIP64: a locator just before the end record points at the ZIP64 end record + if (count == 0xFFFF || dir_len == 0xFFFF_FFFF || dir_at == 0xFFFF_FFFF) + && let Some(loc) = eocd.checked_sub(20) + && tail.get(loc..).is_some_and(|b| b.starts_with(b"PK\x06\x07")) + { + let rec_at = u64_at(&tail, loc + 8).ok_or_else(bad)?; + let rec = read_at(f, rec_at, 56)?; + if !rec.starts_with(b"PK\x06\x06") { + return Err("damaged zip (ZIP64 end record)".into()); + } + count = u64_at(&rec, 32).ok_or_else(bad)?; + dir_len = u64_at(&rec, 40).ok_or_else(bad)?; + dir_at = u64_at(&rec, 48).ok_or_else(bad)?; + } + if count > MAX_ENTRIES || dir_len > MAX_DIRECTORY_BYTES || dir_at.checked_add(dir_len).is_none_or(|end| end > len) { + return Err("damaged zip (central directory out of range)".into()); + } + let dir = read_at(f, dir_at, dir_len)?; + let mut out = Vec::new(); + let mut at = 0usize; + let bad = || "damaged zip (central directory)".to_string(); + for _ in 0..count { + if !dir.get(at..).is_some_and(|b| b.starts_with(b"PK\x01\x02")) { + return Err(bad()); + } + let method = u16_at(&dir, at + 10).ok_or_else(bad)?; + let crc = u32_at(&dir, at + 16).ok_or_else(bad)?; + let mut compressed = u64::from(u32_at(&dir, at + 20).ok_or_else(bad)?); + let mut size = u64::from(u32_at(&dir, at + 24).ok_or_else(bad)?); + let name_len = usize::from(u16_at(&dir, at + 28).ok_or_else(bad)?); + let extra_len = usize::from(u16_at(&dir, at + 30).ok_or_else(bad)?); + let comment_len = usize::from(u16_at(&dir, at + 32).ok_or_else(bad)?); + let mut local = u64::from(u32_at(&dir, at + 42).ok_or_else(bad)?); + let name_at = at + 46; + let name_end = name_at.checked_add(name_len).ok_or_else(bad)?; + let name = String::from_utf8_lossy(dir.get(name_at..name_end).ok_or_else(bad)?).replace('\\', "/"); + let extra = dir.get(name_end..name_end.checked_add(extra_len).ok_or_else(bad)?).ok_or_else(bad)?; + // ZIP64 extra field: the 64-bit values, in this order, of those set to 0xFFFFFFFF + let mut e = 0usize; + while e + 4 <= extra.len() { + let (id, len) = (u16_at(extra, e).ok_or_else(bad)?, usize::from(u16_at(extra, e + 2).ok_or_else(bad)?)); + let body = extra.get(e + 4..e + 4 + len).ok_or_else(bad)?; + if id == 0x0001 { + let mut v = 0usize; + for field in [&mut size, &mut compressed, &mut local] { + if *field == 0xFFFF_FFFF { + *field = u64_at(body, v).ok_or_else(bad)?; + v += 8; + } + } + } + e += 4 + len; + } + at = name_end.checked_add(extra_len).and_then(|x| x.checked_add(comment_len)).ok_or_else(bad)?; + out.push(Entry { name, method, crc, compressed, size, local }); + } + Ok(out) +} + +/// The backup's catalog entry: the one `*.lrcat` file in the zip, outside macOS resource forks. +fn catalog_entry(entries: &[Entry]) -> Result<&Entry, String> { + let mut found = entries.iter().filter(|e| { + let lower = e.name.to_lowercase(); + lower.ends_with(".lrcat") && !lower.split('/').any(|p| p == "__macosx" || p.starts_with("._")) + }); + match (found.next(), found.next()) { + (Some(e), None) => Ok(e), + (None, _) => Err("this zip holds no Lightroom catalog (.lrcat)".into()), + (Some(_), Some(_)) => Err("this zip holds more than one Lightroom catalog (.lrcat); unzip it and import one".into()), + } +} + +/// Remove `lc-lrbackup-*` folders in `root` left by an earlier run that ended without cleaning up. +fn sweep(root: &Path) { + let Ok(dir) = std::fs::read_dir(root) else { return }; + for entry in dir.take(10_000).flatten() { + let stale = + entry.metadata().ok().and_then(|m| m.modified().ok()).and_then(|t| t.elapsed().ok()).is_some_and(|age| age.as_secs() > STALE_SECS); + if stale && entry.file_name().to_str().is_some_and(|n| n.starts_with(PREFIX)) { + let _ = std::fs::remove_dir_all(entry.path()); + } + } +} + +/// Unpack the catalog of the backup `zip` into a new private folder under `root`, checking its +/// size and CRC-32. Stops with an error when `cancel` is set; the folder is removed then too. +pub fn extract(zip: &Path, root: &Path, cancel: &AtomicBool) -> Result { + let mut f = File::open(zip).map_err(|e| format!("{}: {e}", zip.display()))?; + let all = entries(&mut f)?; + let entry = catalog_entry(&all)?.clone(); + if entry.size > MAX_CATALOG_BYTES { + return Err(format!("the catalog in this backup unpacks to {} GB, more than LightCraft reads", entry.size >> 30)); + } + std::fs::create_dir_all(root).map_err(|e| format!("{}: {e}", root.display()))?; + sweep(root); + let stamp = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_nanos()); + let dir = root.join(format!("{PREFIX}{}-{stamp}", std::process::id())); + std::fs::create_dir(&dir).map_err(|e| format!("{}: {e}", dir.display()))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let _ = std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)); + } + let name = Path::new(&entry.name).file_name().map_or_else(|| "catalog.lrcat".into(), |n| n.to_string_lossy().into_owned()); + // the guard exists before the first byte is written: every early return below cleans up + let out = Extracted { catalog: dir.join(&name), dir, entry: entry.name.clone() }; + let local = read_at(&mut f, entry.local, 30)?; + if !local.starts_with(b"PK\x03\x04") { + return Err("damaged zip (local header)".into()); + } + let bad = || "damaged zip (local header)".to_string(); + let skip = u64::from(u16_at(&local, 26).ok_or_else(bad)?) + u64::from(u16_at(&local, 28).ok_or_else(bad)?); + let data_at = entry.local.checked_add(30 + skip).ok_or_else(bad)?; + f.seek(SeekFrom::Start(data_at)).map_err(|e| e.to_string())?; + let mut input = f.take(entry.compressed); + let file = File::create(&out.catalog).map_err(|e| format!("{}: {e}", out.catalog.display()))?; + let mut writer = std::io::BufWriter::with_capacity(CHUNK, file); + let mut crc = crc32fast::Hasher::new(); + let mut written = 0u64; + let mut write = |bytes: &[u8], written: &mut u64| -> Result<(), String> { + *written = written.saturating_add(bytes.len() as u64); + if *written > entry.size { + return Err("damaged zip (the catalog is longer than the zip says)".into()); + } + crc.update(bytes); + writer.write_all(bytes).map_err(|e| format!("writing {}: {e}", out.catalog.display())) + }; + let mut buf = vec![0u8; CHUNK]; + match entry.method { + 0 => loop { + if cancel.load(Ordering::Relaxed) { + return Err("cancelled".into()); + } + let n = input.read(&mut buf).map_err(|e| e.to_string())?; + if n == 0 { + break; + } + write(buf.get(..n).unwrap_or_default(), &mut written)?; + }, + 8 => { + use miniz_oxide::inflate::stream::{InflateState, inflate}; + use miniz_oxide::{DataFormat, MZFlush, MZStatus}; + let mut state = InflateState::new_boxed(DataFormat::Raw); + let mut out_buf = vec![0u8; CHUNK]; + let (mut start, mut end) = (0usize, 0usize); + loop { + if cancel.load(Ordering::Relaxed) { + return Err("cancelled".into()); + } + if start == end { + end = input.read(&mut buf).map_err(|e| e.to_string())?; + start = 0; + } + let flush = if end == 0 { MZFlush::Finish } else { MZFlush::None }; + let r = inflate(&mut state, buf.get(start..end).unwrap_or_default(), &mut out_buf, flush); + start = start.saturating_add(r.bytes_consumed).min(end); + write(out_buf.get(..r.bytes_written).unwrap_or_default(), &mut written)?; + match r.status { + Ok(MZStatus::StreamEnd) => break, + Ok(_) => {} + // no progress with all input given: the stream is cut short + Err(miniz_oxide::MZError::Buf) if end == 0 && r.bytes_written == 0 => return Err("damaged zip (the catalog is cut short)".into()), + Err(miniz_oxide::MZError::Buf) => {} + Err(e) => return Err(format!("damaged zip (deflate: {e:?})")), + } + if r.bytes_consumed == 0 && r.bytes_written == 0 && start < end { + return Err("damaged zip (the catalog's data does not inflate)".into()); + } + } + } + m => return Err(format!("this zip uses compression method {m}, which LightCraft does not read; unzip it and import the .lrcat")), + } + writer.flush().map_err(|e| format!("writing {}: {e}", out.catalog.display()))?; + drop(writer); + if written != entry.size { + return Err("damaged zip (the catalog is shorter than the zip says)".into()); + } + if crc.finalize() != entry.crc { + return Err("damaged zip (the catalog's checksum does not match)".into()); + } + Ok(out) +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + + /// A zip of `files` (name, bytes), each stored or deflated; `zip64` writes ZIP64 records and + /// extra fields (with the 32-bit fields set to 0xFFFFFFFF) as large archives do. + pub(crate) fn zip(files: &[(&str, &[u8])], deflate: bool, zip64: bool) -> Vec { + let mut out = Vec::new(); + let mut central = Vec::new(); + for (name, data) in files { + let body = if deflate { miniz_oxide::deflate::compress_to_vec(data, 6) } else { data.to_vec() }; + let crc = crc32fast::hash(data); + let local = out.len() as u64; + let method: u16 = if deflate { 8 } else { 0 }; + let big = |v: u64| if zip64 { 0xFFFF_FFFFu32 } else { v as u32 }; + let mut extra = Vec::new(); + if zip64 { + extra.extend_from_slice(&1u16.to_le_bytes()); + extra.extend_from_slice(&24u16.to_le_bytes()); + extra.extend_from_slice(&(data.len() as u64).to_le_bytes()); + extra.extend_from_slice(&(body.len() as u64).to_le_bytes()); + extra.extend_from_slice(&local.to_le_bytes()); + } + out.extend_from_slice(b"PK\x03\x04"); + out.extend_from_slice(&[20, 0, 0, 0]); + out.extend_from_slice(&method.to_le_bytes()); + out.extend_from_slice(&[0; 4]); + out.extend_from_slice(&crc.to_le_bytes()); + out.extend_from_slice(&big(body.len() as u64).to_le_bytes()); + out.extend_from_slice(&big(data.len() as u64).to_le_bytes()); + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(name.as_bytes()); + out.extend_from_slice(&body); + central.extend_from_slice(b"PK\x01\x02"); + central.extend_from_slice(&[45, 0, 45, 0, 0, 0]); + central.extend_from_slice(&method.to_le_bytes()); + central.extend_from_slice(&[0; 4]); + central.extend_from_slice(&crc.to_le_bytes()); + central.extend_from_slice(&big(body.len() as u64).to_le_bytes()); + central.extend_from_slice(&big(data.len() as u64).to_le_bytes()); + central.extend_from_slice(&(name.len() as u16).to_le_bytes()); + central.extend_from_slice(&(extra.len() as u16).to_le_bytes()); + // comment length, disk number, internal attributes, external attributes + central.extend_from_slice(&[0; 6]); + central.extend_from_slice(&0u32.to_le_bytes()); + central.extend_from_slice(&big(local).to_le_bytes()); + central.extend_from_slice(name.as_bytes()); + central.extend_from_slice(&extra); + } + let dir_at = out.len() as u64; + out.extend_from_slice(¢ral); + let count = files.len() as u64; + if zip64 { + let rec_at = out.len() as u64; + out.extend_from_slice(b"PK\x06\x06"); + out.extend_from_slice(&44u64.to_le_bytes()); + out.extend_from_slice(&[45, 0, 45, 0]); + out.extend_from_slice(&[0; 8]); + out.extend_from_slice(&count.to_le_bytes()); + out.extend_from_slice(&count.to_le_bytes()); + out.extend_from_slice(&(central.len() as u64).to_le_bytes()); + out.extend_from_slice(&dir_at.to_le_bytes()); + out.extend_from_slice(b"PK\x06\x07"); + out.extend_from_slice(&0u32.to_le_bytes()); + out.extend_from_slice(&rec_at.to_le_bytes()); + out.extend_from_slice(&1u32.to_le_bytes()); + } + out.extend_from_slice(b"PK\x05\x06"); + out.extend_from_slice(&[0; 4]); + let c16 = if zip64 { 0xFFFF } else { count as u16 }; + out.extend_from_slice(&c16.to_le_bytes()); + out.extend_from_slice(&c16.to_le_bytes()); + out.extend_from_slice(&(if zip64 { 0xFFFF_FFFF } else { central.len() as u32 }).to_le_bytes()); + out.extend_from_slice(&(if zip64 { 0xFFFF_FFFF } else { dir_at as u32 }).to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out + } + + fn temp(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("lc-lrbackup-test-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir + } + + fn leftovers(root: &Path) -> Vec { + let Ok(dir) = std::fs::read_dir(root) else { return Vec::new() }; + dir.flatten().map(|e| e.file_name().to_string_lossy().into_owned()).filter(|n| n.starts_with(PREFIX)).collect() + } + + #[test] + fn stored_deflated_and_zip64_backups_unpack_their_catalog_and_clean_up() { + let catalog: Vec = (0..300_000u32).flat_map(|i| (i % 251).to_le_bytes()).collect(); + for (deflate, zip64) in [(false, false), (true, false), (true, true), (false, true)] { + let dir = temp(&format!("ok-{deflate}-{zip64}")); + let path = dir.join("Cat.zip"); + let files: [(&str, &[u8]); 3] = [("Cat.lrcat", &catalog), ("Cat.lrcat-data/CURRENT", b"MANIFEST-1\n"), ("__MACOSX/._Cat.lrcat", b"x")]; + std::fs::write(&path, zip(&files, deflate, zip64)).unwrap(); + let root = dir.join("tmp"); + let got = extract(&path, &root, &AtomicBool::new(false)).unwrap(); + assert_eq!(std::fs::read(&got.catalog).unwrap(), catalog, "deflate {deflate} zip64 {zip64}"); + assert_eq!(got.entry, "Cat.lrcat"); + drop(got); + assert!(leftovers(&root).is_empty(), "the private folder is removed"); + std::fs::remove_dir_all(&dir).unwrap(); + } + } + + #[test] + fn damaged_or_wrong_zips_are_errors_and_leave_nothing_behind() { + let dir = temp("bad"); + let root = dir.join("tmp"); + let catalog = vec![7u8; 100_000]; + let good = zip(&[("Cat.lrcat", &catalog)], true, false); + let cases: Vec<(&str, Vec)> = vec![ + ("not a zip", b"SQLite format 3\0".to_vec()), + ("empty", Vec::new()), + ("no catalog", zip(&[("notes.txt", b"hi")], false, false)), + ("two catalogs", zip(&[("A.lrcat", b"a"), ("B.lrcat", b"b")], false, false)), + ("cut short", good.get(..good.len() / 2).unwrap().to_vec()), + ("bad crc", { + let mut z = good.clone(); + // flip one byte of the deflated data (after the 30-byte local header and the name) + z[30 + 9 + 10] ^= 0xFF; + z + }), + ]; + for (what, bytes) in cases { + let path = dir.join("x.zip"); + std::fs::write(&path, &bytes).unwrap(); + assert!(extract(&path, &root, &AtomicBool::new(false)).is_err(), "{what} must fail"); + if root.exists() { + assert!(leftovers(&root).is_empty(), "{what} left a folder"); + } + } + std::fs::write(dir.join("x.zip"), &good).unwrap(); + assert!(extract(&dir.join("x.zip"), &root, &AtomicBool::new(true)).is_err(), "cancel stops"); + assert!(leftovers(&root).is_empty(), "cancel cleans up"); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn a_zip_that_lies_about_the_catalog_size_is_refused() { + let dir = temp("size"); + let mut z = zip(&[("Cat.lrcat", &[1u8; 1000])], true, true); + // the ZIP64 extra field's uncompressed size: 1 TB + let at = z.windows(4).rposition(|w| w == [1, 0, 24, 0]).unwrap() + 4; + z[at..at + 8].copy_from_slice(&(1u64 << 40).to_le_bytes()); + std::fs::write(dir.join("x.zip"), &z).unwrap(); + let err = extract(&dir.join("x.zip"), &dir.join("tmp"), &AtomicBool::new(false)).unwrap_err(); + assert!(err.contains("more than LightCraft reads"), "{err}"); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[cfg(unix)] + #[test] + fn stale_leftovers_are_swept_and_recent_ones_kept() { + let root = temp("sweep"); + let old = root.join(format!("{PREFIX}1-1")); + let recent = root.join(format!("{PREFIX}2-2")); + std::fs::create_dir_all(&old).unwrap(); + std::fs::create_dir_all(&recent).unwrap(); + let past = std::time::SystemTime::now() - std::time::Duration::from_secs(STALE_SECS + 60); + File::open(&old).unwrap().set_modified(past).unwrap(); + sweep(&root); + assert!(!old.exists() && recent.exists()); + std::fs::remove_dir_all(&root).unwrap(); + } + + #[test] + fn the_original_catalog_is_two_folders_above_a_dated_backup() { + let zip = Path::new("/p/Lightroom/Backups/2023-11-07 0900/Cat-v13.zip"); + assert_eq!(original_catalog(zip, "Cat-v13.lrcat"), Path::new("/p/Lightroom/Cat-v13.lrcat")); + assert_eq!(original_catalog(Path::new("/elsewhere/Cat.zip"), "sub/Cat.lrcat"), Path::new("/elsewhere/Cat.lrcat")); + assert!(is_backup(Path::new("a/B.ZIP")) && !is_backup(Path::new("a/B.lrcat"))); + } +} diff --git a/crates/engine/src/lightroom_catalog.rs b/crates/engine/src/lightroom_catalog.rs index 797a96708b..e84f36e43c 100644 --- a/crates/engine/src/lightroom_catalog.rs +++ b/crates/engine/src/lightroom_catalog.rs @@ -78,6 +78,15 @@ impl Reader { .collect()) } + /// The catalog's `Adobe_storeProviderID` (a UUID), kept by Lightroom in its backups; `None` + /// when the table or the value is missing or doesn't look like an id. + fn catalog_id(&self) -> Option { + let rows = self.table("Adobe_variablesTable", false).ok()?; + let row = rows.iter().find(|r| text(r, "name") == "Adobe_storeProviderID")?; + let id = text(row, "value").trim(); + (!id.is_empty() && id.len() <= 128 && id.chars().all(|c| c.is_ascii_alphanumeric() || c == '-')).then(|| id.to_string()) + } + fn optional_table(&self, name: &str, warnings: &mut Vec) -> Result, String> { match self.table(name, false) { Ok(rows) => Ok(rows), @@ -176,6 +185,11 @@ impl CatalogPhoto { #[derive(Clone, Debug, Serialize, Deserialize)] pub struct CatalogImport { pub source: String, + /// The catalog's own id (`Adobe_storeProviderID` in `Adobe_variablesTable`), the same in the + /// catalog and in every backup of it, so a backup's collections match those of an earlier import + /// of the catalog (issue #2592). `None` when the catalog has none. + #[serde(default)] + pub catalog_id: Option, pub photos: Vec, pub collections: Vec, pub members: Vec, @@ -440,10 +454,42 @@ pub fn read(path: &Path) -> Result { /// stopping when `cancel` is set. Images whose catalog records are incomplete are left out and /// listed in [`CatalogImport::skipped`] instead of failing the read. pub fn read_with_progress(path: &Path, cancel: &AtomicBool, total: &AtomicUsize, done: &AtomicUsize) -> Result { + read_source(path, None, cancel, total, done) +} + +/// Where a Lightroom backup's catalog is unpacked while it is read: the open library's `Interop/` +/// folder (on the library's disk), or `None` for the system's temporary folder. +pub fn backup_temp_root(s: &crate::Session) -> Option { + s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.join("Interop")) +} + +/// [`read_with_progress`] of a catalog or of a Lightroom catalog backup (`.zip`, issue #2592): a +/// backup's catalog is unpacked into a private folder under `temp_root` (else the system's temporary +/// folder), read, and removed again ([`crate::lightroom_backup`]). Paths the catalog records +/// relative to itself are taken from where the catalog was before it was backed up. +pub fn read_source( + path: &Path, + temp_root: Option<&Path>, + cancel: &AtomicBool, + total: &AtomicUsize, + done: &AtomicUsize, +) -> Result { // 13 catalog tables plus one validation stage; ImportJob adds one unit per original path. total.store(14, Ordering::Relaxed); done.store(0, Ordering::Relaxed); ensure_read_active(cancel)?; + if crate::lightroom_backup::is_backup(path) { + let root = temp_root.map_or_else(std::env::temp_dir, Path::to_path_buf); + let extracted = crate::lightroom_backup::extract(path, &root, cancel)?; + let document = crate::lightroom_backup::original_catalog(path, &extracted.entry); + return read_file(&extracted.catalog, &document, path, cancel, done); + } + read_file(path, path, path, cancel, done) +} + +/// Read the catalog file `path`, resolving the paths it records with `document` as the catalog's +/// place and reporting `source` as where it came from. +fn read_file(path: &Path, document: &Path, source: &Path, cancel: &AtomicBool, done: &AtomicUsize) -> Result { let before = std::fs::metadata(path).map_err(|e| e.to_string())?; let wal_path = PathBuf::from(format!("{}-wal", path.display())); let wal_before = std::fs::metadata(&wal_path).ok().map(|m| (m.len(), m.modified().ok())); @@ -529,7 +575,7 @@ pub fn read_with_progress(path: &Path, cancel: &AtomicBool, total: &AtomicUsize, root_ids.sort_unstable(); for (id, root) in root_ids.iter().filter_map(|id| Some((*id, roots.get(id)?))) { let absolute = text(root, "absolutePath"); - let resolved = match relocated_root(absolute, text(root, "relativePathFromCatalog"), path) { + let resolved = match relocated_root(absolute, text(root, "relativePathFromCatalog"), document) { Some(found) => { let found = found.to_string_lossy().to_string(); warnings.push(format!("root folder {}: {absolute} not found; using {found} (relative to the catalog)", text(root, "name"))); @@ -537,7 +583,7 @@ pub fn read_with_progress(path: &Path, cancel: &AtomicBool, total: &AtomicUsize, } None => absolute.to_string(), }; - if let Err(crate::doc_path::NotFollowed::Refused(why)) = crate::doc_path::resolve(&resolved, Some(path)) { + if let Err(crate::doc_path::NotFollowed::Refused(why)) = crate::doc_path::resolve(&resolved, Some(document)) { warnings .push(format!("root folder {}: {resolved} is not opened: {why}; its photos are listed as missing and not added", text(root, "name"))); refused_roots.insert(id); @@ -547,7 +593,8 @@ pub fn read_with_progress(path: &Path, cancel: &AtomicBool, total: &AtomicUsize, let mut history = group(optional_table("Adobe_libraryImageDevelopHistoryStep", &mut warnings)?); let mut snapshots = group(optional_table("Adobe_libraryImageDevelopSnapshot", &mut warnings)?); let mut out = CatalogImport { - source: path.to_string_lossy().into(), + source: source.to_string_lossy().into(), + catalog_id: reader.catalog_id(), photos: Vec::new(), collections: optional_table("AgLibraryCollection", &mut warnings)?, members: optional_table("AgLibraryCollectionImage", &mut warnings)?, @@ -584,7 +631,7 @@ pub fn read_with_progress(path: &Path, cancel: &AtomicBool, total: &AtomicUsize, orphans.insert(id); continue; } - let (photo_path, original) = original_path(root_path, folder_path, name, path); + let (photo_path, original) = original_path(root_path, folder_path, name, document); if let Original::Refused(why) = &original && !refused_roots.contains(&root_id) { @@ -1027,13 +1074,15 @@ pub(crate) fn apply_prepared(s: &mut crate::Session, mut data: CatalogImport, co if number(collection, "systemOnly") != 0 { continue; } - let collection_key = format!("{}:{}", key(&data.source), number(collection, "id_local")); - let id = index - .collections - .get(&collection_key) - .copied() + // keyed on the catalog's own id (the same in its backups); imports made before that was + // read are keyed on the catalog's path, and are still found by it + let path_key = format!("{}:{}", key(&data.source), number(collection, "id_local")); + let collection_key = data.catalog_id.as_ref().map_or_else(|| path_key.clone(), |c| format!("lrcat:{c}:{}", number(collection, "id_local"))); + let id = [&collection_key, &path_key] + .into_iter() + .filter_map(|k| index.collections.get(k).copied()) .map(lightcraft_catalog::AlbumId) - .filter(|id| s.catalog.album(*id).is_some()) + .find(|id| s.catalog.album(*id).is_some()) .unwrap_or_else(|| s.catalog.alloc_album_id()); index.collections.insert(collection_key, id.0); albums.insert(number(collection, "id_local"), id); @@ -1126,6 +1175,7 @@ mod tests { photo.image.insert("captureTime".into(), json!("2024-01-16T12:34:56")); CatalogImport { source: "test.lrcat".into(), + catalog_id: None, photos: vec![copy, photo], collections: vec![ row(json!({"id_local":9,"name":"Travel","creationId":"collection.group"})), @@ -1521,6 +1571,97 @@ mod tests { } } + /// Issue #2592: a catalog in Lightroom's backup layout (`Backups//.zip`), with one + /// collection and the catalog's own id. + fn backup_fixture(root: &Path, catalog_id: &str) -> (PathBuf, PathBuf) { + use SqlValue::{Integer as I, Null as N, Text as T}; + let mut tables: Vec<(&str, &str, Vec>)> = one_row(photo_tables(("/catalog/", ""))); + tables.push(( + "AgLibraryCollection", + "id_local INTEGER PRIMARY KEY, creationId, name, parent, systemOnly", + vec![vec![N, T("com.adobe.ag.library.collection".into()), T("Keepers".into()), N, I(0)]], + )); + tables.push(("AgLibraryCollectionImage", "id_local INTEGER PRIMARY KEY, collection, image", vec![vec![N, I(1), I(1)]])); + tables.push(( + "Adobe_variablesTable", + "id_local INTEGER PRIMARY KEY, id_global, name, type, value", + vec![vec![N, T("v1".into()), T("Adobe_storeProviderID".into()), N, T(catalog_id.into())]], + )); + let bytes = native_catalog_rows(&tables, None); + let folder = root.join("Lightroom"); + let dated = folder.join("Backups").join("2023-11-07 0900"); + std::fs::create_dir_all(&dated).unwrap(); + let live = folder.join("Cat.lrcat"); + std::fs::write(&live, &bytes).unwrap(); + let zip = dated.join("Cat.zip"); + let files: [(&str, &[u8]); 2] = [("Cat.lrcat", &bytes), ("Cat.lrcat-data/CURRENT", b"MANIFEST-000005\n")]; + std::fs::write(&zip, crate::lightroom_backup::tests::zip(&files, true, false)).unwrap(); + (live, zip) + } + + fn albums(s: &crate::Session) -> usize { + s.catalog.albums().count() + } + + #[test] + fn a_backup_zip_imports_like_its_catalog_and_leaves_no_temporary_files() { + let root = std::env::temp_dir().join(format!("lightcraft-lrbackup-zip-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + let (live, zip) = backup_fixture(&root, "968AF29B-EEE2-481E-8AF9-55DD82965F49"); + let zip_before = std::fs::read(&zip).unwrap(); + let from_zip = read(&zip).unwrap(); + let from_live = read(&live).unwrap(); + assert_eq!(from_zip.source, zip.to_string_lossy(), "the zip is the import's source"); + assert_eq!(from_zip.catalog_id.as_deref(), Some("968AF29B-EEE2-481E-8AF9-55DD82965F49")); + assert_eq!((from_zip.photos.len(), from_zip.collections.len()), (from_live.photos.len(), from_live.collections.len())); + assert_eq!(from_zip.photos[0].path, from_live.photos[0].path); + // the live catalog is gone (Lightroom uninstalled): the backup alone imports into a library + std::fs::remove_file(&live).unwrap(); + let lib = root.join("Library"); + let mut s = crate::Session::new(); + s.open_library(&lib, false).unwrap(); + let report = import(&mut s, &zip, false).unwrap(); + assert_eq!((report["photos"].as_u64(), report["collections"].as_u64()), (Some(1), Some(1))); + assert_eq!(report["source"], json!(zip.to_string_lossy())); + assert_eq!(albums(&s), 1); + let interop: Vec<_> = + std::fs::read_dir(lib.join("Interop")).unwrap().flatten().map(|e| e.file_name().to_string_lossy().into_owned()).collect(); + assert!(!interop.iter().any(|n| n.starts_with("lc-lrbackup-")), "the unpacked catalog is removed: {interop:?}"); + assert_eq!(std::fs::read(&zip).unwrap(), zip_before, "the backup is only read"); + let _ = std::fs::remove_dir_all(&root); + } + + #[test] + fn importing_a_backup_after_its_catalog_adds_no_album_twice() { + let root = std::env::temp_dir().join(format!("lightcraft-lrbackup-twice-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + let (live, zip) = backup_fixture(&root, "968AF29B-EEE2-481E-8AF9-55DD82965F49"); + let mut s = crate::Session::new(); + s.open_library(root.join("Library"), false).unwrap(); + import(&mut s, &live, false).unwrap(); + assert_eq!(albums(&s), 1); + let again = import(&mut s, &zip, false).unwrap(); + assert_eq!(again["imported"], 0, "the photos are matched by their Lightroom id"); + assert_eq!(albums(&s), 1, "the collections are matched by the catalog's id, not its path"); + let _ = std::fs::remove_dir_all(&root); + } + + #[test] + fn collections_imported_before_the_catalog_id_was_read_are_still_matched_by_path() { + let dir = std::env::temp_dir().join(format!("lightcraft-lrcat-pathkey-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let mut s = crate::Session::new(); + s.open_library(&dir, false).unwrap(); + // an import made by an earlier release: no catalog id, keys on the path + apply(&mut s, sample(), false).unwrap(); + let before = albums(&s); + let mut data = sample(); + data.catalog_id = Some("A-B".into()); + apply(&mut s, data, false).unwrap(); + assert_eq!(albums(&s), before, "the old path keys still find the albums"); + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn persistent_reimport_keeps_copy_and_collection_ids_and_capture_time() { let dir = std::env::temp_dir().join(format!("lightcraft-lrcat-reimport-{}", std::process::id())); @@ -1608,7 +1749,9 @@ mod tests { /// table's root page. fn native_catalog_rows(tables: &[(&str, &str, Vec>)], last_page: Option) -> Vec { use SqlValue::{Integer as I, Text as T}; - let mut bytes = vec![0u8; 5 * 2048]; + // the schema page, then one page per table (at least four: the photo tables) + let pages = (tables.len() + 1).max(5); + let mut bytes = vec![0u8; pages * 2048]; bytes[..16].copy_from_slice(b"SQLite format 3\0"); bytes[16..18].copy_from_slice(&2048u16.to_be_bytes()); bytes[18] = 1; @@ -1616,7 +1759,7 @@ mod tests { bytes[21] = 64; bytes[22] = 32; bytes[23] = 32; - bytes[28..32].copy_from_slice(&5u32.to_be_bytes()); + bytes[28..32].copy_from_slice(&(pages as u32).to_be_bytes()); bytes[44..48].copy_from_slice(&4u32.to_be_bytes()); bytes[56..60].copy_from_slice(&1u32.to_be_bytes()); let mut schema = Vec::new(); diff --git a/crates/engine/src/lightroom_folder.rs b/crates/engine/src/lightroom_folder.rs index 0b5a85659a..3c91648ffb 100644 --- a/crates/engine/src/lightroom_folder.rs +++ b/crates/engine/src/lightroom_folder.rs @@ -13,7 +13,9 @@ //! `*.lrcat-journal`, `*.lrcat.lock`); //! - a folder named `*.lrdata` or `*.lrcat-data` (Lightroom's previews and data); //! - a folder holding a `*.lrcat` at its top level (the catalog folder itself), or `Backups/` -//! next to one. +//! next to one; +//! - Lightroom's backups (issue #2592), even with the catalog itself gone: a `Backups/` folder +//! (its newest backup) or one of its dated folders, `Backups/YYYY-MM-DD HHMM/`, holding a `*.zip`. //! //! The folder scan is bounded ([`SCAN_LIMIT`] entries, top level only) and every I/O error just //! means "not found": this never fails and never panics. @@ -133,6 +135,34 @@ pub fn lightroom_catalog_at(path: &Path) -> Option { { return Some(LightroomPlace { catalog: Some(catalog) }); } + // `Backups/` alone (the catalog deleted): its newest backup + if lower == "backups" + && let Some(zip) = newest_backup(path) + { + return Some(LightroomPlace { catalog: Some(zip) }); + } + // a dated backup folder, `Backups/2023-11-07 0900/` + if parent.and_then(lower_name).as_deref() == Some("backups") + && let Some(zip) = find_backup_zip(path) + { + return Some(LightroomPlace { catalog: Some(zip) }); + } } None } + +/// The first `*.zip` file (by name) at the top level of `dir`, looking at no more than +/// [`SCAN_LIMIT`] entries. +fn find_backup_zip(dir: &Path) -> Option { + let entries = std::fs::read_dir(dir).ok()?; + entries.take(SCAN_LIMIT).flatten().map(|e| e.path()).filter(|p| crate::lightroom_backup::is_backup(p) && p.is_file()).min() +} + +/// The backup zip of the newest dated folder in `backups` (Lightroom names them +/// `YYYY-MM-DD HHMM`, so the newest sorts last). +fn newest_backup(backups: &Path) -> Option { + let entries = std::fs::read_dir(backups).ok()?; + let mut dated: Vec = entries.take(SCAN_LIMIT).flatten().map(|e| e.path()).filter(|p| p.is_dir()).collect(); + dated.sort(); + dated.iter().rev().find_map(|d| find_backup_zip(d)) +} diff --git a/crates/engine/src/lightroom_job.rs b/crates/engine/src/lightroom_job.rs index 762c8588df..0a2eac60df 100644 --- a/crates/engine/src/lightroom_job.rs +++ b/crates/engine/src/lightroom_job.rs @@ -241,8 +241,14 @@ impl LightroomJob { return Err(cancelled()); } // (reading stops with a message when `cancel` is set: the flag says it was cancelled) - let mut data = lightroom_catalog::read_with_progress(&self.path, cancel, &self.progress.catalog_total, &self.progress.catalog_done) - .map_err(|e| if cancel.load(Ordering::Relaxed) { cancelled() } else { error(e) })?; + let mut data = lightroom_catalog::read_source( + &self.path, + self.archive_dir.as_deref(), + cancel, + &self.progress.catalog_total, + &self.progress.catalog_done, + ) + .map_err(|e| if cancel.load(Ordering::Relaxed) { cancelled() } else { error(e) })?; lightroom_catalog::validate_for_job(&data).map_err(error)?; if cancel.load(Ordering::Relaxed) { return Err(cancelled()); @@ -350,6 +356,7 @@ mod tests { fn empty_data() -> CatalogImport { CatalogImport { source: "fixture.lrcat".into(), + catalog_id: None, photos: Vec::new(), collections: Vec::new(), members: Vec::new(), @@ -384,6 +391,7 @@ mod tests { PreparedLightroom { data: CatalogImport { source: "fixture.lrcat".into(), + catalog_id: None, photos: vec![lightroom_catalog::CatalogPhoto { source_id: 1, uuid: "fixture-photo".into(), diff --git a/crates/engine/src/tests_lightroom_folder.rs b/crates/engine/src/tests_lightroom_folder.rs index eccff23ac9..294473d3b6 100644 --- a/crates/engine/src/tests_lightroom_folder.rs +++ b/crates/engine/src/tests_lightroom_folder.rs @@ -98,6 +98,28 @@ fn an_lrdata_folder_whose_catalog_was_renamed_finds_the_one_next_to_it() { let _ = std::fs::remove_dir_all(&root); } +/// Issue #2592: someone who deleted Lightroom and its catalog keeps only `Backups/`; that folder +/// (its newest backup) and each dated folder in it are refused and name the backup zip to import. +#[test] +fn lightroom_backups_without_their_catalog_are_refused_and_name_the_zip() { + let root = temp_dir("backups"); + let backups = root.join("Lightroom").join("Backups"); + for (dated, zip) in [("2019-03-26 0751", "Cat-2.lrcat.zip"), ("2023-11-07 0900", "Cat-v13.zip")] { + std::fs::create_dir_all(backups.join(dated)).unwrap(); + std::fs::write(backups.join(dated).join(zip), b"PK").unwrap(); + } + let newest = backups.join("2023-11-07 0900").join("Cat-v13.zip"); + refused(&root, &backups, &newest); + refused(&root, &backups.join("2023-11-07 0900"), &newest); + refused(&root, &backups.join("2019-03-26 0751"), &backups.join("2019-03-26 0751").join("Cat-2.lrcat.zip")); + // a folder of zips that isn't in a `Backups/` folder is not Lightroom's + let other = root.join("Downloads"); + std::fs::create_dir_all(&other).unwrap(); + std::fs::write(other.join("photos.zip"), b"PK").unwrap(); + assert_eq!(lightroom_catalog_at(&other), None); + let _ = std::fs::remove_dir_all(&root); +} + #[test] fn an_empty_folder_still_becomes_a_library() { let root = temp_dir("empty"); diff --git a/crates/ui-egui/src/lightroom_import.rs b/crates/ui-egui/src/lightroom_import.rs index 2bda28cf42..6a33e2e728 100644 --- a/crates/ui-egui/src/lightroom_import.rs +++ b/crates/ui-egui/src/lightroom_import.rs @@ -159,6 +159,7 @@ fn busy(app: &LightcraftApp) -> Result<(), String> { #[cfg(not(target_arch = "wasm32"))] fn spawn_inspect( path: PathBuf, + temp: Option, cancel: Arc, progress: LightroomProgress, tx: mpsc::Sender, @@ -168,7 +169,7 @@ fn spawn_inspect( .name("lc-lightroom-inspect".into()) .spawn(move || { let result = lightcraft_engine::guard::catch("Lightroom inspect", || { - lightcraft_engine::lightroom_catalog::read_with_progress(&path, &cancel, &progress.catalog_total, &progress.catalog_done) + lightcraft_engine::lightroom_catalog::read_source(&path, temp.as_deref(), &cancel, &progress.catalog_total, &progress.catalog_done) }) .and_then(|r| r.map(inspect_report)); let _ = tx.send(Message::Inspected(result)); @@ -358,7 +359,8 @@ fn start_inspect(app: &mut LightcraftApp, path: PathBuf, ctx: &egui::Context) -> let cancel = Arc::new(AtomicBool::new(false)); let progress = LightroomProgress::default(); let (tx, rx) = mpsc::channel(); - spawn_inspect(path.clone(), cancel.clone(), progress.clone(), tx, ctx.clone())?; + let temp = lightcraft_engine::lightroom_catalog::backup_temp_root(&app.session); + spawn_inspect(path.clone(), temp, cancel.clone(), progress.clone(), tx, ctx.clone())?; app.lightroom_last = None; app.lightroom = Some(LightroomTask::new(&app.session, Kind::Inspect, path.clone(), cancel, progress, rx)); Ok(json!({"running": true, "kind": "inspect", "path": path})) @@ -721,6 +723,7 @@ mod tests { fn inspect_report_keeps_empty_catalog_shape() { let report = inspect_report(CatalogImport { source: "synthetic.lrcat".into(), + catalog_id: None, photos: Vec::new(), collections: Vec::new(), members: Vec::new(), diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index 04d43e0d34..f251f7782a 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -1617,8 +1617,11 @@ pub fn run_ui_command(app: &mut LightcraftApp, id: &str, p: &Value) -> Option Some(path), None => { - let req = - PickRequest::file(crate::i18n::tr("Import Lightroom Catalog"), crate::i18n::tr("Lightroom Classic Catalog"), &["lrcat"]); + let req = PickRequest::file( + crate::i18n::tr("Import Lightroom Catalog"), + crate::i18n::tr("Lightroom Classic Catalog"), + &["lrcat", "zip"], + ); match crate::pick::ask(app, id, p, "path", req, |s| s.pick_lightroom_catalog.as_mut().and_then(|f| f()).map(|x| vec![x])) { Picked::Now(v) => v.into_iter().next(), Picked::Later | Picked::Unavailable => None, diff --git a/docs/gaps.md b/docs/gaps.md index be411a2050..160a20d35d 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -1,6 +1,6 @@ # Gaps: where LightCraft falls short of Lightroom -> **Last reviewed:** 2026-10-11 · **Last updated:** 2026-10-21 · **Change:** minor (gap 28: F8 shows / hides the right panel; import review album hierarchy, #2362; gap 20: Minolta MRW decodes; gap 10: 551 models verified; gap 15: Print Page Options — page numbers, page info, crop marks; gap 15: Print Page Options — page numbers, page info, crop marks; gap 28: `\` shows / hides the Filter Bar in the grids; gap 21: Indonesian interface shipped, every message but What's New; gap 27: the Windows x64 installer verified on Windows 11; the Settings → Apps uninstall confirmation remains) +> **Last reviewed:** 2026-10-11 · **Last updated:** 2026-10-22 · **Change:** minor (gap 13: Lightroom catalog backup zips import directly, #2592) Every known shortfall, one entry each, ranked by user impact (how many Lightroom users hit it, and how hard it blocks them), then by effort. **Agents pick work from the top** (the first entry nobody is @@ -27,7 +27,7 @@ Hours are Opus 5.5 agent wall-clock hours (see [target-app-parity.md](target-app | 10 B | **Per-model verification thin, plus cheap per-body decode fixes**: 551 of 1,446 Adobe models verified (38%), ~85% of photographers on a verified body; Nikon 5 bodies verified, Canon 34, Leica 14 (the Q3 by a recorded check of 38 private DNGs). Cheap fixes: Z 6II packed 14-bit NEF (#769, PR #671), Sony A100 regression (#702); F828 SRF now discovered and imported as preview-only, sensor decoding still missing, DSLR-A200…A390 / A850 12-bit ARW (#535) | measured: [raw/coverage.tsv](raw/coverage.tsv) | per-body bugs (levels, crops, WB) surface only from users | 25–45 | [raw-parity](raw-parity.md#by-maker-measured) | | 11 | **Enhance: Raw Details, Super Resolution; AI denoise without shippable weights or X-Trans** | ENH rows 0%; denoise Bayer only, RawNIND weights GPL and opt-in | Fujifilm users get no AI denoise; nobody gets it out of the box | 30–60 + model decision | [raw-parity](raw-parity.md#demosaic-levels-crops-and-lens-metadata) | | 12 | **Content-aware / generative remove, detect distractions, reflections removal** | REM rows 75%; patch synthesis missing | cleanup work goes back to Photoshop | 25–45 (+ generative model) | [target-app-parity](target-app-parity.md#ai) | -| 13 | **Lightroom catalog migration is one-way and approximate**: unsupported Adobe profiles / AI settings archived, history kept as data; main-file pages read on demand up to 16 GiB, with dense-table and WAL bounds retained (#798); offline record counts and destination are now shown in a lasting import result | lightroom-catalog-import.md | switching users lose some looks | 8–12 | [file-format-parity](file-format-parity.md) | +| 13 | **Lightroom catalog migration is one-way and approximate**: unsupported Adobe profiles / AI settings archived, history kept as data; main-file pages read on demand up to 16 GiB, with dense-table and WAL bounds retained (#798); offline record counts and destination are now shown in a lasting import result; Lightroom's backup zips import directly (#2592) | lightroom-catalog-import.md | switching users lose some looks | 8–12 | [file-format-parity](file-format-parity.md) | | 14 | **HDR display** (EDR / Windows HDR); the headroom limit and SDR rendition from `crs` XMP (the HDR switch, `HDREditMode`, is read). HLG AVIF export done; the display's SDR/HDR decision is tested, blocked on eframe's 8-bit surface | HDR rows 70%; LR-VIEW-HDR-DISPLAY ⬜ | HDR editing works blind on HDR monitors | 12–25 | [hardware-parity](hardware-parity.md) | | 15 | **Print module**: identity plate / watermark / photo info overlays, Cut Guides for the packages, Custom Package Auto Layout, printer-driver Page Setup and direct printing, JPEG file output (the Print dialog writes a colour-managed PDF) | LRC-PRINT rows | Classic users who print | 15–25 | [target-app-parity](target-app-parity.md#features) | | 16 | **Map view, Book, Slideshow module, publish services** | LRC-MAP / BOOK / SS rows ⬜ | Classic users; publish services matter for Flickr / SmugMug workflows | 60–100 | [target-app-parity](target-app-parity.md#features) | @@ -79,6 +79,7 @@ Beta-blocking gaps (B) total **~230–430 h**; with the stability and platform w | Date | Change | Summary | |---|---|---| +| 2026-10-22 | minor | Gap 13: Lightroom catalog backup zips import directly, and a backup of an imported catalog adds no collection twice (#2592); the gap stays open | | 2026-10-21 | minor | Gap 28 shrinks: F8 shows / hides the right panel (`view.rightPanel`); rating `[` `]`, flag cycling and F1 help were already bound, so the entry lists only filter-bar keys, Tab / ⇧Tab and F5; 2–4 h (was 4–8) | | 2026-10-20 | minor | #2362: import destination albums use the shared hierarchy/search picker instead of a flat name list; existing import workflows retained; ranked gaps/readiness counts unchanged | | 2026-10-19 | minor | Gap 21: Indonesian (`id`) shipped with every catalog message but What's New; four new languages left (Hindi, Arabic, Korean, Vietnamese), 135–239 h | diff --git a/docs/lightroom-catalog-import.md b/docs/lightroom-catalog-import.md index 441becd15e..71b7761072 100644 --- a/docs/lightroom-catalog-import.md +++ b/docs/lightroom-catalog-import.md @@ -8,6 +8,9 @@ File → Open Library… given a Lightroom catalog folder (or an `.lrdata` / `.l doesn't make a LightCraft library there: it offers this import instead, with the catalog found (`crates/engine/src/lightroom_folder.rs`, issue #891). `lightcraft-cli --library` refuses those places too. +A Lightroom Classic catalog backup is imported the same way, as the `.zip` Lightroom writes +(`Backups/YYYY-MM-DD HHMM/.zip`; see *Catalog backups* below). + Equivalent commands: ```text @@ -117,6 +120,33 @@ photo and its virtual copies out of the library. To import originals on a share, from that share. A path that is not a path on this computer (a Windows drive path read on macOS or Linux) is not looked at either and stays catalogued as missing. +## Catalog backups + +Lightroom Classic backs a catalog up as `/Backups/YYYY-MM-DD HHMM/.zip`: one `.lrcat` (plus, +since Classic 12, a small `.lrcat-data/` folder the import does not need). Someone who has removed Lightroom +and its catalog may have nothing else, so `library.importLightroom` / `library.inspectLightroom` and File ▸ Import +Lightroom Catalog… take the zip directly (issue #2592, `crates/engine/src/lightroom_backup.rs`): + +- The `.lrcat` entry is inflated by streaming (stored or deflated, ZIP64 included) into a private folder, + `Interop/lc-lrbackup--