From db0baa1c2d5cd0af9eadc490ba4074020923dbb2 Mon Sep 17 00:00:00 2001 From: chasse20 Date: Thu, 12 Nov 2026 19:43:34 -0500 Subject: [PATCH] Persist library previews in a bounded platform cache --- Cargo.lock | 12 ++ Cargo.toml | 3 + crates/engine/Cargo.toml | 1 + crates/engine/src/lib.rs | 1 + crates/engine/src/preview_fingerprint.rs | 55 +++++ crates/engine/src/preview_fingerprint.test.rs | 51 +++++ crates/ui-egui/Cargo.toml | 3 + crates/ui-egui/src/graph_previews.rs | 15 +- crates/ui-egui/src/material_craft_app.rs | 1 + crates/ui-egui/src/preview_disk_cache.rs | 192 ++++++++++++++++++ crates/ui-egui/src/preview_disk_cache.test.rs | 192 ++++++++++++++++++ crates/ui-egui/src/thumbs.rs | 108 ++++++---- docs/architecture.md | 20 ++ 13 files changed, 613 insertions(+), 41 deletions(-) create mode 100644 crates/engine/src/preview_fingerprint.rs create mode 100644 crates/engine/src/preview_fingerprint.test.rs create mode 100644 crates/ui-egui/src/preview_disk_cache.rs create mode 100644 crates/ui-egui/src/preview_disk_cache.test.rs diff --git a/Cargo.lock b/Cargo.lock index bb61929..20f8491 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -807,6 +807,14 @@ dependencies = [ "libc", ] +[[package]] +name = "craft-atomic" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=1b34eba41e62138fc2438d2734523b302840b45d#1b34eba41e62138fc2438d2734523b302840b45d" +dependencies = [ + "rustix 1.1.6", +] + [[package]] name = "craft-dialogs" version = "0.1.0" @@ -2671,6 +2679,7 @@ dependencies = [ "materialcraft-nodes", "serde", "serde_json", + "sha2", "thiserror 2.0.21", ] @@ -2751,6 +2760,7 @@ name = "materialcraft-ui-egui" version = "0.1.3" dependencies = [ "base64 0.22.1", + "craft-atomic", "craft-dialogs", "craft-keymap", "craft-quit", @@ -2769,7 +2779,9 @@ dependencies = [ "rfd", "serde", "serde_json", + "sha2", "sys-locale", + "tempfile", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", diff --git a/Cargo.toml b/Cargo.toml index ed1a380..2a4e130 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,6 +59,7 @@ materialcraft-ui-egui = { path = "crates/ui-egui" } thiserror = "2" serde = { version = "1", features = ["derive"] } serde_json = "1" +sha2 = "0.10" # MIT OR Apache-2.0; stable content keys for persistent previews. glam = { version = "0.30", features = ["serde", "bytemuck"] } bytemuck = { version = "1", features = ["derive"] } rayon = "1" @@ -98,6 +99,8 @@ craft-menu = { git = "https://github.com/storytold/craft-libs", rev = "1b34eba41 craft-window = { git = "https://github.com/storytold/craft-libs", rev = "1b34eba41e62138fc2438d2734523b302840b45d", features = ["egui"] } # Release checks and self-updates for the desktop app: storytold/craft-libs, MIT OR Apache-2.0. craft-update = { git = "https://github.com/storytold/craft-libs", rev = "1b34eba41e62138fc2438d2734523b302840b45d", features = ["egui"] } +# Atomic persistence, at the suite's existing revision. +craft-atomic = { git = "https://github.com/storytold/craft-libs", rev = "1b34eba41e62138fc2438d2734523b302840b45d" } # egui 0.37 and winit: the suite's forks (storytold/egui, storytold/winit), as craft-libs pins them. [patch.crates-io] diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 5eb7c48..dfada0c 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -17,4 +17,5 @@ materialcraft-formats = { workspace = true } materialcraft-geom = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +sha2 = { workspace = true } thiserror = { workspace = true } diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 454c1ca..8f0e1af 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -37,6 +37,7 @@ mod open_document; mod package_commands; mod package_file_commands; mod preview_cook; +mod preview_fingerprint; mod resource_commands; pub use material_preview::MaterialPreview; pub use material_source::MaterialSource; diff --git a/crates/engine/src/preview_fingerprint.rs b/crates/engine/src/preview_fingerprint.rs new file mode 100644 index 0000000..fa758bf --- /dev/null +++ b/crates/engine/src/preview_fingerprint.rs @@ -0,0 +1,55 @@ +//! Stable keys for disposable library thumbnails, independent of process addresses and timestamps. + +use std::collections::BTreeSet; +use std::sync::atomic::{AtomicBool, Ordering}; + +use sha2::{Digest, Sha256}; + +use crate::NodeLibrary; + +impl NodeLibrary { + /// Hash the source packages and resource bytes used by a library preview. The caller must + /// also key its rendering implementation and display size. Missing dependencies or an + /// excessive dependency closure disable persistence; they must not leave a stale disk hit. + pub fn preview_fingerprint(&self, package: &str, graph: &str, cancel: &AtomicBool) -> Option<[u8; 32]> { + self.packages.get(package)?.graph_by_name(graph)?; + let mut hash = Sha256::new(); + serde_json::to_writer(&mut hash, &("materialcraft-library-preview-v1", package, graph)).ok()?; + let mut pending = BTreeSet::from([package]); + let mut visited = BTreeSet::new(); + while let Some(path) = pending.pop_first() { + if cancel.load(Ordering::Relaxed) || visited.len() >= 4096 { + return None; + } + if !visited.insert(path) { + continue; + } + let source = self.packages.get(path)?; + // Package serialization excludes dependency pointers and resource bytes. Stream the + // model, then hash every byte explicitly, without allocating a second package copy. + serde_json::to_writer(&mut hash, &(path, source.as_ref())).ok()?; + for resource in &source.resources { + hash.update((resource.bytes.len() as u64).to_le_bytes()); + for chunk in resource.bytes.as_slice().chunks(64 * 1024) { + if cancel.load(Ordering::Relaxed) { + return None; + } + hash.update(chunk); + } + } + for instance in source.graphs.iter().flat_map(|graph| graph.instances()) { + if !instance.package.is_empty() && !visited.contains(instance.package.as_str()) { + pending.insert(&instance.package); + if pending.len().saturating_add(visited.len()) > 4096 { + return None; + } + } + } + } + Some(hash.finalize().into()) + } +} + +#[cfg(test)] +#[path = "preview_fingerprint.test.rs"] +mod tests; diff --git a/crates/engine/src/preview_fingerprint.test.rs b/crates/engine/src/preview_fingerprint.test.rs new file mode 100644 index 0000000..1b2cd4a --- /dev/null +++ b/crates/engine/src/preview_fingerprint.test.rs @@ -0,0 +1,51 @@ +use super::*; +use materialcraft_core::{Graph, GraphInstance, Package, Resource, ResourceBytes}; +use std::sync::Arc; + +fn library() -> NodeLibrary { + let mut library = NodeLibrary::default(); + let mut root = Package::new("root"); + let mut graph = Graph::new("main"); + let id = graph.add_node("graph_instance", [0.0, 0.0]); + graph.node_mut(id).unwrap().instance = Some(GraphInstance { + label: String::new(), + package: "child".into(), + graph: "sub".into(), + inputs: vec![], + outputs: vec![], + defaults: Default::default(), + }); + root.graphs.push(graph); + let mut child = Package::new("child"); + child.graphs.push(Graph::new("sub")); + child.resources.push(Resource::new("bitmap", "bitmap.png", ResourceBytes::new(vec![1, 2, 3]))); + library.packages.insert("root".into(), Arc::new(root)); + library.packages.insert("child".into(), Arc::new(child)); + library +} + +fn key(library: &NodeLibrary) -> Option<[u8; 32]> { + library.preview_fingerprint("root", "main", &AtomicBool::new(false)) +} + +#[test] +fn keys_survive_reloading_and_track_nested_content_not_unrelated_packages() { + let original = library(); + assert_eq!(key(&original), key(&library())); + let mut changed = library(); + changed.packages.insert("unrelated".into(), Arc::new(Package::new("unused"))); + assert_eq!(key(&original), key(&changed)); + Arc::make_mut(changed.packages.get_mut("child").unwrap()).graphs[0].seed += 1; + assert_ne!(key(&original), key(&changed)); + let mut changed = library(); + Arc::make_mut(changed.packages.get_mut("child").unwrap()).resources[0].bytes = ResourceBytes::new(vec![1, 2, 4]); + assert_ne!(key(&original), key(&changed), "same-size replacement must invalidate the preview"); +} + +#[test] +fn incomplete_and_cancelled_keys_are_not_persisted() { + let mut library = library(); + assert!(library.preview_fingerprint("root", "main", &AtomicBool::new(true)).is_none()); + library.packages.remove("child"); + assert!(key(&library).is_none()); +} diff --git a/crates/ui-egui/Cargo.toml b/crates/ui-egui/Cargo.toml index 71ab2a3..b3afdb2 100644 --- a/crates/ui-egui/Cargo.toml +++ b/crates/ui-egui/Cargo.toml @@ -24,6 +24,8 @@ image = { workspace = true } zip = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] +craft-atomic = { workspace = true } +sha2 = { workspace = true } sys-locale = "0.3" craft-update = { workspace = true } craft-dialogs = { workspace = true } @@ -42,6 +44,7 @@ web-sys = { version = "0.3", features = ["Window", "Document", "HtmlElement", "H # winit needs a window system on Linux, so the crate's own tests build with X11. [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +tempfile = "3" eframe = { workspace = true, features = ["x11"] } # Offscreen renders of the real UI (tests/snapshot.rs). egui_kittest = { workspace = true, features = ["wgpu", "snapshot", "eframe"] } diff --git a/crates/ui-egui/src/graph_previews.rs b/crates/ui-egui/src/graph_previews.rs index 85215a7..a5c403b 100644 --- a/crates/ui-egui/src/graph_previews.rs +++ b/crates/ui-egui/src/graph_previews.rs @@ -15,6 +15,8 @@ const MAX_VISIBLE_REQUESTS: usize = 64; /// Only one cook may run; every frame replaces the pending work with its visible tiles. #[derive(Default)] pub(super) struct GraphPreviews { + #[cfg(not(target_arch = "wasm32"))] + pub persist: bool, library: Option>, textures: HashMap, u64)>, wanted: Vec, @@ -110,9 +112,18 @@ impl GraphPreviews { let (sender, result) = std::sync::mpsc::sync_channel(1); let cancel = Arc::new(AtomicBool::new(false)); let (thread_cancel, thread_library, thread_key, context) = (cancel.clone(), library.clone(), key.clone(), ctx.clone()); + let persist = self.persist; let spawned = std::thread::Builder::new().name("materialcraft-library-graphs".into()).spawn(move || { - let image = NodeLibrary::preview_cancellable(&thread_library, &thread_key.0, &thread_key.1, LIBRARY_PREVIEW_SIZE, &thread_cancel); - let rgba = image.map(|image| image.resize(LIBRARY_PREVIEW_SIZE, LIBRARY_PREVIEW_SIZE, true).to_rgba8_display()); + let render = || { + NodeLibrary::preview_cancellable(&thread_library, &thread_key.0, &thread_key.1, LIBRARY_PREVIEW_SIZE, &thread_cancel) + .map(|image| image.resize(LIBRARY_PREVIEW_SIZE, LIBRARY_PREVIEW_SIZE, true).to_rgba8_display()) + }; + let disk = persist.then(super::preview_disk_cache::PreviewDiskCache::configured).flatten(); + let fingerprint = disk.as_ref().and_then(|_| thread_library.preview_fingerprint(&thread_key.0, &thread_key.1, &thread_cancel)); + let rgba = match (disk, fingerprint) { + (Some(disk), Some(key)) => disk.get_or_render(key, LIBRARY_PREVIEW_SIZE, &thread_cancel, render), + _ => render(), + }; let _ = sender.send(rgba); context.request_repaint(); }); diff --git a/crates/ui-egui/src/material_craft_app.rs b/crates/ui-egui/src/material_craft_app.rs index ae617c2..38cdbbd 100644 --- a/crates/ui-egui/src/material_craft_app.rs +++ b/crates/ui-egui/src/material_craft_app.rs @@ -77,6 +77,7 @@ impl MaterialCraftApp { #[cfg(not(target_arch = "wasm32"))] { app.dialogs = crate::native_dialogs::attached(&cc.egui_ctx); + app.thumbs.enable_persistence(); } app } diff --git a/crates/ui-egui/src/preview_disk_cache.rs b/crates/ui-egui/src/preview_disk_cache.rs new file mode 100644 index 0000000..1325d38 --- /dev/null +++ b/crates/ui-egui/src/preview_disk_cache.rs @@ -0,0 +1,192 @@ +//! Disposable PNG thumbnails. All filesystem work belongs to the existing preview workers. + +use std::fs::{self, File}; +use std::io::{self, Cursor, Read}; +use std::path::{Path, PathBuf}; +use std::sync::OnceLock; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, SystemTime}; + +use image::{ImageDecoder, ImageEncoder}; +use sha2::{Digest, Sha256}; + +const MAX_ENTRIES: usize = 4096; +const MAX_FILE_BYTES: u64 = 128 * 1024; +const MAX_SIDE: u32 = 128; + +pub(super) struct PreviewDiskCache { + root: PathBuf, + renderer: [u8; 32], + budget: u64, + entries: usize, +} + +impl PreviewDiskCache { + pub fn configured() -> Option { + // Scripted runs must not touch the user's cache. An explicit override lets tests and + // agents exercise persistence in an isolated folder; an empty override disables it. + let root = match std::env::var_os("MATERIALCRAFT_CACHE_DIR") { + Some(path) if path.is_empty() => return None, + Some(path) => PathBuf::from(path), + None if cfg!(test) || std::env::var_os("MATERIALCRAFT_NO_PREFS").is_some() => return None, + // The existing shared path resolver follows the suite's cache-folder policy, + // including redirected Windows profiles and absolute XDG paths. + None => craft_update::Paths::per_user("materialcraft").ok()?.cache, + }; + Self::open(root.join("previews"), 64 * 1024 * 1024) + } + + /// Called on a worker. The executable digest also invalidates development builds that keep + /// the same version number. It is streamed once per process, never on the event-loop thread. + pub fn open(root: PathBuf, budget: u64) -> Option { + static RENDERER: OnceLock> = OnceLock::new(); + let renderer = *RENDERER.get_or_init(|| { + let mut file = File::open(std::env::current_exe().ok()?).ok()?.take((1 << 30) + 1); + let mut hash = Sha256::new(); + let bytes = io::copy(&mut file, &mut hash).ok()?; + (bytes <= 1 << 30).then(|| hash.finalize().into()) + }); + Some(Self { root, renderer: renderer?, budget, entries: MAX_ENTRIES }) + } + + pub fn get_or_render(&self, key: [u8; 32], side: u32, cancel: &AtomicBool, render: impl FnOnce() -> Option>) -> Option> { + if cancel.load(Ordering::Relaxed) { + return None; + } + let path = self.path(key, side); + if let Some(pixels) = self.read(&path, side) { + return (!cancel.load(Ordering::Relaxed)).then_some(pixels); + } + let pixels = render()?; + if cancel.load(Ordering::Relaxed) { + return None; + } + // A cache is optional: unavailable storage must not hide a successfully rendered tile. + if let Err(error) = self.write(&path, side, &pixels) { + log::debug!("Library preview persistence skipped: {error}"); + } + Some(pixels) + } + + fn path(&self, key: [u8; 32], side: u32) -> PathBuf { + let mut hash = Sha256::new(); + hash.update(b"materialcraft-preview-png-v1"); + hash.update(self.renderer); + hash.update(key); + hash.update(side.to_le_bytes()); + self.root.join(format!("{:x}.png", hash.finalize())) + } + + fn read(&self, path: &Path, side: u32) -> Option> { + if side == 0 || side > MAX_SIDE || self.budget == 0 { + return None; + } + let metadata = fs::symlink_metadata(path).ok()?; + if !metadata.is_file() || metadata.len() > MAX_FILE_BYTES { + return None; + } + let mut file = File::open(path).ok()?; + let mut bytes = Vec::new(); + Read::by_ref(&mut file).take(MAX_FILE_BYTES + 1).read_to_end(&mut bytes).ok()?; + if bytes.len() as u64 > MAX_FILE_BYTES { + return None; + } + let mut limits = image::Limits::default(); + limits.max_image_width = Some(MAX_SIDE); + limits.max_image_height = Some(MAX_SIDE); + limits.max_alloc = Some(1024 * 1024); + let decoder = image::codecs::png::PngDecoder::with_limits(Cursor::new(bytes), limits).ok()?; + if decoder.dimensions() != (side, side) || decoder.color_type() != image::ColorType::Rgba8 { + return None; + } + let mut pixels = vec![0; (side * side * 4) as usize]; + decoder.read_image(&mut pixels).ok()?; + // Approximate LRU across launches, without a metadata write on every tile lookup. + if metadata.modified().ok()?.elapsed().is_ok_and(|age| age >= Duration::from_secs(3600)) { + let _ = file.set_modified(SystemTime::now()); + } + Some(pixels) + } + + fn write(&self, path: &Path, side: u32, pixels: &[u8]) -> io::Result<()> { + if side == 0 || side > MAX_SIDE || pixels.len() != (side * side * 4) as usize || self.budget == 0 { + return Ok(()); + } + let mut bytes = Vec::new(); + image::codecs::png::PngEncoder::new(&mut bytes).write_image(pixels, side, side, image::ExtendedColorType::Rgba8).map_err(io::Error::other)?; + if bytes.len() as u64 > MAX_FILE_BYTES || bytes.len() as u64 > self.budget { + return Ok(()); + } + fs::create_dir_all(&self.root)?; + // Coordinate the budget across windows/processes. A busy writer is a harmless cache + // miss; do not block a preview worker waiting for another process to finish. + let lock = File::options().read(true).write(true).create(true).truncate(false).open(self.root.join("cache.lock"))?; + lock.try_lock().map_err(io::Error::other)?; + // craft-atomic deliberately follows document symlinks. Cache entries are disposable + // files owned by this cache, so never follow one into an unrelated document instead. + if fs::symlink_metadata(path).is_ok_and(|metadata| !metadata.is_file()) { + return Err(io::Error::other("preview cache entry is not a regular file")); + } + self.reserve(path, bytes.len() as u64)?; + craft_atomic::WriteOptions::default().with_new_file_permissions(craft_atomic::NewFilePermissions::Private).atomic_write(path, &bytes) + } + + fn reserve(&self, target: &Path, incoming: u64) -> io::Result<()> { + let mut entries = Vec::new(); + let mut total = 0u64; + // Bound both scan time and metadata allocations in an externally populated directory. + for (index, entry) in fs::read_dir(&self.root)?.enumerate() { + if index >= MAX_ENTRIES * 2 { + return Err(io::Error::other("preview cache directory exceeds its entry limit")); + } + let entry = entry?; + if !entry.file_type()?.is_file() { + continue; + } + let name = entry.file_name(); + let name = name.to_string_lossy(); + // The cache lock excludes every live writer. Matching staging files can only be + // leftovers from an interrupted atomic save, so reclaim them before growing again. + if staging_name(&name) { + fs::remove_file(entry.path())?; + continue; + } + if !cache_name(&name) || entry.path() == target { + continue; + } + let metadata = entry.metadata()?; + total = total.saturating_add(metadata.len()); + entries.push((metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH), entry.path(), metadata.len())); + } + entries.sort_unstable_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1))); + let mut count = entries.len(); + for (_, path, size) in entries { + if total.saturating_add(incoming) <= self.budget && count < self.entries { + break; + } + fs::remove_file(path)?; + total = total.saturating_sub(size); + count = count.saturating_sub(1); + } + Ok(()) + } +} + +fn cache_name(name: &str) -> bool { + name.strip_suffix(".png").is_some_and(digest_name) +} + +fn digest_name(stem: &str) -> bool { + stem.len() == 64 && stem.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +fn staging_name(name: &str) -> bool { + let Some((stem, suffix)) = name.strip_prefix('.').and_then(|name| name.strip_suffix(".craft_tmp")).and_then(|name| name.split_once('.')) else { + return false; + }; + digest_name(stem) && suffix.split_once('-').is_some_and(|(pid, counter)| pid.parse::().is_ok_and(|pid| pid > 0) && counter.parse::().is_ok()) +} + +#[cfg(test)] +#[path = "preview_disk_cache.test.rs"] +mod tests; diff --git a/crates/ui-egui/src/preview_disk_cache.test.rs b/crates/ui-egui/src/preview_disk_cache.test.rs new file mode 100644 index 0000000..6fed6f7 --- /dev/null +++ b/crates/ui-egui/src/preview_disk_cache.test.rs @@ -0,0 +1,192 @@ +use super::*; + +const SIDE: u32 = 8; +// A fork can briefly inherit another test's file lock before exec closes it. Keep this +// module's subprocess launches apart from its nonblocking-writer tests. +static TESTS: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +fn cache(root: &Path) -> PreviewDiskCache { + PreviewDiskCache { root: root.to_owned(), renderer: [1; 32], budget: 64 * 1024 * 1024, entries: MAX_ENTRIES } +} + +fn pixels() -> Vec { + (0..SIDE * SIDE * 4).map(|i| i as u8).collect() +} + +#[test] +fn previews_survive_process_restarts_without_rendering() { + let _serial = TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + const CHILD: &str = "MATERIALCRAFT_TEST_DISK_PREVIEW"; + if let Some(root) = std::env::var_os(CHILD) { + let cache = cache(Path::new(&root)); + let actual = cache.get_or_render([2; 32], SIDE, &AtomicBool::new(false), || { + assert_eq!(std::env::var("MATERIALCRAFT_TEST_DISK_PHASE").unwrap(), "cold", "warm launch must not cook"); + Some(pixels()) + }); + assert_eq!(actual, Some(pixels())); + return; + } + let root = tempfile::tempdir().unwrap(); + for phase in ["cold", "warm"] { + let start = std::time::Instant::now(); + let status = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "thumbs::preview_disk_cache::tests::previews_survive_process_restarts_without_rendering", "--nocapture"]) + .env(CHILD, root.path()) + .env("MATERIALCRAFT_TEST_DISK_PHASE", phase) + .status() + .unwrap(); + assert!(status.success()); + eprintln!("{phase} preview subprocess: {:?}", start.elapsed()); + } +} + +#[test] +#[ignore = "renders bundled library graphs and measures cold/warm subprocesses"] +fn real_library_previews_survive_relaunch() { + let _serial = TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + const CHILD: &str = "MATERIALCRAFT_TEST_REAL_PREVIEWS"; + if let Some(root) = std::env::var_os(CHILD) { + let library = materialcraft_engine::NodeLibrary::shared(); + let phase = std::env::var("MATERIALCRAFT_TEST_DISK_PHASE").unwrap(); + let cache = PreviewDiskCache::open(PathBuf::from(root), 64 * 1024 * 1024).unwrap(); + let cancel = AtomicBool::new(false); + for (package, graph) in + [("sbs://blur_hq.sbs", "blur_hq"), ("sbs://noise_clouds_3_v2.sbs", "clouds_3"), ("sbs://materials/pbr/bricks_001.sbs", "bricks_001")] + { + let started = std::time::Instant::now(); + let key = library.preview_fingerprint(package, graph, &cancel).unwrap(); + let image = cache.get_or_render(key, 72, &cancel, || { + assert_eq!(phase, "cold", "warm launch must load {graph} without cooking"); + materialcraft_engine::NodeLibrary::preview_cancellable(&library, package, graph, 72, &cancel) + .map(|image| image.resize(72, 72, true).to_rgba8_display()) + }); + assert_eq!(image.unwrap().len(), 72 * 72 * 4); + eprintln!("{phase} {graph}: {:?}", started.elapsed()); + } + return; + } + let root = tempfile::tempdir().unwrap(); + for phase in ["cold", "warm"] { + let status = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "thumbs::preview_disk_cache::tests::real_library_previews_survive_relaunch", "--ignored", "--nocapture"]) + .env(CHILD, root.path()) + .env("MATERIALCRAFT_TEST_DISK_PHASE", phase) + .status() + .unwrap(); + assert!(status.success()); + } +} + +#[test] +fn keys_separate_sources_sizes_and_renderer_builds() { + let _serial = TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let root = tempfile::tempdir().unwrap(); + let mut cache = cache(root.path()); + let original = cache.path([1; 32], SIDE); + assert_ne!(original, cache.path([2; 32], SIDE)); + assert_ne!(original, cache.path([1; 32], SIDE + 1)); + cache.renderer = [2; 32]; + assert_ne!(original, cache.path([1; 32], SIDE)); +} + +#[test] +fn cancelled_failed_and_invalid_previews_never_persist() { + let _serial = TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let root = tempfile::tempdir().unwrap(); + let cache = cache(root.path()); + let cancel = AtomicBool::new(true); + assert!(cache.get_or_render([1; 32], SIDE, &cancel, || panic!("cancelled cook ran")).is_none()); + cancel.store(false, Ordering::Relaxed); + assert!(cache.get_or_render([1; 32], SIDE, &cancel, || None).is_none()); + assert!( + cache + .get_or_render([1; 32], SIDE, &cancel, || { + cancel.store(true, Ordering::Relaxed); + Some(pixels()) + }) + .is_none() + ); + cache.write(&cache.path([2; 32], SIDE), SIDE, &[1]).unwrap(); + assert_eq!(fs::read_dir(root.path()).unwrap().count(), 0); +} + +#[test] +fn corrupt_oversized_and_wrong_size_files_are_misses_and_repaired() { + let _serial = TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let root = tempfile::tempdir().unwrap(); + let cache = cache(root.path()); + let path = cache.path([1; 32], SIDE); + for bytes in [b"broken PNG".to_vec(), vec![0; MAX_FILE_BYTES as usize + 1]] { + fs::write(&path, bytes).unwrap(); + assert!(cache.read(&path, SIDE).is_none()); + assert_eq!(cache.get_or_render([1; 32], SIDE, &AtomicBool::new(false), || Some(pixels())), Some(pixels())); + assert_eq!(cache.read(&path, SIDE), Some(pixels())); + assert!(cache.read(&path, SIDE + 1).is_none()); + } + let bytes = fs::read(&path).unwrap(); + fs::write(&path, &bytes[..bytes.len() / 2]).unwrap(); + assert!(cache.read(&path, SIDE).is_none()); +} + +#[test] +fn cache_budget_evicts_old_files_and_preserves_unrelated_files() { + let _serial = TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let root = tempfile::tempdir().unwrap(); + let mut cache = cache(root.path()); + let old = cache.path([1; 32], SIDE); + cache.write(&old, SIDE, &pixels()).unwrap(); + let file_bytes = fs::metadata(&old).unwrap().len(); + cache.budget = file_bytes * 2; + File::options().write(true).open(&old).unwrap().set_modified(SystemTime::UNIX_EPOCH).unwrap(); + fs::write(root.path().join("my-image.png"), b"unrelated").unwrap(); + let leftover = root.path().join(craft_atomic::temp_name(old.file_name().unwrap().to_str().unwrap())); + fs::write(&leftover, b"interrupted write").unwrap(); + fs::write(root.path().join("unrelated.craft_tmp"), b"keep").unwrap(); + cache.write(&cache.path([2; 32], SIDE), SIDE, &pixels()).unwrap(); + assert!(!leftover.exists()); + assert!(root.path().join("unrelated.craft_tmp").exists()); + cache.write(&cache.path([3; 32], SIDE), SIDE, &pixels()).unwrap(); + assert!(!old.exists()); + assert_eq!(fs::read(root.path().join("my-image.png")).unwrap(), b"unrelated"); + let total: u64 = + fs::read_dir(root.path()).unwrap().flatten().filter(|f| cache_name(&f.file_name().to_string_lossy())).map(|f| f.metadata().unwrap().len()).sum(); + assert!(total <= cache.budget); + cache.budget = 64 * 1024 * 1024; + cache.entries = 1; + cache.write(&cache.path([4; 32], SIDE), SIDE, &pixels()).unwrap(); + let count = fs::read_dir(root.path()).unwrap().flatten().filter(|file| cache_name(&file.file_name().to_string_lossy())).count(); + assert_eq!(count, 1, "the entry limit also applies when there is space left"); + cache.budget = file_bytes - 1; + let oversized = cache.path([5; 32], SIDE); + cache.write(&oversized, SIDE, &pixels()).unwrap(); + assert!(!oversized.exists()); +} + +#[test] +fn unavailable_or_busy_cache_still_returns_rendered_pixels() { + let _serial = TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let root = tempfile::tempdir().unwrap(); + let blocked = root.path().join("file"); + fs::write(&blocked, b"not a directory").unwrap(); + assert_eq!(cache(&blocked).get_or_render([1; 32], SIDE, &AtomicBool::new(false), || Some(pixels())), Some(pixels())); + let lock = File::options().read(true).write(true).create_new(true).open(root.path().join("cache.lock")).unwrap(); + lock.lock().unwrap(); + let cache = cache(root.path()); + assert_eq!(cache.get_or_render([1; 32], SIDE, &AtomicBool::new(false), || Some(pixels())), Some(pixels())); + assert!(!cache.path([1; 32], SIDE).exists()); +} + +#[cfg(unix)] +#[test] +fn cache_does_not_read_or_evict_symlink_targets() { + let _serial = TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let root = tempfile::tempdir().unwrap(); + let outside = tempfile::NamedTempFile::new().unwrap(); + fs::write(outside.path(), b"keep").unwrap(); + let cache = cache(root.path()); + let path = cache.path([1; 32], SIDE); + std::os::unix::fs::symlink(outside.path(), &path).unwrap(); + assert!(cache.read(&path, SIDE).is_none()); + let _ = cache.write(&path, SIDE, &pixels()); + assert_eq!(fs::read(outside.path()).unwrap(), b"keep"); +} diff --git a/crates/ui-egui/src/thumbs.rs b/crates/ui-egui/src/thumbs.rs index cffb1ad..767d267 100644 --- a/crates/ui-egui/src/thumbs.rs +++ b/crates/ui-egui/src/thumbs.rs @@ -19,6 +19,7 @@ pub struct Thumbs { library: HashMap, library_rx: Option>, library_started: bool, + persist_library: bool, graphs: GraphPreviews, frame: u64, } @@ -32,8 +33,18 @@ pub fn color_image(img: &Image, side: u32) -> ColorImage { #[path = "graph_previews.rs"] mod graph_previews; use graph_previews::GraphPreviews; +#[cfg(not(target_arch = "wasm32"))] +#[path = "preview_disk_cache.rs"] +mod preview_disk_cache; impl Thumbs { + /// Opt in only for a native app window; offscreen harnesses never use the user's cache. + #[cfg(not(target_arch = "wasm32"))] + pub fn enable_persistence(&mut self) { + self.persist_library = true; + self.graphs.persist = true; + } + pub fn clear_nodes(&mut self) { self.nodes.clear(); } @@ -86,7 +97,7 @@ impl Thumbs { pub fn library(&mut self, ctx: &egui::Context, def: &str) -> Option { if !self.library_started { self.library_started = true; - self.library_rx = Some(spawn_library_previews(ctx.clone())); + self.library_rx = Some(spawn_library_previews(ctx.clone(), self.persist_library)); } self.library.get(def).map(TextureHandle::id) } @@ -102,68 +113,87 @@ fn sample_inputs() -> (Image, Image, Image) { (gray, color, normal) } -fn previews() -> Previews { +fn previews(_persist: bool) -> Previews { let (gray, color, normal) = sample_inputs(); let no_res = |_: &str| None; let vars = BTreeMap::new(); let mut out = Vec::new(); + #[cfg(not(target_arch = "wasm32"))] + let disk = _persist.then(preview_disk_cache::PreviewDiskCache::configured).flatten(); for def in catalog() { if def.id == "graph_instance" { continue; } - let inputs: Vec> = def - .inputs - .iter() - .enumerate() - .map(|(i, p)| { - // Masks and secondary inputs stay empty so the main effect shows - if i > 0 && !matches!(def.id, "blend" | "height_blend" | "normal_blend" | "rgba_merge") { - return None; - } - Some(if p.id.contains("normal") || def.category.contains("Normal") && p.kind == materialcraft_engine::nodes::PortKind::Color { - normal.clone() - } else if p.kind == materialcraft_engine::nodes::PortKind::Color { - color.clone() - } else if i == 1 { - gray.map(Kind::Gray, |v| [1.0 - v[0], 0.0, 0.0, 0.0]) - } else { - gray.clone() + let render = || { + let inputs: Vec> = def + .inputs + .iter() + .enumerate() + .map(|(i, p)| { + // Masks and secondary inputs stay empty so the main effect shows + if i > 0 && !matches!(def.id, "blend" | "height_blend" | "normal_blend" | "rgba_merge") { + return None; + } + Some(if p.id.contains("normal") || def.category.contains("Normal") && p.kind == materialcraft_engine::nodes::PortKind::Color { + normal.clone() + } else if p.kind == materialcraft_engine::nodes::PortKind::Color { + color.clone() + } else if i == 1 { + gray.map(Kind::Gray, |v| [1.0 - v[0], 0.0, 0.0, 0.0]) + } else { + gray.clone() + }) }) - }) - .collect(); - let params = resolve_params(def, &BTreeMap::new()); - let ctx = CookContext { - w: LIBRARY_PREVIEW_SIZE, - h: LIBRARY_PREVIEW_SIZE, - seed: 3, - tile: true, - inputs: &inputs, - params: ¶ms, - resource: &no_res, - vars: &vars, + .collect(); + let params = resolve_params(def, &BTreeMap::new()); + let ctx = CookContext { + w: LIBRARY_PREVIEW_SIZE, + h: LIBRARY_PREVIEW_SIZE, + seed: 3, + tile: true, + inputs: &inputs, + params: ¶ms, + resource: &no_res, + vars: &vars, + }; + let imgs = (def.cook)(&ctx).ok()?; + Some(imgs.first()?.resize(LIBRARY_PREVIEW_SIZE, LIBRARY_PREVIEW_SIZE, true).to_rgba8_display()) }; - if let Ok(imgs) = (def.cook)(&ctx) - && let Some(img) = imgs.first() - { - out.push((def.id.to_string(), img.resize(LIBRARY_PREVIEW_SIZE, LIBRARY_PREVIEW_SIZE, true).to_rgba8_display())); + #[cfg(not(target_arch = "wasm32"))] + let rgba = { + use sha2::Digest; + match &disk { + Some(disk) => disk.get_or_render( + sha2::Sha256::digest(format!("materialcraft-atomic-preview-v1:{}", def.id)).into(), + LIBRARY_PREVIEW_SIZE, + &std::sync::atomic::AtomicBool::new(false), + render, + ), + None => render(), + } + }; + #[cfg(target_arch = "wasm32")] + let rgba = render(); + if let Some(rgba) = rgba { + out.push((def.id.to_string(), rgba)); } } out } #[cfg(not(target_arch = "wasm32"))] -fn spawn_library_previews(ctx: egui::Context) -> Receiver { +fn spawn_library_previews(ctx: egui::Context, persist: bool) -> Receiver { let (tx, rx) = std::sync::mpsc::channel(); let _ = std::thread::Builder::new().name("materialcraft-library".into()).spawn(move || { - let _ = tx.send(previews()); + let _ = tx.send(previews(persist)); ctx.request_repaint(); }); rx } #[cfg(target_arch = "wasm32")] -fn spawn_library_previews(_ctx: egui::Context) -> Receiver { +fn spawn_library_previews(_ctx: egui::Context, persist: bool) -> Receiver { let (tx, rx) = std::sync::mpsc::channel(); - let _ = tx.send(previews()); + let _ = tx.send(previews(persist)); rx } diff --git a/docs/architecture.md b/docs/architecture.md index a4259a8..0a5c4c1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -137,6 +137,26 @@ leaving a tile or reloading the library cancels its cook between nodes. The next that worker to finish; hidden tiles cannot accumulate queued work or workers. At most 256 finished previews retain GPU textures (about 5 MiB of pixels). +Native Library previews also persist as PNGs in the platform cache folder selected by +the existing `craft_update::Paths::per_user("materialcraft").cache`, under `previews/`: `%LOCALAPPDATA%` on +Windows, `$XDG_CACHE_HOME` or `~/.cache` on Linux/BSD, and `~/Library/Caches` on macOS. +This follows the [Craft storage policy](https://github.com/storytold/craft-libs/blob/main/docs/settings-storage.md). +The cache covers native nodes and SBS graphs, retains at most 64 MiB / 4096 PNGs across builds, +and evicts older files when admitting new previews (access times refresh at most hourly). It stores disposable +thumbnails, independent of projects, autosaves and temporary render storage. + +Preview workers do all reads, encoding, atomic writes (`craft-atomic`) and eviction; none run +on the UI thread. Keys include the preview size, an executable content digest (including local +rebuilds with an unchanged release version), and, for SBS graphs, the source packages and all +resource bytes in their dependency closure. Changes become visible after `library.reload` or +the next launch, just like the in-memory library. Corrupt, oversized or unavailable cache files +are misses; failed or cancelled renders are never persisted. Concurrent writers use a +nonblocking file lock to share the disk budget. The browser retains its existing behavior. + +`MATERIALCRAFT_CACHE_DIR` overrides the app cache folder (`previews/` is appended); an empty +value disables persistence. Scripted `MATERIALCRAFT_NO_PREFS` runs leave the user's cache +untouched unless that explicit override is provided. Headless UI harnesses keep persistence off. + The local `sbs_library_build` example converts native XML without a model round trip, checking that computational fields survive presentation edits and reference rewriting. It generates independent resource substitutes without reading source resource files. It retains supplied graph implementations;