mirror of
https://github.com/storytold/materialcraft.git
synced 2026-11-22 07:20:57 +01:00
Persist library previews in a bounded platform cache
This commit is contained in:
1 parent
cc2437c0bd
commit
db0baa1c2d
13 files changed
+613
-41
No files matched your search
Generated
+12
@@ -807,6 +807,14 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-atomic"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/storytold/craft-libs?rev=1b34eba41e62138fc2438d2734523b302840b45d#1b34eba41e62138fc2438d2734523b302840b45d"
|
||||
dependencies = [
|
||||
"rustix 1.1.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-dialogs"
|
||||
version = "0.1.0"
|
||||
@@ -2671,6 +2679,7 @@ dependencies = [
|
||||
"materialcraft-nodes",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"thiserror 2.0.21",
|
||||
]
|
||||
|
||||
@@ -2751,6 +2760,7 @@ name = "materialcraft-ui-egui"
|
||||
version = "0.1.3"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"craft-atomic",
|
||||
"craft-dialogs",
|
||||
"craft-keymap",
|
||||
"craft-quit",
|
||||
@@ -2769,7 +2779,9 @@ dependencies = [
|
||||
"rfd",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"sys-locale",
|
||||
"tempfile",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"web-sys",
|
||||
|
||||
@@ -59,6 +59,7 @@ materialcraft-ui-egui = { path = "crates/ui-egui" }
|
||||
thiserror = "2"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
sha2 = "0.10" # MIT OR Apache-2.0; stable content keys for persistent previews.
|
||||
glam = { version = "0.30", features = ["serde", "bytemuck"] }
|
||||
bytemuck = { version = "1", features = ["derive"] }
|
||||
rayon = "1"
|
||||
@@ -98,6 +99,8 @@ craft-menu = { git = "https://github.com/storytold/craft-libs", rev = "1b34eba41
|
||||
craft-window = { git = "https://github.com/storytold/craft-libs", rev = "1b34eba41e62138fc2438d2734523b302840b45d", features = ["egui"] }
|
||||
# Release checks and self-updates for the desktop app: storytold/craft-libs, MIT OR Apache-2.0.
|
||||
craft-update = { git = "https://github.com/storytold/craft-libs", rev = "1b34eba41e62138fc2438d2734523b302840b45d", features = ["egui"] }
|
||||
# Atomic persistence, at the suite's existing revision.
|
||||
craft-atomic = { git = "https://github.com/storytold/craft-libs", rev = "1b34eba41e62138fc2438d2734523b302840b45d" }
|
||||
|
||||
# egui 0.37 and winit: the suite's forks (storytold/egui, storytold/winit), as craft-libs pins them.
|
||||
[patch.crates-io]
|
||||
|
||||
@@ -17,4 +17,5 @@ materialcraft-formats = { workspace = true }
|
||||
materialcraft-geom = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
@@ -37,6 +37,7 @@ mod open_document;
|
||||
mod package_commands;
|
||||
mod package_file_commands;
|
||||
mod preview_cook;
|
||||
mod preview_fingerprint;
|
||||
mod resource_commands;
|
||||
pub use material_preview::MaterialPreview;
|
||||
pub use material_source::MaterialSource;
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
//! Stable keys for disposable library thumbnails, independent of process addresses and timestamps.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::NodeLibrary;
|
||||
|
||||
impl NodeLibrary {
|
||||
/// Hash the source packages and resource bytes used by a library preview. The caller must
|
||||
/// also key its rendering implementation and display size. Missing dependencies or an
|
||||
/// excessive dependency closure disable persistence; they must not leave a stale disk hit.
|
||||
pub fn preview_fingerprint(&self, package: &str, graph: &str, cancel: &AtomicBool) -> Option<[u8; 32]> {
|
||||
self.packages.get(package)?.graph_by_name(graph)?;
|
||||
let mut hash = Sha256::new();
|
||||
serde_json::to_writer(&mut hash, &("materialcraft-library-preview-v1", package, graph)).ok()?;
|
||||
let mut pending = BTreeSet::from([package]);
|
||||
let mut visited = BTreeSet::new();
|
||||
while let Some(path) = pending.pop_first() {
|
||||
if cancel.load(Ordering::Relaxed) || visited.len() >= 4096 {
|
||||
return None;
|
||||
}
|
||||
if !visited.insert(path) {
|
||||
continue;
|
||||
}
|
||||
let source = self.packages.get(path)?;
|
||||
// Package serialization excludes dependency pointers and resource bytes. Stream the
|
||||
// model, then hash every byte explicitly, without allocating a second package copy.
|
||||
serde_json::to_writer(&mut hash, &(path, source.as_ref())).ok()?;
|
||||
for resource in &source.resources {
|
||||
hash.update((resource.bytes.len() as u64).to_le_bytes());
|
||||
for chunk in resource.bytes.as_slice().chunks(64 * 1024) {
|
||||
if cancel.load(Ordering::Relaxed) {
|
||||
return None;
|
||||
}
|
||||
hash.update(chunk);
|
||||
}
|
||||
}
|
||||
for instance in source.graphs.iter().flat_map(|graph| graph.instances()) {
|
||||
if !instance.package.is_empty() && !visited.contains(instance.package.as_str()) {
|
||||
pending.insert(&instance.package);
|
||||
if pending.len().saturating_add(visited.len()) > 4096 {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(hash.finalize().into())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "preview_fingerprint.test.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,51 @@
|
||||
use super::*;
|
||||
use materialcraft_core::{Graph, GraphInstance, Package, Resource, ResourceBytes};
|
||||
use std::sync::Arc;
|
||||
|
||||
fn library() -> NodeLibrary {
|
||||
let mut library = NodeLibrary::default();
|
||||
let mut root = Package::new("root");
|
||||
let mut graph = Graph::new("main");
|
||||
let id = graph.add_node("graph_instance", [0.0, 0.0]);
|
||||
graph.node_mut(id).unwrap().instance = Some(GraphInstance {
|
||||
label: String::new(),
|
||||
package: "child".into(),
|
||||
graph: "sub".into(),
|
||||
inputs: vec![],
|
||||
outputs: vec![],
|
||||
defaults: Default::default(),
|
||||
});
|
||||
root.graphs.push(graph);
|
||||
let mut child = Package::new("child");
|
||||
child.graphs.push(Graph::new("sub"));
|
||||
child.resources.push(Resource::new("bitmap", "bitmap.png", ResourceBytes::new(vec![1, 2, 3])));
|
||||
library.packages.insert("root".into(), Arc::new(root));
|
||||
library.packages.insert("child".into(), Arc::new(child));
|
||||
library
|
||||
}
|
||||
|
||||
fn key(library: &NodeLibrary) -> Option<[u8; 32]> {
|
||||
library.preview_fingerprint("root", "main", &AtomicBool::new(false))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keys_survive_reloading_and_track_nested_content_not_unrelated_packages() {
|
||||
let original = library();
|
||||
assert_eq!(key(&original), key(&library()));
|
||||
let mut changed = library();
|
||||
changed.packages.insert("unrelated".into(), Arc::new(Package::new("unused")));
|
||||
assert_eq!(key(&original), key(&changed));
|
||||
Arc::make_mut(changed.packages.get_mut("child").unwrap()).graphs[0].seed += 1;
|
||||
assert_ne!(key(&original), key(&changed));
|
||||
let mut changed = library();
|
||||
Arc::make_mut(changed.packages.get_mut("child").unwrap()).resources[0].bytes = ResourceBytes::new(vec![1, 2, 4]);
|
||||
assert_ne!(key(&original), key(&changed), "same-size replacement must invalidate the preview");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn incomplete_and_cancelled_keys_are_not_persisted() {
|
||||
let mut library = library();
|
||||
assert!(library.preview_fingerprint("root", "main", &AtomicBool::new(true)).is_none());
|
||||
library.packages.remove("child");
|
||||
assert!(key(&library).is_none());
|
||||
}
|
||||
@@ -24,6 +24,8 @@ image = { workspace = true }
|
||||
zip = { workspace = true }
|
||||
|
||||
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
|
||||
craft-atomic = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
sys-locale = "0.3"
|
||||
craft-update = { workspace = true }
|
||||
craft-dialogs = { workspace = true }
|
||||
@@ -42,6 +44,7 @@ web-sys = { version = "0.3", features = ["Window", "Document", "HtmlElement", "H
|
||||
|
||||
# winit needs a window system on Linux, so the crate's own tests build with X11.
|
||||
[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
|
||||
tempfile = "3"
|
||||
eframe = { workspace = true, features = ["x11"] }
|
||||
# Offscreen renders of the real UI (tests/snapshot.rs).
|
||||
egui_kittest = { workspace = true, features = ["wgpu", "snapshot", "eframe"] }
|
||||
@@ -15,6 +15,8 @@ const MAX_VISIBLE_REQUESTS: usize = 64;
|
||||
/// Only one cook may run; every frame replaces the pending work with its visible tiles.
|
||||
#[derive(Default)]
|
||||
pub(super) struct GraphPreviews {
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub persist: bool,
|
||||
library: Option<Arc<NodeLibrary>>,
|
||||
textures: HashMap<Key, (Option<TextureHandle>, u64)>,
|
||||
wanted: Vec<Key>,
|
||||
@@ -110,9 +112,18 @@ impl GraphPreviews {
|
||||
let (sender, result) = std::sync::mpsc::sync_channel(1);
|
||||
let cancel = Arc::new(AtomicBool::new(false));
|
||||
let (thread_cancel, thread_library, thread_key, context) = (cancel.clone(), library.clone(), key.clone(), ctx.clone());
|
||||
let persist = self.persist;
|
||||
let spawned = std::thread::Builder::new().name("materialcraft-library-graphs".into()).spawn(move || {
|
||||
let image = NodeLibrary::preview_cancellable(&thread_library, &thread_key.0, &thread_key.1, LIBRARY_PREVIEW_SIZE, &thread_cancel);
|
||||
let rgba = image.map(|image| image.resize(LIBRARY_PREVIEW_SIZE, LIBRARY_PREVIEW_SIZE, true).to_rgba8_display());
|
||||
let render = || {
|
||||
NodeLibrary::preview_cancellable(&thread_library, &thread_key.0, &thread_key.1, LIBRARY_PREVIEW_SIZE, &thread_cancel)
|
||||
.map(|image| image.resize(LIBRARY_PREVIEW_SIZE, LIBRARY_PREVIEW_SIZE, true).to_rgba8_display())
|
||||
};
|
||||
let disk = persist.then(super::preview_disk_cache::PreviewDiskCache::configured).flatten();
|
||||
let fingerprint = disk.as_ref().and_then(|_| thread_library.preview_fingerprint(&thread_key.0, &thread_key.1, &thread_cancel));
|
||||
let rgba = match (disk, fingerprint) {
|
||||
(Some(disk), Some(key)) => disk.get_or_render(key, LIBRARY_PREVIEW_SIZE, &thread_cancel, render),
|
||||
_ => render(),
|
||||
};
|
||||
let _ = sender.send(rgba);
|
||||
context.request_repaint();
|
||||
});
|
||||
|
||||
@@ -77,6 +77,7 @@ impl MaterialCraftApp {
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
{
|
||||
app.dialogs = crate::native_dialogs::attached(&cc.egui_ctx);
|
||||
app.thumbs.enable_persistence();
|
||||
}
|
||||
app
|
||||
}
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
//! Disposable PNG thumbnails. All filesystem work belongs to the existing preview workers.
|
||||
|
||||
use std::fs::{self, File};
|
||||
use std::io::{self, Cursor, Read};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::OnceLock;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use image::{ImageDecoder, ImageEncoder};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
const MAX_ENTRIES: usize = 4096;
|
||||
const MAX_FILE_BYTES: u64 = 128 * 1024;
|
||||
const MAX_SIDE: u32 = 128;
|
||||
|
||||
pub(super) struct PreviewDiskCache {
|
||||
root: PathBuf,
|
||||
renderer: [u8; 32],
|
||||
budget: u64,
|
||||
entries: usize,
|
||||
}
|
||||
|
||||
impl PreviewDiskCache {
|
||||
pub fn configured() -> Option<Self> {
|
||||
// Scripted runs must not touch the user's cache. An explicit override lets tests and
|
||||
// agents exercise persistence in an isolated folder; an empty override disables it.
|
||||
let root = match std::env::var_os("MATERIALCRAFT_CACHE_DIR") {
|
||||
Some(path) if path.is_empty() => return None,
|
||||
Some(path) => PathBuf::from(path),
|
||||
None if cfg!(test) || std::env::var_os("MATERIALCRAFT_NO_PREFS").is_some() => return None,
|
||||
// The existing shared path resolver follows the suite's cache-folder policy,
|
||||
// including redirected Windows profiles and absolute XDG paths.
|
||||
None => craft_update::Paths::per_user("materialcraft").ok()?.cache,
|
||||
};
|
||||
Self::open(root.join("previews"), 64 * 1024 * 1024)
|
||||
}
|
||||
|
||||
/// Called on a worker. The executable digest also invalidates development builds that keep
|
||||
/// the same version number. It is streamed once per process, never on the event-loop thread.
|
||||
pub fn open(root: PathBuf, budget: u64) -> Option<Self> {
|
||||
static RENDERER: OnceLock<Option<[u8; 32]>> = OnceLock::new();
|
||||
let renderer = *RENDERER.get_or_init(|| {
|
||||
let mut file = File::open(std::env::current_exe().ok()?).ok()?.take((1 << 30) + 1);
|
||||
let mut hash = Sha256::new();
|
||||
let bytes = io::copy(&mut file, &mut hash).ok()?;
|
||||
(bytes <= 1 << 30).then(|| hash.finalize().into())
|
||||
});
|
||||
Some(Self { root, renderer: renderer?, budget, entries: MAX_ENTRIES })
|
||||
}
|
||||
|
||||
pub fn get_or_render(&self, key: [u8; 32], side: u32, cancel: &AtomicBool, render: impl FnOnce() -> Option<Vec<u8>>) -> Option<Vec<u8>> {
|
||||
if cancel.load(Ordering::Relaxed) {
|
||||
return None;
|
||||
}
|
||||
let path = self.path(key, side);
|
||||
if let Some(pixels) = self.read(&path, side) {
|
||||
return (!cancel.load(Ordering::Relaxed)).then_some(pixels);
|
||||
}
|
||||
let pixels = render()?;
|
||||
if cancel.load(Ordering::Relaxed) {
|
||||
return None;
|
||||
}
|
||||
// A cache is optional: unavailable storage must not hide a successfully rendered tile.
|
||||
if let Err(error) = self.write(&path, side, &pixels) {
|
||||
log::debug!("Library preview persistence skipped: {error}");
|
||||
}
|
||||
Some(pixels)
|
||||
}
|
||||
|
||||
fn path(&self, key: [u8; 32], side: u32) -> PathBuf {
|
||||
let mut hash = Sha256::new();
|
||||
hash.update(b"materialcraft-preview-png-v1");
|
||||
hash.update(self.renderer);
|
||||
hash.update(key);
|
||||
hash.update(side.to_le_bytes());
|
||||
self.root.join(format!("{:x}.png", hash.finalize()))
|
||||
}
|
||||
|
||||
fn read(&self, path: &Path, side: u32) -> Option<Vec<u8>> {
|
||||
if side == 0 || side > MAX_SIDE || self.budget == 0 {
|
||||
return None;
|
||||
}
|
||||
let metadata = fs::symlink_metadata(path).ok()?;
|
||||
if !metadata.is_file() || metadata.len() > MAX_FILE_BYTES {
|
||||
return None;
|
||||
}
|
||||
let mut file = File::open(path).ok()?;
|
||||
let mut bytes = Vec::new();
|
||||
Read::by_ref(&mut file).take(MAX_FILE_BYTES + 1).read_to_end(&mut bytes).ok()?;
|
||||
if bytes.len() as u64 > MAX_FILE_BYTES {
|
||||
return None;
|
||||
}
|
||||
let mut limits = image::Limits::default();
|
||||
limits.max_image_width = Some(MAX_SIDE);
|
||||
limits.max_image_height = Some(MAX_SIDE);
|
||||
limits.max_alloc = Some(1024 * 1024);
|
||||
let decoder = image::codecs::png::PngDecoder::with_limits(Cursor::new(bytes), limits).ok()?;
|
||||
if decoder.dimensions() != (side, side) || decoder.color_type() != image::ColorType::Rgba8 {
|
||||
return None;
|
||||
}
|
||||
let mut pixels = vec![0; (side * side * 4) as usize];
|
||||
decoder.read_image(&mut pixels).ok()?;
|
||||
// Approximate LRU across launches, without a metadata write on every tile lookup.
|
||||
if metadata.modified().ok()?.elapsed().is_ok_and(|age| age >= Duration::from_secs(3600)) {
|
||||
let _ = file.set_modified(SystemTime::now());
|
||||
}
|
||||
Some(pixels)
|
||||
}
|
||||
|
||||
fn write(&self, path: &Path, side: u32, pixels: &[u8]) -> io::Result<()> {
|
||||
if side == 0 || side > MAX_SIDE || pixels.len() != (side * side * 4) as usize || self.budget == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
image::codecs::png::PngEncoder::new(&mut bytes).write_image(pixels, side, side, image::ExtendedColorType::Rgba8).map_err(io::Error::other)?;
|
||||
if bytes.len() as u64 > MAX_FILE_BYTES || bytes.len() as u64 > self.budget {
|
||||
return Ok(());
|
||||
}
|
||||
fs::create_dir_all(&self.root)?;
|
||||
// Coordinate the budget across windows/processes. A busy writer is a harmless cache
|
||||
// miss; do not block a preview worker waiting for another process to finish.
|
||||
let lock = File::options().read(true).write(true).create(true).truncate(false).open(self.root.join("cache.lock"))?;
|
||||
lock.try_lock().map_err(io::Error::other)?;
|
||||
// craft-atomic deliberately follows document symlinks. Cache entries are disposable
|
||||
// files owned by this cache, so never follow one into an unrelated document instead.
|
||||
if fs::symlink_metadata(path).is_ok_and(|metadata| !metadata.is_file()) {
|
||||
return Err(io::Error::other("preview cache entry is not a regular file"));
|
||||
}
|
||||
self.reserve(path, bytes.len() as u64)?;
|
||||
craft_atomic::WriteOptions::default().with_new_file_permissions(craft_atomic::NewFilePermissions::Private).atomic_write(path, &bytes)
|
||||
}
|
||||
|
||||
fn reserve(&self, target: &Path, incoming: u64) -> io::Result<()> {
|
||||
let mut entries = Vec::new();
|
||||
let mut total = 0u64;
|
||||
// Bound both scan time and metadata allocations in an externally populated directory.
|
||||
for (index, entry) in fs::read_dir(&self.root)?.enumerate() {
|
||||
if index >= MAX_ENTRIES * 2 {
|
||||
return Err(io::Error::other("preview cache directory exceeds its entry limit"));
|
||||
}
|
||||
let entry = entry?;
|
||||
if !entry.file_type()?.is_file() {
|
||||
continue;
|
||||
}
|
||||
let name = entry.file_name();
|
||||
let name = name.to_string_lossy();
|
||||
// The cache lock excludes every live writer. Matching staging files can only be
|
||||
// leftovers from an interrupted atomic save, so reclaim them before growing again.
|
||||
if staging_name(&name) {
|
||||
fs::remove_file(entry.path())?;
|
||||
continue;
|
||||
}
|
||||
if !cache_name(&name) || entry.path() == target {
|
||||
continue;
|
||||
}
|
||||
let metadata = entry.metadata()?;
|
||||
total = total.saturating_add(metadata.len());
|
||||
entries.push((metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH), entry.path(), metadata.len()));
|
||||
}
|
||||
entries.sort_unstable_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1)));
|
||||
let mut count = entries.len();
|
||||
for (_, path, size) in entries {
|
||||
if total.saturating_add(incoming) <= self.budget && count < self.entries {
|
||||
break;
|
||||
}
|
||||
fs::remove_file(path)?;
|
||||
total = total.saturating_sub(size);
|
||||
count = count.saturating_sub(1);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn cache_name(name: &str) -> bool {
|
||||
name.strip_suffix(".png").is_some_and(digest_name)
|
||||
}
|
||||
|
||||
fn digest_name(stem: &str) -> bool {
|
||||
stem.len() == 64 && stem.bytes().all(|byte| byte.is_ascii_hexdigit())
|
||||
}
|
||||
|
||||
fn staging_name(name: &str) -> bool {
|
||||
let Some((stem, suffix)) = name.strip_prefix('.').and_then(|name| name.strip_suffix(".craft_tmp")).and_then(|name| name.split_once('.')) else {
|
||||
return false;
|
||||
};
|
||||
digest_name(stem) && suffix.split_once('-').is_some_and(|(pid, counter)| pid.parse::<u32>().is_ok_and(|pid| pid > 0) && counter.parse::<u64>().is_ok())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "preview_disk_cache.test.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,192 @@
|
||||
use super::*;
|
||||
|
||||
const SIDE: u32 = 8;
|
||||
// A fork can briefly inherit another test's file lock before exec closes it. Keep this
|
||||
// module's subprocess launches apart from its nonblocking-writer tests.
|
||||
static TESTS: std::sync::Mutex<()> = std::sync::Mutex::new(());
|
||||
|
||||
fn cache(root: &Path) -> PreviewDiskCache {
|
||||
PreviewDiskCache { root: root.to_owned(), renderer: [1; 32], budget: 64 * 1024 * 1024, entries: MAX_ENTRIES }
|
||||
}
|
||||
|
||||
fn pixels() -> Vec<u8> {
|
||||
(0..SIDE * SIDE * 4).map(|i| i as u8).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn previews_survive_process_restarts_without_rendering() {
|
||||
let _serial = TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
const CHILD: &str = "MATERIALCRAFT_TEST_DISK_PREVIEW";
|
||||
if let Some(root) = std::env::var_os(CHILD) {
|
||||
let cache = cache(Path::new(&root));
|
||||
let actual = cache.get_or_render([2; 32], SIDE, &AtomicBool::new(false), || {
|
||||
assert_eq!(std::env::var("MATERIALCRAFT_TEST_DISK_PHASE").unwrap(), "cold", "warm launch must not cook");
|
||||
Some(pixels())
|
||||
});
|
||||
assert_eq!(actual, Some(pixels()));
|
||||
return;
|
||||
}
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
for phase in ["cold", "warm"] {
|
||||
let start = std::time::Instant::now();
|
||||
let status = std::process::Command::new(std::env::current_exe().unwrap())
|
||||
.args(["--exact", "thumbs::preview_disk_cache::tests::previews_survive_process_restarts_without_rendering", "--nocapture"])
|
||||
.env(CHILD, root.path())
|
||||
.env("MATERIALCRAFT_TEST_DISK_PHASE", phase)
|
||||
.status()
|
||||
.unwrap();
|
||||
assert!(status.success());
|
||||
eprintln!("{phase} preview subprocess: {:?}", start.elapsed());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "renders bundled library graphs and measures cold/warm subprocesses"]
|
||||
fn real_library_previews_survive_relaunch() {
|
||||
let _serial = TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
const CHILD: &str = "MATERIALCRAFT_TEST_REAL_PREVIEWS";
|
||||
if let Some(root) = std::env::var_os(CHILD) {
|
||||
let library = materialcraft_engine::NodeLibrary::shared();
|
||||
let phase = std::env::var("MATERIALCRAFT_TEST_DISK_PHASE").unwrap();
|
||||
let cache = PreviewDiskCache::open(PathBuf::from(root), 64 * 1024 * 1024).unwrap();
|
||||
let cancel = AtomicBool::new(false);
|
||||
for (package, graph) in
|
||||
[("sbs://blur_hq.sbs", "blur_hq"), ("sbs://noise_clouds_3_v2.sbs", "clouds_3"), ("sbs://materials/pbr/bricks_001.sbs", "bricks_001")]
|
||||
{
|
||||
let started = std::time::Instant::now();
|
||||
let key = library.preview_fingerprint(package, graph, &cancel).unwrap();
|
||||
let image = cache.get_or_render(key, 72, &cancel, || {
|
||||
assert_eq!(phase, "cold", "warm launch must load {graph} without cooking");
|
||||
materialcraft_engine::NodeLibrary::preview_cancellable(&library, package, graph, 72, &cancel)
|
||||
.map(|image| image.resize(72, 72, true).to_rgba8_display())
|
||||
});
|
||||
assert_eq!(image.unwrap().len(), 72 * 72 * 4);
|
||||
eprintln!("{phase} {graph}: {:?}", started.elapsed());
|
||||
}
|
||||
return;
|
||||
}
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
for phase in ["cold", "warm"] {
|
||||
let status = std::process::Command::new(std::env::current_exe().unwrap())
|
||||
.args(["--exact", "thumbs::preview_disk_cache::tests::real_library_previews_survive_relaunch", "--ignored", "--nocapture"])
|
||||
.env(CHILD, root.path())
|
||||
.env("MATERIALCRAFT_TEST_DISK_PHASE", phase)
|
||||
.status()
|
||||
.unwrap();
|
||||
assert!(status.success());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keys_separate_sources_sizes_and_renderer_builds() {
|
||||
let _serial = TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let mut cache = cache(root.path());
|
||||
let original = cache.path([1; 32], SIDE);
|
||||
assert_ne!(original, cache.path([2; 32], SIDE));
|
||||
assert_ne!(original, cache.path([1; 32], SIDE + 1));
|
||||
cache.renderer = [2; 32];
|
||||
assert_ne!(original, cache.path([1; 32], SIDE));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancelled_failed_and_invalid_previews_never_persist() {
|
||||
let _serial = TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let cache = cache(root.path());
|
||||
let cancel = AtomicBool::new(true);
|
||||
assert!(cache.get_or_render([1; 32], SIDE, &cancel, || panic!("cancelled cook ran")).is_none());
|
||||
cancel.store(false, Ordering::Relaxed);
|
||||
assert!(cache.get_or_render([1; 32], SIDE, &cancel, || None).is_none());
|
||||
assert!(
|
||||
cache
|
||||
.get_or_render([1; 32], SIDE, &cancel, || {
|
||||
cancel.store(true, Ordering::Relaxed);
|
||||
Some(pixels())
|
||||
})
|
||||
.is_none()
|
||||
);
|
||||
cache.write(&cache.path([2; 32], SIDE), SIDE, &[1]).unwrap();
|
||||
assert_eq!(fs::read_dir(root.path()).unwrap().count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corrupt_oversized_and_wrong_size_files_are_misses_and_repaired() {
|
||||
let _serial = TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let cache = cache(root.path());
|
||||
let path = cache.path([1; 32], SIDE);
|
||||
for bytes in [b"broken PNG".to_vec(), vec![0; MAX_FILE_BYTES as usize + 1]] {
|
||||
fs::write(&path, bytes).unwrap();
|
||||
assert!(cache.read(&path, SIDE).is_none());
|
||||
assert_eq!(cache.get_or_render([1; 32], SIDE, &AtomicBool::new(false), || Some(pixels())), Some(pixels()));
|
||||
assert_eq!(cache.read(&path, SIDE), Some(pixels()));
|
||||
assert!(cache.read(&path, SIDE + 1).is_none());
|
||||
}
|
||||
let bytes = fs::read(&path).unwrap();
|
||||
fs::write(&path, &bytes[..bytes.len() / 2]).unwrap();
|
||||
assert!(cache.read(&path, SIDE).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cache_budget_evicts_old_files_and_preserves_unrelated_files() {
|
||||
let _serial = TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let mut cache = cache(root.path());
|
||||
let old = cache.path([1; 32], SIDE);
|
||||
cache.write(&old, SIDE, &pixels()).unwrap();
|
||||
let file_bytes = fs::metadata(&old).unwrap().len();
|
||||
cache.budget = file_bytes * 2;
|
||||
File::options().write(true).open(&old).unwrap().set_modified(SystemTime::UNIX_EPOCH).unwrap();
|
||||
fs::write(root.path().join("my-image.png"), b"unrelated").unwrap();
|
||||
let leftover = root.path().join(craft_atomic::temp_name(old.file_name().unwrap().to_str().unwrap()));
|
||||
fs::write(&leftover, b"interrupted write").unwrap();
|
||||
fs::write(root.path().join("unrelated.craft_tmp"), b"keep").unwrap();
|
||||
cache.write(&cache.path([2; 32], SIDE), SIDE, &pixels()).unwrap();
|
||||
assert!(!leftover.exists());
|
||||
assert!(root.path().join("unrelated.craft_tmp").exists());
|
||||
cache.write(&cache.path([3; 32], SIDE), SIDE, &pixels()).unwrap();
|
||||
assert!(!old.exists());
|
||||
assert_eq!(fs::read(root.path().join("my-image.png")).unwrap(), b"unrelated");
|
||||
let total: u64 =
|
||||
fs::read_dir(root.path()).unwrap().flatten().filter(|f| cache_name(&f.file_name().to_string_lossy())).map(|f| f.metadata().unwrap().len()).sum();
|
||||
assert!(total <= cache.budget);
|
||||
cache.budget = 64 * 1024 * 1024;
|
||||
cache.entries = 1;
|
||||
cache.write(&cache.path([4; 32], SIDE), SIDE, &pixels()).unwrap();
|
||||
let count = fs::read_dir(root.path()).unwrap().flatten().filter(|file| cache_name(&file.file_name().to_string_lossy())).count();
|
||||
assert_eq!(count, 1, "the entry limit also applies when there is space left");
|
||||
cache.budget = file_bytes - 1;
|
||||
let oversized = cache.path([5; 32], SIDE);
|
||||
cache.write(&oversized, SIDE, &pixels()).unwrap();
|
||||
assert!(!oversized.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unavailable_or_busy_cache_still_returns_rendered_pixels() {
|
||||
let _serial = TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let blocked = root.path().join("file");
|
||||
fs::write(&blocked, b"not a directory").unwrap();
|
||||
assert_eq!(cache(&blocked).get_or_render([1; 32], SIDE, &AtomicBool::new(false), || Some(pixels())), Some(pixels()));
|
||||
let lock = File::options().read(true).write(true).create_new(true).open(root.path().join("cache.lock")).unwrap();
|
||||
lock.lock().unwrap();
|
||||
let cache = cache(root.path());
|
||||
assert_eq!(cache.get_or_render([1; 32], SIDE, &AtomicBool::new(false), || Some(pixels())), Some(pixels()));
|
||||
assert!(!cache.path([1; 32], SIDE).exists());
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn cache_does_not_read_or_evict_symlink_targets() {
|
||||
let _serial = TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let outside = tempfile::NamedTempFile::new().unwrap();
|
||||
fs::write(outside.path(), b"keep").unwrap();
|
||||
let cache = cache(root.path());
|
||||
let path = cache.path([1; 32], SIDE);
|
||||
std::os::unix::fs::symlink(outside.path(), &path).unwrap();
|
||||
assert!(cache.read(&path, SIDE).is_none());
|
||||
let _ = cache.write(&path, SIDE, &pixels());
|
||||
assert_eq!(fs::read(outside.path()).unwrap(), b"keep");
|
||||
}
|
||||
@@ -19,6 +19,7 @@ pub struct Thumbs {
|
||||
library: HashMap<String, TextureHandle>,
|
||||
library_rx: Option<Receiver<Previews>>,
|
||||
library_started: bool,
|
||||
persist_library: bool,
|
||||
graphs: GraphPreviews,
|
||||
frame: u64,
|
||||
}
|
||||
@@ -32,8 +33,18 @@ pub fn color_image(img: &Image, side: u32) -> ColorImage {
|
||||
#[path = "graph_previews.rs"]
|
||||
mod graph_previews;
|
||||
use graph_previews::GraphPreviews;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
#[path = "preview_disk_cache.rs"]
|
||||
mod preview_disk_cache;
|
||||
|
||||
impl Thumbs {
|
||||
/// Opt in only for a native app window; offscreen harnesses never use the user's cache.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub fn enable_persistence(&mut self) {
|
||||
self.persist_library = true;
|
||||
self.graphs.persist = true;
|
||||
}
|
||||
|
||||
pub fn clear_nodes(&mut self) {
|
||||
self.nodes.clear();
|
||||
}
|
||||
@@ -86,7 +97,7 @@ impl Thumbs {
|
||||
pub fn library(&mut self, ctx: &egui::Context, def: &str) -> Option<egui::TextureId> {
|
||||
if !self.library_started {
|
||||
self.library_started = true;
|
||||
self.library_rx = Some(spawn_library_previews(ctx.clone()));
|
||||
self.library_rx = Some(spawn_library_previews(ctx.clone(), self.persist_library));
|
||||
}
|
||||
self.library.get(def).map(TextureHandle::id)
|
||||
}
|
||||
@@ -102,68 +113,87 @@ fn sample_inputs() -> (Image, Image, Image) {
|
||||
(gray, color, normal)
|
||||
}
|
||||
|
||||
fn previews() -> Previews {
|
||||
fn previews(_persist: bool) -> Previews {
|
||||
let (gray, color, normal) = sample_inputs();
|
||||
let no_res = |_: &str| None;
|
||||
let vars = BTreeMap::new();
|
||||
let mut out = Vec::new();
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
let disk = _persist.then(preview_disk_cache::PreviewDiskCache::configured).flatten();
|
||||
for def in catalog() {
|
||||
if def.id == "graph_instance" {
|
||||
continue;
|
||||
}
|
||||
let inputs: Vec<Option<Image>> = def
|
||||
.inputs
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, p)| {
|
||||
// Masks and secondary inputs stay empty so the main effect shows
|
||||
if i > 0 && !matches!(def.id, "blend" | "height_blend" | "normal_blend" | "rgba_merge") {
|
||||
return None;
|
||||
}
|
||||
Some(if p.id.contains("normal") || def.category.contains("Normal") && p.kind == materialcraft_engine::nodes::PortKind::Color {
|
||||
normal.clone()
|
||||
} else if p.kind == materialcraft_engine::nodes::PortKind::Color {
|
||||
color.clone()
|
||||
} else if i == 1 {
|
||||
gray.map(Kind::Gray, |v| [1.0 - v[0], 0.0, 0.0, 0.0])
|
||||
} else {
|
||||
gray.clone()
|
||||
let render = || {
|
||||
let inputs: Vec<Option<Image>> = def
|
||||
.inputs
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, p)| {
|
||||
// Masks and secondary inputs stay empty so the main effect shows
|
||||
if i > 0 && !matches!(def.id, "blend" | "height_blend" | "normal_blend" | "rgba_merge") {
|
||||
return None;
|
||||
}
|
||||
Some(if p.id.contains("normal") || def.category.contains("Normal") && p.kind == materialcraft_engine::nodes::PortKind::Color {
|
||||
normal.clone()
|
||||
} else if p.kind == materialcraft_engine::nodes::PortKind::Color {
|
||||
color.clone()
|
||||
} else if i == 1 {
|
||||
gray.map(Kind::Gray, |v| [1.0 - v[0], 0.0, 0.0, 0.0])
|
||||
} else {
|
||||
gray.clone()
|
||||
})
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
let params = resolve_params(def, &BTreeMap::new());
|
||||
let ctx = CookContext {
|
||||
w: LIBRARY_PREVIEW_SIZE,
|
||||
h: LIBRARY_PREVIEW_SIZE,
|
||||
seed: 3,
|
||||
tile: true,
|
||||
inputs: &inputs,
|
||||
params: ¶ms,
|
||||
resource: &no_res,
|
||||
vars: &vars,
|
||||
.collect();
|
||||
let params = resolve_params(def, &BTreeMap::new());
|
||||
let ctx = CookContext {
|
||||
w: LIBRARY_PREVIEW_SIZE,
|
||||
h: LIBRARY_PREVIEW_SIZE,
|
||||
seed: 3,
|
||||
tile: true,
|
||||
inputs: &inputs,
|
||||
params: ¶ms,
|
||||
resource: &no_res,
|
||||
vars: &vars,
|
||||
};
|
||||
let imgs = (def.cook)(&ctx).ok()?;
|
||||
Some(imgs.first()?.resize(LIBRARY_PREVIEW_SIZE, LIBRARY_PREVIEW_SIZE, true).to_rgba8_display())
|
||||
};
|
||||
if let Ok(imgs) = (def.cook)(&ctx)
|
||||
&& let Some(img) = imgs.first()
|
||||
{
|
||||
out.push((def.id.to_string(), img.resize(LIBRARY_PREVIEW_SIZE, LIBRARY_PREVIEW_SIZE, true).to_rgba8_display()));
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
let rgba = {
|
||||
use sha2::Digest;
|
||||
match &disk {
|
||||
Some(disk) => disk.get_or_render(
|
||||
sha2::Sha256::digest(format!("materialcraft-atomic-preview-v1:{}", def.id)).into(),
|
||||
LIBRARY_PREVIEW_SIZE,
|
||||
&std::sync::atomic::AtomicBool::new(false),
|
||||
render,
|
||||
),
|
||||
None => render(),
|
||||
}
|
||||
};
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
let rgba = render();
|
||||
if let Some(rgba) = rgba {
|
||||
out.push((def.id.to_string(), rgba));
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn spawn_library_previews(ctx: egui::Context) -> Receiver<Previews> {
|
||||
fn spawn_library_previews(ctx: egui::Context, persist: bool) -> Receiver<Previews> {
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
let _ = std::thread::Builder::new().name("materialcraft-library".into()).spawn(move || {
|
||||
let _ = tx.send(previews());
|
||||
let _ = tx.send(previews(persist));
|
||||
ctx.request_repaint();
|
||||
});
|
||||
rx
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
fn spawn_library_previews(_ctx: egui::Context) -> Receiver<Previews> {
|
||||
fn spawn_library_previews(_ctx: egui::Context, persist: bool) -> Receiver<Previews> {
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
let _ = tx.send(previews());
|
||||
let _ = tx.send(previews(persist));
|
||||
rx
|
||||
}
|
||||
@@ -137,6 +137,26 @@ leaving a tile or reloading the library cancels its cook between nodes. The next
|
||||
that worker to finish; hidden tiles cannot accumulate queued work or workers. At most 256
|
||||
finished previews retain GPU textures (about 5 MiB of pixels).
|
||||
|
||||
Native Library previews also persist as PNGs in the platform cache folder selected by
|
||||
the existing `craft_update::Paths::per_user("materialcraft").cache`, under `previews/`: `%LOCALAPPDATA%` on
|
||||
Windows, `$XDG_CACHE_HOME` or `~/.cache` on Linux/BSD, and `~/Library/Caches` on macOS.
|
||||
This follows the [Craft storage policy](https://github.com/storytold/craft-libs/blob/main/docs/settings-storage.md).
|
||||
The cache covers native nodes and SBS graphs, retains at most 64 MiB / 4096 PNGs across builds,
|
||||
and evicts older files when admitting new previews (access times refresh at most hourly). It stores disposable
|
||||
thumbnails, independent of projects, autosaves and temporary render storage.
|
||||
|
||||
Preview workers do all reads, encoding, atomic writes (`craft-atomic`) and eviction; none run
|
||||
on the UI thread. Keys include the preview size, an executable content digest (including local
|
||||
rebuilds with an unchanged release version), and, for SBS graphs, the source packages and all
|
||||
resource bytes in their dependency closure. Changes become visible after `library.reload` or
|
||||
the next launch, just like the in-memory library. Corrupt, oversized or unavailable cache files
|
||||
are misses; failed or cancelled renders are never persisted. Concurrent writers use a
|
||||
nonblocking file lock to share the disk budget. The browser retains its existing behavior.
|
||||
|
||||
`MATERIALCRAFT_CACHE_DIR` overrides the app cache folder (`previews/` is appended); an empty
|
||||
value disables persistence. Scripted `MATERIALCRAFT_NO_PREFS` runs leave the user's cache
|
||||
untouched unless that explicit override is provided. Headless UI harnesses keep persistence off.
|
||||
|
||||
The local `sbs_library_build` example converts native XML without a model round trip, checking that
|
||||
computational fields survive presentation edits and reference rewriting. It generates independent
|
||||
resource substitutes without reading source resource files. It retains supplied graph implementations;
|
||||
|
||||
Reference in new issue
Block a user