diff --git a/Cargo.lock b/Cargo.lock index 8bad56c4..cdee09a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -578,7 +578,7 @@ dependencies = [ "maybe-owned", "rustix 1.1.5", "rustix-linux-procfs", - "windows-sys 0.60.2", + "windows-sys 0.61.2", "winx", ] @@ -746,6 +746,12 @@ dependencies = [ "libc", ] +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -827,9 +833,9 @@ dependencies = [ [[package]] name = "craft-mcmat" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=c376774f15ae7cd3b06a7ac664f01a8645af4b83#c376774f15ae7cd3b06a7ac664f01a8645af4b83" +source = "git+https://github.com/storytold/craft-libs?rev=7a7a18cb40ac4cd5fca29c0dc224ac5178d40a89#7a7a18cb40ac4cd5fca29c0dc224ac5178d40a89" dependencies = [ - "craft-zip 0.1.0 (git+https://github.com/storytold/craft-libs?rev=c376774f15ae7cd3b06a7ac664f01a8645af4b83)", + "craft-zip 0.1.0 (git+https://github.com/storytold/craft-libs?rev=7a7a18cb40ac4cd5fca29c0dc224ac5178d40a89)", "serde", "serde_json", "thiserror 2.0.21", @@ -978,6 +984,16 @@ dependencies = [ "zip", ] +[[package]] +name = "craft-zip" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=7a7a18cb40ac4cd5fca29c0dc224ac5178d40a89#7a7a18cb40ac4cd5fca29c0dc224ac5178d40a89" +dependencies = [ + "flate2", + "thiserror 2.0.21", + "zip", +] + [[package]] name = "craft-zip" version = "0.1.0" @@ -1336,6 +1352,20 @@ dependencies = [ "serde", ] +[[package]] +name = "encoding_rs" +version = "0.8.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89253175e704ab869b04f9db224d38630e0211a53841a79181deae0889e169f5" +dependencies = [ + "cfg-if", + "core_detect", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", +] + [[package]] name = "endi" version = "1.1.1" @@ -1419,7 +1449,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -2652,9 +2682,21 @@ dependencies = [ "objc2-foundation 0.3.2", "png", "thiserror 2.0.21", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + +[[package]] +name = "mutants" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "add0ac067452ff1aca8c5002111bd6b1c895baee6e45fcbc44e0193aea17be56" + [[package]] name = "naga" version = "30.0.1" @@ -3176,7 +3218,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3469,6 +3511,23 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "processkit" +version = "3.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1549b59abbe4fc78aa39860e36c81da89262c1278679b8f7299ef4ad7668c004" +dependencies = [ + "async-trait", + "encoding_rs", + "libc", + "mutants", + "thiserror 2.0.21", + "tokio", + "tokio-stream", + "tokio-util", + "windows-sys 0.61.2", +] + [[package]] name = "profiling" version = "1.0.18" @@ -3820,7 +3879,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3887,7 +3946,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -4240,7 +4299,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -4355,7 +4414,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix 1.1.5", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -4459,10 +4518,13 @@ version = "0.6.1" dependencies = [ "epaint_default_fonts", "image", + "processkit", "rayon", "rustix 1.1.5", "serde", "serde_json", + "sysinfo", + "tempfile", "texturecraft-bake", "texturecraft-compose", "texturecraft-core", @@ -4471,6 +4533,7 @@ dependencies = [ "texturecraft-paint", "texturecraft-texel", "thiserror 2.0.21", + "tokio", "zip", ] @@ -4479,6 +4542,7 @@ name = "texturecraft-formats" version = "0.6.1" dependencies = [ "base64 0.22.1", + "bytemuck", "craft-cms", "craft-materialx", "craft-mcmat", @@ -4493,6 +4557,7 @@ dependencies = [ "half", "image", "lz4_flex", + "png", "quick-xml 0.41.0", "rayon", "serde", @@ -4710,6 +4775,7 @@ dependencies = [ "libc", "mio", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.61.2", @@ -4735,6 +4801,7 @@ dependencies = [ "futures-core", "pin-project-lite", "tokio", + "tokio-util", ] [[package]] @@ -4879,7 +4946,7 @@ checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" dependencies = [ "memoffset", "tempfile", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -5474,7 +5541,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 983ec031..7d3803b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -74,7 +74,7 @@ craft-tcraft = { git = "https://github.com/storytold/craft-libs", rev = "c376774 craft-zip-usdz = { package = "craft-zip", git = "https://github.com/storytold/craft-libs", rev = "5631e99fe126cffb74c673eb458db04299f2bb9b" } craft-zip = { git = "https://github.com/storytold/craft-libs", rev = "c376774f15ae7cd3b06a7ac664f01a8645af4b83" } craft-mesh = { git = "https://github.com/storytold/craft-libs", rev = "70e562aa88436b6b79a997215b4fa4f6a6b1838a", default-features = false, features = ["stl", "ply", "fbx", "obj", "gltf"] } -craft-mcmat = { git = "https://github.com/storytold/craft-libs", rev = "c376774f15ae7cd3b06a7ac664f01a8645af4b83" } +craft-mcmat = { git = "https://github.com/storytold/craft-libs", rev = "7a7a18cb40ac4cd5fca29c0dc224ac5178d40a89" } # Who draws the window frame, and the app-drawn strip as the title bar (crates/ui-egui/src/titlebar.rs). craft-window = { git = "https://github.com/storytold/craft-libs", rev = "390fe98b0dd3fb84db7c3af7cbd958db4e8cc585", features = ["egui"] } # Release checks, the update prompt and the update helper (crates/ui-egui/src/updates_ui.rs). diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 3d66b9df..ae12eea3 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -22,6 +22,16 @@ serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +tempfile = "3" + +[target.'cfg(any(target_os = "macos", target_os = "freebsd"))'.dependencies] +sysinfo = { workspace = true } + +[target.'cfg(windows)'.dependencies] +processkit = { version = "3.3.4", default-features = false, features = ["limits"] } +tokio = { workspace = true } + [target.'cfg(unix)'.dependencies] rustix = { workspace = true } diff --git a/crates/engine/src/file_cmds.rs b/crates/engine/src/file_cmds.rs index 168c438b..c66881d1 100644 --- a/crates/engine/src/file_cmds.rs +++ b/crates/engine/src/file_cmds.rs @@ -2,6 +2,8 @@ //! web; path-based commands are native only. use std::borrow::Cow; +use std::collections::BTreeMap; +use std::sync::Arc; use serde_json::{Value, json}; #[cfg(not(target_arch = "wasm32"))] @@ -212,9 +214,20 @@ impl Session { Ok(s) } - /// The project as a `ProjectFile` (for saving). + /// The project with independently owned resource bytes. pub fn project_file(&self) -> ProjectFile { + let resources = self.doc.resources.iter().map(|(name, bytes)| (name.clone(), bytes.as_ref().clone())).collect(); + self.project_with_resources(resources) + } + + /// A save snapshot that shares immutable resource payloads with the document. + pub fn project_snapshot(&self) -> ProjectFile>> { + self.project_with_resources(self.doc.resources.clone()) + } + + fn project_with_resources(&self, resources: BTreeMap) -> ProjectFile { let mut pixels = self.doc.pixels.clone(); + // Drop pixel images whose size no longer matches their texture set let size_of = |id: texturecraft_core::LayerId| { self.doc.project.locate(id).and_then(|(ts, _, _)| self.doc.project.texture_sets.get(ts)).map(TextureSet::size).or_else(|| { @@ -239,13 +252,14 @@ impl Session { cage, bake_paint: self.saved_bake_paint(), thumbnail: None, - resources: self.doc.resources.iter().map(|(k, v)| (k.clone(), v.as_ref().clone())).collect(), + resources, warnings: Vec::new(), } } pub fn to_project_bytes(&self) -> Result> { - texturecraft_formats::save_project_bytes(&self.project_file()).map_err(fe) + let mut reservation = texturecraft_formats::RamReservation::for_save().map_err(fe)?; + reservation.save_bytes(&self.project_snapshot()).map_err(fe) } /// Render every export file for texture sets `sets` (in memory; works on the web). @@ -405,7 +419,9 @@ mod native { pub fn save_to(s: &mut Session, path: &str, adopt: bool) -> Result { let path = if ext_of(path) == PROJECT_EXT { path.to_string() } else { format!("{path}.{PROJECT_EXT}") }; - texturecraft_formats::save_project(Path::new(&path), &s.project_file()).map_err(fe)?; + let mut reservation = texturecraft_formats::RamReservation::for_save().map_err(fe)?; + reservation.save(Path::new(&path), &s.project_snapshot()).map_err(fe)?; + if adopt { s.path = Some(path.clone()); s.modified = false; diff --git a/crates/engine/src/link_cmds.rs b/crates/engine/src/link_cmds.rs index f8599b32..1fcf5b34 100644 --- a/crates/engine/src/link_cmds.rs +++ b/crates/engine/src/link_cmds.rs @@ -2,20 +2,37 @@ //! channel sources are its cooked maps (image resources) and whose [`MaterialLink`] keeps the //! graph's exposed parameters. Changing one re-cooks the maps through `materialcraft-cli` at the //! texture set's resolution, with the set's baked mesh maps feeding graph inputs of the same name. -//! The apps share the file format and the command line, not code. +//! Both apps use the shared craft-mcmat container and the public CLI. use std::sync::Arc; use serde_json::{Map, Value, json}; -use texturecraft_core::{BakedMap, Channel, Fill, Layer, LayerId, LayerKind, LinkParam, MaterialLink, Source}; -#[cfg(not(target_arch = "wasm32"))] -use texturecraft_formats::mcmat::MCMAT_EXT; -use texturecraft_formats::mcmat::{MCRAFT_EXT, PortableMaterial, is_mcraft, read_mcmat}; +use texturecraft_core::{Channel, Fill, Layer, LayerId, LayerKind, LinkParam, MaterialLink, Source}; +use texturecraft_formats::RamReservation as CookReservation; +use texturecraft_formats::mcmat::{MCRAFT_EXT, is_mcmat, is_mcraft, validate_mcmat_version}; use crate::commands::{CommandSpec, bad, boolp, has_layer, layer_param, strp}; use crate::materials::Place; use crate::{EngineError, Result, Session, cmd}; +#[cfg(not(target_arch = "wasm32"))] +mod cook_report; +mod cooked_map; +mod cooked_material; +mod image_input; +mod link_job; +mod link_memory_budget; +#[cfg(not(target_arch = "wasm32"))] +mod map_file; +mod material_interface; + +pub use cooked_map::CookedMap; +pub use cooked_material::CookedMaterial; +pub use image_input::ImageInput; +pub use link_job::LinkJob; +pub use link_memory_budget::LinkMemoryBudget; +pub use material_interface::MaterialInterface; + /// The undo label of parameter edits and re-cooks (a re-cook merges into the edit before it). const EDIT: &str = "Edit MaterialCraft material"; @@ -56,7 +73,7 @@ pub fn specs() -> Vec { "Save MaterialCraft package", [], None, - "{layer?, path} — writes the MaterialCraft source (.mcraft or .sbs) behind a material layer", + "{layer?, path} — writes the MaterialCraft source (.mcmat, .mcraft or .sbs) behind a material layer", has_layer, save_source ), @@ -64,16 +81,16 @@ pub fn specs() -> Vec { } /// The link of fill layer `id`. -fn link_of(s: &Session, cmd: &str, id: LayerId) -> Result { - match &s.doc.project.layer(id)?.kind { +fn link_of(session: &Session, cmd: &str, id: LayerId) -> Result { + match &session.doc.project.layer(id)?.kind { LayerKind::Fill(Fill { link: Some(link), .. }) => Ok(link.as_ref().clone()), _ => Err(bad(cmd, format!("layer {} is not a MaterialCraft material", id.0))), } } /// Value types other apps may set (MaterialCraft's portable inputs), with their component count. -fn components(t: &str) -> Option { - Some(match t { +fn components(parameter_type: &str) -> Option { + Some(match parameter_type { "float" | "int" | "enum" | "bool" => 1, "float2" | "int2" => 2, "float3" => 3, @@ -82,41 +99,45 @@ fn components(t: &str) -> Option { }) } -fn type_of(v: &Value) -> &str { - v.get("t").and_then(Value::as_str).unwrap_or("") +fn type_of(value: &Value) -> &str { + value.get("t").and_then(Value::as_str).unwrap_or("") } -/// `v` (plain or `{t, v}`) as a typed value for `param`, with finite numbers. -pub fn typed(param: &LinkParam, v: &Value) -> Option { - let t = type_of(¶m.default); - let n = components(t)?; - let payload = if v.get("t").is_some() { v.get("v")? } else { v }; - let num = |x: &Value| x.as_f64().filter(|f| f.is_finite()); - let payload = match (t, payload) { - ("bool", Value::Bool(b)) => json!(b), - ("bool", x) => json!(num(x)? != 0.0), +/// Convert a plain or tagged value to the parameter type, rejecting non-finite numbers. +pub fn typed(parameter: &LinkParam, value: &Value) -> Option { + let parameter_type = type_of(¶meter.default); + let component_count = components(parameter_type)?; + let payload = if value.get("t").is_some() { value.get("v")? } else { value }; + let finite = |value: &Value| value.as_f64().filter(|number| number.is_finite()); + + let payload = match (parameter_type, payload) { + ("bool", Value::Bool(enabled)) => json!(enabled), + ("bool", value) => json!(finite(value)? != 0.0), ("color", Value::String(hex)) => { - let h = hex.trim_start_matches('#'); - if h.len() != 6 || !h.bytes().all(|b| b.is_ascii_hexdigit()) { + let digits = hex.trim_start_matches('#'); + if digits.len() != 6 || !digits.bytes().all(|byte| byte.is_ascii_hexdigit()) { return None; } - let c = |i: usize| Some(f64::from(u8::from_str_radix(h.get(i..i + 2)?, 16).ok()?) / 255.0); - // Hex colors are sRGB; values are linear. - let lin = |x: f64| if x <= 0.04045 { x / 12.92 } else { ((x + 0.055) / 1.055).powf(2.4) }; - json!([lin(c(0)?), lin(c(2)?), lin(c(4)?), 1.0]) + + let component = |index: usize| Some(f64::from(u8::from_str_radix(digits.get(index..index + 2)?, 16).ok()?) / f64::from(u8::MAX)); + + // Convert the sRGB hex input to linear parameter values + let linear = |value: f64| if value <= 0.04045 { value / 12.92 } else { ((value + 0.055) / 1.055).powf(2.4) }; + json!([linear(component(0)?), linear(component(2)?), linear(component(4)?), 1.0]) } - (_, x) if n == 1 => { - let f = num(x)?; - if matches!(t, "int" | "enum") { json!(f.round() as i64) } else { json!(f) } + (_, value) if component_count == 1 => { + let number = finite(value)?; + if matches!(parameter_type, "int" | "enum") { json!(number.round() as i64) } else { json!(number) } } - (_, Value::Array(a)) if a.len() >= n.min(3) => { - let mut out: Vec = a.iter().take(n).map(num).collect::>()?; - out.resize(n, 1.0); - if t == "int2" { json!(out.iter().map(|f| f.round() as i64).collect::>()) } else { json!(out) } + (_, Value::Array(values)) if values.len() >= component_count.min(3) => { + let mut numbers: Vec = values.iter().take(component_count).map(finite).collect::>()?; + numbers.resize(component_count, 1.0); + if parameter_type == "int2" { json!(numbers.iter().map(|number| number.round() as i64).collect::>()) } else { json!(numbers) } } _ => return None, }; - Some(json!({"t": t, "v": payload})) + + Some(json!({"t": parameter_type, "v": payload})) } /// `overrides` checked against the link's parameters. @@ -144,7 +165,8 @@ fn preset(s: &mut Session, p: &Value) -> Result { let name = strp(p, "name").ok_or_else(|| bad(CMD, "`name` is required"))?; let link = link_of(s, CMD, id)?; let values = link.presets.iter().find(|pr| pr.name == name).ok_or_else(|| bad(CMD, format!("`{}` has no preset `{name}`", link.name)))?.values.clone(); - // Presets may name inputs the painter can't set; those stay as cooked. + + // Leave values the painter cannot set unchanged let values = values.into_iter().filter(|(k, _)| link.params.iter().any(|p| p.id == *k)); update(s, CMD, id, values, boolp(p, "cook").unwrap_or(true)) } @@ -156,6 +178,7 @@ fn update(s: &mut Session, cmd: &str, id: LayerId, overrides: impl IntoIterator< if cook_now { return cook(s, id, values); } + s.edit(EDIT, |s| { if let LayerKind::Fill(Fill { link: Some(link), .. }) = &mut s.doc.project.layer_mut(id)?.kind { for p in &mut link.params { @@ -171,8 +194,8 @@ fn update(s: &mut Session, cmd: &str, id: LayerId, overrides: impl IntoIterator< fn cook(s: &mut Session, id: LayerId, overrides: Map) -> Result { let job = s.link_job(id, &overrides)?; - let bytes = job.run()?; - s.apply_link_cook(id, &bytes, false)?; + let material = job.run()?; + s.apply_link_cook(id, &material, false)?; Ok(json!({"layer": id.0, "cooked": true, "size": job.size})) } @@ -194,33 +217,13 @@ fn save_source(_: &mut Session, _: &Value) -> Result { Err(EngineError::Other("file paths are not available in the browser".into())) } -/// A re-cook of a linked layer, to run anywhere (a worker thread) and hand back to -/// [`Session::apply_link_cook`]. -#[derive(Clone, Debug)] -pub struct LinkJob { - pub layer: LayerId, - pub package: Arc>, - /// The source package extension (`mcraft` or `sbs`). - pub source_ext: &'static str, - /// The named graph in a linked material; direct package imports use its default. - pub graph: Option, - /// Pixels across (the texture set's resolution). - pub size: u32, - /// Every parameter's value, by id, in MaterialCraft's typed form. - pub values: Map, - /// Graph inputs fed from baked mesh maps: identifier → 16-bit PNG. - pub inputs: Vec<(String, Vec)>, -} - -#[cfg(not(target_arch = "wasm32"))] -static NEXT_JOB: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); - /// A MaterialCraft executable: `$`, else next to this program, else `name` on `PATH`. #[cfg(not(target_arch = "wasm32"))] pub fn materialcraft_tool(name: &str, env: &str) -> std::path::PathBuf { if let Some(p) = std::env::var_os(env).filter(|p| !p.is_empty()) { return p.into(); } + let exe = format!("{name}{}", std::env::consts::EXE_SUFFIX); std::env::current_exe().ok().and_then(|e| Some(e.parent()?.join(&exe))).filter(|p| p.is_file()).unwrap_or_else(|| exe.into()) } @@ -235,7 +238,7 @@ fn quiet(program: &std::path::Path) -> std::process::Command { c } -/// Open `package` (a `.mcraft` or `.sbs` file) in the MaterialCraft app (`$MATERIALCRAFT`, next to this +/// Open `package` (a published material or authoring package) in the MaterialCraft app (`$MATERIALCRAFT`, next to this /// program, or on `PATH`). #[cfg(not(target_arch = "wasm32"))] pub fn open_in_materialcraft(package: &std::path::Path) -> Result<()> { @@ -247,170 +250,185 @@ pub fn open_in_materialcraft(package: &std::path::Path) -> Result<()> { /// Source file extensions are chosen from the linked resource name, never an untrusted path. pub fn source_extension(name: &str) -> &'static str { - if name.to_ascii_lowercase().ends_with(".sbs") { "sbs" } else { MCRAFT_EXT } -} - -impl LinkJob { - /// Cook through `materialcraft-cli` (`$MATERIALCRAFT_CLI`, next to this program, or on - /// `PATH`); returns `.mcmat` bytes. Blocks for the cook (about a second at 2K). - #[cfg(not(target_arch = "wasm32"))] - pub fn run(&self) -> Result> { - self.run_with_cli(&materialcraft_tool("materialcraft-cli", "MATERIALCRAFT_CLI")) - } - - #[cfg(not(target_arch = "wasm32"))] - fn run_with_cli(&self, cli: &std::path::Path) -> Result> { - let other = |e: String| EngineError::Other(e); - let n = NEXT_JOB.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let dir = std::env::temp_dir().join(format!("texturecraft-link-{}-{n}", std::process::id())); - std::fs::create_dir_all(&dir).map_err(|e| other(format!("{}: {e}", dir.display())))?; - let result = (|| { - let utf8 = |p: &std::path::Path| p.to_str().map(str::to_string).ok_or_else(|| other(format!("{} is not a UTF-8 path", p.display()))); - let write = |name: &str, bytes: &[u8]| { - let p = dir.join(name); - std::fs::write(&p, bytes).map_err(|e| other(format!("{name}: {e}")))?; - utf8(&p) - }; - let ext = if self.source_ext == "sbs" { "sbs" } else { MCRAFT_EXT }; - let src = write(&format!("source.{ext}"), &self.package)?; - let mut inputs = Map::new(); - for (i, (id, png)) in self.inputs.iter().enumerate() { - inputs.insert(id.clone(), json!(write(&format!("input{i}.png"), png)?)); - } - let out = dir.join(format!("out.{MCMAT_EXT}")); - let params = json!({"path": utf8(&out)?, "size": self.size, "values": self.values, "inputs": inputs}); - let mut command = quiet(cli); - if let Some(graph) = &self.graph { - // Opening the source starts on its first graph; select the graph saved in the link. - let script = json!({"open": src, "steps": [ - {"id": "graph.open", "params": {"graph": graph}}, - {"id": "file.exportMaterial", "params": params}, - ]}); - let script_path = write("cook.json", script.to_string().as_bytes())?; - command.arg("batch").arg(script_path); - } else { - command.arg("run").arg(&src).arg("file.exportMaterial").arg(params.to_string()); - } - let run = command.output().map_err(|e| { - other(format!( - "MaterialCraft's command line ({}) did not start: {e}. Install MaterialCraft next to TextureCraft or set MATERIALCRAFT_CLI", - cli.display() - )) - })?; - if !run.status.success() { - return Err(other(format!("MaterialCraft could not cook the material: {}", String::from_utf8_lossy(&run.stderr).trim()))); - } - std::fs::read(&out).map_err(|e| other(format!("{}: {e}", out.display()))) - })(); - let _ = std::fs::remove_dir_all(&dir); - result - } - - #[cfg(target_arch = "wasm32")] - pub fn run(&self) -> Result> { - Err(EngineError::Other("re-cooking a MaterialCraft material needs its command line, which the browser can't run".into())) + if name.to_ascii_lowercase().ends_with(".mcmat") { + "mcmat" + } else if name.to_ascii_lowercase().ends_with(".sbs") { + "sbs" + } else { + MCRAFT_EXT } } -/// Most texels one map may have: 8192 x 8192, the largest texture set. Decoded as RGBA f32 that is -/// 1 GiB, so without this a single 16384 x 16384 map (a tiny PNG) would decode to 4 GiB. -const MAX_MAP_TEXELS: u64 = texturecraft_formats::MAX_RESOURCE_TEXELS; -/// Most texels all of a material's maps may decode to together: nine 4096 x 4096 images (151 M texels). -/// A decoded texel is an RGBA `f32` (16 bytes) and the decoded maps stay in the session, so this holds one -/// `.mcmat` to about 2.25 GiB of pixels however small the file is (a blank 8192 x 8192 PNG is a few hundred -/// KiB). This is a per-import memory policy, not the number of supported channels. It fits nine 4K maps, -/// or an 8K base colour with five 4K maps; larger imports are refused, never shrunk silently. -const MAX_DECODED_TEXELS: u64 = 9 * 4096 * 4096; +/// Prepare the first bake without borrowing the paint document. Its original archive is kept for future cooks. +fn cook_source( + bytes: Vec, + size: u32, + baked: Option, + mut budget: LinkMemoryBudget, + canceled: Arc, + reservation: CookReservation, +) -> Result<(LinkJob, CookedMaterial)> { + check_source_size(bytes.len())?; + budget.reserve(bytes.capacity() as u64)?; -/// Decode every map of `m` (validating them) with the channel it feeds. A usage listed more than -/// once is decoded once (the last listing wins, as it did when each listing was painted over the -/// one before), and the maps together stay within [`MAX_DECODED_TEXELS`], so a `.mcmat` can't -/// multiply one small image into gigabytes of pixels. -fn decoded_maps(m: &PortableMaterial) -> Result, texturecraft_formats::DecodedImage)>> { - decoded_maps_within(m, MAX_DECODED_TEXELS, MAX_MAP_TEXELS) + let source_ext = if is_mcmat(&bytes) { + validate_mcmat_version(&bytes, budget.remaining()).map_err(|error| EngineError::Other(error.to_string()))?; + "mcmat" + } else if is_mcraft(&bytes) { + MCRAFT_EXT + } else { + return Err(EngineError::Other("expected a MaterialCraft material or project archive".into())); + }; + + let job = LinkJob { + layer: LayerId(0), + package: Arc::new(bytes), + source_ext, + graph: None, + size, + values: Map::new(), + input_identifiers: Vec::new(), + baked, + budget, + canceled, + }; + let material = job.run_reserved(reservation)?; + Ok((job, material)) } -/// Decode the maps of `m` (one per usage, the last listing wins) within a total budget of `max_texels`, -/// each map at most `max_map_texels`. The size of each map is checked from its header before decoding: a -/// map over its limit, or one that no longer fits in what is left of the budget, is refused with an error -/// that names the map. -fn decoded_maps_within(m: &PortableMaterial, max_texels: u64, max_map_texels: u64) -> Result, texturecraft_formats::DecodedImage)>> { - let listed: Vec<(Channel, &Vec)> = - m.maps.iter().filter_map(|(usage, bytes)| Some((Channel::ALL.into_iter().find(|c| c.id() == usage)?, bytes))).collect(); - let mut left = max_texels; - let mut out = Vec::new(); - for (i, (c, bytes)) in listed.iter().enumerate() { - if listed.iter().skip(i + 1).any(|(later, _)| later == c) { - continue; - } - let img = texturecraft_formats::decode_image_within(bytes, left.min(max_map_texels)).map_err(|e| EngineError::Other(format!("{} map: {e}", c.id())))?; - left = left.saturating_sub(u64::from(img.width) * u64::from(img.height)); - out.push((*c, (*bytes).clone(), img)); +fn check_source_size(bytes: usize) -> Result<()> { + let limit = crate::resource_cmds::MAX_BYTES; + if bytes > limit { + return Err(EngineError::Other(format!("linked source exceeds the project resource limit of {} MiB", limit >> 20))); } - Ok(out) + + Ok(()) +} + +/// File extensions for initial linked-material imports. +#[cfg(not(target_arch = "wasm32"))] +pub fn is_link_file(path: &std::path::Path) -> bool { + path.extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| extension.eq_ignore_ascii_case("mcmat") || extension.eq_ignore_ascii_case(MCRAFT_EXT)) +} + +#[cfg(not(target_arch = "wasm32"))] +pub fn cook_link_file( + path: &std::path::Path, + size: u32, + baked: Option, + budget: LinkMemoryBudget, + canceled: Arc, +) -> Result<(LinkJob, CookedMaterial)> { + let reservation = CookReservation::for_cook().map_err(|error| EngineError::Other(error.to_string()))?; + let metadata = std::fs::metadata(path).map_err(|error| bad("material.load", format!("cannot read {}: {error}", path.display())))?; + budget.check(metadata.len())?; + + let bytes = crate::resource_cmds::read_resource_file("material.load", path, budget.remaining().min(crate::resource_cmds::MAX_BYTES as u64))?; + cook_source(bytes, size, baked, budget, canceled, reservation) } impl Session { /// Add a MaterialCraft material (`.mcmat` bytes, or a `.mcraft` package cooked first) as a /// fill layer at `place` (one undo step). Returns the new layer. pub fn add_linked_material_bytes_at(&mut self, bytes: &[u8], place: Place) -> Result { - let cooked; - let bytes = if is_mcraft(bytes) { - let ts = self.ts(); - let size = self.doc.project.texture_set(ts)?.resolution; - let job = LinkJob { - layer: LayerId(0), - package: Arc::new(bytes.to_vec()), - source_ext: MCRAFT_EXT, - graph: None, - size, - values: Map::new(), - inputs: Vec::new(), - }; - cooked = job.run()?; - cooked.as_slice() - } else { - bytes - }; - let m = read_mcmat(bytes).map_err(|e| EngineError::Other(e.to_string()))?; - let maps = decoded_maps(&m)?; - if maps.is_empty() { - return Err(EngineError::Other(format!("{} has no maps TextureCraft can paint with", m.manifest.name))); + check_source_size(bytes.len())?; + let ts = self.ts(); + let size = self.doc.project.texture_set(ts)?.resolution; + let reservation = CookReservation::for_cook().map_err(|error| EngineError::Other(error.to_string()))?; + let budget = self.link_memory_budget(); + budget.check(bytes.len() as u64)?; + + let mut source = Vec::new(); + source.try_reserve_exact(bytes.len()).map_err(|error| EngineError::Other(format!("cannot allocate linked source: {error}")))?; + source.extend_from_slice(bytes); + let (job, material) = cook_source(source, size, self.doc.baked.get(&ts).cloned(), budget, Arc::default(), reservation)?; + self.add_cooked_material(&material, job.package, job.source_ext, place) + } + + /// Install validated CLI output in one undo step. Keep the original source for future variations. + pub fn add_cooked_material(&mut self, m: &CookedMaterial, source: Arc>, source_ext: &str, place: Place) -> Result { + self.add_cooked_material_to_set(m, source, source_ext, self.ts(), place) + } + + /// Install a background import in the texture set selected when it started. + pub fn add_cooked_material_to_set(&mut self, m: &CookedMaterial, source: Arc>, source_ext: &str, ts: usize, place: Place) -> Result { + check_source_size(source.len())?; + self.link_memory_budget().check(material_bytes(m).saturating_add(source.capacity() as u64))?; + if m.maps.is_empty() { + return Err(EngineError::Other(format!("{} has no maps TextureCraft can paint with", m.interface.name))); } - let label = format!("Add {}", m.manifest.name); + let label = format!("Add {}", m.interface.name); let (id, names) = self.edit(&label, |s| { - let ts = s.ts(); - let stem = texturecraft_formats::sanitize_resource_name(&m.manifest.name); - let package = unique(s, &format!("{stem}.{}", source_extension(&m.manifest.source))); - s.doc.resources.insert(package.clone(), Arc::new(m.source.clone())); + let stem = texturecraft_formats::sanitize_resource_name(&m.interface.name); + let ext = match source_ext { + "mcmat" | "sbs" | "mcraft" => source_ext, + _ => return Err(EngineError::Other("unknown linked source format".into())), + }; + let package = unique(s, &stem, &format!(".{ext}")); + s.doc.resources.insert(package.clone(), source); let link = MaterialLink { - name: m.manifest.name.clone(), + name: m.interface.name.clone(), package, - graph: m.manifest.graph.clone(), - params: m.manifest.parameters.iter().filter(|p| components(type_of(&p.default)).is_some()).cloned().collect(), - presets: m.manifest.presets.clone(), - inputs: m.manifest.inputs.iter().map(|i| i.identifier.clone()).collect(), - size: m.manifest.size[0], + graph: m.interface.graph.clone(), + params: m.interface.parameters.iter().filter(|p| components(type_of(&p.default)).is_some()).cloned().collect(), + presets: m.interface.presets.clone(), + inputs: m.interface.inputs.iter().map(|i| i.identifier.clone()).collect(), + size: m.size, }; let mut fill = Fill { link: Some(Box::new(link)), ..Fill::default() }; - let names = put_maps(s, &stem, &maps, &mut fill); + let names = put_maps(s, &stem, &m.maps, &mut fill); for c in fill.sources.keys() { crate::set_cmds::add_channel(s, ts, *c)?; } let channels: Vec = fill.sources.keys().copied().collect(); let lid = s.doc.project.alloc_id(); - let name = s.doc.project.next_layer_name(ts, &m.manifest.name); - let lid = crate::layer_cmds::insert_selected(s, Layer::new(lid, name, LayerKind::Fill(fill), &channels))?; + let name = s.doc.project.next_layer_name(ts, &m.interface.name); + let layer = Layer::new(lid, name, LayerKind::Fill(fill), &channels); + let lid = if ts == s.ts() { crate::layer_cmds::insert_selected(s, layer)? } else { s.doc.project.insert_layer(ts, layer, None, false)? }; place.apply(s, lid)?; - s.select_one(ts, lid); - s.target = crate::session::StackTarget::Content; + if ts == s.ts() { + s.select_one(ts, lid); + s.target = crate::session::StackTarget::Content; + } Ok((lid, names)) })?; - self.cache_decoded(names, maps); + self.cache_decoded(names, &m.maps); Ok(id) } + /// Account for existing encoded resources and decoded images when admitting a linked bake. + pub fn link_memory_budget(&self) -> LinkMemoryBudget { + #[cfg(target_arch = "wasm32")] + let cap_bytes = u64::MAX; + #[cfg(not(target_arch = "wasm32"))] + let cap_bytes = texturecraft_formats::configured_ram_budget(); + let mut budget = LinkMemoryBudget::new(cap_bytes); + let mut resources = std::collections::HashSet::new(); + let mut mesh_maps = std::collections::HashSet::new(); + let documents = std::iter::once(&self.doc).chain(self.undo.iter().chain(&self.redo).map(|(_, document)| document)); + let mut retained_bytes = 0u64; + for document in documents { + for bytes in document.resources.values().filter(|bytes| resources.insert(Arc::as_ptr(bytes))) { + retained_bytes = retained_bytes.saturating_add(bytes.capacity() as u64); + } + + for data in document.baked.values().flat_map(|baked| baked.maps.values()).filter(|data| mesh_maps.insert(Arc::as_ptr(data))) { + retained_bytes = retained_bytes.saturating_add(data.capacity() as u64 * std::mem::size_of::() as u64); + } + } + + let mut images = std::collections::HashSet::new(); + for image in self.images.values().filter(|image| images.insert(Arc::as_ptr(image))) { + retained_bytes = retained_bytes.saturating_add(image.rgba.capacity() as u64 * std::mem::size_of::<[f32; 4]>() as u64); + } + + budget.set_retained(retained_bytes); + budget + } + /// The source package behind linked layer `id`. pub fn link_package(&self, id: LayerId) -> Result>> { let link = link_of(self, "material.link", id)?; @@ -429,17 +447,6 @@ impl Session { let mut values = self.link_values(id)?; values.extend(overrides.clone()); - let mut inputs = Vec::new(); - if let Some(baked) = self.doc.baked.get(&texture_set) { - for identifier in &link.inputs { - let Some((map, data)) = BakedMap::parse(identifier).and_then(|map| Some((map, baked.maps.get(&map)?))) else { continue }; - - let png = texturecraft_formats::mcmat::mesh_map_png(baked.resolution, map.components(), data) - .map_err(|error| EngineError::Other(error.to_string()))?; - inputs.push((identifier.clone(), png)); - } - } - // UDIM tiles share one cook at the texture set's base resolution let size = self.doc.project.texture_set(texture_set)?.resolution; Ok(LinkJob { @@ -449,17 +456,18 @@ impl Session { graph: Some(link.graph.clone()), size, values, - inputs, + input_identifiers: link.inputs, + baked: self.doc.baked.get(&texture_set).cloned(), + budget: self.link_memory_budget(), + canceled: Arc::default(), }) } - /// Swap in maps a [`LinkJob`] cooked (`.mcmat` bytes) and the values they were cooked with. + /// Install the maps and exposed values returned by a completed [`LinkJob`]. /// With `merge`, this folds into the previous undo step when that was an edit of the same kind /// (a parameter set while the cook ran). - pub fn apply_link_cook(&mut self, id: LayerId, bytes: &[u8], merge: bool) -> Result<()> { - let material = read_mcmat(bytes).map_err(|error| EngineError::Other(error.to_string()))?; - let maps = decoded_maps(&material)?; - + pub fn apply_link_cook(&mut self, id: LayerId, material: &CookedMaterial, merge: bool) -> Result<()> { + self.link_memory_budget().check(material_bytes(material))?; let names = self.edit_merging(EDIT, merge, |session| { let LayerKind::Fill(mut fill) = session.doc.project.layer(id)?.kind.clone() else { return Err(EngineError::Other(format!("layer {} is not a fill", id.0))); @@ -467,24 +475,25 @@ impl Session { let link = fill.link.as_mut().ok_or_else(|| EngineError::Other(format!("layer {} is not a MaterialCraft material", id.0)))?; for parameter in &mut link.params { - if let Some(cooked) = material.manifest.parameters.iter().find(|candidate| candidate.id == parameter.id) { + if let Some(cooked) = material.interface.parameters.iter().find(|candidate| candidate.id == parameter.id) { parameter.value = cooked.value.clone(); } } - link.size = material.manifest.size[0]; + link.size = material.size; let stem = texturecraft_formats::sanitize_resource_name(&link.name); let old_names: Vec = fill.sources.values().filter_map(|source| if let Source::Image { name, .. } = source { Some(name.clone()) } else { None }).collect(); - let names = put_maps(session, &stem, &maps, &mut fill); + let names = put_maps(session, &stem, &material.maps, &mut fill); session.doc.project.layer_mut(id)?.kind = LayerKind::Fill(fill); // Duplicated layers may still use the earlier maps let mut used_names = Vec::new(); - for set in 0..session.doc.project.texture_sets.len() { - for root in session.doc.project.texture_sets.get(set).map(|texture_set| texture_set.stack.clone()).unwrap_or_default() { + for texture_set in &session.doc.project.texture_sets { + for &root in &texture_set.stack { crate::material_lib::images_used(&session.doc.project, root, &mut used_names, 0); } } + for name in old_names.iter().filter(|name| !used_names.contains(name)) { session.doc.resources.remove(name); } @@ -493,184 +502,66 @@ impl Session { })?; // Reuse the validated maps before synchronization considers decoding missing images - self.cache_decoded(names, maps); + self.cache_decoded(names, &material.maps); self.sync_images(); Ok(()) } /// Keep the images decoded while validating maps instead of decoding them again. - fn cache_decoded(&mut self, names: Vec, maps: Vec<(Channel, Vec, texturecraft_formats::DecodedImage)>) { - for (name, (_, _, img)) in names.into_iter().zip(maps) { - self.images.insert(name, crate::resource_cmds::decoded(img)); + fn cache_decoded(&mut self, names: Vec, maps: &[CookedMap]) { + for (name, map) in names.into_iter().zip(maps) { + self.images.insert(name, Arc::clone(&map.image)); } } } -/// A resource name not in use: `name`, else `2_name`, `3_name`… -fn unique(s: &Session, name: &str) -> String { - (1..).map(|n| if n == 1 { name.to_string() } else { format!("{n}_{name}") }).find(|n| !s.doc.resources.contains_key(n)).unwrap_or_else(|| name.to_string()) +fn material_bytes(material: &CookedMaterial) -> u64 { + material.maps.iter().fold(0u64, |total, map| { + total.saturating_add(map.bytes.capacity() as u64).saturating_add(map.image.rgba.capacity() as u64 * std::mem::size_of::<[f32; 4]>() as u64) + }) +} + +/// A resource name that preserves its suffix through a project round trip. +fn unique(session: &Session, stem: &str, suffix: &str) -> String { + for number in 1.. { + let prefix = if number == 1 { String::new() } else { format!("{number}_") }; + let stem_limit = texturecraft_formats::MAX_RESOURCE_NAME_LEN.saturating_sub(prefix.len() + suffix.len()); + let stem = stem.get(..stem.len().min(stem_limit)).unwrap_or(stem); + let name = format!("{prefix}{stem}{suffix}"); + if !session.doc.resources.contains_key(&name) { + return name; + } + } + + format!("{stem}{suffix}") } /// Store `maps` as new resources (names carry a fresh id, so the decoded-image cache never /// confuses two cooks) and point `fill`'s channels at them. Returns the names, in map order. -fn put_maps(s: &mut Session, stem: &str, maps: &[(Channel, Vec, texturecraft_formats::DecodedImage)], fill: &mut Fill) -> Vec { +fn put_maps(s: &mut Session, stem: &str, maps: &[CookedMap], fill: &mut Fill) -> Vec { let cook = s.doc.project.alloc_id().0; let mut names = Vec::new(); - for (c, bytes, _) in maps { - let name = unique(s, &format!("{stem}_{}_{cook}.png", c.id())); - s.doc.resources.insert(name.clone(), Arc::new(bytes.clone())); - // A normal map the user marked DirectX stays marked through re-cooks. - let normal_format = match fill.sources.get(c) { + for map in maps { + let channel = map.channel; + let extension = if map.bytes.starts_with(crate::resource_cmds::EXR_SIGNATURE) { "exr" } else { "png" }; + let name = unique(s, stem, &format!("_{}_{cook}.{extension}", channel.id())); + s.doc.resources.insert(name.clone(), Arc::clone(&map.bytes)); + + // Preserve a normal convention the user chose for this layer + let normal_format = match fill.sources.get(&channel) { Some(Source::Image { normal_format, .. }) => *normal_format, _ => Default::default(), }; - fill.sources.insert(*c, Source::Image { name: name.clone(), normal_format }); + fill.sources.insert(channel, Source::Image { name: name.clone(), normal_format }); names.push(name); } + names } -#[cfg(all(test, unix))] -mod subprocess_tests { - use super::*; - use std::os::unix::fs::PermissionsExt; - - #[test] - fn cook_uses_the_saved_graph_and_source_type_without_global_environment() { - let id = NEXT_JOB.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let dir = std::env::temp_dir().join(format!("texturecraft-cook-test-{}-{id}", std::process::id())); - std::fs::create_dir_all(&dir).unwrap(); - let cli = dir.join("materialcraft-cli"); - let script = r#"#!/bin/sh -set -eu -case "$1" in - batch) - grep -F '"graph":"Second"' "$2" >/dev/null - grep -F '"id":"file.exportMaterial"' "$2" >/dev/null - test -f "$(dirname "$2")/source.sbs" - test -f "$(dirname "$2")/input0.png" - cp "$(dirname "$0")/result.mcmat" "$(dirname "$2")/out.mcmat" - ;; - run) - test "$(basename "$2")" = "source.mcraft" - cp "$(dirname "$0")/result.mcmat" "$(dirname "$2")/out.mcmat" - ;; - *) exit 1 ;; -esac -"#; - std::fs::write(&cli, script).unwrap(); - std::fs::set_permissions(&cli, std::fs::Permissions::from_mode(0o700)).unwrap(); - std::fs::write(dir.join("result.mcmat"), b"cooked").unwrap(); - let mut job = LinkJob { - layer: LayerId(1), - package: Arc::new(b"source".to_vec()), - source_ext: "sbs", - graph: Some("Second".into()), - size: 64, - values: Map::new(), - inputs: vec![("curvature".into(), b"png".to_vec())], - }; - assert_eq!(job.run_with_cli(&cli).unwrap(), b"cooked"); - job.graph = None; - job.source_ext = "mcraft"; - assert_eq!(job.run_with_cli(&cli).unwrap(), b"cooked"); - job.graph = Some("wrong".into()); - assert!(job.run_with_cli(&cli).is_err()); - assert!(job.run_with_cli(&dir.join("missing")).is_err()); - let _ = std::fs::remove_dir_all(dir); - } -} - #[cfg(test)] mod tests { - use super::*; - use texturecraft_formats::mcmat::{Manifest, MapEntry}; - - /// A blank `side` x `side` RGBA PNG, as bytes. - fn png(side: u32) -> Vec { - let mut out = Vec::new(); - image::RgbaImage::new(side, side).write_to(&mut std::io::Cursor::new(&mut out), image::ImageFormat::Png).unwrap(); - out - } - - /// A portable material that lists the given (usage, image bytes) maps in order. - fn material(maps: Vec<(&str, Vec)>) -> PortableMaterial { - let manifest = Manifest { - format: "craft-material".into(), - version: 1, - name: "M".into(), - graph: "M".into(), - size: [4, 4], - maps: maps.iter().map(|(usage, _)| MapEntry { usage: usage.to_string(), file: "m.png".into() }).collect(), - parameters: vec![], - inputs: vec![], - presets: vec![], - source: "source.mcraft".into(), - }; - PortableMaterial { manifest, maps: maps.into_iter().map(|(u, b)| (u.to_string(), b)).collect(), source: vec![] } - } - - /// However many times one usage is listed, it is decoded only once. - #[test] - fn a_usage_listed_many_times_is_decoded_once() { - let m = material((0..64).map(|_| ("baseColor", png(4))).collect()); - assert_eq!(decoded_maps(&m).unwrap().len(), 1, "64 listings of one usage, one decoded image"); - } - - /// When a usage is listed twice, the later image is the one that is kept. - #[test] - fn the_last_listing_of_a_usage_wins() { - let m = material(vec![("baseColor", png(4)), ("roughness", png(2)), ("baseColor", png(8))]); - let maps = decoded_maps(&m).unwrap(); - let base = maps.iter().find(|(c, ..)| *c == Channel::BaseColor).unwrap(); - assert_eq!(base.2.width, 8); - assert_eq!(maps.len(), 2); - } - - /// A single map over the per-map limit is refused even when the total budget would allow it; one exactly at it is kept. - #[test] - fn one_map_over_the_per_map_limit_is_refused_though_the_total_would_allow_it() { - let m = material(vec![("baseColor", png(5))]); - assert!(decoded_maps_within(&m, 1000, 24).is_err(), "25 texels over a per-map limit of 24, total 1000"); - assert_eq!(decoded_maps_within(&m, 1000, 25).unwrap().len(), 1, "exactly at the per-map limit"); - // The real limits: one map is at most 8192 x 8192, and the total holds nine maps at 4096 x 4096. - assert_eq!(MAX_MAP_TEXELS, 8192 * 8192); - assert_eq!(MAX_DECODED_TEXELS, 9 * 4096 * 4096); - } - - /// The maps together stay within the total budget, exactly: three 8 x 8 maps fit in 192 texels but not in 191. - #[test] - fn decoded_pixels_are_bounded_across_maps() { - let m = material(vec![("baseColor", png(8)), ("roughness", png(8)), ("metallic", png(8))]); - assert_eq!(decoded_maps_within(&m, 192, 64).unwrap().len(), 3, "at the budget"); - assert!(decoded_maps_within(&m, 191, 64).is_err(), "one texel under: the third map goes over"); - } - /// Shared version-2/SBS parsing retains its behavior before duplicate decode filtering. - #[test] - fn shared_version_two_archive_keeps_sbs_and_decodes_repeated_usage_once() { - use std::io::Write; - - let manifest = serde_json::json!({ - "format": "craft-material", "version": 2, "name": "M", "graph": "Second", "size": [4, 4], - "maps": (0..64).map(|_| serde_json::json!({"usage": "baseColor", "file": "maps/base.png"})).collect::>(), - "source": "source.sbs" - }); - let mut archive = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); - for (name, bytes) in [("material.json", manifest.to_string().into_bytes()), ("maps/base.png", png(4)), ("source.sbs", b"source package".to_vec())] { - archive.start_file(name, zip::write::SimpleFileOptions::default()).unwrap(); - archive.write_all(&bytes).unwrap(); - } - let material = read_mcmat(&archive.finish().unwrap().into_inner()).unwrap(); - assert_eq!(material.manifest.version, 2); - assert_eq!(material.manifest.graph, "Second"); - assert_eq!(material.manifest.source, "source.sbs"); - assert_eq!(material.source, b"source package"); - let maps = decoded_maps(&material).unwrap(); - assert_eq!(maps.len(), 1); - assert_eq!((maps[0].2.width, maps[0].2.height), (4, 4)); - } - use super::typed; use serde_json::json; use texturecraft_core::layer::LinkParam; diff --git a/crates/engine/src/link_cmds/cook_report.rs b/crates/engine/src/link_cmds/cook_report.rs new file mode 100644 index 00000000..86b8fbad --- /dev/null +++ b/crates/engine/src/link_cmds/cook_report.rs @@ -0,0 +1,8 @@ +use super::{MaterialInterface, map_file::MapFile}; +use serde::Deserialize; + +#[derive(Deserialize)] +pub(super) struct CookReport { + pub(super) material: MaterialInterface, + pub(super) maps: Vec, +} diff --git a/crates/engine/src/link_cmds/cooked_map.rs b/crates/engine/src/link_cmds/cooked_map.rs new file mode 100644 index 00000000..e7c8f439 --- /dev/null +++ b/crates/engine/src/link_cmds/cooked_map.rs @@ -0,0 +1,70 @@ +use std::sync::Arc; + +use texturecraft_compose::ImageRes; +use texturecraft_core::Channel; + +use super::LinkMemoryBudget; +use crate::{EngineError, Result}; + +/// A validated bake ready to install in the document and decoded-image cache. +#[derive(Clone, Debug)] +pub struct CookedMap { + pub channel: Channel, + pub bytes: Arc>, + pub image: Arc, +} + +impl CookedMap { + pub(super) fn decoded_bytes(texels: u64) -> u64 { + texels.saturating_mul(std::mem::size_of::<[f32; 4]>() as u64) + } + + pub(super) fn decoder_peak_bytes(texels: u64) -> u64 { + // Native image samples can coexist with the linear-sized float output during expansion + Self::decoded_bytes(texels).saturating_mul(2) + } + + pub fn decode(channel: Channel, bytes: Vec, size: [u32; 2], budget: &mut LinkMemoryBudget) -> Result { + let texels = u64::from(size[0]) * u64::from(size[1]); + if texels > texturecraft_formats::MAX_RESOURCE_TEXELS { + return Err(EngineError::Other(format!("{} map exceeds the project resource limit", channel.id()))); + } + + let encoded_bytes = bytes.capacity() as u64; + let decoder_peak_bytes = Self::decoder_peak_bytes(texels); + budget.check(encoded_bytes.saturating_add(decoder_peak_bytes))?; + #[cfg(not(target_arch = "wasm32"))] + budget.check_available(decoder_peak_bytes)?; + + let decoder_bytes = budget.remaining().saturating_sub(encoded_bytes).saturating_sub(Self::decoded_bytes(texels)); + let image = texturecraft_formats::decode_cooked_image(&bytes, size, decoder_bytes) + .map_err(|error| EngineError::Other(format!("{} map: {error}", channel.id())))?; + if [image.width, image.height] != size { + return Err(EngineError::Other(format!("{} map dimensions disagree with the export report", channel.id()))); + } + + budget.reserve(encoded_bytes.saturating_add(Self::decoded_bytes(image.rgba.capacity() as u64)))?; + Ok(Self { channel, bytes: Arc::new(bytes), image: crate::resource_cmds::decoded(image) }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn each_retained_map_reduces_the_next_decoders_budget() { + let bytes = texturecraft_formats::gray16_png(4, 4, [0.5; 16].into_iter()).unwrap(); + let copy = bytes.clone(); + let size = [4, 4]; + let first_bytes = copy.capacity() as u64 + CookedMap::decoded_bytes(16); + let second_peak = bytes.capacity() as u64 + CookedMap::decoder_peak_bytes(16); + let mut budget = LinkMemoryBudget::new(first_bytes + second_peak - 1); + + let first = CookedMap::decode(Channel::Roughness, copy, size, &mut budget).unwrap(); + let second = CookedMap::decode(Channel::Metallic, bytes, size, &mut budget); + + assert_eq!(first.image.rgba.len(), 16); + assert!(second.unwrap_err().to_string().contains("configured cap")); + } +} diff --git a/crates/engine/src/link_cmds/cooked_material.rs b/crates/engine/src/link_cmds/cooked_material.rs new file mode 100644 index 00000000..a2b437f4 --- /dev/null +++ b/crates/engine/src/link_cmds/cooked_material.rs @@ -0,0 +1,25 @@ +use std::sync::Arc; + +use super::{CookReservation, CookedMap, MaterialInterface}; + +#[derive(Clone, Debug)] +pub struct CookedMaterial { + pub interface: MaterialInterface, + pub maps: Vec, + pub size: u32, + // A queued result still owns its large buffers + _reservation: Option>, +} + +impl CookedMaterial { + /// Assemble decoded maps without holding a CLI worker slot. + pub fn new(interface: MaterialInterface, maps: Vec, size: u32) -> Self { + Self { interface, maps, size, _reservation: None } + } + + #[cfg(not(target_arch = "wasm32"))] + pub(super) fn with_reservation(mut self, reservation: CookReservation) -> Self { + self._reservation = Some(Arc::new(reservation)); + self + } +} diff --git a/crates/engine/src/link_cmds/image_input.rs b/crates/engine/src/link_cmds/image_input.rs new file mode 100644 index 00000000..99cde77b --- /dev/null +++ b/crates/engine/src/link_cmds/image_input.rs @@ -0,0 +1,4 @@ +#[derive(Clone, Debug, serde::Deserialize)] +pub struct ImageInput { + pub identifier: String, +} diff --git a/crates/engine/src/link_cmds/link_job.rs b/crates/engine/src/link_cmds/link_job.rs new file mode 100644 index 00000000..2bfb8595 --- /dev/null +++ b/crates/engine/src/link_cmds/link_job.rs @@ -0,0 +1,308 @@ +use std::sync::{Arc, atomic::AtomicBool}; + +use super::{CookReservation, CookedMaterial, EngineError, LinkMemoryBudget, Result}; +use serde_json::{Map, Value}; +use texturecraft_core::LayerId; + +#[cfg(not(target_arch = "wasm32"))] +use { + super::{CookedMap, MaterialInterface, cook_report::CookReport, materialcraft_tool, quiet}, + std::{io::Read, path::Path}, + texturecraft_core::{BakedMap, Channel}, +}; + +/// A re-cook of a linked layer, to run anywhere (a worker thread) and hand back to +/// [`crate::Session::apply_link_cook`]. +#[derive(Clone, Debug)] +pub struct LinkJob { + pub layer: LayerId, + pub package: Arc>, + /// The source package extension (`mcmat`, `mcraft` or `sbs`). + pub source_ext: &'static str, + /// The named graph in a linked material; direct package imports use its default. + pub graph: Option, + /// Pixels across (the texture set's resolution). + pub size: u32, + /// Every parameter's value, by id, in MaterialCraft's typed form. + pub values: Map, + /// Graph inputs matched to mesh maps and encoded one at a time in the worker. + pub input_identifiers: Vec, + pub(super) baked: Option, + pub(super) budget: LinkMemoryBudget, + pub canceled: Arc, +} + +#[cfg(not(target_arch = "wasm32"))] +fn io(error: impl std::fmt::Display) -> EngineError { + EngineError::Other(error.to_string()) +} + +#[cfg(not(target_arch = "wasm32"))] +pub(super) fn read_bounded(path: &Path, limit: u64) -> Result> { + let metadata = std::fs::symlink_metadata(path).map_err(io)?; + if !metadata.is_file() || metadata.len() > limit { + return Err(io(format!("invalid or oversized cook output: {}", path.display()))); + } + + let mut bytes = Vec::new(); + let capacity = usize::try_from(metadata.len()).map_err(io)?; + bytes.try_reserve_exact(capacity).map_err(io)?; + #[cfg(unix)] + let mut file = { + use rustix::fs::{Mode, OFlags, open}; + + // The generated output must stay a regular file between inspection and open + let descriptor = open(path, OFlags::RDONLY | OFlags::NONBLOCK | OFlags::NOFOLLOW | OFlags::CLOEXEC, Mode::empty()).map_err(io)?; + std::fs::File::from(descriptor) + }; + + #[cfg(not(unix))] + let mut file = std::fs::File::open(path).map_err(io)?; + if !file.metadata().map_err(io)?.is_file() { + return Err(io(format!("invalid cook output: {}", path.display()))); + } + + file.by_ref().take(metadata.len()).read_to_end(&mut bytes).map_err(io)?; + if file.read(&mut [0u8]).map_err(io)? != 0 { + return Err(io("cook output grew while reading")); + } + + if bytes.len() as u64 > limit { + return Err(io("cook output exceeds the remaining budget")); + } + + Ok(bytes) +} + +#[cfg(not(target_arch = "wasm32"))] +const CANCEL_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(20); + +#[cfg(windows)] +#[path = "windows_cook.rs"] +mod windows_cook; + +#[cfg(not(target_arch = "wasm32"))] +fn invoke(command: std::process::Command, dir: &Path, budget: LinkMemoryBudget, canceled: &AtomicBool) -> Result> { + use std::sync::atomic::Ordering; + + const MAX_DIAGNOSTIC_BYTES: u64 = 1 << 20; + const MAX_REPORT_BYTES: u64 = 8 << 20; + if canceled.load(Ordering::Relaxed) { + return Err(io("MaterialCraft cook canceled")); + } + + let stdout = dir.join("reply.json"); + let stderr = dir.join("diagnostic.txt"); + let child_allowance = budget.child_allowance(); + let (success, status) = run_bounded(&command, &stdout, &stderr, child_allowance, canceled)?; + if !success { + let diagnostic = read_bounded(&stderr, MAX_DIAGNOSTIC_BYTES.min(budget.remaining()))?; + return Err(io(format!( + "MaterialCraft cook failed ({status}, {child_allowance} bytes of RAM available within the configured cap): {}", + String::from_utf8_lossy(&diagnostic).trim() + ))); + } + + read_bounded(&stdout, MAX_REPORT_BYTES.min(budget.remaining())) +} + +#[cfg(windows)] +use windows_cook::run_bounded; + +#[cfg(not(any(target_arch = "wasm32", windows)))] +fn run_bounded(command: &std::process::Command, stdout: &Path, stderr: &Path, remaining_bytes: u64, canceled: &AtomicBool) -> Result<(bool, String)> { + use std::sync::atomic::Ordering; + + let mut command = bounded_cook_command(command, remaining_bytes)?; + let mut child = command + .stdout(std::fs::File::create(stdout).map_err(io)?) + .stderr(std::fs::File::create(stderr).map_err(io)?) + .spawn() + .map_err(|error| io(format!("could not start the MaterialCraft cook: {error}. Install MaterialCraft or set MATERIALCRAFT_CLI")))?; + + loop { + if canceled.load(Ordering::Relaxed) { + let _ = child.kill(); + child.wait().map_err(io)?; + return Err(io("MaterialCraft cook canceled")); + } + + match child.try_wait() { + Ok(Some(status)) => return Ok((status.success(), status.to_string())), + Ok(None) => std::thread::sleep(CANCEL_POLL_INTERVAL), + Err(error) => { + let _ = child.kill(); + let _ = child.wait(); + return Err(io(error)); + } + } + } +} + +#[cfg(target_os = "linux")] +fn bounded_cook_command(command: &std::process::Command, remaining_bytes: u64) -> Result { + // Cap address space before exec so a large allocation cannot race the parent + let mut limited = quiet(Path::new("prlimit")); + limited.arg(format!("--as={remaining_bytes}")).arg("--").arg(command.get_program()).args(command.get_args()); + Ok(limited) +} + +#[cfg(any(target_os = "macos", target_os = "freebsd"))] +fn bounded_cook_command(command: &std::process::Command, remaining_bytes: u64) -> Result { + #[cfg(target_os = "macos")] + const SHELL: &str = "/bin/bash"; + #[cfg(target_os = "freebsd")] + const SHELL: &str = "/bin/sh"; + + // The shell applies the process limit before replacing itself with MaterialCraft + let mut limited = quiet(Path::new(SHELL)); + limited + .env_remove("BASH_ENV") + .env_remove("ENV") + .args(["-c", r#"ulimit -v "$1" || exit 1; shift; exec -- "$@""#, "texturecraft-cook"]) + .arg((remaining_bytes / 1024).to_string()) + .arg(command.get_program()) + .args(command.get_args()); + Ok(limited) +} + +#[cfg(not(any(target_arch = "wasm32", target_os = "linux", target_os = "macos", target_os = "freebsd", windows)))] +fn bounded_cook_command(_: &std::process::Command, remaining_bytes: u64) -> Result { + Err(io(format!("This platform cannot enforce the configured MaterialCraft RAM allowance ({remaining_bytes} bytes remaining)"))) +} + +#[cfg(not(target_arch = "wasm32"))] +fn channel_for_usage(usage: &str) -> Option { + if usage == "diffuse" { + return Some(Channel::BaseColor); + } + + Channel::ALL.into_iter().find(|channel| channel.id() == usage) +} + +#[cfg(not(target_arch = "wasm32"))] +impl LinkJob { + pub(super) fn run_reserved(&self, reservation: CookReservation) -> Result { + use std::sync::atomic::Ordering; + + let mut budget = self.budget; + let pixels = u64::from(self.size) * u64::from(self.size); + budget.check(CookedMap::decoded_bytes(pixels))?; + + let cli = materialcraft_tool("materialcraft-cli", "MATERIALCRAFT_CLI"); + let temp = tempfile::Builder::new().prefix("texturecraft-material-").tempdir().map_err(io)?; + let dir = temp.path(); + let ext = match self.source_ext { + "mcmat" | "sbs" | "mcraft" => self.source_ext, + _ => return Err(io("unknown linked source format")), + }; + + let source = dir.join(format!("source.{ext}")); + std::fs::write(&source, &*self.package).map_err(io)?; + + let out = dir.join("maps"); + let mut export = quiet(&cli); + export.arg("export").arg(&source).arg(&out).args(["--profile", "maps", "--size"]).arg(format!("{}x{}", self.size, self.size)); + if self.source_ext != "mcmat" + && let Some(graph) = &self.graph + { + export.arg("--graph-path").arg(graph); + } + + for (id, value) in &self.values { + export.arg("--set").arg(format!("{id}={value}")); + } + + if let Some(bakes) = self.baked.as_ref().filter(|bakes| !bakes.maps.is_empty()) { + let mut identifiers = self.input_identifiers.clone(); + if self.layer == LayerId(0) { + let mut inspect = quiet(&cli); + if ext == "mcmat" { + inspect.arg("info").arg(&source); + } else { + inspect.arg("run").arg(&source).arg("material.info"); + } + + let interface: MaterialInterface = serde_json::from_slice(&invoke(inspect, dir, budget, &self.canceled)?).map_err(io)?; + identifiers = interface.inputs.into_iter().map(|input| input.identifier).collect(); + } + + for (index, identifier) in identifiers.iter().enumerate() { + let Some((map, data)) = BakedMap::parse(identifier).and_then(|map| Some((map, bakes.maps.get(&map)?))) else { + continue; + }; + + let temporary_bytes = texturecraft_formats::mcmat::mesh_map_png_working_bytes(bakes.resolution, map.components()); + budget.check(temporary_bytes)?; + budget.check_available(temporary_bytes)?; + if self.canceled.load(Ordering::Relaxed) { + return Err(io("MaterialCraft cook canceled")); + } + + let file = dir.join(format!("input{index}.png")); + let output = std::fs::File::create(&file).map_err(io)?; + texturecraft_formats::mcmat::write_mesh_map_png(output, bakes.resolution, map.components(), data).map_err(io)?; + export.arg("--input").arg(format!("{identifier}={}", file.display())); + } + } + + let reply: CookReport = serde_json::from_slice(&invoke(export, dir, budget, &self.canceled)?).map_err(io)?; + let mut maps = Vec::::new(); + let mut size = 0; + let has_base_color = reply.maps.iter().any(|map| map.usage == Channel::BaseColor.id()); + for map in reply.maps.into_iter().rev() { + if self.canceled.load(Ordering::Relaxed) { + return Err(io("MaterialCraft cook canceled")); + } + + let Some(channel) = channel_for_usage(&map.usage) else { continue }; + + if (map.usage == "diffuse" && has_base_color) || maps.iter().any(|map| map.channel == channel) { + continue; + } + + let path = Path::new(&map.file); + if path.file_name().and_then(|p| p.to_str()) != Some(&map.file) || map.file.contains(['\\', ':']) { + return Err(io("invalid map filename in MaterialCraft's export report")); + } + + let [width, height] = map.size; + let texels = u64::from(width) * u64::from(height); + if width == 0 || height == 0 || width > self.size || height > self.size || texels > texturecraft_formats::MAX_RESOURCE_TEXELS { + return Err(io(format!( + "{} map dimensions {width}x{height} exceed the requested {} px bake or the project resource limit", + map.usage, self.size + ))); + } + + let file = out.join(path); + let metadata = std::fs::symlink_metadata(&file).map_err(io)?; + let temporary_bytes = metadata.len().saturating_add(CookedMap::decoder_peak_bytes(texels)); + budget.check(temporary_bytes)?; + budget.check_available(metadata.len())?; + let bytes = read_bounded(&file, metadata.len().min(crate::resource_cmds::MAX_BYTES as u64))?; + let decoded = CookedMap::decode(channel, bytes, map.size, &mut budget)?; + size = size.max(width).max(height); + maps.push(decoded); + } + + if maps.is_empty() { + return Err(io("MaterialCraft returned no paintable map usages")); + } + + Ok(CookedMaterial::new(reply.material, maps, size).with_reservation(reservation)) + } +} + +#[cfg(target_arch = "wasm32")] +impl LinkJob { + pub(super) fn run_reserved(&self, _: CookReservation) -> Result { + Err(EngineError::Other("cooking a procedural material requires materialcraft-cli on a desktop".into())) + } +} + +impl LinkJob { + pub fn run(&self) -> Result { + self.run_reserved(CookReservation::for_cook().map_err(|error| EngineError::Other(error.to_string()))?) + } +} diff --git a/crates/engine/src/link_cmds/link_memory_budget.rs b/crates/engine/src/link_cmds/link_memory_budget.rs new file mode 100644 index 00000000..197af25a --- /dev/null +++ b/crates/engine/src/link_cmds/link_memory_budget.rs @@ -0,0 +1,129 @@ +use crate::{EngineError, Result}; + +/// Budget shared by retained link data and all phases of one cook. +#[derive(Clone, Copy, Debug)] +pub struct LinkMemoryBudget { + cap_bytes: u64, + retained_bytes: u64, +} + +impl LinkMemoryBudget { + pub fn new(cap_bytes: u64) -> Self { + Self { cap_bytes, retained_bytes: 0 } + } + + pub(crate) fn reserve(&mut self, bytes: u64) -> Result<()> { + self.retained_bytes = self.checked_total(bytes)?; + Ok(()) + } + + pub(crate) fn check(self, additional_bytes: u64) -> Result<()> { + self.checked_total(additional_bytes).map(|_| ()) + } + + fn checked_total(self, additional_bytes: u64) -> Result { + let required_bytes = u128::from(self.retained_bytes) + u128::from(additional_bytes); + if required_bytes > u128::from(self.cap_bytes) { + return Err(EngineError::Other(format!( + "MaterialCraft cook requires an estimated {required_bytes} bytes of RAM, exceeding the configured cap of {} bytes", + self.cap_bytes + ))); + } + + Ok(required_bytes as u64) + } + + pub(super) fn set_retained(&mut self, bytes: u64) { + self.retained_bytes = bytes; + } + + pub(super) fn remaining(self) -> u64 { + self.cap_bytes.saturating_sub(self.retained_bytes) + } + + #[cfg(not(target_arch = "wasm32"))] + pub(super) fn child_allowance(self) -> u64 { + let allowance = self.remaining().min(texturecraft_formats::available_save_resources().1); + #[cfg(any(target_os = "linux", target_os = "macos", target_os = "freebsd"))] + let allowance = allowance.min(rustix::process::getrlimit(rustix::process::Resource::As).current.unwrap_or(u64::MAX)); + + allowance + } + + #[cfg(not(target_arch = "wasm32"))] + pub(super) fn check_available(self, temporary_bytes: u64) -> Result<()> { + let available_bytes = texturecraft_formats::available_save_resources().1; + #[cfg(any(target_os = "linux", target_os = "macos", target_os = "freebsd"))] + let available_bytes = match rustix::process::getrlimit(rustix::process::Resource::As).current { + // Physical memory can be plentiful while the host has little address space left + Some(limit) => available_bytes.min(limit.saturating_sub(host_virtual_bytes()?)), + None => available_bytes, + }; + + if temporary_bytes > available_bytes { + return Err(EngineError::Other(format!( + "MaterialCraft cook requires an estimated {temporary_bytes} additional bytes of RAM, with {available_bytes} bytes currently available and a configured cap of {} bytes", + self.cap_bytes + ))); + } + + Ok(()) + } +} + +#[cfg(target_os = "linux")] +fn host_virtual_bytes() -> Result { + let status = std::fs::read_to_string("/proc/self/status").map_err(|error| EngineError::Other(format!("cannot check host address space: {error}")))?; + status + .lines() + .find_map(|line| line.strip_prefix("VmSize:")) + .and_then(|value| value.split_whitespace().next()?.parse::().ok()) + .and_then(|kib| kib.checked_mul(1024)) + .ok_or_else(|| EngineError::Other("cannot determine the memory available under the host address-space limit".into())) +} + +#[cfg(any(target_os = "macos", target_os = "freebsd"))] +fn host_virtual_bytes() -> Result { + use sysinfo::{Pid, ProcessRefreshKind, ProcessesToUpdate, System}; + + let mut system = System::new().map_err(|error| EngineError::Other(format!("cannot check host address space: {error}")))?; + let pid = Pid::from_u32(std::process::id()); + system.refresh_processes_specifics(ProcessesToUpdate::Some(&[pid]), false, ProcessRefreshKind::nothing().with_memory()); + system + .process(pid) + .map(|process| process.virtual_memory()) + .filter(|bytes| *bytes != 0) + .ok_or_else(|| EngineError::Other("cannot determine the memory available under the host address-space limit".into())) +} + +#[cfg(all(test, target_os = "linux"))] +mod tests { + use super::*; + + #[test] + fn inherited_address_limit_restricts_host_and_companion() { + const PROCESS_LIMIT_BYTES: u64 = 192 << 20; + const CONSTRAINED_TEST: &str = "TC_LINK_TEST_ADDRESS_LIMIT"; + + // Isolate the inherited limit from unrelated tests in the parent process + if std::env::var_os(CONSTRAINED_TEST).is_none() { + let status = std::process::Command::new("prlimit") + .arg(format!("--as={PROCESS_LIMIT_BYTES}")) + .arg("--") + .arg(std::env::current_exe().unwrap()) + .args(["--exact", std::thread::current().name().unwrap()]) + .env(CONSTRAINED_TEST, "1") + .status() + .unwrap(); + assert!(status.success()); + return; + } + + let budget = LinkMemoryBudget::new(8 << 30); + let error = budget.check_available(PROCESS_LIMIT_BYTES).unwrap_err(); + + assert!(error.to_string().contains("configured cap")); + assert!(budget.child_allowance() <= PROCESS_LIMIT_BYTES); + budget.check_available(1).unwrap(); + } +} diff --git a/crates/engine/src/link_cmds/map_file.rs b/crates/engine/src/link_cmds/map_file.rs new file mode 100644 index 00000000..2e2f40ad --- /dev/null +++ b/crates/engine/src/link_cmds/map_file.rs @@ -0,0 +1,8 @@ +use serde::Deserialize; + +#[derive(Deserialize)] +pub(super) struct MapFile { + pub(super) usage: String, + pub(super) file: String, + pub(super) size: [u32; 2], +} diff --git a/crates/engine/src/link_cmds/material_interface.rs b/crates/engine/src/link_cmds/material_interface.rs new file mode 100644 index 00000000..477bb126 --- /dev/null +++ b/crates/engine/src/link_cmds/material_interface.rs @@ -0,0 +1,11 @@ +use super::ImageInput; +use texturecraft_core::{LinkParam, LinkPreset}; + +#[derive(Clone, Debug, serde::Deserialize)] +pub struct MaterialInterface { + pub name: String, + pub graph: String, + pub parameters: Vec, + pub inputs: Vec, + pub presets: Vec, +} diff --git a/crates/engine/src/link_cmds/windows_cook.rs b/crates/engine/src/link_cmds/windows_cook.rs new file mode 100644 index 00000000..8436f599 --- /dev/null +++ b/crates/engine/src/link_cmds/windows_cook.rs @@ -0,0 +1,43 @@ +use std::path::Path; +use std::process::Command; +use std::sync::atomic::{AtomicBool, Ordering}; + +use processkit::{ProcessGroup, ProcessGroupOptions}; + +use super::{CANCEL_POLL_INTERVAL, EngineError, Result, io}; + +pub(super) fn run_bounded(command: &Command, stdout: &Path, stderr: &Path, remaining_bytes: u64, canceled: &AtomicBool) -> Result<(bool, String)> { + // A synchronous engine command may already be running inside a Tokio context + std::thread::scope(|scope| { + std::thread::Builder::new() + .spawn_scoped(scope, || { + let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().map_err(io)?; + runtime.block_on(async { + let group = ProcessGroup::with_options(ProcessGroupOptions::default().max_memory(remaining_bytes)) + .map_err(|error| io(format!("could not enforce the MaterialCraft RAM allowance of {remaining_bytes} bytes: {error}")))?; + let command = + processkit::Command::new(command.get_program()).args(command.get_args()).create_no_window().stdout_file(stdout).stderr_file(stderr); + let process = group.start(&command).await.map_err(io)?; + let mut wait = std::pin::pin!(process.wait()); + let cancellation = async { + while !canceled.load(Ordering::Relaxed) { + tokio::time::sleep(CANCEL_POLL_INTERVAL).await; + } + }; + let status = tokio::select! { + status = &mut wait => status.map_err(io)?, + _ = cancellation => { + group.kill_all().map_err(io)?; + wait.await.map_err(io)?; + return Err(io("MaterialCraft cook canceled")); + } + }; + + Ok((status.code() == Some(0), format!("{status:?}"))) + }) + }) + .map_err(io)? + .join() + .map_err(|_| EngineError::Other("the MaterialCraft process worker stopped".into()))? + }) +} diff --git a/crates/engine/src/material_lib.rs b/crates/engine/src/material_lib.rs index 10c664d4..5931761e 100644 --- a/crates/engine/src/material_lib.rs +++ b/crates/engine/src/material_lib.rs @@ -81,10 +81,12 @@ impl Session { /// The smart material file holding `mini` (a document from [`one_set_document`]) and the images layer `root` of it uses. pub(crate) fn file_bytes(&self, mini: Document, root: LayerId) -> Result> { + let mut reservation = texturecraft_formats::RamReservation::for_save().map_err(fe)?; let mut used = Vec::new(); images_used(&mini.project, root, &mut used, 0); - let resources = used.iter().filter_map(|n| self.doc.resources.get(n).map(|b| (n.clone(), b.as_ref().clone()))).collect(); - let pf = texturecraft_formats::ProjectFile { + let resources = used.into_iter().filter_map(|name| self.doc.resources.get(&name).map(|bytes| (name, Arc::clone(bytes)))).collect(); + + let snapshot = texturecraft_formats::ProjectFile { project: mini.project, pixels: mini.pixels, baked: BTreeMap::new(), @@ -96,7 +98,7 @@ impl Session { resources, warnings: Vec::new(), }; - texturecraft_formats::save_project_bytes(&pf).map_err(fe) + reservation.save_bytes(&snapshot).map_err(fe) } /// Add a smart material file above the selection (one undo step). Returns the new layer. @@ -111,23 +113,26 @@ impl Session { return self.add_linked_material_bytes_at(bytes, place); } - let pf = texturecraft_formats::load_project_bytes(bytes).map_err(fe)?; - let root = pf + self.add_smart_material_project(texturecraft_formats::load_project_bytes(bytes).map_err(fe)?, place) + } + + fn add_smart_material_project(&mut self, project_file: texturecraft_formats::ProjectFile, place: Place) -> Result { + let root = project_file .project .texture_sets .first() .and_then(|t| t.stack.last().copied()) .ok_or_else(|| EngineError::Other("the smart material file has no layer".into()))?; - let resources: BTreeMap>> = pf.resources.into_iter().map(|(k, v)| (k, Arc::new(v))).collect(); - let src = Document { bake_paint: BTreeMap::new(), project: pf.project, pixels: pf.pixels, baked: BTreeMap::new(), resources: resources.clone() }; + let resources: BTreeMap>> = project_file.resources.into_iter().map(|(k, v)| (k, Arc::new(v))).collect(); + let source = Document { bake_paint: BTreeMap::new(), project: project_file.project, pixels: project_file.pixels, baked: BTreeMap::new(), resources }; let id = self.edit("Add smart material", |s| { - for (k, v) in &resources { - s.doc.resources.entry(k.clone()).or_insert_with(|| v.clone()); + for (name, bytes) in &source.resources { + s.doc.resources.entry(name.clone()).or_insert_with(|| Arc::clone(bytes)); } let ts = s.ts(); let sel = s.selected_layer(); let inside = sel.and_then(|l| s.doc.project.layer(l).ok()).is_some_and(|l| l.is_group()); - let id = crate::layer_ops::copy_tree(&src, root, &mut s.doc, ts, sel, inside, 0)?; + let id = crate::layer_ops::copy_tree(&source, root, &mut s.doc, ts, sel, inside, 0)?; place.apply(s, id)?; s.select_one(ts, id); s.target = StackTarget::Content; @@ -161,8 +166,23 @@ fn save(s: &mut Session, p: &Value) -> Result { #[cfg(not(target_arch = "wasm32"))] fn load(s: &mut Session, p: &Value) -> Result { let path = strp(p, "path").ok_or_else(|| bad("material.load", "`path` is required"))?; - let bytes = std::fs::read(path).map_err(|e| bad("material.load", format!("cannot read {path}: {e}")))?; - let id = s.add_smart_material_bytes_at(&bytes, Place::from_params(p))?; + let source = std::path::Path::new(path); + let place = Place::from_params(p); + let project = if crate::link_cmds::is_link_file(source) { + None + } else { + let file = crate::resource_cmds::open_resource_file("material.load", source)?; + texturecraft_formats::project_file::load_smart_material_project(file).map_err(fe)? + }; + + let id = if let Some(project) = project { + s.add_smart_material_project(project, place)? + } else { + let set = s.ts(); + let resolution = s.doc.project.texture_set(set)?.resolution; + let (source, cooked) = crate::link_cmds::cook_link_file(source, resolution, s.baked(set).cloned(), s.link_memory_budget(), Arc::default())?; + s.add_cooked_material(&cooked, source.package, source.source_ext, place)? + }; Ok(json!({ "layer": id.0 })) } diff --git a/crates/engine/src/materials.rs b/crates/engine/src/materials.rs index b13f03b0..f91efe86 100644 --- a/crates/engine/src/materials.rs +++ b/crates/engine/src/materials.rs @@ -844,8 +844,8 @@ pub fn apply_at(s: &mut Session, id: &str, place: Place) -> Result { } impl Place { - /// From `above` (+ `inside`) or `index` params; the selection by default. - pub(crate) fn from_params(p: &serde_json::Value) -> Place { + /// Parse a drop position from the command parameters, defaulting to the selection. + pub fn from_params(p: &serde_json::Value) -> Place { match (crate::commands::u64p(p, "above"), crate::commands::u64p(p, "below"), crate::commands::u64p(p, "index")) { (Some(above), _, _) => Place::Above(LayerId(above), crate::commands::boolp(p, "inside").unwrap_or(false)), (None, Some(below), _) => Place::Below(LayerId(below)), diff --git a/crates/engine/src/resource_cmds.rs b/crates/engine/src/resource_cmds.rs index 97d0bd9a..aa4647ff 100644 --- a/crates/engine/src/resource_cmds.rs +++ b/crates/engine/src/resource_cmds.rs @@ -39,6 +39,7 @@ pub fn specs() -> Vec { /// Largest accepted resource file. pub(crate) const MAX_BYTES: usize = 512 << 20; +pub(crate) const EXR_SIGNATURE: &[u8] = &[0x76, 0x2f, 0x31, 0x01]; pub(crate) fn decoded(d: texturecraft_formats::DecodedImage) -> Arc { Arc::new(ImageRes { width: d.width, height: d.height, rgba: d.rgba, srgb: d.srgb }) @@ -102,14 +103,34 @@ impl Session { pub fn sync_images(&mut self) { self.images.retain(|name, _| self.doc.resources.contains_key(name)); - // Linked material packages are resources but cannot be decoded as images - let package_suffix = format!(".{}", texturecraft_formats::mcmat::MCRAFT_EXT); + let mut linked_packages = std::collections::HashSet::new(); + let mut linked_images = std::collections::HashMap::new(); + for layer in self.doc.project.layers.values() { + let LayerKind::Fill(Fill { link: Some(link), sources, .. }) = &layer.kind else { continue }; + + linked_packages.insert(link.package.as_str()); + for source in sources.values() { + if let Source::Image { name, .. } = source { + linked_images.entry(name.as_str()).and_modify(|size: &mut u32| *size = (*size).max(link.size)).or_insert(link.size); + } + } + } + for (name, bytes) in &self.doc.resources { - if self.images.contains_key(name) || name.ends_with(&package_suffix) { + if self.images.contains_key(name) || linked_packages.contains(name.as_str()) { continue; } - match texturecraft_formats::decode_image(bytes) { + // EXR may need a full float decode buffer even when encoded project bytes are small + let image = match linked_images.get(name.as_str()) { + Some(&size) if size > 0 && bytes.starts_with(EXR_SIGNATURE) => { + let texels = (u64::from(size) * u64::from(size)).min(texturecraft_formats::MAX_RESOURCE_TEXELS); + let decoder_bytes = texels * std::mem::size_of::<[f32; 4]>() as u64; + texturecraft_formats::decode_cooked_image(bytes, [size; 2], decoder_bytes).or_else(|_| texturecraft_formats::decode_image(bytes)) + } + _ => texturecraft_formats::decode_image(bytes), + }; + match image { Ok(image) => { self.images.insert(name.clone(), decoded(image)); } @@ -143,27 +164,57 @@ fn import(s: &mut Session, p: &Value) -> Result { /// Read a regular image file within both the per-file cap and the caller's remaining byte budget #[cfg(not(target_arch = "wasm32"))] pub(crate) fn read_image_file(cmd: &str, path: &str, max_bytes: u64) -> Result> { - use std::io::Read; + read_resource_file(cmd, std::path::Path::new(path), max_bytes.min(MAX_BYTES as u64)) +} - let max_bytes = max_bytes.min(MAX_BYTES as u64); - let read_error = |error| bad(cmd, format!("cannot read {path}: {error}")); - let too_large = || bad(cmd, format!("{path} is too large (limit {max_bytes} bytes)")); - let metadata = std::fs::metadata(path).map_err(read_error)?; +#[cfg(not(target_arch = "wasm32"))] +pub(crate) fn open_resource_file(cmd: &str, path: &std::path::Path) -> Result { + let display = path.display(); + let read_error = |error| bad(cmd, format!("cannot read {display}: {error}")); // Pipes can block on open, and devices may never reach EOF - if !metadata.is_file() { - return Err(bad(cmd, format!("{path} is not a regular file"))); + if !std::fs::metadata(path).map_err(read_error)?.is_file() { + return Err(bad(cmd, format!("{display} is not a regular file"))); } + #[cfg(unix)] + let file = { + use rustix::fs::{Mode, OFlags, open}; + + // Follow user-selected symlinks without waiting on a substituted pipe + let descriptor = open(path, OFlags::RDONLY | OFlags::NONBLOCK | OFlags::CLOEXEC, Mode::empty()).map_err(|error| read_error(error.into()))?; + std::fs::File::from(descriptor) + }; + #[cfg(not(unix))] + let file = std::fs::File::open(path).map_err(read_error)?; + + if !file.metadata().map_err(read_error)?.is_file() { + return Err(bad(cmd, format!("{display} is not a regular file"))); + } + + Ok(file) +} + +#[cfg(not(target_arch = "wasm32"))] +pub(crate) fn read_resource_file(cmd: &str, path: &std::path::Path, max_bytes: u64) -> Result> { + use std::io::Read; + + let display = path.display(); + let read_error = |error| bad(cmd, format!("cannot read {display}: {error}")); + let too_large = || bad(cmd, format!("{display} is too large (limit {max_bytes} bytes)")); + let mut file = open_resource_file(cmd, path)?; + let metadata = file.metadata().map_err(read_error)?; if metadata.len() > max_bytes { return Err(too_large()); } - // Bound the read even if the file grows after the metadata check - let file = std::fs::File::open(path).map_err(read_error)?; - let mut bytes = Vec::with_capacity(metadata.len() as usize); - file.take(max_bytes + 1).read_to_end(&mut bytes).map_err(read_error)?; - if bytes.len() as u64 > max_bytes { + let mut bytes = Vec::new(); + let capacity = usize::try_from(metadata.len()).map_err(|_| too_large())?; + bytes.try_reserve_exact(capacity).map_err(|error| bad(cmd, format!("cannot allocate {display}: {error}")))?; + + // A growing file must not trigger allocations beyond the checked size + file.by_ref().take(metadata.len()).read_to_end(&mut bytes).map_err(read_error)?; + if file.read(&mut [0u8]).map_err(read_error)? != 0 { return Err(too_large()); } diff --git a/crates/engine/src/session.rs b/crates/engine/src/session.rs index f8ce6c32..12970dc1 100644 --- a/crates/engine/src/session.rs +++ b/crates/engine/src/session.rs @@ -922,6 +922,8 @@ impl Session { } fn after_history(&mut self, other: &Document) { + // Pending operations may still hold layer IDs from the discarded snapshot + self.doc.project.next_id = self.doc.project.next_id.max(other.project.next_id); self.modified = true; // Layers picked together belong to the stack they were picked in diff --git a/crates/engine/tests/it/materialcraft_link.rs b/crates/engine/tests/it/materialcraft_link.rs index cbe75532..ea361a89 100644 --- a/crates/engine/tests/it/materialcraft_link.rs +++ b/crates/engine/tests/it/materialcraft_link.rs @@ -10,6 +10,7 @@ use texturecraft_engine::compose::BakedMaps; use texturecraft_engine::core::{BakedMap, Channel, LayerId, LayerKind, NormalFormat, Source}; use texturecraft_engine::formats::mcmat::mesh_map_png; use texturecraft_engine::geom::primitives; +use texturecraft_engine::link_cmds::{CookedMap, CookedMaterial, LinkMemoryBudget}; use texturecraft_engine::materials::Place; use texturecraft_engine::{Session, execute}; @@ -19,21 +20,11 @@ fn session() -> Session { s } -/// A `.mcmat` as MaterialCraft writes it: uniform maps, one float input `wear` cooked at `wear`. -fn mcmat(roughness: f32, wear: f32) -> Vec { - mcmat_version(roughness, wear, 1) -} - -fn mcmat_version(roughness: f32, wear: f32, version: u32) -> Vec { - let source = if version == 2 { "source.sbs" } else { "source.mcraft" }; +/// A CLI bake with known pixels for testing the linked layer independently of the subprocess. +fn cooked(roughness: f32, wear: f32) -> CookedMaterial { let gray = |v: f32| mesh_map_png(4, 1, &[v; 16]).unwrap(); - let manifest = json!({ - "format": "craft-material", "version": version, "name": "Brick wall", "graph": "Brick_wall", "size": [4, 4], - "maps": [ - {"usage": "baseColor", "file": "maps/baseColor.png", "colorSpace": "srgb"}, - {"usage": "roughness", "file": "maps/roughness.png", "colorSpace": "linear"}, - {"usage": "ambientOcclusion", "file": "maps/ambientOcclusion.png", "colorSpace": "linear"} - ], + let interface = json!({ + "name": "Brick wall", "graph": "Brick_wall", "parameters": [ {"id": "wear", "label": "Wear", "default": {"t": "float", "v": 0.5}, "min": 0.0, "max": 1.0, "value": {"t": "float", "v": wear}}, {"id": "tint", "label": "Tint", "default": {"t": "color", "v": [1.0, 1.0, 1.0, 1.0]}, "value": {"t": "color", "v": [1.0, 1.0, 1.0, 1.0]}}, @@ -47,21 +38,17 @@ fn mcmat_version(roughness: f32, wear: f32, version: u32) -> Vec { {"identifier": "bentNormal", "kind": "color"} ], "presets": [{"name": "Worn", "values": {"wear": {"t": "float", "v": 1.0}, "note": {"t": "text", "v": "x"}}}], - "source": source }); - let mut z = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); - let o = zip::write::SimpleFileOptions::default(); - for (name, bytes) in [ - ("material.json", manifest.to_string().into_bytes()), - ("maps/baseColor.png", mesh_map_png(4, 3, &[0.5; 48]).unwrap()), - ("maps/roughness.png", gray(roughness)), - ("maps/ambientOcclusion.png", gray(1.0)), - (source, b"package bytes".to_vec()), - ] { - z.start_file(name, o).unwrap(); - z.write_all(&bytes).unwrap(); - } - z.finish().unwrap().into_inner() + let mut budget = LinkMemoryBudget::new(1 << 20); + let maps = [(Channel::BaseColor, mesh_map_png(4, 3, &[0.5; 48]).unwrap()), (Channel::Roughness, gray(roughness)), (Channel::AmbientOcclusion, gray(1.0))] + .into_iter() + .map(|(channel, bytes)| CookedMap::decode(channel, bytes, [4, 4], &mut budget).unwrap()) + .collect(); + CookedMaterial::new(serde_json::from_value(interface).unwrap(), maps, 4) +} + +fn install(session: &mut Session) -> LayerId { + session.add_cooked_material(&cooked(0.25, 0.5), Arc::new(b"package bytes".to_vec()), "mcmat", Place::Selection).unwrap() } fn roughness(s: &mut Session) -> f32 { @@ -86,31 +73,47 @@ fn map_name(s: &Session, id: LayerId, c: Channel) -> String { } } +// Real cook reservations must be independent of saves in other parallel integration tests +fn run_in_private_process() -> bool { + const CHILD_TEST: &str = "TC_MATERIALCRAFT_CHILD_TEST"; + + let thread = std::thread::current(); + let name = thread.name().unwrap(); + if std::env::var(CHILD_TEST).as_deref() == Ok(name) { + return false; + } + + let status = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", name, "--include-ignored", "--nocapture"]) + .env(CHILD_TEST, name) + .status() + .unwrap(); + assert!(status.success(), "{name} returned {status}"); + true +} + #[test] -fn version_two_preserves_the_sbs_source_for_future_cooks() { +fn version_two_requires_republishing_before_install() { + if run_in_private_process() { + return; + } + + let mut zip = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + zip.start_file("material.json", zip::write::SimpleFileOptions::default()).unwrap(); + zip.write_all(br#"{"version":2,"format":"craft-material"}"#).unwrap(); + let bytes = zip.finish().unwrap().into_inner(); let mut session = session(); - let id = session.add_linked_material_bytes_at(&mcmat_version(0.25, 0.5, 2), Place::Selection).unwrap(); - let linked = link(&session, id); - assert!(linked.package.ends_with(".sbs")); - let job = session.link_job(id, &serde_json::Map::new()).unwrap(); - assert_eq!(job.source_ext, "sbs"); - assert_eq!(job.graph.as_deref(), Some("Brick_wall")); - assert_eq!(job.package.as_slice(), b"package bytes"); - let base = std::env::temp_dir().join(format!("texturecraft-sbs-source-{}", std::process::id())); - let saved = execute(&mut session, "material.link.saveSource", &json!({"layer": id.0, "path": base.to_str().unwrap()})).unwrap(); - let path = saved["path"].as_str().unwrap(); - assert!(path.ends_with(".sbs")); - assert_eq!(std::fs::read(path).unwrap(), b"package bytes"); - let _ = std::fs::remove_file(path); + assert!(session.add_linked_material_bytes_at(&bytes, Place::Selection).unwrap_err().to_string().contains("Republish")); } #[test] fn imports_as_a_linked_fill_with_its_maps() { let mut s = session(); - let id = s.add_linked_material_bytes_at(&mcmat(0.25, 0.5), Place::Selection).unwrap(); + let id = install(&mut s); assert_eq!(s.selected_layer(), Some(id)); assert!((roughness(&mut s) - 0.25).abs() < 2e-3); - // The material's channels join the texture set; text inputs are not editable here. + + // The fill adds its mapped channels and omits text parameters assert!(s.doc.project.texture_sets[0].has_channel(Channel::AmbientOcclusion)); let l = link(&s, id); assert_eq!(l.graph, "Brick_wall"); @@ -118,13 +121,15 @@ fn imports_as_a_linked_fill_with_its_maps() { assert_eq!(l.inputs, ["curvature", "normal", "worldNormal", "ambientOcclusion", "bentNormal"]); assert_eq!(s.doc.resources.get(&l.package).map(|b| b.as_slice()), Some(&b"package bytes"[..])); assert!(!s.images.contains_key(&l.package), "the package is not decoded as an image"); - // Inspect shows the link; one undo removes everything. + + // One undo removes the link exposed through inspect let inspect = execute(&mut s, "project.inspect", &Value::Null).unwrap().to_string(); assert!(inspect.contains("Brick_wall"), "{inspect}"); let n = s.doc.resources.len(); execute(&mut s, "edit.undo", &Value::Null).unwrap(); assert!(s.doc.project.layer(id).is_err() && s.doc.resources.len() < n); - // Garbage is an error, not a layer. + + // Invalid archives leave the stack untouched assert!(s.add_linked_material_bytes_at(b"PK not really", Place::Selection).is_err()); } @@ -132,10 +137,10 @@ fn imports_as_a_linked_fill_with_its_maps() { #[test] fn recooks_keep_the_normal_format_of_a_map() { let mut session = session(); - let id = session.add_linked_material_bytes_at(&mcmat(0.25, 0.5), Place::Selection).unwrap(); + let id = install(&mut session); execute(&mut session, "layer.fill.set", &json!({"layer": id.0, "channel": "roughness", "normalFormat": "directX"})).unwrap(); - session.apply_link_cook(id, &mcmat(0.75, 0.5), false).unwrap(); + session.apply_link_cook(id, &cooked(0.75, 0.5), false).unwrap(); let LayerKind::Fill(fill) = &session.doc.project.layer(id).unwrap().kind else { panic!("a fill") }; @@ -145,7 +150,7 @@ fn recooks_keep_the_normal_format_of_a_map() { #[test] fn parameters_are_checked_stored_and_recooked_in_one_undo_step() { let mut session = session(); - let id = session.add_linked_material_bytes_at(&mcmat(0.25, 0.5), Place::Selection).unwrap(); + let id = install(&mut session); let set = |session: &mut Session, params: Value| execute(session, "material.link.set", ¶ms); assert!(set(&mut session, json!({"id": "nope", "value": 1, "cook": false})).is_err()); @@ -164,7 +169,7 @@ fn parameters_are_checked_stored_and_recooked_in_one_undo_step() { // Merge the cooked maps into the parameter edit let old = map_name(&session, id, Channel::Roughness); - session.apply_link_cook(id, &mcmat(0.75, 0.9), true).unwrap(); + session.apply_link_cook(id, &cooked(0.75, 0.9), true).unwrap(); assert!((roughness(&mut session) - 0.75).abs() < 2e-3); assert!(!session.doc.resources.contains_key(&old), "old maps are dropped"); @@ -185,27 +190,34 @@ fn parameters_are_checked_stored_and_recooked_in_one_undo_step() { #[test] fn duplicates_keep_their_maps_and_mesh_maps_feed_inputs() { let mut session = session(); - let id = session.add_linked_material_bytes_at(&mcmat(0.25, 0.5), Place::Selection).unwrap(); + let id = install(&mut session); let duplicate = LayerId(execute(&mut session, "layer.duplicate", &json!({"layer": id.0})).unwrap()["layer"].as_u64().unwrap()); let shared = map_name(&session, duplicate, Channel::Roughness); - session.apply_link_cook(id, &mcmat(0.75, 0.5), false).unwrap(); + session.apply_link_cook(id, &cooked(0.75, 0.5), false).unwrap(); assert!(session.doc.resources.contains_key(&shared), "the duplicate still shows the first cook"); let maps = [BakedMap::Curvature, BakedMap::Normal, BakedMap::WorldNormal, BakedMap::AmbientOcclusion, BakedMap::BentNormal]; session.doc.baked.insert(0, BakedMaps { resolution: 4, maps: maps.into_iter().map(|map| (map, Arc::new(vec![0.5f32; 16 * map.components()]))).collect() }); let job = session.link_job(id, &serde_json::Map::new()).unwrap(); - assert_eq!(job.inputs.iter().map(|(identifier, _)| identifier.as_str()).collect::>(), maps.map(BakedMap::id)); + assert_eq!(job.input_identifiers.iter().map(String::as_str).collect::>(), maps.map(BakedMap::id)); assert_eq!(maps.map(BakedMap::id), ["curvature", "normal", "worldNormal", "ambientOcclusion", "bentNormal"]); } #[test] fn projects_save_the_link_and_its_package() { let mut s = session(); - let id = s.add_linked_material_bytes_at(&mcmat(0.25, 0.5), Place::Selection).unwrap(); - let back = Session::from_project_bytes(&s.to_project_bytes().unwrap()).unwrap(); + let id = install(&mut s); + let snapshot = s.project_snapshot(); + assert!(snapshot.resources.iter().all(|(name, bytes)| Arc::ptr_eq(bytes, &s.doc.resources[name]))); + let owned = s.project_file(); + let saved = s.to_project_bytes().unwrap(); + + assert_eq!(saved, texturecraft_engine::formats::save_project_bytes(&owned).unwrap()); + let back = Session::from_project_bytes(&saved).unwrap(); assert_eq!(link(&back, id), link(&s, id)); assert!(back.doc.resources.contains_key(&link(&s, id).package)); - // Smart materials made from the layer carry the package too. + + // Layer exports also carry the source package let tcmat = s.smart_material_bytes(id, "Bricks").unwrap(); let mut other = session(); let copy = other.add_smart_material_bytes(&tcmat).unwrap(); @@ -219,45 +231,210 @@ fn cooks_through_materialcraft_cli_when_available() { eprintln!("MATERIALCRAFT_CLI is not set: skipping the materialcraft-cli round trip"); return; }; - let dir = std::env::temp_dir().join(format!("tc-mc-link-{}", std::process::id())); - std::fs::create_dir_all(&dir).unwrap(); + + if run_in_private_process() { + return; + } + + let temp = tempfile::tempdir().unwrap(); + let dir = temp.path(); let pkg = dir.join("bricks.mcraft"); let ok = std::process::Command::new(&cli).args(["new", "pbr"]).arg(&pkg).args(["--size", "64"]).status().unwrap(); assert!(ok.success()); let exposed = dir.join("exposed.mcraft"); + let rectangular = dir.join("rectangular.mcraft"); + let resize = + std::process::Command::new(&cli).arg("run").arg(&pkg).args(["graph.size", r#"{"width":64,"height":32}"#, "-o"]).arg(&rectangular).status().unwrap(); + assert!(resize.success()); let ok = std::process::Command::new(&cli) .arg("run") - .arg(&pkg) + .arg(&rectangular) .arg("graph.exposed.add") - .arg(r#"{"id": "wear", "default": 0.5}"#) + .arg(r#"{"id":"wear","bind":{"node":5,"param":"color"}}"#) .arg("-o") .arg(&exposed) .status() .unwrap(); assert!(ok.success()); + + // Feed metallic from the texture set's baked curvature + let with_input = dir.join("with_input.mcraft"); + let add = std::process::Command::new(&cli) + .arg("run") + .arg(&exposed) + .arg("graph.node.add") + .arg(r#"{"def":"input_grayscale","params":{"identifier":"curvature"}}"#) + .arg("-o") + .arg(&with_input) + .output() + .unwrap(); + assert!(add.status.success(), "{}", String::from_utf8_lossy(&add.stderr)); + let new_node: Value = serde_json::from_slice(&add.stdout).unwrap(); + let connect = json!({"from":new_node["id"],"to":8,"toPort":"input"}); + let ok = + std::process::Command::new(&cli).arg("run").arg(&with_input).arg("graph.connect").arg(connect.to_string()).arg("-o").arg(&exposed).status().unwrap(); + assert!(ok.success()); let mut s = session(); + s.doc.baked.insert(0, BakedMaps { resolution: 16, maps: [(BakedMap::Curvature, Arc::new(vec![0.35; 256]))].into() }); let id = s.add_linked_material_bytes_at(&std::fs::read(&exposed).unwrap(), Place::Selection).unwrap(); assert_eq!(link(&s, id).size, 128, "cooked at the texture set's resolution"); + let image = &s.images[&map_name(&s, id, Channel::Roughness)]; + assert_eq!([image.width, image.height], [128, 128]); assert!(link(&s, id).params.iter().any(|p| p.id == "wear")); - execute(&mut s, "material.link.set", &json!({"layer": id.0, "id": "wear", "value": 0.8})).unwrap(); - let wear = link(&s, id).params.iter().find(|p| p.id == "wear").unwrap().value["v"].as_f64().unwrap(); + assert!((roughness(&mut s) - 0.5).abs() < 0.01); + assert!((s.caches[0].composite.value(Channel::Metallic, 64, 64)[0] - 0.35).abs() < 0.01); + execute(&mut s, "material.link.set", &json!({"layer": id.0, "id": "wear", "value": [0.8,0.8,0.8,1.0]})).unwrap(); + let wear = link(&s, id).params.iter().find(|p| p.id == "wear").unwrap().value["v"][0].as_f64().unwrap(); assert!((wear - 0.8).abs() < 1e-6); - let out = dir.join("back.sbs"); + assert!((roughness(&mut s) - 0.8).abs() < 0.01); + let out = dir.join("back.mcraft"); execute(&mut s, "material.link.saveSource", &json!({"layer": id.0, "path": out.to_str().unwrap()})).unwrap(); - assert!(std::fs::metadata(&out).unwrap().len() > 0); + assert_eq!(std::fs::read(&out).unwrap(), std::fs::read(&exposed).unwrap()); - // The portable material can name a graph other than the first in the embedded source. + // Publish a graph with distinct pixels to verify graph selection let multi = dir.join("multi.mcraft"); let ok = std::process::Command::new(&cli).arg("run").arg(&exposed).arg("graph.duplicate").arg("{}").arg("-o").arg(&multi).status().unwrap(); assert!(ok.success()); + let edited = dir.join("edited.mcraft"); + let script = dir.join("edit-graph.json"); + std::fs::write( + &script, + json!({ + "open": multi, + "steps": [ + {"id":"graph.open","params":{"graph":1}}, + {"id":"graph.exposed.set","params":{"id":"wear","value":[0.2,0.2,0.2,1.0]}} + ], + "save": edited + }) + .to_string(), + ) + .unwrap(); + let changed = std::process::Command::new(&cli).arg("batch").arg(&script).output().unwrap(); + assert!(changed.status.success(), "{}", String::from_utf8_lossy(&changed.stderr)); + let info = std::process::Command::new(&cli).arg("run").arg(&edited).args(["material.info", r#"{"graph":1}"#]).output().unwrap(); + assert!(info.status.success(), "{}", String::from_utf8_lossy(&info.stderr)); + let interface: Value = serde_json::from_slice(&info.stdout).unwrap(); + let expected_graph = interface["graph"].as_str().unwrap(); let portable = dir.join("second.mcmat"); - let params = json!({"graph": 1, "size": 64, "path": portable.to_str().unwrap()}); - let ok = std::process::Command::new(&cli).arg("run").arg(&multi).arg("file.exportMaterial").arg(params.to_string()).status().unwrap(); + let ok = std::process::Command::new(&cli).arg("publish").arg(&edited).arg(&portable).args(["--graph", "1"]).status().unwrap(); assert!(ok.success()); - let second = s.add_linked_material_bytes_at(&std::fs::read(&portable).unwrap(), Place::Selection).unwrap(); - let expected_graph = link(&s, second).graph; + let renamed = dir.join("second.asset"); + std::fs::rename(portable, &renamed).unwrap(); + let loaded = execute(&mut s, "material.load", &json!({"path":renamed})).unwrap(); + let second = LayerId(loaded["layer"].as_u64().unwrap()); + assert_eq!(link(&s, second).graph, expected_graph); + let second_map = map_name(&s, second, Channel::Roughness); + assert!((s.images[&second_map].rgba[0][0] - 0.2).abs() < 0.01); + s.update(); + assert!((s.caches[0].composite.value(Channel::Metallic, 64, 64)[0] - 0.35).abs() < 0.01); let job = s.link_job(second, &serde_json::Map::new()).unwrap(); - let result = texturecraft_engine::formats::mcmat::read_mcmat(&job.run().unwrap()).unwrap(); - assert_eq!(result.manifest.graph, expected_graph); - let _ = std::fs::remove_dir_all(&dir); + assert_eq!(job.source_ext, "mcmat"); + assert_eq!(job.package.as_slice(), std::fs::read(&renamed).unwrap()); + let result = job.run().unwrap(); + assert_eq!(result.interface.graph, expected_graph); + assert_eq!(result.size, 128); +} + +#[test] +fn installing_a_bake_reuses_its_encoded_map_buffers() { + let material = cooked(0.25, 0.5); + let prepared = &material.maps[1]; + let mut session = session(); + + let id = session.add_cooked_material(&material, Arc::new(Vec::new()), "mcmat", Place::Selection).unwrap(); + let name = map_name(&session, id, Channel::Roughness); + + assert!(Arc::ptr_eq(&session.doc.resources[&name], &prepared.bytes)); + assert!(Arc::ptr_eq(&session.images[&name], &prepared.image)); +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "requires MaterialCraft CLI and at least 8 GiB of available memory"] +fn three_8k_maps_respect_budget_and_survive_reopen() { + use texturecraft_engine::formats::save_ram_budget; + use texturecraft_engine::link_cmds::cook_link_file; + + const RESOLUTION: u32 = 8192; + const FIXTURE_AVAILABLE_BYTES: u64 = 16 << 30; + if run_in_private_process() { + return; + } + + let cli = std::env::var_os("MATERIALCRAFT_CLI").expect("set MATERIALCRAFT_CLI to the compatible CLI"); + let temp = tempfile::tempdir().unwrap(); + let mut authoring = temp.path().join("pbr.mcraft"); + assert!(std::process::Command::new(&cli).args(["new", "pbr"]).arg(&authoring).args(["--size", "32"]).status().unwrap().success()); + + // Keep base color, roughness and metallic from the PBR fixture + for node in [4, 10, 12] { + let next = temp.path().join(format!("after-{node}.mcraft")); + let args = json!({"to":node,"toPort":"input"}).to_string(); + let output = std::process::Command::new(&cli).arg("run").arg(&authoring).args(["graph.disconnect", &args, "-o"]).arg(&next).output().unwrap(); + assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + authoring = next; + } + + let path = temp.path().join("three.mcmat"); + assert!(std::process::Command::new(&cli).arg("publish").arg(&authoring).arg(&path).args(["--graph", "0"]).status().unwrap().success()); + let mut session = Session::new_project("Budget", primitives::cube(1.0), "builtin:cube", None, RESOLUTION).unwrap(); + let original = session.to_project_bytes().unwrap(); + + let save = texturecraft_engine::formats::RamReservation::for_save().unwrap(); + let blocked = cook_link_file(&path, RESOLUTION, None, LinkMemoryBudget::new(FIXTURE_AVAILABLE_BYTES), Arc::default()).unwrap_err(); + assert!(blocked.to_string().contains("a project save is using the RAM allowance")); + drop(save); + + let low = LinkMemoryBudget::new(save_ram_budget(FIXTURE_AVAILABLE_BYTES, 1)); + let companion_limited = LinkMemoryBudget::new(save_ram_budget(FIXTURE_AVAILABLE_BYTES, 10)); + + let error = cook_link_file(&path, RESOLUTION, None, low, Arc::default()).unwrap_err(); + + assert!(error.to_string().contains("configured cap"), "{error}"); + assert_eq!(session.to_project_bytes().unwrap(), original); + + let error = cook_link_file(&path, RESOLUTION, None, companion_limited, Arc::default()).unwrap_err(); + + assert!(error.to_string().contains("MaterialCraft cook failed"), "{error}"); + assert!(error.to_string().contains("configured cap"), "{error}"); + assert_eq!(session.to_project_bytes().unwrap(), original); + + let high = LinkMemoryBudget::new(save_ram_budget(FIXTURE_AVAILABLE_BYTES, 50)); + let (source, cook) = cook_link_file(&path, RESOLUTION, None, high, Arc::default()).unwrap(); + let probes: Vec<_> = cook.maps.iter().map(|map| Arc::downgrade(&map.image)).collect(); + + assert_eq!(cook.maps.len(), 3); + assert!(cook.maps.iter().all(|map| [map.image.width, map.image.height] == [RESOLUTION; 2])); + + let layer = session.add_cooked_material(&cook, source.package, source.source_ext, Place::Selection).unwrap(); + let save_blocked = session.to_project_bytes().unwrap_err(); + assert!(save_blocked.to_string().contains("a MaterialCraft cook is using the RAM allowance")); + drop(cook); + let encoded = session.to_project_bytes().unwrap(); + + assert!(probes.iter().all(|probe| probe.upgrade().is_some())); + execute(&mut session, "edit.undo", &Value::Null).unwrap(); + assert!(probes.iter().all(|probe| probe.upgrade().is_none())); + drop(session); + + let reopened = Session::from_project_bytes(&encoded).unwrap(); + + assert!(reopened.doc.project.layer(layer).is_ok()); + for channel in [Channel::BaseColor, Channel::Roughness, Channel::Metallic] { + let name = map_name(&reopened, layer, channel); + let image = &reopened.images[&name]; + assert_eq!([image.width, image.height], [RESOLUTION; 2]); + } +} + +#[test] +fn image_with_material_extension_reopens_as_an_image() { + let mut session = session(); + let bytes = mesh_map_png(2, 1, &[0.5; 4]).unwrap(); + let name = session.import_resource_bytes("swatch.sbs", bytes).unwrap(); + + let reopened = Session::from_project_bytes(&session.to_project_bytes().unwrap()).unwrap(); + + assert_eq!(reopened.images[&name].rgba.len(), 4); } diff --git a/crates/formats/Cargo.toml b/crates/formats/Cargo.toml index a70198cf..b8203bcf 100644 --- a/crates/formats/Cargo.toml +++ b/crates/formats/Cargo.toml @@ -23,6 +23,8 @@ texturecraft-texel = { workspace = true } texturecraft-geom = { workspace = true } texturecraft-compose = { workspace = true } image = { workspace = true } +png = "0.18" +bytemuck = { workspace = true, features = ["extern_crate_alloc"] } gltf = { workspace = true } flate2 = { workspace = true } lz4_flex = { workspace = true } diff --git a/crates/formats/src/image_res.rs b/crates/formats/src/image_res.rs index d766bb68..c0dff433 100644 --- a/crates/formats/src/image_res.rs +++ b/crates/formats/src/image_res.rs @@ -21,6 +21,20 @@ pub fn decode_image(bytes: &[u8]) -> Result { /// [`decode_image`] for an image of at most `max_texels` (width × height), checked from the /// header, before any pixel is decoded or allocated. pub fn decode_image_within(bytes: &[u8], max_texels: u64) -> Result { + decode_image_with_limits(bytes, max_texels, image::Limits::default()) +} + +/// Decode a cook output under both its declared dimensions and decoder allocation budget. +pub fn decode_cooked_image(bytes: &[u8], size: [u32; 2], max_decoder_bytes: u64) -> Result { + let mut limits = image::Limits::default(); + limits.max_alloc = Some(max_decoder_bytes); + limits.max_image_width = Some(size[0]); + limits.max_image_height = Some(size[1]); + let texels = (u64::from(size[0]) * u64::from(size[1])).min(MAX_RESOURCE_TEXELS); + decode_image_with_limits(bytes, texels, limits) +} + +fn decode_image_with_limits(bytes: &[u8], max_texels: u64, limits: image::Limits) -> Result { let mut reader = image::ImageReader::new(std::io::Cursor::new(bytes)) .with_guessed_format() .map_err(|error| FormatError::Malformed { kind: "image", msg: error.to_string() })?; @@ -28,8 +42,11 @@ pub fn decode_image_within(bytes: &[u8], max_texels: u64) -> Result MAX_IMAGE_SIZE || height > MAX_IMAGE_SIZE { return Err(FormatError::TooLarge(format!("image is {width}x{height} (limit {MAX_IMAGE_SIZE})"))); } @@ -38,13 +55,21 @@ pub fn decode_image_within(bytes: &[u8], max_texels: u64) -> Result rgba, + Err((_, samples)) => { + let mut rgba = Vec::new(); + rgba.try_reserve_exact(samples.len() / 4).map_err(|error| FormatError::TooLarge(error.to_string()))?; + rgba.extend_from_slice(samples.as_chunks::<4>().0); + rgba + } + }; Ok(DecodedImage { width, height, rgba, srgb }) } diff --git a/crates/formats/src/lib.rs b/crates/formats/src/lib.rs index 73b6a055..36808c45 100644 --- a/crates/formats/src/lib.rs +++ b/crates/formats/src/lib.rs @@ -44,15 +44,15 @@ pub mod zip_read; pub use export::{ Dest, ExportInput, ExportOptions, ExportPreset, ExportedFile, FileFormat, MapSpec, Padding, PlannedFile, SourceRef, VirtualMap, list_outputs, render_maps, }; -pub use image_res::{DecodedImage, coverage_png, decode_image, decode_image_within, gray16_png, gray16_resource}; +pub use image_res::{DecodedImage, coverage_png, decode_cooked_image, decode_image, decode_image_within, gray16_png, gray16_resource}; pub use mesh_export::{GlbTextures, NamedFiles, UsdExport, UsdFormat, glb_bytes, glb_export_options, obj_bytes, obj_with_mtl, usd_bytes, usd_export_options}; pub use mesh_import::{HighPolyMesh, ImportedMesh, MeshFile, MeshReport, import_high_poly_bytes, import_mesh_bytes, mesh_report, self_contained_mesh}; #[cfg(not(target_arch = "wasm32"))] pub use presets::load_preset; pub use presets::{builtin_preset, builtin_presets, preset_from_json, preset_to_json}; pub use project_file::{ - BakePaintData, DEFAULT_PROJECT_OPEN_LIMIT_GIB, MAX_RESOURCE_NAME_LEN, ProjectFile, load_project_bytes, project_open_limit_gib, sanitize_resource_name, - save_project_bytes, set_project_open_limit_gib, + BakePaintData, DEFAULT_PROJECT_OPEN_LIMIT_GIB, MAX_RESOURCE_NAME_LEN, ProjectFile, RamReservation, load_project_bytes, project_open_limit_gib, + sanitize_resource_name, save_project_bytes, set_project_open_limit_gib, }; #[cfg(not(target_arch = "wasm32"))] @@ -61,7 +61,8 @@ pub use export::write_files; pub use mesh_import::{import_mesh, read_mesh_file}; #[cfg(not(target_arch = "wasm32"))] pub use project_file::{ - DEFAULT_RAM_PERCENT, DEFAULT_RESERVED_CORES, available_save_resources, load_project, save_project, save_ram_budget, set_save_resources, + DEFAULT_RAM_PERCENT, DEFAULT_RESERVED_CORES, available_save_resources, configured_ram_budget, load_project, save_project, save_ram_budget, + set_save_resources, }; #[derive(Debug, thiserror::Error)] diff --git a/crates/formats/src/mcmat.rs b/crates/formats/src/mcmat.rs index 5040f320..4aa59481 100644 --- a/crates/formats/src/mcmat.rs +++ b/crates/formats/src/mcmat.rs @@ -1,251 +1,170 @@ -//! MaterialCraft portable materials (`.mcmat`, read only): a zip with `material.json` (the -//! manifest: maps by usage, exposed parameters, image inputs, presets), `maps/.png` -//! (8-bit sRGB for color usages, 16-bit linear otherwise; OpenGL normals) and the -//! source package. The shared reader handles the portable container and manifest. -//! -//! Also the 16-bit PNG encoder that hands baked mesh maps to MaterialCraft as graph inputs. - -use std::io::Cursor; - -use serde::Deserialize; -use texturecraft_core::{LinkParam, LinkPreset}; - +//! Validate a procedural MCMAT and encode caller-supplied mesh maps for MaterialCraft. use crate::{FormatError, Result}; +use std::io::Write; + +mod version_marker; +use version_marker::VersionMarker; -/// The portable material extension. pub const MCMAT_EXT: &str = "mcmat"; -/// MaterialCraft's package extension. pub const MCRAFT_EXT: &str = "mcraft"; -const VERSION: u32 = 2; -/// Largest entry read (maps are at most 8192² × 16 bits, compressed). -const MAX_ENTRY: u64 = 512 << 20; -/// Most map entries one manifest may name (a material has at most nine usages). -const MAX_MAPS: usize = 64; -/// Largest total payload read from one `.mcmat`: maps and source together (audit: TextureCraft #4). -const MAX_TOTAL: u64 = 1 << 30; -const KIND: &str = "MaterialCraft material"; -#[derive(Clone, Debug, PartialEq, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Manifest { - pub format: String, - pub version: u32, - pub name: String, - pub graph: String, - pub size: [u32; 2], - pub maps: Vec, - #[serde(default)] - pub parameters: Vec, - #[serde(default)] - pub inputs: Vec, - #[serde(default)] - pub presets: Vec, - pub source: String, -} +/// Diagnose old versions before starting the CLI. The shared reader in MaterialCraft +/// validates and extracts the full archive. +pub fn validate_mcmat_version(bytes: &[u8], manifest_budget_bytes: u64) -> Result<()> { + let limits = craft_mcmat::Limits::default(); + let mut archive = craft_zip::Archive::open(bytes, craft_zip::Limits { max_entries: limits.max_entries, max_total_bytes: manifest_budget_bytes }) + .map_err(mcmat_archive_error)?; + let bytes = archive + .read(craft_mcmat::MANIFEST_ENTRY, limits.max_manifest_bytes.min(manifest_budget_bytes)) + .map_err(mcmat_archive_error)? + .ok_or_else(|| mcmat_invalid("missing material.json"))?; -#[derive(Clone, Debug, PartialEq, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct MapEntry { - /// `baseColor`, `normal`, `roughness`, `metallic`, `height`, `ambientOcclusion`, `emissive`, - /// `opacity` or `specular`. - pub usage: String, - pub file: String, -} - -#[derive(Clone, Debug, PartialEq, Deserialize)] -pub struct ImageInput { - pub identifier: String, -} - -/// A read `.mcmat`: the manifest, each map's PNG bytes by usage, the source package. -#[derive(Clone, Debug)] -pub struct PortableMaterial { - pub manifest: Manifest, - pub maps: Vec<(String, Vec)>, - pub source: Vec, -} - -fn malformed(msg: impl Into) -> FormatError { - FormatError::Malformed { kind: KIND, msg: msg.into() } -} - -fn shared_error(error: craft_mcmat::Error) -> FormatError { - match error { - craft_mcmat::Error::Version(version) => FormatError::Unsupported(format!("{KIND} version {version} (this version reads up to {VERSION})")), - craft_mcmat::Error::Limit(message) => malformed(message), - craft_mcmat::Error::Archive(ref archive) if archive.is_limit() => FormatError::TooLarge(error.to_string()), - other => malformed(other.to_string()), + // A marker must be an object even though Serde also accepts positional struct fields + if !bytes.trim_ascii_start().starts_with(b"{") { + return Err(mcmat_invalid("expected a version object")); } + + let marker: VersionMarker = serde_json::from_slice(&bytes).map_err(mcmat_invalid)?; + if marker.version != craft_mcmat::VERSION { + return Err(FormatError::Unsupported(craft_mcmat::Error::Version(marker.version).to_string())); + } + + Ok(()) } -/// Read `.mcmat` bytes within TextureCraft's existing per-entry and total payload limits. -pub fn read_mcmat(bytes: &[u8]) -> Result { - let limits = craft_mcmat::Limits { - min_version: 0, - max_version: VERSION, - max_entries: 100_000, - max_maps: MAX_MAPS, - max_manifest_bytes: MAX_ENTRY, - max_entry_bytes: MAX_ENTRY, - max_total_bytes: MAX_TOTAL, - }; - let mut archive = craft_mcmat::Material::open(bytes, limits).map_err(shared_error)?; - // Consume the JSON values rather than cloning the entire untrusted manifest into a second tree. - let parameters = std::mem::take(&mut archive.manifest.parameters) - .into_iter() - .map(serde_json::from_value) - .collect::>() - .map_err(|e| malformed(format!("material.json: {e}")))?; - let inputs = std::mem::take(&mut archive.manifest.inputs) - .into_iter() - .map(serde_json::from_value) - .collect::>() - .map_err(|e| malformed(format!("material.json: {e}")))?; - let presets = std::mem::take(&mut archive.manifest.presets) - .into_iter() - .map(serde_json::from_value) - .collect::>() - .map_err(|e| malformed(format!("material.json: {e}")))?; - let shared = &archive.manifest; - let manifest = Manifest { - format: shared.format.clone(), - version: shared.version, - name: shared.name.clone(), - graph: shared.graph.clone(), - size: shared.size, - maps: shared.maps.iter().map(|m| MapEntry { usage: m.usage.clone(), file: m.file.clone() }).collect(), - parameters, - inputs, - presets, - source: shared.source.clone(), - }; - let maps = manifest.maps.iter().map(|map| Ok((map.usage.clone(), archive.read(&map.file).map_err(shared_error)?))).collect::>>()?; - let source = archive.read_source().map_err(shared_error)?; - Ok(PortableMaterial { manifest, maps, source }) +fn mcmat_invalid(error: impl std::fmt::Display) -> FormatError { + FormatError::Malformed { kind: "MaterialCraft material", msg: error.to_string() } +} + +fn mcmat_archive_error(error: craft_zip::Error) -> FormatError { + if error.is_limit() { + return FormatError::TooLarge(error.to_string()); + } + + mcmat_invalid(error) } /// Whether `bytes` contain a portable material manifest within the ZIP entry limit. pub fn is_mcmat(bytes: &[u8]) -> bool { - craft_mcmat::is_material(bytes, 100_000) + craft_mcmat::is_material(bytes, craft_mcmat::Limits::default().max_entries) } pub fn is_mcraft(bytes: &[u8]) -> bool { - let Ok(mut archive) = craft_zip::Archive::open(bytes, craft_zip::Limits { max_entries: 100_000, max_total_bytes: 0 }) else { return false }; - // A portable material may itself contain source.sbs, so it is not a source archive. - if archive.has_duplicate_names() || archive.contains("material.json") { + let limits = craft_zip::Limits { max_entries: craft_mcmat::Limits::default().max_entries, max_total_bytes: 0 }; + let Ok(mut archive) = craft_zip::Archive::open(bytes, limits) else { + return false; + }; + + // A portable material may contain source.sbs inside its envelope + if archive.has_duplicate_names() || archive.contains(craft_mcmat::MANIFEST_ENTRY) { return false; } + if archive.contains("package.json") { return true; } - (0..archive.len()).any(|i| archive.entry_name(i).ok().flatten().is_some_and(|name| !name.contains('/') && name.to_ascii_lowercase().ends_with(".sbs"))) + + (0..archive.len()).any(|index| archive.entry_name(index).ok().flatten().as_deref().is_some_and(is_root_sbs)) +} + +pub(crate) fn is_root_sbs(name: &str) -> bool { + !name.contains('/') && name.rsplit_once('.').is_some_and(|(_, extension)| extension.eq_ignore_ascii_case("sbs")) } /// A baked mesh map (`components` floats per texel, 0..1, rows top first) as a 16-bit PNG /// (grayscale for one component, RGB for three). pub fn mesh_map_png(width: u32, components: usize, data: &[f32]) -> Result> { - let per_row = (width as usize).saturating_mul(components); - let height = data.len().checked_div(per_row).unwrap_or(0) as u32; - if height == 0 || !matches!(components, 1 | 3) { - return Err(FormatError::Invalid(format!("mesh map of {} values, {width} wide, {components} components", data.len()))); + let mut encoded = Vec::new(); + write_mesh_map_png(&mut encoded, width, components, data)?; + Ok(encoded) +} + +/// Estimated workspace for one quantized row, three encoder rows and the deflate state. +pub fn mesh_map_png_working_bytes(width_px: u32, components: usize) -> u64 { + const DEFLATE_WORKSPACE_BYTES: u64 = 1 << 20; + + u64::from(width_px).saturating_mul(components as u64).saturating_mul(4 * std::mem::size_of::() as u64).saturating_add(DEFLATE_WORKSPACE_BYTES) +} + +/// Encode a mesh map directly into its destination using one row of quantized samples. +pub fn write_mesh_map_png(output: impl Write, width_px: u32, components: usize, data: &[f32]) -> Result<()> { + let invalid_size = || FormatError::Invalid(format!("mesh map of {} values, {width_px} wide, {components} components", data.len())); + let color = match components { + 1 => png::ColorType::Grayscale, + 3 => png::ColorType::Rgb, + _ => return Err(invalid_size()), + }; + let samples_per_row = (width_px as usize).checked_mul(components).filter(|&count| count != 0).ok_or_else(invalid_size)?; + if data.is_empty() || !data.len().is_multiple_of(samples_per_row) { + return Err(invalid_size()); } - let px: Vec = data.iter().take(per_row * height as usize).map(|v| (v.clamp(0.0, 1.0) * 65535.0).round() as u16).collect(); - let img = if components == 1 { - image::ImageBuffer::, _>::from_raw(width, height, px).map(image::DynamicImage::ImageLuma16) - } else { - image::ImageBuffer::, _>::from_raw(width, height, px).map(image::DynamicImage::ImageRgb16) + + let height_px = u32::try_from(data.len() / samples_per_row).map_err(|_| invalid_size())?; + let row_bytes = samples_per_row.checked_mul(std::mem::size_of::()).ok_or_else(invalid_size)?; + let mut scanline = Vec::new(); + scanline.try_reserve_exact(row_bytes).map_err(|error| FormatError::TooLarge(error.to_string()))?; + + let encoding_error = |error: png::EncodingError| FormatError::Invalid(error.to_string()); + let mut encoder = png::Encoder::new(output, width_px, height_px); + encoder.set_color(color); + encoder.set_depth(png::BitDepth::Sixteen); + + // The file exists only while the companion reads the input + encoder.set_compression(png::Compression::Fast); + let mut writer = encoder.write_header().map_err(encoding_error)?; + { + let mut stream = writer.stream_writer().map_err(encoding_error)?; + for row in data.chunks_exact(samples_per_row) { + scanline.clear(); + scanline.extend(row.iter().flat_map(|value| ((value.clamp(0.0, 1.0) * f32::from(u16::MAX)).round() as u16).to_be_bytes())); + stream.write_all(&scanline)?; + } + + stream.finish().map_err(encoding_error)?; } - .ok_or_else(|| FormatError::Invalid("mesh map size".into()))?; - let mut out = Vec::new(); - img.write_to(&mut Cursor::new(&mut out), image::ImageFormat::Png).map_err(|e| FormatError::Invalid(e.to_string()))?; - Ok(out) + + writer.finish().map_err(encoding_error) } #[cfg(test)] mod tests { use super::*; - use std::io::Write; - - /// A `.mcmat` as MaterialCraft writes it (see its `formats/src/material.rs`). - pub(crate) fn sample(version: u32) -> Vec { - let source = if version >= 2 { "source.sbs" } else { "source.mcraft" }; - let manifest = serde_json::json!({ - "format": "craft-material", "version": version, "name": "Brick wall", "graph": "Brick_wall", "size": [4, 4], - "maps": [{"usage": "baseColor", "file": "maps/baseColor.png", "colorSpace": "srgb"}], - "parameters": [{"id": "wear", "label": "Wear", "default": {"t": "float", "v": 0.5}, "min": 0.0, "max": 1.0, "value": {"t": "float", "v": 0.25}}], - "inputs": [{"identifier": "curvature", "kind": "gray"}], - "presets": [{"name": "Worn", "values": {"wear": {"t": "float", "v": 1.0}}}], - "source": source, "futureField": true - }); - let mut z = zip::ZipWriter::new(Cursor::new(Vec::new())); - let o = zip::write::SimpleFileOptions::default(); - z.start_file("material.json", o).unwrap(); - z.write_all(manifest.to_string().as_bytes()).unwrap(); - z.start_file("maps/baseColor.png", o).unwrap(); - z.write_all(&mesh_map_png(4, 3, &[0.5; 48]).unwrap()).unwrap(); - z.start_file(source, o).unwrap(); - z.write_all(b"pkg").unwrap(); - z.finish().unwrap().into_inner() - } + use std::io::Cursor; #[test] - fn reads_manifest_maps_and_source() { - let bytes = sample(1); - - assert!(is_mcmat(&bytes) && !is_mcraft(&bytes)); - let material = read_mcmat(&bytes).unwrap(); - - assert_eq!(material.manifest.graph, "Brick_wall"); - assert_eq!(material.manifest.parameters[0].id, "wear"); - assert_eq!(material.manifest.parameters[0].value, serde_json::json!({"t": "float", "v": 0.25})); - assert_eq!(material.manifest.presets[0].name, "Worn"); - assert_eq!(material.manifest.inputs[0].identifier, "curvature"); - assert_eq!(material.maps[0].0, "baseColor"); - assert_eq!(crate::decode_image(&material.maps[0].1).unwrap().width, 4); - assert_eq!(material.source, b"pkg"); - let current = read_mcmat(&sample(2)).unwrap(); - assert_eq!(current.manifest.source, "source.sbs"); - assert_eq!(current.source, b"pkg"); - assert!(matches!(read_mcmat(&sample(3)), Err(FormatError::Unsupported(_)))); - assert!(read_mcmat(b"junk").is_err()); - } - - #[test] - fn recognizes_both_generations_of_source_archive_without_confusing_portable_materials() { - for entry in ["package.json", "package.sbs"] { + fn recognizes_v3_and_directs_older_authors_to_republish() { + let source = craft_mcmat::write(b"", []).unwrap(); + assert!(is_mcmat(&source) && !is_mcraft(&source)); + validate_mcmat_version(&source, 1 << 20).unwrap(); + for version in [1, 2] { let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new())); - zip.start_file(entry, zip::write::SimpleFileOptions::default()).unwrap(); - zip.write_all(b"source").unwrap(); - let bytes = zip.finish().unwrap().into_inner(); - assert!(is_mcraft(&bytes), "{entry}"); + zip.start_file("material.json", zip::write::SimpleFileOptions::default()).unwrap(); + write!(zip, r#"{{"version":{version}}}"#).unwrap(); + let err = validate_mcmat_version(&zip.finish().unwrap().into_inner(), 1 << 20).unwrap_err(); + assert!(err.to_string().contains("Republish")); } - assert!(!is_mcraft(&sample(2)), "a portable material contains source.sbs too"); } #[test] - fn an_unbounded_map_list_is_refused() { - let maps: Vec<_> = (0..=MAX_MAPS).map(|_| serde_json::json!({"usage": "baseColor", "file": "maps/baseColor.png"})).collect(); - let manifest = serde_json::json!({"format": craft_mcmat::FORMAT, "version": VERSION, "name": "n", "graph": "g", "size": [4, 4], "maps": maps, "source": "source.mcraft"}); - let mut archive = zip::ZipWriter::new(Cursor::new(Vec::new())); - let options = zip::write::SimpleFileOptions::default(); - archive.start_file("material.json", options).unwrap(); - archive.write_all(manifest.to_string().as_bytes()).unwrap(); - archive.start_file("maps/baseColor.png", options).unwrap(); - archive.write_all(&mesh_map_png(4, 3, &[0.5; 48]).unwrap()).unwrap(); - archive.start_file("source.mcraft", options).unwrap(); - archive.write_all(b"pkg").unwrap(); + fn version_marker_requires_one_named_version() { + for marker in [r#"{"version":2,"version":3}"#, "[3]"] { + let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new())); + zip.start_file(craft_mcmat::MANIFEST_ENTRY, zip::write::SimpleFileOptions::default()).unwrap(); + zip.write_all(marker.as_bytes()).unwrap(); + let bytes = zip.finish().unwrap().into_inner(); - let result = read_mcmat(&archive.finish().unwrap().into_inner()); - - assert!(matches!(result, Err(FormatError::Malformed { .. }))); + assert!(validate_mcmat_version(&bytes, 1 << 20).is_err()); + } } #[test] fn mesh_maps_encode_as_16_bit() { let png = mesh_map_png(2, 1, &[0.0, 0.25, 0.5, 1.0]).unwrap(); - let d = crate::decode_image(&png).unwrap(); - assert_eq!((d.width, d.height), (2, 2)); - assert!((d.rgba[1][0] - 0.25).abs() < 1e-4); + let decoded = crate::decode_image(&png).unwrap(); + + assert_eq!((decoded.width, decoded.height), (2, 2)); + assert!((decoded.rgba[1][0] - 0.25).abs() < 1e-4); + assert!(mesh_map_png(2, 1, &[0.0; 3]).is_err()); assert!(mesh_map_png(0, 1, &[]).is_err()); assert!(mesh_map_png(2, 2, &[0.0; 8]).is_err()); } diff --git a/crates/formats/src/mcmat/version_marker.rs b/crates/formats/src/mcmat/version_marker.rs new file mode 100644 index 00000000..374047d8 --- /dev/null +++ b/crates/formats/src/mcmat/version_marker.rs @@ -0,0 +1,5 @@ +/// Only the version is needed before handing the full package to MaterialCraft. +#[derive(serde::Deserialize)] +pub(super) struct VersionMarker { + pub(super) version: u32, +} diff --git a/crates/formats/src/project_file/entries.rs b/crates/formats/src/project_file/entries.rs index 1ad13a9d..a3d00c97 100644 --- a/crates/formats/src/project_file/entries.rs +++ b/crates/formats/src/project_file/entries.rs @@ -60,7 +60,8 @@ mod limits { pub fn available_ram() -> u64 { let Ok(mut sys) = sysinfo::System::new() else { return FALLBACK_AVAILABLE_RAM_BYTES }; sys.refresh_memory(); - reported_ram(sys.available_memory(), sys.total_memory()) + let available_bytes = reported_ram(sys.available_memory(), sys.total_memory()); + sys.cgroup_limits().map_or(available_bytes, |limits| available_bytes.min(limits.free_memory)) } fn reported_ram(available: u64, total: u64) -> u64 { @@ -109,6 +110,12 @@ pub fn save_ram_budget(available_ram_bytes: u64, ram_percent: u8) -> u64 { u64::try_from(u128::from(available_ram_bytes) * u128::from(ram_percent.clamp(1, 100)) / 100).unwrap_or(u64::MAX) } +/// The configured Settings share of currently available RAM, in bytes. +#[cfg(not(target_arch = "wasm32"))] +pub fn configured_ram_budget() -> u64 { + save_ram_budget(limits::available_ram(), limits::get().1) +} + /// Worker threads and in-flight byte budget for a save of `entries` entries. #[cfg(not(target_arch = "wasm32"))] pub(super) fn plan(cpus: usize, reserved_cores: usize, entries: usize, available_ram: u64, ram_percent: u8) -> (usize, u64) { diff --git a/crates/formats/src/project_file/mod.rs b/crates/formats/src/project_file/mod.rs index 9f2053fc..c5bf3bdb 100644 --- a/crates/formats/src/project_file/mod.rs +++ b/crates/formats/src/project_file/mod.rs @@ -18,6 +18,7 @@ //! count of 0, followed by `width × height × components` f32 LE. Pixels of a UV-tiled texture //! set cover its whole atlas (`resolution` per tile, see `TileGrid`). +use std::borrow::Borrow; use std::collections::{BTreeMap, HashSet}; use std::io::{Cursor, Read, Seek, Write}; use std::sync::Arc; @@ -35,9 +36,12 @@ use crate::{FormatError, MAX_IMAGE_SIZE, Result, malformed}; mod entries; mod load_budget; mod project_open_limit; +mod ram_operation; +mod ram_reservation; #[cfg(not(target_arch = "wasm32"))] -pub use entries::{DEFAULT_RAM_PERCENT, DEFAULT_RESERVED_CORES, available_save_resources, save_ram_budget, set_save_resources}; +pub use entries::{DEFAULT_RAM_PERCENT, DEFAULT_RESERVED_CORES, available_save_resources, configured_ram_budget, save_ram_budget, set_save_resources}; pub use project_open_limit::{DEFAULT_PROJECT_OPEN_LIMIT_GIB, project_open_limit_gib, set_project_open_limit_gib}; +pub use ram_reservation::RamReservation; const KIND: &str = "project file"; #[cfg(test)] @@ -61,7 +65,7 @@ pub use bake_paint_data::BakePaintData; /// Everything a project file holds. #[derive(Clone, Debug)] -pub struct ProjectFile { +pub struct ProjectFile> { pub project: Project, pub pixels: PixelStore, /// Baked mesh maps per texture set index. @@ -76,7 +80,7 @@ pub struct ProjectFile { pub bake_paint: BTreeMap<(usize, BakePaintTarget), BakePaintData>, pub thumbnail: Option>, /// Imported resources (images used by fills, alphas, stencils), by name: original bytes. - pub resources: BTreeMap>, + pub resources: BTreeMap, /// Import warnings for the caller's log; never saved in the archive pub warnings: Vec, } @@ -95,14 +99,17 @@ pub fn sanitize_resource_name(name: &str) -> String { // Save. pub fn save_project_bytes(pf: &ProjectFile) -> Result> { - let mut out = Cursor::new(Vec::new()); - write_project(&mut out, pf)?; - Ok(out.into_inner()) + RamReservation::for_save()?.save_bytes(pf) } /// Save atomically through a newly created temporary file beside `path`. #[cfg(not(target_arch = "wasm32"))] pub fn save_project(path: &std::path::Path, project_file: &ProjectFile) -> Result<()> { + RamReservation::for_save()?.save(path, project_file) +} + +#[cfg(not(target_arch = "wasm32"))] +fn write_project_atomic(path: &std::path::Path, project_file: &ProjectFile>>) -> Result<()> { let (temporary_path, file) = crate::create_temp(path)?; let result = (|| -> Result<()> { let mut file = std::io::BufWriter::new(file); @@ -180,7 +187,7 @@ fn write_baked(w: &mut dyn Write, res: u32, h: u32, comps: usize, data: &[f32]) } /// The zip entries of a project file, in file order. Bodies run later, possibly on other threads. -fn entries(pf: &ProjectFile) -> Result>> { +fn entries(pf: &ProjectFile>>) -> Result>> { let deflate = SimpleFileOptions::default().compression_method(CompressionMethod::Deflated).large_file(true); let stored = SimpleFileOptions::default().compression_method(CompressionMethod::Stored); let json = serde_json::to_vec_pretty(&pf.project).map_err(|e| FormatError::Invalid(e.to_string()))?; @@ -206,6 +213,7 @@ fn entries(pf: &ProjectFile) -> Result>> { if map.width == 0 || map.height == 0 || map.values.len() as u64 != u64::from(map.width) * u64::from(map.height) { return Err(FormatError::Invalid("bake paint map dimensions do not match its values".into())); } + let target_name = match target { BakePaintTarget::Cage => "cage", BakePaintTarget::Skew => "skew", @@ -237,11 +245,12 @@ fn entries(pf: &ProjectFile) -> Result>> { for (ts, maps) in &pf.baked { let r = maps.resolution; for (m, data) in &maps.maps { - // A map whose size disagrees with its width cannot be loaded; it can be re-baked. + // Skip inconsistent maps so the saved project can reopen and bake them again let h = maps.height(*m); if h == 0 || data.len() != (r as usize) * (h as usize) * m.components() { continue; } + let comps = m.components(); out.push(Entry { name: format!("baked/{ts}_{}.bin", m.id()), @@ -255,7 +264,8 @@ fn entries(pf: &ProjectFile) -> Result>> { out.push(Entry { name: "thumbnail.png".into(), options: stored, weight: png.len() as u64, body: Box::new(move |w| w.write_all(png)) }); } used.clear(); - for (name, bytes) in &pf.resources { + for (name, resource) in &pf.resources { + let bytes = resource.borrow(); out.push(Entry { name: unique_entry_name("resources", name, &mut used), options: stored, @@ -266,7 +276,7 @@ fn entries(pf: &ProjectFile) -> Result>> { Ok(out) } -fn write_project(out: W, pf: &ProjectFile) -> Result<()> { +fn write_project(out: W, pf: &ProjectFile>>) -> Result<()> { let mut zip = ZipWriter::new(out); entries::write_entries(&mut zip, entries(pf)?)?; zip.finish().map_err(zip_err)?; @@ -286,6 +296,23 @@ pub fn load_project(path: &std::path::Path) -> Result { crate::guard(KIND, || load(std::io::BufReader::new(file))) } +/// Read a native smart material in place. MaterialCraft packages go to the bounded cook reader. +#[cfg(not(target_arch = "wasm32"))] +pub fn load_smart_material_project(file: std::fs::File) -> Result> { + crate::guard(KIND, || { + let archive = open_archive(std::io::BufReader::new(file), MAX_ZIP_METADATA_BYTES)?; + if archive.index_for_name(craft_mcmat::MANIFEST_ENTRY).is_some() + || archive.index_for_name("package.json").is_some() + || archive.file_names().any(|name| name.is_ok_and(|name| crate::mcmat::is_root_sbs(&name))) + { + return Ok(None); + } + + let budget = LoadBudget::new(u64::from(project_open_limit_gib()) << 30); + load_archive(archive, budget).map(Some) + }) +} + fn read_all(reader: &mut impl Read, cap: u64, budget: &mut LoadBudget, what: &str) -> Result> { let mut out = Vec::new(); @@ -359,8 +386,11 @@ fn open_archive(reader: R, metadata_bytes: u64) -> Result(reader: R, mut budget: LoadBudget) -> Result { - let mut zip = open_archive(reader, MAX_ZIP_METADATA_BYTES)?; +fn load_with_budget(reader: R, budget: LoadBudget) -> Result { + load_archive(open_archive(reader, MAX_ZIP_METADATA_BYTES)?, budget) +} + +fn load_archive(mut zip: ZipArchive, mut budget: LoadBudget) -> Result { check_archive(&mut zip, budget.remaining())?; let json = read_all(&mut zip.by_name("project.json").map_err(zip_err)?, MAX_JSON, &mut budget, "project.json")?; @@ -394,6 +424,8 @@ fn load_with_budget(reader: R, mut budget: LoadBudget) -> Result let declared_bytes = f.size(); let name = f.name().map_err(zip_err)?.into_owned(); let size_of = |id: LayerId| texture_set_of(&pf.project, id).and_then(|ts| pf.project.texture_sets.get(ts)).map(|t| t.size()); + + // Unrecognized entries from newer minor revisions may be skipped if name == "thumbnail.png" { pf.thumbnail = Some(read_all(&mut f, MAX_THUMBNAIL, &mut budget, &name)?); } else if let Some(res_name) = name.strip_prefix("resources/") { @@ -434,6 +466,7 @@ fn load_with_budget(reader: R, mut budget: LoadBudget) -> Result let (id, chan) = rest.split_once('_').ok_or_else(|| malformed(KIND, format!("bad entry name {name}")))?; let id = LayerId(id.parse().map_err(|_| malformed(KIND, format!("bad entry name {name}")))?); let c = Channel::ALL.into_iter().find(|c| c.id() == chan).ok_or_else(|| malformed(KIND, format!("unknown channel in {name}")))?; + // Drop pixels of layers that are no longer in any stack let Some(size) = size_of(id) else { continue }; @@ -464,9 +497,9 @@ fn load_with_budget(reader: R, mut budget: LoadBudget) -> Result } else if maps.resolution != res || maps.maps.keys().next().is_some_and(|k| maps.height(*k) != height) { return Err(malformed(KIND, format!("{name}: baked maps of one texture set differ in size"))); } + maps.maps.insert(m, Arc::new(data)); } - // Ignore other entries from newer minor revisions } if version < TANGENT_BENT_NORMAL_VERSION { diff --git a/crates/formats/src/project_file/ram_operation.rs b/crates/formats/src/project_file/ram_operation.rs new file mode 100644 index 00000000..1fb40c40 --- /dev/null +++ b/crates/formats/src/project_file/ram_operation.rs @@ -0,0 +1,5 @@ +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum RamOperation { + Cook, + Save, +} diff --git a/crates/formats/src/project_file/ram_reservation.rs b/crates/formats/src/project_file/ram_reservation.rs new file mode 100644 index 00000000..72857c15 --- /dev/null +++ b/crates/formats/src/project_file/ram_reservation.rs @@ -0,0 +1,84 @@ +use std::borrow::Borrow; +use std::io::Cursor; +use std::sync::{Condvar, Mutex}; + +use super::{ProjectFile, ram_operation::RamOperation, write_project}; +use crate::{FormatError, Result}; + +static ACTIVE_OPERATION: Mutex> = Mutex::new(None); +static RELEASED: Condvar = Condvar::new(); + +/// Shares the Settings allowance between linked cooks and project save workers. +#[derive(Debug)] +pub struct RamReservation(RamOperation); + +impl RamReservation { + /// Reserve the allowance through cooking, decoding and result delivery. + pub fn for_cook() -> Result { + let mut active = ACTIVE_OPERATION.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let message = match *active { + None => { + *active = Some(RamOperation::Cook); + return Ok(Self(RamOperation::Cook)); + } + Some(RamOperation::Cook) => "another MaterialCraft cook is still running", + Some(RamOperation::Save) => "a project save is using the RAM allowance; retry the cook after saving", + }; + + Err(FormatError::Invalid(message.into())) + } + + /// Serialize saves while giving the UI backpressure if a cook owns the allowance. + pub fn for_save() -> Result { + let mut active = ACTIVE_OPERATION.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + while *active == Some(RamOperation::Save) { + active = RELEASED.wait(active).unwrap_or_else(std::sync::PoisonError::into_inner); + } + + if active.is_some() { + return Err(FormatError::Invalid("a MaterialCraft cook is using the RAM allowance; retry saving when it finishes".into())); + } + + *active = Some(RamOperation::Save); + Ok(Self(RamOperation::Save)) + } + + /// Reserve an autosave before cloning its snapshot without blocking the UI. + pub fn try_for_save() -> Option { + let mut active = ACTIVE_OPERATION.try_lock().ok()?; + if active.is_some() { + return None; + } + + *active = Some(RamOperation::Save); + Some(Self(RamOperation::Save)) + } + + pub fn save_bytes(&mut self, project_file: &ProjectFile>>) -> Result> { + self.check_save()?; + let mut out = Cursor::new(Vec::new()); + write_project(&mut out, project_file)?; + Ok(out.into_inner()) + } + + #[cfg(not(target_arch = "wasm32"))] + pub fn save(&mut self, path: &std::path::Path, project_file: &ProjectFile>>) -> Result<()> { + self.check_save()?; + super::write_project_atomic(path, project_file) + } + + fn check_save(&self) -> Result<()> { + if self.0 == RamOperation::Save { + return Ok(()); + } + + Err(FormatError::Invalid("saving requires a save reservation".into())) + } +} + +impl Drop for RamReservation { + fn drop(&mut self) { + *ACTIVE_OPERATION.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = None; + RELEASED.notify_all(); + } +} diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index f97008a3..4ba94f88 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -134,12 +134,27 @@ pub struct ResizeJob { pub rx: std::sync::mpsc::Receiver>, } +mod link_bake_input; +#[cfg(not(target_arch = "wasm32"))] +mod link_import; + /// A MaterialCraft material re-cooking on a worker thread (one per layer at a time). pub struct LinkCook { pub layer: texturecraft_engine::core::LayerId, + project: u64, + mesh: u64, + size: u32, + baked_inputs: Vec, + canceled: std::sync::Arc, /// The parameter values it cooks: a result whose values are no longer the layer's is redone. pub values: serde_json::Map, - pub rx: std::sync::mpsc::Receiver>>, + pub rx: std::sync::mpsc::Receiver>, +} + +impl Drop for LinkCook { + fn drop(&mut self) { + self.canceled.store(true, std::sync::atomic::Ordering::Relaxed); + } } /// The engine's refusal to paint while a bake shows (translated where it is toasted). @@ -175,6 +190,8 @@ pub struct TextureCraftApp { pub pbr_import: Option, pub resize: Option, pub link_cooks: Vec, + #[cfg(not(target_arch = "wasm32"))] + link_imports: Vec, pub thumbs: thumbs::Thumbs, /// The Layers panel's thumbnails and folded groups. pub layer_panel: layer_panel::LayerPanel, @@ -284,6 +301,8 @@ impl TextureCraftApp { pbr_import: None, resize: None, link_cooks: Vec::new(), + #[cfg(not(target_arch = "wasm32"))] + link_imports: Vec::new(), thumbs: thumbs::Thumbs::default(), layer_panel: layer_panel::LayerPanel::default(), viewport_rect: egui::Rect::NOTHING, @@ -347,6 +366,14 @@ impl TextureCraftApp { /// Run an engine command, reporting errors as toasts. Returns the result on success. pub fn run(&mut self, id: &str, params: Value) -> Option { self.ui.prefs.apply_resource_limits(); + #[cfg(not(target_arch = "wasm32"))] + if id == "material.load" + && self.session.project_open + && let Some(path) = params.get("path").and_then(Value::as_str) + && texturecraft_engine::link_cmds::is_link_file(std::path::Path::new(path)) + { + return self.start_link_import(path, ¶ms); + } // Going to a saved camera or a fixed view glides there; the control channel and CLI jump straight away let glides = id.starts_with("camera.select") || id.starts_with("camera.bookmark") || id.starts_with("camera.view"); @@ -362,6 +389,9 @@ impl TextureCraftApp { if matches!(id, "file.open" | "file.new" | "file.close" | "project.new" | "project.sample") { self.pbr_import = None; self.resize = None; + #[cfg(not(target_arch = "wasm32"))] + self.link_imports.clear(); + self.link_cooks.clear(); self.layer_panel = Default::default(); } @@ -448,11 +478,37 @@ impl TextureCraftApp { if self.link_cooks.iter().any(|c| c.layer == layer) { return; } + + #[cfg(not(target_arch = "wasm32"))] + if !self.link_imports.is_empty() || !self.link_cooks.is_empty() { + self.toast("Another MaterialCraft cook is still running", true); + return; + } + + self.ui.prefs.apply_resource_limits(); match self.session.link_job(layer, &serde_json::Map::new()) { Ok(job) => { + let baked = self.session.doc.project.locate(layer).and_then(|(set, _, _)| self.session.baked(set)); + let baked_inputs = job + .input_identifiers + .iter() + .filter_map(|identifier| texturecraft_engine::core::BakedMap::parse(identifier)) + .map(|map| link_bake_input::LinkBakeInput::capture(map, baked)) + .collect(); let values = job.values.clone(); + let size = job.size; + let canceled = std::sync::Arc::clone(&job.canceled); let rx = spawn_worker("texturecraft-link", move || job.run()); - self.link_cooks.push(LinkCook { layer, values, rx }); + self.link_cooks.push(LinkCook { + layer, + project: self.session.project_id, + mesh: self.session.mesh_id, + size, + baked_inputs, + canceled, + values, + rx, + }); } Err(e) => self.toast(e.to_string(), true), } @@ -462,27 +518,45 @@ impl TextureCraftApp { if self.link_cooks.is_empty() { return; } + ctx.request_repaint_after(std::time::Duration::from_millis(50)); + if self.session.stroke_active() { + return; + } + + self.link_cooks.retain(|cook| { + cook.project == self.session.project_id && cook.mesh == self.session.mesh_id && self.session.doc.project.locate(cook.layer).is_some() + }); let mut done = Vec::new(); self.link_cooks.retain(|c| match c.rx.try_recv() { Err(std::sync::mpsc::TryRecvError::Empty) => true, - Ok(r) => { - done.push((c.layer, c.values.clone(), r)); + Ok(result) => { + let Some((set, _, _)) = self.session.doc.project.locate(c.layer) else { + return false; + }; + + let resized = self.session.doc.project.texture_set(set).is_ok_and(|set| set.resolution != c.size); + let baked = self.session.baked(set); + let stale = resized || c.baked_inputs.iter().any(|input| !input.is_current(baked)); + done.push((c.layer, c.values.clone(), stale, result)); false } Err(std::sync::mpsc::TryRecvError::Disconnected) => { - done.push((c.layer, c.values.clone(), Err(texturecraft_engine::EngineError::Other("the MaterialCraft cook stopped".into())))); + done.push((c.layer, c.values.clone(), false, Err(texturecraft_engine::EngineError::Other("the MaterialCraft cook stopped".into())))); false } }); - for (layer, values, result) in done { + for (layer, values, stale, result) in done { let current = self.session.link_values(layer); match (result, current) { - // The layer is gone (undone, deleted): nothing to show. + // The layer can disappear while a cook is pending (_, Err(_)) => {} - (Ok(_), Ok(now)) if now != values => self.cook_link(layer), - (Ok(bytes), Ok(_)) => { - if let Err(e) = self.session.apply_link_cook(layer, &bytes, true) { + (result, Ok(now)) if now != values || stale => { + drop(result); + self.cook_link(layer); + } + (Ok(material), Ok(_)) => { + if let Err(e) = self.session.apply_link_cook(layer, &material, true) { self.toast(e.to_string(), true); } } @@ -711,6 +785,10 @@ impl TextureCraftApp { return; } + let Some(mut reservation) = texturecraft_engine::formats::RamReservation::try_for_save() else { + return; + }; + self.next_autosave = now + period; let dir = user_dir("autosave"); let path = dir.join(format!( @@ -718,11 +796,13 @@ impl TextureCraftApp { texturecraft_engine::formats::export::sanitize_file_stem(&self.session.doc.project.name), texturecraft_engine::core::PROJECT_EXT )); - let pf = self.session.project_file(); + let snapshot = self.session.project_snapshot(); let spawned = std::thread::Builder::new().name("texturecraft-autosave".into()).spawn(move || { - let result = std::fs::create_dir_all(&dir) - .map_err(|e| e.to_string()) - .and_then(|_| texturecraft_engine::formats::save_project(&path, &pf).map_err(|e| e.to_string())); + let result = + std::fs::create_dir_all(&dir).map_err(|e| e.to_string()).and_then(|_| reservation.save(&path, &snapshot).map_err(|error| error.to_string())); + drop(snapshot); + drop(reservation); + match result { Ok(()) => log::info!("autosaved {}", path.display()), Err(e) => log::warn!("autosave to {} failed: {e}", path.display()), @@ -828,6 +908,8 @@ impl eframe::App for TextureCraftApp { self.poll_bake(ctx); self.poll_resize(ctx); self.poll_link_cooks(ctx); + #[cfg(not(target_arch = "wasm32"))] + self.poll_link_imports(ctx); self.thumbs.sync_images(&self.session.images, self.session.revision); // The library shelf rescans when the window comes back (MaterialCraft may have sent a material) diff --git a/crates/ui-egui/src/link_bake_input.rs b/crates/ui-egui/src/link_bake_input.rs new file mode 100644 index 00000000..c3703101 --- /dev/null +++ b/crates/ui-egui/src/link_bake_input.rs @@ -0,0 +1,52 @@ +use std::sync::{Arc, Weak}; + +use texturecraft_engine::{compose::BakedMaps, core::BakedMap}; + +/// The identity of one baked map read by a linked cook. +pub(super) struct LinkBakeInput { + map: BakedMap, + resolution: Option, + pixels: Option>>, +} + +impl LinkBakeInput { + pub(super) fn capture(map: BakedMap, baked: Option<&BakedMaps>) -> Self { + let pixels = baked.and_then(|baked| baked.maps.get(&map)).map(Arc::downgrade); + + Self { map, resolution: baked.filter(|_| pixels.is_some()).map(|baked| baked.resolution), pixels } + } + + pub(super) fn is_current(&self, baked: Option<&BakedMaps>) -> bool { + let current = Self::capture(self.map, baked); + match (&self.pixels, current.pixels) { + (Some(previous), Some(now)) => self.resolution == current.resolution && previous.ptr_eq(&now), + (None, None) => true, + _ => false, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn replacing_a_baked_input_at_the_same_resolution_invalidates_its_cook() { + let earlier = Arc::new(vec![0.25; 4]); + let mut baked = BakedMaps { resolution: 2, maps: [(BakedMap::AmbientOcclusion, Arc::clone(&earlier))].into() }; + let ambient_occlusion = LinkBakeInput::capture(BakedMap::AmbientOcclusion, Some(&baked)); + let curvature = LinkBakeInput::capture(BakedMap::Curvature, Some(&baked)); + + assert!(ambient_occlusion.is_current(Some(&baked))); + assert_eq!(Arc::strong_count(&earlier), 2); + + baked.maps.insert(BakedMap::AmbientOcclusion, Arc::new(vec![0.75; 4])); + + assert!(!ambient_occlusion.is_current(Some(&baked))); + assert!(curvature.is_current(Some(&baked))); + + baked.maps.insert(BakedMap::Curvature, Arc::new(vec![0.5; 4])); + + assert!(!curvature.is_current(Some(&baked))); + } +} diff --git a/crates/ui-egui/src/link_import.rs b/crates/ui-egui/src/link_import.rs new file mode 100644 index 00000000..0e7d8cd8 --- /dev/null +++ b/crates/ui-egui/src/link_import.rs @@ -0,0 +1,159 @@ +//! The first procedural bake can take as long as a re-cook. Keep the native UI responsive. +use std::path::PathBuf; +use std::sync::mpsc::{Receiver, TryRecvError}; +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; + +use crate::TextureCraftApp; +use serde_json::{Value, json}; +use texturecraft_engine::link_cmds::{CookedMaterial, cook_link_file}; +use texturecraft_engine::materials::Place; + +type ImportedMaterial = (Arc>, &'static str, CookedMaterial); + +pub(super) struct LinkImport { + project: u64, + mesh: u64, + canceled: Arc, + set: usize, + resolution: u32, + place: Place, + result: Receiver>, +} + +impl Drop for LinkImport { + fn drop(&mut self) { + self.canceled.store(true, Ordering::Relaxed); + } +} + +impl TextureCraftApp { + pub(super) fn start_link_import(&mut self, path: &str, params: &Value) -> Option { + if !self.link_imports.is_empty() || !self.link_cooks.is_empty() { + self.toast("Another MaterialCraft cook is still running", true); + return None; + } + + let set = self.session.ts(); + let texture_set = match self.session.doc.project.texture_set(set) { + Ok(set) => set, + Err(error) => { + self.toast(error.to_string(), true); + return None; + } + }; + + // The artist can select a different layer while this import cooks + let place = match Place::from_params(params) { + Place::Selection => self.session.selected_layer().map_or_else( + || texture_set.stack.last().map_or(Place::Index(0), |target| Place::Above(*target, false)), + |target| Place::Above(target, self.session.doc.project.layer(target).is_ok_and(|layer| layer.is_group())), + ), + explicit => explicit, + }; + let resolution = texture_set.resolution; + let baked = self.session.baked(set).cloned(); + let path = PathBuf::from(path); + let canceled = Arc::new(AtomicBool::new(false)); + let worker_canceled = Arc::clone(&canceled); + let budget = self.session.link_memory_budget(); + let rx = super::spawn_worker("texturecraft-material-import", move || { + let (source, cooked) = cook_link_file(&path, resolution, baked, budget, worker_canceled)?; + + // A pending installation only needs its source package and cooked outputs + Ok((source.package, source.source_ext, cooked)) + }); + self.link_imports.push(LinkImport { project: self.session.project_id, mesh: self.session.mesh_id, canceled, set, resolution, place, result: rx }); + self.toast("Cooking MaterialCraft material", false); + Some(json!({"pending":true})) + } + + pub(super) fn poll_link_imports(&mut self, ctx: &egui::Context) { + if self.link_imports.is_empty() { + return; + } + + ctx.request_repaint_after(std::time::Duration::from_millis(50)); + if self.session.stroke_active() { + return; + } + + for mut job in std::mem::take(&mut self.link_imports) { + if job.project != self.session.project_id + || job.mesh != self.session.mesh_id + || !self.session.doc.project.texture_set(job.set).is_ok_and(|set| set.resolution == job.resolution) + { + self.toast("Material import canceled because the target texture set changed", true); + continue; + } + + if let Place::Above(target, _) | Place::Below(target) = job.place + && !self.session.doc.project.locate(target).is_some_and(|(set, _, _)| set == job.set) + { + self.toast("Material import canceled because the target layer changed", true); + continue; + } + + let result = match job.result.try_recv() { + Err(TryRecvError::Empty) => { + self.link_imports.push(job); + continue; + } + Err(TryRecvError::Disconnected) => Err(texturecraft_engine::EngineError::Other("the material import stopped".into())), + Ok(result) => result, + }; + + match result.and_then(|(package, source_ext, cooked)| { + self.session.add_cooked_material_to_set(&cooked, package, source_ext, job.set, std::mem::replace(&mut job.place, Place::Selection)) + }) { + Ok(_) => self.toast("Material imported", false), + Err(error) => self.toast(error.to_string(), true), + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + use texturecraft_engine::{Session, execute}; + + #[test] + fn pending_import_keeps_its_original_layer_position() { + let Some(cli) = std::env::var_os("MATERIALCRAFT_CLI") else { return }; + let path = std::env::temp_dir().join(format!("texturecraft-pending-import-{}.mcraft", std::process::id())); + let source = std::process::Command::new(cli).args(["new", "pbr"]).arg(&path).args(["--size", "32"]).output().unwrap(); + assert!(source.status.success()); + let session = Session::new_project("Paint", texturecraft_engine::geom::primitives::cube(1.0), "builtin:cube", None, 128).unwrap(); + let context = eframe::CreationContext::_new_kittest(egui::Context::default()); + let mut app = TextureCraftApp::new(&context, session); + execute(&mut app.session, "layer.add.paint", &json!({"name":"Drop target"})).unwrap(); + let target = app.session.selected_layer().unwrap(); + + let pixels = Arc::new(vec![0.25; 128 * 128]); + let replaced = Arc::downgrade(&pixels); + let baked = texturecraft_engine::compose::BakedMaps { resolution: 128, maps: [(texturecraft_engine::core::BakedMap::Curvature, pixels)].into() }; + app.session.doc.baked.insert(0, baked); + + app.start_link_import(path.to_str().unwrap(), &Value::Null).unwrap(); + app.session.doc.baked.remove(&0); + execute(&mut app.session, "layer.add.paint", &json!({"name":"Later selection"})).unwrap(); + let later = app.session.selected_layer().unwrap(); + let job = app.link_imports.first_mut().unwrap(); + let result = job.result.recv_timeout(Duration::from_secs(60)).unwrap().unwrap(); + assert!(replaced.upgrade().is_none(), "a pending import retained replaced bake pixels"); + + let (sender, receiver) = std::sync::mpsc::channel(); + sender.send(Ok(result)).unwrap(); + job.result = receiver; + app.poll_link_imports(&context.egui_ctx); + std::fs::remove_file(path).unwrap(); + + let imported = app.session.selected_layer().unwrap(); + let stack = &app.session.doc.project.texture_sets[0].stack; + assert!(stack.ends_with(&[target, imported, later]), "{stack:?}"); + } +} diff --git a/docs/architecture.md b/docs/architecture.md index ff4e04df..f7910199 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -122,11 +122,7 @@ on the CPU by `render::displace` from the composited height. ## MaterialCraft materials A fill can carry a `MaterialLink` (`core::layer`): a MaterialCraft graph whose cooked maps are the -fill's image sources. `engine::link_cmds` imports `.mcmat` files (`formats::mcmat`), keeps the -`.mcraft` or `.sbs` source as a resource, and re-cooks its named graph by running `materialcraft-cli` with the link's -values, the texture set's resolution and its baked mesh maps (`LinkJob::run`, on a worker thread -in the UI). Every cook stores its maps under fresh resource names, so undo, redo and the decoded -image cache never disagree. The apps share the `craft-mcmat` format library and the command line; neither depends on the other app. +fill's image sources. `engine::link_cmds` imports a procedural MCMAT v3 by asking MaterialCraft to bake the entry graph. It keeps the archive and the interface reported by the CLI with the layer. Parameter edits can re-cook at the texture set's resolution. After a bake or a resize, run `material.link.cook` to use the new mesh maps or resolution. Each bake installs fresh map resources so duplicates, undo and redo preserve their own results. Both apps use the shared `craft-mcmat` library and communicate through the CLI. ## Commands and automation diff --git a/docs/control-protocol.md b/docs/control-protocol.md index 23bf09f3..f0e01239 100644 --- a/docs/control-protocol.md +++ b/docs/control-protocol.md @@ -502,12 +502,11 @@ view, Alt+Shift+left drag orbits in 90° steps. ## MaterialCraft materials -A `.mcmat` portable material (MaterialCraft's format, read here by `formats::mcmat`; the manifest is -documented in MaterialCraft's `docs/control-protocol.md` § Portable materials) becomes a fill layer -whose channel sources are its cooked maps and whose `link` (shown by `project.inspect`) holds the -graph's exposed inputs, presets and image inputs. Both apps use the shared `craft-mcmat` container. -Versions 1 and 2 are accepted; version 1 contains a legacy JSON `.mcraft` source that current -MaterialCraft cannot reopen, so its cooked maps remain usable but re-cooking needs a compatible older version. +A v3 `.mcmat` holds a published SBS graph and its source assets. TextureCraft uses the shared `craft-mcmat` reader and cooks the archive through `materialcraft-cli`. The resulting fill stores its channel textures and the original archive in the project. Linked cooks use the Settings percentage of available RAM. A cook that exceeds the estimated budget reports the required and configured byte counts without replacing the current fill. Its exposed values, presets and mesh-map inputs appear in `project.inspect`. Republish older MCMAT files from their original authoring project. Procedural imports and re-cooks require the desktop CLI. Saved projects retain their baked maps for use in the browser. + +Saving and linked cooks share one reservation for the configured RAM allowance. Autosave waits +until a cook releases the reservation. An explicit save or cook that conflicts with the other +operation returns a busy error and can be retried. | Command | Params | |---|---| @@ -515,9 +514,9 @@ MaterialCraft cannot reopen, so its cooked maps remain usable but re-cooking nee | `material.link.set` | `{layer?, id, value, cook?: true}`: a number, `[..]`, a bool, `"#rrggbb"` or a typed `{"t", "v"}`; re-cooks unless `cook: false` | | `material.link.preset` | `{layer?, name, cook?: true}` | | `material.link.cook` | `{layer?}`: re-cook at the texture set's resolution with its baked mesh maps (after a bake or a resize) | -| `material.link.saveSource` | `{layer?, path}`: the `.mcraft` or `.sbs` source, to open in MaterialCraft | +| `material.link.saveSource` | `{layer?, path}`: the original linked source with its `.mcmat`, `.mcraft` or `.sbs` extension | -Re-cooks use `materialcraft-cli batch` to open the saved graph by name and run `file.exportMaterial` with its size, values and inputs: +For a 1024-pixel texture set, TextureCraft requests `materialcraft-cli export source.mcmat maps --profile maps --size 1024x1024` with repeated `--set` and `--input` arguments. It discovers the published image inputs through `info` when the texture set supplies mesh maps. The desktop worker decodes the returned maps before installing the fill at its original drop target. The executable is resolved from `$MATERIALCRAFT_CLI`, else `materialcraft-cli` next to the TextureCraft executable, else on `PATH` (the app is `$MATERIALCRAFT` / `materialcraft`). Baked maps feed graph input nodes whose identifier matches the map's shared id (`ambientOcclusion`, `curvature`, `thickness`, `position`,