From a75497d63405b448b03227ebcffbb08fca18ef48 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Thu, 12 Nov 2026 06:29:07 -0500 Subject: [PATCH] security: importers and preset application index instead of rescanning * security: importers and preset application index instead of rescanning * address review: keep first match when indexing names, skip used mask ids, move presets instead of cloning * address review: a dry-run import keeps its rows, not the parsed presets * merge main into fix/sec-importers-indexed * address review: key duplicate presets by the settings value, so 0.0 and -0.0 are one preset * preset import index: key numbers as f64 so 0 and 0.0 cannot collide --- crates/develop/src/controls.rs | 13 +++-- crates/develop/src/presets.rs | 37 ++++++++++++- crates/engine/src/cmd/curves.rs | 15 +++++- crates/engine/src/cmd/lut_profiles.rs | 11 ++-- crates/engine/src/cmd/mask_effect_files.rs | 21 +++++++- crates/engine/src/cmd/preset_files.rs | 29 +++++++--- crates/engine/src/crs_masks.rs | 7 ++- crates/engine/src/presets.rs | 46 +++++++++++----- crates/engine/src/tests/presets_scale.rs | 63 ++++++++++++++++++++++ 9 files changed, 211 insertions(+), 31 deletions(-) diff --git a/crates/develop/src/controls.rs b/crates/develop/src/controls.rs index 589bcc6f19..d3447c2348 100644 --- a/crates/develop/src/controls.rs +++ b/crates/develop/src/controls.rs @@ -455,9 +455,16 @@ fn indexed_field<'a>(s: &'a mut DevelopSettings, id: &str) -> Option<&'a mut f64 } fn indexed_get(s: &DevelopSettings, id: &str) -> Option { - // (a clone keeps one accessor for get and set; these lists are tiny) - let mut c = DevelopSettings { point_colors: s.point_colors.clone(), red_eye: s.red_eye.clone(), ..DevelopSettings::default() }; - indexed_field(&mut c, id).map(|v| *v) + // one accessor for get and set: a copy of the one element read (never of the lists, which a + // read of every indexed control would copy once per control) + let (fam, i, field) = split_indexed(id)?; + let mut c = DevelopSettings::default(); + match fam { + "pointColor" => c.point_colors.push(*s.point_colors.get(i)?), + "redEye" => c.red_eye.push(*s.red_eye.get(i)?), + _ => return None, + } + indexed_field(&mut c, &format!("{fam}.0.{field}")).map(|v| *v) } fn indexed_set(s: &mut DevelopSettings, id: &str, v: f64) -> bool { diff --git a/crates/develop/src/presets.rs b/crates/develop/src/presets.rs index a1072a89ac..0ea3d12f8c 100644 --- a/crates/develop/src/presets.rs +++ b/crates/develop/src/presets.rs @@ -313,13 +313,28 @@ impl Preset { /// The preset's masks, added to the photo's own (ones it already has are skipped), their effect /// scaled by the amount. fn add_masks(out: &mut DevelopSettings, masks: &[Value], amount: f64) { + // the masks by name, kept up to date: a preset of many masks compares each with its + // namesakes only, not with every mask added before it + let mut by_name: std::collections::HashMap> = std::collections::HashMap::new(); + for (i, q) in out.masks.iter().enumerate() { + by_name.entry(q.name.clone()).or_default().push(i); + } + let mut next_id = out.next_mask_id(); + let mut used: std::collections::HashSet = out.masks.iter().map(|q| q.id).collect(); for m in masks { let Ok(mut m) = serde_json::from_value::(m.clone()) else { continue }; - if out.masks.iter().any(|q| q.name == m.name && q.components == m.components) { + if by_name.get(&m.name).is_some_and(|ix| ix.iter().any(|&i| out.masks.get(i).is_some_and(|q| q.components == m.components))) { continue; } - m.id = out.next_mask_id(); + // ids run on from the first free one, past any a mask already has (a scan for the + // next free id per mask would be another pass over them) + while used.contains(&next_id) { + next_id = next_id.checked_add(1).unwrap_or(1); + } + m.id = next_id; + used.insert(next_id); m.adjust.amount = (m.adjust.amount * amount).clamp(0.0, 200.0); + by_name.entry(m.name.clone()).or_default().push(out.masks.len()); out.masks.push(m); } } @@ -604,4 +619,22 @@ mod tests { ids.dedup(); assert_eq!(ids.len(), 3, "{:?}", out.masks); } + + #[test] + fn preset_masks_skip_ids_an_existing_mask_has_after_the_first_free_one() { + // the first free id is 1, and 2 is taken: the second added mask must not get 2 + let theirs = |id| crate::Mask { id, name: format!("T{id}"), ..Default::default() }; + let s = DevelopSettings { masks: vec![theirs(u32::MAX), theirs(2)], ..Default::default() }; + let p = Preset { + id: "p".into(), + name: "P".into(), + group: "User".into(), + settings: json!({"masks": [{"name": "A"}, {"name": "B"}]}), + favorite: false, + builtin: false, + }; + let mut ids: Vec = p.apply(&s, 1.0).masks.iter().map(|m| m.id).collect(); + ids.sort_unstable(); + assert_eq!(ids, [1, 2, 3, u32::MAX]); + } } diff --git a/crates/engine/src/cmd/curves.rs b/crates/engine/src/cmd/curves.rs index 7388f8e573..0e52ad0929 100644 --- a/crates/engine/src/cmd/curves.rs +++ b/crates/engine/src/cmd/curves.rs @@ -311,6 +311,13 @@ fn import_presets(s: &mut Session, p: &Value) -> Result { return Err(bad(C, "missing `path` / `paths`")); } let (mut imported, mut failed) = (Vec::new(), Vec::new()); + // the user presets by name, kept up to date: a file of many presets replaces or adds each + // without a scan of all the others + // (the first of two same-named presets, as `upsert` and the lookups find it) + let mut by_name: std::collections::HashMap = std::collections::HashMap::new(); + for (i, x) in s.curve_presets.iter().enumerate() { + by_name.entry(x.name.to_ascii_lowercase()).or_insert(i); + } for f in &paths { match std::fs::read(f).map_err(|e| e.to_string()).and_then(|b| parse_file(&b)) { Ok(list) => { @@ -321,7 +328,13 @@ fn import_presets(s: &mut Session, p: &Value) -> Result { preset.name = format!("{} (imported)", preset.name); } imported.push(preset.name.clone()); - upsert(s, preset); + match by_name.get(&preset.name.to_ascii_lowercase()).and_then(|&i| s.curve_presets.get_mut(i)) { + Some(x) => *x = preset, + None => { + by_name.insert(preset.name.to_ascii_lowercase(), s.curve_presets.len()); + s.curve_presets.push(preset); + } + } } } Err(e) => failed.push(json!([f, e])), diff --git a/crates/engine/src/cmd/lut_profiles.rs b/crates/engine/src/cmd/lut_profiles.rs index 3f3c2b3428..50cb122300 100644 --- a/crates/engine/src/cmd/lut_profiles.rs +++ b/crates/engine/src/cmd/lut_profiles.rs @@ -103,6 +103,10 @@ fn import(s: &mut Session, p: &Value) -> Result { } let (files, mut failed) = cube_files(&paths); let mut imported = Vec::new(); + // the ids taken and the next suffix per slug: a bundle of same-titled LUTs doesn't probe the + // whole list from 2 for each one + let mut taken: std::collections::HashSet = s.lut_profiles.iter().map(|x| x.id.clone()).collect(); + let mut next_suffix: std::collections::HashMap = std::collections::HashMap::new(); for (name, text, group) in files { let lut = match lightcraft_pipeline::lut::Lut3d::parse_cube(&text) { Ok(l) => l, @@ -115,11 +119,12 @@ fn import(s: &mut Session, p: &Value) -> Result { let group = super::super::preset_import::group_from_dir(&group.unwrap_or_default()).unwrap_or_else(|| "Imported".into()); let slug = crate::presets::slug(&format!("{group}-{display}")); let mut id = format!("lut:{slug}"); - let mut n = 2; - while s.lut_profiles.iter().any(|x| x.id == id) { + let n = next_suffix.entry(slug.clone()).or_insert(2); + while taken.contains(&id) { id = format!("lut:{slug}-{n}"); - n += 1; + *n += 1; } + taken.insert(id.clone()); let file = match &dir { Some(d) => { let f = d.join(format!("{}.cube", id.trim_start_matches("lut:"))); diff --git a/crates/engine/src/cmd/mask_effect_files.rs b/crates/engine/src/cmd/mask_effect_files.rs index 4b94acb64b..0b36c73d17 100644 --- a/crates/engine/src/cmd/mask_effect_files.rs +++ b/crates/engine/src/cmd/mask_effect_files.rs @@ -10,7 +10,7 @@ use lightcraft_develop::LocalAdjustments; use serde_json::{Map, Value, json}; -use super::mask_effects::{MaskEffect, upsert}; +use super::mask_effects::MaskEffect; use super::{bad, str_param}; use crate::preset_import::{Lua, group_from_dir, parse_lua, read_zip}; use crate::{Result, Session}; @@ -397,8 +397,25 @@ pub fn add_found(s: &mut Session, found: Vec, group: Option<&str>, dry: b return Ok((rows, div_rows)); } s.mask_effects.retain(|e| !(e.is_empty() && dividers.iter().any(|(d, _)| d.effect.name.eq_ignore_ascii_case(&e.name)))); + // as `upsert`, with the names indexed once: a file of many effects doesn't scan the list per effect + // (the first of two same-named effects, as `upsert` and readers find it) + let mut by_name: std::collections::HashMap = std::collections::HashMap::new(); + for (i, x) in s.mask_effects.iter().enumerate() { + by_name.entry(x.name.trim().to_ascii_lowercase()).or_insert(i); + } for f in effects { - upsert(s, f.effect); + let e = f.effect; + let key = e.name.trim().to_ascii_lowercase(); + match by_name.get(&key).and_then(|&i| s.mask_effects.get_mut(i)) { + Some(x) => { + let favorite = x.favorite || e.favorite; + *x = MaskEffect { favorite, ..e }; + } + None => { + by_name.insert(key, s.mask_effects.len()); + s.mask_effects.push(e); + } + } } s.save_prefs()?; Ok((rows, div_rows)) diff --git a/crates/engine/src/cmd/preset_files.rs b/crates/engine/src/cmd/preset_files.rs index b64a69fc76..9c555f9fb5 100644 --- a/crates/engine/src/cmd/preset_files.rs +++ b/crates/engine/src/cmd/preset_files.rs @@ -23,6 +23,8 @@ fn import(s: &mut Session, p: &Value) -> Result { // .lceffect, also inside zips — become mask effects let mut effects = Vec::new(); let mut effect_failed = Vec::new(); + // every file's presets, added in one go below (adding one at a time indexed the library each time) + let mut pending: Vec<(lightcraft_develop::Preset, Value)> = Vec::new(); for top in &paths { let from_dir = std::path::Path::new(top).is_dir(); for f in expand_preset_paths(std::slice::from_ref(top)) { @@ -65,13 +67,14 @@ fn import(s: &mut Session, p: &Value) -> Result { it.unmapped.extend(crate::profile_refs::report(&look, &r)); profile = json!({"name": look["name"], "id": it.preset.settings["profile"].get("id")}); } - let (name, grp) = (it.preset.name.clone(), it.preset.group.clone()); - let id = if dry { Some(it.preset.id.clone()) } else { s.add_presets(vec![it.preset]).pop() }; - match id { - Some(id) => { - imported.push(json!({"id": id, "name": name, "group": grp, "file": f, "unmapped": it.unmapped, "profile": profile})) - } - None => skipped += 1, + let mut row = + json!({"name": it.preset.name, "group": it.preset.group, "file": f, "unmapped": it.unmapped, "profile": profile}); + if dry { + // a dry run keeps nothing: the row is final, the parsed preset is dropped + row["id"] = json!(it.preset.id); + imported.push(row); + } else { + pending.push((it.preset, row)); } } } @@ -79,6 +82,18 @@ fn import(s: &mut Session, p: &Value) -> Result { } } } + // split, not cloned: a large import holds one copy of the parsed presets + let (presets, rows): (Vec<_>, Vec<_>) = pending.into_iter().unzip(); + let ids = if dry { Vec::new() } else { s.add_presets_each(presets) }; + for (mut row, id) in rows.into_iter().zip(ids) { + match id { + Some(id) => { + row["id"] = json!(id); + imported.push(row); + } + None => skipped += 1, + } + } failed.extend(effect_failed.into_iter().map(|(f, e)| json!([f, e]))); let (effects, dividers) = mef::add_found(s, effects, group.as_deref(), dry)?; Ok(json!({"imported": imported, "skipped": skipped, "failed": failed, "effects": effects, "effectDividers": dividers})) diff --git a/crates/engine/src/crs_masks.rs b/crates/engine/src/crs_masks.rs index a54bd620d5..2bc2298573 100644 --- a/crates/engine/src/crs_masks.rs +++ b/crates/engine/src/crs_masks.rs @@ -274,15 +274,20 @@ pub fn masks(values: &Values, aspect: f64) -> (Vec, Vec) { skipped.extend(unmapped_local(c)); // brush dabs of one correction are one brush component let mut merged: Vec = Vec::new(); + // where the brush of each op sits in `merged` (no scan of it per dab) + let mut brush_at: std::collections::HashMap = std::collections::HashMap::new(); for p in parts { let is_brush = p["shape"]["kind"] == "brush"; if is_brush - && let Some(prev) = merged.iter_mut().find(|q| q["shape"]["kind"] == "brush" && q["op"] == p["op"]) + && let Some(prev) = brush_at.get(&p["op"].to_string()).and_then(|&i| merged.get_mut(i)) && let (Some(a), Some(b)) = (prev["shape"]["strokes"].as_array_mut(), p["shape"]["strokes"].as_array()) { a.extend(b.iter().cloned()); continue; } + if is_brush { + brush_at.insert(p["op"].to_string(), merged.len()); + } merged.push(p); } let n = out.len() + 1; diff --git a/crates/engine/src/presets.rs b/crates/engine/src/presets.rs index 2ad6e5d2ef..a1aa603c83 100644 --- a/crates/engine/src/presets.rs +++ b/crates/engine/src/presets.rs @@ -198,40 +198,62 @@ impl Session { /// Add imported presets. Identical presets (same name, group and settings) already present are /// skipped; id clashes get a fresh id. Returns the ids added. pub fn add_presets(&mut self, presets: Vec) -> Vec { + self.add_presets_each(presets).into_iter().flatten().collect() + } + + /// [`Self::add_presets`] reporting per preset, in order: its id, or `None` when it was skipped + /// as already present. One call indexes the library once, so an importer adds a whole file's + /// presets through it rather than one at a time. + pub fn add_presets_each(&mut self, presets: Vec) -> Vec> { use std::collections::{HashMap, HashSet}; // indexed: a collection of 10 000+ presets was quadratic (minutes) to import let mut ids: HashSet = self.presets.iter().map(|q| q.id.clone()).collect(); - let mut by_name: HashMap<(String, String), Vec> = HashMap::new(); - for (i, q) in self.presets.iter().enumerate() { - by_name.entry((q.name.clone(), q.group.clone())).or_default().push(i); - } - let mut added = Vec::new(); + // (name, group, settings): a preset is looked up rather than compared with every namesake. + // Every number is keyed as an f64 with -0.0 folded into 0.0: `0` and `0.0` are unequal + // `Value`s with equal hashes, so a crafted file of them would collide in one bucket. + // shortcut: integers past 2^53 lose precision in the key, so two such presets are one + let key_of = |p: &Preset| (p.name.clone(), p.group.clone(), canonical_numbers(&p.settings)); + let mut present: HashSet<(String, String, serde_json::Value)> = self.presets.iter().map(key_of).collect(); + // the next suffix to try per generated id, so a batch of clashes doesn't probe from 2 each time + let mut next_suffix: HashMap = HashMap::new(); + let mut added = Vec::with_capacity(presets.len()); for mut p in presets { - let key = (p.name.clone(), p.group.clone()); - let same = by_name.get(&key).is_some_and(|ix| ix.iter().any(|&i| self.presets.get(i).is_some_and(|q| q.settings == p.settings))); - if same { + let key = key_of(&p); + if present.contains(&key) { + added.push(None); continue; } if p.id.trim().is_empty() || p.id.starts_with("lc.") || ids.contains(&p.id) { let base = format!("user.{}", slug(&format!("{}-{}", p.group, p.name))); let mut id = base.clone(); - let mut n = 2; + let n = next_suffix.entry(base.clone()).or_insert(2); while ids.contains(&id) { id = format!("{base}-{n}"); - n += 1; + *n += 1; } p.id = id; } p.builtin = false; ids.insert(p.id.clone()); - by_name.entry(key).or_default().push(self.presets.len()); - added.push(p.id.clone()); + present.insert(key); + added.push(Some(p.id.clone())); self.presets.push(p); } added } } +/// `v` with every number replaced by its f64 (`-0.0` as `0.0`), so equal numbers hash and compare alike. +fn canonical_numbers(v: &serde_json::Value) -> serde_json::Value { + use serde_json::Value; + match v { + Value::Number(n) => n.as_f64().and_then(|f| serde_json::Number::from_f64(f + 0.0)).map_or_else(|| v.clone(), Value::Number), + Value::Array(a) => Value::Array(a.iter().map(canonical_numbers).collect()), + Value::Object(o) => Value::Object(o.iter().map(|(k, x)| (k.clone(), canonical_numbers(x))).collect()), + _ => v.clone(), + } +} + fn p(group: &str, id: &str, name: &str, settings: serde_json::Value) -> Preset { Preset { id: format!("lc.{id}"), name: name.into(), group: group.into(), settings, favorite: false, builtin: true } } diff --git a/crates/engine/src/tests/presets_scale.rs b/crates/engine/src/tests/presets_scale.rs index 866b7deb31..149c82ccce 100644 --- a/crates/engine/src/tests/presets_scale.rs +++ b/crates/engine/src/tests/presets_scale.rs @@ -94,6 +94,50 @@ fn importing_thousands_of_presets_is_not_quadratic() { assert!(t.elapsed().as_secs() < 20, "took {:?}", t.elapsed()); } +#[test] +fn a_preset_whose_settings_differ_only_in_the_sign_of_zero_is_a_repeat() { + let mut s = Session::new(); + let preset = |x: f64| Preset { + id: "user.z".into(), + name: "Zero".into(), + group: "Coll".into(), + settings: json!({"light": {"exposure": x}}), + favorite: false, + builtin: false, + }; + assert!(s.add_presets_each(vec![preset(0.0)])[0].is_some()); + assert_eq!(s.add_presets_each(vec![preset(-0.0)]), vec![None]); +} + +#[test] +fn thousands_of_presets_sharing_a_name_and_id_get_unique_ids_in_one_pass() { + let mut s = Session::new(); + let t = Instant::now(); + // the same name, group and id throughout, each with its own settings: every one clashes + let same: Vec = (0..20_000) + .map(|i| Preset { + id: "user.x".into(), + name: "Look".into(), + group: "Coll".into(), + settings: json!({"light": {"exposure": i}}), + favorite: false, + builtin: false, + }) + .collect(); + let each = s.add_presets_each(same); + assert_eq!(each.len(), 20_000); + assert!(each.iter().all(Option::is_some)); + // (the session may hold built-in presets too: only the added ones are counted) + let ids: std::collections::HashSet<_> = s.presets.iter().filter(|p| p.group == "Coll").map(|p| p.id.as_str()).collect(); + assert_eq!(ids.len(), 20_000); + // a repeat is skipped, per preset + let last = s.presets.last().cloned().unwrap(); + let again = s.add_presets_each(vec![last.clone(), Preset { settings: json!({"light": {"exposure": -1}}), ..last }]); + assert_eq!(again[0], None); + assert!(again[1].is_some()); + assert!(t.elapsed().as_secs() < 20, "took {:?}", t.elapsed()); +} + /// Timing with a real preset collection: `LIGHTCRAFT_BENCH_PRESETS=path/to/presets.json cargo test /// -p lightcraft-engine --release presets_scale_bench -- --ignored --nocapture`. The file is copied /// into a temporary library; the original is only read. @@ -124,3 +168,22 @@ fn presets_scale_bench() { eprintln!("preset.favorite (saves presets.json): {:?}", t.elapsed()); let _ = std::fs::remove_dir_all(&lib); } + +#[test] +fn integer_and_float_spellings_of_one_number_are_one_preset() { + // 15 slots of `0` / `0.0` / `-0.0`: all spellings are the same preset, so none collide in the index + let mut s = Session::new(); + let preset = |i: usize| { + let spell = |bit: usize| match (i >> bit) % 3 { + 0 => json!(0), + 1 => json!(0.0), + _ => json!(-0.0), + }; + let slots: Vec<_> = (0..15).map(spell).collect(); + Preset { id: "user.n".into(), name: "N".into(), group: "G".into(), settings: json!({"v": slots}), favorite: false, builtin: false } + }; + let t = Instant::now(); + let added = s.add_presets_each((0..4000).map(preset).collect()); + assert_eq!(added.iter().filter(|a| a.is_some()).count(), 1); + assert!(t.elapsed().as_secs() < 20); +}