security: importers and preset application index instead of rescanning

* security: importers and preset application index instead of rescanning

* address review: keep first match when indexing names, skip used mask ids, move presets instead of cloning

* address review: a dry-run import keeps its rows, not the parsed presets

* merge main into fix/sec-importers-indexed

* address review: key duplicate presets by the settings value, so 0.0 and -0.0 are one preset

* preset import index: key numbers as f64 so 0 and 0.0 cannot collide
This commit is contained in:
Dylan M. Taylor authored and GitHub committed 2026-11-12 06:29:07 -05:00
1 parent 9563f22969
commit a75497d634
9 files changed
+211 -31

No files matched your search

+10 -3
View File
@@ -455,9 +455,16 @@ fn indexed_field<'a>(s: &'a mut DevelopSettings, id: &str) -> Option<&'a mut f64
}
fn indexed_get(s: &DevelopSettings, id: &str) -> Option<f64> {
// (a clone keeps one accessor for get and set; these lists are tiny)
let mut c = DevelopSettings { point_colors: s.point_colors.clone(), red_eye: s.red_eye.clone(), ..DevelopSettings::default() };
indexed_field(&mut c, id).map(|v| *v)
// one accessor for get and set: a copy of the one element read (never of the lists, which a
// read of every indexed control would copy once per control)
let (fam, i, field) = split_indexed(id)?;
let mut c = DevelopSettings::default();
match fam {
"pointColor" => c.point_colors.push(*s.point_colors.get(i)?),
"redEye" => c.red_eye.push(*s.red_eye.get(i)?),
_ => return None,
}
indexed_field(&mut c, &format!("{fam}.0.{field}")).map(|v| *v)
}
fn indexed_set(s: &mut DevelopSettings, id: &str, v: f64) -> bool {
+35 -2
View File
@@ -313,13 +313,28 @@ impl Preset {
/// The preset's masks, added to the photo's own (ones it already has are skipped), their effect
/// scaled by the amount.
fn add_masks(out: &mut DevelopSettings, masks: &[Value], amount: f64) {
// the masks by name, kept up to date: a preset of many masks compares each with its
// namesakes only, not with every mask added before it
let mut by_name: std::collections::HashMap<String, Vec<usize>> = std::collections::HashMap::new();
for (i, q) in out.masks.iter().enumerate() {
by_name.entry(q.name.clone()).or_default().push(i);
}
let mut next_id = out.next_mask_id();
let mut used: std::collections::HashSet<u32> = out.masks.iter().map(|q| q.id).collect();
for m in masks {
let Ok(mut m) = serde_json::from_value::<crate::Mask>(m.clone()) else { continue };
if out.masks.iter().any(|q| q.name == m.name && q.components == m.components) {
if by_name.get(&m.name).is_some_and(|ix| ix.iter().any(|&i| out.masks.get(i).is_some_and(|q| q.components == m.components))) {
continue;
}
m.id = out.next_mask_id();
// ids run on from the first free one, past any a mask already has (a scan for the
// next free id per mask would be another pass over them)
while used.contains(&next_id) {
next_id = next_id.checked_add(1).unwrap_or(1);
}
m.id = next_id;
used.insert(next_id);
m.adjust.amount = (m.adjust.amount * amount).clamp(0.0, 200.0);
by_name.entry(m.name.clone()).or_default().push(out.masks.len());
out.masks.push(m);
}
}
@@ -604,4 +619,22 @@ mod tests {
ids.dedup();
assert_eq!(ids.len(), 3, "{:?}", out.masks);
}
#[test]
fn preset_masks_skip_ids_an_existing_mask_has_after_the_first_free_one() {
// the first free id is 1, and 2 is taken: the second added mask must not get 2
let theirs = |id| crate::Mask { id, name: format!("T{id}"), ..Default::default() };
let s = DevelopSettings { masks: vec![theirs(u32::MAX), theirs(2)], ..Default::default() };
let p = Preset {
id: "p".into(),
name: "P".into(),
group: "User".into(),
settings: json!({"masks": [{"name": "A"}, {"name": "B"}]}),
favorite: false,
builtin: false,
};
let mut ids: Vec<u32> = p.apply(&s, 1.0).masks.iter().map(|m| m.id).collect();
ids.sort_unstable();
assert_eq!(ids, [1, 2, 3, u32::MAX]);
}
}
+14 -1
View File
@@ -311,6 +311,13 @@ fn import_presets(s: &mut Session, p: &Value) -> Result<Value> {
return Err(bad(C, "missing `path` / `paths`"));
}
let (mut imported, mut failed) = (Vec::new(), Vec::new());
// the user presets by name, kept up to date: a file of many presets replaces or adds each
// without a scan of all the others
// (the first of two same-named presets, as `upsert` and the lookups find it)
let mut by_name: std::collections::HashMap<String, usize> = std::collections::HashMap::new();
for (i, x) in s.curve_presets.iter().enumerate() {
by_name.entry(x.name.to_ascii_lowercase()).or_insert(i);
}
for f in &paths {
match std::fs::read(f).map_err(|e| e.to_string()).and_then(|b| parse_file(&b)) {
Ok(list) => {
@@ -321,7 +328,13 @@ fn import_presets(s: &mut Session, p: &Value) -> Result<Value> {
preset.name = format!("{} (imported)", preset.name);
}
imported.push(preset.name.clone());
upsert(s, preset);
match by_name.get(&preset.name.to_ascii_lowercase()).and_then(|&i| s.curve_presets.get_mut(i)) {
Some(x) => *x = preset,
None => {
by_name.insert(preset.name.to_ascii_lowercase(), s.curve_presets.len());
s.curve_presets.push(preset);
}
}
}
}
Err(e) => failed.push(json!([f, e])),
+8 -3
View File
@@ -103,6 +103,10 @@ fn import(s: &mut Session, p: &Value) -> Result<Value> {
}
let (files, mut failed) = cube_files(&paths);
let mut imported = Vec::new();
// the ids taken and the next suffix per slug: a bundle of same-titled LUTs doesn't probe the
// whole list from 2 for each one
let mut taken: std::collections::HashSet<String> = s.lut_profiles.iter().map(|x| x.id.clone()).collect();
let mut next_suffix: std::collections::HashMap<String, u32> = std::collections::HashMap::new();
for (name, text, group) in files {
let lut = match lightcraft_pipeline::lut::Lut3d::parse_cube(&text) {
Ok(l) => l,
@@ -115,11 +119,12 @@ fn import(s: &mut Session, p: &Value) -> Result<Value> {
let group = super::super::preset_import::group_from_dir(&group.unwrap_or_default()).unwrap_or_else(|| "Imported".into());
let slug = crate::presets::slug(&format!("{group}-{display}"));
let mut id = format!("lut:{slug}");
let mut n = 2;
while s.lut_profiles.iter().any(|x| x.id == id) {
let n = next_suffix.entry(slug.clone()).or_insert(2);
while taken.contains(&id) {
id = format!("lut:{slug}-{n}");
n += 1;
*n += 1;
}
taken.insert(id.clone());
let file = match &dir {
Some(d) => {
let f = d.join(format!("{}.cube", id.trim_start_matches("lut:")));
+19 -2
View File
@@ -10,7 +10,7 @@
use lightcraft_develop::LocalAdjustments;
use serde_json::{Map, Value, json};
use super::mask_effects::{MaskEffect, upsert};
use super::mask_effects::MaskEffect;
use super::{bad, str_param};
use crate::preset_import::{Lua, group_from_dir, parse_lua, read_zip};
use crate::{Result, Session};
@@ -397,8 +397,25 @@ pub fn add_found(s: &mut Session, found: Vec<Found>, group: Option<&str>, dry: b
return Ok((rows, div_rows));
}
s.mask_effects.retain(|e| !(e.is_empty() && dividers.iter().any(|(d, _)| d.effect.name.eq_ignore_ascii_case(&e.name))));
// as `upsert`, with the names indexed once: a file of many effects doesn't scan the list per effect
// (the first of two same-named effects, as `upsert` and readers find it)
let mut by_name: std::collections::HashMap<String, usize> = std::collections::HashMap::new();
for (i, x) in s.mask_effects.iter().enumerate() {
by_name.entry(x.name.trim().to_ascii_lowercase()).or_insert(i);
}
for f in effects {
upsert(s, f.effect);
let e = f.effect;
let key = e.name.trim().to_ascii_lowercase();
match by_name.get(&key).and_then(|&i| s.mask_effects.get_mut(i)) {
Some(x) => {
let favorite = x.favorite || e.favorite;
*x = MaskEffect { favorite, ..e };
}
None => {
by_name.insert(key, s.mask_effects.len());
s.mask_effects.push(e);
}
}
}
s.save_prefs()?;
Ok((rows, div_rows))
+22 -7
View File
@@ -23,6 +23,8 @@ fn import(s: &mut Session, p: &Value) -> Result<Value> {
// .lceffect, also inside zips — become mask effects
let mut effects = Vec::new();
let mut effect_failed = Vec::new();
// every file's presets, added in one go below (adding one at a time indexed the library each time)
let mut pending: Vec<(lightcraft_develop::Preset, Value)> = Vec::new();
for top in &paths {
let from_dir = std::path::Path::new(top).is_dir();
for f in expand_preset_paths(std::slice::from_ref(top)) {
@@ -65,13 +67,14 @@ fn import(s: &mut Session, p: &Value) -> Result<Value> {
it.unmapped.extend(crate::profile_refs::report(&look, &r));
profile = json!({"name": look["name"], "id": it.preset.settings["profile"].get("id")});
}
let (name, grp) = (it.preset.name.clone(), it.preset.group.clone());
let id = if dry { Some(it.preset.id.clone()) } else { s.add_presets(vec![it.preset]).pop() };
match id {
Some(id) => {
imported.push(json!({"id": id, "name": name, "group": grp, "file": f, "unmapped": it.unmapped, "profile": profile}))
}
None => skipped += 1,
let mut row =
json!({"name": it.preset.name, "group": it.preset.group, "file": f, "unmapped": it.unmapped, "profile": profile});
if dry {
// a dry run keeps nothing: the row is final, the parsed preset is dropped
row["id"] = json!(it.preset.id);
imported.push(row);
} else {
pending.push((it.preset, row));
}
}
}
@@ -79,6 +82,18 @@ fn import(s: &mut Session, p: &Value) -> Result<Value> {
}
}
}
// split, not cloned: a large import holds one copy of the parsed presets
let (presets, rows): (Vec<_>, Vec<_>) = pending.into_iter().unzip();
let ids = if dry { Vec::new() } else { s.add_presets_each(presets) };
for (mut row, id) in rows.into_iter().zip(ids) {
match id {
Some(id) => {
row["id"] = json!(id);
imported.push(row);
}
None => skipped += 1,
}
}
failed.extend(effect_failed.into_iter().map(|(f, e)| json!([f, e])));
let (effects, dividers) = mef::add_found(s, effects, group.as_deref(), dry)?;
Ok(json!({"imported": imported, "skipped": skipped, "failed": failed, "effects": effects, "effectDividers": dividers}))
+6 -1
View File
@@ -274,15 +274,20 @@ pub fn masks(values: &Values, aspect: f64) -> (Vec<Value>, Vec<String>) {
skipped.extend(unmapped_local(c));
// brush dabs of one correction are one brush component
let mut merged: Vec<Value> = Vec::new();
// where the brush of each op sits in `merged` (no scan of it per dab)
let mut brush_at: std::collections::HashMap<String, usize> = std::collections::HashMap::new();
for p in parts {
let is_brush = p["shape"]["kind"] == "brush";
if is_brush
&& let Some(prev) = merged.iter_mut().find(|q| q["shape"]["kind"] == "brush" && q["op"] == p["op"])
&& let Some(prev) = brush_at.get(&p["op"].to_string()).and_then(|&i| merged.get_mut(i))
&& let (Some(a), Some(b)) = (prev["shape"]["strokes"].as_array_mut(), p["shape"]["strokes"].as_array())
{
a.extend(b.iter().cloned());
continue;
}
if is_brush {
brush_at.insert(p["op"].to_string(), merged.len());
}
merged.push(p);
}
let n = out.len() + 1;
+34 -12
View File
@@ -198,40 +198,62 @@ impl Session {
/// Add imported presets. Identical presets (same name, group and settings) already present are
/// skipped; id clashes get a fresh id. Returns the ids added.
pub fn add_presets(&mut self, presets: Vec<Preset>) -> Vec<String> {
self.add_presets_each(presets).into_iter().flatten().collect()
}
/// [`Self::add_presets`] reporting per preset, in order: its id, or `None` when it was skipped
/// as already present. One call indexes the library once, so an importer adds a whole file's
/// presets through it rather than one at a time.
pub fn add_presets_each(&mut self, presets: Vec<Preset>) -> Vec<Option<String>> {
use std::collections::{HashMap, HashSet};
// indexed: a collection of 10 000+ presets was quadratic (minutes) to import
let mut ids: HashSet<String> = self.presets.iter().map(|q| q.id.clone()).collect();
let mut by_name: HashMap<(String, String), Vec<usize>> = HashMap::new();
for (i, q) in self.presets.iter().enumerate() {
by_name.entry((q.name.clone(), q.group.clone())).or_default().push(i);
}
let mut added = Vec::new();
// (name, group, settings): a preset is looked up rather than compared with every namesake.
// Every number is keyed as an f64 with -0.0 folded into 0.0: `0` and `0.0` are unequal
// `Value`s with equal hashes, so a crafted file of them would collide in one bucket.
// shortcut: integers past 2^53 lose precision in the key, so two such presets are one
let key_of = |p: &Preset| (p.name.clone(), p.group.clone(), canonical_numbers(&p.settings));
let mut present: HashSet<(String, String, serde_json::Value)> = self.presets.iter().map(key_of).collect();
// the next suffix to try per generated id, so a batch of clashes doesn't probe from 2 each time
let mut next_suffix: HashMap<String, u32> = HashMap::new();
let mut added = Vec::with_capacity(presets.len());
for mut p in presets {
let key = (p.name.clone(), p.group.clone());
let same = by_name.get(&key).is_some_and(|ix| ix.iter().any(|&i| self.presets.get(i).is_some_and(|q| q.settings == p.settings)));
if same {
let key = key_of(&p);
if present.contains(&key) {
added.push(None);
continue;
}
if p.id.trim().is_empty() || p.id.starts_with("lc.") || ids.contains(&p.id) {
let base = format!("user.{}", slug(&format!("{}-{}", p.group, p.name)));
let mut id = base.clone();
let mut n = 2;
let n = next_suffix.entry(base.clone()).or_insert(2);
while ids.contains(&id) {
id = format!("{base}-{n}");
n += 1;
*n += 1;
}
p.id = id;
}
p.builtin = false;
ids.insert(p.id.clone());
by_name.entry(key).or_default().push(self.presets.len());
added.push(p.id.clone());
present.insert(key);
added.push(Some(p.id.clone()));
self.presets.push(p);
}
added
}
}
/// `v` with every number replaced by its f64 (`-0.0` as `0.0`), so equal numbers hash and compare alike.
fn canonical_numbers(v: &serde_json::Value) -> serde_json::Value {
use serde_json::Value;
match v {
Value::Number(n) => n.as_f64().and_then(|f| serde_json::Number::from_f64(f + 0.0)).map_or_else(|| v.clone(), Value::Number),
Value::Array(a) => Value::Array(a.iter().map(canonical_numbers).collect()),
Value::Object(o) => Value::Object(o.iter().map(|(k, x)| (k.clone(), canonical_numbers(x))).collect()),
_ => v.clone(),
}
}
fn p(group: &str, id: &str, name: &str, settings: serde_json::Value) -> Preset {
Preset { id: format!("lc.{id}"), name: name.into(), group: group.into(), settings, favorite: false, builtin: true }
}
+63
View File
@@ -94,6 +94,50 @@ fn importing_thousands_of_presets_is_not_quadratic() {
assert!(t.elapsed().as_secs() < 20, "took {:?}", t.elapsed());
}
#[test]
fn a_preset_whose_settings_differ_only_in_the_sign_of_zero_is_a_repeat() {
let mut s = Session::new();
let preset = |x: f64| Preset {
id: "user.z".into(),
name: "Zero".into(),
group: "Coll".into(),
settings: json!({"light": {"exposure": x}}),
favorite: false,
builtin: false,
};
assert!(s.add_presets_each(vec![preset(0.0)])[0].is_some());
assert_eq!(s.add_presets_each(vec![preset(-0.0)]), vec![None]);
}
#[test]
fn thousands_of_presets_sharing_a_name_and_id_get_unique_ids_in_one_pass() {
let mut s = Session::new();
let t = Instant::now();
// the same name, group and id throughout, each with its own settings: every one clashes
let same: Vec<Preset> = (0..20_000)
.map(|i| Preset {
id: "user.x".into(),
name: "Look".into(),
group: "Coll".into(),
settings: json!({"light": {"exposure": i}}),
favorite: false,
builtin: false,
})
.collect();
let each = s.add_presets_each(same);
assert_eq!(each.len(), 20_000);
assert!(each.iter().all(Option::is_some));
// (the session may hold built-in presets too: only the added ones are counted)
let ids: std::collections::HashSet<_> = s.presets.iter().filter(|p| p.group == "Coll").map(|p| p.id.as_str()).collect();
assert_eq!(ids.len(), 20_000);
// a repeat is skipped, per preset
let last = s.presets.last().cloned().unwrap();
let again = s.add_presets_each(vec![last.clone(), Preset { settings: json!({"light": {"exposure": -1}}), ..last }]);
assert_eq!(again[0], None);
assert!(again[1].is_some());
assert!(t.elapsed().as_secs() < 20, "took {:?}", t.elapsed());
}
/// Timing with a real preset collection: `LIGHTCRAFT_BENCH_PRESETS=path/to/presets.json cargo test
/// -p lightcraft-engine --release presets_scale_bench -- --ignored --nocapture`. The file is copied
/// into a temporary library; the original is only read.
@@ -124,3 +168,22 @@ fn presets_scale_bench() {
eprintln!("preset.favorite (saves presets.json): {:?}", t.elapsed());
let _ = std::fs::remove_dir_all(&lib);
}
#[test]
fn integer_and_float_spellings_of_one_number_are_one_preset() {
// 15 slots of `0` / `0.0` / `-0.0`: all spellings are the same preset, so none collide in the index
let mut s = Session::new();
let preset = |i: usize| {
let spell = |bit: usize| match (i >> bit) % 3 {
0 => json!(0),
1 => json!(0.0),
_ => json!(-0.0),
};
let slots: Vec<_> = (0..15).map(spell).collect();
Preset { id: "user.n".into(), name: "N".into(), group: "G".into(), settings: json!({"v": slots}), favorite: false, builtin: false }
};
let t = Instant::now();
let added = s.add_presets_each((0..4000).map(preset).collect());
assert_eq!(added.iter().filter(|a| a.is_some()).count(), 1);
assert!(t.elapsed().as_secs() < 20);
}