mirror of
https://github.com/storytold/lightcraft.git
synced 2026-11-21 18:03:34 +01:00
security: importers and preset application index instead of rescanning
* security: importers and preset application index instead of rescanning * address review: keep first match when indexing names, skip used mask ids, move presets instead of cloning * address review: a dry-run import keeps its rows, not the parsed presets * merge main into fix/sec-importers-indexed * address review: key duplicate presets by the settings value, so 0.0 and -0.0 are one preset * preset import index: key numbers as f64 so 0 and 0.0 cannot collide
This commit is contained in:
1 parent
9563f22969
commit
a75497d634
9 files changed
+211
-31
No files matched your search
@@ -455,9 +455,16 @@ fn indexed_field<'a>(s: &'a mut DevelopSettings, id: &str) -> Option<&'a mut f64
|
||||
}
|
||||
|
||||
fn indexed_get(s: &DevelopSettings, id: &str) -> Option<f64> {
|
||||
// (a clone keeps one accessor for get and set; these lists are tiny)
|
||||
let mut c = DevelopSettings { point_colors: s.point_colors.clone(), red_eye: s.red_eye.clone(), ..DevelopSettings::default() };
|
||||
indexed_field(&mut c, id).map(|v| *v)
|
||||
// one accessor for get and set: a copy of the one element read (never of the lists, which a
|
||||
// read of every indexed control would copy once per control)
|
||||
let (fam, i, field) = split_indexed(id)?;
|
||||
let mut c = DevelopSettings::default();
|
||||
match fam {
|
||||
"pointColor" => c.point_colors.push(*s.point_colors.get(i)?),
|
||||
"redEye" => c.red_eye.push(*s.red_eye.get(i)?),
|
||||
_ => return None,
|
||||
}
|
||||
indexed_field(&mut c, &format!("{fam}.0.{field}")).map(|v| *v)
|
||||
}
|
||||
|
||||
fn indexed_set(s: &mut DevelopSettings, id: &str, v: f64) -> bool {
|
||||
|
||||
@@ -313,13 +313,28 @@ impl Preset {
|
||||
/// The preset's masks, added to the photo's own (ones it already has are skipped), their effect
|
||||
/// scaled by the amount.
|
||||
fn add_masks(out: &mut DevelopSettings, masks: &[Value], amount: f64) {
|
||||
// the masks by name, kept up to date: a preset of many masks compares each with its
|
||||
// namesakes only, not with every mask added before it
|
||||
let mut by_name: std::collections::HashMap<String, Vec<usize>> = std::collections::HashMap::new();
|
||||
for (i, q) in out.masks.iter().enumerate() {
|
||||
by_name.entry(q.name.clone()).or_default().push(i);
|
||||
}
|
||||
let mut next_id = out.next_mask_id();
|
||||
let mut used: std::collections::HashSet<u32> = out.masks.iter().map(|q| q.id).collect();
|
||||
for m in masks {
|
||||
let Ok(mut m) = serde_json::from_value::<crate::Mask>(m.clone()) else { continue };
|
||||
if out.masks.iter().any(|q| q.name == m.name && q.components == m.components) {
|
||||
if by_name.get(&m.name).is_some_and(|ix| ix.iter().any(|&i| out.masks.get(i).is_some_and(|q| q.components == m.components))) {
|
||||
continue;
|
||||
}
|
||||
m.id = out.next_mask_id();
|
||||
// ids run on from the first free one, past any a mask already has (a scan for the
|
||||
// next free id per mask would be another pass over them)
|
||||
while used.contains(&next_id) {
|
||||
next_id = next_id.checked_add(1).unwrap_or(1);
|
||||
}
|
||||
m.id = next_id;
|
||||
used.insert(next_id);
|
||||
m.adjust.amount = (m.adjust.amount * amount).clamp(0.0, 200.0);
|
||||
by_name.entry(m.name.clone()).or_default().push(out.masks.len());
|
||||
out.masks.push(m);
|
||||
}
|
||||
}
|
||||
@@ -604,4 +619,22 @@ mod tests {
|
||||
ids.dedup();
|
||||
assert_eq!(ids.len(), 3, "{:?}", out.masks);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preset_masks_skip_ids_an_existing_mask_has_after_the_first_free_one() {
|
||||
// the first free id is 1, and 2 is taken: the second added mask must not get 2
|
||||
let theirs = |id| crate::Mask { id, name: format!("T{id}"), ..Default::default() };
|
||||
let s = DevelopSettings { masks: vec![theirs(u32::MAX), theirs(2)], ..Default::default() };
|
||||
let p = Preset {
|
||||
id: "p".into(),
|
||||
name: "P".into(),
|
||||
group: "User".into(),
|
||||
settings: json!({"masks": [{"name": "A"}, {"name": "B"}]}),
|
||||
favorite: false,
|
||||
builtin: false,
|
||||
};
|
||||
let mut ids: Vec<u32> = p.apply(&s, 1.0).masks.iter().map(|m| m.id).collect();
|
||||
ids.sort_unstable();
|
||||
assert_eq!(ids, [1, 2, 3, u32::MAX]);
|
||||
}
|
||||
}
|
||||
@@ -311,6 +311,13 @@ fn import_presets(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
return Err(bad(C, "missing `path` / `paths`"));
|
||||
}
|
||||
let (mut imported, mut failed) = (Vec::new(), Vec::new());
|
||||
// the user presets by name, kept up to date: a file of many presets replaces or adds each
|
||||
// without a scan of all the others
|
||||
// (the first of two same-named presets, as `upsert` and the lookups find it)
|
||||
let mut by_name: std::collections::HashMap<String, usize> = std::collections::HashMap::new();
|
||||
for (i, x) in s.curve_presets.iter().enumerate() {
|
||||
by_name.entry(x.name.to_ascii_lowercase()).or_insert(i);
|
||||
}
|
||||
for f in &paths {
|
||||
match std::fs::read(f).map_err(|e| e.to_string()).and_then(|b| parse_file(&b)) {
|
||||
Ok(list) => {
|
||||
@@ -321,7 +328,13 @@ fn import_presets(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
preset.name = format!("{} (imported)", preset.name);
|
||||
}
|
||||
imported.push(preset.name.clone());
|
||||
upsert(s, preset);
|
||||
match by_name.get(&preset.name.to_ascii_lowercase()).and_then(|&i| s.curve_presets.get_mut(i)) {
|
||||
Some(x) => *x = preset,
|
||||
None => {
|
||||
by_name.insert(preset.name.to_ascii_lowercase(), s.curve_presets.len());
|
||||
s.curve_presets.push(preset);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => failed.push(json!([f, e])),
|
||||
|
||||
@@ -103,6 +103,10 @@ fn import(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
}
|
||||
let (files, mut failed) = cube_files(&paths);
|
||||
let mut imported = Vec::new();
|
||||
// the ids taken and the next suffix per slug: a bundle of same-titled LUTs doesn't probe the
|
||||
// whole list from 2 for each one
|
||||
let mut taken: std::collections::HashSet<String> = s.lut_profiles.iter().map(|x| x.id.clone()).collect();
|
||||
let mut next_suffix: std::collections::HashMap<String, u32> = std::collections::HashMap::new();
|
||||
for (name, text, group) in files {
|
||||
let lut = match lightcraft_pipeline::lut::Lut3d::parse_cube(&text) {
|
||||
Ok(l) => l,
|
||||
@@ -115,11 +119,12 @@ fn import(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
let group = super::super::preset_import::group_from_dir(&group.unwrap_or_default()).unwrap_or_else(|| "Imported".into());
|
||||
let slug = crate::presets::slug(&format!("{group}-{display}"));
|
||||
let mut id = format!("lut:{slug}");
|
||||
let mut n = 2;
|
||||
while s.lut_profiles.iter().any(|x| x.id == id) {
|
||||
let n = next_suffix.entry(slug.clone()).or_insert(2);
|
||||
while taken.contains(&id) {
|
||||
id = format!("lut:{slug}-{n}");
|
||||
n += 1;
|
||||
*n += 1;
|
||||
}
|
||||
taken.insert(id.clone());
|
||||
let file = match &dir {
|
||||
Some(d) => {
|
||||
let f = d.join(format!("{}.cube", id.trim_start_matches("lut:")));
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
use lightcraft_develop::LocalAdjustments;
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
use super::mask_effects::{MaskEffect, upsert};
|
||||
use super::mask_effects::MaskEffect;
|
||||
use super::{bad, str_param};
|
||||
use crate::preset_import::{Lua, group_from_dir, parse_lua, read_zip};
|
||||
use crate::{Result, Session};
|
||||
@@ -397,8 +397,25 @@ pub fn add_found(s: &mut Session, found: Vec<Found>, group: Option<&str>, dry: b
|
||||
return Ok((rows, div_rows));
|
||||
}
|
||||
s.mask_effects.retain(|e| !(e.is_empty() && dividers.iter().any(|(d, _)| d.effect.name.eq_ignore_ascii_case(&e.name))));
|
||||
// as `upsert`, with the names indexed once: a file of many effects doesn't scan the list per effect
|
||||
// (the first of two same-named effects, as `upsert` and readers find it)
|
||||
let mut by_name: std::collections::HashMap<String, usize> = std::collections::HashMap::new();
|
||||
for (i, x) in s.mask_effects.iter().enumerate() {
|
||||
by_name.entry(x.name.trim().to_ascii_lowercase()).or_insert(i);
|
||||
}
|
||||
for f in effects {
|
||||
upsert(s, f.effect);
|
||||
let e = f.effect;
|
||||
let key = e.name.trim().to_ascii_lowercase();
|
||||
match by_name.get(&key).and_then(|&i| s.mask_effects.get_mut(i)) {
|
||||
Some(x) => {
|
||||
let favorite = x.favorite || e.favorite;
|
||||
*x = MaskEffect { favorite, ..e };
|
||||
}
|
||||
None => {
|
||||
by_name.insert(key, s.mask_effects.len());
|
||||
s.mask_effects.push(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
s.save_prefs()?;
|
||||
Ok((rows, div_rows))
|
||||
|
||||
@@ -23,6 +23,8 @@ fn import(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
// .lceffect, also inside zips — become mask effects
|
||||
let mut effects = Vec::new();
|
||||
let mut effect_failed = Vec::new();
|
||||
// every file's presets, added in one go below (adding one at a time indexed the library each time)
|
||||
let mut pending: Vec<(lightcraft_develop::Preset, Value)> = Vec::new();
|
||||
for top in &paths {
|
||||
let from_dir = std::path::Path::new(top).is_dir();
|
||||
for f in expand_preset_paths(std::slice::from_ref(top)) {
|
||||
@@ -65,13 +67,14 @@ fn import(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
it.unmapped.extend(crate::profile_refs::report(&look, &r));
|
||||
profile = json!({"name": look["name"], "id": it.preset.settings["profile"].get("id")});
|
||||
}
|
||||
let (name, grp) = (it.preset.name.clone(), it.preset.group.clone());
|
||||
let id = if dry { Some(it.preset.id.clone()) } else { s.add_presets(vec![it.preset]).pop() };
|
||||
match id {
|
||||
Some(id) => {
|
||||
imported.push(json!({"id": id, "name": name, "group": grp, "file": f, "unmapped": it.unmapped, "profile": profile}))
|
||||
}
|
||||
None => skipped += 1,
|
||||
let mut row =
|
||||
json!({"name": it.preset.name, "group": it.preset.group, "file": f, "unmapped": it.unmapped, "profile": profile});
|
||||
if dry {
|
||||
// a dry run keeps nothing: the row is final, the parsed preset is dropped
|
||||
row["id"] = json!(it.preset.id);
|
||||
imported.push(row);
|
||||
} else {
|
||||
pending.push((it.preset, row));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -79,6 +82,18 @@ fn import(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
}
|
||||
}
|
||||
}
|
||||
// split, not cloned: a large import holds one copy of the parsed presets
|
||||
let (presets, rows): (Vec<_>, Vec<_>) = pending.into_iter().unzip();
|
||||
let ids = if dry { Vec::new() } else { s.add_presets_each(presets) };
|
||||
for (mut row, id) in rows.into_iter().zip(ids) {
|
||||
match id {
|
||||
Some(id) => {
|
||||
row["id"] = json!(id);
|
||||
imported.push(row);
|
||||
}
|
||||
None => skipped += 1,
|
||||
}
|
||||
}
|
||||
failed.extend(effect_failed.into_iter().map(|(f, e)| json!([f, e])));
|
||||
let (effects, dividers) = mef::add_found(s, effects, group.as_deref(), dry)?;
|
||||
Ok(json!({"imported": imported, "skipped": skipped, "failed": failed, "effects": effects, "effectDividers": dividers}))
|
||||
|
||||
@@ -274,15 +274,20 @@ pub fn masks(values: &Values, aspect: f64) -> (Vec<Value>, Vec<String>) {
|
||||
skipped.extend(unmapped_local(c));
|
||||
// brush dabs of one correction are one brush component
|
||||
let mut merged: Vec<Value> = Vec::new();
|
||||
// where the brush of each op sits in `merged` (no scan of it per dab)
|
||||
let mut brush_at: std::collections::HashMap<String, usize> = std::collections::HashMap::new();
|
||||
for p in parts {
|
||||
let is_brush = p["shape"]["kind"] == "brush";
|
||||
if is_brush
|
||||
&& let Some(prev) = merged.iter_mut().find(|q| q["shape"]["kind"] == "brush" && q["op"] == p["op"])
|
||||
&& let Some(prev) = brush_at.get(&p["op"].to_string()).and_then(|&i| merged.get_mut(i))
|
||||
&& let (Some(a), Some(b)) = (prev["shape"]["strokes"].as_array_mut(), p["shape"]["strokes"].as_array())
|
||||
{
|
||||
a.extend(b.iter().cloned());
|
||||
continue;
|
||||
}
|
||||
if is_brush {
|
||||
brush_at.insert(p["op"].to_string(), merged.len());
|
||||
}
|
||||
merged.push(p);
|
||||
}
|
||||
let n = out.len() + 1;
|
||||
|
||||
@@ -198,40 +198,62 @@ impl Session {
|
||||
/// Add imported presets. Identical presets (same name, group and settings) already present are
|
||||
/// skipped; id clashes get a fresh id. Returns the ids added.
|
||||
pub fn add_presets(&mut self, presets: Vec<Preset>) -> Vec<String> {
|
||||
self.add_presets_each(presets).into_iter().flatten().collect()
|
||||
}
|
||||
|
||||
/// [`Self::add_presets`] reporting per preset, in order: its id, or `None` when it was skipped
|
||||
/// as already present. One call indexes the library once, so an importer adds a whole file's
|
||||
/// presets through it rather than one at a time.
|
||||
pub fn add_presets_each(&mut self, presets: Vec<Preset>) -> Vec<Option<String>> {
|
||||
use std::collections::{HashMap, HashSet};
|
||||
// indexed: a collection of 10 000+ presets was quadratic (minutes) to import
|
||||
let mut ids: HashSet<String> = self.presets.iter().map(|q| q.id.clone()).collect();
|
||||
let mut by_name: HashMap<(String, String), Vec<usize>> = HashMap::new();
|
||||
for (i, q) in self.presets.iter().enumerate() {
|
||||
by_name.entry((q.name.clone(), q.group.clone())).or_default().push(i);
|
||||
}
|
||||
let mut added = Vec::new();
|
||||
// (name, group, settings): a preset is looked up rather than compared with every namesake.
|
||||
// Every number is keyed as an f64 with -0.0 folded into 0.0: `0` and `0.0` are unequal
|
||||
// `Value`s with equal hashes, so a crafted file of them would collide in one bucket.
|
||||
// shortcut: integers past 2^53 lose precision in the key, so two such presets are one
|
||||
let key_of = |p: &Preset| (p.name.clone(), p.group.clone(), canonical_numbers(&p.settings));
|
||||
let mut present: HashSet<(String, String, serde_json::Value)> = self.presets.iter().map(key_of).collect();
|
||||
// the next suffix to try per generated id, so a batch of clashes doesn't probe from 2 each time
|
||||
let mut next_suffix: HashMap<String, u32> = HashMap::new();
|
||||
let mut added = Vec::with_capacity(presets.len());
|
||||
for mut p in presets {
|
||||
let key = (p.name.clone(), p.group.clone());
|
||||
let same = by_name.get(&key).is_some_and(|ix| ix.iter().any(|&i| self.presets.get(i).is_some_and(|q| q.settings == p.settings)));
|
||||
if same {
|
||||
let key = key_of(&p);
|
||||
if present.contains(&key) {
|
||||
added.push(None);
|
||||
continue;
|
||||
}
|
||||
if p.id.trim().is_empty() || p.id.starts_with("lc.") || ids.contains(&p.id) {
|
||||
let base = format!("user.{}", slug(&format!("{}-{}", p.group, p.name)));
|
||||
let mut id = base.clone();
|
||||
let mut n = 2;
|
||||
let n = next_suffix.entry(base.clone()).or_insert(2);
|
||||
while ids.contains(&id) {
|
||||
id = format!("{base}-{n}");
|
||||
n += 1;
|
||||
*n += 1;
|
||||
}
|
||||
p.id = id;
|
||||
}
|
||||
p.builtin = false;
|
||||
ids.insert(p.id.clone());
|
||||
by_name.entry(key).or_default().push(self.presets.len());
|
||||
added.push(p.id.clone());
|
||||
present.insert(key);
|
||||
added.push(Some(p.id.clone()));
|
||||
self.presets.push(p);
|
||||
}
|
||||
added
|
||||
}
|
||||
}
|
||||
|
||||
/// `v` with every number replaced by its f64 (`-0.0` as `0.0`), so equal numbers hash and compare alike.
|
||||
fn canonical_numbers(v: &serde_json::Value) -> serde_json::Value {
|
||||
use serde_json::Value;
|
||||
match v {
|
||||
Value::Number(n) => n.as_f64().and_then(|f| serde_json::Number::from_f64(f + 0.0)).map_or_else(|| v.clone(), Value::Number),
|
||||
Value::Array(a) => Value::Array(a.iter().map(canonical_numbers).collect()),
|
||||
Value::Object(o) => Value::Object(o.iter().map(|(k, x)| (k.clone(), canonical_numbers(x))).collect()),
|
||||
_ => v.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
fn p(group: &str, id: &str, name: &str, settings: serde_json::Value) -> Preset {
|
||||
Preset { id: format!("lc.{id}"), name: name.into(), group: group.into(), settings, favorite: false, builtin: true }
|
||||
}
|
||||
|
||||
@@ -94,6 +94,50 @@ fn importing_thousands_of_presets_is_not_quadratic() {
|
||||
assert!(t.elapsed().as_secs() < 20, "took {:?}", t.elapsed());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_preset_whose_settings_differ_only_in_the_sign_of_zero_is_a_repeat() {
|
||||
let mut s = Session::new();
|
||||
let preset = |x: f64| Preset {
|
||||
id: "user.z".into(),
|
||||
name: "Zero".into(),
|
||||
group: "Coll".into(),
|
||||
settings: json!({"light": {"exposure": x}}),
|
||||
favorite: false,
|
||||
builtin: false,
|
||||
};
|
||||
assert!(s.add_presets_each(vec![preset(0.0)])[0].is_some());
|
||||
assert_eq!(s.add_presets_each(vec![preset(-0.0)]), vec![None]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thousands_of_presets_sharing_a_name_and_id_get_unique_ids_in_one_pass() {
|
||||
let mut s = Session::new();
|
||||
let t = Instant::now();
|
||||
// the same name, group and id throughout, each with its own settings: every one clashes
|
||||
let same: Vec<Preset> = (0..20_000)
|
||||
.map(|i| Preset {
|
||||
id: "user.x".into(),
|
||||
name: "Look".into(),
|
||||
group: "Coll".into(),
|
||||
settings: json!({"light": {"exposure": i}}),
|
||||
favorite: false,
|
||||
builtin: false,
|
||||
})
|
||||
.collect();
|
||||
let each = s.add_presets_each(same);
|
||||
assert_eq!(each.len(), 20_000);
|
||||
assert!(each.iter().all(Option::is_some));
|
||||
// (the session may hold built-in presets too: only the added ones are counted)
|
||||
let ids: std::collections::HashSet<_> = s.presets.iter().filter(|p| p.group == "Coll").map(|p| p.id.as_str()).collect();
|
||||
assert_eq!(ids.len(), 20_000);
|
||||
// a repeat is skipped, per preset
|
||||
let last = s.presets.last().cloned().unwrap();
|
||||
let again = s.add_presets_each(vec![last.clone(), Preset { settings: json!({"light": {"exposure": -1}}), ..last }]);
|
||||
assert_eq!(again[0], None);
|
||||
assert!(again[1].is_some());
|
||||
assert!(t.elapsed().as_secs() < 20, "took {:?}", t.elapsed());
|
||||
}
|
||||
|
||||
/// Timing with a real preset collection: `LIGHTCRAFT_BENCH_PRESETS=path/to/presets.json cargo test
|
||||
/// -p lightcraft-engine --release presets_scale_bench -- --ignored --nocapture`. The file is copied
|
||||
/// into a temporary library; the original is only read.
|
||||
@@ -124,3 +168,22 @@ fn presets_scale_bench() {
|
||||
eprintln!("preset.favorite (saves presets.json): {:?}", t.elapsed());
|
||||
let _ = std::fs::remove_dir_all(&lib);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn integer_and_float_spellings_of_one_number_are_one_preset() {
|
||||
// 15 slots of `0` / `0.0` / `-0.0`: all spellings are the same preset, so none collide in the index
|
||||
let mut s = Session::new();
|
||||
let preset = |i: usize| {
|
||||
let spell = |bit: usize| match (i >> bit) % 3 {
|
||||
0 => json!(0),
|
||||
1 => json!(0.0),
|
||||
_ => json!(-0.0),
|
||||
};
|
||||
let slots: Vec<_> = (0..15).map(spell).collect();
|
||||
Preset { id: "user.n".into(), name: "N".into(), group: "G".into(), settings: json!({"v": slots}), favorite: false, builtin: false }
|
||||
};
|
||||
let t = Instant::now();
|
||||
let added = s.add_presets_each((0..4000).map(preset).collect());
|
||||
assert_eq!(added.iter().filter(|a| a.is_some()).count(), 1);
|
||||
assert!(t.elapsed().as_secs() < 20);
|
||||
}
|
||||
Reference in new issue
Block a user