mirror of
https://github.com/storytold/effectcraft.git
synced 2026-11-20 01:14:53 +01:00
Footage cache (#3271): review fixes
- A covered batch is served before a remembered failure; failures are kept per request, so a broken part or too many channels never hides the picture; a picture too large with its effects' channels is read alone. - Hostile headers: parts whose data window or uncompressed block is out of bounds are refused before any decoder sees them (index, reader and the image crate's whole-file path, import). - Seek and read errors are transient (Stopped), not remembered; the copy path notes its file. - The index drops the headers' other attributes and previews, is bounded by bytes and checks the file's identity before it is reused. - A picture read alone isn't kept twice (plane store and frame cache). - A waiter whose decodes keep being called off decodes a copy; rounds are bounded. - The Cryptomatte probe asks with its own effect's need; animated parameters are found in nested groups. - Test hooks and caps are keyed by path; new tests: data window offsets, failures per request, too large with channels, the file bound, hostile data windows, damaged files, rows and levels on every compression and on tiles.
This commit is contained in:
1 parent
63d32fa33e
commit
bf48ec8b8e
14 files changed
+636
-174
No files matched your search
@@ -116,8 +116,8 @@ pub use time_fx::{PixelMotion, TimewarpPlan, TwStep, pmb_plan, posterized_time,
|
||||
pub use ocio::color_stabilizer_maps;
|
||||
// GPU effects, part B (3D Channel, Immersive Video, colour management, simulation render passes).
|
||||
pub use channel3d::{
|
||||
CRYPTO_LAYERS, CryptoAux, MATTE_ONLY_IN_ALPHA, aux_point, crypto_aux_image, crypto_aux_image_with, crypto_layer_choices, crypto_layer_menu,
|
||||
aux_needs, crypto_layer_name, crypto_layer_named, id_color, matte_only_px, reads_aux, selection_hashes,
|
||||
CRYPTO_LAYERS, CryptoAux, MATTE_ONLY_IN_ALPHA, aux_needs, aux_point, crypto_aux_image, crypto_aux_image_with, crypto_layer_choices, crypto_layer_menu,
|
||||
crypto_layer_name, crypto_layer_named, id_color, matte_only_px, reads_aux, selection_hashes,
|
||||
};
|
||||
pub use ocio::{ColorOp, Straight, Tf, color_program};
|
||||
pub use sim::{Acc, Blob, Bubble, Post, Shape, Sprite, SpritePlan, Tint, bubble_list, bubble_sample_bounds, drizzle_drops, mercury_blobs, sprite_plan};
|
||||
|
||||
@@ -20,7 +20,7 @@ mod create;
|
||||
mod edit;
|
||||
mod effect;
|
||||
pub use effect::{CryptoCoverage, CryptoProbeJob, PLUGINS_DIR};
|
||||
pub(crate) use effect::{CryptoProbeSlot, crypto_coverage, crypto_probe_job, crypto_probe_ready, keep_crypto_probe, kept_crypto_probe};
|
||||
pub(crate) use effect::{CryptoProbeSlot, crypto_coverage, crypto_probe_job, crypto_probe_ready, keep_crypto_probe, kept_crypto_probe, moves};
|
||||
pub mod essential;
|
||||
pub mod expr_tools;
|
||||
mod file;
|
||||
|
||||
@@ -520,7 +520,14 @@ impl CryptoProbe {
|
||||
reads: None,
|
||||
};
|
||||
let params = effectcraft_effects::flatten_params(g, &mut |pr| ctx.value(layer, pr));
|
||||
let aux = r.layer_aux(&ctx, layer).ok_or(NO_CRYPTO)?;
|
||||
// The channels the layer's effects read and this effect's (it may be switched off).
|
||||
let own = match &g.kind {
|
||||
effectcraft_project::GroupKind::Effect { effect } => effectcraft_effects::aux_needs(effect, ¶ms),
|
||||
_ => None,
|
||||
};
|
||||
let need =
|
||||
r.layer_aux_need(&ctx, layer).union(own.map_or(effectcraft_raster::channels3d::AuxNeed::All, effectcraft_raster::channels3d::AuxNeed::Lookups));
|
||||
let aux = r.layer_aux_with(&ctx, layer, &need).ok_or(NO_CRYPTO)?;
|
||||
if !live() {
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -540,7 +547,7 @@ impl CryptoProbe {
|
||||
}
|
||||
|
||||
/// Whether a property in `g` other than `skip` changes over time (keyframes or an expression).
|
||||
fn moves(g: &PropGroup, skip: Option<Uid>) -> bool {
|
||||
pub(crate) fn moves(g: &PropGroup, skip: Option<Uid>) -> bool {
|
||||
g.children.iter().any(|c| match c {
|
||||
effectcraft_project::Node::Prop(p) => Some(p.uid) != skip && (p.is_animated() || p.has_expression()),
|
||||
effectcraft_project::Node::Group(g) => moves(g, skip),
|
||||
|
||||
@@ -169,7 +169,7 @@ impl Session {
|
||||
/// doesn't declare, or whose parameters change over time (keyframes, an expression), reads all
|
||||
/// channels. Several layers of one file add up.
|
||||
pub(crate) fn aux_needs(project: &effectcraft_project::Project) -> Vec<(String, effectcraft_raster::channels3d::AuxNeed)> {
|
||||
use effectcraft_project::{GroupKind, ItemKind, LayerSource, Node};
|
||||
use effectcraft_project::{GroupKind, ItemKind, LayerSource};
|
||||
use effectcraft_raster::channels3d::AuxNeed;
|
||||
let mut out: std::collections::BTreeMap<String, AuxNeed> = std::collections::BTreeMap::new();
|
||||
for l in project.comps().flat_map(|(_, c)| c.layers.iter()).filter(|l| l.switches.effects) {
|
||||
@@ -183,8 +183,7 @@ pub(crate) fn aux_needs(project: &effectcraft_project::Project) -> Vec<(String,
|
||||
if !effectcraft_effects::reads_aux(effect) {
|
||||
continue;
|
||||
}
|
||||
let moving = g.children.iter().any(|c| matches!(c, Node::Prop(p) if p.is_animated() || p.has_expression()));
|
||||
let need = if moving {
|
||||
let need = if commands::moves(g, None) {
|
||||
AuxNeed::All
|
||||
} else {
|
||||
let params = effectcraft_effects::flatten_params(g, &mut |p| p.value.clone());
|
||||
|
||||
@@ -272,6 +272,7 @@ pub(crate) fn add_decodes(n: [u64; 2]) {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use effectcraft_raster::channels3d::BACKGROUND_DEPTH;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// An EXR file of `layers` (with their display window `display`).
|
||||
fn file(layers: Vec<exr::prelude::Layer<exr::prelude::AnyChannels<exr::prelude::FlatSamples>>>, display: exr::prelude::IntegerBounds) -> Vec<u8> {
|
||||
@@ -368,7 +369,14 @@ mod tests {
|
||||
assert!(read_exr_channels(&bytes).is_none());
|
||||
let pool = crate::MediaPool::new();
|
||||
pool.add_bytes("/474-too-large.exr", bytes.as_slice().into());
|
||||
let f = effectcraft_project::Footage { path: "/474-too-large.exr".into(), kind: effectcraft_project::FootageKind::Still, has_video: true, width: 16384, height: 16384, ..Default::default() };
|
||||
let f = effectcraft_project::Footage {
|
||||
path: "/474-too-large.exr".into(),
|
||||
kind: effectcraft_project::FootageKind::Still,
|
||||
has_video: true,
|
||||
width: 16384,
|
||||
height: 16384,
|
||||
..Default::default()
|
||||
};
|
||||
use effectcraft_render::FootageSource;
|
||||
assert!(pool.aux(effectcraft_project::ItemId(1), &f, effectcraft_time::Tick::ZERO).is_none());
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
//! with a bad offset is never read).
|
||||
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::io::{Read, Seek, SeekFrom};
|
||||
use std::io::{Read, SeekFrom};
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
|
||||
use effectcraft_raster::AuxChannels;
|
||||
@@ -160,8 +160,42 @@ fn headers_and_tables(src: &mut dyn ReadSeek, len: u64) -> Result<(MetaData, exr
|
||||
}
|
||||
}
|
||||
|
||||
/// The most bytes one uncompressed block of a part may take: the decompressor allocates that
|
||||
/// much for a block before it looks at the data, so a hostile header (a data window a billion
|
||||
/// pixels wide over one tiny chunk) would ask for gigabytes.
|
||||
pub(crate) const MAX_BLOCK_BYTES: usize = 256 << 20;
|
||||
|
||||
/// The most pixels of a part's data window (the display window's limit, four times over: data
|
||||
/// windows may reach past it).
|
||||
const MAX_DATA_PIXELS: usize = 4 * MAX_DISPLAY_PIXELS;
|
||||
|
||||
/// The bytes of one uncompressed block of `h` (its pixel bytes times a block's pixels, checked):
|
||||
/// `None` past [`MAX_BLOCK_BYTES`] or when it doesn't fit in a `usize`.
|
||||
pub(crate) fn block_bytes(h: &exr::meta::header::Header) -> Option<usize> {
|
||||
let (w, rows) = match h.blocks {
|
||||
exr::meta::BlockDescription::Tiles(t) => (t.tile_size.0, t.tile_size.1),
|
||||
exr::meta::BlockDescription::ScanLines => (h.layer_size.0, h.compression.scan_lines_per_block().min(h.layer_size.1.max(1))),
|
||||
};
|
||||
let per_pixel = h.channels.list.iter().map(|c| c.sample_type.bytes_per_sample()).try_fold(0usize, usize::checked_add)?;
|
||||
per_pixel.checked_mul(w)?.checked_mul(rows).filter(|b| *b <= MAX_BLOCK_BYTES)
|
||||
}
|
||||
|
||||
/// Every part of `meta` is safe to decode: its data window and its blocks within bounds.
|
||||
pub(crate) fn parts_in_bounds(meta: &MetaData) -> bool {
|
||||
meta.headers.iter().all(|h| h.layer_size.0.checked_mul(h.layer_size.1).is_some_and(|n| n <= MAX_DATA_PIXELS) && block_bytes(h).is_some())
|
||||
}
|
||||
|
||||
/// The headers of the OpenEXR `bytes` (read leniently) are safe to hand to a decoder
|
||||
/// ([`parts_in_bounds`]); `false` for anything else.
|
||||
pub(crate) fn safe_to_decode(bytes: &[u8]) -> bool {
|
||||
MetaData::read_from_buffered(std::io::Cursor::new(bytes), false).is_ok_and(|meta| parts_in_bounds(&meta))
|
||||
}
|
||||
|
||||
/// Refuse headers this reader doesn't take or that ask for more than the file can hold.
|
||||
fn check(meta: &MetaData, len: u64, header_end: u64) -> Result<(), IndexError> {
|
||||
if !parts_in_bounds(meta) {
|
||||
return Err(IndexError::Unreadable);
|
||||
}
|
||||
if let Some(why) = unsupported(meta) {
|
||||
return Err(IndexError::Unsupported(why));
|
||||
}
|
||||
@@ -195,7 +229,15 @@ pub(crate) fn unsupported(meta: &MetaData) -> Option<&'static str> {
|
||||
/// offset tables only.
|
||||
pub(crate) fn build(src: &mut dyn ReadSeek, len: u64, stamp: FileStamp) -> Result<ExrIndex, IndexError> {
|
||||
src.seek(SeekFrom::Start(0)).map_err(|_| IndexError::Io)?;
|
||||
let (meta, tables, chunks_start) = headers_and_tables(&mut *src, len)?;
|
||||
let (mut meta, tables, chunks_start) = headers_and_tables(&mut *src, len)?;
|
||||
// The manifests are parsed, and the other attributes (and previews) aren't kept: an index
|
||||
// stays small however much a renderer writes into the headers.
|
||||
let manifests = manifests(&meta);
|
||||
for h in meta.headers.iter_mut() {
|
||||
h.shared_attributes.other.clear();
|
||||
h.own_attributes.other.clear();
|
||||
h.own_attributes.preview = None;
|
||||
}
|
||||
let first = meta.headers.first().ok_or(IndexError::Unreadable)?;
|
||||
let display = first.shared_attributes.display_window;
|
||||
// Where each chunk ends: at the next chunk's start, or at the end of the file.
|
||||
@@ -211,7 +253,7 @@ pub(crate) fn build(src: &mut dyn ReadSeek, len: u64, stamp: FileStamp) -> Resul
|
||||
let table = tables.get(pi).ok_or(IndexError::Unreadable)?;
|
||||
// A chunk is at most its block's bytes plus its own small header (part number, position
|
||||
// and sizes): never read more for one.
|
||||
let most = (h.max_block_byte_size() as u64).saturating_add(64);
|
||||
let most = (block_bytes(h).ok_or(IndexError::Unreadable)? as u64).saturating_add(64);
|
||||
let mut list = Vec::with_capacity(h.chunk_count);
|
||||
for (i, tile) in h.blocks_increasing_y_order().enumerate() {
|
||||
let bounds = h.get_absolute_block_pixel_coordinates(tile.location).map_err(|_| IndexError::Unreadable)?;
|
||||
@@ -249,7 +291,6 @@ pub(crate) fn build(src: &mut dyn ReadSeek, len: u64, stamp: FileStamp) -> Resul
|
||||
channels.push(IndexChannel { name, part: pi, channel: ci, fill });
|
||||
}
|
||||
}
|
||||
let manifests = manifests(&meta);
|
||||
let rgb = rgb_of(&meta, &channels);
|
||||
let mut names = AuxChannels::new(display.size.0 as u32, display.size.1 as u32, 1.0);
|
||||
names.channels = channels.iter().map(|c| (c.name.clone(), vec![])).collect();
|
||||
@@ -257,12 +298,7 @@ pub(crate) fn build(src: &mut dyn ReadSeek, len: u64, stamp: FileStamp) -> Resul
|
||||
names.aliases = aliases;
|
||||
let picture = picture_names(&names).map(|(rgb, a)| {
|
||||
// Planes are looked up by name ignoring case, so every channel that could answer is read.
|
||||
let ids = channels
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, c)| rgb.iter().chain(&a).any(|k| k.eq_ignore_ascii_case(&c.name)))
|
||||
.map(|(i, _)| i)
|
||||
.collect();
|
||||
let ids = channels.iter().enumerate().filter(|(_, c)| rgb.iter().chain(&a).any(|k| k.eq_ignore_ascii_case(&c.name))).map(|(i, _)| i).collect();
|
||||
(ids, rgb, a)
|
||||
});
|
||||
Ok(ExrIndex { meta, stamp, display, parts, channels, rgb, picture, names: Arc::new(names) })
|
||||
@@ -292,8 +328,7 @@ fn manifests(meta: &MetaData) -> Vec<(String, Vec<(String, u32)>)> {
|
||||
for h in meta.headers.iter() {
|
||||
collect(&h.own_attributes.other);
|
||||
}
|
||||
let mut out: Vec<(String, Vec<(String, u32)>)> =
|
||||
crypto.into_values().filter_map(|(name, manifest)| Some((name?, parse_manifest(&manifest?)))).collect();
|
||||
let mut out: Vec<(String, Vec<(String, u32)>)> = crypto.into_values().filter_map(|(name, manifest)| Some((name?, parse_manifest(&manifest?)))).collect();
|
||||
out.sort();
|
||||
out
|
||||
}
|
||||
@@ -321,42 +356,77 @@ fn rgb_of(meta: &MetaData, channels: &[IndexChannel]) -> Rgb {
|
||||
Rgb::None
|
||||
}
|
||||
|
||||
/// At most this many files are indexed (a few kilobytes each); the oldest goes first.
|
||||
/// At most this many files are indexed, and their indexes take at most [`INDEXED_BYTES`]; the
|
||||
/// oldest goes first.
|
||||
const INDEXED_FILES: usize = 4096;
|
||||
const INDEXED_BYTES: usize = 64 << 20;
|
||||
|
||||
impl ExrIndex {
|
||||
/// About the bytes it holds: its chunks, channels and manifests (the headers' other
|
||||
/// attributes are dropped when it is built).
|
||||
pub(crate) fn bytes(&self) -> usize {
|
||||
let chunks: usize = self.parts.iter().map(|p| p.chunks.len()).sum();
|
||||
let names: usize = self.channels.iter().map(|c| c.name.len() * 2 + 64).sum();
|
||||
let manifests: usize = self.names.manifests.iter().map(|(l, m)| l.len() + m.iter().map(|(n, _)| n.len() + 32).sum::<usize>()).sum();
|
||||
let headers = self.meta.headers.len().saturating_mul(1024);
|
||||
(chunks * std::mem::size_of::<IndexChunk>()).saturating_add(names).saturating_add(manifests.saturating_mul(2)).saturating_add(headers)
|
||||
}
|
||||
}
|
||||
|
||||
/// Indexed files by path.
|
||||
struct Indexes {
|
||||
map: HashMap<String, Arc<ExrIndex>>,
|
||||
order: VecDeque<String>,
|
||||
bytes: usize,
|
||||
}
|
||||
|
||||
fn indexes() -> &'static Mutex<Indexes> {
|
||||
static INDEXES: std::sync::OnceLock<Mutex<Indexes>> = std::sync::OnceLock::new();
|
||||
INDEXES.get_or_init(|| Mutex::new(Indexes { map: HashMap::new(), order: VecDeque::new() }))
|
||||
INDEXES.get_or_init(|| Mutex::new(Indexes { map: HashMap::new(), order: VecDeque::new(), bytes: 0 }))
|
||||
}
|
||||
|
||||
fn lock() -> std::sync::MutexGuard<'static, Indexes> {
|
||||
indexes().lock().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
|
||||
/// Keep `index` as the index of `path`.
|
||||
/// Keep `index` as the index of `path`, dropping the oldest beyond the bounds (the newest stays).
|
||||
pub(crate) fn keep(path: &str, index: Arc<ExrIndex>) {
|
||||
let mut ix = lock();
|
||||
if ix.map.insert(path.to_string(), index).is_none() {
|
||||
ix.order.push_back(path.to_string());
|
||||
let bytes = index.bytes();
|
||||
match ix.map.insert(path.to_string(), index) {
|
||||
Some(old) => ix.bytes = ix.bytes.saturating_sub(old.bytes()),
|
||||
None => ix.order.push_back(path.to_string()),
|
||||
}
|
||||
while ix.map.len() > INDEXED_FILES {
|
||||
ix.bytes = ix.bytes.saturating_add(bytes);
|
||||
for _ in 0..=ix.order.len() {
|
||||
if (ix.map.len() <= INDEXED_FILES && ix.bytes <= INDEXED_BYTES) || ix.map.len() <= 1 {
|
||||
break;
|
||||
}
|
||||
let Some(old) = ix.order.pop_front() else { break };
|
||||
ix.map.remove(&old);
|
||||
if old == path {
|
||||
ix.order.push_back(old);
|
||||
continue;
|
||||
}
|
||||
if let Some(gone) = ix.map.remove(&old) {
|
||||
ix.bytes = ix.bytes.saturating_sub(gone.bytes());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The index of the file at `path` (read from `source` the first time; kept until
|
||||
/// [`forget`]). Header-only users take it as it is; a read checks the file's identity and
|
||||
/// indexes it again when it changed.
|
||||
/// [`forget`]). A file whose identity changed since (size, modification time) is indexed again;
|
||||
/// a read checks it again on the file it opens.
|
||||
pub(crate) fn get(path: &str, source: &ExrSource) -> Result<Arc<ExrIndex>, IndexError> {
|
||||
if let Some(ix) = lock().map.get(path).cloned() {
|
||||
return Ok(ix);
|
||||
let kept = lock().map.get(path).cloned();
|
||||
if let Some(ix) = kept {
|
||||
match source.stamp() {
|
||||
Ok(stamp) if stamp == ix.stamp => return Ok(ix),
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
log::debug!("media: {path}: {e}");
|
||||
return Err(IndexError::Io);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut opened = source.open().map_err(|e| {
|
||||
log::debug!("media: {path}: {e}");
|
||||
@@ -376,7 +446,8 @@ pub(crate) fn cached(path: &str) -> Option<Arc<ExrIndex>> {
|
||||
/// Drop the index of `path` (the file changed on disk).
|
||||
pub(crate) fn forget(path: &str) {
|
||||
let mut ix = lock();
|
||||
if ix.map.remove(path).is_some() {
|
||||
if let Some(gone) = ix.map.remove(path) {
|
||||
ix.bytes = ix.bytes.saturating_sub(gone.bytes());
|
||||
ix.order.retain(|p| p != path);
|
||||
}
|
||||
}
|
||||
+202
-102
@@ -27,9 +27,9 @@ use effectcraft_raster::{AuxChannels, Image};
|
||||
use effectcraft_render::reads::ReadFile;
|
||||
|
||||
use crate::convert::AlphaOp;
|
||||
use crate::exr_channels::CACHE_BUDGET;
|
||||
use crate::exr_channels::{Rgb, planes_picture};
|
||||
use crate::exr_index::{self, ExrIndex, IndexError};
|
||||
use crate::exr_channels::CACHE_BUDGET;
|
||||
use crate::exr_read::{ExrSource, ReadError, ReadRequest, read_planes};
|
||||
|
||||
/// At most this many entries are kept (unreadable files hold no bytes).
|
||||
@@ -44,8 +44,36 @@ const HELD_FILES: usize = 4;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
const WAIT_LIMIT: std::time::Duration = std::time::Duration::from_secs(10);
|
||||
|
||||
/// How often a request asks again after a decode it waited for was called off or forgotten.
|
||||
/// How often a request asks again after a decode it waited for was called off or forgotten,
|
||||
/// and how many rounds it waits at most, before it decodes a copy itself.
|
||||
const RETRIES: u32 = 3;
|
||||
const ROUNDS: u32 = 8;
|
||||
|
||||
/// At most this many failed requests are remembered per entry.
|
||||
const FAILED_KEPT: usize = 8;
|
||||
|
||||
/// What a round of [`PlaneStore::get`] ended with.
|
||||
enum Step {
|
||||
Done(Result<Arc<Batch>, Refused>),
|
||||
/// Ask again (`retry`: the decode waited for was called off, stopped or forgotten).
|
||||
Again {
|
||||
retry: bool,
|
||||
},
|
||||
}
|
||||
|
||||
/// A request for `want` asks for at least what `failed` asked for.
|
||||
fn asks_at_least(want: &Want, failed: &Want) -> bool {
|
||||
want.all || (!failed.all && failed.ids.is_subset(&want.ids))
|
||||
}
|
||||
|
||||
/// Decode `want` for this request alone (kept nowhere), noting the file it read.
|
||||
fn copy(path: &str, source: &ExrSource, index: &Arc<ExrIndex>, want: &Want) -> Result<Arc<Batch>, Refused> {
|
||||
let b = run(path, source, &Flight::new(index.clone(), want.clone()), None);
|
||||
if let Ok(b) = &b {
|
||||
crate::reads::note(b.file.clone());
|
||||
}
|
||||
b
|
||||
}
|
||||
|
||||
/// Decoded planes of a file: the union of the channels read so far.
|
||||
pub(crate) struct Batch {
|
||||
@@ -154,7 +182,9 @@ struct Entry {
|
||||
level: u32,
|
||||
batch: Option<Arc<Batch>>,
|
||||
flight: Option<Arc<Flight>>,
|
||||
failed: Option<Refused>,
|
||||
/// Requests that failed for good (unreadable, too large) and why: a request asking for at
|
||||
/// least as much fails the same way until the file is forgotten.
|
||||
failed: Vec<(Want, Refused)>,
|
||||
bytes: usize,
|
||||
/// When it was last asked for or finished decoding (a counter).
|
||||
used: u64,
|
||||
@@ -260,11 +290,37 @@ pub(crate) fn decodes_of(path: &str) -> Vec<DecodeRecord> {
|
||||
#[cfg(test)]
|
||||
type Hook = Arc<dyn Fn(&str) + Send + Sync>;
|
||||
#[cfg(test)]
|
||||
static BEFORE_READ: Mutex<Option<Hook>> = Mutex::new(None);
|
||||
static BEFORE_READ: Mutex<Vec<(String, Hook)>> = Mutex::new(Vec::new());
|
||||
|
||||
/// Run `hook` before each decode of `path` reads its chunks (`None`: no longer). Keyed by path,
|
||||
/// so tests running at the same time don't see each other's.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_before_read(path: &str, hook: Option<Hook>) {
|
||||
let mut hooks = BEFORE_READ.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
hooks.retain(|(p, _)| p != path);
|
||||
if let Some(h) = hook {
|
||||
hooks.push((path.to_string(), h));
|
||||
}
|
||||
}
|
||||
|
||||
/// Planes of `path` may take at most this many bytes in a read (tests: a small cap in place of
|
||||
/// [`crate::exr_read::MAX_DECODE_BYTES`]).
|
||||
#[cfg(test)]
|
||||
static CAPS: Mutex<Vec<(String, usize)>> = Mutex::new(Vec::new());
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_before_read(hook: Option<Hook>) {
|
||||
*BEFORE_READ.lock().unwrap_or_else(PoisonError::into_inner) = hook;
|
||||
pub(crate) fn set_cap(path: &str, bytes: usize) {
|
||||
CAPS.lock().unwrap_or_else(PoisonError::into_inner).push((path.to_string(), bytes));
|
||||
}
|
||||
|
||||
/// The most bytes of planes one read of `path` may allocate.
|
||||
fn cap_of(path: &str) -> usize {
|
||||
#[cfg(test)]
|
||||
if let Some((_, c)) = CAPS.lock().unwrap_or_else(PoisonError::into_inner).iter().find(|(p, _)| p == path) {
|
||||
return *c;
|
||||
}
|
||||
let _ = path;
|
||||
crate::exr_read::MAX_DECODE_BYTES
|
||||
}
|
||||
|
||||
impl PlaneStore {
|
||||
@@ -278,77 +334,96 @@ impl PlaneStore {
|
||||
|
||||
/// The planes of `want` of the file at `path` (`index`, read from `source`): from the store,
|
||||
/// from a decode in progress, or decoded now (only what the store doesn't hold yet). The
|
||||
/// file it was read from is noted ([`crate::reads`]).
|
||||
/// file it was read from is noted ([`crate::reads`]). A request that keeps finding the
|
||||
/// decodes it waits for called off decodes a copy itself.
|
||||
pub(crate) fn get(&self, path: &str, source: &ExrSource, index: &Arc<ExrIndex>, want: &Want, by: Use) -> Result<Arc<Batch>, Refused> {
|
||||
let level = 0;
|
||||
let mut retries = 0;
|
||||
loop {
|
||||
let mut st = self.lock();
|
||||
st.clock += 1;
|
||||
let clock = st.clock;
|
||||
let at = match st.entries.iter().position(|e| e.path == path && e.level == level) {
|
||||
Some(i) => i,
|
||||
None => {
|
||||
st.entries.push(Entry { path: path.to_string(), level, batch: None, flight: None, failed: None, bytes: 0, used: clock });
|
||||
st.entries.len() - 1
|
||||
}
|
||||
};
|
||||
let Some(e) = st.entries.get_mut(at) else { return Err(Refused::Unreadable) };
|
||||
e.used = clock;
|
||||
if let Some(f) = &e.failed {
|
||||
return Err(f.clone());
|
||||
for _ in 0..ROUNDS {
|
||||
match self.step(path, source, index, want, by) {
|
||||
Step::Done(r) => return r,
|
||||
Step::Again { retry: true } if retries >= RETRIES => break,
|
||||
Step::Again { retry } => retries += u32::from(retry),
|
||||
}
|
||||
if let Some(b) = e.batch.clone().filter(|b| b.covers(index, want)) {
|
||||
}
|
||||
copy(path, source, index, want)
|
||||
}
|
||||
|
||||
/// One round of [`PlaneStore::get`].
|
||||
fn step(&self, path: &str, source: &ExrSource, index: &Arc<ExrIndex>, want: &Want, by: Use) -> Step {
|
||||
let level = 0;
|
||||
let mut st = self.lock();
|
||||
st.clock += 1;
|
||||
let clock = st.clock;
|
||||
let at = match st.entries.iter().position(|e| e.path == path && e.level == level) {
|
||||
Some(i) => i,
|
||||
None => {
|
||||
st.entries.push(Entry { path: path.to_string(), level, batch: None, flight: None, failed: vec![], bytes: 0, used: clock });
|
||||
st.entries.len() - 1
|
||||
}
|
||||
};
|
||||
let Some(e) = st.entries.get_mut(at) else { return Step::Done(Err(Refused::Unreadable)) };
|
||||
e.used = clock;
|
||||
// What the store holds first: a failure of a larger request doesn't hide it.
|
||||
if let Some(b) = e.batch.clone().filter(|b| b.covers(index, want)) {
|
||||
Self::hold(&mut st, path, &b, by);
|
||||
drop(st);
|
||||
crate::reads::note(b.file.clone());
|
||||
return Step::Done(Ok(b));
|
||||
}
|
||||
if let Some((_, why)) = e.failed.iter().find(|(failed, _)| asks_at_least(want, failed)) {
|
||||
return Step::Done(Err(why.clone()));
|
||||
}
|
||||
if let Some(flight) = e.flight.clone() {
|
||||
{
|
||||
let mut fs = flight.state();
|
||||
if !fs.sealed && Arc::ptr_eq(&fs.index, index) {
|
||||
fs.want.add(want);
|
||||
}
|
||||
}
|
||||
drop(st);
|
||||
return match self.wait(&flight) {
|
||||
Waited::Got(Ok(b)) if b.covers(index, want) => {
|
||||
crate::reads::note(b.file.clone());
|
||||
Step::Done(Ok(b))
|
||||
}
|
||||
// Not all it asks for: ask again (the next decode reads what is missing).
|
||||
Waited::Got(Ok(_)) => Step::Again { retry: false },
|
||||
Waited::Got(Err(Refused::Cancelled | Refused::Forgotten | Refused::Stopped)) => Step::Again { retry: true },
|
||||
Waited::Got(Err(e)) => Step::Done(Err(e)),
|
||||
Waited::Cancelled => Step::Done(Err(Refused::Cancelled)),
|
||||
// A copy, kept nowhere.
|
||||
Waited::No => Step::Done(copy(path, source, index, want)),
|
||||
};
|
||||
}
|
||||
// This request decodes: what it asks for, minus what the store holds.
|
||||
let base = e.batch.clone().filter(|b| Arc::ptr_eq(&b.index, index));
|
||||
let flight = Flight::new(index.clone(), want.clone());
|
||||
e.flight = Some(flight.clone());
|
||||
drop(st);
|
||||
#[cfg(test)]
|
||||
crate::exr_channels::count_decode(true);
|
||||
let mut owning = Owning { store: self, path, level, flight: &flight, ended: false };
|
||||
let outcome = run(path, source, &flight, base);
|
||||
match owning.end(outcome) {
|
||||
Ok(b) if b.covers(index, want) => {
|
||||
let mut st = self.lock();
|
||||
Self::hold(&mut st, path, &b, by);
|
||||
drop(st);
|
||||
crate::reads::note(b.file.clone());
|
||||
return Ok(b);
|
||||
}
|
||||
if let Some(flight) = e.flight.clone() {
|
||||
{
|
||||
let mut fs = flight.state();
|
||||
if !fs.sealed && Arc::ptr_eq(&fs.index, index) {
|
||||
fs.want.add(want);
|
||||
}
|
||||
}
|
||||
drop(st);
|
||||
match self.wait(&flight) {
|
||||
Waited::Got(Ok(b)) if b.covers(index, want) => {
|
||||
crate::reads::note(b.file.clone());
|
||||
return Ok(b);
|
||||
}
|
||||
// Not all it asks for: ask again (the next decode reads what is missing).
|
||||
Waited::Got(Ok(_)) => {}
|
||||
Waited::Got(Err(Refused::Cancelled | Refused::Forgotten | Refused::Stopped)) if retries < RETRIES => retries += 1,
|
||||
Waited::Got(Err(e)) => return Err(e),
|
||||
Waited::Cancelled => return Err(Refused::Cancelled),
|
||||
// A copy, kept nowhere.
|
||||
Waited::No => return run(path, source, &Flight::new(index.clone(), want.clone()), None),
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// This request decodes: what it asks for, minus what the store holds.
|
||||
let base = e.batch.clone().filter(|b| Arc::ptr_eq(&b.index, index));
|
||||
let flight = Flight::new(index.clone(), want.clone());
|
||||
e.flight = Some(flight.clone());
|
||||
drop(st);
|
||||
#[cfg(test)]
|
||||
crate::exr_channels::count_decode(true);
|
||||
let mut owning = Owning { store: self, path, level, flight: &flight, ended: false };
|
||||
let outcome = run(path, source, &flight, base);
|
||||
let outcome = owning.end(outcome);
|
||||
match outcome {
|
||||
Ok(b) if b.covers(index, want) => {
|
||||
let mut st = self.lock();
|
||||
Self::hold(&mut st, path, &b, by);
|
||||
drop(st);
|
||||
crate::reads::note(b.file.clone());
|
||||
return Ok(b);
|
||||
}
|
||||
Ok(_) | Err(Refused::Forgotten | Refused::Stopped) if retries < RETRIES => retries += 1,
|
||||
Ok(_) | Err(Refused::Forgotten) => return Err(Refused::Cancelled),
|
||||
Err(e) => return Err(e),
|
||||
Step::Done(Ok(b))
|
||||
}
|
||||
Ok(_) | Err(Refused::Forgotten | Refused::Stopped) => Step::Again { retry: true },
|
||||
Err(e) => Step::Done(Err(e)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Stop keeping the planes `b` of `path` (a picture alone, converted: the pool keeps the
|
||||
/// picture, the planes would be a second copy of it).
|
||||
pub(crate) fn release(&self, path: &str, b: &Arc<Batch>) {
|
||||
let mut st = self.lock();
|
||||
if let Some(e) = st.entries.iter_mut().find(|e| e.path == path && e.batch.as_ref().is_some_and(|k| Arc::ptr_eq(k, b))) {
|
||||
e.batch = None;
|
||||
e.bytes = 0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -387,8 +462,13 @@ impl PlaneStore {
|
||||
e.bytes = b.bytes();
|
||||
e.batch = Some(b.clone());
|
||||
}
|
||||
Err(Refused::Unreadable) => e.failed = Some(Refused::Unreadable),
|
||||
Err(Refused::TooLarge(why)) => e.failed = Some(Refused::TooLarge(why.clone())),
|
||||
Err(why @ (Refused::Unreadable | Refused::TooLarge(_))) => {
|
||||
let want = flight.state().want.clone();
|
||||
if e.failed.len() >= FAILED_KEPT {
|
||||
e.failed.remove(0);
|
||||
}
|
||||
e.failed.push((want, why.clone()));
|
||||
}
|
||||
// (the file changed: what is held is of the old one)
|
||||
Err(Refused::Changed) => {
|
||||
e.batch = None;
|
||||
@@ -473,7 +553,7 @@ impl PlaneStore {
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn cached(&self) -> Vec<String> {
|
||||
self.lock().entries.iter().filter(|e| e.batch.is_some() || e.failed.is_some()).map(|e| e.path.clone()).collect()
|
||||
self.lock().entries.iter().filter(|e| e.batch.is_some() || !e.failed.is_empty()).map(|e| e.path.clone()).collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -520,7 +600,7 @@ fn decode(path: &str, source: &ExrSource, flight: &Arc<Flight>, base: Option<Arc
|
||||
}
|
||||
#[cfg(test)]
|
||||
{
|
||||
let hook = BEFORE_READ.lock().unwrap_or_else(PoisonError::into_inner).clone();
|
||||
let hook = BEFORE_READ.lock().unwrap_or_else(PoisonError::into_inner).iter().find(|(p, _)| p == path).map(|(_, h)| h.clone());
|
||||
if let Some(h) = hook {
|
||||
h(path);
|
||||
}
|
||||
@@ -531,10 +611,12 @@ fn decode(path: &str, source: &ExrSource, flight: &Arc<Flight>, base: Option<Arc
|
||||
fs.sealed = true;
|
||||
fs.want.clone()
|
||||
};
|
||||
let wanted: Vec<usize> = if want.all { (0..index.channels.len()).collect() } else { want.ids.iter().copied().filter(|i| *i < index.channels.len()).collect() };
|
||||
let wanted: Vec<usize> =
|
||||
if want.all { (0..index.channels.len()).collect() } else { want.ids.iter().copied().filter(|i| *i < index.channels.len()).collect() };
|
||||
let base = base.filter(|b| Arc::ptr_eq(&b.index, &index));
|
||||
let missing: Vec<usize> = wanted.iter().copied().filter(|id| !base.as_ref().is_some_and(|b| b.ids.binary_search(id).is_ok())).collect();
|
||||
let (planes, stats) = read_planes(&mut *opened.reader, &index, &ReadRequest::full(missing.clone()));
|
||||
let request = ReadRequest { max_bytes: cap_of(path), ..ReadRequest::full(missing.clone()) };
|
||||
let (planes, stats) = read_planes(&mut *opened.reader, &index, &request);
|
||||
#[cfg(test)]
|
||||
DECODES.lock().unwrap_or_else(PoisonError::into_inner).push((
|
||||
path.to_string(),
|
||||
@@ -544,6 +626,11 @@ fn decode(path: &str, source: &ExrSource, flight: &Arc<Flight>, base: Option<Arc
|
||||
let _ = &stats;
|
||||
let planes = planes.map_err(|e| match e {
|
||||
ReadError::Cancelled => Refused::Cancelled,
|
||||
// (a read that failed just now: the next request reads again)
|
||||
ReadError::Io(why) => {
|
||||
log::warn!("media: {path}: {why}");
|
||||
Refused::Stopped
|
||||
}
|
||||
ReadError::TooLarge(why) => {
|
||||
log::warn!("media: {path}: {why}");
|
||||
Refused::TooLarge(why)
|
||||
@@ -613,14 +700,16 @@ pub(crate) enum Picture {
|
||||
/// [`crate::exr_channels::layered_image`]). With `extra` (an effect reads the footage's
|
||||
/// channels) those channels are read in the same pass and kept for the effects.
|
||||
pub(crate) fn picture(path: &str, source: &ExrSource, op: AlphaOp, extra: Option<&AuxNeed>) -> Picture {
|
||||
for _ in 0..2 {
|
||||
let mut extra = extra;
|
||||
for _ in 0..3 {
|
||||
let index = match index_of(path, source) {
|
||||
Ok(ix) => ix,
|
||||
Err(IndexError::Unsupported(why)) => return Picture::Done(Err(why.to_string())),
|
||||
Err(_) => return Picture::Legacy,
|
||||
};
|
||||
let Some(ids) = index.picture_ids() else { return Picture::Legacy };
|
||||
let mut want = Want { all: false, ids: ids.into_iter().collect() };
|
||||
let alone = Want { all: false, ids: ids.into_iter().collect() };
|
||||
let mut want = alone.clone();
|
||||
if let Some(need) = extra {
|
||||
match need_ids(&index, need) {
|
||||
Some(more) => want.ids.extend(more),
|
||||
@@ -631,33 +720,21 @@ pub(crate) fn picture(path: &str, source: &ExrSource, op: AlphaOp, extra: Option
|
||||
let batch = match store().get(path, source, &index, &want, by) {
|
||||
Ok(b) => b,
|
||||
Err(Refused::Changed) => continue,
|
||||
// The channels with it are too many: the picture alone (the effects ask for theirs).
|
||||
Err(Refused::TooLarge(_)) if extra.is_some() => {
|
||||
extra = None;
|
||||
continue;
|
||||
}
|
||||
Err(Refused::TooLarge(why)) => return Picture::Done(Err(why)),
|
||||
Err(Refused::Cancelled | Refused::Forgotten) => return Picture::Cancelled,
|
||||
// (the whole file's decode says why)
|
||||
Err(Refused::Stopped) => return Picture::Legacy,
|
||||
Err(Refused::TooLarge(why)) => return Picture::Done(Err(why)),
|
||||
Err(Refused::Unreadable) => return Picture::Legacy,
|
||||
};
|
||||
let aux = &batch.aux;
|
||||
let img = match index.rgb {
|
||||
Rgb::Planes([r, g, b], a) => {
|
||||
let a = match a {
|
||||
Some(a) => batch.plane(a).map(Some),
|
||||
None => Some(None),
|
||||
};
|
||||
match (batch.plane(r), batch.plane(g), batch.plane(b), a) {
|
||||
(Some(r), Some(g), Some(b), Some(a)) => planes_picture(aux, [r, g, b], a, op),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
Rgb::None => index.picture.as_ref().and_then(|(_, rgb, a)| {
|
||||
let a = match a {
|
||||
Some(n) => Some(aux.get(n)?),
|
||||
None => None,
|
||||
};
|
||||
planes_picture(aux, [aux.get(&rgb[0])?, aux.get(&rgb[1])?, aux.get(&rgb[2])?], a, op)
|
||||
}),
|
||||
Rgb::Unsupported => None,
|
||||
Err(Refused::Stopped | Refused::Unreadable) => return Picture::Legacy,
|
||||
};
|
||||
let img = picture_of(&index, &batch, op);
|
||||
// A picture read alone is kept by the pool once converted, not twice.
|
||||
if extra.is_none() && !batch.all && batch.ids.len() == want.ids.len() {
|
||||
store().release(path, &batch);
|
||||
}
|
||||
return match img {
|
||||
Some(img) => Picture::Done(Ok(img)),
|
||||
None => Picture::Legacy,
|
||||
@@ -666,6 +743,29 @@ pub(crate) fn picture(path: &str, source: &ExrSource, op: AlphaOp, extra: Option
|
||||
Picture::Legacy
|
||||
}
|
||||
|
||||
/// The picture of the file `index` describes, made of the planes of `batch`.
|
||||
fn picture_of(index: &ExrIndex, batch: &Batch, op: AlphaOp) -> Option<Image> {
|
||||
let aux = &batch.aux;
|
||||
match index.rgb {
|
||||
Rgb::Planes([r, g, b], a) => {
|
||||
let a = match a {
|
||||
Some(a) => Some(batch.plane(a)?),
|
||||
None => None,
|
||||
};
|
||||
planes_picture(aux, [batch.plane(r)?, batch.plane(g)?, batch.plane(b)?], a, op)
|
||||
}
|
||||
Rgb::None => {
|
||||
let (_, rgb, a) = index.picture.as_ref()?;
|
||||
let a = match a {
|
||||
Some(n) => Some(aux.get(n)?),
|
||||
None => None,
|
||||
};
|
||||
planes_picture(aux, [aux.get(&rgb[0])?, aux.get(&rgb[1])?, aux.get(&rgb[2])?], a, op)
|
||||
}
|
||||
Rgb::Unsupported => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The channels `need` reads of the OpenEXR at `path` (all for [`AuxNeed::All`]), from the
|
||||
/// store. `None` when it isn't a readable OpenEXR (or the request was called off).
|
||||
pub(crate) fn aux(path: &str, source: &ExrSource, need: &AuxNeed) -> Option<Arc<AuxChannels>> {
|
||||
|
||||
@@ -7,7 +7,9 @@ use effectcraft_raster::channels3d::{AuxLookup, AuxNeed};
|
||||
use effectcraft_render::reads::FileStamp;
|
||||
use effectcraft_render::{FloatCurve, FootageSource};
|
||||
use effectcraft_time::Tick;
|
||||
use exr::prelude::{AnyChannel, AnyChannels, Blocks, Compression, Encoding, FlatSamples, ImageAttributes, IntegerBounds, Layer, LayerAttributes, LineOrder, WritableImage, f16};
|
||||
use exr::prelude::{
|
||||
AnyChannel, AnyChannels, Blocks, Compression, Encoding, FlatSamples, ImageAttributes, IntegerBounds, Layer, LayerAttributes, LineOrder, WritableImage, f16,
|
||||
};
|
||||
|
||||
use super::*;
|
||||
use crate::exr_read::ExrSource;
|
||||
@@ -36,7 +38,9 @@ fn part(name: Option<&str>, channels: &[(&str, bool)], size: (usize, usize), com
|
||||
.collect();
|
||||
let mut attrs = name.map_or_else(LayerAttributes::default, LayerAttributes::named);
|
||||
if name == Some("Crypto_Material") {
|
||||
attrs.other.insert(exr::prelude::Text::from("cryptomatte/abc1234/name"), exr::prelude::AttributeValue::Text(exr::prelude::Text::from("Crypto_Material")));
|
||||
attrs
|
||||
.other
|
||||
.insert(exr::prelude::Text::from("cryptomatte/abc1234/name"), exr::prelude::AttributeValue::Text(exr::prelude::Text::from("Crypto_Material")));
|
||||
attrs.other.insert(
|
||||
exr::prelude::Text::from("cryptomatte/abc1234/manifest"),
|
||||
exr::prelude::AttributeValue::Text(exr::prelude::Text::from(r#"{"bunny":"13851a76"}"#)),
|
||||
@@ -56,7 +60,8 @@ fn write(parts: Vec<Layer<AnyChannels<FlatSamples>>>, size: (usize, usize)) -> V
|
||||
|
||||
const SIZE: (usize, usize) = (64, 64);
|
||||
const RGBA: [(&str, bool); 4] = [("R", true), ("G", true), ("B", true), ("A", true)];
|
||||
const CRYPTO: [(&str, bool); 4] = [("Crypto_Material00.R", false), ("Crypto_Material00.G", false), ("Crypto_Material00.B", false), ("Crypto_Material00.A", false)];
|
||||
const CRYPTO: [(&str, bool); 4] =
|
||||
[("Crypto_Material00.R", false), ("Crypto_Material00.G", false), ("Crypto_Material00.B", false), ("Crypto_Material00.A", false)];
|
||||
|
||||
/// A multi-part file as renderers write one part per pass: the beauty, a Cryptomatte layer whose
|
||||
/// channels keep their full names, and depth; one scan line per chunk.
|
||||
@@ -349,14 +354,17 @@ fn a_file_forgotten_while_it_decodes_is_not_kept() {
|
||||
let forgot = Arc::new(AtomicU64::new(0));
|
||||
{
|
||||
let (store, forgot) = (store.clone(), forgot.clone());
|
||||
set_before_read(Some(Arc::new(move |p: &str| {
|
||||
if p == "/3271-forgotten.exr" && forgot.fetch_add(1, Ordering::SeqCst) == 0 {
|
||||
store.forget(p);
|
||||
}
|
||||
})));
|
||||
set_before_read(
|
||||
path,
|
||||
Some(Arc::new(move |p: &str| {
|
||||
if p == "/3271-forgotten.exr" && forgot.fetch_add(1, Ordering::SeqCst) == 0 {
|
||||
store.forget(p);
|
||||
}
|
||||
})),
|
||||
);
|
||||
}
|
||||
let got = store.get(path, &source, &index, &Want { all: true, ids: Default::default() }, Use::Other);
|
||||
set_before_read(None);
|
||||
set_before_read(path, None);
|
||||
assert!(got.is_ok());
|
||||
assert_eq!(decodes_of(path).len(), 2, "decoded again after the reload");
|
||||
assert_eq!(store.cached(), [path], "only the second decode is kept");
|
||||
@@ -403,7 +411,7 @@ fn given_aux_footage_decodes_once_from_the_first_frame() {
|
||||
assert_eq!(take_decodes(), [0, 2]);
|
||||
let d = decodes_of(learned);
|
||||
assert_eq!(d[0].channels.len(), 4, "the picture alone");
|
||||
assert_eq!(d[1].channels.len() + 4, crate::exr_index::cached(learned).unwrap().channels.len(), "the rest");
|
||||
assert_eq!(d[1].channels.len(), crate::exr_index::cached(learned).unwrap().channels.len(), "all: a picture read alone isn't kept twice");
|
||||
}
|
||||
|
||||
/// #474: the picture made of the planes of a file with exact R, G, B channels is the `image`
|
||||
@@ -417,7 +425,10 @@ fn picture_from_planes_is_the_image_crates() {
|
||||
write(vec![part(None, &[("R", true), ("G", true), ("B", true), ("A", false), ("Z", false)], display, Compression::ZIP16)], display),
|
||||
write(vec![part(None, &[("R", false), ("G", false), ("B", false), ("Image.R", true)], display, Compression::RLE)], display),
|
||||
write(
|
||||
vec![part(Some("diffuse"), &[("R", true), ("G", true), ("B", true)], display, Compression::ZIP1), part(Some("beauty"), &RGBA, display, Compression::PIZ)],
|
||||
vec![
|
||||
part(Some("diffuse"), &[("R", true), ("G", true), ("B", true)], display, Compression::ZIP1),
|
||||
part(Some("beauty"), &RGBA, display, Compression::PIZ),
|
||||
],
|
||||
display,
|
||||
),
|
||||
];
|
||||
@@ -451,11 +462,14 @@ fn a_decode_that_panics_is_not_left_in_the_store() {
|
||||
let panicked = Arc::new(AtomicU64::new(0));
|
||||
{
|
||||
let (watch, panicked) = (path.clone(), panicked.clone());
|
||||
set_before_read(Some(Arc::new(move |p: &str| {
|
||||
if p == watch && panicked.fetch_add(1, Ordering::SeqCst) == 0 {
|
||||
panic!("a decoder bug");
|
||||
}
|
||||
})));
|
||||
set_before_read(
|
||||
&path,
|
||||
Some(Arc::new(move |p: &str| {
|
||||
if p == watch && panicked.fetch_add(1, Ordering::SeqCst) == 0 {
|
||||
panic!("a decoder bug");
|
||||
}
|
||||
})),
|
||||
);
|
||||
}
|
||||
let ask = || store.get(&path, &source, &index, &all, Use::Other);
|
||||
if inline {
|
||||
@@ -469,9 +483,9 @@ fn a_decode_that_panics_is_not_left_in_the_store() {
|
||||
// Handed to the decode threads: the request asks again by itself.
|
||||
assert!(ask().is_ok());
|
||||
}
|
||||
set_before_read(None);
|
||||
set_before_read(&path, None);
|
||||
assert_eq!(panicked.load(Ordering::SeqCst), 2, "inline: {inline}");
|
||||
assert_eq!(store.cached(), [path.clone()]);
|
||||
assert_eq!(store.cached(), std::slice::from_ref(&path));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -495,15 +509,18 @@ fn a_file_is_used_as_of_its_decode() {
|
||||
{
|
||||
let get = get.clone();
|
||||
let nested = Arc::new(AtomicU64::new(0));
|
||||
set_before_read(Some(Arc::new(move |p: &str| {
|
||||
// Another frame's file is asked for and decoded while this one decodes.
|
||||
if p == "/3271-used-first.exr" && nested.fetch_add(1, Ordering::SeqCst) == 0 {
|
||||
get("/3271-used-second.exr").unwrap();
|
||||
}
|
||||
})));
|
||||
set_before_read(
|
||||
"/3271-used-first.exr",
|
||||
Some(Arc::new(move |p: &str| {
|
||||
// Another frame's file is asked for and decoded while this one decodes.
|
||||
if p == "/3271-used-first.exr" && nested.fetch_add(1, Ordering::SeqCst) == 0 {
|
||||
get("/3271-used-second.exr").unwrap();
|
||||
}
|
||||
})),
|
||||
);
|
||||
}
|
||||
get("/3271-used-first.exr").unwrap();
|
||||
set_before_read(None);
|
||||
set_before_read("/3271-used-first.exr", None);
|
||||
get("/3271-used-third.exr").unwrap();
|
||||
assert_eq!(store.cached(), ["/3271-used-first.exr", "/3271-used-third.exr"], "the file decoded first went first");
|
||||
}
|
||||
@@ -574,3 +591,216 @@ fn a_new_aux_footage_set_returns_to_picture_only() {
|
||||
pool.set_aux_footage(&[]);
|
||||
assert_eq!(frame(3), Some(4), "a new set drops what was learned");
|
||||
}
|
||||
|
||||
/// [`part`] with its data window at `pos`.
|
||||
fn part_at(name: Option<&str>, channels: &[(&str, bool)], size: (usize, usize), pos: (i32, i32), compression: Compression) -> Layer<AnyChannels<FlatSamples>> {
|
||||
let mut p = part(name, channels, size, compression);
|
||||
p.attributes.layer_position = exr::prelude::Vec2(pos.0, pos.1);
|
||||
p
|
||||
}
|
||||
|
||||
/// The bytes of a file of `parts` over the display window `display`.
|
||||
fn write_in(parts: Vec<Layer<AnyChannels<FlatSamples>>>, display: IntegerBounds) -> Vec<u8> {
|
||||
let image = exr::prelude::Image::from_layers(ImageAttributes::new(display), parts);
|
||||
let mut bytes = std::io::Cursor::new(Vec::new());
|
||||
image.write().non_parallel().to_buffered(&mut bytes).unwrap();
|
||||
bytes.into_inner()
|
||||
}
|
||||
|
||||
/// #474 review: data windows placed inside and around the display window make the `image`
|
||||
/// crate's picture too.
|
||||
#[test]
|
||||
fn picture_from_planes_with_data_window_offsets_is_the_image_crates() {
|
||||
let display = IntegerBounds::new((1, -1), (5, 4));
|
||||
let files = [
|
||||
write_in(vec![part_at(None, &[("R", true), ("G", true), ("B", true), ("A", false), ("Z", false)], (5, 4), (1, -1), Compression::ZIP16)], display),
|
||||
write_in(vec![part_at(None, &[("R", false), ("G", false), ("B", false), ("Image.R", true)], (3, 2), (0, 0), Compression::RLE)], display),
|
||||
write_in(
|
||||
vec![
|
||||
part_at(Some("diffuse"), &[("R", true), ("G", true), ("B", true)], (5, 4), (1, -1), Compression::ZIP1),
|
||||
part_at(Some("beauty"), &RGBA, (2, 2), (2, 0), Compression::PIZ),
|
||||
],
|
||||
display,
|
||||
),
|
||||
];
|
||||
for (k, bytes) in files.iter().enumerate() {
|
||||
let path = format!("/3271-offsets-{k}.exr");
|
||||
let source = ExrSource::Bytes(bytes.as_slice().into());
|
||||
assert!(matches!(crate::exr_index::get(&path, &source).unwrap().rgb, Rgb::Planes(..)), "file {k}");
|
||||
let f = Footage::default();
|
||||
let Picture::Done(Ok(got)) = picture(&path, &source, AlphaOp::of(&f, FloatCurve::Linear), None) else { panic!("file {k}: a picture") };
|
||||
let img = crate::probe::load_oriented(bytes, Some(image::ImageFormat::OpenExr)).unwrap();
|
||||
assert_eq!(bits(&got), bits(&crate::convert::dynamic_to_image(&img, AlphaOp::of(&f, FloatCurve::Linear))), "file {k}");
|
||||
}
|
||||
}
|
||||
|
||||
/// #3271 review: a request that failed (a broken part) doesn't hide what the store holds, nor
|
||||
/// fail smaller requests: the picture read before is served, and read again when asked alone.
|
||||
#[test]
|
||||
fn a_failed_request_fails_only_requests_as_large() {
|
||||
let mut bytes = multipart();
|
||||
// The depth part is last: its chunks now point past the end of the file.
|
||||
bytes.truncate(bytes.len() - 200);
|
||||
let path = "/3271-broken-part.exr";
|
||||
let source = ExrSource::Bytes(bytes.into());
|
||||
let store = PlaneStore::new(CACHE_BUDGET);
|
||||
let index = crate::exr_index::get(path, &source).unwrap();
|
||||
let picture = Want { all: false, ids: index.picture_ids().unwrap().into_iter().collect() };
|
||||
let all = Want { all: true, ids: Default::default() };
|
||||
let held = store.get(path, &source, &index, &picture, Use::Picture).unwrap();
|
||||
assert_eq!(store.get(path, &source, &index, &all, Use::Other).err(), Some(Refused::Unreadable));
|
||||
assert!(Arc::ptr_eq(&store.get(path, &source, &index, &picture, Use::Other).unwrap(), &held), "served from the store");
|
||||
assert_eq!(store.get(path, &source, &index, &all, Use::Other).err(), Some(Refused::Unreadable), "remembered");
|
||||
assert_eq!(decodes_of(path).len(), 2);
|
||||
// Another store: the picture alone after the failure decodes.
|
||||
let other = PlaneStore::new(CACHE_BUDGET);
|
||||
assert!(other.get(path, &source, &index, &all, Use::Other).is_err());
|
||||
assert!(other.get(path, &source, &index, &picture, Use::Other).is_ok(), "a smaller request isn't refused");
|
||||
}
|
||||
|
||||
/// #3271 review: when the picture with the channels its effects read is more than one file may
|
||||
/// allocate, the picture is read alone instead of failing.
|
||||
#[test]
|
||||
fn a_picture_too_large_with_its_channels_is_read_alone() {
|
||||
let bytes: Arc<[u8]> = single_part_zips().into();
|
||||
let path = "/3271-too-large-with-aux.exr";
|
||||
let pool = crate::MediaPool::new();
|
||||
pool.add_bytes(path, bytes.clone());
|
||||
let f = crate::probe_bytes(path, bytes.clone()).unwrap();
|
||||
// Room for the picture's 4 planes, not for all of them.
|
||||
set_cap(path, 5 * SIZE.0 * SIZE.1 * 4);
|
||||
pool.set_aux_footage(&[path.to_string()]);
|
||||
let img = pool.frame_at(&f, Tick::ZERO).unwrap();
|
||||
assert_eq!(bits(&img), bits(&image_crates(&bytes, &f)));
|
||||
let d = decodes_of(path);
|
||||
assert_eq!(d.last().map(|d| d.channels.len()), Some(4), "{d:?}");
|
||||
}
|
||||
|
||||
/// #474: at most CACHE_FILES entries are kept, however small.
|
||||
#[test]
|
||||
fn the_store_keeps_a_bounded_number_of_files() {
|
||||
let bytes: Arc<[u8]> = write(vec![part(None, &[("R", true)], (2, 2), Compression::ZIP1)], (2, 2)).into();
|
||||
let store = PlaneStore::new(CACHE_BUDGET);
|
||||
let all = Want { all: true, ids: Default::default() };
|
||||
for i in 0..CACHE_FILES + 10 {
|
||||
let path = format!("/3271-many/{i}.exr");
|
||||
let source = ExrSource::Bytes(bytes.clone());
|
||||
let index = crate::exr_index::get(&path, &source).unwrap();
|
||||
store.get(&path, &source, &index, &all, Use::Other).unwrap();
|
||||
}
|
||||
assert!(store.cached().len() <= CACHE_FILES);
|
||||
}
|
||||
|
||||
/// `bytes` with the data window's right edge (`xMax`) moved to `x_max`.
|
||||
fn with_data_window_x_max(mut bytes: Vec<u8>, x_max: i32) -> Vec<u8> {
|
||||
let tag = b"dataWindow\0box2i\0\x10\0\0\0";
|
||||
let at = bytes.windows(tag.len()).position(|w| w == tag).unwrap() + tag.len() + 8;
|
||||
bytes[at..at + 4].copy_from_slice(&x_max.to_le_bytes());
|
||||
bytes
|
||||
}
|
||||
|
||||
/// #3271 review (never crash): a tiny file whose data window is a billion pixels wide over one
|
||||
/// chunk made the decompressor reserve gigabytes for a block (an abort, not an error). Every
|
||||
/// way in refuses it with an error: import, the picture, the channels, the names.
|
||||
#[test]
|
||||
fn a_hostile_data_window_is_refused_not_allocated() {
|
||||
for compression in [Compression::PIZ, Compression::ZIP1, Compression::B44] {
|
||||
let small = write(vec![part(None, &[("R", true)], (1, 1), compression)], (1, 1));
|
||||
let bytes = with_data_window_x_max(small, (1 << 30) - 3);
|
||||
let path = format!("/3271-hostile-{compression:?}.exr");
|
||||
assert!(!crate::exr_index::safe_to_decode(&bytes));
|
||||
assert!(crate::exr_index::build(&mut std::io::Cursor::new(bytes.as_slice()), bytes.len() as u64, FileStamp::Virtual).is_err());
|
||||
assert!(crate::exr_channels::read_exr_channels(&bytes).is_none());
|
||||
assert!(crate::exr_channels::layered_image(&bytes).is_none());
|
||||
let b: Arc<[u8]> = bytes.into();
|
||||
assert!(crate::probe_bytes(&path, b.clone()).is_err(), "import refuses it");
|
||||
let pool = crate::MediaPool::new();
|
||||
pool.add_bytes(&path, b);
|
||||
let f = Footage { path: path.clone(), kind: effectcraft_project::FootageKind::Still, has_video: true, width: 1, height: 1, ..Default::default() };
|
||||
assert!(pool.frame_at(&f, Tick::ZERO).is_err());
|
||||
assert!(pool.aux(ItemId(1), &f, Tick::ZERO).is_none());
|
||||
assert!(pool.aux_names(ItemId(1), &f, Tick::ZERO).is_none());
|
||||
}
|
||||
}
|
||||
|
||||
/// #3271 review (never crash): damaged files (bytes changed in the offset table, the chunk
|
||||
/// headers or the pixels) read as an error or as some pixels, never a panic.
|
||||
#[test]
|
||||
fn damaged_files_never_panic() {
|
||||
let files = [single_part_zips(), multipart(), write(vec![part(None, &RGBA, (16, 40), Compression::PIZ)], (16, 40))];
|
||||
for good in &files {
|
||||
let start = crate::exr_index::build(&mut std::io::Cursor::new(good.as_slice()), good.len() as u64, FileStamp::Virtual)
|
||||
.unwrap()
|
||||
.parts
|
||||
.iter()
|
||||
.flat_map(|p| p.chunks.iter().map(|c| c.offset))
|
||||
.min()
|
||||
.unwrap() as usize;
|
||||
// From a little before the chunks (the offset tables) to the end, in steps.
|
||||
let from = start.saturating_sub(64);
|
||||
for (i, at) in (from..good.len()).step_by(((good.len() - from) / 150).max(1)).enumerate() {
|
||||
let mut bad = good.clone();
|
||||
bad[at] ^= [0xff, 0x80, 0x01, 0x7f][i % 4];
|
||||
let _ = crate::exr_channels::read_exr_channels(&bad);
|
||||
let _ = crate::exr_channels::layered_image(&bad);
|
||||
let mut src = std::io::Cursor::new(bad.as_slice());
|
||||
if let Ok(ix) = crate::exr_index::build(&mut src, bad.len() as u64, FileStamp::Virtual) {
|
||||
let req = ReadRequest { rows: Some(3..9), level: 1, ..ReadRequest::full((0..ix.channels.len()).collect()) };
|
||||
let _ = crate::exr_read::read_planes(&mut src, &ix, &req);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// #3271 review: rows and levels on every compression: the rows read are the full read's,
|
||||
/// whole chunks are decompressed (1, 16, 32 or 256 lines), and a level is the full read's
|
||||
/// every other row and column.
|
||||
#[test]
|
||||
fn rows_and_levels_on_every_compression() {
|
||||
let cases = [
|
||||
(Compression::Uncompressed, 1),
|
||||
(Compression::RLE, 1),
|
||||
(Compression::ZIP1, 1),
|
||||
(Compression::ZIP16, 16),
|
||||
(Compression::PXR24, 16),
|
||||
(Compression::PIZ, 32),
|
||||
(Compression::B44, 32),
|
||||
(Compression::DWAA(None), 32),
|
||||
(Compression::DWAB(None), 256),
|
||||
];
|
||||
for (compression, per_chunk) in cases {
|
||||
let bytes = write(vec![part(None, &RGBA, (32, 64), compression)], (32, 64));
|
||||
let (ix, mut src) = indexed(&bytes);
|
||||
let all = ReadRequest::full(vec![0, 1, 2, 3]);
|
||||
let full = crate::exr_read::read_planes(&mut src, &ix, &all).0.unwrap();
|
||||
let (band, stats) = crate::exr_read::read_planes(&mut src, &ix, &ReadRequest { rows: Some(10..20), ..all.clone() });
|
||||
let band = band.unwrap();
|
||||
// The chunks holding rows 10..20, each `per_chunk` lines (the last one shorter).
|
||||
let lines: usize = (0..64usize).step_by(per_chunk).filter(|y0| *y0 < 20 && y0 + per_chunk > 10).map(|y0| per_chunk.min(64 - y0)).sum();
|
||||
assert_eq!(stats.lines, lines, "{compression:?}");
|
||||
for (b, f) in band.iter().zip(&full) {
|
||||
assert_eq!(
|
||||
b[10 * 32..20 * 32].iter().map(|v| v.to_bits()).collect::<Vec<_>>(),
|
||||
f[10 * 32..20 * 32].iter().map(|v| v.to_bits()).collect::<Vec<_>>(),
|
||||
"{compression:?}"
|
||||
);
|
||||
}
|
||||
let (half, _) = crate::exr_read::read_planes(&mut src, &ix, &ReadRequest { level: 1, ..all.clone() });
|
||||
let half = half.unwrap();
|
||||
for y in 0..32 {
|
||||
for x in 0..16 {
|
||||
assert_eq!(half[0][y * 16 + x].to_bits(), full[0][y * 2 * 32 + x * 2].to_bits(), "{compression:?} ({x}, {y})");
|
||||
}
|
||||
}
|
||||
}
|
||||
// Tiles: rows read as in the full read.
|
||||
let mut tiled = part(None, &RGBA, (32, 64), Compression::ZIP16);
|
||||
tiled.encoding.blocks = Blocks::Tiles(exr::prelude::Vec2(16, 16));
|
||||
let bytes = write(vec![tiled], (32, 64));
|
||||
let (ix, mut src) = indexed(&bytes);
|
||||
let all = ReadRequest::full(vec![0, 1, 2, 3]);
|
||||
let full = crate::exr_read::read_planes(&mut src, &ix, &all).0.unwrap();
|
||||
let (band, stats) = crate::exr_read::read_planes(&mut src, &ix, &ReadRequest { rows: Some(10..20), ..all.clone() });
|
||||
assert_eq!(stats.chunks, [4], "two rows of two tiles");
|
||||
assert_eq!(band.unwrap()[0][10 * 32..20 * 32], full[0][10 * 32..20 * 32]);
|
||||
}
|
||||
@@ -47,6 +47,14 @@ pub(crate) struct Opened {
|
||||
}
|
||||
|
||||
impl ExrSource {
|
||||
/// Its identity now (a file's size and modification time), without opening it.
|
||||
pub(crate) fn stamp(&self) -> std::io::Result<FileStamp> {
|
||||
match self {
|
||||
ExrSource::Bytes(_) => Ok(FileStamp::Virtual),
|
||||
ExrSource::File(path) => std::fs::metadata(path).map(|m| FileStamp::of_metadata(&m)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Open it: a file only when it is a regular file (a project can name `/dev/zero` or a FIFO,
|
||||
/// #2134), with its identity from the open file.
|
||||
pub(crate) fn open(&self) -> std::io::Result<Opened> {
|
||||
@@ -74,12 +82,14 @@ pub(crate) struct ReadRequest {
|
||||
pub(crate) rows: Option<Range<i64>>,
|
||||
/// Every 2^level-th row and column of the display window (0: full resolution).
|
||||
pub(crate) level: u32,
|
||||
/// The most bytes of planes the read may allocate.
|
||||
pub(crate) max_bytes: usize,
|
||||
}
|
||||
|
||||
impl ReadRequest {
|
||||
/// Every row of `channels` at full resolution.
|
||||
pub(crate) fn full(channels: Vec<usize>) -> ReadRequest {
|
||||
ReadRequest { channels, rows: None, level: 0 }
|
||||
ReadRequest { channels, rows: None, level: 0, max_bytes: MAX_DECODE_BYTES }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,8 +107,11 @@ pub(crate) struct ReadStats {
|
||||
/// Why a read gave no planes.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub(crate) enum ReadError {
|
||||
/// A chunk can't be read or decompressed.
|
||||
/// A chunk can't be decompressed, or the file doesn't hold it.
|
||||
Unreadable(String),
|
||||
/// Reading the file failed just now (a seek or read error, no memory for a chunk): another
|
||||
/// read may succeed.
|
||||
Io(String),
|
||||
/// The planes need more memory than one read may allocate (the reason, to show).
|
||||
TooLarge(String),
|
||||
/// The request was called off.
|
||||
@@ -148,14 +161,15 @@ fn read_into(src: &mut dyn ReadSeek, index: &ExrIndex, req: &ReadRequest, stats:
|
||||
let (ow, oh) = level_size((w, h), level);
|
||||
let n = ow.checked_mul(oh).ok_or_else(|| ReadError::Unreadable("display window".into()))?;
|
||||
let need = n.saturating_mul(req.channels.len()).saturating_mul(std::mem::size_of::<f32>());
|
||||
if need > MAX_DECODE_BYTES {
|
||||
let most = req.max_bytes.min(MAX_DECODE_BYTES);
|
||||
if need > most {
|
||||
let gb = |b: usize| b as f64 / f64::from(1u32 << 30);
|
||||
return Err(ReadError::TooLarge(format!(
|
||||
"its {}×{} OpenEXR channels need {:.1} GB of memory, more than the {:.1} GB one file may use",
|
||||
w,
|
||||
h,
|
||||
gb(need),
|
||||
gb(MAX_DECODE_BYTES)
|
||||
gb(most)
|
||||
)));
|
||||
}
|
||||
// Per part: the plane slot of each of its channels (`None`: not requested).
|
||||
@@ -219,12 +233,12 @@ fn read_into(src: &mut dyn ReadSeek, index: &ExrIndex, req: &ReadRequest, stats:
|
||||
let mut raw = Vec::with_capacity(batch.len());
|
||||
for (pi, offset, len) in batch {
|
||||
if pos != Some(*offset) {
|
||||
src.seek(SeekFrom::Start(*offset)).map_err(|e| ReadError::Unreadable(e.to_string()))?;
|
||||
src.seek(SeekFrom::Start(*offset)).map_err(|e| ReadError::Io(e.to_string()))?;
|
||||
}
|
||||
let size = usize::try_from(*len).map_err(|_| ReadError::Unreadable("chunk size".into()))?;
|
||||
let mut buf = Vec::new();
|
||||
buf.try_reserve_exact(size).map_err(|e| ReadError::Unreadable(e.to_string()))?;
|
||||
(&mut *src).take(*len).read_to_end(&mut buf).map_err(|e| ReadError::Unreadable(e.to_string()))?;
|
||||
buf.try_reserve_exact(size).map_err(|e| ReadError::Io(e.to_string()))?;
|
||||
(&mut *src).take(*len).read_to_end(&mut buf).map_err(|e| ReadError::Io(e.to_string()))?;
|
||||
pos = Some(offset.saturating_add(buf.len() as u64));
|
||||
stats.bytes = stats.bytes.saturating_add(buf.len() as u64);
|
||||
if let Some(c) = stats.chunks.get_mut(*pi) {
|
||||
|
||||
@@ -6,8 +6,8 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Condvar, Mutex, OnceLock};
|
||||
|
||||
use effectcraft_project::{AlphaMode, Footage, FootageKind, ItemId};
|
||||
use effectcraft_raster::{Image, Px};
|
||||
use effectcraft_raster::channels3d::AuxNeed;
|
||||
use effectcraft_raster::{Image, Px};
|
||||
use effectcraft_render::reads::{FileStamp, ReadFile, ReadFiles, ReadLog};
|
||||
use effectcraft_render::{FloatCurve, FootageSource};
|
||||
use effectcraft_time::{FrameRate, TICKS_PER_SECOND, Tick};
|
||||
@@ -1328,6 +1328,11 @@ impl Inner {
|
||||
{
|
||||
return Err(MediaError::Decode(format!("{path}: {why}")));
|
||||
}
|
||||
// A hostile OpenEXR header (a data window a billion pixels wide over one tiny chunk) made
|
||||
// the decoder allocate gigabytes for a block: refused before any decoder sees it.
|
||||
if crate::still_format(path, &bytes) == Some(image::ImageFormat::OpenExr) && !crate::exr_index::safe_to_decode(&bytes) {
|
||||
return Err(MediaError::Decode(format!("{path}: an OpenEXR part larger than this reader takes")));
|
||||
}
|
||||
#[cfg(test)]
|
||||
crate::exr_channels::count_decode(false);
|
||||
match crate::probe::load_oriented(&bytes, crate::still_format(path, &bytes)) {
|
||||
|
||||
@@ -257,6 +257,9 @@ fn probe_still_bytes(path: &str, bytes: &[u8], fmt: image::ImageFormat) -> Resul
|
||||
{
|
||||
return Err(MediaError::Decode(format!("{path}: {why}")));
|
||||
}
|
||||
if fmt == image::ImageFormat::OpenExr && !crate::exr_index::safe_to_decode(bytes) {
|
||||
return Err(MediaError::Decode(format!("{path}: an OpenEXR part larger than this reader takes")));
|
||||
}
|
||||
let reader = image::ImageReader::with_format(std::io::Cursor::new(bytes), fmt);
|
||||
let (w, h, has_alpha) = match reader.into_decoder() {
|
||||
Ok(mut dec) => {
|
||||
|
||||
@@ -17,7 +17,6 @@
|
||||
//! the object name reinterpreted as a float with the exponent nudged off 0 and 255, and the
|
||||
//! manifest maps names to hashes.
|
||||
|
||||
|
||||
/// Depth written where nothing was rendered.
|
||||
pub const BACKGROUND_DEPTH: f32 = 1.0e10;
|
||||
|
||||
@@ -624,7 +623,16 @@ mod tests {
|
||||
/// the planes; a set holding only some channels keeps listing every Cryptomatte layer.
|
||||
#[test]
|
||||
fn needs_resolve_as_the_effects_look_up() {
|
||||
let all = named(&["A.CryptoAsset00.R", "A.CryptoAsset00.G", "B.CryptoObject00.R", "B.CryptoObject00.G", "Image.R", "depth.Z", "zz.Custom00.R", "zz.Custom00.G"]);
|
||||
let all = named(&[
|
||||
"A.CryptoAsset00.R",
|
||||
"A.CryptoAsset00.G",
|
||||
"B.CryptoObject00.R",
|
||||
"B.CryptoObject00.G",
|
||||
"Image.R",
|
||||
"depth.Z",
|
||||
"zz.Custom00.R",
|
||||
"zz.Custom00.G",
|
||||
]);
|
||||
let need = AuxNeed::Lookups(vec![
|
||||
AuxLookup::Name(" image.r ".into()),
|
||||
AuxLookup::find(&DEPTH_NAMES),
|
||||
|
||||
@@ -220,13 +220,18 @@ impl<'a> Renderer<'a> {
|
||||
/// OpenEXR footage's at the layer's source time (Time Remap included), or a precomp's
|
||||
/// generated 3D channels at its nested time. `None` for other layers.
|
||||
pub fn layer_aux(&self, ctx: &EvalCtx, layer: &Layer) -> Option<Arc<effectcraft_raster::AuxChannels>> {
|
||||
self.layer_aux_with(ctx, layer, &self.layer_aux_need(ctx, layer))
|
||||
}
|
||||
|
||||
/// [`Renderer::layer_aux`] of what `need` reads (a footage layer's channels; a precomp's
|
||||
/// generated channels are all made anyway).
|
||||
pub fn layer_aux_with(&self, ctx: &EvalCtx, layer: &Layer, need: &effectcraft_raster::channels3d::AuxNeed) -> Option<Arc<effectcraft_raster::AuxChannels>> {
|
||||
match &layer.source {
|
||||
LayerSource::Footage { item } => {
|
||||
let item = ctx.media_item(layer, *item);
|
||||
let ItemKind::Footage(f) = &self.project.item(item)?.kind else { return None };
|
||||
// Only what the layer's effects read (#3271).
|
||||
let need = self.layer_aux_need(ctx, layer);
|
||||
let a = self.read(|| self.footage.aux_for(item, f, ctx.source_time(layer), &need))?;
|
||||
let a = self.read(|| self.footage.aux_for(item, f, ctx.source_time(layer), need))?;
|
||||
// Aux pixels per layer pixel (the layer is the footage at its nominal size). The
|
||||
// planes are only copied when that changes the scale.
|
||||
let scale = a.width as f64 / f.width as f64;
|
||||
|
||||
@@ -52,7 +52,13 @@ pub trait FootageSource: Send + Sync {
|
||||
/// [`FootageSource::aux`] of only what `need` reads (#3271): the channels its lookups find,
|
||||
/// or more (sources may hand over a set that holds them; [`effectcraft_raster::AuxChannels::listed`]
|
||||
/// then names every channel). Sources that read every channel anyway can leave the default.
|
||||
fn aux_for(&self, item: ItemId, footage: &Footage, t: Tick, need: &effectcraft_raster::channels3d::AuxNeed) -> Option<Arc<effectcraft_raster::AuxChannels>> {
|
||||
fn aux_for(
|
||||
&self,
|
||||
item: ItemId,
|
||||
footage: &Footage,
|
||||
t: Tick,
|
||||
need: &effectcraft_raster::channels3d::AuxNeed,
|
||||
) -> Option<Arc<effectcraft_raster::AuxChannels>> {
|
||||
let _ = need;
|
||||
self.aux(item, footage, t)
|
||||
}
|
||||
@@ -191,7 +197,13 @@ impl FootageSource for CancellableFootage<'_> {
|
||||
fn aux(&self, item: ItemId, footage: &Footage, t: Tick) -> Option<Arc<effectcraft_raster::AuxChannels>> {
|
||||
self.inner.aux(item, footage, t)
|
||||
}
|
||||
fn aux_for(&self, item: ItemId, footage: &Footage, t: Tick, need: &effectcraft_raster::channels3d::AuxNeed) -> Option<Arc<effectcraft_raster::AuxChannels>> {
|
||||
fn aux_for(
|
||||
&self,
|
||||
item: ItemId,
|
||||
footage: &Footage,
|
||||
t: Tick,
|
||||
need: &effectcraft_raster::channels3d::AuxNeed,
|
||||
) -> Option<Arc<effectcraft_raster::AuxChannels>> {
|
||||
self.inner.aux_for(item, footage, t, need)
|
||||
}
|
||||
fn aux_names(&self, item: ItemId, footage: &Footage, t: Tick) -> Option<Arc<effectcraft_raster::AuxChannels>> {
|
||||
|
||||
Reference in new issue
Block a user