mirror of
https://github.com/storytold/texturecraft.git
synced 2026-11-21 23:10:43 +01:00
Use fallible shared USD Crate LZ4 writing
Propagate compression errors through USD writer values, sections and finish. Bound worst-case output reservation by the existing 2GiB app ceiling; preserve normal wire bytes and scene policy. Co-authored-by: bflatastic <bombayapirat@gmail.com>
This commit is contained in:
1 parent
8003210b64
commit
939b12bfd0
6 files changed
+35
-24
No files matched your search
Generated
+1
-1
@@ -906,7 +906,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "craft-usd"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/storytold/craft-libs?rev=456abf7b7a2e6919b5a16e35fa2bf4e796f88674#456abf7b7a2e6919b5a16e35fa2bf4e796f88674"
|
||||
source = "git+https://github.com/storytold/craft-libs?rev=02ab4e01ef584345c4c5fc43d3bad2e9fdfc28bf#02ab4e01ef584345c4c5fc43d3bad2e9fdfc28bf"
|
||||
dependencies = [
|
||||
"lz4_flex",
|
||||
"thiserror 2.0.21",
|
||||
|
||||
+1
-1
@@ -12,7 +12,7 @@ rust-version = "1.95"
|
||||
repository = "https://github.com/storytold/texturecraft"
|
||||
|
||||
[workspace.dependencies]
|
||||
craft-usd = { version = "0.1.0", git = "https://github.com/storytold/craft-libs", rev = "456abf7b7a2e6919b5a16e35fa2bf4e796f88674" }
|
||||
craft-usd = { version = "0.1.0", git = "https://github.com/storytold/craft-libs", rev = "02ab4e01ef584345c4c5fc43d3bad2e9fdfc28bf" }
|
||||
# internal
|
||||
texturecraft-core = { path = "crates/core" }
|
||||
texturecraft-texel = { path = "crates/texel" }
|
||||
|
||||
@@ -154,7 +154,7 @@ impl Writer {
|
||||
UsdValue::Ints(v) if v.len() >= codec::MIN_COMPRESSED_ARRAY => {
|
||||
let at = self.offset();
|
||||
self.out.extend_from_slice(&(v.len() as u64).to_le_bytes());
|
||||
self.ints(&v.iter().map(|&i| i64::from(i)).collect::<Vec<_>>());
|
||||
self.ints(&v.iter().map(|&i| i64::from(i)).collect::<Vec<_>>())?;
|
||||
rep(ty::INT, codec::IS_ARRAY | codec::IS_COMPRESSED, at)
|
||||
}
|
||||
UsdValue::Ints(v) => self.raw_array(ty::INT, v.len(), v.iter().flat_map(|i| i.to_le_bytes())),
|
||||
@@ -174,10 +174,11 @@ impl Writer {
|
||||
}
|
||||
|
||||
/// Compressed integers: their compressed size, then the LZ4-framed coding.
|
||||
fn ints(&mut self, v: &[i64]) {
|
||||
let comp = codec::compress(&codec::encode_ints(v, 4));
|
||||
fn ints(&mut self, v: &[i64]) -> Result<()> {
|
||||
let comp = codec::compress(&codec::encode_ints(v, 4))?;
|
||||
self.out.extend_from_slice(&(comp.len() as u64).to_le_bytes());
|
||||
self.out.extend_from_slice(&comp);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn spec(&mut self, path: usize, kind: i64, fields: Vec<(&str, u64)>) {
|
||||
@@ -303,7 +304,7 @@ impl Writer {
|
||||
self.out.extend_from_slice(&(v as u64).to_le_bytes());
|
||||
}
|
||||
|
||||
fn sections(&mut self) {
|
||||
fn sections(&mut self) -> Result<()> {
|
||||
let mut toc = Vec::new();
|
||||
let start = self.offset();
|
||||
let mut raw = Vec::new();
|
||||
@@ -311,7 +312,7 @@ impl Writer {
|
||||
raw.extend_from_slice(t.as_bytes());
|
||||
raw.push(0);
|
||||
}
|
||||
let comp = codec::compress(&raw);
|
||||
let comp = codec::compress(&raw)?;
|
||||
self.u64(self.tokens.len());
|
||||
self.u64(raw.len());
|
||||
self.u64(comp.len());
|
||||
@@ -327,9 +328,9 @@ impl Writer {
|
||||
let start = self.offset();
|
||||
let fields = std::mem::take(&mut self.fields);
|
||||
self.u64(fields.len());
|
||||
self.ints(&fields.iter().map(|f| i64::from(f.0)).collect::<Vec<_>>());
|
||||
self.ints(&fields.iter().map(|f| i64::from(f.0)).collect::<Vec<_>>())?;
|
||||
let reps: Vec<u8> = fields.iter().flat_map(|f| f.1.to_le_bytes()).collect();
|
||||
let comp = codec::compress(&reps);
|
||||
let comp = codec::compress(&reps)?;
|
||||
self.u64(comp.len());
|
||||
self.out.extend_from_slice(&comp);
|
||||
self.section(&mut toc, "FIELDS", start);
|
||||
@@ -337,7 +338,7 @@ impl Writer {
|
||||
let start = self.offset();
|
||||
let sets = std::mem::take(&mut self.fieldsets);
|
||||
self.u64(sets.len());
|
||||
self.ints(&sets);
|
||||
self.ints(&sets)?;
|
||||
self.section(&mut toc, "FIELDSETS", start);
|
||||
|
||||
let start = self.offset();
|
||||
@@ -345,7 +346,7 @@ impl Writer {
|
||||
self.u64(self.paths.len());
|
||||
self.u64(self.paths.len());
|
||||
for column in &table {
|
||||
self.ints(column);
|
||||
self.ints(column)?;
|
||||
}
|
||||
self.section(&mut toc, "PATHS", start);
|
||||
|
||||
@@ -353,7 +354,7 @@ impl Writer {
|
||||
let specs = std::mem::take(&mut self.specs);
|
||||
self.u64(specs.len());
|
||||
for k in 0..3 {
|
||||
self.ints(&specs.iter().map(|s| s.get(k).copied().unwrap_or_default()).collect::<Vec<_>>());
|
||||
self.ints(&specs.iter().map(|s| s.get(k).copied().unwrap_or_default()).collect::<Vec<_>>())?;
|
||||
}
|
||||
self.section(&mut toc, "SPECS", start);
|
||||
|
||||
@@ -371,6 +372,7 @@ impl Writer {
|
||||
if let Some(slot) = self.out.get_mut(16..24) {
|
||||
slot.copy_from_slice(&toc_at.to_le_bytes());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -400,9 +402,9 @@ impl Writer {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn finish(mut self) -> Vec<u8> {
|
||||
self.sections();
|
||||
self.out
|
||||
fn finish(mut self) -> Result<Vec<u8>> {
|
||||
self.sections()?;
|
||||
Ok(self.out)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -413,7 +415,7 @@ pub(crate) fn write(layer: &UsdLayer) -> Result<Vec<u8>> {
|
||||
w.add_prim_paths(0, "/", root);
|
||||
w.root_spec(&root.name)?;
|
||||
w.prim_specs(&format!("/{}", root.name), root)?;
|
||||
Ok(w.finish())
|
||||
w.finish()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -504,7 +506,7 @@ mod tests {
|
||||
for (path, value) in attrs.into_iter().zip([points, counts, indices]) {
|
||||
w.spec(path, SPEC_ATTRIBUTE, vec![("default", value)]);
|
||||
}
|
||||
let bytes = w.finish();
|
||||
let bytes = w.finish().unwrap();
|
||||
|
||||
let m = crate::import_mesh_bytes("crafted.usdc", &bytes).unwrap().mesh;
|
||||
assert_eq!(m.triangle_count(), 1, "the class prim itself is not drawn");
|
||||
|
||||
@@ -108,10 +108,9 @@ pub(crate) fn decompress(input: &[u8], max_out: usize) -> Result<Vec<u8>> {
|
||||
}
|
||||
|
||||
/// Crate's LZ4 framing for one chunk (inputs are far below LZ4's ~2 GB chunk limit).
|
||||
pub(crate) fn compress(input: &[u8]) -> Vec<u8> {
|
||||
let mut out = vec![0u8];
|
||||
out.extend_from_slice(&lz4_flex::block::compress(input));
|
||||
out
|
||||
pub(crate) fn compress(input: &[u8]) -> Result<Vec<u8>> {
|
||||
// Keep one compressed output reservation within the app's decoded-data ceiling.
|
||||
craft_usd::compress::compress(input, super::MAX_DECODED_BYTES).map_err(|e| malformed(KIND, e.to_string()))
|
||||
}
|
||||
|
||||
/// Upper bound of the encoded size of `n` integers of `width` bytes.
|
||||
|
||||
@@ -416,10 +416,20 @@ fn integer_coding_round_trips() {
|
||||
let wide = vec![i64::MAX, i64::MIN, 0, 1 << 40];
|
||||
assert_eq!(codec::decode_ints(&codec::encode_ints(&wide, 8), 4, 8).unwrap(), wide);
|
||||
let big: Vec<u8> = (0..5000u32).flat_map(|i| (i % 7).to_le_bytes()).collect();
|
||||
assert_eq!(codec::decompress(&codec::compress(&big), big.len()).unwrap(), big);
|
||||
assert_eq!(codec::decompress(&codec::compress(&big).unwrap(), big.len()).unwrap(), big);
|
||||
assert!(codec::decompress(&[0, 0xF0], 10).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shared_crate_lz4_writer_preserves_bytes() {
|
||||
for len in [0, 1, 15, 16, 255, 4096] {
|
||||
let data: Vec<u8> = (0..len).map(|i| (i % 251) as u8).collect();
|
||||
let mut previous = vec![0];
|
||||
previous.extend(lz4_flex::block::compress(&data));
|
||||
assert_eq!(codec::compress(&data).unwrap(), previous);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shared_integer_decoder_rejects_unsupported_width_and_oversized_output() {
|
||||
assert!(codec::decode_ints(&[], 1, 3).is_err());
|
||||
|
||||
@@ -660,7 +660,7 @@ mod tests {
|
||||
}
|
||||
|
||||
fn compressed_ints(out: &mut Vec<u8>, v: &[i64]) {
|
||||
let comp = codec::compress(&codec::encode_ints(v, 4));
|
||||
let comp = codec::compress(&codec::encode_ints(v, 4)).unwrap();
|
||||
out.extend_from_slice(&(comp.len() as u64).to_le_bytes());
|
||||
out.extend_from_slice(&comp);
|
||||
}
|
||||
@@ -721,7 +721,7 @@ mod tests {
|
||||
compressed_ints(&mut b, &idx);
|
||||
let i64_at = b.len() as u64;
|
||||
b.extend_from_slice(&(n as u64).to_le_bytes());
|
||||
let comp = codec::compress(&codec::encode_ints(&(0..n as i64).map(|i| i << 33).collect::<Vec<_>>(), 8));
|
||||
let comp = codec::compress(&codec::encode_ints(&(0..n as i64).map(|i| i << 33).collect::<Vec<_>>(), 8)).unwrap();
|
||||
b.extend_from_slice(&(comp.len() as u64).to_le_bytes());
|
||||
b.extend_from_slice(&comp);
|
||||
let c = crate_over(&b, (0, 8, 0));
|
||||
|
||||
Reference in new issue
Block a user