From 54f58fff31946b8061f9fccd68c2d1e306d63ba0 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Thu, 19 Nov 2026 21:05:18 -0500 Subject: [PATCH 1/5] eps: cut over PostScript scanner (lex.rs) to craft-ps (#8793) * eps: cut over PostScript scanner (lex.rs) to craft-ps * chore: repin craft-ps to craft-libs main --- Cargo.lock | 14 +- Cargo.toml | 2 +- crates/eps/src/import/lex.rs | 304 ++++------------------------------- 3 files changed, 42 insertions(+), 278 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f6b5e507e6..c793b70f9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1048,6 +1048,11 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "craft-eps" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=7258e42d8#7258e42d8dd909f2176fb08f09aae048611f3eaf" + [[package]] name = "craft-eps" version = "0.1.0" @@ -1248,10 +1253,15 @@ dependencies = [ [[package]] name = "craft-ps" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=d7c9632bfb3040cf45db394e7500cc93c81c49e0#d7c9632bfb3040cf45db394e7500cc93c81c49e0" +source = "git+https://github.com/storytold/craft-libs?rev=7258e42d8#7258e42d8dd909f2176fb08f09aae048611f3eaf" dependencies = [ + "craft-eps 0.1.0 (git+https://github.com/storytold/craft-libs?rev=7258e42d8)", "flate2", "kurbo 0.13.1", + "miniz_oxide 0.8.9", + "pdf-writer", + "skrifa 0.47.0", + "weezl 0.1.12", ] [[package]] @@ -6866,7 +6876,7 @@ dependencies = [ name = "vectorcraft-eps" version = "0.24.1" dependencies = [ - "craft-eps", + "craft-eps 0.1.0 (git+https://github.com/storytold/craft-libs?rev=d7c9632bfb3040cf45db394e7500cc93c81c49e0)", "craft-image-codecs", "craft-pdf", "craft-ps", diff --git a/Cargo.toml b/Cargo.toml index 1a33442d15..d7d90bb1a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,7 +41,7 @@ craft-xml = { git = "https://github.com/storytold/craft-libs", rev = "d7c9632bfb craft-image-codecs = { git = "https://github.com/storytold/craft-libs", rev = "d7c9632bfb3040cf45db394e7500cc93c81c49e0" } craft-gif = { git = "https://github.com/storytold/craft-libs", rev = "d7c9632bfb3040cf45db394e7500cc93c81c49e0" } craft-tiff = { git = "https://github.com/storytold/craft-libs", rev = "d7c9632bfb3040cf45db394e7500cc93c81c49e0" } -craft-ps = { git = "https://github.com/storytold/craft-libs", rev = "d7c9632bfb3040cf45db394e7500cc93c81c49e0", default-features = false, features = ["write"] } +craft-ps = { git = "https://github.com/storytold/craft-libs", rev = "7258e42d8", default-features = false, features = ["read", "write"] } craft-psd = { git = "https://github.com/storytold/craft-libs", rev = "d7c9632bfb3040cf45db394e7500cc93c81c49e0" } craft-stream-codecs = { git = "https://github.com/storytold/craft-libs", rev = "d7c9632bfb3040cf45db394e7500cc93c81c49e0" } craft-jxl = { git = "https://github.com/storytold/craft-libs", rev = "d7c9632bfb3040cf45db394e7500cc93c81c49e0" } diff --git a/crates/eps/src/import/lex.rs b/crates/eps/src/import/lex.rs index 0ce1707f61..4e7c2b2a19 100644 --- a/crates/eps/src/import/lex.rs +++ b/crates/eps/src/import/lex.rs @@ -1,293 +1,76 @@ //! The scanner: PostScript source into objects (numbers, names, strings, procedures), with the //! position the program's own data (`currentfile`) is read from. +//! +//! Backed by `craft_ps::read::lex`. use std::rc::Rc; -use super::obj::{Obj, PsError, Res}; -use crate::ps; +use craft_ps::read::lex::{LexError, Number, Token}; -/// Deepest procedure nesting the scanner reads. -const MAX_NESTING: usize = 256; +use super::obj::{Obj, PsError, Res}; + +pub(crate) use craft_ps::read::lex::{IMMEDIATE, find, hex_decode}; pub(crate) struct Lexer<'a> { - src: &'a [u8], - pos: usize, - /// The whitespace character that ended the last token is still unread (data the program - /// reads from itself starts after it). - fresh: bool, + inner: craft_ps::read::lex::Lexer<'a>, /// The last object read has immediately evaluated names (`//name`) in it, read as executable /// names starting with [`IMMEDIATE`] for the interpreter to replace by their values. pub immediate: bool, } -/// How an immediately evaluated name (`//name`) is read, until the interpreter replaces it. -pub(crate) const IMMEDIATE: &str = "//"; - -fn is_space(b: u8) -> bool { - matches!(b, b' ' | b'\t' | b'\r' | b'\n' | b'\x0c' | b'\0') -} - -fn is_delim(b: u8) -> bool { - matches!(b, b'(' | b')' | b'<' | b'>' | b'[' | b']' | b'{' | b'}' | b'/' | b'%') -} - -fn syntax(what: &str) -> Res { - Err(PsError::Ps("syntaxerror", what.to_string())) -} - impl<'a> Lexer<'a> { pub fn new(src: &'a [u8]) -> Self { - Self { src, pos: 0, fresh: false, immediate: false } + Self { inner: craft_ps::read::lex::Lexer::new(src), immediate: false } } /// What is left to read. pub fn rest(&self) -> &'a [u8] { - self.src.get(self.pos..).unwrap_or_default() + self.inner.rest() } /// What is left to read as data: after the whitespace that ended the last token (a CR LF /// pair counts as one). pub fn data(&mut self) -> &'a [u8] { - if std::mem::take(&mut self.fresh) { - match self.peek() { - Some(b'\r') => { - self.pos += 1; - if self.peek() == Some(b'\n') { - self.pos += 1; - } - } - Some(b) if is_space(b) => self.pos += 1, - _ => {} - } - } - self.rest() + self.inner.data() } /// Skip `n` bytes (the data a filter or `readstring` took). pub fn advance(&mut self, n: usize) { - self.fresh = false; - self.pos = self.pos.saturating_add(n).min(self.src.len()); - } - - fn peek(&self) -> Option { - self.src.get(self.pos).copied() - } - - fn skip_space(&mut self) { - while let Some(b) = self.peek() { - if is_space(b) { - self.pos += 1; - } else if b == b'%' { - while let Some(c) = self.peek() { - if c == b'\n' || c == b'\r' { - break; - } - self.pos += 1; - } - } else { - break; - } - } + self.inner.advance(n); } /// The next object at the top level (`None` at the end of the file). pub fn next(&mut self) -> Res> { - self.immediate = false; - let t = self.token(0); - self.fresh = true; - t - } - - fn token(&mut self, depth: usize) -> Res> { - self.skip_space(); - let Some(b) = self.peek() else { return Ok(None) }; - self.pos += 1; - Ok(Some(match b { - b'(' => Obj::string(self.string()?), - b'<' => match self.peek() { - Some(b'<') => { - self.pos += 1; - Obj::Exec(Rc::from("<<")) - } - Some(b'~') => { - self.pos += 1; - Obj::string(self.ascii85()?) - } - _ => Obj::string(self.hex()?), - }, - b'>' => { - if self.peek() != Some(b'>') { - return syntax(">"); - } - self.pos += 1; - Obj::Exec(Rc::from(">>")) - } - b'[' => Obj::Exec(Rc::from("[")), - b']' => Obj::Exec(Rc::from("]")), - b'{' => { - if depth >= MAX_NESTING { - return Err(PsError::Limit("procedures nested too deeply")); - } - let mut items = vec![]; - loop { - self.skip_space(); - match self.peek() { - None => return syntax("{"), - Some(b'}') => { - self.pos += 1; - break; - } - _ => match self.token(depth + 1)? { - Some(o) => items.push(o), - None => return syntax("{"), - }, - } - } - Obj::proc(items) - } - b'}' | b')' => return syntax(&char::from(b).to_string()), - b'/' => { - // `//name` stands for the name's value when it is read. - let immediate = self.peek() == Some(b'/'); - if immediate { - self.pos += 1; - self.immediate = true; - } - let name = self.word(); - if immediate { Obj::Exec(Rc::from(format!("{IMMEDIATE}{name}").as_str())) } else { Obj::Name(Rc::from(name.as_str())) } - } - _ => { - self.pos -= 1; - let w = self.word(); - number(&w).unwrap_or_else(|| Obj::Exec(Rc::from(w.as_str()))) - } - })) - } - - /// A regular token's characters. - fn word(&mut self) -> String { - let start = self.pos; - while let Some(b) = self.peek() { - if is_space(b) || is_delim(b) { - break; - } - self.pos += 1; - } - String::from_utf8_lossy(self.src.get(start..self.pos).unwrap_or_default()).into_owned() - } - - /// A literal string after its `(`. - fn string(&mut self) -> Res> { - let mut out = vec![]; - let mut depth = 0usize; - loop { - let Some(b) = self.peek() else { return syntax("(") }; - self.pos += 1; - match b { - b'(' => { - depth += 1; - out.push(b); - } - b')' if depth == 0 => return Ok(out), - b')' => { - depth -= 1; - out.push(b); - } - b'\\' => { - let Some(e) = self.peek() else { return syntax("(") }; - self.pos += 1; - match e { - b'n' => out.push(b'\n'), - b'r' => out.push(b'\r'), - b't' => out.push(b'\t'), - b'b' => out.push(8), - b'f' => out.push(12), - b'\r' => { - if self.peek() == Some(b'\n') { - self.pos += 1; - } - } - b'\n' => {} - b'0'..=b'7' => { - let mut v = u32::from(e - b'0'); - for _ in 0..2 { - match self.peek() { - Some(d @ b'0'..=b'7') => { - v = v * 8 + u32::from(d - b'0'); - self.pos += 1; - } - _ => break, - } - } - out.push((v & 0xff) as u8); - } - other => out.push(other), - } - } - _ => out.push(b), - } - } - } - - /// A hexadecimal string after its `<`. - fn hex(&mut self) -> Res> { - let end = self.rest().iter().position(|b| *b == b'>').ok_or(PsError::Ps("syntaxerror", "<".into()))?; - let body = self.rest().get(..end).unwrap_or_default(); - let out = hex_decode(body).ok_or(PsError::Ps("syntaxerror", "<".into()))?; - self.advance(end + 1); - Ok(out) - } - - /// An ASCII85 string after its `<~`. - fn ascii85(&mut self) -> Res> { - let end = find(self.rest(), b"~>").ok_or(PsError::Ps("syntaxerror", "<~".into()))?; - let text = String::from_utf8_lossy(self.rest().get(..end + 2).unwrap_or_default()).into_owned(); - self.advance(end + 2); - ps::ascii85_decode(&text).ok_or(PsError::Ps("syntaxerror", "<~".into())) + let res = self.inner.next().map(|opt| opt.map(into_obj)).map_err(into_err); + self.immediate = self.inner.immediate; + res } } -/// Where `needle` first starts in `hay`. -pub(crate) fn find(hay: &[u8], needle: &[u8]) -> Option { - if needle.is_empty() { - return Some(0); +fn into_err(err: LexError) -> PsError { + match err { + LexError::Syntax(what) => PsError::Ps("syntaxerror", what), + LexError::Limit(msg) => PsError::Limit(msg), } - memchr::memmem::find(hay, needle) } -/// Hexadecimal digits (whitespace ignored, an odd last digit padded with 0); `None` on others. -pub(crate) fn hex_decode(body: &[u8]) -> Option> { - // Callers pass an already-delimited body. Do not let the filter decoder's EOD - // handling silently accept a stray terminator or discard bytes after it. - if body.contains(&b'>') { - return None; +fn into_obj(tok: Token) -> Obj { + match tok { + Token::Int(i) => Obj::Int(i), + Token::Real(r) => Obj::Real(r), + Token::Name(n) => Obj::Name(Rc::from(n.as_str())), + Token::Exec(n) => Obj::Exec(Rc::from(n.as_str())), + Token::Str(s) => Obj::string(s), + Token::Proc(items) => Obj::proc(items.into_iter().map(into_obj).collect()), } - craft_stream_codecs::asciihex::decode(body, body.len().div_ceil(2)).ok() } /// A number token (`12`, `-3.5`, `.5`, `1e-3`, `16#ff`), or `None`. pub(crate) fn number(w: &str) -> Option { - let first = w.bytes().next()?; - if !(first.is_ascii_digit() || matches!(first, b'+' | b'-' | b'.')) { - return None; + match craft_ps::read::lex::number(w)? { + Number::Int(i) => Some(Obj::Int(i)), + Number::Real(r) => Some(Obj::Real(r)), } - if let Ok(i) = w.parse::() { - return Some(Obj::Int(i)); - } - if let Some((base, digits)) = w.split_once('#') { - let base: u32 = base.parse().ok().filter(|b| (2..=36).contains(b))?; - // `from_str_radix` accepts a leading sign; a PostScript radix number has digits only. - if digits.starts_with(['+', '-']) { - return None; - } - return i64::from_str_radix(digits, base).ok().map(Obj::Int); - } - // Rust reads `inf` and `nan`; PostScript doesn't. - if w.bytes().any(|b| b.is_ascii_alphabetic() && !matches!(b, b'e' | b'E')) { - return None; - } - let v: f64 = w.parse().ok()?; - v.is_finite().then_some(Obj::Real(v)) } #[cfg(test)] @@ -300,33 +83,4 @@ mod tests { assert!(number("16#-ff").is_none()); assert!(matches!(number("16#ff"), Some(Obj::Int(255)))); } - #[test] - fn shared_hex_decode_keeps_body_boundaries_and_odd_nibbles() { - assert_eq!(hex_decode(b" 4\t1\n4\r2\x0c4\0"), Some(vec![0x41, 0x42, 0x40])); - assert_eq!(hex_decode(b""), Some(vec![])); - for invalid in [b"41>42".as_slice(), b">", b"4g", b"4\x0b1", b"\xff"] { - assert_eq!(hex_decode(invalid), None); - } - // Every byte pairs with each possible nibble, including all whitespace - // and invalid-byte combinations; this is the former body-only contract. - for first in 0..=255u8 { - for second in 0..=255u8 { - let input = [first, second]; - let mut digits = Vec::new(); - let mut valid = true; - for b in input { - if is_space(b) { - continue; - } - match char::from(b).to_digit(16) { - Some(d) => digits.push(d as u8), - None => valid = false, - } - } - let expected = - if !valid { None } else { Some(digits.chunks(2).map(|pair| (pair[0] << 4) | pair.get(1).copied().unwrap_or(0)).collect()) }; - assert_eq!(hex_decode(&input), expected, "{input:?}"); - } - } - } } From 2848526796e197a27866e9c9d24ac773bffd3583 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 20 Nov 2026 02:32:03 -0500 Subject: [PATCH 2/5] fix: resolve two coverage test failures on main (#8813) - eps: restore linear-time memchr::memmem::find in lex::find so search_in_a_loop_runs_out_of_the_budget does not hit O(N*M) under craft_ps::read::lex::find. - ui-egui: wait for FontDb background scan and ensure >20 font families in menu_access_stationary_pointer_repeat_and_rerun_do_not_retarget_or_double_invoke so coverage/minimal runners do not race or fall short of 20 families. --- crates/eps/src/import/lex.rs | 10 +++++++++- crates/ui-egui/tests/menu_access_keys.rs | 9 +++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/crates/eps/src/import/lex.rs b/crates/eps/src/import/lex.rs index 4e7c2b2a19..b967f52676 100644 --- a/crates/eps/src/import/lex.rs +++ b/crates/eps/src/import/lex.rs @@ -9,7 +9,15 @@ use craft_ps::read::lex::{LexError, Number, Token}; use super::obj::{Obj, PsError, Res}; -pub(crate) use craft_ps::read::lex::{IMMEDIATE, find, hex_decode}; +pub(crate) use craft_ps::read::lex::{IMMEDIATE, hex_decode}; + +/// Where `needle` first starts in `hay` (uses `memmem` so large inputs stay linear-time). +pub(crate) fn find(hay: &[u8], needle: &[u8]) -> Option { + if needle.is_empty() { + return Some(0); + } + memchr::memmem::find(hay, needle) +} pub(crate) struct Lexer<'a> { inner: craft_ps::read::lex::Lexer<'a>, diff --git a/crates/ui-egui/tests/menu_access_keys.rs b/crates/ui-egui/tests/menu_access_keys.rs index 2389716d61..2d21b77d88 100644 --- a/crates/ui-egui/tests/menu_access_keys.rs +++ b/crates/ui-egui/tests/menu_access_keys.rs @@ -764,6 +764,15 @@ fn menu_access_stationary_pointer_repeat_and_rerun_do_not_retarget_or_double_inv assert_eq!(s.app.ui.view.rulers, !before, "one Text/Key pair invokes exactly once across passes"); s.end_access(); + // Wait for any background font scan and ensure > 20 families even on minimal runners. + let db = vectorcraft_text::FontDb::global(); + let _ = db.families(); + for i in 0..24 { + if db.menu_family_list_now().len() > 20 { + break; + } + db.add_font(vectorcraft_testkit::fonts::renamed(&format!("Menu Test Font {i:02}"))); + } let text = s.app.run("text.create", json!({"x": 250, "y": 200, "text": "Menu", "font": "Inter", "size": 12})).unwrap()["id"].clone(); s.app.run("select.set", json!({"ids": [text]})).unwrap(); let size = vec2(640.0, 420.0); From f987648e96b0238df46c6ec788de64863ff4f744 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 20 Nov 2026 03:01:58 -0500 Subject: [PATCH 3/5] eps: delegate scene-writer shading, image stream, and line style syntax to craft-ps (#8810) * eps: delegate scene-writer shading, image stream, and line style syntax to craft-ps * chore(eps): repin craft-ps to merged craft-libs main (cfcc86242) --- Cargo.lock | 6 ++-- Cargo.toml | 2 +- crates/eps/src/ai_ops.rs | 28 +++++---------- crates/eps/src/ps.rs | 31 ++++++++++++++++- crates/eps/src/scene.rs | 75 +++++++--------------------------------- 5 files changed, 54 insertions(+), 88 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c793b70f9d..7da3db5266 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1051,7 +1051,7 @@ dependencies = [ [[package]] name = "craft-eps" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=7258e42d8#7258e42d8dd909f2176fb08f09aae048611f3eaf" +source = "git+https://github.com/storytold/craft-libs?rev=cfcc86242d56fe68117cd5dc2e5dd3b7a0e5b0ec#cfcc86242d56fe68117cd5dc2e5dd3b7a0e5b0ec" [[package]] name = "craft-eps" @@ -1253,9 +1253,9 @@ dependencies = [ [[package]] name = "craft-ps" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=7258e42d8#7258e42d8dd909f2176fb08f09aae048611f3eaf" +source = "git+https://github.com/storytold/craft-libs?rev=cfcc86242d56fe68117cd5dc2e5dd3b7a0e5b0ec#cfcc86242d56fe68117cd5dc2e5dd3b7a0e5b0ec" dependencies = [ - "craft-eps 0.1.0 (git+https://github.com/storytold/craft-libs?rev=7258e42d8)", + "craft-eps 0.1.0 (git+https://github.com/storytold/craft-libs?rev=cfcc86242d56fe68117cd5dc2e5dd3b7a0e5b0ec)", "flate2", "kurbo 0.13.1", "miniz_oxide 0.8.9", diff --git a/Cargo.toml b/Cargo.toml index d7d90bb1a5..0279fa081a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,7 +41,7 @@ craft-xml = { git = "https://github.com/storytold/craft-libs", rev = "d7c9632bfb craft-image-codecs = { git = "https://github.com/storytold/craft-libs", rev = "d7c9632bfb3040cf45db394e7500cc93c81c49e0" } craft-gif = { git = "https://github.com/storytold/craft-libs", rev = "d7c9632bfb3040cf45db394e7500cc93c81c49e0" } craft-tiff = { git = "https://github.com/storytold/craft-libs", rev = "d7c9632bfb3040cf45db394e7500cc93c81c49e0" } -craft-ps = { git = "https://github.com/storytold/craft-libs", rev = "7258e42d8", default-features = false, features = ["read", "write"] } +craft-ps = { git = "https://github.com/storytold/craft-libs", rev = "cfcc86242d56fe68117cd5dc2e5dd3b7a0e5b0ec", default-features = false, features = ["read", "write"] } craft-psd = { git = "https://github.com/storytold/craft-libs", rev = "d7c9632bfb3040cf45db394e7500cc93c81c49e0" } craft-stream-codecs = { git = "https://github.com/storytold/craft-libs", rev = "d7c9632bfb3040cf45db394e7500cc93c81c49e0" } craft-jxl = { git = "https://github.com/storytold/craft-libs", rev = "d7c9632bfb3040cf45db394e7500cc93c81c49e0" } diff --git a/crates/eps/src/ai_ops.rs b/crates/eps/src/ai_ops.rs index d74932e74a..9d0ff5ddc0 100644 --- a/crates/eps/src/ai_ops.rs +++ b/crates/eps/src/ai_ops.rs @@ -3,8 +3,6 @@ //! Format Specification, version 7.0, §5.3, §5.4 and §6), and the colors they write in place of //! the paint they can't hold. -use std::fmt::Write; - use vectorcraft_color::{Color, Gradient}; use vectorcraft_doc::{LAYER_COLORS, LayerColor, LineCap, LineJoin, StrokeLayer}; use vectorcraft_geom::{Affine, Anchor, AnchorKind, PathData, Point, Shape, SubPath}; @@ -184,25 +182,15 @@ pub(crate) fn window_samples(samples: &[u8], w: u32, channels: usize, [x0, y0, x /// The stroke's width, cap, join, miter limit and dashes: `w J j M d`. pub(crate) fn line_style(s: &mut String, st: &StrokeLayer) { let cap = match st.cap { - LineCap::Butt => 0, - LineCap::Round => 1, - LineCap::Square => 2, + LineCap::Butt => ps::LineCap::Butt, + LineCap::Round => ps::LineCap::Round, + LineCap::Square => ps::LineCap::Square, }; let join = match st.join { - LineJoin::Miter => 0, - LineJoin::Round => 1, - LineJoin::Bevel => 2, + LineJoin::Miter => ps::LineJoin::Miter, + LineJoin::Round => ps::LineJoin::Round, + LineJoin::Bevel => ps::LineJoin::Bevel, }; - ps::push_num(s, st.width); - let _ = write!(s, " w {cap} J {join} j "); - ps::push_num(s, if st.miter_limit.is_finite() { st.miter_limit.max(1.0) } else { 10.0 }); - s.push_str(" M ["); - let dash = st.dash.as_ref().filter(|d| d.is_dashed()); - if let Some(d) = dash { - ps::push_nums(s, &d.pattern); - s.pop(); - } - s.push_str("] "); - ps::push_num(s, dash.map_or(0.0, |d| d.offset)); - s.push_str(" d\n"); + let dash = st.dash.as_ref().filter(|d| d.is_dashed()).map(|d| (d.pattern.as_slice(), d.offset)); + ps::push_line_style(s, st.width, cap, join, st.miter_limit, dash); } diff --git a/crates/eps/src/ps.rs b/crates/eps/src/ps.rs index f43b08fcb7..a2b075df67 100644 --- a/crates/eps/src/ps.rs +++ b/crates/eps/src/ps.rs @@ -3,7 +3,10 @@ use vectorcraft_geom::{BezPath, FillRule}; -pub(crate) use craft_ps::write::{bytes, deflate, inflate, matrix, num, push_num, push_num_places, push_nums, string, written}; +pub(crate) use craft_ps::write::{ + LineCap, LineJoin, ShadingKind, bytes, deflate, inflate, matrix, num, push_image_array_ref, push_inline_image, push_line_style, push_num, + push_num_places, push_nums, push_reusable_image_ref, push_reusable_image_resource, push_shading, string, written, +}; /// The procedures the page uses, in `VCdict`. pub(crate) const PROLOG: &str = "/VCdict 64 dict def @@ -121,4 +124,30 @@ mod tests { assert_eq!(places(2.5, 7), "2.5"); assert_eq!(places(f64::NAN, 7), "0"); } + + #[test] + fn shared_scene_syntax_preserves_vc_names_and_shading() { + let mut line = String::new(); + super::push_line_style(&mut line, 3.0, super::LineCap::Square, super::LineJoin::Round, 4.0, Some((&[5.0, 1.5], 0.5))); + assert_eq!(line, "3 w 2 J 1 j 4 M [5 1.5] 0.5 d\n"); + + let mut shading = String::new(); + super::push_shading( + &mut shading, + super::ShadingKind::Axial, + "/DeviceCMYK", + &[0.0, 0.0, 100.0, 0.0], + 4, + &[(0.0, vec![0.0, 1.0, 1.0, 0.0]), (1.0, vec![1.0, 0.0, 1.0, 0.0])], + ); + assert!(shading.contains("/ShadingType 2 /ColorSpace /DeviceCMYK")); + assert!(shading.ends_with(">> shfill\n")); + + let mut inline = String::new(); + super::push_inline_image(&mut inline, "VCsrc", "", "<< /ImageType 1", "/FlateDecode", "image", "~>\n"); + assert_eq!( + inline, + "{ /VCsrc currentfile /ASCII85Decode filter def\n<< /ImageType 1\n/DataSource VCsrc /FlateDecode filter >> image\nVCsrc flushfile } exec\n~>\n" + ); + } } diff --git a/crates/eps/src/scene.rs b/crates/eps/src/scene.rs index 349bc2a16b..be26ba8c4f 100644 --- a/crates/eps/src/scene.rs +++ b/crates/eps/src/scene.rs @@ -50,8 +50,6 @@ const SPREAD_CAPPED: &str = "gradients that repeat or reflect are written with at most 256 runs of their stops (at most 16384 stops); they pad past those"; /// Wedges a conic gradient is written as: one a degree. const CONIC_WEDGE_COUNT: usize = 360; -/// Decimals of a smooth shading's stitching function bounds (stop offsets, 0 to 1). -const BOUNDS_PLACES: u32 = 7; /// The most colorants a DeviceN color space may have (PostScript's limit). const MAX_DEVICE_N: usize = 32; const MISSING_PATTERN: &str = "missing patterns are written as mid-grey"; @@ -63,8 +61,6 @@ const MAX_BODY: usize = 256 << 20; /// Most compressed bytes in one string of an image's samples at Level 2 (under PostScript's /// 65,535-byte string limit). const IMAGE_CHUNK: usize = 48 << 10; -/// The start of the procedure that reads an image's data from the lines after it. -const INLINE_SOURCE: &str = "{ /VCsrc currentfile /ASCII85Decode filter def\n"; /// An image as written: what sets up its color, its image dictionary and its compressed samples. /// Placed once, it is written where it is drawn, its data following the operators that read it. @@ -108,33 +104,20 @@ impl ImageCode { /// The operators and data that draw the image where it is placed. fn push_inline(&self, s: &mut String) { - s.push_str(&self.prelude); - s.push_str(INLINE_SOURCE); - s.push_str(&self.dict); - s.push_str(&self.inline_read()); - s.push_str(&self.ascii); - } - - /// The end of the procedure [`ImageCode::push_inline`] writes. The procedure is read whole - /// before it runs: the image data follows it, and flushing the ASCII85 filter afterwards reads - /// past its end marker. - fn inline_read(&self) -> String { - format!("\n/DataSource VCsrc {} filter >> {}\nVCsrc flushfile }} exec\n", self.filter, self.op) + ps::push_inline_image(s, "VCsrc", &self.prelude, &self.dict, self.filter, self.op, &self.ascii); } /// The operators that draw the image `VCim` holds, for an image placed more than once. fn reference(&self, i: usize, level: Level) -> String { + let mut s = String::new(); + let name = format!("VCim{i}"); match level { // The stream is read again from its start. - Level::Three => { - format!("{}VCim{i} resetfile\n{}\n/DataSource VCim{i} {} filter >> {}\n", self.prelude, self.dict, self.filter, self.op) - } + Level::Three => ps::push_reusable_image_ref(&mut s, &name, &self.prelude, &self.dict, self.filter, self.op), // A procedure gives the strings in turn, then an empty string: the end of the data. - Level::Two => format!( - "{}/VCi 0 def\n{}\n/DataSource {{VCi VCim{i} length lt {{VCim{i} VCi get /VCi VCi 1 add def}} {{()}} ifelse}} {} filter >> {}\n", - self.prelude, self.dict, self.filter, self.op - ), + Level::Two => ps::push_image_array_ref(&mut s, "VCi", &name, &self.prelude, &self.dict, self.filter, self.op), } + s } /// The setup's definition of `VCim`: the image's compressed samples, once. @@ -145,11 +128,7 @@ impl ImageCode { // The stream reads all of its data when it is made, so the data follows the // procedure that makes it. let intent = if self.op == "imagemask" { 1 } else { 0 }; - let _ = writeln!( - s, - "{{ /VCsrc currentfile /ASCII85Decode filter def\n/VCim{i} VCsrc << /Intent {intent} /AsyncRead false /CloseSource false >> /ReusableStreamDecode filter def\nVCsrc flushfile }} exec" - ); - s.push_str(&self.ascii); + ps::push_reusable_image_resource(&mut s, "VCsrc", &format!("VCim{i}"), intent, &self.ascii); } Level::Two => { let _ = writeln!(s, "/VCim{i} ["); @@ -1489,44 +1468,12 @@ impl<'a> Scene<'a> { /// A Level 3 smooth shading of `kind` (2 axial, 3 radial) at `coords`, extended both ways. fn shading(&mut self, kind: u8, space: &GradientSpace, coords: &[f64], stops: &[(f64, Vec)]) { - let n = space.n(); - let comps = |c: &[f32]| -> String { - let mut s = String::from("["); - push_nums(&mut s, &c.iter().take(n).map(|v| f64::from(*v)).collect::>()); - s.pop(); - s.push(']'); - s - }; - let segments: Vec<_> = stops.windows(2).filter(|w| w[1].0 > w[0].0).collect(); - let interp = |w: &&[(f64, Vec)]| format!("<< /FunctionType 2 /Domain [0 1] /C0 {} /C1 {} /N 1 >>", comps(&w[0].1), comps(&w[1].1)); - let function = match segments.as_slice() { - [one] => interp(one), - many => { - let mut f = String::from("<< /FunctionType 3 /Domain [0 1] /Functions [\n"); - for w in many { - f.push_str(&interp(w)); - f.push('\n'); - } - f.push_str("] /Bounds ["); - // Seven decimals: a gradient's stops repeated hundreds or thousands of times - // (`Gradient::padded_over`) keep their places in each run. - for w in many.iter().skip(1) { - ps::push_num_places(&mut f, w[0].0, BOUNDS_PLACES); - f.push(' '); - } - f.push_str("] /Encode ["); - f.push_str(&"0 1 ".repeat(many.len())); - f.push_str("] >>"); - f - } - }; + let kind = if kind == 3 { ps::ShadingKind::Radial } else { ps::ShadingKind::Axial }; let name = match space { GradientSpace::Process(s) => s.name(), GradientSpace::Inks(name, _) => name, }; - let _ = write!(self.body, "<< /ShadingType {kind} /ColorSpace {name} /Coords ["); - push_nums(&mut self.body, coords); - let _ = writeln!(self.body, "] /Extend [true true]\n/Function {function} >> shfill"); + ps::push_shading(&mut self.body, kind, name, coords, space.n(), stops); } /// The bands of each stretch between stops: how many, for colours `a` to `b` over `length` @@ -2090,7 +2037,9 @@ impl<'a> Scene<'a> { /// `data`, not yet placed. fn image_code(prelude: String, dict: String, filter: &'static str, op: &'static str, data: Vec) -> ImageCode { let mut code = ImageCode { prelude, dict, filter, op, ascii: ps::ascii85(&data), data, uses: 0, inline_len: 0, ref_len: 0 }; - code.inline_len = code.prelude.len() + INLINE_SOURCE.len() + code.dict.len() + code.inline_read().len() + code.ascii.len(); + let mut inline = String::new(); + ps::push_inline_image(&mut inline, "VCsrc", &code.prelude, &code.dict, code.filter, code.op, ""); + code.inline_len = inline.len() + code.ascii.len(); code } From 49be5fb8754b15894e5c60b30dbdf619215e1e67 Mon Sep 17 00:00:00 2001 From: dotCooCoo Date: Thu, 19 Nov 2026 23:59:06 -0800 Subject: [PATCH 4/5] CI: the gate builds and runs the tests in five shards The gate's one job builds all 106 test binaries on a 4-CPU, 16 GB runner, and it is often stopped with exit code 143 ("The runner has received a shutdown signal") during that build. ci.yml now runs the gate as five legs: - Shards 1 to 4 run `cargo xtask ci shard`, which builds and runs the tests and doctests of its own crates: ui-egui, engine, the apps with mcp and xtask, and every other crate. - Shard 5 runs the other `cargo xtask ci` steps. - On pushes to main, a `cache warm` leg runs `cargo xtask ci warm`, which builds what every shard builds and saves the dependency cache. `gate (ubuntu-24.04)` passes only when all five legs passed. `cargo xtask ci` with no argument runs every step, as before. `cargo xtask shards` lists each shard's crates. It fails when a crate named in xtask/src/shards.rs is not a workspace member or is named twice, and when a crate builds with other features in its shard than in `cargo test --workspace`. Every shard runs it first. vectorcraft-affinity and vectorcraft-pdf now turn on `synth` and `read-count` in their own dev-dependencies, which vectorcraft-engine's dev-dependencies turned on before. Without that, 39 tests of affinity's `it` binary do not build in a shard without the engine. The gate also sets dev-debug "0" and dev-strip: true, so the workspace crates build without line tables and the test binaries without debug info or symbols. --- .github/workflows/ci.yml | 11 ++ Cargo.lock | 2 + crates/affinity/Cargo.toml | 3 + crates/pdf/Cargo.toml | 3 + xtask/src/main.rs | 188 ++++++++++++++++++++---- xtask/src/shards.rs | 294 +++++++++++++++++++++++++++++++++++++ 6 files changed, 471 insertions(+), 30 deletions(-) create mode 100644 xtask/src/shards.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7b3b6bf5b8..773a82e145 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,17 @@ jobs: # Same craft-fonts commit as release.yml. craft-fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83 timeout-minutes: 90 + # The workspace crates build without line tables, and the test binaries without debug info or + # symbols. A panic still names its file and line. + dev-debug: "0" + dev-strip: true + # Five legs instead of one: shards 1 to 4 each build and run the tests of their own crates, + # and shard 5 runs the other `cargo xtask ci` steps (xtask/src/shards.rs). `gate ()` + # passes only when all five did. On pushes to main, a `cache warm` leg builds what every + # shard builds and saves the dependency cache. + test-shards: 5 + shard-command: cargo xtask ci shard + warm-command: cargo xtask ci warm packages: >- pkg-config libxkbcommon-dev libwayland-dev libx11-dev libxcb1-dev libxcursor-dev libxrandr-dev libxi-dev libfontconfig1-dev libfreetype-dev mesa-vulkan-drivers diff --git a/Cargo.lock b/Cargo.lock index c793b70f9d..358891a86f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6683,6 +6683,7 @@ dependencies = [ "craft-image-codecs", "crc32fast", "proptest", + "vectorcraft-affinity", ] [[package]] @@ -7044,6 +7045,7 @@ dependencies = [ "vectorcraft-doc", "vectorcraft-effects", "vectorcraft-geom 0.24.1", + "vectorcraft-pdf", "vectorcraft-testkit", "vectorcraft-text", ] diff --git a/crates/affinity/Cargo.toml b/crates/affinity/Cargo.toml index 8e3bce12a9..92977884cf 100644 --- a/crates/affinity/Cargo.toml +++ b/crates/affinity/Cargo.toml @@ -21,3 +21,6 @@ proptest = { workspace = true } # The tests build containers and decode previews. crc32fast = { workspace = true } craft-image-codecs = { workspace = true } +# tests/it's `import_limits`, `outline_limits` and `text` modules need `synth`, also when this crate +# is tested without vectorcraft-engine (`cargo xtask ci shard`). +vectorcraft-affinity = { path = ".", features = ["synth"] } diff --git a/crates/pdf/Cargo.toml b/crates/pdf/Cargo.toml index 69bcf84f7f..f63a23a2ce 100644 --- a/crates/pdf/Cargo.toml +++ b/crates/pdf/Cargo.toml @@ -56,6 +56,9 @@ serde_json = { workspace = true } krilla = { workspace = true, features = ["simple-text"] } # How many times the symbol list is read to find symbols by name. vectorcraft-doc = { workspace = true, features = ["symbol-read-count"] } +# The counters vectorcraft-engine's tests turn on, also when this crate is tested without +# vectorcraft-engine (`cargo xtask ci shard`). +vectorcraft-pdf = { path = ".", features = ["read-count"] } # An independent raster oracle for gradient placement in PDF forms. These are test-only macOS # bindings; the exporter stays portable Rust and does not depend on an OS PDF renderer. diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 2c279e36aa..3c2a94c14d 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -13,6 +13,7 @@ mod layers; mod pdf_deps; mod pngsuite; mod sha256; +mod shards; mod stats; mod test262; mod ui_commands; @@ -37,6 +38,11 @@ commands: wasm cargo check --target wasm32-unknown-unknown for the wasm-safe crates and the web app test the workspace tests (cargo nextest when installed, else cargo test), then the doctests ci fmt --check, clippy -D warnings, test, doctests, no-bundled-fonts, pdf-deps, assets, brands, cursors (warnings only), ui-commands --check, layers, wasm (stops at first failure) + ci shard shard $GATE_SHARD of $GATE_SHARDS of ci (xtask/src/shards.rs): a test shard runs + the tests and doctests of its crates, the last shard the other steps + ci warm build what every shard builds, without running it (fills the gate's cache) + shards list the crates each shard tests; fails when a member is in no shard or in two, + or builds with other features in its shard than in the whole workspace corpus [--download] [--affinity] show where test corpora live; --download fetches PngSuite into corpus/pngsuite, --affinity the pinned public Affinity documents into corpus/affinity @@ -66,7 +72,10 @@ fn main() -> ExitCode { Some("pdf-deps") => pdf_deps::run(&root()), Some("wasm") => cmd_wasm(), Some("test") => cmd_test().and_then(|()| cmd_doctests()), + Some("ci") if rest.first() == Some(&"shard") => cmd_ci_shard(), + Some("ci") if rest.first() == Some(&"warm") => cmd_ci_warm(), Some("ci") => cmd_ci(), + Some("shards") => shards::run().map(drop), Some("corpus") if rest.contains(&"--affinity") => affinity_corpus::fetch(), Some("corpus") => cmd_corpus(rest.contains(&"--download")), Some("bundle") => bundle::run(&root()), @@ -246,30 +255,162 @@ fn has_nextest() -> bool { /// The workspace's unit and integration tests, without examples (clippy still checks those). /// nextest runs each test in its own process, in parallel across all test binaries. fn cmd_test() -> Result<(), String> { + test_packages(&["--workspace".to_owned()], "--workspace") +} + +/// [`cmd_test`] for the packages `select` picks (`--workspace` or `-p` ids), shown as `shown`. +fn test_packages(select: &[String], shown: &str) -> Result<(), String> { const TARGETS: [&str; 3] = ["--lib", "--bins", "--tests"]; let mut c = cargo(); if has_nextest() { - c.args(["nextest", "run", "--workspace"]).args(TARGETS); - run(c, "cargo nextest run --workspace --lib --bins --tests") + c.args(["nextest", "run"]).args(select).args(TARGETS); + run(c, &format!("cargo nextest run {shown} --lib --bins --tests")) } else { eprintln!("note: cargo-nextest is not installed; running cargo test (cargo install cargo-nextest for faster runs)"); - c.args(["test", "--workspace"]).args(TARGETS); - run(c, "cargo test --workspace --lib --bins --tests") + c.arg("test").args(select).args(TARGETS); + run(c, &format!("cargo test {shown} --lib --bins --tests")) } } /// The doctests, which nextest doesn't run. fn cmd_doctests() -> Result<(), String> { - let mut c = cargo(); - c.args(["test", "--workspace", "--doc"]); - run(c, "cargo test --workspace --doc") + doctest_packages(&["--workspace".to_owned()], "--workspace") } +/// [`cmd_doctests`] for the packages `select` picks, shown as `shown`. +fn doctest_packages(select: &[String], shown: &str) -> Result<(), String> { + let mut c = cargo(); + c.arg("test").args(select).arg("--doc"); + run(c, &format!("cargo test {shown} --doc")) +} + +/// One `cargo xtask ci` step: its name and what it runs. +type Step = (&'static str, Box Result<(), String>>); + +/// The `ci` steps that build and run tests of the whole workspace; `ci shard` runs them per shard. +const TEST_STEPS: [&str; 3] = ["test", "doctests", "i18n-report"]; + /// `cargo xtask ci`: each step in turn, stopping at the first that fails; the summary names the /// steps that passed and, after a failure, the one that failed and those skipped. fn cmd_ci() -> Result<(), String> { - type Step = (&'static str, Box Result<(), String>>); - let steps: Vec = vec![ + run_steps(&ci_steps()) +} + +/// `cargo xtask ci shard`: shard `GATE_SHARD` of `GATE_SHARDS` (1-based, as the shared gate +/// workflow sets them). A test shard runs the shard checks, then the tests and doctests of its +/// crates, and the i18n report where it tests vectorcraft-ui-egui. The checks shard runs every +/// other `ci` step and the shard checks. +fn cmd_ci_shard() -> Result<(), String> { + let number = |key: &str| -> Result { + let value = std::env::var(key).map_err(|_| format!("ci shard: {key} is not set (the gate workflow sets it)"))?; + value.trim().parse().map_err(|_| format!("ci shard: {key} is `{value}`, not a number")) + }; + let (shard, count) = (number("GATE_SHARD")?, number("GATE_SHARDS")?); + if count != shards::GROUPS.len() { + return Err(format!( + "ci shard: GATE_SHARDS is {count}, and xtask/src/shards.rs has {} shards (test-shards in .github/workflows/ci.yml)", + shards::GROUPS.len() + )); + } + if !(1..=count).contains(&shard) { + return Err(format!("ci shard: GATE_SHARD is {shard}, not 1 to {count}")); + } + let checked = shards::run()?; + let mut steps: Vec = Vec::new(); + match &checked[shard - 1] { + shards::Shard::Tests(crates) => { + let names = shards::names(crates); + eprintln!("ci shard {shard}/{count}: the tests of {names}"); + let select = shards::select(crates); + let shown = crates.iter().map(|m| format!("-p {}", m.name)).collect::>().join(" "); + let (s, w) = (select.clone(), shown.clone()); + steps.push(("test", Box::new(move || test_packages(&s, &w)))); + let (s, w) = (select.clone(), shown.clone()); + steps.push(("doctests", Box::new(move || doctest_packages(&s, &w)))); + if crates.iter().any(|m| m.name == "vectorcraft-ui-egui") { + steps.push(("i18n-report", Box::new(move || i18n_report_packages(&select, &shown)))); + } + } + shards::Shard::Checks => { + eprintln!("ci shard {shard}/{count}: the ci steps other than the tests"); + steps.extend(ci_steps().into_iter().filter(|(name, _)| !TEST_STEPS.contains(name))); + } + } + run_steps(&steps) +} + +/// `cargo xtask ci warm`: for each test shard, builds its crates' library and binary tests without +/// running them, then builds what the checks shard builds, so one dependency-cache entry holds the +/// dependencies of every shard with the features that shard builds them with (cargo resolves +/// features over the packages each command selects). The integration tests are left out: they +/// build no dependency that the library tests do not, since both take the crate's dev-dependencies. +fn cmd_ci_warm() -> Result<(), String> { + let checked = shards::run()?; + let mut steps: Vec = Vec::new(); + for shard in checked { + if let shards::Shard::Tests(crates) = shard { + steps.push(( + "build tests", + Box::new(move || { + let mut c = cargo(); + c.arg("test").args(shards::select(&crates)).args(["--lib", "--bins", "--no-run"]); + run(c, &format!("cargo test --lib --bins --no-run ({})", shards::names(&crates))) + }), + )); + } + } + steps.push(( + "clippy", + Box::new(|| { + let mut c = cargo(); + c.args(["clippy", "--workspace", "--all-targets", "--keep-going"]); + run(c, "cargo clippy --workspace --all-targets --keep-going") + }), + )); + steps.push(( + "no-bundled-fonts", + Box::new(|| { + let mut c = cargo(); + c.args(["test", "-p", "vectorcraft-text", "--no-default-features", "--test", "without_bundled_fonts", "--no-run"]); + run(c, "cargo test -p vectorcraft-text --no-default-features --test without_bundled_fonts --no-run")?; + let mut c = cargo(); + c.args(["clippy", "-p", "vectorcraft-ui-egui", "--no-default-features", "--all-targets"]); + run(c, "cargo clippy -p vectorcraft-ui-egui --no-default-features --all-targets")?; + let mut c = cargo(); + c.args(["test", "-p", "vectorcraft-ui-egui", "--no-default-features", "--test", "install_fonts", "--no-run"]); + run(c, "cargo test -p vectorcraft-ui-egui --no-default-features --test install_fonts --no-run") + }), + )); + steps.push(("wasm", Box::new(cmd_wasm))); + run_steps(&steps) +} + +/// Runs `steps` in turn, stopping at the first that fails; the summary names the steps that passed +/// and, after a failure, the one that failed and those skipped. +fn run_steps(steps: &[Step]) -> Result<(), String> { + let mut done = Vec::new(); + for (name, f) in steps { + eprintln!("\n=== ci: {name} ==="); + if let Err(e) = f() { + println!("\nCI summary:"); + for d in &done { + println!(" ok {d}"); + } + println!(" FAIL {name}: {e}"); + for (n, _) in steps.iter().skip(done.len() + 1) { + println!(" skip {n}"); + } + return Err(format!("ci failed at `{name}`")); + } + done.push(*name); + } + println!("\nCI summary: all {} steps passed ({})", done.len(), done.join(", ")); + Ok(()) +} + +/// Every `cargo xtask ci` step, in order. +fn ci_steps() -> Vec { + vec![ ( "fmt", Box::new(|| { @@ -316,25 +457,7 @@ fn cmd_ci() -> Result<(), String> { ("ui-commands", Box::new(|| ui_commands::run(&root(), true))), ("layers", Box::new(cmd_layers)), ("wasm", Box::new(cmd_wasm)), - ]; - let mut done = Vec::new(); - for (name, f) in &steps { - eprintln!("\n=== ci: {name} ==="); - if let Err(e) = f() { - println!("\nCI summary:"); - for d in &done { - println!(" ok {d}"); - } - println!(" FAIL {name}: {e}"); - for (n, _) in steps.iter().skip(done.len() + 1) { - println!(" skip {n}"); - } - return Err(format!("ci failed at `{name}`")); - } - done.push(*name); - } - println!("\nCI summary: all {} steps passed ({})", done.len(), done.join(", ")); - Ok(()) + ] } fn cmd_corpus(download: bool) -> Result<(), String> { @@ -379,7 +502,12 @@ fn cmd_i18n_report() -> Result<(), String> { /// differently and recompiled for about 3.5 minutes. Every other test binary runs with no test /// matching the filter. fn cmd_ci_i18n_report() -> Result<(), String> { + i18n_report_packages(&["--workspace".to_owned()], "--workspace") +} + +/// [`cmd_ci_i18n_report`] for the packages `select` picks (a `ci shard`'s), shown as `shown`. +fn i18n_report_packages(select: &[String], shown: &str) -> Result<(), String> { let mut c = cargo(); - c.args(["test", "--workspace", "--lib", "--bins", "--tests", "--", "i18n::tests::untranslated_report", "--exact", "--nocapture"]); - run(c, "cargo test --workspace --lib --bins --tests -- i18n::tests::untranslated_report --exact --nocapture") + c.arg("test").args(select).args(["--lib", "--bins", "--tests", "--", "i18n::tests::untranslated_report", "--exact", "--nocapture"]); + run(c, &format!("cargo test {shown} --lib --bins --tests -- i18n::tests::untranslated_report --exact --nocapture")) } diff --git a/xtask/src/shards.rs b/xtask/src/shards.rs new file mode 100644 index 0000000000..a885619b15 --- /dev/null +++ b/xtask/src/shards.rs @@ -0,0 +1,294 @@ +//! The gate's shards (`cargo xtask ci shard`, with `GATE_SHARD` of `GATE_SHARDS` set by the shared +//! gate workflow): which workspace crates each shard tests, and the checks that keep the split +//! complete. +//! +//! Each test shard builds and runs the tests of its own crates and what they depend on. The last +//! shard runs the `cargo xtask ci` steps that are not tests. [`GROUPS`] names the crates of the +//! first shards; every other workspace member goes to the [`Group::Rest`] shard, so a new crate is +//! tested without a change here. [`check`] fails when a named crate is not a workspace member or +//! is named twice, when a group is empty, and when a crate's features in its shard differ from +//! its features in `cargo test --workspace`. Cargo resolves features over the packages a command +//! selects, so a feature that only another crate's dev-dependencies turn on is off in a shard +//! without that crate, and the tests behind it do not build. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::Value; + +/// What one shard runs. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Group { + /// The tests of these workspace crates (package names). + Crates(&'static [&'static str]), + /// The tests of every workspace member no other group names. + Rest, + /// The `cargo xtask ci` steps that are not tests. + Checks, +} + +/// The shards in order; `test-shards` in `.github/workflows/ci.yml` is its length. +pub const GROUPS: &[Group] = &[ + Group::Crates(&["vectorcraft-ui-egui"]), + Group::Crates(&["vectorcraft-engine"]), + Group::Crates(&["vectorcraft", "vectorcraft-web", "vectorcraft-cli", "vectorcraft-mcp", "xtask"]), + Group::Rest, + Group::Checks, +]; + +/// A workspace member, as `cargo metadata` gives it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Member { + pub name: String, + /// The package id, which `-p` takes where the name would be ambiguous: craft-libs builds + /// against this repo's geom and pathops by git, so the graph holds a second copy of each. + pub id: String, + pub version: String, +} + +/// What one shard runs, with its crates resolved. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Shard { + Tests(Vec), + Checks, +} + +/// The workspace members of `cargo metadata --no-deps` output, in its order. +pub fn members(meta: &Value) -> Result, String> { + let ids: BTreeSet<&str> = meta["workspace_members"].as_array().into_iter().flatten().filter_map(Value::as_str).collect(); + let members: Vec = meta["packages"] + .as_array() + .into_iter() + .flatten() + .filter(|p| p["id"].as_str().is_some_and(|id| ids.contains(id))) + .map(|p| { + let field = |key: &str| p[key].as_str().map(str::to_owned).ok_or_else(|| format!("cargo metadata: a workspace package has no `{key}`")); + Ok(Member { name: field("name")?, id: field("id")?, version: field("version")? }) + }) + .collect::>()?; + if members.is_empty() { + return Err("cargo metadata lists no workspace members".into()); + } + Ok(members) +} + +/// Each group of `groups` with its crates: a named crate must be a workspace member and named +/// once, there must be exactly one [`Group::Rest`], and no test shard may be empty. +pub fn assign(groups: &[Group], members: &[Member]) -> Result, String> { + const WHERE: &str = "xtask/src/shards.rs › GROUPS"; + let mut named: BTreeMap<&str, usize> = BTreeMap::new(); + for (i, g) in groups.iter().enumerate() { + if let Group::Crates(names) = g { + if names.is_empty() { + return Err(format!("shard {} names no crates ({WHERE})", i + 1)); + } + for name in *names { + if !members.iter().any(|m| m.name == *name) { + return Err(format!("shard {} names `{name}`, which is not a workspace member ({WHERE})", i + 1)); + } + if let Some(first) = named.insert(name, i) { + return Err(format!("`{name}` is in shard {} and shard {} ({WHERE})", first + 1, i + 1)); + } + } + } + } + let rests = groups.iter().filter(|g| **g == Group::Rest).count(); + if rests != 1 { + return Err(format!("{rests} shards take the remaining crates, not 1 ({WHERE})")); + } + let pick = |f: &dyn Fn(&Member) -> bool| members.iter().filter(|m| f(m)).cloned().collect::>(); + groups + .iter() + .enumerate() + .map(|(i, g)| match g { + Group::Crates(names) => Ok(Shard::Tests(pick(&|m| names.contains(&m.name.as_str())))), + Group::Rest => { + let rest = pick(&|m| !named.contains_key(m.name.as_str())); + if rest.is_empty() { + return Err(format!("shard {} takes the remaining crates, and there are none ({WHERE})", i + 1)); + } + Ok(Shard::Tests(rest)) + } + Group::Checks => Ok(Shard::Checks), + }) + .collect() +} + +/// The features `cargo tree -e normal,dev,build --prefix none -f '{p}|{f}'` shows for each of +/// `members`, by name; a crate the tree holds twice (once for the host, say) has two entries. +/// Other copies of a member's name (craft-libs' git copies of geom and pathops) are not members. +pub fn tree_features(tree: &str, members: &[Member]) -> BTreeMap> { + let mut found: BTreeMap> = BTreeMap::new(); + for line in tree.lines() { + let line = line.trim_end(); + let line = line.strip_suffix(" (*)").unwrap_or(line); + let Some((package, features)) = line.split_once('|') else { continue }; + let Some(m) = + members.iter().find(|m| package.strip_prefix(&format!("{} v{} (", m.name, m.version)).is_some_and(|source| !source.contains("://"))) + else { + continue; + }; + let mut features: Vec<&str> = features.split(',').map(str::trim).filter(|f| !f.is_empty()).collect(); + features.sort_unstable(); + found.entry(m.name.clone()).or_default().insert(features.join(",")); + } + found +} + +/// One line for each crate of `shard` whose features in `alone` (the shard's tree) differ from +/// `workspace` (the whole workspace's tree). +pub fn feature_differences( + shard: usize, + crates: &[Member], + workspace: &BTreeMap>, + alone: &BTreeMap>, +) -> Vec { + let show = |set: Option<&BTreeSet>| match set { + None => "absent".to_owned(), + Some(set) => set.iter().map(|f| format!("[{f}]")).collect::>().join(" "), + }; + crates + .iter() + .filter(|m| workspace.get(&m.name) != alone.get(&m.name)) + .map(|m| { + format!("{}: features {} in shard {shard}, {} with the whole workspace", m.name, show(alone.get(&m.name)), show(workspace.get(&m.name))) + }) + .collect() +} + +/// `cargo tree` with the features of each package, for `select` (`--workspace` or `-p` ids). +fn tree(select: &[String]) -> Result { + let mut c = crate::cargo(); + // Without `--color never`, CARGO_TERM_COLOR=always (as the gate sets it) colors the ` (*)` marks. + c.args(["tree", "--color", "never", "-e", "normal,dev,build", "--prefix", "none", "-f", "{p}|{f}"]).args(select); + let out = c.output().map_err(|e| format!("cargo tree: failed to spawn: {e}"))?; + if !out.status.success() { + return Err(format!("cargo tree {}: exited with {}:\n{}", select.join(" "), out.status, String::from_utf8_lossy(&out.stderr))); + } + Ok(String::from_utf8_lossy(&out.stdout).into_owned()) +} + +/// `-p ` for each crate. +pub fn select(crates: &[Member]) -> Vec { + crates.iter().flat_map(|m| ["-p".to_owned(), m.id.clone()]).collect() +} + +/// The package names of `crates`, for messages. +pub fn names(crates: &[Member]) -> String { + crates.iter().map(|m| m.name.as_str()).collect::>().join(", ") +} + +/// The shards of this workspace, after the checks in [`assign`] and [`feature_differences`]. +pub fn check() -> Result, String> { + let members = members(&crate::metadata()?)?; + let shards = assign(GROUPS, &members)?; + let workspace = tree_features(&tree(&["--workspace".to_owned()])?, &members); + let mut differences = Vec::new(); + for (i, shard) in shards.iter().enumerate() { + if let Shard::Tests(crates) = shard { + let alone = tree_features(&tree(&select(crates))?, &members); + differences.extend(feature_differences(i + 1, crates, &workspace, &alone)); + } + } + if !differences.is_empty() { + return Err(format!( + "{} crate(s) build with other features in their shard than in `cargo test --workspace`, so some of their tests \ + would build differently or not at all (turn the feature on in the crate's own dev-dependencies, or move the crate \ + to the shard of the crate that turns it on):\n {}", + differences.len(), + differences.join("\n ") + )); + } + Ok(shards) +} + +/// `cargo xtask shards`: prints each shard's crates after [`check`], and returns the shards. +pub fn run() -> Result, String> { + let shards = check()?; + for (i, shard) in shards.iter().enumerate() { + match shard { + Shard::Tests(crates) => println!("shard {}/{}: tests of {} crate(s): {}", i + 1, shards.len(), crates.len(), names(crates)), + Shard::Checks => println!("shard {}/{}: the other ci steps", i + 1, shards.len()), + } + } + println!("OK: every workspace member is tested on one shard, with the features `cargo test --workspace` gives it."); + Ok(shards) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn m(name: &str) -> Member { + Member { name: name.into(), id: format!("path+file:///w/{name}#0.1.0"), version: "0.1.0".into() } + } + + fn crates(shard: &Shard) -> Vec<&str> { + match shard { + Shard::Tests(c) => c.iter().map(|m| m.name.as_str()).collect(), + Shard::Checks => Vec::new(), + } + } + + #[test] + fn the_rest_shard_takes_every_crate_no_group_names() { + let ws = [m("a"), m("b"), m("c"), m("d")]; + let shards = assign(&[Group::Crates(&["c"]), Group::Rest, Group::Checks], &ws).unwrap(); + assert_eq!(shards.iter().map(crates).collect::>(), [vec!["c"], vec!["a", "b", "d"], vec![]]); + assert_eq!(shards[2], Shard::Checks); + } + + #[test] + fn a_crate_named_twice_or_unknown_fails() { + let ws = [m("a"), m("b")]; + let twice = assign(&[Group::Crates(&["a"]), Group::Crates(&["a"]), Group::Rest], &ws).unwrap_err(); + assert!(twice.contains("`a` is in shard 1 and shard 2"), "{twice}"); + let unknown = assign(&[Group::Crates(&["z"]), Group::Rest], &ws).unwrap_err(); + assert!(unknown.contains("`z`, which is not a workspace member"), "{unknown}"); + } + + #[test] + fn empty_groups_and_a_missing_or_second_rest_fail() { + let ws = [m("a"), m("b")]; + assert!(assign(&[Group::Crates(&[]), Group::Rest], &ws).unwrap_err().contains("names no crates")); + assert!(assign(&[Group::Crates(&["a", "b"]), Group::Rest], &ws).unwrap_err().contains("there are none")); + assert!(assign(&[Group::Crates(&["a"])], &ws).unwrap_err().contains("0 shards take the remaining crates")); + assert!(assign(&[Group::Rest, Group::Rest], &ws).unwrap_err().contains("2 shards take the remaining crates")); + } + + #[test] + fn every_workspace_member_is_tested_on_exactly_one_shard() { + let ws = members(&crate::metadata().unwrap()).unwrap(); + let shards = assign(GROUPS, &ws).unwrap(); + let mut tested: Vec<&str> = shards.iter().flat_map(crates).collect(); + tested.sort_unstable(); + let mut all: Vec<&str> = ws.iter().map(|m| m.name.as_str()).collect(); + all.sort_unstable(); + assert_eq!(tested, all); + assert_eq!(shards.iter().filter(|s| **s == Shard::Checks).count(), 1); + } + + #[test] + fn a_feature_only_another_shard_turns_on_is_a_difference() { + let ws = [m("lib"), m("geom")]; + let workspace = "\ +lib v0.1.0 (/w/lib)|default,synth +geom v0.1.0 (/w/geom)| +geom v0.18.0 (https://github.com/x/y?rev=1#1)|old +lib v0.1.0 (/w/lib)|default,synth (*) +"; + let alone = "\ +lib v0.1.0 (/w/lib)|default +geom v0.1.0 (/w/geom)| +"; + let workspace = tree_features(workspace, &ws); + assert_eq!(workspace["lib"], BTreeSet::from(["default,synth".to_owned()])); + assert_eq!(workspace["geom"], BTreeSet::from([String::new()])); + let alone = tree_features(alone, &ws); + assert_eq!(feature_differences(4, &ws, &workspace, &alone), ["lib: features [default] in shard 4, [default,synth] with the whole workspace"]); + assert!(feature_differences(4, &ws, &workspace, &workspace).is_empty()); + assert_eq!( + feature_differences(4, &[m("lib")], &workspace, &BTreeMap::new()), + ["lib: features absent in shard 4, [default,synth] with the whole workspace"] + ); + } +} From 291ea9fd00b8de32dd87d6fc3aa2de03e61993f1 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Fri, 20 Nov 2026 19:41:48 +0700 Subject: [PATCH 5/5] ci: build the gate's test binaries without debug info (#8829) Every gate since 20 Nov 00:38 UTC except one has been stopped mid-link with "The runner has received a shutdown signal" (exit 143), on main and on pull requests. Turn on the shared gate's dev-strip input, as photocraft does, so the ~100 test binaries link without line tables or symbols. Co-authored-by: FredApps Co-authored-by: Claude Opus 5.5 --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7b3b6bf5b8..0c38980b72 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,12 @@ jobs: # Same craft-fonts commit as release.yml. craft-fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83 timeout-minutes: 90 + # The gate's test binaries carry no debug info and no symbols (CARGO_PROFILE_TEST_DEBUG=0, + # CARGO_PROFILE_TEST_STRIP=symbols), as photocraft's do: linking ~100 test binaries with line + # tables ran the hosted runner out of resources, and it stopped mid-link with "received a + # shutdown signal" (exit 143). Panics still name file and line; local builds keep Cargo.toml's + # line tables. + dev-strip: true packages: >- pkg-config libxkbcommon-dev libwayland-dev libx11-dev libxcb1-dev libxcursor-dev libxrandr-dev libxi-dev libfontconfig1-dev libfreetype-dev mesa-vulkan-drivers