From 27bb1ebf065a58fc415ac9eb05c04a834d3cbb38 Mon Sep 17 00:00:00 2001 From: totoz64 <47389794+totoz64@users.noreply.github.com> Date: Tue, 3 Nov 2026 11:19:25 +0100 Subject: [PATCH 001/473] CP: Adapt French proofing to shared Language packs Keep French locales in Pack(PackLocale), reusing the shared dictionary cache, affix reader and suggestion engine. Add French elisions, bounded accent and split tuning, conservative typography and Quebec punctuation rules. Validate displayed suggestion casing before OCONV and retain the original eligible form when an explicit target is excluded. Preserve historical text fixtures with three documented pre-existing upstream chain differences. Ported from WordCraft #3185; no dictionary or hyphenation data is bundled. --- crates/proof/README.md | 41 ++ crates/proof/src/affix.rs | 66 ++ crates/proof/src/affix_suggest.rs | 30 + crates/proof/src/fr/grammar.rs | 665 ++++++++++++++++++ crates/proof/src/fr/mod.rs | 181 +++++ crates/proof/src/fr/suggest.rs | 81 +++ crates/proof/src/fr/test_dict.rs | 234 ++++++ crates/proof/src/hunspell.rs | 17 +- crates/proof/src/lib.rs | 5 + crates/proof/src/pack_lang.rs | 8 +- crates/proof/tests/affix_compatibility.rs | 92 +++ .../fixtures/affix_compatibility.compact | 7 + .../fixtures/affix_compatibility.expected | 497 +++++++++++++ .../fixtures/affix_compatibility_cases.rs | 61 ++ crates/proof/tests/french_dictionary.rs | 107 +++ crates/proof/tests/french_packs.rs | 124 ++++ 16 files changed, 2210 insertions(+), 6 deletions(-) create mode 100644 crates/proof/src/fr/grammar.rs create mode 100644 crates/proof/src/fr/mod.rs create mode 100644 crates/proof/src/fr/suggest.rs create mode 100644 crates/proof/src/fr/test_dict.rs create mode 100644 crates/proof/tests/affix_compatibility.rs create mode 100644 crates/proof/tests/fixtures/affix_compatibility.compact create mode 100644 crates/proof/tests/fixtures/affix_compatibility.expected create mode 100644 crates/proof/tests/fixtures/affix_compatibility_cases.rs create mode 100644 crates/proof/tests/french_dictionary.rs create mode 100644 crates/proof/tests/french_packs.rs diff --git a/crates/proof/README.md b/crates/proof/README.md index 970670143..b4b4b5ba8 100644 --- a/crates/proof/README.md +++ b/crates/proof/README.md @@ -16,6 +16,7 @@ craft-proof = { git = "https://github.com/storytold/craft-libs", rev = " | English (en-US, en-CA, en-AU, en-GB and the British-spelling Englishes such as en-NZ) | SCOWL 2026 Hunspell dictionary per English + `en-extra.txt`, dialect policy (`en-variants.txt`); a dictionaries-folder English first; without SCOWL the installed Hunspell English, else the built-in Moby list ([below](#english-is-always-checked)) | yes | Moby list, our patterns, heuristic | | Polish, Portuguese, Italian | the installed Hunspell dictionary | yes | TeX patterns (embedded, or a Language pack) | | Croatian, Turkish, Russian, Czech, Indonesian, Spanish (es, es-MX, es-419, es-UY) | the installed Hunspell dictionary (Russian also `ru-extra.txt`) | yes | Turkish and Russian rules; English fallback otherwise | +| French (fr-FR, fr-CA and other French pack locales) | the installed Language pack or Hunspell dictionary; dictionary-driven elisions and accent-oriented suggestions | conservative typography and basic checks, with Quebec punctuation spacing in fr-CA | the Language pack (generic fr fallback) | | German | — | — | TeX patterns (embedded, or a Language pack) | | Bulgarian | — | — | syllable rules | @@ -41,6 +42,12 @@ application data (`%LOCALAPPDATA%\WordCraft\dictionaries` and `%APPDATA%\WordCra `~/Library/Application Support/WordCraft/dictionaries`, `$XDG_DATA_HOME/wordcraft/dictionaries`), then the usual Hunspell and LibreOffice locations. None on the web. +French remains a `ProofLang::Pack` locale: use its full `tag()` for settings and issue identity. +The shared pack loader keeps regional dictionary overrides, fallback to `fr`, reloads and its +memory budget; no separate French dictionary slot or embedded French patterns are added. +Only French packs receive the French spelling tuning and typography; other pack languages +keep their existing behavior. French agreement and conjugation are outside these checks. + State is process-wide: `add_word`, `ignore_word`, `set_dictionaries_folder`, `load_user_dictionary`/`save_user_dictionary`. The crate reads no document model; callers pass plain text and language spans. @@ -221,6 +228,40 @@ CI: `.github/workflows/proof.yml`. storytold/craft-libs#30/#31 by brzuchal (Michał Marcin Brzuchalski) started the shared crate. +### French adaptation validation + +The French rules and original synthetic fixtures are adapted from +[totoz64’s WordCraft contribution #3185](https://github.com/storytold/wordcraft/pull/3185). +They use the shared affix reader and Language pack cache; WCAFF1 remains deliberately rejected. +The historical text fixtures retain their original expected data. Three observations for +Indonesian `bermainkanlah` are explicitly updated in the compatibility test because the +shared reader already supports that prefix plus two suffixes. This is a finite regression +check, not a universal compatibility claim. + +The ordinary tests cover French packs, regional overrides and reloads, options, user words, +NFC/NFD lookup with original ranges, suggestions and pack hyphenation. The opt-in real French +dictionary gate fails when its input is missing; it never silently skips its assertions: + +```sh +WORDCRAFT_DICTIONARIES=/absolute/isolated/french-dictionary cargo test -p craft-proof --test french_dictionary -- --ignored +``` + +For a reproducible real-dictionary run, use Dicollecte 7.7 from LibreOffice/dictionaries +revision `67032ee957631908098a72edf48dc81cac27e369`: +[fr.aff](https://raw.githubusercontent.com/LibreOffice/dictionaries/67032ee957631908098a72edf48dc81cac27e369/fr_FR/dictionaries/fr.aff) and +[fr.dic](https://raw.githubusercontent.com/LibreOffice/dictionaries/67032ee957631908098a72edf48dc81cac27e369/fr_FR/dictionaries/fr.dic). +Verify SHA-256 before invoking the gate (the test checks location and behavior, not hashes): + +| File | SHA-256 | +|---|---| +| fr.aff | `c176610cd5dc4846806a65ddd029f422d87978bf58f224aa44222662a16a2de5` | +| fr.dic | `b78a868e31dd6e373b6c3217969afb898a9acde828a5e7ef97308da42218c88c` | + +Use a dedicated directory containing only the intended `fr.aff`/`fr.dic` pair (or explicit +regional pairs for both regions). The gate requires more than 50,000 dictionary entries and +checks the actual source folder, spelling, accent/elision suggestions, regional dispatch, +combining-mark ranges and hyphenated compounds. No real dictionary is committed here. + ## Owners WordCraft maintainers: echelon, dylanmtaylor, aiongg. diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 0df68665b..32de07659 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -2212,6 +2212,37 @@ impl AffixDict { self.check_except(w, self.excluded(true)) } + /// Validate a collected suggestion in its displayed case, before OCONV. Only identity, + /// first-letter and uppercase variants are allowed (including uppercase that keeps sharp s). + /// An exact target excluded from suggestions cannot be rescued by a lowercase fallback. + /// This preserves the checker's homonym, compound and CHECKSHARPS rules without scanning + /// stems or changing which words the spelling checker accepts. + pub(crate) fn can_suggest_cased(&self, original: &str, changed: &str) -> bool { + if original.len() > MAX_INPUT_BYTES || changed.len() > MAX_INPUT_BYTES { + return false; + } + if changed != original + && changed != capitalize(original) + && changed != original.to_uppercase() + && changed != original.split('ß').map(str::to_uppercase).collect::>().join("ß") + { + return false; + } + let target = self.normalize(changed); + if self.is_forbidden(&target) { + return false; + } + // The exact simple-word path includes affixes; asking with no exclusions detects a + // target whose only derivations carry NOSUGGEST, SUBSTANDARD or FORBIDDENWORD. Spend + // the same shared suggestion budget as the normal checker, and abstain if it runs out. + match self.simple_word_budgeted(&target, Excluded::default(), true) { + None => return false, + Some(true) if !self.simple_word(&target, self.excluded(true)) => return false, + _ => {} + } + self.can_suggest(changed) + } + /// Whether `w` is a word only through a `KEEPCASE` stem, so it is correct in this letter case /// alone (`km` from `km/K`, which `Km` and `KM` are not). pub(crate) fn case_locked(&self, w: &str) -> bool { @@ -3298,6 +3329,41 @@ pub(crate) mod tests { assert!(d.check("ASSASSASSASSASS") && !d.check(&"SS".repeat(60))); } + #[test] + fn displayed_casing_rejects_exact_exclusions_before_case_fallback() { + for flag in ["N", "F", "S"] { + let aff = "SET UTF-8\nNOSUGGEST N\nFORBIDDENWORD F\nSUBSTANDARD S\nSFX A Y 1\nSFX A 0 s .\nSFX B Y 1\nSFX B 0 S .\n"; + let dic = format!("5\nnormal/A\nNORMAL/{flag}B\nmixCase\npour\nkm/K\n"); + let d = AffixDict::from_hunspell(&format!("{aff}KEEPCASE K\n"), &dic, &[]).unwrap(); + for (original, changed) in [("normal", "NORMAL"), ("normals", "NORMALS")] { + assert!(!d.can_suggest_cased(changed, changed), "{flag}: {changed}"); + assert!(!d.can_suggest_cased(original, changed), "{flag}: {original} → {changed}"); + } + assert!(d.can_suggest_cased("normal", "Normal")); + assert!(d.can_suggest_cased("mixCase", "MIXCASE")); + assert!(d.can_suggest_cased("km", "km") && !d.can_suggest_cased("km", "KM")); + assert!(!d.can_suggest_cased("normal", "pour"), "no arbitrary replacement"); + let back = AffixDict::from_bytes(&d.to_bytes().unwrap()).unwrap(); + assert!(!back.can_suggest_cased("normal", "NORMAL")); + } + } + + #[test] + fn displayed_casing_uses_normalization_and_native_homonym_policy() { + let aff = "SET UTF-8\nNOSUGGEST N\nSUBSTANDARD S\nIGNORE ()\nICONV 1\nICONV Q NORMAL\n"; + for flag in ["N", "S"] { + let d = AffixDict::from_hunspell(aff, &format!("3\nnormal\nNORMAL/{flag}\nNORMAL\n"), &[]).unwrap(); + // Native homonyms carry exclusions: a second entry does not erase NOSUGGEST. + assert!(!d.contains_suggestible("NORMAL")); + assert!(!d.can_suggest_cased("(normal)", "(NORMAL)")); + assert!(!d.can_suggest_cased("Q", "Q")); + } + let plain = AffixDict::from_hunspell(aff, "3\nnormal\nNORMAL\nNORMAL\n", &[]).unwrap(); + assert!(plain.can_suggest_cased("normal", "NORMAL")); + assert!(plain.can_suggest_cased("Q", "Q")); + assert!(!plain.can_suggest_cased(&"x".repeat(MAX_INPUT_BYTES + 1), "NORMAL")); + } + #[test] fn oconv_converts_suggestions_only() { // Answers as Hunspell 1.7 gives them on the same files: `kbt` → `k’t`. diff --git a/crates/proof/src/affix_suggest.rs b/crates/proof/src/affix_suggest.rs index 3b498542c..68336de16 100644 --- a/crates/proof/src/affix_suggest.rs +++ b/crates/proof/src/affix_suggest.rs @@ -426,6 +426,18 @@ pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> V _ => vec![text.clone()], }; for t in recased { + let eligible = |candidate: &str| { + let original: Vec<_> = text.split(' ').collect(); + let changed: Vec<_> = candidate.split(' ').collect(); + original.len() == changed.len() && original.iter().zip(changed).all(|(a, b)| d.can_suggest_cased(a, b)) + }; + let t = if eligible(&t) { + t + } else if eligible(&text) { + text.clone() + } else { + continue; + }; // Shown as the dictionary's OCONV table writes it (`'` → `’` in French). let t = d.output(&t).into_owned(); if t != word && !out.contains(&t) { @@ -763,6 +775,24 @@ mod tests { assert_eq!(suggest_in(&d, "Piesx", 4, &Tuning::DEFAULT).first().map(String::as_str), Some("Pies")); } + #[test] + fn recasing_keeps_valid_original_when_exact_target_is_excluded() { + for flag in ["N", "F", "S"] { + let aff = "SET UTF-8\nNOSUGGEST N\nFORBIDDENWORD F\nSUBSTANDARD S\nTRY normal\n"; + let d = AffixDict::from_hunspell(aff, &format!("2\nnormal\nNORMAL/{flag}\n"), &[]).unwrap(); + let got = suggest_in(&d, "NORMALL", 4, &Tuning::DEFAULT); + assert!(got.iter().any(|w| w == "normal"), "{flag}: {got:?}"); + assert!(!got.iter().any(|w| w == "NORMAL"), "{flag}: {got:?}"); + let split = AffixDict::from_hunspell(aff, &format!("3\nnormal\nNORMAL/{flag}\npour\n"), &[]).unwrap(); + let got = suggest_in(&split, "NORMALPOUR", 4, &Tuning::DEFAULT); + assert!(got.iter().any(|w| w == "normal pour"), "{flag}: {got:?}"); + assert!(!got.iter().any(|w| w == "NORMAL POUR"), "{flag}: {got:?}"); + } + let d = AffixDict::from_hunspell("NOSUGGEST N\n", "3\nnormal\nNORMAL/N\npour\n", &[]).unwrap(); + assert!(!d.can_suggest_cased("normal", "NORMAL")); + assert!(!d.can_suggest_cased("normal", "pour")); + } + /// A language's tables come after its dictionary's, a TRY alphabet stands in for a missing TRY /// line, and a split rule decides which missing spaces are offered. #[test] diff --git a/crates/proof/src/fr/grammar.rs b/crates/proof/src/fr/grammar.rs new file mode 100644 index 000000000..1d5449b1f --- /dev/null +++ b/crates/proof/src/fr/grammar.rs @@ -0,0 +1,665 @@ +//! French grammar and typography checks: cheap, rule-based and tuned to stay quiet when unsure +//! (a missed mistake costs less than a wrong mark on correct text). Messages are in French. +//! +//! - **Space before `;` `!` `?` `:`**: French typography puts a non-breaking space there — a +//! narrow one (U+202F) before `;` `!` `?`, a full one (U+00A0) before `:`. Only a missing +//! space is flagged (`Bonjour!`): an ordinary one stays quiet until typing turns it into a +//! non-breaking one (no French AutoFormat yet). Quebec French (`fr-CA`) has no space before +//! `;` `!` `?`, only before `:`. Addresses (`https://…`) and times (`10:30`) are left alone. +//! - **Space inside `« »`**: a missing one after `«` or before `»` (`«oui»`) → U+00A0. +//! - **Straight double quotes** around a phrase (`"bonjour"`) → `« bonjour »`; inch marks +//! (`27"`) and unbalanced quotes are left alone. +//! - **Repeated word** (`le le`, `que que`), except repetitions French makes on purpose +//! (`nous nous lavons`, `vous vous trompez`, `si si`, interjections) and a proper noun after +//! the same lower-case word (`de De…`). +//! - **Capital letter at the start of a sentence** after `.` `!` `?`, knowing French +//! abbreviations (`etc.`, `M.`, `p.`, `cf.`), initials, numbers, ellipses, addresses, +//! dialogue (`« Tu viens ? » demanda-t-elle`), inversions (`Quoi ? dit-il`) and exclamations +//! after an interjection (`Ah ! que c'est beau`). +//! - **Elision** (`de avoir` → `d’avoir`, `je ai` → `j’ai`, `le homme` → `l’homme`) for `le`, +//! `la`, `de`, `je`, `me`, `te`, `se`, `ne`, `que` before a vowel. Before an `h` only for words +//! known to have a mute `h`, so aspirated ones (`le hibou`, `la hache`, `le héros`) stay quiet; +//! never before a proper noun, `onze`, `oui`, `y…` words (`le yaourt`) or `la une`. +//! +//! No agreement or conjugation rule, and no English rule (such as `a`/`an`), applies to French +//! text. + +use crate::{Issue, IssueKind, ProofLang, words}; + +/// Non-breaking space, before `:` and inside `« »`. +const NBSP: char = '\u{A0}'; +/// Narrow non-breaking space, before `;` `!` `?`. +const NNBSP: char = '\u{202F}'; + +/// Words French repeats on purpose: pronominal verbs (`nous nous lavons`), emphasis, +/// interjections, onomatopoeia. +const ALLOWED_REPEATS: &[&str] = &[ + "nous", "vous", "si", "oui", "non", "très", "bon", "vite", "là", "ah", "oh", "eh", "ha", "hé", "ho", "hi", "bla", "chut", "toc", "pan", "bravo", + "merci", "allez", "encore", "beaucoup", "fort", "tchin", "bye", "couci", "tam", "miam", "glou", +]; + +/// Abbreviations written with a final period after which a sentence goes on in lower case +/// (lower case, without the period). +const ABBREVIATIONS: &[&str] = &[ + "etc", "cf", "ex", "env", "av", "apr", "mm", "mme", "mmes", "mlle", "mlles", "mgr", "dr", "pr", "me", "st", "ste", "sts", "pp", "vol", "chap", + "fig", "éd", "coll", "trad", "réf", "tél", "art", "al", "op", "cit", "ibid", "id", "resp", "min", "max", "sq", "sqq", "suiv", "hab", "janv", + "févr", "avr", "juil", "sept", "oct", "nov", "déc", "bd", "boul", "cie", "éts", "vve", "nb", "ps", "no", "nos", "vs", "qqch", "qqn", "ca", + "approx", "prof", "dir", "sté", "dépt", "adj", "adv", "fém", "masc", "sing", "plur", "à-d", "ms", "mss", "éds", "trim", "hôp", +]; + +/// Words after which `!` or `?` doesn't end the sentence (`Ah ! que c'est beau`, `Eh bien ? +/// et alors`). +const INTERJECTIONS: &[&str] = &[ + "ah", "oh", "eh", "hé", "hein", "hélas", "bah", "ô", "aïe", "ouf", "oups", "quoi", "comment", "ha", "ho", "bon", "tiens", "allons", "zut", + "chut", "bravo", "hourra", "dame", "bien", "là", "oui", "non", "pardon", "voyons", +]; + +/// Pronouns ending an inversion (`dit-il`, `demanda-t-elle`): after one, a `!` or `?` doesn't +/// end the sentence (`Quoi ? dit-il`). +const INVERTED: &[&str] = &["-il", "-elle", "-on", "-ils", "-elles", "-je", "-tu", "-nous", "-vous"]; + +/// Speech verbs of a dialogue tag: after one, a `!` or `?` doesn't end the sentence (`— Tu +/// viens ? demanda le voisin`). +const SPEECH_VERBS: &[&str] = &[ + "dit", + "demanda", + "répondit", + "cria", + "murmura", + "ajouta", + "reprit", + "fit", + "lança", + "souffla", + "répliqua", + "interrogea", + "hurla", + "poursuivit", + "continua", + "déclara", + "expliqua", + "protesta", + "soupira", + "chuchota", + "insista", + "s'écria", + "s’écria", + "s'exclama", + "s’exclama", + "objecta", + "conclut", + "dis", + "dit-on", +]; + +/// The words that elide before a vowel, and their elided form. +const ELIDING: &[(&str, &str)] = + &[("le", "l"), ("la", "l"), ("de", "d"), ("je", "j"), ("me", "m"), ("te", "t"), ("se", "s"), ("ne", "n"), ("que", "qu")]; + +/// Words starting with a vowel that don't take an elision (`le onze`, `le oui`, `de ouate`). +const NO_ELISION: &[&str] = &["onze", "onzième", "onzièmes", "oui", "ouistiti", "ouistitis", "ouate", "ouates", "uhlan", "uhlans"]; + +/// Words with a mute `h`, which take an elision (`l’homme`, `d’habitude`, `j’habite`). Only these +/// are flagged: an `h` word outside the list may be aspirated (`le hibou`, `la hache`, `le +/// héros`), and staying quiet is the safe side. +const MUTE_H: &[&str] = &[ + "homme", + "hommes", + "heure", + "heures", + "histoire", + "histoires", + "hiver", + "hivers", + "hôtel", + "hôtels", + "herbe", + "herbes", + "habitude", + "habitudes", + "habitant", + "habitants", + "habit", + "habits", + "hôpital", + "hôpitaux", + "humain", + "humains", + "humanité", + "honneur", + "honneurs", + "horizon", + "horizons", + "huile", + "huiles", + "hypothèse", + "hypothèses", + "harmonie", + "héritage", + "héroïne", + "héroïnes", + "hésitation", + "hommage", + "hommages", + "horaire", + "horaires", + "horreur", + "horreurs", + "humeur", + "humour", + "hygiène", + "hymne", + "habiter", + "hériter", + "hésiter", + "habiller", + "honorer", + "habite", + "habites", + "habitent", + "habitais", + "habitait", + "hésite", + "hésites", + "hésitent", + "hésitais", + "hésitait", + "habille", + "habillent", + "honore", + "hier", + "hôte", + "hôtes", + "hôtesse", + "humidité", + "haleine", + "hebdomadaire", + "hectare", + "hectares", + "hélicoptère", + "hémisphère", + "heureux", + "heureuse", + "hippopotame", + "hirondelle", + "honnête", + "honnêteté", + "huître", + "huîtres", + "humble", + "humilité", + "hydrogène", + "hypocrite", + "horloge", + "hôtellerie", + "hérédité", + "historien", + "historique", + "humanitaire", + "hormone", + "hormones", + "hospitalité", +]; + +fn issue(start: usize, end: usize, message: String, suggestions: Vec) -> Issue { + Issue { start, end, kind: IssueKind::Grammar, message, suggestions, lang: ProofLang::from_tag(Some("fr-FR")) } +} + +/// Is `c` a space that doesn't break (or a thin space some writers use instead)? +fn is_fixed_space(c: char) -> bool { + matches!(c, '\u{A0}' | '\u{202F}' | '\u{2007}' | '\u{2009}' | '\u{2060}') +} + +/// The byte range of the whitespace-delimited token around byte `i` of `text`. +fn token_bounds(text: &str, i: usize) -> (usize, usize) { + let start = text.get(..i).and_then(|t| t.char_indices().rev().find(|(_, c)| c.is_whitespace())).map_or(0, |(k, c)| k + c.len_utf8()); + let end = text.get(i..).and_then(|t| t.find(char::is_whitespace)).map_or(text.len(), |k| i + k); + (start, end) +} + +/// The whitespace-delimited token around byte `i` of `text`. +fn token_at(text: &str, i: usize) -> &str { + let (start, end) = token_bounds(text, i); + text.get(start..end).unwrap_or("") +} + +/// Is `token` a web or e-mail address, a path or an HTML entity, where punctuation isn't prose? +fn is_address(token: &str) -> bool { + token.contains("://") || token.contains('@') || token.starts_with("www.") || token.contains('&') || token.contains('\\') +} + +/// `«` + text + `»` in a message, with French spacing. +fn quoted(s: &str) -> String { + format!("«{NBSP}{s}{NBSP}»") +} + +/// `;` `!` `?` `:` without the non-breaking space French puts before them. +/// Quebec typography (`quebec`) puts no space before `;` `!` `?`, only before `:`. +fn high_punctuation(text: &str, quebec: bool, v: &mut Vec) { + let mut token: Option<(usize, usize, bool)> = None; + for (i, p) in text.char_indices() { + if !matches!(p, ';' | ':' | '!' | '?') || (quebec && p != ':') { + continue; + } + let before = text.get(..i).and_then(|t| t.chars().next_back()); + let after = text.get(i + p.len_utf8()..).and_then(|t| t.chars().next()); + // One look per token, so a long run of marks stays linear. + let address = match token { + Some((start, end, address)) if start <= i && i < end => address, + _ => { + let (start, end) = token_bounds(text, i); + let address = is_address(text.get(start..end).unwrap_or("")); + token = Some((start, end, address)); + address + } + }; + if address { + continue; + } + let (space, kind) = if p == ':' { (NBSP, "insécable") } else { (NNBSP, "fine insécable") }; + let message = format!("Espace {kind} attendue avant {}", quoted(&p.to_string())); + let fix = format!("{space}{p}"); + // Only a missing space is flagged: an ordinary one stays quiet until typing turns it into + // a non-breaking one by itself (French AutoFormat). + if before.is_some_and(char::is_alphabetic) { + // `10:30`, `a:b`, `C:` and `x;y` aren't prose: the mark must end the word. + let ends = match p { + ':' => after.is_none_or(char::is_whitespace), + _ => after.is_none_or(|a| a.is_whitespace() || matches!(a, '!' | '?' | '»' | '"' | '”' | ')')), + }; + if ends { + v.push(issue(i, i + p.len_utf8(), message, vec![fix])); + } + } + } +} + +/// `«` and `»` without a non-breaking space inside. +fn guillemets(text: &str, v: &mut Vec) { + for (i, c) in text.char_indices() { + let len = c.len_utf8(); + match c { + '«' => { + let mut next = text.get(i + len..).unwrap_or("").chars(); + let message = format!("Espace insécable attendue après {}", quoted("«")); + if next.next().is_some_and(char::is_alphanumeric) { + v.push(issue(i, i + len, message, vec![format!("«{NBSP}")])); + } + } + '»' => { + let prev = text.get(..i).unwrap_or("").chars().next_back(); + let message = format!("Espace insécable attendue avant {}", quoted("»")); + if prev.is_some_and(|n| n.is_alphanumeric() || matches!(n, '.' | '!' | '?' | '…')) { + v.push(issue(i, i + len, message, vec![format!("{NBSP}»")])); + } + } + _ => {} + } + } +} + +/// `"…"` around a phrase, where French uses `« … »`. +fn straight_quotes(text: &str, v: &mut Vec) { + let quotes: Vec = text.match_indices('"').map(|(i, _)| i).collect(); + // An odd count leaves no telling which quote opens and which closes. + if !quotes.len().is_multiple_of(2) { + return; + } + for pair in quotes.chunks(2) { + let [a, b] = pair else { continue }; + let (Some(inner), Some(before), Some(after)) = (text.get(a + 1..*b), text.get(..*a), text.get(b + 1..)) else { continue }; + let opens = before.chars().next_back().is_none_or(|c| c.is_whitespace() || matches!(c, '(' | '[' | '—' | '–')); + let closes = after.chars().next().is_none_or(|c| c.is_whitespace() || matches!(c, '.' | ',' | ';' | ':' | '!' | '?' | ')' | ']' | '…')); + let phrase = inner.trim(); + if !opens || !closes || !phrase.chars().any(char::is_alphabetic) || phrase.chars().count() > 120 || phrase.contains('\n') { + continue; + } + v.push(issue(*a, b + 1, format!("Guillemets français attendus{NBSP}: {}", quoted(phrase)), vec![quoted(phrase)])); + } +} + +fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec) { + for pair in ws.windows(2) { + let [(a0, b0), (a1, b1)] = pair else { continue }; + let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue }; + let lower = w0.to_lowercase(); + let starts_upper = |w: &str| w.chars().next().is_some_and(char::is_uppercase); + if !gap.is_empty() + && gap.chars().all(char::is_whitespace) + && w0.chars().all(char::is_alphabetic) + && lower == w1.to_lowercase() + && !ALLOWED_REPEATS.contains(&lower.as_str()) + // `de De…`, `Bora Bora`: a capitalized second word is a name. + && !starts_upper(w1) + { + v.push(issue(*a0, *b1, format!("Mot répété{NBSP}: {}", quoted(w1)), vec![w0.to_string()])); + } + } +} + +/// Is `prev`, before a period, an abbreviation, an initial, a number or a Roman numeral (after +/// which a lower-case word doesn't start a sentence)? +fn abbreviation_or_number(prev: &str) -> bool { + prev.chars().count() <= 1 + || prev.chars().any(|c| c.is_ascii_digit()) + || prev.chars().all(|c| matches!(c, 'I' | 'V' | 'X' | 'L' | 'C' | 'D' | 'M')) + || ABBREVIATIONS.contains(&prev.to_lowercase().as_str()) +} + +/// Does `gap`, between the words `prev` and `next`, end a sentence: a `.`, `!` or `?` (French +/// puts a space before the last two), then a space, with only quotes and brackets around? Not +/// an ellipsis, not after an abbreviation; for `!` and `?`, not after an interjection, not before +/// a closing quote (dialogue goes on: `« Tu viens ? » demanda-t-elle`), not before an inversion +/// or a speech verb (`Quoi ? dit-il`, `— Viens ! cria-t-on`), and never in a `dialogue` line (one +/// opening with a dash). +fn ends_sentence(gap: &str, prev: &str, next: &str, dialogue: bool) -> bool { + let closing = ['"', '”', '’', '\'', ')', '»', ']']; + let t = gap.trim_start_matches(|c: char| c.is_whitespace() || is_fixed_space(c) || closing.contains(&c)); + let Some(end) = t.chars().next() else { return false }; + if !matches!(end, '.' | '!' | '?') || t.starts_with("..") { + return false; + } + if end == '.' && abbreviation_or_number(prev) { + return false; + } + let exclamation = end != '.'; + let lower_next = next.to_lowercase(); + if exclamation + && (dialogue + || INTERJECTIONS.contains(&prev.to_lowercase().as_str()) + || INVERTED.iter().any(|p| lower_next.ends_with(p)) + || SPEECH_VERBS.contains(&lower_next.as_str())) + { + return false; + } + let rest = t.trim_start_matches(['.', '!', '?']); + let after_marks = rest.trim_start_matches(is_fixed_space); + if after_marks.starts_with(closing) { + if exclamation { + return false; + } + return ends_sentence_after(after_marks.trim_start_matches(|c: char| closing.contains(&c) || is_fixed_space(c))); + } + ends_sentence_after(rest) +} + +/// Is `rest` (after the final mark) a space, then nothing but opening quotes and brackets? +fn ends_sentence_after(rest: &str) -> bool { + rest.starts_with(char::is_whitespace) + && rest.trim_start().chars().all(|c| c.is_whitespace() || is_fixed_space(c) || matches!(c, '"' | '“' | '«' | '(' | '[' | '—' | '–')) +} + +fn sentence_capitals(text: &str, ws: &[(usize, usize)], v: &mut Vec) { + let dialogue = text.trim_start().starts_with(['—', '–']); + for pair in ws.windows(2) { + let [(a0, b0), (a1, b1)] = pair else { continue }; + let (Some(prev), Some(gap), Some(w)) = (text.get(*a0..*b0), text.get(*b0..*a1), text.get(*a1..*b1)) else { continue }; + let word_like = w.chars().all(|c| c.is_alphabetic() || matches!(c, '\'' | '’' | '-')); + if word_like && w.chars().next().is_some_and(char::is_lowercase) && ends_sentence(gap, prev, w, dialogue) && !is_address(token_at(text, *a1)) + { + let fixed: String = w.chars().next().map(|f| f.to_uppercase().chain(w.chars().skip(1)).collect()).unwrap_or_default(); + v.push(issue(*a1, *b1, "Majuscule attendue en début de phrase".into(), vec![fixed])); + } + } +} + +/// Does `w` (lower case) start with a sound that makes `le`, `de`, `que`… elide? +fn takes_elision(w: &str) -> bool { + let Some(first) = w.chars().next() else { return false }; + if NO_ELISION.contains(&w) { + return false; + } + match first { + 'a' | 'à' | 'â' | 'ä' | 'e' | 'é' | 'è' | 'ê' | 'ë' | 'i' | 'î' | 'ï' | 'o' | 'ô' | 'ö' | 'u' | 'ù' | 'û' | 'ü' | 'œ' | 'æ' => { + true + } + 'h' => MUTE_H.contains(&w), + // Only `y` itself (`j’y vais`): `le yaourt`, `de yoga`. + 'y' => w == "y", + _ => false, + } +} + +fn elisions(text: &str, ws: &[(usize, usize)], v: &mut Vec) { + for pair in ws.windows(2) { + let [(a0, b0), (a1, b1)] = pair else { continue }; + let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue }; + if gap.is_empty() || !gap.chars().all(|c| c == ' ') { + continue; + } + let lower0 = w0.to_lowercase(); + let Some((_, elided)) = ELIDING.iter().find(|(full, _)| *full == lower0) else { continue }; + // `le`/`Le` only, not `LE`; never before a proper noun (`de Orléans`) or a capital. + let capital = w0.chars().next().is_some_and(char::is_uppercase); + if w0.chars().skip(1).any(char::is_uppercase) || !w1.chars().next().is_some_and(char::is_lowercase) { + continue; + } + if !w1.chars().all(|c| c.is_alphabetic() || c == '-') { + continue; + } + let lower1 = w1.to_lowercase(); + // `le un`, `la une` (the number, the front page); a single letter only where it is a word + // the elision is certain for (`n’a`, `qu’a`, `j’y`). + if matches!(lower0.as_str(), "le" | "la") && matches!(lower1.as_str(), "un" | "une") { + continue; + } + let certain = (lower1 == "y" && matches!(lower0.as_str(), "je" | "me" | "te" | "se" | "ne")) + || (lower1 == "a" && matches!(lower0.as_str(), "ne" | "que")); + if lower1.chars().count() == 1 && !certain { + continue; + } + if !takes_elision(&lower1) { + continue; + } + let head = if capital { crate::affix::capitalize(elided) } else { (*elided).to_string() }; + let fixed = format!("{head}’{w1}"); + v.push(issue(*a0, *b1, format!("Élision{NBSP}: {}", quoted(&fixed)), vec![fixed])); + } +} + +/// Grammar issues in a paragraph of French text (France's typography), in order. +pub fn check(text: &str) -> Vec { + check_in(text, ProofLang::from_tag(Some("fr-FR"))) +} + +/// Grammar issues in a French Language pack locale, with Quebec typography in `fr-CA`. +/// Other locales have no French issues; the original locale is retained on every issue. +pub fn check_in(text: &str, lang: ProofLang) -> Vec { + let Some(locale) = lang.pack_locale().filter(|l| l.primary() == "fr") else { return Vec::new() }; + let ws: Vec<(usize, usize)> = words(text); + let mut v = Vec::new(); + high_punctuation(text, crate::lang::region(locale.as_str()).is_some_and(|r| r.eq_ignore_ascii_case("CA")), &mut v); + guillemets(text, &mut v); + straight_quotes(text, &mut v); + repeated_words(text, &ws, &mut v); + sentence_capitals(text, &ws, &mut v); + elisions(text, &ws, &mut v); + for i in &mut v { + i.lang = lang; + } + v.sort_by_key(|i| (i.start, i.end)); + v +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The issues of `text` as (flagged text, first suggestion). + fn found(text: &str) -> Vec<(String, String)> { + check(text) + .iter() + .map(|i| (text.get(i.start..i.end).unwrap_or("?").to_string(), i.suggestions.first().cloned().unwrap_or_default())) + .collect() + } + + fn quiet(text: &str) { + let v = check(text); + assert!(v.is_empty(), "{text:?} → {v:?}"); + } + + fn one(text: &str, flagged: &str, fix: &str) { + assert_eq!(found(text), [(flagged.to_string(), fix.to_string())], "{text:?}"); + } + + #[test] + fn missing_space_before_high_punctuation() { + one("Bonjour!", "!", "\u{202F}!"); + one("Tu viens?", "?", "\u{202F}?"); + one("Il pleut; il vente.", ";", "\u{202F};"); + one("Note: rien.", ":", "\u{A0}:"); + // `?!`: only the first mark needs the space. + one("Quoi?!", "?", "\u{202F}?"); + assert!(check("Bonjour!").iter().all(|i| i.lang == ProofLang::from_tag(Some("fr-FR")) && i.message.contains("fine insécable"))); + assert!(check("Note: rien.").iter().all(|i| i.message.contains("insécable") && !i.message.contains("fine"))); + } + + #[test] + fn an_ordinary_space_stays_quiet_until_french_autoformat() { + // Typing can't produce a non-breaking space yet, so an ordinary one is accepted. + quiet("Vraiment ? Bonjour ! Il pleut ; note : rien."); + quiet("Il dit « oui » hier."); + } + + #[test] + fn quebec_has_no_space_before_semicolon_exclamation_and_question() { + let fr_ca = |t: &str| check_in(t, ProofLang::from_tag(Some("fr-CA"))); + assert!(fr_ca("Bonjour! Tu viens? Il pleut; il vente.").is_empty(), "{:?}", fr_ca("Bonjour! Tu viens?")); + // The colon and the guillemets still want their space in Quebec. + let v = fr_ca("Note: «oui»."); + assert_eq!(v.len(), 3, "{v:?}"); + assert!(v.iter().all(|i| i.lang == ProofLang::from_tag(Some("fr-CA"))), "{v:?}"); + assert_eq!(check_in("Bonjour!", ProofLang::from_tag(Some("fr-FR"))).len(), 1, "France still wants the space"); + } + + #[test] + fn high_punctuation_stays_quiet() { + quiet("Bonjour\u{202F}! Tu viens\u{202F}? Il pleut\u{202F}; note\u{A0}: rien."); + quiet("Bonjour\u{A0}! Quoi\u{202F}?! Ah\u{2009}!"); + quiet("Il est 10:30."); + quiet("Voir https://exemple.org:8080/a?b=c pour la suite."); + quiet("Écrire à contact@exemple.org!"); + quiet("Super :) et ;-) aussi :D"); + quiet("a:b et x;y"); + quiet("Le fichier C:\\dossier est là."); + } + + #[test] + fn missing_space_inside_guillemets() { + assert_eq!(found("Il dit «oui» hier."), [("«".to_string(), "«\u{A0}".to_string()), ("»".to_string(), "\u{A0}»".to_string())]); + quiet("Il dit «\u{A0}oui\u{A0}» hier."); + quiet("Il dit «\u{202F}oui\u{202F}» hier."); + } + + #[test] + fn straight_quotes_become_guillemets() { + one("Il a dit \"bonjour\" hier.", "\"bonjour\"", "«\u{A0}bonjour\u{A0}»"); + one("\"Bonne idée\", dit-elle.", "\"Bonne idée\"", "«\u{A0}Bonne idée\u{A0}»"); + // Inch marks and unbalanced quotes are left alone. + quiet("Un écran de 27\" et un de 32\" ici."); + quiet("Il a dit \"bonjour hier."); + } + + #[test] + fn repeated_words_are_flagged() { + one("Le le chat dort.", "Le le", "Le"); + one("Je pense que que tu viens.", "que que", "que"); + one("Il mange la la pomme.", "la la", "la"); + } + + #[test] + fn deliberate_repetitions_stay_quiet() { + quiet("Nous nous lavons et vous vous trompez."); + quiet("Si si, très très bien."); + quiet("Ah ah, bla bla."); + quiet("Je crois que qu’il viendra."); + quiet("Le livre de De Gaulle."); + quiet("Nous partons à Bora Bora."); + quiet("Ils ont dit bye bye. Tchin tchin\u{202F}!"); + } + + #[test] + fn sentence_capitals_are_flagged() { + one("Il pleut. il fait froid.", "il", "Il"); + one("Il pleut\u{202F}! il fait froid.", "il", "Il"); + one("Tu viens\u{202F}? on part.", "on", "On"); + one("Il pleut. l’orage arrive.", "l’orage", "L’orage"); + one("Il pleut. «\u{A0}tout va bien\u{A0}», dit-il.", "tout", "Tout"); + } + + #[test] + fn sentence_capitals_stay_quiet() { + quiet("Voir p. 12 et cf. la note, etc. puis la suite."); + quiet("M. le maire est là."); + quiet("Il attend… puis part. Il attend... puis part."); + quiet("«\u{A0}Tu viens\u{202F}?\u{A0}» demanda-t-elle."); + quiet("Ah\u{202F}! que c’est beau."); + quiet("Quoi\u{202F}? dit-il."); + quiet("Tu viens\u{202F}? demanda le voisin."); + quiet("— Tu viens\u{202F}? reprends-tu, inquiet."); + quiet("– Viens\u{202F}! on part."); + quiet("Voir www.exemple.fr pour la suite."); + quiet("En 2026. ensuite rien."); + quiet("il commence en minuscule."); + } + + #[test] + fn missing_elisions_are_flagged() { + one("Il refuse de avoir peur.", "de avoir", "d’avoir"); + one("Je ai faim.", "Je ai", "J’ai"); + one("Voici le homme.", "le homme", "l’homme"); + one("Je sais que il viendra.", "que il", "qu’il"); + one("Le arbre est grand.", "Le arbre", "L’arbre"); + one("Il ne a pas faim.", "ne a", "n’a"); + one("Bon, je y vais.", "je y", "j’y"); + one("Il se est levé.", "se est", "s’est"); + one("Voici la école.", "la école", "l’école"); + one("Il me aime.", "me aime", "m’aime"); + } + + #[test] + fn aspirated_h_and_other_exceptions_stay_quiet() { + for w in [ + "hibou", + "héros", + "haricot", + "hasard", + "haut", + "hall", + "hamac", + "hangar", + "hockey", + "homard", + "huit", + "hérisson", + "hors-jeu", + "onze", + "oui", + "yaourt", + ] { + quiet(&format!("Voici le {w}.")); + } + // hiérarchie has an aspirated h, despite the mute h in hiéroglyphe: + // https://www.academie-francaise.fr/le-hieroglyphe-pour-lhieroglyphe + for w in ["honte", "hache", "harpe", "hausse", "hiérarchie"] { + quiet(&format!("Voici la {w}.")); + } + quiet("Une question de hiérarchie."); + quiet("Le onze de France. À la une du journal."); + quiet("Il vient de Orléans."); + quiet("De a à z, le a et le b."); + quiet("Le y et le x sont des variables. La dérivée de y."); + quiet("Je mange de la viande."); + } + + #[test] + fn hostile_input_never_panics() { + for t in ["", " ", "!", " !", "«", "»", " »", "« ", "\"", "\"\"", ". a", "de", "de ", "\u{202F}?", "é!é", "« » « »", ":", " :", "a :)"] + { + let _ = check(t); + } + // A long run of marks in one token: linear, not one token scan per mark. + let run = format!("mot{}", "?".repeat(200_000)); + assert_eq!(check(&run).len(), 1); + } +} diff --git a/crates/proof/src/fr/mod.rs b/crates/proof/src/fr/mod.rs new file mode 100644 index 000000000..541fcac0d --- /dev/null +++ b/crates/proof/src/fr/mod.rs @@ -0,0 +1,181 @@ +//! French proofing through the shared Language pack cache. +//! +//! A French locale remains [`ProofLang::Pack`]: dictionary selection, installation, reloads, +//! memory limits and hyphenation use the same paths as other packs. This module adds French +//! elisions, accent-oriented suggestions and conservative typography, including Quebec spacing. +//! No dictionary or hyphenation data is bundled here. + +pub mod grammar; +mod suggest; +#[cfg(test)] +#[allow(clippy::unwrap_used)] +pub(crate) mod test_dict; + +use crate::affix::AffixDict; +use crate::{Options, ProofLang}; + +pub use suggest::suggest_in; + +/// Words after whose elision a user-dictionary entry is accepted. Ordinary words and their +/// elisions are always decided by the dictionary (`l'arbre`, but not `l'arbres`). +const ELISIONS: &[&str] = &["l", "d", "j", "m", "t", "s", "n", "c", "qu", "jusqu", "lorsqu", "puisqu", "quoiqu", "presqu", "quelqu"]; + +/// Check French spelling in the installed France/generic pack, with the default options. +/// Regional callers should use [`crate::is_correct_in`] with their full locale. +pub fn is_correct(word: &str) -> bool { + crate::is_correct_in(word, ProofLang::from_tag(Some("fr-FR"))) +} + +/// French suggestions from the installed France/generic pack, best first, at most `max`. +/// Regional callers should use [`crate::suggest_in`] with their full locale. +pub fn suggest(word: &str, max: usize) -> Vec { + crate::suggest_in(word, max, ProofLang::from_tag(Some("fr-FR"))) +} + +/// Check against a supplied dictionary, without changing any installed pack or cache. +pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { + check(d, &crate::composed(word), &Options::default()) +} + +pub(crate) fn check(d: &AffixDict, word: &str, opts: &Options) -> bool { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { return true }; + // A regional dictionary may store the curly apostrophe without ICONV. Honor that entry + // before the French ASCII-apostrophe fallback. + if known(d, w) { + return true; + } + let w = w.replace('’', "'"); + if known(d, &w) { + return true; + } + if let Some((head, tail)) = w.split_once('\'') + && ELISIONS.contains(&head.to_lowercase().as_str()) + && !tail.is_empty() + && !tail.contains('\'') + && crate::in_user_dictionary(&tail.to_lowercase()) + { + return true; + } + w.contains('-') && w.split('-').all(|p| !p.is_empty() && (p.chars().count() <= 1 || known(d, p))) +} + +fn known(d: &AffixDict, word: &str) -> bool { + d.check(word) || crate::in_user_dictionary(&word.to_lowercase()) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + #[test] + fn original_french_words_and_elisions() { + let d = test_dict::dictionary(); + for word in [ + "maison", + "maisons", + "école", + "écoles", + "grand", + "grande", + "grands", + "grandes", + "aimer", + "aime", + "aimons", + "aiment", + "suis", + "est", + "sommes", + "sont", + "a", + "avons", + "ont", + "Paris", + "Montréal", + "œuvre", + "sœur", + "aujourd'hui", + "aujourd’hui", + "peut-être", + "arc-en-ciel", + "km", + "Maison", + "MAISON", + "SNCF", + "2026", + "très", + "rivière", + "l'arbre", + "l’arbre", + "L'arbre", + "d'enfants", + "l'homme", + "qu'elle", + "qu’il", + "j'aime", + "j’ai", + "d'avoir", + "l'école", + "jusqu'à", + "maison-école", + "est-il", + "a-t-il", + ] { + assert!(is_correct_with(&d, word), "{word}"); + } + for word in [ + "maisonn", + "ecole", + "arbrre", + "beaucoups", + "aimmer", + "soeur", + "Km", + "grandd", + "aim", + "l'hasard", + "l'chat", + "l'arbres", + "l'arbrre", + "qu'maison", + "maison-ecole", + "maison--école", + ] { + assert!(!is_correct_with(&d, word), "{word}"); + } + assert!(is_correct_with(&d, "e\u{301}cole")); + assert!(!is_correct_with(&d, "maison\u{338}"), "a trailing mark is part of the word"); + } + + #[test] + fn explicit_curly_apostrophe_without_iconv() { + let d = AffixDict::from_hunspell("SET UTF-8\n", "1\naujourd’hui\n", &[]).unwrap(); + assert!(is_correct_with(&d, "aujourd’hui")); + assert!(!is_correct_with(&d, "aujourd’huui")); + } + + #[test] + fn user_word_after_an_elision() { + let d = test_dict::dictionary(); + let word = "zxqmotadaptfr"; + assert!(!is_correct_with(&d, word) && !is_correct_with(&d, &format!("l'{word}"))); + crate::add_word(word); + assert!(is_correct_with(&d, word) && is_correct_with(&d, &format!("l'{word}")) && is_correct_with(&d, &format!("d’{word}"))); + assert!(!is_correct_with(&d, &format!("chat'{word}"))); + crate::remove_word(word); + } + + #[test] + fn options_and_missing_dictionary() { + let d = test_dict::dictionary(); + let defaults = Options::default(); + let strict = Options { ignore_uppercase: false, ignore_numbers: false, ..defaults }; + assert!(check(&d, "SNCF", &defaults) && check(&d, "mp3x", &defaults)); + assert!(!check(&d, "SNCF", &strict) && !check(&d, "mp3x", &strict)); + assert!(check(&d, "MAISON", &strict) && check(&d, "2026", &strict)); + for word in ["école", "l'arbre", "xqzv", "SNCF", ""] { + assert!(check(&AffixDict::default(), word, &strict)); + } + } +} diff --git a/crates/proof/src/fr/suggest.rs b/crates/proof/src/fr/suggest.rs new file mode 100644 index 000000000..86477d33f --- /dev/null +++ b/crates/proof/src/fr/suggest.rs @@ -0,0 +1,81 @@ +//! French tuning of the shared, bounded affix suggestion engine. + +use crate::affix::AffixDict; +use crate::affix_suggest::{Split, Tuning}; + +/// Short words often run into the next one (`et la`, `de Paris`, `pour moi`). +const LEADING_WORDS: &[&str] = &[ + "à", "a", "au", "aux", "de", "du", "des", "le", "la", "les", "un", "une", "et", "ou", "où", "en", "y", "ce", "se", "ne", "je", "tu", "il", "on", + "nous", "vous", "ils", "elle", "par", "pour", "sur", "sous", "dans", "avec", "sans", "chez", "vers", "entre", "très", "plus", "moins", "bien", + "pas", "que", "qui", "mais", "donc", "car", "si", "son", "sa", "ses", "mon", "ma", "mes", "ton", "ta", "tes", "est", "tout", "tous", "toute", + "comme", "quand", "puis", "encore", "déjà", "jamais", "toujours", +]; +/// Short words often run into the one before (`dis-moi` without the hyphen, `là-bas`, or a +/// dropped space: `pourmoi` → `pour moi`). +const TRAILING_WORDS: &[&str] = + &["moi", "toi", "lui", "elle", "nous", "vous", "eux", "le", "la", "les", "en", "y", "là", "ci", "il", "on", "ils", "elles", "est", "et"]; + +const TUNING: Tuning = Tuning { + split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, + alphabet: "esaitnrulodcpmévqfbghjàxèêyçôîâûùœ", + ..Tuning::DEFAULT +}; + +/// French suggestions from a supplied dictionary; keeps the writer's apostrophe and case. +/// Dictionary OCONV remains active for every other output conversion. +pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { + let word = crate::composed(word); + let curly = word.contains('’'); + let apostrophe = if curly { '’' } else { '\'' }; + let mut out = Vec::new(); + for text in crate::affix_suggest::suggest_in(d, &word, max, &TUNING) { + let styled: String = text.chars().map(|c| if c == '\'' || c == '’' { apostrophe } else { c }).collect(); + // A pack may spell only one apostrophe and have no ICONV. Keep its valid output + // rather than turn it back into the spelling mistake the caller is correcting. + let text = if styled.split(' ').all(|p| d.can_suggest_cased(p, p)) { styled } else { text }; + if text != word && !out.contains(&text) { + out.push(text); + } + } + out +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + #[test] + fn french_accents_confusions_splits_and_elisions() { + let d = super::super::test_dict::dictionary(); + for (typo, expected) in [ + ("ecole", "école"), + ("riviere", "rivière"), + ("soeur", "sœur"), + ("maisonn", "maison"), + ("pourmoi", "pour moi"), + ("l'arbrre", "l'arbre"), + ("l’arbrre", "l’arbre"), + ] { + let got = suggest_in(&d, typo, 8); + assert!(got.iter().any(|s| s == expected), "{typo}: {got:?}"); + } + assert!(suggest_in(&d, "e\u{301}cole", 4).iter().all(|s| s != "école")); + assert!(suggest_in(&d, "ecole", 0).is_empty()); + assert!(suggest_in(&d, &"a".repeat(41), 4).is_empty()); + } + + #[test] + fn curly_dictionary_without_iconv_remains_suggestible() { + let d = AffixDict::from_hunspell("SET UTF-8\n", "1\naujourd’hui\n", &[]).unwrap(); + assert!(suggest_in(&d, "aujourd’huui", 4).iter().any(|s| s == "aujourd’hui")); + assert!(suggest_in(&d, "aujourd'hui", 4).iter().any(|s| s == "aujourd’hui")); + } + + #[test] + fn output_conversion_preserves_typed_apostrophe() { + let d = AffixDict::from_hunspell("SET UTF-8\nICONV 1\nICONV ’ '\nOCONV 1\nOCONV ' ’\n", "1\nl'arbre\n", &[]).unwrap(); + assert!(suggest_in(&d, "l'arbrre", 4).iter().any(|s| s == "l'arbre")); + assert!(suggest_in(&d, "l’arbrre", 4).iter().any(|s| s == "l’arbre")); + } +} diff --git a/crates/proof/src/fr/test_dict.rs b/crates/proof/src/fr/test_dict.rs new file mode 100644 index 000000000..23637254c --- /dev/null +++ b/crates/proof/src/fr/test_dict.rs @@ -0,0 +1,234 @@ +//! A tiny synthetic French Hunspell dictionary for tests (written for these tests; not taken +//! from any real dictionary). Real dictionaries are installed on the system, never bundled. +//! +//! It exercises what the Dicollecte French dictionary uses: two-character flags (`FLAG long`), +//! stems that need an affix (`NEEDAFFIX`, with an empty suffix for the singular), elisions as +//! prefixes licensed by a suffix's continuation (`l'arbre` but not `l'arbres`), irregular verbs +//! through `FULLSTRIP` (`être` → `suis`), `KEEPCASE` (`km`), `NOSUGGEST` (`bah`), `ICONV` (`’` +//! read as `'`) and the +//! suggestion tables (`TRY`, `MAP` for accents, `REP` for common confusions). + +pub const AFF: &str = "SET UTF-8 +FLAG long +NEEDAFFIX () +KEEPCASE || +NOSUGGEST -- +FULLSTRIP +ICONV 1 +ICONV ’ ' +TRY esaitnrulodcpmévqfbghjàxèêyçôîâûùœ +MAP 6 +MAP aàâä +MAP eéèêë +MAP iîï +MAP oôö +MAP uùûü +MAP cç +REP 3 +REP oe œ +REP ph f +REP f ph +PFX L' Y 1 +PFX L' 0 l' [aàâeéèêiîoôuœh] +PFX D' Y 1 +PFX D' 0 d' [aàâeéèêiîoôuœh] +PFX Q' Y 1 +PFX Q' 0 qu' [aàâeéèêiîoôuœh] +PFX J' Y 1 +PFX J' 0 j' [aàâeéèêiîoôuœhy] +SFX S. Y 2 +SFX S. 0 0/L'D'Q' [^sxz] +SFX S. 0 s/D'Q' [^sxz] +SFX S= Y 2 +SFX S= 0 0 [^sxz] +SFX S= 0 s [^sxz] +SFX F. Y 4 +SFX F. 0 0/L'D'Q' . +SFX F. 0 e/L'D'Q' . +SFX F. 0 s/D'Q' . +SFX F. 0 es/D'Q' . +SFX Er Y 6 +SFX Er er er/D' er +SFX Er er e/J' er +SFX Er er es er +SFX Er er ons er +SFX Er er ez er +SFX Er er ent er +SFX Et N 7 +SFX Et être être être +SFX Et être suis être +SFX Et être es être +SFX Et être est être +SFX Et être sommes être +SFX Et être êtes être +SFX Et être sont être +SFX Av N 7 +SFX Av avoir avoir/D' avoir +SFX Av avoir ai/J' avoir +SFX Av avoir as avoir +SFX Av avoir a avoir +SFX Av avoir avons avoir +SFX Av avoir avez avoir +SFX Av avoir ont/Q' avoir +"; + +/// Stems (the count line first, as in Hunspell files). +pub const DIC: &str = "150 +maison/S.() +arbre/S.() +enfant/S.() +ami/S.() +amie/S.() +homme/S.() +école/S.() +café/S.() +chat/S.() +rivière/S.() +ville/S.() +heure/S.() +histoire/S.() +sœur/S.() +œuvre/S.() +photo/S.() +téléphone/S.() +année/S.() +vie/S.() +livre/S.() +table/S.() +porte/S.() +femme/S.() +garçon/S.() +fille/S.() +père/S.() +mère/S.() +langue/S.() +mot/S.() +texte/S.() +phrase/S.() +exemple/S.() +jour/S.() +nuit/S.() +monde/S.() +hasard/S=() +haricot/S=() +grand/F.() +petit/F.() +noir/F.() +aimer/Er() +parler/Er() +chanter/Er() +arriver/Er() +être/Et() +avoir/Av() +il/Q' +ils/Q' +elle/Q' +elles/Q' +on/Q'L' +un/D'Q'L' +une/D'Q'L' +en/Q'J' +y/J' +à/Q' +jusqu'à +aujourd'hui/D' +peut-être +arc-en-ciel +km/|| +bah/-- +Paris +France +Montréal +le +la +les +de +des +du +et +à +je +tu +nous +vous +que +qui +ne +pas +se +me +te +ce +ça +avec +pour +dans +sur +sous +très +près +beaucoup +si +oui +non +mais +ou +où +dit +demanda +quand +comme +plus +bien +tout +toujours +jamais +ici +là +alors +donc +aussi +encore +déjà +mon +ma +mes +son +sa +ses +notre +votre +leur +cette +ces +au +aux +par +entre +vers +chez +sans +moi +toi +lui +rien +chose +temps +fois +eau +voici +voilà +bonjour +merci +beau +belle +la +etc +M +p +"; + +/// The original synthetic fixture, parsed independently of the installed pack cache. +pub fn dictionary() -> crate::affix::AffixDict { + crate::affix::AffixDict::from_hunspell(AFF, DIC, &[]).unwrap() +} diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index 966ded72f..948d0e25b 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -754,9 +754,13 @@ mod tests { /// spec's default `~/.local/share` applies, and with no home there is no folder. #[test] fn xdg_data_home_follows_the_spec() { - let home = Path::new("/home/a"); - assert_eq!(xdg_data_home(Some("/data".into()), Some(home)), Some(PathBuf::from("/data"))); - assert_eq!(xdg_data_home(Some("/data".into()), None), Some(PathBuf::from("/data"))); + // On Windows a rooted path without a drive is not absolute. Keep the same + // absolute/relative contract on each platform instead of changing the loader. + let (home, data) = if cfg!(windows) { (r"C:\Users\a", r"C:\data") } else { ("/home/a", "/data") }; + let (home, data) = (Path::new(home), Path::new(data)); + assert!(data.is_absolute()); + assert_eq!(xdg_data_home(Some(data.as_os_str().into()), Some(home)), Some(data.to_path_buf())); + assert_eq!(xdg_data_home(Some(data.as_os_str().into()), None), Some(data.to_path_buf())); for xdg in [None, Some(""), Some("relative/data")] { assert_eq!(xdg_data_home(xdg.map(Into::into), Some(home)), Some(home.join(".local/share")), "{xdg:?}"); assert_eq!(xdg_data_home(xdg.map(Into::into), None), None, "{xdg:?}"); @@ -1010,7 +1014,12 @@ mod tests { assert!(load(&aff, &big).unwrap_err().contains("too large")); // A directory, and on Unix an endless device, are not dictionary files. (No clock: a // reader that read `/dev/zero` would never return, so the error itself is the proof.) - assert!(load(&aff, &dir).unwrap_err().contains("not a regular file")); + // Windows rejects directories in File::open, before the regular-file guard. The + // rejection is required everywhere; its diagnostic is platform-dependent. + let directory_error = load(&aff, &dir).unwrap_err(); + assert!(directory_error.contains(&dir.display().to_string()), "{directory_error}"); + #[cfg(unix)] + assert!(directory_error.contains("not a regular file"), "{directory_error}"); #[cfg(unix)] assert!(load(&aff, Path::new("/dev/zero")).unwrap_err().contains("not a regular file")); // Turkish reads through the same reader. diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 3df6cd017..f0f33d007 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -42,6 +42,9 @@ //! existing English fallback. //! - **German** ([`ProofLang::De`]): hyphenation by `hyph-utf8`'s German patterns (reformed //! spelling, MIT); no spelling or grammar marks yet. +//! - **French** (a [`ProofLang::Pack`] locale; module [`fr`]): the pack dictionary with French +//! elisions and accent-oriented suggestions, plus conservative typography and basic checks. +//! France and Canada retain their regional tags; Quebec uses its own punctuation spacing. //! - **Any other language** ([`ProofLang::Pack`], module [`packs`]): spelling and suggestions //! from its Hunspell dictionary (an installed Language pack, the dictionaries folder or the //! system's) with default tuning, hyphenation from its pack, no grammar rules; without a @@ -73,6 +76,7 @@ pub mod dict; pub mod en_variants; mod english; pub mod es; +pub mod fr; pub mod hr; pub mod hunspell; pub mod hyphen; @@ -976,6 +980,7 @@ fn grammar_in(text: &str, lang: ProofLang, capital_first: bool) -> Vec { match lang { ProofLang::En => grammar_en(text, capital_first), ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa => grammar_en(text, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect(), + ProofLang::Pack(locale) if locale.primary() == "fr" => fr::grammar::check_in(text, lang), // `pt-pt` shares Portuguese's rules, its issues in its own language. _ => language(lang.built_in()) .map_or_else(Vec::new, |l| (l.grammar)(text, lang, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect()), diff --git a/crates/proof/src/pack_lang.rs b/crates/proof/src/pack_lang.rs index 7b4d19315..8b6cea902 100644 --- a/crates/proof/src/pack_lang.rs +++ b/crates/proof/src/pack_lang.rs @@ -1,6 +1,7 @@ //! Proofing for any language by its locale ([`crate::ProofLang::Pack`]): its Hunspell dictionary (the //! dictionaries folder, then an installed Language pack, then the system's), read by the shared -//! affix engine, with default suggestion tuning and no grammar rules. +//! affix engine. Most use default suggestion tuning and no grammar rules; French locales add +//! the spelling tuning and conservative typography in [`crate::fr`] without a separate cache. //! //! The loaded dictionaries share a memory budget ([`MAX_LOADED_BYTES`], estimated from their //! files); the one used least recently goes first, but never one used during the current @@ -320,6 +321,9 @@ pub(crate) fn is_correct(word: &str, locale: PackLocale, opts: &crate::Options) if other_script(w, &scripts) { return true; } + if locale.primary() == "fr" { + return crate::fr::check(&d, w, opts); + } let known = |w: &str| d.check(w) || crate::in_user_dictionary(&w.to_lowercase()); if w.contains('-') && !known(w) { return w.split('-').all(|p| p.chars().count() <= 1 || known(p)); @@ -333,7 +337,7 @@ pub(crate) fn suggest(word: &str, max: usize, locale: PackLocale) -> Vec if d.is_empty() { return Vec::new(); } - crate::affix_suggest::suggest_in(&d, word, max, &Tuning::DEFAULT) + if locale.primary() == "fr" { crate::fr::suggest_in(&d, word, max) } else { crate::affix_suggest::suggest_in(&d, word, max, &Tuning::DEFAULT) } } /// The pack locales whose hyphenation patterns serve a [`crate::ProofLang::Pack`] language: the locale, diff --git a/crates/proof/tests/affix_compatibility.rs b/crates/proof/tests/affix_compatibility.rs new file mode 100644 index 000000000..6d09e5f90 --- /dev/null +++ b/crates/proof/tests/affix_compatibility.rs @@ -0,0 +1,92 @@ +//! Frozen public output from the pre-French shared affix engine at upstream +//! e6c5a701ddff5be16e266028168c70658aae39d9, using original synthetic fixtures. +//! Do not regenerate expectations from the new implementation: they establish the baseline. +//! Legacy WCAFF1 bytes remain frozen, but are explicitly refused by the WCAFF2 engine. + +use craft_proof::affix::{AffixDict, capitalize}; +use std::fmt::Write; + +#[path = "fixtures/affix_compatibility_cases.rs"] +mod fixtures; + +fn snapshot(case: &fixtures::Case, dict: &AffixDict) -> String { + let mut out = format!("{}\nlen={} rules={}\n", case.name, dict.len(), dict.rule_count()); + writeln!(out, "try={:?} key={:?} map={:?} rep={:?} comments={:?}", dict.try_chars(), dict.key(), dict.map(), dict.rep(), dict.comments()) + .unwrap(); + let mut stems: Vec<_> = dict.stems().collect(); + stems.sort_unstable(); + for stem in &stems { + let mut forms = dict.forms(stem); + forms.sort_unstable(); + forms.dedup(); + writeln!(out, "forms {stem:?}: {forms:?}").unwrap(); + } + let mut probes: Vec = case.probes.iter().map(|word| (*word).to_string()).collect(); + for stem in stems { + probes.extend(dict.forms(stem)); + } + let originals = probes.clone(); + for word in originals { + probes.extend([word.to_uppercase(), word.to_lowercase(), capitalize(&word)]); + } + probes.sort_unstable(); + probes.dedup(); + for word in probes { + writeln!( + out, + "{word:?}: contains={} check={} suggest={} raw_suggest={}", + dict.contains(&word), + dict.check(&word), + dict.can_suggest(&word), + dict.contains_suggestible(&word) + ) + .unwrap(); + } + out +} + +#[test] +fn non_french_affix_outputs_match_pre_french_baseline() { + let expected = include_str!("fixtures/affix_compatibility.expected"); + let mut actual = String::new(); + for case in fixtures::cases() { + let dict = AffixDict::from_hunspell(&case.aff, &case.dic, &["original compatibility fixture"]).unwrap(); + let result = snapshot(&case, &dict); + let restored = AffixDict::from_bytes(&dict.to_bytes().unwrap()).unwrap(); + assert_eq!(snapshot(&case, &restored), result, "current compact round trip: {}", case.name); + actual.push_str(&result); + } + // craft-libs 55332dd predates this adaptation and supports a prefix plus a two-suffix + // chain (native test affix::tests::a_prefix_with_a_suffix_chain). The old Indonesian + // fixture's bermain + kan + lah therefore became a word, in these three cases only. + // Keep the frozen historical file untouched and assert every explicit transition once. + let mut expected = expected.to_string(); + let mut transitions = 0; + for word in ["BERMAINKANLAH", "Bermainkanlah", "bermainkanlah"] { + let before = format!("{word:?}: contains=false check=false suggest=false raw_suggest=false\n"); + let after = if word == "bermainkanlah" { + format!("{word:?}: contains=true check=true suggest=true raw_suggest=true\n") + } else { + format!("{word:?}: contains=false check=true suggest=true raw_suggest=false\n") + }; + assert_eq!(expected.matches(&before).count(), 1, "one historical observable: {word}"); + expected = expected.replacen(&before, &after, 1); + transitions += 1; + } + assert_eq!(transitions, 3, "the documented upstream exception has exactly three observables"); + assert_eq!(actual, expected, "shared engine output changed beyond the explicit upstream chain exception"); +} + +#[test] +fn legacy_compact_data_is_refused_without_reinterpreting_homonyms() { + let encoded = include_str!("fixtures/affix_compatibility.compact"); + let cases = fixtures::cases(); + let lines: Vec<_> = encoded.lines().collect(); + assert_eq!(lines.len(), cases.len()); + for (case, hex) in cases.iter().zip(lines) { + assert_eq!(hex.len() % 2, 0); + let bytes: Vec<_> = + hex.as_bytes().as_chunks::<2>().0.iter().map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap()).collect(); + assert!(AffixDict::from_bytes(&bytes).is_err(), "WCAFF1 must be regenerated as WCAFF2: {}", case.name); + } +} diff --git a/crates/proof/tests/fixtures/affix_compatibility.compact b/crates/proof/tests/fixtures/affix_compatibility.compact new file mode 100644 index 000000000..5c8c627de --- /dev/null +++ b/crates/proof/tests/fixtures/affix_compatibility.compact @@ -0,0 +1,7 @@ +4d8bc10a825014446b7bbf62a07dd62744e5ae120b2ca2e02aafb8bceac54b295d067e91db76ea7f2516d16666ce30138c47ae3ba41e8c95a35cf884c89caf1c4b282789531ce4112756d1cadf809591a47e6a4bb39107aef2d64d5924e44f3dd42fd8ac4d09ca823c778d101b0c7011853e2d1be696536cf7e9ee57280dcd8dfe2dd4f97b88306faa2a0753b0f0274bea046c6f19dfd989a8a34dec70485d6b3e7c157553da617a03 +1d8c410e82301045bb9e534ce2deea11a8d8c48d10304196058b990418534aa25ec7852770e9aa17b3b27c79ffbf6a9768bd8515b2a32b8da6c796879bf1d4504ffe811dddfdec2c9c8a1a8d259ec33bbcc2077456a8439aee8f5556a418be90eb33e658e3069dc535941193888c462ae485d5a2a768ffa71244c32379968902d19a290e4174e6699dcc41d0308f17962aa67f +1dc7410ac2301005d0ace7141fdc1b3d42aa64a9a15d5497d392ca40eac818416f6fe8eaf1c65388f1483ba8c9439e5c30ebfae22a9314a93f2cf2ad1fcb94e20d09771c60823d0dada15521d06d1d2e6064308dd7fe3c90eb75657233bfd97789dcc2967db3c864ea03fd01 +1d8a3d0a02311406b77ea7f8c0dee8114437651453ac220a51a33cc88fc408e62c9e451bf55e1bb6199861baf94cca298d10135f39188753f43793f9c88e73c1859ff9912cade4060a5b4c102cc6a4abea410b7687b21f425b0f83df0b86bae57aa1a9f1ce9cbf6fa1a8b9e714bd50b51551f1ffd860444b3d +1d8c310e82401444b1dd53fcc45ef408282cb11008ab414b346a3641310413ed0cad95072034f450488c46f10a7f6fe4aeddcc9bcc0b4606a503d28538e15bbe0f2358c5bb4398f2258f787a860d3fa5c7644d268607d8881ba1ae3f1c9ba6e504ae6f0296c471d9ccb62d3605cc8947e780052ca00ff8c1067a842952fd89b8caae6e8c6878c716df3a56a4f3c01abfe222ed998ea5222f6c552e64d64426e7a794d53ae612fc00 +2dc84b0e8230100050d6738a49dc8b1e0114589800a11864398522c3a768a951120f2f212edf2b4e5e181e618793e13b6b1ab09ac60759963cb05db0e18f7d1905795622c9aa56dc6bb804253edfcad8e54b73dd4016a45877d8419c886b140522c718d2f08629967840a90cee41acf636f7a45dff3ffe3603b5ab8b243b0b702455e47ae08cc4daf552700cb534f37a31fc00 +35c7b98e82600045e1a9790a92e967dc97d20df71515b51345455114c1ed6d2616c6c2cad8d8fe2f36e67269cec9a7e5328a1296be65db31e7e6666cc9137bbd1dbba66e5aa67b9667e6c9f51c435295813c961b72e8d31f4887746a024da82934a50cc8a066d08c9a43736a012d281332a925b4a456d08ab2208b5a436b6a036d281bb2a92db4a576d08e722087da437bca855cca833cea001da82374a44ed0893a4367ea025da80c94a1b25096ca41392a0fe5a90254a01448a18a50912a4125aa0c95a90a54a1aa5095aa4135aa0ed5a906d4a09a50936a412daa0db5a90ed4a15448a5ba5097ea413daa0ff5290dd2a80134a086d0901a41232a0485a83014a62250848a42512a06c5a83814a71250824a42492a05a5a83494a6c41ff8197df77d0ffcf0fd08fcf4fd0cfcf6fd0e7cf37d0b7cf57d0dfcf2fdfa586b76f2aaf4657bae63ff8a97f475301cddfe0d8523d1583c914ca533d95cbea0144be54ab5566f345bed8edaedf5b5c17034d6275363365f98cb95b5ded8db9db377bdc3f174be883f71153771170ff1142ff196fe01 diff --git a/crates/proof/tests/fixtures/affix_compatibility.expected b/crates/proof/tests/fixtures/affix_compatibility.expected new file mode 100644 index 000000000..8fe3a9929 --- /dev/null +++ b/crates/proof/tests/fixtures/affix_compatibility.expected @@ -0,0 +1,497 @@ +pl: conditional strip, prefix cross products, diacritics +len=4 rules=5 +try="aeoiułkr" key="" map=[['a', 'ą'], ['o', 'ó', 'u']] rep=[("ż", "rz"), ("u", "ó")] comments=["original compatibility fixture"] +forms "Warszawa": ["Warszawa", "Warszawą"] +forms "kot": ["kot", "kotem", "koty", "niekot", "niekotem", "niekoty"] +forms "krowa": ["krowa", "krową"] +forms "piesek": ["piesek", "piesekem", "pieseky", "pieska"] +"": contains=false check=false suggest=false raw_suggest=false +"KOT": contains=false check=true suggest=true raw_suggest=false +"KOTEM": contains=false check=true suggest=true raw_suggest=false +"KOTY": contains=false check=true suggest=true raw_suggest=false +"KOTĄ": contains=false check=false suggest=false raw_suggest=false +"KROWA": contains=false check=true suggest=true raw_suggest=false +"KROWY": contains=false check=false suggest=false raw_suggest=false +"KROWĄ": contains=false check=true suggest=true raw_suggest=false +"Kot": contains=false check=true suggest=true raw_suggest=false +"Kotem": contains=false check=true suggest=true raw_suggest=false +"Koty": contains=false check=true suggest=true raw_suggest=false +"Kotą": contains=false check=false suggest=false raw_suggest=false +"Krowa": contains=false check=true suggest=true raw_suggest=false +"Krowy": contains=false check=false suggest=false raw_suggest=false +"Krową": contains=false check=true suggest=true raw_suggest=false +"NIE": contains=false check=false suggest=false raw_suggest=false +"NIEKOT": contains=false check=true suggest=true raw_suggest=false +"NIEKOTEM": contains=false check=true suggest=true raw_suggest=false +"NIEKOTY": contains=false check=true suggest=true raw_suggest=false +"NIEPIESKA": contains=false check=false suggest=false raw_suggest=false +"Nie": contains=false check=false suggest=false raw_suggest=false +"Niekot": contains=false check=true suggest=true raw_suggest=false +"Niekotem": contains=false check=true suggest=true raw_suggest=false +"Niekoty": contains=false check=true suggest=true raw_suggest=false +"Niepieska": contains=false check=false suggest=false raw_suggest=false +"PIESEK": contains=false check=true suggest=true raw_suggest=false +"PIESEKA": contains=false check=false suggest=false raw_suggest=false +"PIESEKEM": contains=false check=true suggest=true raw_suggest=false +"PIESEKY": contains=false check=true suggest=true raw_suggest=false +"PIESKA": contains=false check=true suggest=true raw_suggest=false +"Piesek": contains=false check=true suggest=true raw_suggest=false +"Pieseka": contains=false check=false suggest=false raw_suggest=false +"Piesekem": contains=false check=true suggest=true raw_suggest=false +"Pieseky": contains=false check=true suggest=true raw_suggest=false +"Pieska": contains=false check=true suggest=true raw_suggest=false +"WARSZAWA": contains=false check=true suggest=true raw_suggest=false +"WARSZAWĄ": contains=false check=true suggest=true raw_suggest=false +"Warszawa": contains=true check=true suggest=true raw_suggest=true +"Warszawą": contains=true check=true suggest=true raw_suggest=true +"kot": contains=true check=true suggest=true raw_suggest=true +"kotem": contains=true check=true suggest=true raw_suggest=true +"koty": contains=true check=true suggest=true raw_suggest=true +"kotą": contains=false check=false suggest=false raw_suggest=false +"krowa": contains=true check=true suggest=true raw_suggest=true +"krowy": contains=false check=false suggest=false raw_suggest=false +"krową": contains=true check=true suggest=true raw_suggest=true +"nie": contains=false check=false suggest=false raw_suggest=false +"niekot": contains=true check=true suggest=true raw_suggest=true +"niekotem": contains=true check=true suggest=true raw_suggest=true +"niekoty": contains=true check=true suggest=true raw_suggest=true +"niepieska": contains=false check=false suggest=false raw_suggest=false +"piesek": contains=true check=true suggest=true raw_suggest=true +"pieseka": contains=false check=false suggest=false raw_suggest=false +"piesekem": contains=true check=true suggest=true raw_suggest=true +"pieseky": contains=true check=true suggest=true raw_suggest=true +"pieska": contains=true check=true suggest=true raw_suggest=true +"warszawa": contains=false check=false suggest=false raw_suggest=false +"warszawą": contains=false check=false suggest=false raw_suggest=false +pt: UTF-8 flags, forbidden words, two suffixes +len=4 rules=3 +try="aeiouçãõ" key="" map=[] rep=[] comments=["original compatibility fixture"] +forms "bonito": ["bonita", "bonitas", "bonito", "bonitos"] +forms "casa": ["casa", "casas"] +forms "fazer": ["fazer", "refazer"] +forms "imundo": [] +"BONITA": contains=false check=true suggest=true raw_suggest=false +"BONITAS": contains=false check=true suggest=true raw_suggest=false +"BONITASS": contains=false check=false suggest=false raw_suggest=false +"BONITO": contains=false check=true suggest=true raw_suggest=false +"BONITOS": contains=false check=true suggest=true raw_suggest=false +"Bonita": contains=false check=true suggest=true raw_suggest=false +"Bonitas": contains=false check=true suggest=true raw_suggest=false +"Bonitass": contains=false check=false suggest=false raw_suggest=false +"Bonito": contains=false check=true suggest=true raw_suggest=false +"Bonitos": contains=false check=true suggest=true raw_suggest=false +"CASA": contains=false check=true suggest=true raw_suggest=false +"CASAA": contains=false check=false suggest=false raw_suggest=false +"CASAS": contains=false check=true suggest=true raw_suggest=false +"Casa": contains=false check=true suggest=true raw_suggest=false +"Casaa": contains=false check=false suggest=false raw_suggest=false +"Casas": contains=false check=true suggest=true raw_suggest=false +"FAZER": contains=false check=true suggest=true raw_suggest=false +"Fazer": contains=false check=true suggest=true raw_suggest=false +"IMUNDO": contains=false check=false suggest=false raw_suggest=false +"IMUNDOS": contains=false check=false suggest=false raw_suggest=false +"Imundo": contains=false check=false suggest=false raw_suggest=false +"Imundos": contains=false check=false suggest=false raw_suggest=false +"RECASAS": contains=false check=false suggest=false raw_suggest=false +"REFAZER": contains=false check=true suggest=true raw_suggest=false +"Recasas": contains=false check=false suggest=false raw_suggest=false +"Refazer": contains=false check=true suggest=true raw_suggest=false +"bonita": contains=true check=true suggest=true raw_suggest=true +"bonitas": contains=true check=true suggest=true raw_suggest=true +"bonitass": contains=false check=false suggest=false raw_suggest=false +"bonito": contains=true check=true suggest=true raw_suggest=true +"bonitos": contains=true check=true suggest=true raw_suggest=true +"casa": contains=true check=true suggest=true raw_suggest=true +"casaa": contains=false check=false suggest=false raw_suggest=false +"casas": contains=true check=true suggest=true raw_suggest=true +"fazer": contains=true check=true suggest=true raw_suggest=true +"imundo": contains=false check=false suggest=false raw_suggest=false +"imundos": contains=false check=false suggest=false raw_suggest=false +"recasas": contains=false check=false suggest=false raw_suggest=false +"refazer": contains=true check=true suggest=true raw_suggest=true +it: stripping, non-crossing suffix, proper-name casing +len=4 rules=3 +try="" key="" map=[] rep=[] comments=["original compatibility fixture"] +forms "Roma": ["Roma"] +forms "casa": ["casa", "case", "ricasa"] +forms "fare": ["fare", "rifare"] +forms "libro": ["libri", "libro"] +"": contains=false check=false suggest=false raw_suggest=false +"CASA": contains=false check=true suggest=true raw_suggest=false +"CASE": contains=false check=true suggest=true raw_suggest=false +"Casa": contains=false check=true suggest=true raw_suggest=false +"Case": contains=false check=true suggest=true raw_suggest=false +"FARE": contains=false check=true suggest=true raw_suggest=false +"Fare": contains=false check=true suggest=true raw_suggest=false +"LIBRI": contains=false check=true suggest=true raw_suggest=false +"LIBRO": contains=false check=true suggest=true raw_suggest=false +"LIBROS": contains=false check=false suggest=false raw_suggest=false +"Libri": contains=false check=true suggest=true raw_suggest=false +"Libro": contains=false check=true suggest=true raw_suggest=false +"Libros": contains=false check=false suggest=false raw_suggest=false +"R ILIBRI": contains=false check=false suggest=false raw_suggest=false +"R ilibri": contains=false check=false suggest=false raw_suggest=false +"RICAS A": contains=false check=false suggest=false raw_suggest=false +"RICASA": contains=false check=true suggest=true raw_suggest=false +"RICASE": contains=false check=false suggest=false raw_suggest=false +"RIFARE": contains=false check=true suggest=true raw_suggest=false +"ROMA": contains=false check=true suggest=true raw_suggest=false +"Ricas a": contains=false check=false suggest=false raw_suggest=false +"Ricasa": contains=false check=true suggest=true raw_suggest=false +"Ricase": contains=false check=false suggest=false raw_suggest=false +"Rifare": contains=false check=true suggest=true raw_suggest=false +"Roma": contains=true check=true suggest=true raw_suggest=true +"casa": contains=true check=true suggest=true raw_suggest=true +"case": contains=true check=true suggest=true raw_suggest=true +"fare": contains=true check=true suggest=true raw_suggest=true +"libri": contains=true check=true suggest=true raw_suggest=true +"libro": contains=true check=true suggest=true raw_suggest=true +"libros": contains=false check=false suggest=false raw_suggest=false +"r ilibri": contains=false check=false suggest=false raw_suggest=false +"ricas a": contains=false check=false suggest=false raw_suggest=false +"ricasa": contains=true check=true suggest=true raw_suggest=true +"ricase": contains=false check=false suggest=false raw_suggest=false +"rifare": contains=true check=true suggest=true raw_suggest=true +"roma": contains=false check=false suggest=false raw_suggest=false +cs: negated conditions and shared Unicode stems +len=4 rules=3 +try="" key="" map=[] rep=[] comments=["original compatibility fixture"] +forms "mladý": ["mladý", "nemladý"] +forms "strom": ["nestrom", "nestromy", "strom", "stromy"] +forms "y": ["y"] +forms "žena": ["žena", "ženě"] +"MLADÝ": contains=false check=true suggest=true raw_suggest=false +"MLADÝY": contains=false check=false suggest=false raw_suggest=false +"Mladý": contains=false check=true suggest=true raw_suggest=false +"Mladýy": contains=false check=false suggest=false raw_suggest=false +"NEMLADÝ": contains=false check=true suggest=true raw_suggest=false +"NESTROM": contains=false check=true suggest=true raw_suggest=false +"NESTROMY": contains=false check=true suggest=true raw_suggest=false +"NEŽENĚ": contains=false check=false suggest=false raw_suggest=false +"Nemladý": contains=false check=true suggest=true raw_suggest=false +"Nestrom": contains=false check=true suggest=true raw_suggest=false +"Nestromy": contains=false check=true suggest=true raw_suggest=false +"Neženě": contains=false check=false suggest=false raw_suggest=false +"STROM": contains=false check=true suggest=true raw_suggest=false +"STROMY": contains=false check=true suggest=true raw_suggest=false +"Strom": contains=false check=true suggest=true raw_suggest=false +"Stromy": contains=false check=true suggest=true raw_suggest=false +"Y": contains=false check=true suggest=true raw_suggest=false +"YY": contains=false check=false suggest=false raw_suggest=false +"Yy": contains=false check=false suggest=false raw_suggest=false +"mladý": contains=true check=true suggest=true raw_suggest=true +"mladýy": contains=false check=false suggest=false raw_suggest=false +"nemladý": contains=true check=true suggest=true raw_suggest=true +"nestrom": contains=true check=true suggest=true raw_suggest=true +"nestromy": contains=true check=true suggest=true raw_suggest=true +"neženě": contains=false check=false suggest=false raw_suggest=false +"strom": contains=true check=true suggest=true raw_suggest=true +"stromy": contains=true check=true suggest=true raw_suggest=true +"y": contains=true check=true suggest=true raw_suggest=true +"yy": contains=false check=false suggest=false raw_suggest=false +"ŽENA": contains=false check=true suggest=true raw_suggest=false +"ŽENY": contains=false check=false suggest=false raw_suggest=false +"ŽENĚ": contains=false check=true suggest=true raw_suggest=false +"Žena": contains=false check=true suggest=true raw_suggest=false +"Ženy": contains=false check=false suggest=false raw_suggest=false +"Ženě": contains=false check=true suggest=true raw_suggest=false +"žena": contains=true check=true suggest=true raw_suggest=true +"ženy": contains=false check=false suggest=false raw_suggest=false +"ženě": contains=true check=true suggest=true raw_suggest=true +ru: Cyrillic flags and suggestions exclusion +len=4 rules=2 +try="" key="" map=[['е', 'ё']] rep=[] comments=["original compatibility fixture"] +forms "дом": ["дом", "домы"] +forms "запрет": [] +forms "кот": ["кот", "коты", "некот", "некоты"] +forms "тайна": ["тайна", "тайнаы"] +"ДОМ": contains=false check=true suggest=true raw_suggest=false +"ДОМЫ": contains=false check=true suggest=true raw_suggest=false +"Дом": contains=false check=true suggest=true raw_suggest=false +"Домы": contains=false check=true suggest=true raw_suggest=false +"ЗАПРЕТ": contains=false check=false suggest=false raw_suggest=false +"ЗАПРЕТЫ": contains=false check=false suggest=false raw_suggest=false +"Запрет": contains=false check=false suggest=false raw_suggest=false +"Запреты": contains=false check=false suggest=false raw_suggest=false +"КОТ": contains=false check=true suggest=true raw_suggest=false +"КОТЫ": contains=false check=true suggest=true raw_suggest=false +"КОт": contains=false check=false suggest=false raw_suggest=false +"Кот": contains=false check=true suggest=true raw_suggest=false +"Коты": contains=false check=true suggest=true raw_suggest=false +"НЕДОМЫ": contains=false check=false suggest=false raw_suggest=false +"НЕКОТ": contains=false check=true suggest=true raw_suggest=false +"НЕКОТЫ": contains=false check=true suggest=true raw_suggest=false +"Недомы": contains=false check=false suggest=false raw_suggest=false +"Некот": contains=false check=true suggest=true raw_suggest=false +"Некоты": contains=false check=true suggest=true raw_suggest=false +"ТАЙНА": contains=false check=true suggest=false raw_suggest=false +"ТАЙНАЫ": contains=false check=true suggest=false raw_suggest=false +"Тайна": contains=false check=true suggest=false raw_suggest=false +"Тайнаы": contains=false check=true suggest=false raw_suggest=false +"дом": contains=true check=true suggest=true raw_suggest=true +"домы": contains=true check=true suggest=true raw_suggest=true +"запрет": contains=false check=false suggest=false raw_suggest=false +"запреты": contains=false check=false suggest=false raw_suggest=false +"кОт": contains=false check=false suggest=false raw_suggest=false +"кот": contains=true check=true suggest=true raw_suggest=true +"коты": contains=true check=true suggest=true raw_suggest=true +"недомы": contains=false check=false suggest=false raw_suggest=false +"некот": contains=true check=true suggest=true raw_suggest=true +"некоты": contains=true check=true suggest=true raw_suggest=true +"тайна": contains=true check=true suggest=false raw_suggest=false +"тайнаы": contains=true check=true suggest=false raw_suggest=false +id: continuation and NOSUGGEST through derivations +len=3 rules=3 +try="abcdeikn" key="qwerty|asdf" map=[] rep=[("dj", "j")] comments=["original compatibility fixture"] +forms "baca": ["baca", "bacakan", "bacakanlah"] +forms "main": ["bermain", "bermainkan", "main", "mainkan", "mainkanlah"] +forms "rahasia": ["rahasia", "rahasiakan", "rahasiakanlah"] +"BACA": contains=false check=true suggest=true raw_suggest=false +"BACAKAN": contains=false check=true suggest=true raw_suggest=false +"BACAKANLAH": contains=false check=true suggest=true raw_suggest=false +"BERBACA": contains=false check=false suggest=false raw_suggest=false +"BERMAIN": contains=false check=true suggest=true raw_suggest=false +"BERMAINKAN": contains=false check=true suggest=true raw_suggest=false +"BERMAINKANLAH": contains=false check=false suggest=false raw_suggest=false +"Baca": contains=false check=true suggest=true raw_suggest=false +"Bacakan": contains=false check=true suggest=true raw_suggest=false +"Bacakanlah": contains=false check=true suggest=true raw_suggest=false +"Berbaca": contains=false check=false suggest=false raw_suggest=false +"Bermain": contains=false check=true suggest=true raw_suggest=false +"Bermainkan": contains=false check=true suggest=true raw_suggest=false +"Bermainkanlah": contains=false check=false suggest=false raw_suggest=false +"MAIN": contains=false check=true suggest=true raw_suggest=false +"MAINKAN": contains=false check=true suggest=true raw_suggest=false +"MAINKANLAH": contains=false check=true suggest=true raw_suggest=false +"MAINKANLAHLAH": contains=false check=false suggest=false raw_suggest=false +"MAINLAH": contains=false check=false suggest=false raw_suggest=false +"Main": contains=false check=true suggest=true raw_suggest=false +"Mainkan": contains=false check=true suggest=true raw_suggest=false +"Mainkanlah": contains=false check=true suggest=true raw_suggest=false +"Mainkanlahlah": contains=false check=false suggest=false raw_suggest=false +"Mainlah": contains=false check=false suggest=false raw_suggest=false +"RAHASIA": contains=false check=true suggest=false raw_suggest=false +"RAHASIAKAN": contains=false check=true suggest=false raw_suggest=false +"RAHASIAKANLAH": contains=false check=true suggest=false raw_suggest=false +"Rahasia": contains=false check=true suggest=false raw_suggest=false +"Rahasiakan": contains=false check=true suggest=false raw_suggest=false +"Rahasiakanlah": contains=false check=true suggest=false raw_suggest=false +"baca": contains=true check=true suggest=true raw_suggest=true +"bacakan": contains=true check=true suggest=true raw_suggest=true +"bacakanlah": contains=true check=true suggest=true raw_suggest=true +"berbaca": contains=false check=false suggest=false raw_suggest=false +"bermain": contains=true check=true suggest=true raw_suggest=true +"bermainkan": contains=true check=true suggest=true raw_suggest=true +"bermainkanlah": contains=false check=false suggest=false raw_suggest=false +"main": contains=true check=true suggest=true raw_suggest=true +"mainkan": contains=true check=true suggest=true raw_suggest=true +"mainkanlah": contains=true check=true suggest=true raw_suggest=true +"mainkanlahlah": contains=false check=false suggest=false raw_suggest=false +"mainlah": contains=false check=false suggest=false raw_suggest=false +"rahasia": contains=true check=true suggest=false raw_suggest=false +"rahasiakan": contains=true check=true suggest=false raw_suggest=false +"rahasiakanlah": contains=true check=true suggest=false raw_suggest=false +pt-sized: 70 distinct flags across the old u128 boundary +len=2 rules=70 +try="" key="" map=[] rep=[] comments=["original compatibility fixture"] +forms "outro": ["outro", "outroõ"] +forms "verbo": ["verbo", "verbo1", "verbo2", "verbo3", "verbo4", "verbo5", "verbo6", "verbo7", "verbo8", "verbo9", "verboA", "verboB", "verboC", "verboD", "verboE", "verboF", "verboG", "verboH", "verboI", "verboJ", "verboK", "verboL", "verboM", "verboN", "verboO", "verboP", "verboQ", "verboR", "verboS", "verboT", "verboU", "verboV", "verboW", "verboX", "verboY", "verboZ", "verboa", "verbob", "verboc", "verbod", "verboe", "verbof", "verbog", "verboh", "verboi", "verboj", "verbok", "verbol", "verbom", "verbon", "verboo", "verbop", "verboq", "verbor", "verbos", "verbot", "verbou", "verbov", "verbow", "verbox", "verboy", "verboz", "verboá", "verboã", "verboç", "verboé", "verboí", "verboó", "verboõ", "verboú"] +"OUTRO": contains=false check=true suggest=true raw_suggest=false +"OUTROA": contains=false check=false suggest=false raw_suggest=false +"OUTROÕ": contains=false check=true suggest=true raw_suggest=false +"Outro": contains=false check=true suggest=true raw_suggest=false +"Outroa": contains=false check=false suggest=false raw_suggest=false +"Outroõ": contains=false check=true suggest=true raw_suggest=false +"VERBO": contains=false check=true suggest=true raw_suggest=false +"VERBO0": contains=false check=false suggest=false raw_suggest=false +"VERBO1": contains=false check=true suggest=true raw_suggest=false +"VERBO2": contains=false check=true suggest=true raw_suggest=false +"VERBO3": contains=false check=true suggest=true raw_suggest=false +"VERBO4": contains=false check=true suggest=true raw_suggest=false +"VERBO5": contains=false check=true suggest=true raw_suggest=false +"VERBO6": contains=false check=true suggest=true raw_suggest=false +"VERBO7": contains=false check=true suggest=true raw_suggest=false +"VERBO8": contains=false check=true suggest=true raw_suggest=false +"VERBO9": contains=false check=true suggest=true raw_suggest=false +"VERBOA": contains=false check=true suggest=true raw_suggest=false +"VERBOB": contains=false check=true suggest=true raw_suggest=false +"VERBOC": contains=false check=true suggest=true raw_suggest=false +"VERBOD": contains=false check=true suggest=true raw_suggest=false +"VERBOE": contains=false check=true suggest=true raw_suggest=false +"VERBOF": contains=false check=true suggest=true raw_suggest=false +"VERBOG": contains=false check=true suggest=true raw_suggest=false +"VERBOH": contains=false check=true suggest=true raw_suggest=false +"VERBOI": contains=false check=true suggest=true raw_suggest=false +"VERBOJ": contains=false check=true suggest=true raw_suggest=false +"VERBOK": contains=false check=true suggest=true raw_suggest=false +"VERBOL": contains=false check=true suggest=true raw_suggest=false +"VERBOM": contains=false check=true suggest=true raw_suggest=false +"VERBON": contains=false check=true suggest=true raw_suggest=false +"VERBOO": contains=false check=true suggest=true raw_suggest=false +"VERBOP": contains=false check=true suggest=true raw_suggest=false +"VERBOQ": contains=false check=true suggest=true raw_suggest=false +"VERBOR": contains=false check=true suggest=true raw_suggest=false +"VERBOS": contains=false check=true suggest=true raw_suggest=false +"VERBOSS": contains=false check=false suggest=false raw_suggest=false +"VERBOT": contains=false check=true suggest=true raw_suggest=false +"VERBOU": contains=false check=true suggest=true raw_suggest=false +"VERBOV": contains=false check=true suggest=true raw_suggest=false +"VERBOW": contains=false check=true suggest=true raw_suggest=false +"VERBOX": contains=false check=true suggest=true raw_suggest=false +"VERBOY": contains=false check=true suggest=true raw_suggest=false +"VERBOZ": contains=false check=true suggest=true raw_suggest=false +"VERBOÁ": contains=false check=true suggest=true raw_suggest=false +"VERBOÃ": contains=false check=true suggest=true raw_suggest=false +"VERBOÇ": contains=false check=true suggest=true raw_suggest=false +"VERBOÉ": contains=false check=true suggest=true raw_suggest=false +"VERBOÍ": contains=false check=true suggest=true raw_suggest=false +"VERBOÓ": contains=false check=true suggest=true raw_suggest=false +"VERBOÕ": contains=false check=true suggest=true raw_suggest=false +"VERBOÚ": contains=false check=true suggest=true raw_suggest=false +"Verbo": contains=false check=true suggest=true raw_suggest=false +"Verbo0": contains=false check=false suggest=false raw_suggest=false +"Verbo1": contains=false check=true suggest=true raw_suggest=false +"Verbo2": contains=false check=true suggest=true raw_suggest=false +"Verbo3": contains=false check=true suggest=true raw_suggest=false +"Verbo4": contains=false check=true suggest=true raw_suggest=false +"Verbo5": contains=false check=true suggest=true raw_suggest=false +"Verbo6": contains=false check=true suggest=true raw_suggest=false +"Verbo7": contains=false check=true suggest=true raw_suggest=false +"Verbo8": contains=false check=true suggest=true raw_suggest=false +"Verbo9": contains=false check=true suggest=true raw_suggest=false +"VerboA": contains=false check=false suggest=false raw_suggest=false +"VerboB": contains=false check=false suggest=false raw_suggest=false +"VerboC": contains=false check=false suggest=false raw_suggest=false +"VerboD": contains=false check=false suggest=false raw_suggest=false +"VerboE": contains=false check=false suggest=false raw_suggest=false +"VerboF": contains=false check=false suggest=false raw_suggest=false +"VerboG": contains=false check=false suggest=false raw_suggest=false +"VerboH": contains=false check=false suggest=false raw_suggest=false +"VerboI": contains=false check=false suggest=false raw_suggest=false +"VerboJ": contains=false check=false suggest=false raw_suggest=false +"VerboK": contains=false check=false suggest=false raw_suggest=false +"VerboL": contains=false check=false suggest=false raw_suggest=false +"VerboM": contains=false check=false suggest=false raw_suggest=false +"VerboN": contains=false check=false suggest=false raw_suggest=false +"VerboO": contains=false check=false suggest=false raw_suggest=false +"VerboP": contains=false check=false suggest=false raw_suggest=false +"VerboQ": contains=false check=false suggest=false raw_suggest=false +"VerboR": contains=false check=false suggest=false raw_suggest=false +"VerboS": contains=false check=false suggest=false raw_suggest=false +"VerboT": contains=false check=false suggest=false raw_suggest=false +"VerboU": contains=false check=false suggest=false raw_suggest=false +"VerboV": contains=false check=false suggest=false raw_suggest=false +"VerboW": contains=false check=false suggest=false raw_suggest=false +"VerboX": contains=false check=false suggest=false raw_suggest=false +"VerboY": contains=false check=false suggest=false raw_suggest=false +"VerboZ": contains=false check=false suggest=false raw_suggest=false +"Verboa": contains=false check=true suggest=true raw_suggest=false +"Verbob": contains=false check=true suggest=true raw_suggest=false +"Verboc": contains=false check=true suggest=true raw_suggest=false +"Verbod": contains=false check=true suggest=true raw_suggest=false +"Verboe": contains=false check=true suggest=true raw_suggest=false +"Verbof": contains=false check=true suggest=true raw_suggest=false +"Verbog": contains=false check=true suggest=true raw_suggest=false +"Verboh": contains=false check=true suggest=true raw_suggest=false +"Verboi": contains=false check=true suggest=true raw_suggest=false +"Verboj": contains=false check=true suggest=true raw_suggest=false +"Verbok": contains=false check=true suggest=true raw_suggest=false +"Verbol": contains=false check=true suggest=true raw_suggest=false +"Verbom": contains=false check=true suggest=true raw_suggest=false +"Verbon": contains=false check=true suggest=true raw_suggest=false +"Verboo": contains=false check=true suggest=true raw_suggest=false +"Verbop": contains=false check=true suggest=true raw_suggest=false +"Verboq": contains=false check=true suggest=true raw_suggest=false +"Verbor": contains=false check=true suggest=true raw_suggest=false +"Verbos": contains=false check=true suggest=true raw_suggest=false +"Verbot": contains=false check=true suggest=true raw_suggest=false +"Verbou": contains=false check=true suggest=true raw_suggest=false +"Verbov": contains=false check=true suggest=true raw_suggest=false +"Verbow": contains=false check=true suggest=true raw_suggest=false +"Verbox": contains=false check=true suggest=true raw_suggest=false +"Verboy": contains=false check=true suggest=true raw_suggest=false +"Verboz": contains=false check=true suggest=true raw_suggest=false +"Verboß": contains=false check=false suggest=false raw_suggest=false +"Verboá": contains=false check=true suggest=true raw_suggest=false +"Verboã": contains=false check=true suggest=true raw_suggest=false +"Verboç": contains=false check=true suggest=true raw_suggest=false +"Verboé": contains=false check=true suggest=true raw_suggest=false +"Verboí": contains=false check=true suggest=true raw_suggest=false +"Verboó": contains=false check=true suggest=true raw_suggest=false +"Verboõ": contains=false check=true suggest=true raw_suggest=false +"Verboú": contains=false check=true suggest=true raw_suggest=false +"outro": contains=true check=true suggest=true raw_suggest=true +"outroa": contains=false check=false suggest=false raw_suggest=false +"outroõ": contains=true check=true suggest=true raw_suggest=true +"verbo": contains=true check=true suggest=true raw_suggest=true +"verbo0": contains=false check=false suggest=false raw_suggest=false +"verbo1": contains=true check=true suggest=true raw_suggest=true +"verbo2": contains=true check=true suggest=true raw_suggest=true +"verbo3": contains=true check=true suggest=true raw_suggest=true +"verbo4": contains=true check=true suggest=true raw_suggest=true +"verbo5": contains=true check=true suggest=true raw_suggest=true +"verbo6": contains=true check=true suggest=true raw_suggest=true +"verbo7": contains=true check=true suggest=true raw_suggest=true +"verbo8": contains=true check=true suggest=true raw_suggest=true +"verbo9": contains=true check=true suggest=true raw_suggest=true +"verboA": contains=true check=true suggest=true raw_suggest=true +"verboB": contains=true check=true suggest=true raw_suggest=true +"verboC": contains=true check=true suggest=true raw_suggest=true +"verboD": contains=true check=true suggest=true raw_suggest=true +"verboE": contains=true check=true suggest=true raw_suggest=true +"verboF": contains=true check=true suggest=true raw_suggest=true +"verboG": contains=true check=true suggest=true raw_suggest=true +"verboH": contains=true check=true suggest=true raw_suggest=true +"verboI": contains=true check=true suggest=true raw_suggest=true +"verboJ": contains=true check=true suggest=true raw_suggest=true +"verboK": contains=true check=true suggest=true raw_suggest=true +"verboL": contains=true check=true suggest=true raw_suggest=true +"verboM": contains=true check=true suggest=true raw_suggest=true +"verboN": contains=true check=true suggest=true raw_suggest=true +"verboO": contains=true check=true suggest=true raw_suggest=true +"verboP": contains=true check=true suggest=true raw_suggest=true +"verboQ": contains=true check=true suggest=true raw_suggest=true +"verboR": contains=true check=true suggest=true raw_suggest=true +"verboS": contains=true check=true suggest=true raw_suggest=true +"verboT": contains=true check=true suggest=true raw_suggest=true +"verboU": contains=true check=true suggest=true raw_suggest=true +"verboV": contains=true check=true suggest=true raw_suggest=true +"verboW": contains=true check=true suggest=true raw_suggest=true +"verboX": contains=true check=true suggest=true raw_suggest=true +"verboY": contains=true check=true suggest=true raw_suggest=true +"verboZ": contains=true check=true suggest=true raw_suggest=true +"verboa": contains=true check=true suggest=true raw_suggest=true +"verbob": contains=true check=true suggest=true raw_suggest=true +"verboc": contains=true check=true suggest=true raw_suggest=true +"verbod": contains=true check=true suggest=true raw_suggest=true +"verboe": contains=true check=true suggest=true raw_suggest=true +"verbof": contains=true check=true suggest=true raw_suggest=true +"verbog": contains=true check=true suggest=true raw_suggest=true +"verboh": contains=true check=true suggest=true raw_suggest=true +"verboi": contains=true check=true suggest=true raw_suggest=true +"verboj": contains=true check=true suggest=true raw_suggest=true +"verbok": contains=true check=true suggest=true raw_suggest=true +"verbol": contains=true check=true suggest=true raw_suggest=true +"verbom": contains=true check=true suggest=true raw_suggest=true +"verbon": contains=true check=true suggest=true raw_suggest=true +"verboo": contains=true check=true suggest=true raw_suggest=true +"verbop": contains=true check=true suggest=true raw_suggest=true +"verboq": contains=true check=true suggest=true raw_suggest=true +"verbor": contains=true check=true suggest=true raw_suggest=true +"verbos": contains=true check=true suggest=true raw_suggest=true +"verbot": contains=true check=true suggest=true raw_suggest=true +"verbou": contains=true check=true suggest=true raw_suggest=true +"verbov": contains=true check=true suggest=true raw_suggest=true +"verbow": contains=true check=true suggest=true raw_suggest=true +"verbox": contains=true check=true suggest=true raw_suggest=true +"verboy": contains=true check=true suggest=true raw_suggest=true +"verboz": contains=true check=true suggest=true raw_suggest=true +"verboß": contains=false check=false suggest=false raw_suggest=false +"verboá": contains=true check=true suggest=true raw_suggest=true +"verboã": contains=true check=true suggest=true raw_suggest=true +"verboç": contains=true check=true suggest=true raw_suggest=true +"verboé": contains=true check=true suggest=true raw_suggest=true +"verboí": contains=true check=true suggest=true raw_suggest=true +"verboó": contains=true check=true suggest=true raw_suggest=true +"verboõ": contains=true check=true suggest=true raw_suggest=true +"verboú": contains=true check=true suggest=true raw_suggest=true diff --git a/crates/proof/tests/fixtures/affix_compatibility_cases.rs b/crates/proof/tests/fixtures/affix_compatibility_cases.rs new file mode 100644 index 000000000..d0c2973b4 --- /dev/null +++ b/crates/proof/tests/fixtures/affix_compatibility_cases.rs @@ -0,0 +1,61 @@ +// Original, deliberately small fixtures for behavior supported before French proofing. +// These are not extracts from installed dictionaries and do not claim full language coverage. +pub struct Case { + pub name: &'static str, + pub aff: String, + pub dic: String, + pub probes: Vec<&'static str>, +} + +pub fn cases() -> Vec { + let mut out = vec![ + Case { + name: "pl: conditional strip, prefix cross products, diacritics", + aff: "SET UTF-8\nTRY aeoiułkr\nPFX b Y 1\nPFX b 0 nie .\nSFX a Y 3\nSFX a 0 y [^y]\nSFX a ek ka ek\nSFX a 0 em .\nSFX c N 1\nSFX c a ą a\nMAP 2\nMAP aą\nMAP oóu\nREP 2\nREP ż rz\nREP u ó\n".into(), + dic: "4\nkot/ab\npiesek/a\nkrowa/c\nWarszawa/c\n".into(), + probes: vec!["kot", "koty", "kotem", "niekoty", "pieska", "pieseka", "niepieska", "krową", "krowy", "Warszawą", "warszawą", "kotą", "nie", ""], + }, + Case { + name: "pt: UTF-8 flags, forbidden words, two suffixes", + aff: "SET UTF-8\nFLAG UTF-8\nFORBIDDENWORD ý\nTRY aeiouçãõ\nSFX A Y 1\nSFX A o a/B o\nSFX B Y 1\nSFX B 0 s .\nPFX P Y 1\nPFX P 0 re .\n".into(), + dic: "4\nbonito/AB\ncasa/B\nfazer/P\nimundo/ýB\n".into(), + probes: vec!["bonito", "bonita", "bonitos", "bonitas", "bonitass", "casa", "casas", "casaa", "refazer", "recasas", "imundo", "imundos", "Imundo", "IMUNDO"], + }, + Case { + name: "it: stripping, non-crossing suffix, proper-name casing", + aff: "SET UTF-8\nSFX A Y 1\nSFX A o i o\nSFX B N 1\nSFX B a e a\nPFX P Y 1\nPFX P 0 ri .\n".into(), + dic: "4\nlibro/A\ncasa/BP\nfare/P\nRoma\n".into(), + probes: vec!["libri", "libros", "case", "ricas a", "ricasa", "ricase", "rifare", "r ilibri", "Roma", "roma", "ROMA", ""], + }, + Case { + name: "cs: negated conditions and shared Unicode stems", + aff: "SET UTF-8\nPFX N Y 1\nPFX N 0 ne .\nSFX S Y 1\nSFX S 0 y [^y]\nSFX E N 1\nSFX E a ě a\n".into(), + dic: "4\nstrom/NS\nžena/E\nmladý/N\ny/S\n".into(), + probes: vec!["stromy", "nestromy", "nestrom", "ženě", "ženy", "neženě", "nemladý", "mladýy", "yy", "Ženě", "STROMY"], + }, + Case { + name: "ru: Cyrillic flags and suggestions exclusion", + aff: "SET UTF-8\nFLAG UTF-8\nNOSUGGEST Н\nFORBIDDENWORD Ф\nSFX Я Y 1\nSFX Я 0 ы .\nPFX П Y 1\nPFX П 0 не .\nMAP 1\nMAP её\n".into(), + dic: "4\nкот/ЯП\nдом/Я\nтайна/НЯ\nзапрет/ФЯ\n".into(), + probes: vec!["кот", "коты", "некоты", "домы", "недомы", "тайна", "тайнаы", "ТАЙНАЫ", "запрет", "запреты", "КОТЫ", "кОт"], + }, + Case { + name: "id: continuation and NOSUGGEST through derivations", + aff: "SET UTF-8\nNOSUGGEST N\nKEY qwerty|asdf\nTRY abcdeikn\nREP 1\nREP dj j\nPFX P Y 1\nPFX P 0 ber .\nSFX A Y 1\nSFX A 0 kan/B .\nSFX B Y 1\nSFX B 0 lah .\n".into(), + dic: "3\nmain/PA\nbaca/A\nrahasia/NA\n".into(), + probes: vec!["main", "bermain", "mainkan", "mainkanlah", "bermainkanlah", "mainlah", "mainkanlahlah", "bacakanlah", "berbaca", "rahasia", "rahasiakan", "RAHASIAKANLAH"], + }, + ]; + let flags: Vec = ('a'..='z').chain('A'..='Z').chain('0'..='9').chain(['á', 'é', 'í', 'ó', 'ú', 'ç', 'ã', 'õ']).collect(); + let mut aff = String::from("SET UTF-8\nFLAG UTF-8\n"); + for flag in &flags { + aff.push_str(&format!("SFX {flag} N 1\nSFX {flag} 0 {flag} .\n")); + } + out.push(Case { + name: "pt-sized: 70 distinct flags across the old u128 boundary", + aff, + dic: format!("2\nverbo/{}\noutro/õ\n", flags.iter().collect::()), + probes: vec!["verboa", "verboz", "verboA", "verboZ", "verbo0", "verbo9", "verboç", "verboõ", "verboß", "outroõ", "outroa"], + }); + out +} diff --git a/crates/proof/tests/french_dictionary.rs b/crates/proof/tests/french_dictionary.rs new file mode 100644 index 000000000..5612a0e61 --- /dev/null +++ b/crates/proof/tests/french_dictionary.rs @@ -0,0 +1,107 @@ +//! Explicit manual real-dictionary gate. Verify the pinned runtime inputs as documented in +//! crates/proof/README.md before running with `--ignored`. Missing data is a failure, +//! never a silently successful test that checked nothing. + +#![cfg(not(target_arch = "wasm32"))] +#![allow(clippy::unwrap_used, clippy::expect_used)] + +use craft_proof::{IssueKind, Options, ProofLang, fr}; + +#[test] +#[ignore = "requires a real French dictionary in WORDCRAFT_DICTIONARIES"] +fn real_french_dictionary_spelling_suggestions_and_dispatch() { + let directory = std::env::var_os("WORDCRAFT_DICTIONARIES").expect("set WORDCRAFT_DICTIONARIES to the test dictionary folder"); + craft_proof::set_dictionaries_folder(Some(std::path::PathBuf::from(&directory))); + let french = ProofLang::from_tag(Some("fr-FR")); + let installed = craft_proof::pack_dictionary_file(french).expect("the real French dictionary must be installed"); + assert_eq!(installed.parent(), Some(std::path::Path::new(&directory)), "do not silently use another system dictionary"); + let aff = installed.with_extension("aff"); + let dictionary = craft_proof::hunspell::load(&aff, &installed).expect("the real French dictionary loads"); + assert!(dictionary.len() > 50_000, "expected Dicollecte, not the synthetic fixture: {}", dictionary.len()); + for word in [ + "maison", + "maisons", + "Paris", + "mangeons", + "finissaient", + "étaient", + "l'arbre", + "l’arbre", + "qu'elles", + "école", + "œuvre", + "aujourd'hui", + "chevaux", + "heureuse", + "va-t-il", + ] { + assert!(fr::is_correct(word), "correct French word: {word}"); + } + for (typo, expected) in [("maisonn", "maison"), ("ecole", "école"), ("l’ecole", "l’école"), ("documant", "document")] { + assert!(!fr::is_correct(typo), "misspelling: {typo}"); + let suggestions = fr::suggest(typo, 8); + assert!(suggestions.iter().any(|s| s == expected), "{typo} -> {suggestions:?}, expected {expected}"); + } + let text = "Une maisonn près de l’école."; + for lang in [ProofLang::from_tag(Some("fr-FR")), ProofLang::from_tag(Some("fr-CA"))] { + let issues = craft_proof::check_text(text, &[(0..text.len(), lang)]); + assert_eq!(issues.len(), 1, "{lang:?}: {issues:?}"); + let issue = &issues[0]; + assert_eq!(&text[issue.start..issue.end], "maisonn"); + assert_eq!(issue.lang, lang); + assert_eq!(issue.message, "Faute d’orthographe possible"); + } + check_combining_marks_and_compounds(&dictionary); + assert!(craft_proof::dictionary_load_error(ProofLang::from_tag(Some("fr-FR"))).is_none()); +} + +fn check_combining_marks_and_compounds(dictionary: &craft_proof::affix::AffixDict) { + let spelling = |text: &str, spans: &[(std::ops::Range, ProofLang)]| { + craft_proof::check_text_with(text, spans, &Options::default(), false, false) + .into_iter() + .filter(|i| i.kind == IssueKind::Spelling) + .map(|i| (i.start, i.end, i.lang)) + .collect::>() + }; + let mut failures = Vec::new(); + for lang in [ProofLang::from_tag(Some("fr-FR")), ProofLang::from_tag(Some("fr-CA"))] { + for text in + ["école", "e\u{301}cole", "l’école", "l’e\u{301}cole", "café", "cafe\u{301}", "porte-monnaie", "va-t-il", "a-t-il", "maison-école"] + { + assert!( + fr::is_correct_with(dictionary, text) || ["va-t-il", "a-t-il", "maison-école"].contains(&text), + "real dictionary recognizes {text:?}" + ); + let issues = spelling(text, &[(0..text.len(), lang)]); + if !issues.is_empty() { + failures.push(format!("correct {lang:?} {text:?}: {issues:?}")); + } + } + for text in ["porte--monnaie", "va--t-il", "maison--école", "e\u{301}colllle", "cafffe\u{301}"] { + assert!(!fr::is_correct_with(dictionary, text), "the explicit dictionary entry must not already accept {text:?}"); + let issues = spelling(text, &[(0..text.len(), lang)]); + let expected = vec![(0, text.len(), lang)]; + if issues != expected { + failures.push(format!("typo {lang:?} {text:?}: {issues:?}, wanted {expected:?}")); + } + } + let text = "e\u{301}cole cafe\u{301} xyzxyz porte--monnaie"; + let foreign = text.find("xyzxyz").unwrap(); + let compound = text.find("porte").unwrap(); + let spans = [(0..foreign, lang), (foreign..compound, ProofLang::Other), (compound..text.len(), lang)]; + let issues = spelling(text, &spans); + if issues != [(compound, text.len(), lang)] { + failures.push(format!("mixed {lang:?}: {issues:?}")); + } + assert!(spelling(text, &[(0..text.len(), ProofLang::Other)]).is_empty()); + // An English word remains in its own dictionary next to the original French bytes. + let text = "e\u{301}cole hello cafe\u{301}"; + let en = text.find("hello").unwrap(); + let spans = [(0..en, lang), (en..en + 5, ProofLang::En), (en + 5..text.len(), lang)]; + let issues = spelling(text, &spans); + if !issues.is_empty() { + failures.push(format!("English boundary {lang:?}: {issues:?}")); + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} diff --git a/crates/proof/tests/french_packs.rs b/crates/proof/tests/french_packs.rs new file mode 100644 index 000000000..64ad46748 --- /dev/null +++ b/crates/proof/tests/french_packs.rs @@ -0,0 +1,124 @@ +//! French uses ordinary Language packs and keeps each regional locale at the public API. +#![allow(clippy::unwrap_used, clippy::expect_used)] + +use craft_proof::affix; +use craft_proof::{IssueKind, Options, ProofLang, check_text_with, is_correct_in_with, suggest_in}; +use std::path::{Path, PathBuf}; + +#[path = "../src/fr/test_dict.rs"] +mod fixture; + +struct Tools(PathBuf); +impl Drop for Tools { + fn drop(&mut self) { + craft_proof::set_dictionaries_folder(None); + craft_proof::packs::set_packs_dir(None); + craft_proof::remove_word("zxqrechargefr"); + let _ = std::fs::remove_dir_all(&self.0); + } +} + +fn pair(dir: &Path, name: &str, aff: &str, dic: &str) { + std::fs::create_dir_all(dir).unwrap(); + std::fs::write(dir.join(format!("{name}.aff")), aff).unwrap(); + std::fs::write(dir.join(format!("{name}.dic")), dic).unwrap(); +} + +#[test] +fn french_packs_regions_reload_and_original_ranges() { + assert!(!fixture::dictionary().is_empty()); + let root = std::env::temp_dir().join(format!("craft-fr-packs-{}", std::process::id())); + let _tools = Tools(root.clone()); + let pack = root.join("packs/fr"); + pair(&pack, "spelling", fixture::AFF, fixture::DIC); + std::fs::write(pack.join("pack.toml"), "locale = 'fr'\nname = 'Français'\nlicence = 'MIT'\n").unwrap(); + std::fs::write(pack.join("hyphenation.pat"), "ma1ison\n").unwrap(); + let folder = root.join("dictionaries"); + std::fs::create_dir_all(&folder).unwrap(); + craft_proof::packs::set_packs_dir(Some(root.join("packs"))); + craft_proof::set_dictionaries_folder(Some(folder.clone())); + let defaults = Options::default(); + let strict = Options { ignore_uppercase: false, ignore_numbers: false, ..defaults }; + check_locales(defaults, strict); + check_typography(strict); + let fr = ProofLang::from_tag(Some("fr-FR")); + let ca = ProofLang::from_tag(Some("fr-CA")); + check_user_word(fr, ca, strict); + check_overrides(folder, fr, ca, strict); +} + +fn check_locales(defaults: Options, strict: Options) { + for tag in ["fr", "fr-FR", "fr-CA", "fr-BE", "fr-CA-u-nu-latn", "fr-u-ca-gregory"] { + let lang = ProofLang::from_tag(Some(tag)); + assert!(matches!(lang, ProofLang::Pack(_)), "{tag}"); + assert!(craft_proof::has_dictionary(Some(tag)), "{tag}"); + assert!(craft_proof::pack_dictionary_file(lang).unwrap().ends_with("fr/spelling.dic"), "{tag}"); + for word in ["école", "e\u{301}cole", "l'arbre", "l’arbre", "maison-école"] { + assert!(is_correct_in_with(word, lang, &strict), "{tag}: {word}"); + } + assert!(!is_correct_in_with("maison--école", lang, &strict)); + assert!(!is_correct_in_with("l'arbres", lang, &strict)); + assert!(is_correct_in_with("SNCF", lang, &defaults)); + assert!(!is_correct_in_with("SNCF", lang, &strict)); + assert!(is_correct_in_with("mp3x", lang, &defaults)); + assert!(!is_correct_in_with("mp3x", lang, &strict)); + assert!(suggest_in("ecole", 6, lang).iter().any(|s| s == "école")); + let text = "🙂 maison\u{338} école"; + let issues = check_text_with(text, &[(0..text.len(), lang)], &strict, false, false); + assert_eq!(issues.len(), 1, "{tag}: {issues:?}"); + assert_eq!(&text[issues[0].start..issues[0].end], "maison\u{338}"); + assert_eq!(issues[0].lang, lang); + assert_eq!(issues[0].kind, IssueKind::Spelling); + assert_eq!(issues[0].message, "Faute d’orthographe possible"); + } +} + +fn check_typography(strict: Options) { + let fr = ProofLang::from_tag(Some("fr-FR")); + let ca = ProofLang::from_tag(Some("fr-CA")); + for lang in [fr, ca] { + assert_eq!(craft_proof::hyphen::hyphen_points_in("maison", &craft_proof::hyphen::Limits::default(), lang), [2]); + } + for tag in ["fr-FR", "fr-BE", "fr-u-ca-gregory", "fr-FR-u-rg-cazzzz"] { + let issues = craft_proof::check_grammar_in("Bonjour!", ProofLang::from_tag(Some(tag))); + assert_eq!(issues.len(), 1, "{tag}: {issues:?}"); + assert_eq!(issues[0].suggestions, ["\u{202f}!"]); + } + assert!(craft_proof::check_grammar_in("Bonjour!", ca).is_empty()); + assert!(craft_proof::check_grammar_in("Bonjour!", ProofLang::from_tag(Some("fr-Latn-CA"))).is_empty()); + assert!(craft_proof::check_grammar_in("Bonjour!", ProofLang::from_tag(Some("nl-NL"))).is_empty()); + assert!(craft_proof::fr::grammar::check_in("Bonjour!", ProofLang::En).is_empty()); + let text = "Bonjour! Salut!"; + let issues = check_text_with(text, &[(0..8, fr), (8..text.len(), ca)], &strict, true, false); + let spacing: Vec<_> = issues.iter().filter(|i| i.kind == IssueKind::Grammar && i.message.contains("fine insécable")).collect(); + assert_eq!(spacing.len(), 1, "{issues:?}"); + assert_eq!(spacing[0].lang, fr); +} + +fn check_user_word(fr: ProofLang, ca: ProofLang, strict: Options) { + craft_proof::add_word("zxqrechargefr"); + for lang in [fr, ca] { + assert!(is_correct_in_with("l’zxqrechargefr", lang, &strict)); + } + craft_proof::remove_word("zxqrechargefr"); + assert!(!is_correct_in_with("l’zxqrechargefr", ca, &strict)); +} + +fn check_overrides(folder: PathBuf, fr: ProofLang, ca: ProofLang, strict: Options) { + // Regional overrides keep the generic pack fallback and the shared reload generation. + pair(&folder, "fr_FR", "SET UTF-8\n", "1\nhexagonefixture\n"); + pair(&folder, "fr_CA", "SET UTF-8\n", "1\nquebecfixture\n"); + let generation = craft_proof::dictionary_generation(); + craft_proof::set_dictionaries_folder(Some(folder.clone())); + assert!(craft_proof::dictionary_generation() > generation); + assert!(is_correct_in_with("hexagonefixture", fr, &strict)); + assert!(!is_correct_in_with("hexagonefixture", ca, &strict)); + assert!(is_correct_in_with("quebecfixture", ca, &strict)); + assert!(!is_correct_in_with("quebecfixture", fr, &strict)); + assert!(craft_proof::pack_dictionary_file(ca).unwrap().ends_with("fr_CA.dic")); + std::fs::remove_file(folder.join("fr_CA.dic")).unwrap(); + std::fs::remove_file(folder.join("fr_CA.aff")).unwrap(); + craft_proof::set_dictionaries_folder(Some(folder)); + assert!(is_correct_in_with("école", ca, &strict), "removed regional override falls back to fr pack"); + assert!(craft_proof::pack_dictionary_file(ca).unwrap().ends_with("fr/spelling.dic")); +} From 3c5ffadd3b3fb4bcbf657a245ad9c480d2b6e304 Mon Sep 17 00:00:00 2001 From: totoz64 <47389794+totoz64@users.noreply.github.com> Date: Tue, 3 Nov 2026 11:53:24 +0100 Subject: [PATCH 002/473] Proofing: retain exact apostrophes in French compounds Check each original compound component before its ASCII-apostrophe alias. Add a curly-only dictionary regression, prove legacy fixture headers before rejecting WCAFF1, and clarify French-specific Pack grammar in the README. Preserve the existing one-letter component policy. --- crates/proof/README.md | 13 ++++++++----- crates/proof/src/fr/mod.rs | 14 +++++++++++++- crates/proof/tests/affix_compatibility.rs | 2 ++ 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/crates/proof/README.md b/crates/proof/README.md index b4b4b5ba8..4cc4b2ca4 100644 --- a/crates/proof/README.md +++ b/crates/proof/README.md @@ -89,19 +89,22 @@ pub fn load_errors() -> Vec<(String, String)>; // + (locale or folder, pub fn embedded_locales() -> &'static [&'static str]; // + locales with embedded upstream data, not our supplements ("en-us", "en-gb", "en-ca", "en-au", "pl", "pt", "it", "de" by default; the four English ones with CRAFT_PROOF_EMBED=en) // craft_proof (crate root) -pub enum ProofLang { …, Pack(PackLocale), Other } // + Pack: any language without tools of its own (nl, fr, sr-Latn…) +pub enum ProofLang { …, Pack(PackLocale), Other } // + Pack: a language without a dedicated built-in variant (nl, fr, sr-Latn…) pub struct PackLocale; // + Copy; PackLocale::new(tag) -> Option, .as_str() "nl-nl", .primary() "nl", Display impl ProofLang { pub fn pack_locale(self) -> Option; } // + pub fn pack_dictionary_file(lang: ProofLang) -> Option; // + the .dic a Pack language reads now; None = unchecked ``` -`ProofLang::from_tag` now gives `Pack(locale)` for every valid language tag it has no tools for +`ProofLang::from_tag` now gives `Pack(locale)` for every valid language tag without a dedicated built-in variant (`nl-NL` → `Pack("nl-nl")`, `fr-FR`, `ja-JP`, `sr-Latn`…); `Other` is left for `x-none`, `und`, `zxx`, `mul`, `mis`, `i-…` and malformed tags. A `Pack` language is spell-checked and given suggestions from its Hunspell dictionary (dictionaries folder → installed pack → system Hunspell) -through the shared affix engine with default tuning, and hyphenated from its pack's -`hyphenation.pat`; it has no grammar rules and the generic English spelling message. Without a -dictionary or pack it behaves exactly as `Other` did (no marks, English hyphenation fallback). +through the shared affix engine with default tuning (French tuning for `fr` packs), and +hyphenated from its pack's `hyphenation.pat`. Most pack languages use no dedicated grammar +rules; regional variants can reuse a built-in language's rules. French packs use the French +spelling message and the conservative French grammar and typography described above. Without a dictionary or pack, +spelling has no marks and hyphenation falls back to English; French grammar and typography +still apply independently of a spelling dictionary. `has_dictionary(tag)` / `ProofLang::is_proofed()` say whether a `Pack` language has a dictionary now (read on first use, cached). Loaded `Pack` dictionaries share a 256 MB budget (estimated from their files); the least recently used goes first, but never one used in the paragraph being checked, and languages with no dictionary take no room. Downstream `match`es on `ProofLang` need a `Pack(_)` arm; `code()` stays `&'static str` and is `""` for it, as for `Other`: anything that keys, saves or reports a language (settings, caches, JSON/MCP output) uses `tag()`, which is `code()` for the built-ins and the locale (`nl`, `fr-fr`) for a pack language. diff --git a/crates/proof/src/fr/mod.rs b/crates/proof/src/fr/mod.rs index 541fcac0d..c67c3ffc0 100644 --- a/crates/proof/src/fr/mod.rs +++ b/crates/proof/src/fr/mod.rs @@ -44,6 +44,7 @@ pub(crate) fn check(d: &AffixDict, word: &str, opts: &Options) -> bool { if known(d, w) { return true; } + let original = w; let w = w.replace('’', "'"); if known(d, &w) { return true; @@ -56,7 +57,7 @@ pub(crate) fn check(d: &AffixDict, word: &str, opts: &Options) -> bool { { return true; } - w.contains('-') && w.split('-').all(|p| !p.is_empty() && (p.chars().count() <= 1 || known(d, p))) + original.contains('-') && original.split('-').all(|p| !p.is_empty() && (p.chars().count() <= 1 || known(d, p) || known(d, &p.replace('’', "'")))) } fn known(d: &AffixDict, word: &str) -> bool { @@ -148,6 +149,17 @@ mod tests { assert!(!is_correct_with(&d, "maison\u{338}"), "a trailing mark is part of the word"); } + #[test] + fn curly_apostrophes_in_compounds_keep_exact_dictionary_parts() { + let d = AffixDict::from_hunspell("SET UTF-8\n", "4\naujourd’hui\nécole\ncafé\nl'école\n", &[]).unwrap(); + for word in ["aujourd’hui-école", "aujourd’hui-e\u{301}cole", "café-aujourd’hui", "l’école-aujourd’hui"] { + assert!(is_correct_with(&d, word), "{word}"); + } + for word in ["aujourd’hui-écolle", "aujourd’hui--école", "aujourd'hui-école"] { + assert!(!is_correct_with(&d, word), "{word}"); + } + } + #[test] fn explicit_curly_apostrophe_without_iconv() { let d = AffixDict::from_hunspell("SET UTF-8\n", "1\naujourd’hui\n", &[]).unwrap(); diff --git a/crates/proof/tests/affix_compatibility.rs b/crates/proof/tests/affix_compatibility.rs index 6d09e5f90..c18d9fd03 100644 --- a/crates/proof/tests/affix_compatibility.rs +++ b/crates/proof/tests/affix_compatibility.rs @@ -87,6 +87,8 @@ fn legacy_compact_data_is_refused_without_reinterpreting_homonyms() { assert_eq!(hex.len() % 2, 0); let bytes: Vec<_> = hex.as_bytes().as_chunks::<2>().0.iter().map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap()).collect(); + let raw = miniz_oxide::inflate::decompress_to_vec(&bytes).unwrap(); + assert!(raw.starts_with(b"WCAFF1\n"), "fixture must contain WCAFF1 data: {}", case.name); assert!(AffixDict::from_bytes(&bytes).is_err(), "WCAFF1 must be regenerated as WCAFF2: {}", case.name); } } From 870bad52e20a62e0260c90d5d18c5643181377b7 Mon Sep 17 00:00:00 2001 From: totoz64 <47389794+totoz64@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:43:22 +0100 Subject: [PATCH 003/473] M15: Preserve accepted suggestions after the shared collection budget runs out Collector::keep has already validated the stored text. Avoid rechecking identity candidates against an exhausted SuggestSteps budget; validate only changed case and fall back to the accepted original if that validation cannot succeed. Keep the same time/work caps, exclusion rules, parser and ranking. An integrated regression uses the real REP/TRY/affix pipeline and normal budget: it loses the accepted affixed suggestion on parent b5 and retains it with this change, for lower and upper input. No injected state or relaxed limit. Exact-source fmt and strict proof Clippy passed; full local/hosted validation is still being collected. Addresses CodeRabbit4470300723. --- crates/proof/src/affix_suggest.rs | 40 +++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/crates/proof/src/affix_suggest.rs b/crates/proof/src/affix_suggest.rs index 07194c548..b3f07e43c 100644 --- a/crates/proof/src/affix_suggest.rs +++ b/crates/proof/src/affix_suggest.rs @@ -452,13 +452,9 @@ pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> V let changed: Vec<_> = candidate.split(' ').collect(); original.len() == changed.len() && original.iter().zip(changed).all(|(a, b)| d.can_suggest_cased(a, b)) }; - let t = if eligible(&t) { - t - } else if eligible(&text) { - text.clone() - } else { - continue; - }; + // Collector::keep already accepted text. Collection may have exhausted the shared + // affix budget, so recheck only a changed case and otherwise retain that accepted form. + let t = if t == text || eligible(&t) { t } else { text.clone() }; // Shown as the dictionary's OCONV table writes it (`'` → `’` in French). let t = d.output(&t).into_owned(); if t != word && !out.contains(&t) { @@ -904,4 +900,34 @@ mod tests { assert!(suggest_in(&d, "neer", 4, &english).contains(&"ne'er".to_string())); assert_eq!(suggest_in(&d, "neer", 4, &Tuning::DEFAULT).first().map(String::as_str), Some("ne'er"), "an elision"); } + /// A REP candidate is admitted before costly TRY checks consume the normal shared budget. + /// Finalization must retain that accepted affixed word, including a conservative case fallback. + #[test] + fn accepted_affixed_suggestions_survive_exhausted_collection_budget() { + let input = format!("{}x{}", "a".repeat(20), "a".repeat(19)); + let mut aff = String::from("SET UTF-8\nFLAG num\nTRY "); + aff.extend(('a'..='z').chain((0x100..0x166).filter_map(char::from_u32))); + aff.push_str(&format!("\nREP 1\nREP {input} kots\n")); + // The bounded adversarial rule shape of the existing shared-budget regression. + for f in 1..=1000u32 { + let c = char::from_u32(0x4e00 + f).unwrap(); + aff.push_str(&format!("PFX {f} Y 1\nPFX {f} {c} 0 .\n")); + } + let conts: Vec = (0..286).map(|k| (5000 + k * 7).to_string()).collect(); + for len in 1..=40 { + for k in 0..50 { + let f = 5000 + (len - 1) * 50 + k; + aff.push_str(&format!("SFX {f} Y 1\nSFX {f} 0 {}/{} .\n", "a".repeat(len), conts.join(","))); + } + } + aff.push_str("SFX 9000 Y 1\nSFX 9000 0 s .\n"); + let d = AffixDict::from_hunspell(&aff, "3\nb/1,5000\nab/2\nkot/9000\n", &[]).unwrap(); + assert!(!d.stems().any(|s| s == "kots") && d.can_suggest("kots")); + for word in [input.clone(), input.to_uppercase()] { + let got = suggest_in(&d, &word, 1, &Tuning::DEFAULT); + assert_eq!(got.len(), 1, "accepted affixed candidate lost: {word}: {got:?}"); + assert!(got[0].eq_ignore_ascii_case("kots"), "{word}: {got:?}"); + assert!(d.can_suggest(&got[0])); + } + } } From 55a16b7a724cd7ab3f9873a415fbd6fbcd8b3c96 Mon Sep 17 00:00:00 2001 From: totoz64 <47389794+totoz64@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:56:56 +0100 Subject: [PATCH 004/473] M15: Validate exact suggestion eligibility before admitting candidates Collector::keep now uses the exact-case eligibility guard before insertion, preserving NOSUGGEST, SUBSTANDARD, FORBIDDENWORD and KEEPCASE rules. Finalization can safely retain the admitted original after the collection budget is exhausted, and still validates actual case variants. No fresh budget, parser, ranking or historical fixture change. This completes CodeRabbit4470300723 without the provisional870 exclusion regression. On this exact source: workspace fmt and strict proof Clippy passed; 458 unit tests passed with52 ignored, compatibility2 and French Pack integration1 passed, and the real pinned Dicollecte test passed. The integrated normal-budget regression and all15 suggestion/casing/exclusion tests passed together in the fresh complete execution. Hosted validation of this new SHA is still required. --- crates/proof/src/affix_suggest.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/proof/src/affix_suggest.rs b/crates/proof/src/affix_suggest.rs index b3f07e43c..4d8bfea59 100644 --- a/crates/proof/src/affix_suggest.rs +++ b/crates/proof/src/affix_suggest.rs @@ -248,7 +248,8 @@ impl Collector<'_> { /// Keep a suggestion known to be correct, unless the dictionary says never to offer it. fn keep(&mut self, text: String, cost: u32) { - if !text.split(' ').all(|part| self.d.can_suggest(part)) { + // Reject exact excluded spellings at admission, before later candidates spend the budget. + if !text.split(' ').all(|part| self.d.can_suggest_cased(part, part)) { return; } self.seq += 1; From 88864de95f194ea7b81273a241868fcb94930153 Mon Sep 17 00:00:00 2001 From: David Mann Date: Thu, 12 Nov 2026 16:52:07 +0100 Subject: [PATCH 005/473] fix(flags): the review findings on craft-flags (#1119) - text handed to Flags::from_parts gets the same size limit as a file - the file is read once, at most one byte past the limit - a file named by CRAFT_FLAGS_FILE that does not exist is reported - set() holds a lock file beside the flags file from read to write, so two programs changing flags at once keep both changes - set() keeps a comment after the flag it changes - set() refuses to write a file larger than the reader accepts Refs #1118 --- crates/flags/src/lib.rs | 7 ++++- crates/flags/src/store.rs | 65 +++++++++++++++++++++++++++++---------- crates/flags/src/tests.rs | 59 +++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 17 deletions(-) diff --git a/crates/flags/src/lib.rs b/crates/flags/src/lib.rs index c65e9218f..8552bab2c 100644 --- a/crates/flags/src/lib.rs +++ b/crates/flags/src/lib.rs @@ -38,7 +38,7 @@ use std::collections::BTreeMap; use std::path::PathBuf; #[cfg(not(target_arch = "wasm32"))] -mod store; +pub(crate) mod store; #[cfg(not(target_arch = "wasm32"))] pub use store::{FlagError, load, set}; @@ -135,6 +135,11 @@ impl Flags { } fn read_file(&mut self, text: &str) { + // The same limit as a file read from disk, also for text handed in directly. + if text.len() as u64 > MAX_FILE_BYTES { + self.problems.push(format!("the flags file is larger than {MAX_FILE_BYTES} bytes, so every flag is at its default")); + return; + } let doc = match text.parse::() { Ok(d) => d, Err(e) => { diff --git a/crates/flags/src/store.rs b/crates/flags/src/store.rs index a601d8731..6f422aa09 100644 --- a/crates/flags/src/store.rs +++ b/crates/flags/src/store.rs @@ -1,9 +1,10 @@ //! Reading and changing the flags file (desktop only). use std::fmt; -use std::path::Path; +use std::io::Read; +use std::path::{Path, PathBuf}; -use crate::{ENV, Flags, MAX_FILE_BYTES, default_path, valid_id}; +use crate::{ENV, FILE_ENV, Flags, MAX_FILE_BYTES, default_path, valid_id}; /// Why a flag could not be changed. #[derive(Debug)] @@ -32,32 +33,47 @@ impl std::error::Error for FlagError {} /// The file's text, `None` when it does not exist. A file that isn't a regular file, is too big /// or isn't UTF-8 is an error. fn read_text(path: &Path) -> Result, FlagError> { - let meta = match std::fs::metadata(path) { - Ok(m) => m, + let file = match std::fs::File::open(path) { + Ok(f) => f, Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), Err(e) => return Err(FlagError::Io(e)), }; + let meta = file.metadata().map_err(FlagError::Io)?; if !meta.is_file() { return Err(FlagError::Unreadable(format!("{} is not a regular file", path.display()))); } - if meta.len() > MAX_FILE_BYTES { + // Read at most one byte past the limit from the file opened: one that grew or was replaced + // after a size check can't bring more. + let mut bytes = Vec::new(); + file.take(MAX_FILE_BYTES + 1).read_to_end(&mut bytes).map_err(FlagError::Io)?; + if bytes.len() as u64 > MAX_FILE_BYTES { return Err(FlagError::Unreadable(format!("{} is larger than {MAX_FILE_BYTES} bytes", path.display()))); } - let bytes = std::fs::read(path).map_err(FlagError::Io)?; String::from_utf8(bytes).map(Some).map_err(|_| FlagError::Unreadable(format!("{} is not UTF-8 text", path.display()))) } /// The flags of this run: [`default_path`] and `CRAFT_FLAGS`. Never fails; what could not be /// read is in [`Flags::problems`]. pub fn load() -> Flags { - let env = std::env::var(ENV).ok(); - let Some(path) = default_path() else { - return Flags::from_parts(None, env.as_deref()); + let named = std::env::var_os(FILE_ENV).is_some_and(|p| !p.is_empty()); + load_from(default_path(), named, std::env::var(ENV).ok().as_deref()) +} + +/// [`load`] from `path`; `named` when `CRAFT_FLAGS_FILE` chose it, so a missing file is worth a +/// word (the usual file not existing yet is not). +pub(crate) fn load_from(path: Option, named: bool, env: Option<&str>) -> Flags { + let Some(path) = path else { + return Flags::from_parts(None, env); }; match read_text(&path) { - Ok(text) => Flags::from_parts(text.as_deref(), env.as_deref()), + Ok(None) if named => { + let mut flags = Flags::from_parts(None, env); + flags.problems.insert(0, format!("{}, named by {FILE_ENV}, does not exist; every flag is at its default", path.display())); + flags + } + Ok(text) => Flags::from_parts(text.as_deref(), env), Err(e) => { - let mut flags = Flags::from_parts(None, env.as_deref()); + let mut flags = Flags::from_parts(None, env); flags.problems.insert(0, format!("{}: {e}; every flag is at its default", path.display())); flags } @@ -65,11 +81,21 @@ pub fn load() -> Flags { } /// Turn the flag `id` on or off in the file at `path` (made if missing). Everything else in the -/// file, comments included, is kept; the write is atomic. +/// file, comments included, is kept; the write is atomic. Two programs changing flags at once +/// (the launcher and an app) take turns, so neither change is lost. pub fn set(path: &Path, id: &str, on: bool) -> Result<(), FlagError> { if !valid_id(id) { return Err(FlagError::BadId(id.to_string())); } + if let Some(dir) = path.parent() { + std::fs::create_dir_all(dir).map_err(FlagError::Io)?; + } + // A lock file beside the flags file, which itself is replaced on every write. Released when + // `_lock` is dropped (and by the system if the process ends). + let mut lock_path = path.as_os_str().to_owned(); + lock_path.push(".lock"); + let _lock = std::fs::OpenOptions::new().create(true).truncate(false).write(true).open(PathBuf::from(lock_path)).map_err(FlagError::Io)?; + _lock.lock().map_err(FlagError::Io)?; let mut doc = match read_text(path)? { Some(text) => text.parse::().map_err(|e| FlagError::Unreadable(e.to_string()))?, None => toml_edit::DocumentMut::new(), @@ -78,9 +104,16 @@ pub fn set(path: &Path, id: &str, on: bool) -> Result<(), FlagError> { let Some(table) = item.as_table_like_mut() else { return Err(FlagError::Unreadable("`flags` is not a table".into())); }; - table.insert(id, toml_edit::value(on)); - if let Some(dir) = path.parent() { - std::fs::create_dir_all(dir).map_err(FlagError::Io)?; + // Keep what surrounds the old value, such as a comment after it. + let mut value = toml_edit::Value::from(on); + if let Some(old) = table.get(id).and_then(toml_edit::Item::as_value) { + *value.decor_mut() = old.decor().clone(); } - craft_atomic::atomic_write(path, doc.to_string().as_bytes()).map_err(FlagError::Io) + table.insert(id, toml_edit::Item::Value(value)); + let text = doc.to_string(); + // What can't be read back is not written: the file stays as it was. + if text.len() as u64 > MAX_FILE_BYTES { + return Err(FlagError::Unreadable(format!("the flags file would grow past {MAX_FILE_BYTES} bytes"))); + } + craft_atomic::atomic_write(path, text.as_bytes()).map_err(FlagError::Io) } diff --git a/crates/flags/src/tests.rs b/crates/flags/src/tests.rs index 3f4bef667..92f690037 100644 --- a/crates/flags/src/tests.rs +++ b/crates/flags/src/tests.rs @@ -48,6 +48,15 @@ fn bad_entries_are_reported_and_left_out_never_fatal() { assert!(!Flags::from_parts(Some("flags = 3"), None).problems.is_empty()); } +#[test] +fn oversized_text_is_not_read() { + let mut text = String::from("[flags]\nAI-MCP = true\n"); + text.push_str(&"#".repeat(MAX_FILE_BYTES as usize)); + let flags = Flags::from_parts(Some(&text), None); + assert!(!flags.enabled(&AI_MCP)); + assert!(flags.problems.iter().any(|p| p.contains("larger than"))); +} + #[test] fn many_entries_are_capped() { let mut text = String::from("[flags]\n"); @@ -79,6 +88,7 @@ fn ids_are_checked() { #[cfg(not(target_arch = "wasm32"))] mod file { use super::super::*; + use crate::store; #[test] fn set_creates_the_file_and_keeps_what_else_is_in_it() { @@ -98,6 +108,55 @@ mod file { assert!(flags.enabled(&AI_MODELS)); } + #[test] + fn set_keeps_the_comment_after_a_flag() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(FILE_NAME); + std::fs::write(&path, "[flags]\nAI-MCP = true # on for testing\n").unwrap(); + set(&path, "AI-MCP", false).unwrap(); + let text = std::fs::read_to_string(&path).unwrap(); + assert!(text.contains("AI-MCP = false # on for testing"), "{text}"); + } + + #[test] + fn set_refuses_to_write_what_could_not_be_read_back() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(FILE_NAME); + let mut text = String::from("[flags]\n"); + while (text.len() as u64) < MAX_FILE_BYTES - 4 { + text.push_str("# padding\n"); + } + std::fs::write(&path, &text).unwrap(); + assert!(matches!(set(&path, "AI-MODELS", true), Err(FlagError::Unreadable(_)))); + assert_eq!(std::fs::read_to_string(&path).unwrap(), text, "left as it was"); + } + + #[test] + fn a_missing_file_is_mentioned_only_when_named() { + let dir = tempfile::tempdir().unwrap(); + let missing = dir.path().join("nope.toml"); + let named = store::load_from(Some(missing.clone()), true, None); + assert!(named.problems.iter().any(|p| p.contains("does not exist")), "{:?}", named.problems); + assert!(store::load_from(Some(missing), false, None).problems.is_empty(), "the usual file not existing yet is normal"); + } + + #[test] + fn two_writers_keep_both_changes() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(FILE_NAME); + let ids: Vec = (0..8).map(|i| format!("F{i}")).collect(); + std::thread::scope(|s| { + for id in &ids { + let path = &path; + s.spawn(move || set(path, id, true).unwrap()); + } + }); + let flags = Flags::from_parts(Some(&std::fs::read_to_string(&path).unwrap()), None); + for id in &ids { + assert!(flags.enabled_id(id), "{id} lost"); + } + } + #[test] fn set_never_overwrites_a_file_it_cannot_read() { let dir = tempfile::tempdir().unwrap(); From 4a3bb60b828feaae94de67e00db3c25040f19ec7 Mon Sep 17 00:00:00 2001 From: David Mann Date: Thu, 12 Nov 2026 17:07:23 +0100 Subject: [PATCH 006/473] test(flags): the oversized-write fixture is readable, so the test reaches the output-size check The padding loop made the fixture 65,538 bytes, past the read limit, so set() refused while reading and the test passed without the output-size check. The fixture is now exactly MAX_FILE_BYTES - 4 bytes; with the check removed the test fails. --- crates/flags/src/tests.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/crates/flags/src/tests.rs b/crates/flags/src/tests.rs index 92f690037..66c00855b 100644 --- a/crates/flags/src/tests.rs +++ b/crates/flags/src/tests.rs @@ -122,12 +122,15 @@ mod file { fn set_refuses_to_write_what_could_not_be_read_back() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join(FILE_NAME); - let mut text = String::from("[flags]\n"); - while (text.len() as u64) < MAX_FILE_BYTES - 4 { - text.push_str("# padding\n"); - } + // Readable (just under the limit), so the refusal comes from the size of what `set` would + // write, not from reading. + let mut text = String::from("[flags]\n# "); + let pad = MAX_FILE_BYTES as usize - 4 - text.len(); + text.push_str(&"x".repeat(pad)); + assert_eq!(text.len() as u64, MAX_FILE_BYTES - 4); std::fs::write(&path, &text).unwrap(); - assert!(matches!(set(&path, "AI-MODELS", true), Err(FlagError::Unreadable(_)))); + assert!(Flags::from_parts(Some(&text), None).problems.is_empty(), "the fixture itself reads"); + assert!(matches!(set(&path, "AI-MODELS", true), Err(FlagError::Unreadable(e)) if e.contains("would grow"))); assert_eq!(std::fs::read_to_string(&path).unwrap(), text, "left as it was"); } From 18f529112e1dcf09a873f3ebf9818c00f1600b97 Mon Sep 17 00:00:00 2001 From: blenux Date: Fri, 13 Nov 2026 15:51:20 +1100 Subject: [PATCH 007/473] raster-codecs: expose DDS header info and bounded mip imports Keep import clients on the shared decoder after the extraction from PhotoCraft. Charge the entire mip chain before each allocation, check cancellation between levels, and accept mip counts without the optional writer flag. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- crates/raster-codecs/src/dds.rs | 77 ++++++++++++++++++++++++++++--- crates/raster-codecs/tests/dds.rs | 76 ++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 6 deletions(-) diff --git a/crates/raster-codecs/src/dds.rs b/crates/raster-codecs/src/dds.rs index 1249cb5b7..0f848e38c 100644 --- a/crates/raster-codecs/src/dds.rs +++ b/crates/raster-codecs/src/dds.rs @@ -91,6 +91,7 @@ impl Field { } } +#[derive(Clone, Copy)] enum Encoding { /// Uncompressed: bytes per pixel, then red (or luminance), green, blue and alpha. Masks { @@ -202,7 +203,25 @@ fn dx10_encoding(b: &[u8]) -> Result { } } -pub fn decode(b: &[u8], limits: &Limits) -> Result { +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DdsInfo { + pub width: u32, + pub height: u32, + pub levels: u32, + pub alpha: bool, + pub compression: Option, +} + +struct Parsed<'a> { + encoding: Encoding, + data: &'a [u8], + width: u32, + height: u32, + levels: u32, + pitch: Option, +} + +fn parse(b: &[u8]) -> Result, CodecError> { if !b.starts_with(b"DDS ") { return Err(bad("missing the `DDS ` signature")); } @@ -218,17 +237,63 @@ pub fn decode(b: &[u8], limits: &Limits) -> Result { } let dx10 = le32(b, 80)? & DDPF_FOURCC != 0 && &le32(b, 84)?.to_le_bytes() == b"DX10"; let (encoding, start) = if dx10 { (dx10_encoding(b)?, HEADER + DX10_HEADER) } else { (legacy_encoding(b)?, HEADER) }; - let layout = encoding.layout(); - limits.check(width, height, layout, SampleType::U8)?; + let declared = le32(b, 28)?.max(1); + let levels = declared.min(mip_levels(width, height)); // The same header field is a total level size for BC, but a row stride when DDSD_PITCH is set. let pitch = if flags & DDSD_PITCH != 0 { Some(le32(b, 20)?) } else { None }; + Ok(Parsed { encoding, data: b.get(start..).unwrap_or_default(), width, height, levels, pitch }) +} + +pub fn info(b: &[u8]) -> Result { + let p = parse(b)?; + let compression = match p.encoding { + Encoding::Masks { .. } => DdsCompression::None, + Encoding::Bc1 => DdsCompression::Bc1, + Encoding::Bc2 => DdsCompression::Bc2, + Encoding::Bc3 => DdsCompression::Bc3, + }; + let alpha = match p.encoding { + Encoding::Masks { fields, .. } => fields[3].is_some(), + _ => true, + }; + Ok(DdsInfo { width: p.width, height: p.height, levels: p.levels, alpha, compression: Some(compression) }) +} + +pub fn decode(b: &[u8], limits: &Limits) -> Result { + let p = parse(b)?; + level_image(p.encoding, p.data, p.width, p.height, p.pitch, limits).map(|(img, _)| img) +} + +pub fn decode_levels(b: &[u8], limits: &Limits, cancel: &dyn Fn() -> bool) -> Result, CodecError> { + let p = parse(b)?; + let (mut data, mut width, mut height, mut remaining) = (p.data, p.width, p.height, limits.max_alloc); + let mut levels = Vec::with_capacity(p.levels as usize); + for i in 0..p.levels { + if cancel() { + return Err(bad("DDS import cancelled")); + } + let level_limits = Limits { max_alloc: remaining, ..*limits }; + let (img, size) = level_image(p.encoding, data, width, height, if i == 0 { p.pitch } else { None }, &level_limits)?; + remaining = + remaining.checked_sub(img.data().len() as u64).ok_or_else(|| CodecError::LimitExceeded("DDS mip chain exceeds max_alloc".into()))?; + levels.push(img); + data = data.get(size..).ok_or_else(|| bad("truncated mip data"))?; + width = (width / 2).max(1); + height = (height / 2).max(1); + } + Ok(levels) +} + +fn level_image(encoding: Encoding, data: &[u8], width: u32, height: u32, pitch: Option, limits: &Limits) -> Result<(Image, usize), CodecError> { + let layout = encoding.layout(); + limits.check(width, height, layout, SampleType::U8)?; let (row_pitch, size) = encoding.level_layout(width, height, pitch)?; - let data = b.get(start..).and_then(|d| d.get(..size)).ok_or_else(|| bad(format!("truncated pixel data: {size} bytes needed")))?; + let data = data.get(..size).ok_or_else(|| bad(format!("truncated pixel data: {size} bytes needed")))?; let (w, h) = (width as usize, height as usize); let channels = layout.channels(); let row_size = w.checked_mul(channels).ok_or_else(|| bad("pixel size overflow"))?; let total = row_size.checked_mul(h).ok_or_else(|| bad("pixel size overflow"))?; - let mut pixels = vec![0u8; total]; + let mut pixels = crate::alloc_zeroed::(total)?; match encoding { Encoding::Masks { bytes, gray, fields } => { for (src_row, dst_row) in data.chunks_exact(row_pitch).zip(pixels.chunks_exact_mut(row_size)) { @@ -263,7 +328,7 @@ pub fn decode(b: &[u8], limits: &Limits) -> Result { } } } - Image::from_u8(width, height, layout, pixels) + Ok((Image::from_u8(width, height, layout, pixels)?, size)) } /// A BC1 colour block. BC2 and BC3 colour blocks always have four colours; only BC1's diff --git a/crates/raster-codecs/tests/dds.rs b/crates/raster-codecs/tests/dds.rs index afb9066aa..60fe1366d 100644 --- a/crates/raster-codecs/tests/dds.rs +++ b/crates/raster-codecs/tests/dds.rs @@ -410,3 +410,79 @@ fn rejects_bogus_sizes_and_pixel_formats() { no_format.extend([0; 4]); assert!(matches!(decode(&no_format), Err(CodecError::Unsupported { .. }))); } + +fn mip_chain() -> Vec { + let mut b = compressed(4, 4, b"DXT1"); + b[8..12].copy_from_slice(&(0x1007u32 | 0x2_0000).to_le_bytes()); + b[28..32].copy_from_slice(&3u32.to_le_bytes()); + for c in [RED, GREEN, BLUE] { + b.extend(bc1_block(c, c, [0; 4])); + } + b +} + +#[test] +fn import_info_and_all_mips_use_the_shared_decoder() { + let b = mip_chain(); + let info = dds::info(&b[..128]).unwrap(); + assert_eq!((info.width, info.height, info.levels, info.alpha), (4, 4, 3, true)); + assert_eq!(info.compression, Some(dds::DdsCompression::Bc1)); + let levels = dds::decode_levels(&b, &Limits::default(), &|| false).unwrap(); + assert_eq!(levels.iter().map(Image::dimensions).collect::>(), [(4, 4), (2, 2), (1, 1)]); + for (img, color) in levels.iter().zip([[255, 0, 0, 255], [0, 255, 0, 255], [0, 0, 255, 255]]) { + assert!(img.data().as_chunks::<4>().0.iter().all(|p| *p == color)); + } + assert_eq!(decode(&b).unwrap().data(), levels[0].data()); +} + +#[test] +fn mip_chain_charges_the_combined_allocation() { + for (budget, accepted) in [(63, false), (64, false), (80, false), (83, false), (84, true)] { + let result = dds::decode_levels(&mip_chain(), &Limits { max_alloc: budget, ..Default::default() }, &|| false); + if accepted { + assert_eq!(result.unwrap().len(), 3); + } else { + assert!(matches!(result, Err(CodecError::LimitExceeded(_)))); + } + } +} + +#[test] +fn cancellation_is_checked_after_a_level_was_decoded() { + let calls = std::cell::Cell::new(0); + let result = dds::decode_levels(&mip_chain(), &Limits::default(), &|| { + let n = calls.get(); + calls.set(n + 1); + n == 1 + }); + assert!(matches!(result, Err(CodecError::Malformed { message, .. }) if message.contains("cancelled"))); + assert_eq!(calls.get(), 2); +} + +#[test] +fn mip_import_rejects_truncation_and_bounds_declared_count() { + let mut b = mip_chain(); + for n in 0..b.len() { + assert!(dds::decode_levels(&b[..n], &Limits::default(), &|| false).is_err(), "length {n}"); + } + b[28..32].copy_from_slice(&u32::MAX.to_le_bytes()); + assert_eq!(dds::decode_levels(&b, &Limits::default(), &|| false).unwrap().len(), 3); + b[8..12].copy_from_slice(&0x1007u32.to_le_bytes()); + assert_eq!(dds::info(&b).unwrap().levels, 3); + assert_eq!(dds::decode_levels(&b, &Limits::default(), &|| false).unwrap().len(), 3); + b[28..32].copy_from_slice(&0u32.to_le_bytes()); + assert_eq!(dds::info(&b).unwrap().levels, 1); + assert_eq!(dds::decode_levels(&b, &Limits::default(), &|| false).unwrap().len(), 1); +} + +#[test] +fn lower_mips_do_not_inherit_top_level_pitch() { + let mut b = header(2, 2, DDPF_LUMINANCE, &[0; 4], 8, [0xff, 0, 0, 0]); + set_pitch(&mut b, 4); + b[8..12].copy_from_slice(&(0x1007u32 | DDSD_PITCH | 0x2_0000).to_le_bytes()); + b[28..32].copy_from_slice(&2u32.to_le_bytes()); + b.extend([1, 2, 99, 99, 3, 4, 99, 99, 5]); + let levels = dds::decode_levels(&b, &Limits::default(), &|| false).unwrap(); + assert_eq!(levels[0].data(), [1, 2, 3, 4]); + assert_eq!(levels[1].data(), [5]); +} From 332c5abd8d8fbc1f07c919e249c76a01efb5b3ee Mon Sep 17 00:00:00 2001 From: blenux Date: Fri, 13 Nov 2026 19:50:43 +1100 Subject: [PATCH 008/473] raster-codecs: reject incomplete DDS metadata headers Keep complete header-only probes valid while rejecting truncated legacy and DX10 headers before returning metadata. Cover every truncated prefix and preserve missing-pixel-data errors in both decoding paths. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- crates/raster-codecs/src/dds.rs | 3 ++- crates/raster-codecs/tests/dds.rs | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/crates/raster-codecs/src/dds.rs b/crates/raster-codecs/src/dds.rs index 0f848e38c..b9a716688 100644 --- a/crates/raster-codecs/src/dds.rs +++ b/crates/raster-codecs/src/dds.rs @@ -241,7 +241,8 @@ fn parse(b: &[u8]) -> Result, CodecError> { let levels = declared.min(mip_levels(width, height)); // The same header field is a total level size for BC, but a row stride when DDSD_PITCH is set. let pitch = if flags & DDSD_PITCH != 0 { Some(le32(b, 20)?) } else { None }; - Ok(Parsed { encoding, data: b.get(start..).unwrap_or_default(), width, height, levels, pitch }) + let data = b.get(start..).ok_or_else(|| bad("truncated header"))?; + Ok(Parsed { encoding, data, width, height, levels, pitch }) } pub fn info(b: &[u8]) -> Result { diff --git a/crates/raster-codecs/tests/dds.rs b/crates/raster-codecs/tests/dds.rs index 60fe1366d..c0a06b6de 100644 --- a/crates/raster-codecs/tests/dds.rs +++ b/crates/raster-codecs/tests/dds.rs @@ -421,6 +421,31 @@ fn mip_chain() -> Vec { b } +#[test] +fn import_info_rejects_truncated_legacy_headers() { + let b = compressed(4, 4, b"DXT1"); + for n in 0..b.len() { + assert!(matches!(dds::info(&b[..n]), Err(CodecError::Malformed { .. })), "accepted {n} of {} header bytes", b.len()); + } +} + +#[test] +fn import_info_rejects_truncated_dx10_headers() { + let b = dx10(4, 4, 71, 3, 0, 1); + for n in 0..b.len() { + assert!(matches!(dds::info(&b[..n]), Err(CodecError::Malformed { .. })), "accepted {n} of {} header bytes", b.len()); + } +} + +#[test] +fn import_info_accepts_complete_headers_without_pixel_data() { + for b in [compressed(4, 4, b"DXT1"), dx10(4, 4, 71, 3, 0, 1)] { + assert_eq!(dds::info(&b).unwrap(), dds::DdsInfo { width: 4, height: 4, levels: 1, alpha: true, compression: Some(dds::DdsCompression::Bc1) }); + assert!(matches!(decode(&b), Err(CodecError::Malformed { .. }))); + assert!(matches!(dds::decode_levels(&b, &Limits::default(), &|| false), Err(CodecError::Malformed { .. }))); + } +} + #[test] fn import_info_and_all_mips_use_the_shared_decoder() { let b = mip_chain(); From 6ac5f83bb23e46518c3aaea33b4d3c3995d41abe Mon Sep 17 00:00:00 2001 From: alehel Date: Sun, 15 Nov 2026 00:44:35 +0100 Subject: [PATCH 009/473] feat(media,video,audio): add Sierra VMD demuxer and decoders Sierra VMD (Video and Music Data) is the video format of Sierra and Coktel Vision games of the 1990s. Written clean-room from the public format description at https://wiki.multimedia.cx/index.php/VMD, with ffmpeg as a black-box oracle. - craft-media: `vmd` feature, a demuxer for the 816-byte header and the table of contents; `Container::Vmd`, `CodecId::VmdVideo` and `CodecId::VmdAudio`. - craft-video: `vmd` feature, a decoder for 8-bit paletted pictures (rendering methods 1 to 3, LZ packing, palette changes). - craft-audio: `vmd-decode` feature, a decoder for 8-bit PCM and 16-bit DPCM sound, mono and stereo. - craft-media-testkit: a VMD fixture writer, since ffmpeg reads VMD but cannot write it. Tests compare the decoders with ffmpeg on generated files, and on real files when CRAFT_MEDIA_VMD_SAMPLES names a directory of them. 16-bit, 24-bit and Indeo 3 pictures, IMA ADPCM sound and the later header revisions are not decoded. --- README.md | 2 +- crates/audio/Cargo.toml | 8 + crates/audio/README.md | 1 + crates/audio/src/lib.rs | 15 +- crates/audio/src/vmd.rs | 178 +++++++++++++ crates/audio/tests/vmd.rs | 265 ++++++++++++++++++++ crates/media-testkit/src/lib.rs | 2 + crates/media-testkit/src/vmd.rs | 343 +++++++++++++++++++++++++ crates/media/Cargo.toml | 6 + crates/media/README.md | 56 ++++- crates/media/src/id.rs | 12 + crates/media/src/lib.rs | 5 +- crates/media/src/vmd/mod.rs | 405 ++++++++++++++++++++++++++++++ crates/media/src/vmd/tests.rs | 161 ++++++++++++ crates/media/tests/oracle_vmd.rs | 68 +++++ crates/video/Cargo.toml | 12 +- crates/video/README.md | 1 + crates/video/src/lib.rs | 4 + crates/video/src/vmd/mod.rs | 382 ++++++++++++++++++++++++++++ crates/video/tests/vmd/main.rs | 415 +++++++++++++++++++++++++++++++ 20 files changed, 2329 insertions(+), 12 deletions(-) create mode 100644 crates/audio/src/vmd.rs create mode 100644 crates/audio/tests/vmd.rs create mode 100644 crates/media-testkit/src/vmd.rs create mode 100644 crates/media/src/vmd/mod.rs create mode 100644 crates/media/src/vmd/tests.rs create mode 100644 crates/media/tests/oracle_vmd.rs create mode 100644 crates/video/src/vmd/mod.rs create mode 100644 crates/video/tests/vmd/main.rs diff --git a/README.md b/README.md index c6e95bd4f..2f72adfe8 100644 --- a/README.md +++ b/README.md @@ -254,7 +254,7 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit - [`craft-math`](crates/math/README.md): equations as one tree (the Office Math object model) with its syntaxes, read and written: OMML, presentation MathML, LaTeX math and the UnicodeMath linear format. - [`craft-media`](crates/media/README.md): bit-level I/O (`bitstream`), clean-room MP4/MOV (`isobmff`), Matroska/WebM (`matroska`) and MXF (`mxf`) demuxers and muxers, Ogg (`ogg`), AVI (`avi`) and - MPEG-TS/PS (`mpegts`) demuxers, from FilmCraft, and an ISO BMFF box reader. Containers only; codecs are in `craft-audio`, and + MPEG-TS/PS (`mpegts`) demuxers, from FilmCraft, a Sierra VMD (`vmd`) demuxer, and an ISO BMFF box reader. Containers only; codecs are in `craft-audio`, and video codecs will go in a separate crate. - [`craft-lottie`](crates/lottie/README.md): Lottie (Bodymovin) JSON: the document model of lottie-spec 1.0 (animation, assets, layers by type, shape items, masks, effects, text, markers) diff --git a/crates/audio/Cargo.toml b/crates/audio/Cargo.toml index d132f23da..55698d192 100644 --- a/crates/audio/Cargo.toml +++ b/crates/audio/Cargo.toml @@ -49,6 +49,8 @@ ac3-encode = ["craft-media/bitstream"] eac3-encode = ["craft-media/bitstream"] # MPEG-1/2 Layer III (MP3) encoder (`mp3`). mp3-encode = ["craft-media/bitstream"] +# Sierra VMD sound decoder (`vmd::Decoder`): 8-bit PCM and 16-bit DPCM. +vmd-decode = [] # Multi-resolution waveform overviews (`peaks`). peaks = [] # Encode AAC channels in parallel (with `aac-encode`). @@ -67,6 +69,8 @@ symphonia = { version = "0.6.1", optional = true, default-features = false, feat [dev-dependencies] craft-media-testkit = { workspace = true } +# The VMD demuxer, for the VMD sound decoder's tests. +craft-media = { workspace = true, features = ["vmd"] } proptest = { workspace = true } serde_json = { workspace = true } @@ -74,6 +78,10 @@ serde_json = { workspace = true } name = "audio" required-features = ["decode", "lossless-encode", "peaks"] +[[test]] +name = "vmd" +required-features = ["vmd-decode"] + [[test]] name = "codecs" required-features = ["decode", "encode"] diff --git a/crates/audio/README.md b/crates/audio/README.md index 5a5df4f2f..59256822a 100644 --- a/crates/audio/README.md +++ b/crates/audio/README.md @@ -42,6 +42,7 @@ No default features. The shared types (`AudioBuffer`, `AudioInfo`, `ChannelLayou | `mpa-decode` | `mpa::Decoder` | craft-media `bitstream` | | `mp3-decode` | `mp3::Decoder`, `mp3::InfoTag` | `mpa-decode` | | `mp3-encode` | `mp3::Encoder`; `encode`, `FileEncoder` for MP3 | craft-media `bitstream` | +| `vmd-decode` | `vmd::Decoder`: Sierra VMD sound (8-bit PCM and 16-bit DPCM), frame by frame from `craft_media::vmd` | none | | `peaks` | `peaks::Peaks` | none | | `rayon` | parallel AAC encoding (with `aac-encode`) | `rayon` | | `serde` | `Serialize`/`Deserialize` on the shared types | `serde` | diff --git a/crates/audio/src/lib.rs b/crates/audio/src/lib.rs index 1c6c73dcf..cd250bd21 100644 --- a/crates/audio/src/lib.rs +++ b/crates/audio/src/lib.rs @@ -27,7 +27,8 @@ //! - Codec modules for packet-level use, each with its own features: [`aac`] (`aac-decode`, //! `aac-encode`), [`alac`] (`alac-decode`, `alac-encode`), [`opus`] (`opus-decode`, and //! [`opus::encoder`] with `opus-encode`), [`ac3`] (`ac3-decode`), [`mpa`] (`mpa-decode`), [`mp3`] -//! (`mp3-decode`, `mp3-encode`), [`flac`] (`flac-encode`: the packet encoder). +//! (`mp3-decode`, `mp3-encode`), [`flac`] (`flac-encode`: the packet encoder), [`vmd`] +//! (`vmd-decode`: Sierra VMD sound). //! //! The shared types below build with no features. @@ -53,6 +54,8 @@ pub mod peaks; mod read; #[cfg(any(feature = "aac-decode", feature = "alac-decode", feature = "opus-decode", feature = "ac3-decode", feature = "mpa-decode"))] mod track; +#[cfg(feature = "vmd-decode")] +pub mod vmd; #[cfg(any(feature = "decode-native", feature = "pcm-encode"))] mod wav; #[cfg(any( @@ -224,6 +227,16 @@ impl From for AudioError { } } +#[cfg(feature = "vmd-decode")] +impl From for AudioError { + fn from(e: vmd::Error) -> Self { + match e { + vmd::Error::Unsupported(_) => AudioError::Unsupported(e.to_string()), + _ => AudioError::Malformed(e.to_string()), + } + } +} + #[cfg(feature = "opus-encode")] impl From for AudioError { fn from(e: opus::encoder::Error) -> Self { diff --git a/crates/audio/src/vmd.rs b/crates/audio/src/vmd.rs new file mode 100644 index 000000000..6c2a0b829 --- /dev/null +++ b/crates/audio/src/vmd.rs @@ -0,0 +1,178 @@ +//! Sierra VMD sound decoder (feature `vmd-decode`): the 8-bit PCM and 16-bit DPCM sound of VMD +//! (Video and Music Data) files, as in Sierra's and Coktel Vision's 1990s games. +//! +//! Clean-room, from the public format description at +//! ("Description of the Sierra Video and Music Data (VMD) Format", Mike Melanson and Vladimir +//! Gneushev). The description gives the coding but not the size of a block; the sizes here are +//! what ffmpeg was seen to expect of files made by this crate's tests: +//! +//! | Sound | Bytes in a block of `n` samples a channel | +//! |---|---| +//! | 8-bit | `n` for each channel, interleaved | +//! | 16-bit | 2 for each channel's first sample, then one code for each further sample, interleaved | +//! +//! The header's frame length is `n`, except in old-style stereo (audio flag 0x8000), where it +//! counts both channels. Old-style 16-bit stereo blocks begin with their first samples like the +//! others, which is what ffmpeg reads; the description says they carry the last block's samples +//! over instead. No such file was at hand to settle it. +//! +//! A [`Decoder`] is made from the 816-byte file header. [`Decoder::decode`] takes a sound +//! frame's flags (from its record in the file's table of contents; `craft_media::vmd` reads it) +//! and its data, and returns interleaved `i16` samples: one block of sound or silence, or, for a +//! frame with a silence mask, several. 8-bit samples are widened to 16 bits. Blocks do not depend +//! on each other, so decoding can start at any frame. +//! +//! Not decoded: IMA ADPCM sound (audio flag 0x10) and the external codecs of later Coktel Vision +//! files ([`Error::Unsupported`]). +//! +//! Input is treated as hostile: a frame yields at most 32 blocks (the width of the silence +//! mask) of at most 65 535 samples a channel, and a block is decoded only if all of it is there. No `unsafe`, no panics. + +#![warn(missing_docs)] + +/// Bytes of the VMD file header a [`Decoder`] is made from. +pub const HEADER_LEN: usize = 816; + +/// Errors from decoding VMD sound. +#[non_exhaustive] +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum Error { + /// The header or a frame is not valid. + #[error("invalid VMD sound: {0}")] + Invalid(&'static str), + /// Valid, but not something this decoder handles. + #[error("unsupported VMD sound: {0}")] + Unsupported(&'static str), +} + +/// The steps of the 16-bit coding: a code's low seven bits pick one, its top bit the sign. +const STEPS: [u16; 128] = [ + 0, 8, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, + 480, 496, 512, 520, 528, 536, 544, 552, 560, 568, 576, 584, 592, 600, 608, 616, 624, 632, 640, 648, 656, 664, 672, 680, 688, 696, 704, 712, 720, + 728, 736, 744, 752, 760, 768, 776, 784, 792, 800, 808, 816, 824, 832, 840, 848, 856, 864, 872, 880, 888, 896, 904, 912, 920, 928, 936, 944, 952, + 960, 968, 976, 984, 992, 1000, 1008, 1016, 1024, 1088, 1152, 1216, 1280, 1344, 1408, 1472, 1536, 1600, 1664, 1728, 1792, 1856, 1920, 1984, 2048, + 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 5120, 6144, 7168, 8192, 12288, 16384, +]; + +/// A VMD sound decoder. +#[derive(Clone, Debug)] +pub struct Decoder { + sample_rate: u32, + channels: usize, + sixteen_bit: bool, + /// Samples of one block, all channels counted. + block_samples: usize, +} + +impl Decoder { + /// A decoder for the file whose header is `header` (its first 816 bytes; more are ignored). + pub fn new(header: &[u8]) -> Result { + let h = header.get(..HEADER_LEN).ok_or(Error::Invalid("the header is shorter than 816 bytes"))?; + let u16_at = |at: usize| u16::from_le_bytes([h[at], h[at + 1]]); + if usize::from(u16_at(0)) != HEADER_LEN - 2 { + return Err(Error::Invalid("not a VMD header")); + } + if u16_at(16) & 0x1000 == 0 { + return Err(Error::Invalid("the file has no sound")); + } + let flags = u16_at(810); + if flags & 0x10 != 0 { + return Err(Error::Unsupported("IMA ADPCM sound")); + } + let length = u16_at(806) as i16; + let frame_length = usize::from(length.unsigned_abs()); + // old-style stereo counts both channels, new-style one + let (channels, block_samples) = match flags { + f if f & 0x8000 != 0 => (2, frame_length & !1), + f if f & 0x200 != 0 => (2, frame_length * 2), + _ => (1, frame_length), + }; + Ok(Decoder { sample_rate: u32::from(u16_at(804)), channels, sixteen_bit: length < 0, block_samples }) + } + + /// Samples a second. + pub fn sample_rate(&self) -> u32 { + self.sample_rate + } + + /// 1 or 2. + pub fn channels(&self) -> usize { + self.channels + } + + /// Bits a sample in the file: 8 (PCM) or 16 (DPCM). The decoder returns `i16` for both. + pub fn bits(&self) -> u8 { + if self.sixteen_bit { 16 } else { 8 } + } + + /// Samples of one block for each channel. + pub fn block_samples(&self) -> usize { + self.block_samples / self.channels + } + + /// Bytes of one block of sound in a frame. + pub fn block_bytes(&self) -> usize { + if self.sixteen_bit { (self.block_samples + self.channels).min(self.block_samples * 2) } else { self.block_samples } + } + + /// Decode one sound frame: `flags` is the frame's flag byte from its record, `data` the + /// frame data. Returns interleaved samples, [`Decoder::block_samples`] for each channel and + /// block. + /// + /// - flags 1: one block of sound; + /// - flags 2: a 32-bit mask, then blocks of sound: the sound a player loads ahead at the + /// start of a file. From bit 0 up, a set bit is a block of silence and a clear bit the next + /// block of data, while there is one. ffmpeg plays the silent blocks first instead; + /// the order here is the description's; + /// - flags 3: one block of silence. + /// + /// An empty frame gives no samples. A block cut short is an error. + pub fn decode(&self, flags: u8, data: &[u8]) -> Result, Error> { + let mut out = Vec::new(); + match flags { + 3 => out.resize(self.block_samples, 0), + _ if data.is_empty() => {} + 1 => self.block(data, &mut out)?, + 2 => { + let (mask, mut data) = data.split_first_chunk::<4>().ok_or(Error::Invalid("the silence mask is cut short"))?; + let mask = u32::from_le_bytes(*mask); + // bit by bit: a set bit is a silent block, a clear one takes the next block of data + // while there is one. The header's buffer count is not needed: game files hold + // one block fewer than it says. + for i in 0..32 { + if mask >> i & 1 == 1 { + out.resize(out.len() + self.block_samples, 0); + } else if !data.is_empty() && data.len() >= self.block_bytes() { + self.block(data, &mut out)?; + data = &data[self.block_bytes()..]; + } + } + } + _ => return Err(Error::Unsupported("unknown sound frame flags")), + } + Ok(out) + } + + /// Decode the block at the start of `data` onto `out`. + fn block(&self, data: &[u8], out: &mut Vec) -> Result<(), Error> { + let data = data.get(..self.block_bytes()).ok_or(Error::Invalid("a block of sound is cut short"))?; + if !self.sixteen_bit { + out.extend(data.iter().map(|&b| (i16::from(b) - 0x80) << 8)); + return Ok(()); + } + // each channel's first sample as it is, then a signed step for each further sample + let mut last = [0i32; 2]; + let Some((first, codes)) = data.split_at_checked(self.channels * 2) else { return Ok(()) }; + for (ch, s) in first.as_chunks::<2>().0.iter().enumerate() { + last[ch] = i32::from(i16::from_le_bytes(*s)); + out.push(last[ch] as i16); + } + for (i, &code) in codes.iter().enumerate() { + let ch = i % self.channels; + let step = i32::from(STEPS[usize::from(code & 0x7F)]); + last[ch] = (if code & 0x80 != 0 { last[ch] - step } else { last[ch] + step }).clamp(-32768, 32767); + out.push(last[ch] as i16); + } + Ok(()) + } +} diff --git a/crates/audio/tests/vmd.rs b/crates/audio/tests/vmd.rs new file mode 100644 index 000000000..d7004a888 --- /dev/null +++ b/crates/audio/tests/vmd.rs @@ -0,0 +1,265 @@ +//! VMD sound decoder tests. ffmpeg reads VMD but cannot write it, so the files come from the +//! test kit's VMD writer. The decoder must return the samples that were packed, and the same +//! samples as ffmpeg (skipped without ffmpeg). + +use std::process::Command; + +use craft_audio::vmd::{Decoder, Error}; +use craft_media::vmd::{FrameKind, open}; +use craft_media_testkit::hostile::Rng; +use craft_media_testkit::vmd::{Audio, Frame, Spec, build, dpcm_pack}; + +/// (frame length field, audio flags, channels, 16-bit): every layout the decoder handles. +const FORMATS: [(i16, u16, usize, bool); 6] = + [(50, 0, 1, false), (50, 0x200, 2, false), (50, 0x8000, 2, false), (-50, 0, 1, true), (-50, 0x200, 2, true), (-50, 0x8000, 2, true)]; + +fn spec(frame_length: i16, flags: u16) -> Spec { + let mut spec = Spec::new(4, 4); + spec.audio = Some(Audio { sample_rate: 22050, frame_length, buffers: 5, flags }); + spec +} + +fn decoder(frame_length: i16, flags: u16) -> Decoder { + Decoder::new(&build(&spec(frame_length, flags), &[])).unwrap() +} + +/// One block of random sound: its bytes in the file and the samples it decodes to. +fn block(rng: &mut Rng, d: &Decoder) -> (Vec, Vec) { + let n = d.block_samples() * d.channels(); + if d.bits() == 8 { + let bytes: Vec = (0..n).map(|_| rng.next_u64() as u8).collect(); + let samples = bytes.iter().map(|&b| (i16::from(b) - 0x80) << 8).collect(); + return (bytes, samples); + } + // a random walk, so that the steps are of every size + let mut last = [0i32; 2]; + let samples: Vec = (0..n) + .map(|i| { + let bits = 1 + rng.below(15); + let step = rng.below(1 << bits) as i32 - (1 << (bits - 1)); + let s = &mut last[i % d.channels()]; + *s = (*s + step).clamp(-32768, 32767); + *s as i16 + }) + .collect(); + dpcm_pack(&samples, d.channels()) +} + +/// A file of sound frames (with a small picture each, as ffmpeg expects video) and the samples +/// expected of it. The first frame is a mask frame: one silent block, then four of sound. +fn movie(seed: u64, frame_length: i16, flags: u16) -> (Vec, Vec) { + let spec = spec(frame_length, flags); + let d = decoder(frame_length, flags); + let mut rng = Rng(seed | 1); + let mut picture = vec![2u8]; + picture.extend([9; 16]); + let silence = vec![0i16; d.block_samples() * d.channels()]; + let mut want = Vec::new(); + let mut blocks = Vec::new(); + for n in 0..12 { + let sound = match n { + 0 => { + let mut data = 1u32.to_le_bytes().to_vec(); + want.extend_from_slice(&silence); + for _ in 0..4 { + let (bytes, samples) = block(&mut rng, &d); + data.extend_from_slice(&bytes); + want.extend_from_slice(&samples); + } + Frame::audio(2, data) + } + _ if rng.below(4) == 0 => { + want.extend_from_slice(&silence); + Frame::audio(3, vec![]) + } + _ => { + let (bytes, samples) = block(&mut rng, &d); + want.extend_from_slice(&samples); + Frame::audio(1, bytes) + } + }; + blocks.push(vec![sound, Frame::video([0, 0, 3, 3], false, picture.clone())]); + } + (build(&spec, &blocks), want) +} + +fn decode_all(bytes: &[u8]) -> Vec { + let file = open(bytes).unwrap(); + let d = Decoder::new(file.header_bytes()).unwrap(); + let mut out = Vec::new(); + for (i, frame) in file.frames.iter().enumerate() { + if let FrameKind::Audio { flags } = frame.kind { + out.extend(d.decode(flags, &file.read_frame(bytes, i).unwrap()).unwrap()); + } + } + out +} + +#[test] +fn decodes_the_samples_that_were_packed() { + for (i, (frame_length, flags, channels, sixteen)) in FORMATS.into_iter().enumerate() { + let d = decoder(frame_length, flags); + assert_eq!((d.channels(), d.bits(), d.sample_rate()), (channels, if sixteen { 16 } else { 8 }, 22050), "format {i}"); + // old-style stereo counts both channels in the frame length + assert_eq!(d.block_samples(), if flags & 0x8000 != 0 { 25 } else { 50 }, "format {i}"); + let (bytes, want) = movie(i as u64 + 1, frame_length, flags); + assert_eq!(want.len(), 16 * d.block_samples() * channels); + assert!(decode_all(&bytes) == want, "format {i}"); + } +} + +#[test] +fn matches_ffmpeg() { + let Some(ffmpeg) = craft_media_testkit::ffmpeg_or_skip("vmd sound oracle") else { return }; + let dir = craft_media_testkit::private_dir("vmd-sound"); + for (i, (frame_length, flags, ..)) in FORMATS.into_iter().enumerate() { + let (bytes, _) = movie(i as u64 + 1, frame_length, flags); + let path = dir.join(format!("sound{i}.vmd")); + std::fs::write(&path, &bytes).unwrap(); + let out = Command::new(&ffmpeg).args(["-v", "error", "-i"]).arg(&path).args(["-vn", "-f", "s16le", "-"]).output().unwrap(); + assert!(out.status.success() && out.stderr.is_empty(), "format {i}: ffmpeg: {}", String::from_utf8_lossy(&out.stderr)); + let theirs: Vec = out.stdout.as_chunks::<2>().0.iter().map(|b| i16::from_le_bytes(*b)).collect(); + assert!(theirs == decode_all(&bytes), "format {i} differs from ffmpeg"); + } + std::fs::remove_dir_all(&dir).unwrap(); +} + +#[test] +fn sixteen_bit_steps() { + // values ffmpeg gave for this block: the first sample, then signed steps from the table + let mut data = 1000i16.to_le_bytes().to_vec(); + data.extend([1, 1, 1, 0x81, 0x81, 2, 3, 0x7F, 0xFF, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]); + let d = decoder(-20, 0); + assert_eq!(d.block_bytes(), 21); + assert_eq!( + d.decode(1, &data).unwrap(), + [1000, 1008, 1016, 1024, 1016, 1008, 1024, 1056, 17440, 1056, 1056, 1056, 1056, 1056, 1056, 1056, 1056, 1056, 1056, 1064] + ); + // stereo: the channels take turns + let mut data = [1000i16.to_le_bytes(), (-1000i16).to_le_bytes()].concat(); + data.extend([1, 0x82].repeat(3)); + let d = decoder(-4, 0x200); + assert_eq!(d.decode(1, &data).unwrap(), [1000, -1000, 1008, -1016, 1016, -1032, 1024, -1048]); + // steps past 16 bits stop at the ends + let mut data = 30000i16.to_le_bytes().to_vec(); + data.extend([0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF]); + assert_eq!(decoder(-7, 0).decode(1, &data).unwrap(), [30000, 32767, 16383, -1, -16385, -32768, -32768]); +} + +#[test] +fn eight_bit_and_silence() { + let d = decoder(4, 0); + assert_eq!(d.decode(1, &[0x80, 0x00, 0xFF, 0x70]).unwrap(), [0, -32768, 0x7F00, -0x1000]); + assert_eq!(d.decode(3, &[]).unwrap(), [0; 4]); + assert_eq!(d.decode(1, &[]).unwrap(), [0i16; 0]); + assert_eq!(decoder(4, 0x200).decode(3, &[]).unwrap(), [0; 8]); + assert_eq!(decoder(-4, 0x8000).decode(3, &[]).unwrap(), [0; 4]); +} + +#[test] +fn silence_mask_runs_bit_by_bit() { + let d = decoder(2, 0); + let frame = |mask: u32, data: &[u8]| d.decode(2, &[&mask.to_le_bytes()[..], data].concat()); + let (a, b) = ([0x90, 0x91], [0xA0, 0xA1]); + let (sa, sb) = ([0x1000, 0x1100], [0x2000, 0x2100]); + assert_eq!(frame(0, &[a, b].concat()).unwrap(), [sa, sb].concat()); + // silent, sound, silent, sound + assert_eq!(frame(0b0101, &[a, b].concat()).unwrap(), [[0, 0], sa, [0, 0], sb].concat()); + // game files set a bit for every block when all are silent + assert_eq!(frame(0b0111, &[]).unwrap(), [0; 6]); + // a trailing part of a block is left out + assert_eq!(frame(0, &[0x90, 0x91, 0xA0]).unwrap(), sa); + assert_eq!(d.decode(2, &[0, 0, 0]), Err(Error::Invalid("the silence mask is cut short"))); +} + +#[test] +fn bad_input_is_an_error() { + let d = decoder(-20, 0); + assert_eq!(d.decode(1, &[0; 20]), Err(Error::Invalid("a block of sound is cut short"))); + assert_eq!(d.decode(4, &[0; 21]), Err(Error::Unsupported("unknown sound frame flags"))); + + let good = build(&spec(20, 0), &[]); + let with = |patch: &dyn Fn(&mut Vec)| { + let mut h = good.clone(); + patch(&mut h); + Decoder::new(&h).err() + }; + assert_eq!(with(&|_| {}), None); + assert_eq!(with(&|h| h.truncate(815)), Some(Error::Invalid("the header is shorter than 816 bytes"))); + assert_eq!(with(&|h| h[0] = 0), Some(Error::Invalid("not a VMD header"))); + assert_eq!(with(&|h| h[17] = 0), Some(Error::Invalid("the file has no sound"))); + assert_eq!(with(&|h| h[810] = 0x10), Some(Error::Unsupported("IMA ADPCM sound"))); + // a frame length of nothing decodes to nothing + let mut empty = good.clone(); + empty[806..808].fill(0); + let d = Decoder::new(&empty).unwrap(); + assert_eq!(d.decode(1, &[1, 2, 3]).unwrap(), [0i16; 0]); + assert_eq!(d.decode(2, &[0, 0, 0, 0, 1, 2]).unwrap(), [0i16; 0]); +} + +#[test] +fn hostile_input_never_panics() { + for (i, (frame_length, flags, ..)) in FORMATS.into_iter().enumerate() { + let (bytes, _) = movie(i as u64 + 1, frame_length, flags); + craft_media_testkit::hostile::hammer(&format!("vmd sound format {i}"), &bytes, 10_000, |d| { + let Ok(file) = open(d) else { return }; + let Ok(decoder) = Decoder::new(file.header_bytes()) else { return }; + for (i, frame) in file.frames.iter().enumerate() { + let FrameKind::Audio { flags } = frame.kind else { continue }; + if let Ok(data) = file.read_frame(d, i) { + let _ = decoder.decode(flags, &data); + } + } + }); + } +} + +/// Real files, for whoever has some: the sound of every `.vmd` in the directory +/// `CRAFT_MEDIA_VMD_SAMPLES` names must decode to the samples ffmpeg gets. Game files are not +/// redistributable, so none are in the repository and the test is skipped without the variable. +#[test] +fn real_files_match_ffmpeg() { + let Some(dir) = std::env::var_os("CRAFT_MEDIA_VMD_SAMPLES").filter(|d| !d.is_empty()) else { + craft_media_testkit::skip_test("vmd samples", "CRAFT_MEDIA_VMD_SAMPLES is not set"); + return; + }; + let Some(ffmpeg) = craft_media_testkit::ffmpeg_or_skip("vmd sound samples") else { return }; + let mut paths: Vec<_> = std::fs::read_dir(&dir) + .unwrap() + .filter_map(|e| Some(e.ok()?.path())) + .filter(|p| p.extension().is_some_and(|e| e.eq_ignore_ascii_case("vmd"))) + .collect(); + paths.sort(); + assert!(!paths.is_empty(), "no .vmd files in CRAFT_MEDIA_VMD_SAMPLES"); + let (mut files, mut samples) = (0, 0); + for path in &paths { + let name = path.display(); + let source = std::fs::File::open(path).unwrap(); + let file = open(&source).unwrap(); + if file.audio_params().is_none() { + continue; + } + let d = Decoder::new(file.header_bytes()).unwrap(); + let mut ours = Vec::new(); + for (i, frame) in file.frames.iter().enumerate() { + let FrameKind::Audio { flags } = frame.kind else { continue }; + let mut data = file.read_frame(&source, i).unwrap(); + if flags == 2 && data.len() >= 4 { + // ffmpeg plays a mask frame's silent blocks before its sound; put ours in that + // order to compare + let silent = u32::from_le_bytes([data[0], data[1], data[2], data[3]]).count_ones() as usize; + ours.resize(ours.len() + silent * d.block_samples() * d.channels(), 0); + data[..4].fill(0); + } + ours.extend(d.decode(flags, &data).unwrap_or_else(|e| panic!("{name}: frame {i}: {e}"))); + } + let out = Command::new(&ffmpeg).args(["-v", "quiet", "-i"]).arg(path).args(["-vn", "-f", "s16le", "-"]).output().unwrap(); + let theirs: Vec = out.stdout.as_chunks::<2>().0.iter().map(|b| i16::from_le_bytes(*b)).collect(); + if let Some(at) = (0..ours.len().max(theirs.len())).find(|&i| ours.get(i) != theirs.get(i)) { + panic!("{name}: sample {at} differs from ffmpeg ({} samples here, {} there)", ours.len(), theirs.len()); + } + files += 1; + samples += ours.len(); + } + println!("{files} files, {samples} samples match ffmpeg"); +} diff --git a/crates/media-testkit/src/lib.rs b/crates/media-testkit/src/lib.rs index 6d3a75525..8894014dd 100644 --- a/crates/media-testkit/src/lib.rs +++ b/crates/media-testkit/src/lib.rs @@ -13,11 +13,13 @@ //! //! [`hostile::hammer`] runs a parser over truncations and mutations of a valid input and fails on //! a panic or a slow run; `CRAFT_MEDIA_HOSTILE_MULT=10` runs ten times the iterations. +//! [`vmd`] writes Sierra VMD fixtures, which ffmpeg can read but not write. //! //! Failing loudly is what a test helper is for: a wrong `CRAFT_MEDIA_*` setting or an unwritable //! fixture directory panics, which fails the test that asked. pub mod hostile; +pub mod vmd; use std::path::{Path, PathBuf}; use std::process::Command; diff --git a/crates/media-testkit/src/vmd.rs b/crates/media-testkit/src/vmd.rs new file mode 100644 index 000000000..c7068f19c --- /dev/null +++ b/crates/media-testkit/src/vmd.rs @@ -0,0 +1,343 @@ +//! Sierra VMD fixture writer. ffmpeg reads VMD but cannot write it, so the VMD tests build their +//! files here and use ffmpeg only to decode them. Written from the public format description +//! (); the packers favour coverage of every coding +//! mode over compression. + +use crate::hostile::Rng; + +/// Bytes of the file header. +pub const HEADER_LEN: usize = 816; + +/// The audio fields of the header. +#[derive(Clone, Copy, Debug)] +pub struct Audio { + /// Samples per second. + pub sample_rate: u16, + /// The frame length field as stored: negative for 16-bit sound. + pub frame_length: i16, + /// Number of sound buffers. + pub buffers: u16, + /// Audio flags (0x8000 old stereo, 0x200 new stereo). + pub flags: u16, +} + +/// What the header says. +#[derive(Clone, Debug)] +pub struct Spec { + /// Picture width in pixels. + pub width: u16, + /// Picture height in pixels. + pub height: u16, + /// Left coordinate of the picture. + pub left: u16, + /// Top coordinate of the picture. + pub top: u16, + /// 256 R-G-B triplets of 6-bit components. + pub palette: [u8; 768], + /// Size of the unpack buffer for LZ-packed frames. + pub unpack: u32, + /// Video codec version (header bytes 4-5). + pub codec_version: u16, + /// The sound, if the file has any. + pub audio: Option