From 9101d242b3cfdda7489e7fa17bcb1e6f02745be3 Mon Sep 17 00:00:00 2001 From: Skef Iterum Date: Sun, 25 Jan 2026 23:52:51 -0800 Subject: [PATCH] Update depend API docs and add explicit test mode to fuzzer Documentation updates: - Add comprehensive context_set documentation to depend-api.md with encoding details and example code for check_context_satisfied() - Update depend-for-closure.md: remove obsolete over-approximation analysis, reflect that recent testing shows no over-approximation with proper context_set filtering - Update depend-implementation.md: correct descriptions of active glyph filtering, sequential accumulation, ligature sets, and context recording to match current implementation - Fix terminology: change 'binning' to 'segmentation' throughout - Fix hb-depend-data.hh:557: use HB_CODEPOINT_INVALID instead of 0 for layout_tag default value Fuzzer enhancements (hb-depend-closure-parity): - Add explicit test mode with command-line options similar to hb-subset conventions: --unicodes=LIST Unicode codepoints (U+XXXX, decimal, ranges) --gids=LIST Glyph IDs (decimal, ranges) --no-layout-closure Skip GSUB closure --layout-features=LIST GSUB features (comma-separated or *) - Support multiple input formats: --unicodes=U+0041,U+0042,U+0043 --unicodes=65-90 --gids=10-50 --no-layout-closure --layout-features=liga,calt or --layout-features='*' - Add robust input parsing with validation to prevent infinite loops on invalid input (e.g., --gids=abc or --unicodes=U+GGGG) - Add error checking for conflicting options This allows reproducing random test cases manually and testing specific scenarios without finding the right seed. Tested: All modes produce identical results to seed-based tests. --- docs/depend-api.md | 134 ++- docs/depend-for-closure.md | 502 ++------ docs/depend-implementation.md | 94 +- src/hb-depend-data.hh | 296 ++++- src/hb-depend.cc | 12 +- src/hb-depend.h | 3 +- src/hb-depend.hh | 6 +- src/hb-ot-layout-gsub-table.hh | 5 + src/hb-ot-layout-gsubgpos.hh | 253 +++- test/api/test-ot-depend.c | 11 +- test/fuzzing/hb-depend-closure-parity-main.cc | 182 ++- test/fuzzing/hb-depend-closure-parity.cc | 1042 ++++++++++++----- test/fuzzing/hb-depend-fuzzer.cc | 5 +- 13 files changed, 1719 insertions(+), 826 deletions(-) diff --git a/docs/depend-api.md b/docs/depend-api.md index 5839d0230..f11091de2 100644 --- a/docs/depend-api.md +++ b/docs/depend-api.md @@ -42,11 +42,12 @@ hb_tag_t table_tag; hb_codepoint_t dependent; hb_tag_t layout_tag; hb_codepoint_t ligature_set; +hb_codepoint_t context_set; printf("Dependencies for glyph %u:\n", gid); while (hb_depend_get_glyph_entry(depend, gid, index++, &table_tag, &dependent, - &layout_tag, &ligature_set)) { + &layout_tag, &ligature_set, &context_set)) { printf(" -> glyph %u via %c%c%c%c\n", dependent, HB_UNTAG(table_tag)); } @@ -69,11 +70,12 @@ hb_tag_t table_tag; hb_codepoint_t dependent; hb_tag_t layout_tag; hb_codepoint_t ligature_set; +hb_codepoint_t context_set; // Iterate through all dependencies for this glyph while (hb_depend_get_glyph_entry(depend, gid, index, &table_tag, &dependent, - &layout_tag, &ligature_set)) { + &layout_tag, &ligature_set, &context_set)) { // Process dependency information if (table_tag == HB_OT_TAG_GSUB) { // GSUB dependency: layout_tag contains feature tag @@ -112,10 +114,11 @@ void find_reachable_glyphs(hb_depend_t *depend, hb_set_t *reachable) hb_codepoint_t dependent; hb_tag_t layout_tag; hb_codepoint_t ligature_set; + hb_codepoint_t context_set; while (hb_depend_get_glyph_entry(depend, gid, index++, &table_tag, &dependent, - &layout_tag, &ligature_set)) { + &layout_tag, &ligature_set, &context_set)) { if (!hb_set_has(reachable, dependent)) { hb_set_add(reachable, dependent); hb_set_add(to_process, dependent); @@ -137,12 +140,13 @@ printf("Found %u reachable glyphs\n", hb_set_get_population(reachable)); hb_set_destroy(reachable); ``` -**Note:** This simple algorithm doesn't handle ligature dependencies correctly. -Ligatures should only be added when all component glyphs in their ligature set -are present. For a production-quality implementation that correctly handles -ligatures and feature filtering to compute closures similar to HarfBuzz's subset -API, see `compute_depend_closure()` in `test/fuzzing/hb-depend-fuzzer.cc` and -`docs/depend-for-closure.md`. +**Note:** This simple algorithm doesn't handle ligature dependencies or context sets +correctly. Ligatures should only be added when all component glyphs in their ligature +set are present, and contextual dependencies should only be followed when their positional +requirements are satisfied. For a production-quality implementation that correctly handles +ligatures, context filtering, and feature filtering to compute closures similar to +HarfBuzz's subset API, see `compute_depend_closure()` in `test/fuzzing/hb-depend-closure-parity.cc` +and `docs/depend-for-closure.md`. ## Working with Ligature Sets @@ -165,10 +169,11 @@ if (hb_depend_get_set_from_index(depend, ligature_set, ligature_glyphs)) { hb_codepoint_t lig_dependent; hb_tag_t lig_layout_tag; hb_codepoint_t lig_ligature_set; + hb_codepoint_t lig_context_set; while (hb_depend_get_glyph_entry(depend, lig_gid, lig_index++, &lig_table_tag, &lig_dependent, - &lig_layout_tag, &lig_ligature_set)) { + &lig_layout_tag, &lig_ligature_set, &lig_context_set)) { if (lig_ligature_set == ligature_set) { printf(" gid=%u, index=%u\n", lig_gid, lig_index - 1); break; @@ -179,6 +184,103 @@ if (hb_depend_get_set_from_index(depend, ligature_set, ligature_glyphs)) { hb_set_destroy(ligature_glyphs); ``` +## Working with Context Sets + +Context and ChainContext GSUB rules apply lookups only when specific backtrack and/or +lookahead glyphs are present. The depend API records these requirements in context_set +indices as optional information that can refine dependency traversal. + +**Note:** Context sets are optional refinement information. Uses of the dependency graph +that ignore context_set (treating it as always satisfied) work fine - they produce a +conservative over-approximation. Context sets allow more precise closure computation when +desired. + +### Understanding Context Sets + +When `context_set != HB_CODEPOINT_INVALID`, the edge includes positional requirements +from a Context or ChainContext rule. Context set elements use two encodings: + +1. **Direct GID reference** (value < 0x80000000): A specific glyph ID +2. **Indirect set reference** (value >= 0x80000000): An index into the sets array + (mask off high bit with `value & 0x7FFFFFFF` to get the set index) + +The high bit (0x80000000) distinguishes between direct and indirect references. For an +edge to apply, ALL elements in the context set must be satisfied: +- Direct references: that specific glyph must be present in the closure +- Indirect references: at least ONE glyph from the referenced set must be present (disjunction) + +### Checking Context Requirements + +```c +bool check_context_satisfied(hb_depend_t *depend, + hb_codepoint_t context_set_idx, + hb_set_t *current_closure) +{ + if (context_set_idx == HB_CODEPOINT_INVALID) + return true; // No context requirements + + hb_set_t *context_elements = hb_set_create(); + if (!hb_depend_get_set_from_index(depend, context_set_idx, context_elements)) + { + hb_set_destroy(context_elements); + return false; // Error retrieving context + } + + bool satisfied = true; + hb_codepoint_t elem = HB_SET_VALUE_INVALID; + + while (hb_set_next(context_elements, &elem)) + { + if (elem < 0x80000000) + { + // Direct reference: check if specific glyph is in closure + if (!hb_set_has(current_closure, elem)) + { + satisfied = false; + break; + } + } + else + { + // Indirect reference: check if ANY glyph from set is in closure + hb_codepoint_t set_idx = elem & 0x7FFFFFFF; + hb_set_t *required_set = hb_set_create(); + + if (hb_depend_get_set_from_index(depend, set_idx, required_set)) + { + // Check if ANY element from required_set is in current_closure + bool any_found = false; + hb_codepoint_t gid = HB_SET_VALUE_INVALID; + while (hb_set_next(required_set, &gid)) + { + if (hb_set_has(current_closure, gid)) + { + any_found = true; + break; + } + } + if (!any_found) + satisfied = false; + } + else + { + satisfied = false; // Error retrieving set + } + + hb_set_destroy(required_set); + if (!satisfied) + break; + } + } + + hb_set_destroy(context_elements); + return satisfied; +} +``` + +For a complete implementation of context-aware closure computation, see the +`compute_depend_closure()` function in `test/fuzzing/hb-depend-closure-parity.cc`. + # API Details ## Dependency Entry Fields @@ -186,7 +288,8 @@ hb_set_destroy(ligature_glyphs); Each dependency entry returned by `hb_depend_get_glyph_entry()` contains: - **table_tag**: Source table (e.g., `HB_OT_TAG_GSUB`, `HB_TAG('c','m','a','p')`, - `HB_TAG('g','l','y','f')`, `HB_TAG('C','O','L','R')`, `HB_TAG('M','A','T','H')`) + `HB_TAG('g','l','y','f')`, `HB_TAG('C','F','F',' ')`, `HB_TAG('C','O','L','R')`, + `HB_TAG('M','A','T','H')`) - **dependent**: The dependent glyph ID - **layout_tag**: - For GSUB: the feature tag (e.g., `HB_TAG('l','i','g','a')`) @@ -194,6 +297,11 @@ Each dependency entry returned by `hb_depend_get_glyph_entry()` contains: - Otherwise: `HB_CODEPOINT_INVALID` - **ligature_set**: For ligatures, identifies which ligature set; otherwise `HB_CODEPOINT_INVALID` +- **context_set**: Optional refinement information for Context and ChainContext GSUB rules. + When not `HB_CODEPOINT_INVALID`, identifies positional requirements (backtrack/lookahead) + for this dependency. Can be ignored (treated as always satisfied) for conservative + over-approximation, or checked for more precise closure computation. See "Working with + Context Sets" section for details. *(Experimental feature from Phase 3 implementation)* ## Compilation @@ -246,9 +354,9 @@ characters, accounting for all OpenType substitutions and compositions. Analyze which features or scripts require which glyphs, useful for font optimization and planning. -## Font Binning +## Font Segmentation -Partition a font into smaller subsets where each bin contains glyphs reachable +Partition a font into smaller subsets where each segment contains glyphs reachable from a specific set of input characters, enabling more efficient font delivery for web applications. diff --git a/docs/depend-for-closure.md b/docs/depend-for-closure.md index 64483d1e5..5e1aac835 100644 --- a/docs/depend-for-closure.md +++ b/docs/depend-for-closure.md @@ -2,21 +2,12 @@ The depend API can be used to compute glyph closures - determining which glyphs are reachable from a given set of input codepoints and active features. This -document explains how to compute closures using the depend graph, provides a -reference implementation, and characterizes the over-approximation that results -from this approach compared to subset-based closure. +document explains how to compute closures using the depend graph and provides a +reference implementation. ## Computing Closure with Depend -The depend API can be used to compute an approximation of the glyph closure that -HarfBuzz's subset API would produce for a given set of input codepoints and -active features. While the depend-based closure will include all glyphs that -subset would include (no false negatives), it typically includes additional -glyphs that subset would not (over-approximation). - -### Basic Approach - -To emulate a subset closure using the depend graph: +To compute a closure using the depend graph: 1. **Map input codepoints to starting glyphs** using the cmap table 2. **Initialize the reachable set** with these starting glyphs @@ -24,467 +15,142 @@ To emulate a subset closure using the depend graph: 4. **Filter by active features** to only follow edges for enabled GSUB features 5. **Handle ligatures correctly** - only add ligature outputs when all component glyphs are present +6. **Check context requirements** - only follow edges when positional requirements + (backtrack/lookahead) are satisfied -### Why Depend Over-Approximates +### Context Set Filtering -The depend graph answers: "If this glyph is present and this feature is active, -what other glyphs **might** be needed?" It records all possible outputs from -GSUB substitution rules without evaluating contextual conditions. +Context and ChainContext GSUB rules have positional requirements encoded in the +context_set field. Each context_set specifies which backtrack and/or lookahead +glyphs must be present for a dependency edge to apply. -In contrast, subset performs actual text shaping and only includes glyphs that -are **actually** reached for the specific input. Context-dependent rules (context -substitutions, chaining context substitutions) only trigger when specific neighbor -glyphs are present in the actual input sequence. +When context_set filtering is implemented, recent testing has not generated any +over-approximation cases - the depend-based closure matches the subset-based +closure in all tested scenarios. -**Result**: Depend closures are **safe but conservative** - they guarantee all -needed glyphs are included (no under-approximation) but may include extra glyphs -that won't actually be used for the specific input. +**With context_set filtering:** Recent testing shows no over-approximation +**Without context_set filtering:** Conservative over-approximation (safe but may include extra glyphs) + +See `docs/depend-api.md` section "Working with Context Sets" for details on the +context_set encoding and how to check context requirements. ### Reference Implementation The test suite includes a production-quality implementation of depend-based -closure in `test/fuzzing/hb-depend-fuzzer.cc`. The `compute_depend_closure()` +closure in `test/fuzzing/hb-depend-closure-parity.cc`. The `compute_depend_closure()` function demonstrates proper handling of: - Feature filtering (only following edges for active GSUB features) - Ligature sets (only adding ligature outputs when all components present) -- Non-GSUB dependencies (cmap, glyf, COLR, MATH) - -See `test/fuzzing/hb-depend-fuzzer.cc` lines 417-552 for the complete implementation. +- Context sets (only following edges when positional requirements satisfied) +- Non-GSUB dependencies (cmap, glyf, CFF, COLR, MATH) Key aspects of the implementation: ```c++ // Feature filtering - only follow GSUB edges for active features if (table_tag == HB_OT_TAG_GSUB) { - if (!hb_set_has(active_features, layout_tag)) + if (active_features && !hb_set_has(active_features, layout_tag)) continue; // Skip edge if feature not active } +// Context filtering - only follow edges when requirements satisfied +if (context_set != HB_CODEPOINT_INVALID) { + if (!check_context_satisfied(depend, context_set, glyphs)) + continue; // Skip edge if context not satisfied +} + // Ligature handling - only add ligature when all components present if (ligature_set != HB_CODEPOINT_INVALID) { - // Get all glyphs in this ligature set hb_set_t *ligature_glyphs = hb_set_create(); hb_depend_get_set_from_index(depend, ligature_set, ligature_glyphs); // Check if all component glyphs are in reachable set - bool all_present = true; - hb_codepoint_t lig_gid = HB_SET_VALUE_INVALID; - while (hb_set_next(ligature_glyphs, &lig_gid)) { - if (!hb_set_has(reachable, lig_gid)) { - all_present = false; - break; - } - } - - if (!all_present) + if (!hb_set_is_subset(ligature_glyphs, glyphs)) { + hb_set_destroy(ligature_glyphs); continue; // Skip ligature if not all components present + } + hb_set_destroy(ligature_glyphs); +} + +// Add dependent glyph to closure +if (!hb_set_has(glyphs, dependent)) { + hb_set_add(glyphs, dependent); + hb_set_add(to_process, dependent); } ``` -## Understanding Over-Approximation +## Optional Over-Approximation -When using depend for closure computation, the resulting closure typically includes -more glyphs than subset would include. To characterize these over-approximation -patterns and their scale, we analyzed 2,100 cases across 7 fonts representing -6 script families. The analysis collected cases where depend closure included -glyphs that subset closure did not include, then categorized these cases by -mechanism and measured their frequency and size. +You can choose to skip context_set checking for: -### Key Finding: GSUB-Only Phenomenon +- **Simpler implementation** - Fewer steps, less complex code +- **Coverage analysis** - Finding all glyphs that participate in a feature, regardless + of whether they're reachable for specific input +- **Conservative estimates** - Ensuring no glyphs are missed -**100% of over-approximations are GSUB-related.** Analysis of 350 deeply-analyzed -cases found: -- GSUB tests: 350 cases (100%) -- Non-GSUB tests: 0 cases (0%) +Skipping context_set produces a safe over-approximation - all needed glyphs are +included, possibly with some additional glyphs that wouldn't be reached for the +specific input. -Over-approximation does not occur with non-GSUB dependencies: -- Character mapping (cmap) - exact -- Composite glyphs (glyf) - exact -- Color layers (COLR) - exact -- Math variants (MATH) - exact - -Over-approximation is fundamentally a contextual shaping phenomenon arising from -GSUB substitution rules that have contextual conditions. - -## Categories of Over-Approximation - -Analysis identified five primary categories that explain 95% of observed cases: - -### 1. Positional Forms + Ligatures (21.4% of cases) - -**Average:** 33.7 extra glyphs per case (range 1-191) - -**Mechanism:** -- Arabic and related scripts have 2-4 positional variants per letter: initial - (`init`), medial (`medi`), final (`fina`), isolated (`isol`) -- Most positional variants lack independent codepoint mappings and are only - accessible through GSUB features -- Required ligatures (`rlig`) add additional context-dependent forms -- Depend includes ALL positional variants and ligature outputs when features - are active -- Subset includes only the specific positional variant for each letter's actual - position in the input text - -**Scripts affected:** -- Arabic (Nastaliq calligraphic style): 78% of over-approximation cases -- Arabic (Kufi geometric style): 72% of over-approximation cases - -**Example:** -``` -Arabic letter (U+0647) -- Base form (glyph 48): HAS codepoint U+0647 -- Initial form (glyph 992): NO codepoint, only via 'init' feature -- Medial form (glyph 1015): NO codepoint, only via 'medi' feature -- Final form (glyph 1030): NO codepoint, only via 'fina' feature - -Depend: If glyph 48 + 'init' active → include all variants (992, 1015, 1030) -Subset: Only includes glyph 992 if input has letter in initial position -``` - -**Features involved:** -- `ccmp` (66%), `fina` (66%), `medi` (64%), `init` (62%), `isol` (62%), `rlig` (39%) - -**Size note:** Nastaliq (calligraphic) shows ~4× larger over-approximation than -Kufi (geometric) due to more extensive contextual variants. - ---- - -### 2. Indic Complex Syllables (18.6% of cases) - -**Average:** 37.2 extra glyphs per case (range 1-216) -**HIGHEST average over-approximation** - -**Mechanism:** -- Indic scripts have complex syllable structure with pre-base, post-base, - above-base, below-base forms -- Consonant conjuncts (`cjct`) have many conditional forms based on syllable - structure -- Marks have position-dependent variants (above/below/pre/post base) -- Half-forms and special forms depend on syllable composition -- Syllable reordering rules are highly contextual -- Depend includes all possible syllable component variants when 4+ Indic features - are active -- Subset includes only components for the actual syllable structure in input text - -**Scripts affected:** -- Oriya: 84% of over-approximation cases (47.1 avg glyphs per case!) -- Multi-script fonts when Indic features active: 46% of cases - -**Notable pattern:** -- Oriya has LOW over-approximation *rate* (2.5% of random tests) but HIGH *size* - when it occurs -- Interpretation: Oriya's GSUB rules are highly contextual - rarely triggered - by random input, but when they are, many glyphs participate - -**Features involved:** -- `cjct` (72%), `blwf` (78%), `abvs` (76%), `blws` (70%), `nukt` (72%), - `haln` (78%), `pres` (78%), `psts` (74%) - -**Extreme cases:** Up to 216 extra glyphs observed in Oriya font - ---- - -### 3. Mark Positioning (19.4% of cases) - -**Average:** 19.1 extra glyphs per case (range 1-105) - -**Mechanism:** -- Above-base marks (`abvs`) have variants depending on base character shape -- Below-base marks (`blws`, `blwf`) have position-dependent forms -- Pre-base and post-base marks (`pres`, `psts`) adjust to context -- Localized forms (`locl`) add script-specific positioning variants -- Depend includes all mark variants when mark positioning features are active -- Subset includes only variants for the actual base characters in input text - -**Scripts affected:** -- Myanmar: 96% of over-approximation cases (21.8 avg glyphs per case) - - This is THE PRIMARY cause of Myanmar's 27.8% over-approximation rate -- Oriya: 16% of cases (when fewer than 4 Indic features active) -- Multi-script fonts: 24% of cases - -**Features involved:** -- `blws` (86%), `locl` (78%), `abvs` (72%), `blwf` (68%) - -**Note:** Mark positioning is about MARK glyphs with position-dependent variants. -This is distinct from positional forms (category 1), which is about BASE glyphs -with positional variants. - ---- - -### 4. Ligatures Only (24.3% of cases) - -**Average:** 2.7 extra glyphs per case (range 1-13) -**LARGEST category by case count, SMALLEST by average size** - -**Mechanism:** -- Standard ligatures (`liga`): common ligatures like fi, fl, ffi, ffl -- Required ligatures (`rlig`): mandatory ligatures for proper rendering -- Discretionary ligatures (`dlig`): optional typographic ligatures -- Contextual ligatures (`clig`): ligatures with contextual conditions -- Some ligatures require specific neighbor glyphs to form -- Depend includes all ligature outputs when ligature features are active -- Subset only forms ligatures when the full component sequence is present in - input text - -**Scripts affected:** -- Hebrew: 84% of over-approximation cases (3.6 avg glyphs per case) -- Thai: 80% of over-approximation cases (2.1 avg glyphs per case) -- Some Arabic: 6% of cases - -**Features involved:** -- `ccmp` (74%), `aalt` (65%), `dlig` (42%), `rlig` (36%), `liga` (36%), `locl` (33%) - -**Size note:** Very small over-approximation, typically 1-3 glyphs. Hebrew median: -3 glyphs. Thai median: 1 glyph. - ---- - -### 5. Typographic Alternates (4.9% of cases) - -**Average:** 4.1 extra glyphs per case (range 1-24) - -**Mechanism:** -- Small caps (`smcp`, `c2sc`): each letter has a small-cap variant -- Fractions (`frac`, `numr`, `dnom`): numbers have numerator/denominator variants -- Superscripts/subscripts (`sups`, `subs`): superior/inferior forms -- Number styles (`onum`, `lnum`, `pnum`, `tnum`): oldstyle/lining, - proportional/tabular variants -- Special forms (`zero` slashed zero, `ordn` ordinal indicators) -- Depend includes all typographic variants when features are active -- Subset includes only the variant actually used for the specific text - -**Scripts affected:** -- Multi-script fonts with Latin support: 24% of over-approximation cases -- Arabic-Kufi: 10% of cases (includes `numr`, `dnom` for Eastern Arabic numerals) - -**Features involved:** -- `numr` (7), `dnom` (6), `onum` (5), `zero` (5), `ordn` (4), `c2sc` (4) - -**Example:** Letter "A" might have regular A, small-cap A, superscript A, -subscript A. Depend includes all if `c2sc`, `subs`, `sups` are active. Subset -includes only the variant needed for the specific text. - ---- - -## Script-Specific Patterns - -### Arabic Scripts (Nastaliq vs Kufi) - -**Arabic-Nastaliq** (22.9% over-approximation rate) -- Average: 41.2 extra glyphs per case -- Categories: 78% Positional+Ligatures, 10% Positional Only -- Calligraphic style with extensive contextual variants - -**Arabic-Kufi** (14.8% over-approximation rate) -- Average: 10.6 extra glyphs per case -- Categories: 72% Positional+Ligatures, 10% Typographic -- Geometric style with fewer contextual forms - -**Key difference:** Nastaliq's calligraphic nature creates ~4× more -over-approximation than Kufi. - -### Myanmar (27.8% rate - HIGHEST) - -**Myanmar/Burmese** -- Average: 21.8 extra glyphs per case -- Categories: 96% Mark Positioning -- Features: `blws` (86%), `locl` (78%), `abvs` (72%), `blwf` (68%) - -**Why highest rate:** Extensive above-base and below-base mark positioning with -many position-dependent variants. Almost ALL over-approximations fall in a single -category (mark positioning). - -### Indic Scripts (Oriya) - -**Oriya** (2.5% rate - LOWEST, but 47.1 avg glyphs - HIGHEST SIZE) -- Average: 47.1 extra glyphs per case (highest!) -- Categories: 84% Indic Complex, 16% Mark Positioning -- Features: 10 Indic features commonly active - -**Paradox explained:** LOW rate means over-approximation rarely occurs. HIGH size -means when it DOES occur, it's large. This suggests Oriya's GSUB rules are highly -contextual - most random tests don't trigger complex syllable rules, but when -rules ARE triggered, many glyphs participate. - -### Hebrew (9.4% rate) - -**Hebrew** -- Average: 3.6 extra glyphs per case -- Categories: 84% Ligatures Only, 14% Contextual Only -- Features: `ccmp` (86%), `dlig` (84%), `aalt` (64%) - -**Pattern:** Discretionary ligatures are primary cause. Very small -over-approximation size (1-3 glyphs typically). Related to Arabic but much -simpler (no positional forms). - -### Thai (9.2% rate) - -**Thai** -- Average: 2.1 extra glyphs per case (2nd lowest) -- Categories: 80% Ligatures Only, 10% Stylistic -- Features: `ccmp` (94%), `aalt` (78%), `liga` (72%), `locl` (66%), `rlig` (66%) - -**Pattern:** Primarily ligatures and composition. Very small over-approximation -(median: 1 glyph!). - -### Latin (in multi-script fonts) - -**NotoSans-Bold** (9.2% rate) -- Average: 6.9 extra glyphs per case -- Categories: 46% Indic Complex (!), 24% Mark Positioning, 24% Typographic - -**Important note:** NotoSans-Bold is a multi-script font supporting Latin + -Indic scripts. 70% of over-approximations are NOT Latin features, but Indic -features in the font. True Latin over-approximations (24%): typographic alternates -(small caps, fractions, etc.). - -### CJK and Simple Scripts - -**CJK (Chinese, Japanese, Korean)** - 0% over-approximation -- Primarily ideographic characters without contextual shaping -- GSUB features mainly for glyph variants (one-to-one mappings) -- No contextual conditions, thus no over-approximation - -**Simple Latin fonts** - 0% over-approximation -- Fonts with minimal or no GSUB features -- Basic ligatures (fi, fl) without contextual conditions - -**Emoji fonts** - 0% over-approximation -- Use COLR (Color Layer) table, not GSUB -- No contextual shaping - -**Math fonts** - 0% over-approximation -- Use MATH table for size variants, not GSUB contextual rules - -## Frequency and Scale - -### Overall Statistics (2,100 cases across 7 fonts) - -**Size Distribution:** -- Tiny (1-2 glyphs): 35.4% of cases -- Small (3-10 glyphs): 40.7% of cases -- Medium (11-50 glyphs): 11.6% of cases -- Large (51-100 glyphs): 9.7% of cases -- Very large (100+ glyphs): 2.6% of cases - -**By Script Family:** - -| Script | Over-approx Rate | Avg Glyphs | Median | Range | -|--------|------------------|------------|--------|-------| -| Myanmar | 27.8% | 21.8 | 13 | 1-161 | -| Arabic (Nastaliq) | 22.9% | 41.2 | 21 | 1-191 | -| Arabic (Kufi) | 14.8% | 10.6 | 5 | 1-87 | -| Hebrew | 9.4% | 3.6 | 3 | 1-16 | -| Thai | 9.2% | 2.1 | 1 | 1-12 | -| Latin (multi-script) | 9.2% | 6.9 | 3 | 1-86 | -| Oriya | 2.5% | 47.1 | 27 | 1-231 | - -### Feature Co-occurrence - -Most common feature pairs that appear together in over-approximating cases: - -1. **`abvs` + `blwf`** (23.4%) - Above-base + below-base forms (Indic, Myanmar) -2. **`aalt` + `ccmp`** (22.9%) - Alternates + composition (Hebrew, Thai) -3. **`blwf` + `blws`** (22.0%) - Below-base forms + substitutions (Indic, Myanmar) -4. **`abvs` + `blws`** (20.6%) - Above/below-base (Indic, Myanmar) -5. **`ccmp` + `rlig`** (19.4%) - Composition + required ligatures (Arabic) - -Feature co-occurrence reveals script families: -- Arabic: positional features (`init`, `medi`, `fina`, `isol`) + `ccmp` + `rlig` -- Indic/Myanmar: mark features (`abvs`, `blws`, `blwf`) cluster together -- Hebrew/Thai: `aalt` + `ccmp` + ligatures - -## Impact on Use Cases +## Use Cases ### Font Subsetting -**For complex-script fonts:** -- Depend-based subset will render correctly but may be 5-15% larger than - subset-based subset -- Arabic fonts: 10-15% overhead typical -- Indic fonts: Varies widely (2-28% rate × 20-50 glyphs average) -- Myanmar fonts: 28% of subsets affected, 20 glyphs average overhead +Determine which glyphs must be retained to properly render a specific set of +characters, accounting for all OpenType substitutions and compositions. -**For Latin-only fonts:** -- Minimal or no overhead if font lacks contextual GSUB features -- Fonts with extensive typographic features: 5-10% overhead possible - -**For CJK, emoji, math fonts:** -- No overhead (zero over-approximation) +With context_set filtering, depend-based subsetting produces results that match +subset-based subsetting in recent testing. ### Coverage Analysis -Over-approximation may be DESIRABLE for coverage analysis. It accurately reflects -"what glyphs participate in this feature" even if not all are reachable for -every input. This is often the desired behavior for: -- Feature analysis and optimization -- Understanding font coverage -- Planning font modifications +Analyze which features or scripts require which glyphs. For this use case, +skipping context_set filtering may be desirable - it shows all glyphs that +participate in a feature, not just those reachable for specific input. -### Font Binning +### Font Segmentation -For web font delivery with code point binning: -- Depend provides upper bound on required glyphs per bin -- Bins may be slightly larger than strictly necessary (5-15% for complex scripts) -- Trade-off: Simpler computation vs slightly larger bins -- May be acceptable depending on delivery constraints +Partition a font into smaller subsets where each segment contains glyphs reachable +from a specific set of input characters, enabling more efficient font delivery +for web applications. -## Recommendations +### Testing -### When to Use Depend for Closure +Verify that font modifications haven't inadvertently broken glyph references or +substitution chains. The depend graph provides a complete view of font structure +that can be compared before and after modifications. -**Good fit:** -- Coverage analysis and feature analysis -- Conservative estimates where missing glyphs is unacceptable -- Performance-sensitive applications (depend is faster than full shaping) -- Applications where 5-15% size overhead is acceptable +## Implementation Notes -**Poor fit:** -- Minimum-size subsetting where every byte matters -- Complex-script fonts where precision is critical -- Applications that need exact glyph lists +### Non-GSUB Dependencies -### Mitigating Over-Approximation +Dependencies from non-GSUB tables do not require context filtering: +- **Character mapping (cmap)**: Direct Unicode to glyph mappings +- **Composite glyphs (glyf, CFF)**: Structural component relationships +- **Color layers (COLR)**: Layer composition +- **Math variants (MATH)**: Size variant relationships -If using depend for closure computation: +These dependencies should always be followed during closure computation. -1. **Filter by active features** - Always filter GSUB edges by active features. - Don't include dependencies for inactive features. +### Feature Filtering -2. **Handle ligatures correctly** - Only add ligature outputs when all component - glyphs are present. See reference implementation in `test/fuzzing/hb-depend-fuzzer.cc`. +GSUB edges should be filtered by active features. The layout_tag field contains +the feature tag for GSUB dependencies. Only follow edges where the feature is +active in your shaping configuration. -3. **Consider script** - For Latin-only or CJK fonts, over-approximation is - minimal. For Arabic/Indic/Myanmar, be aware of 10-50% overhead. +### Ligature Filtering -4. **Hybrid approach** - Use depend for initial coverage analysis, then subset - for final optimization if size is critical. - -### Testing for Over-Approximation - -To test a font for over-approximation characteristics: - -```bash -# Build with depend API enabled -meson setup build -Ddepend_api=true -meson compile -C build - -# Run fuzzer with -r flag to report over-approximations -build/test/fuzzing/hb-depend-fuzzer -r -n 1000 your-font.ttf - -# Analyze output -grep "Depend found .* extra glyphs" output.txt | wc -l # Count cases -``` +Ligature dependencies should only be followed when ALL component glyphs in the +ligature_set are present in the current closure. This prevents adding ligature +outputs prematurely. ## Further Reading -For implementation details, data structures, and performance characteristics, see: -- `docs/depend-implementation.md` - Implementation details -- `docs/depend-api.md` - API usage guide -- `test/fuzzing/hb-depend-fuzzer.cc` - Reference implementation +For detailed API documentation: +- `docs/depend-api.md` - API usage guide with examples +- `docs/depend-implementation.md` - Implementation details and architecture -For detailed analysis methodology and raw data: -- Analysis code and data: Contact HarfBuzz maintainers -- 2,100 cases analyzed across 7 fonts representing 6 script families -- Stratified sampling with 350 deep-analyzed cases +For reference implementation: +- `test/fuzzing/hb-depend-closure-parity.cc` - Production-quality closure computation +- `test/api/test-ot-depend.c` - Unit tests for depend API diff --git a/docs/depend-implementation.md b/docs/depend-implementation.md index d6bf71b54..a45f96e6d 100644 --- a/docs/depend-implementation.md +++ b/docs/depend-implementation.md @@ -23,27 +23,44 @@ The closure only tells you what's reachable from a specific starting point. The ## Graph Extraction Strategy -The depend implementation uses a simple strategy for graph extraction: it processes the entire font structure once to record all possible dependency edges. +The depend implementation extracts the complete dependency graph by walking the +font structure once to record all possible dependency edges. -The `context_depend_recurse_lookups()` function (around line 1827) handles contextual and chaining contextual lookups during graph extraction. +### Active Glyph Filtering -### Graph Extraction Implementation +During graph extraction, depend filters by active glyphs using `parent_active_glyphs()` +(same as closure does). This prevents recording spurious edges for glyphs that can +never participate in a particular substitution. -During graph construction, depend walks the font structure similarly to closure, but with a critical difference: +**Critical difference from closure:** Depend does NOT do sequential accumulation. +In a contextual rule with multiple lookups, closure accumulates outputs from earlier +lookups as inputs to later lookups. Depend records edges from ALL lookups independently +to capture the complete graph structure. -**Goal**: Record ALL possible dependency edges that exist in the font. +### Context Requirement Recording -**Strategy**: Every nested lookup sees ALL glyphs in the font (via `c->glyphs`), not just contextually relevant ones. +For Context and ChainContext GSUB rules, depend records positional requirements +(backtrack/lookahead) in context_set indices. This allows filtering during closure +computation - edges are only followed when their positional requirements are satisfied. -**Implementation**: Uses simple recursion without sequential accumulation. Each lookup in a contextual rule independently processes the full glyph universe. +Implementation in `context_depend_recurse_lookups()` (hb-ot-layout-gsubgpos.hh): +1. Build backtrack and lookahead glyph sets for the rule +2. Call `build_context_set()` to store these requirements +3. Set `current_context_set_index` so nested edges inherit the context +4. Recurse into nested lookups +5. Clear context index when done -**Why process all glyphs?**: We want to record all edges that exist in the font structure, not just those reachable from a specific starting set. Limiting glyph visibility during graph extraction would cause us to miss edges. +### Graph Completeness -The tradeoff is that graph extraction may record edges that are "theoretically possible but practically unreachable", but this is acceptable - it's better to over-approximate than under-approximate the dependency graph. +The depend graph captures all edges that exist in the font structure. Recent testing +with proper ligature_set, context_set, and feature filtering has not generated +over-approximation cases when computing closures. ### Using the Graph for Closure Computation -To compute closures using the extracted graph, see `depend-for-closure.md` which documents how the fuzzer implements closure by iteratively following edges in the dependency graph. +To compute closures using the extracted graph, see `depend-for-closure.md` which +documents how to implement closure by iteratively following edges in the dependency +graph with appropriate filtering. ## Data Structures @@ -64,14 +81,16 @@ Depend builds a persistent graph with deduplication: struct hb_depend_data_t { hb_vector_t glyph_dependencies; // Per-glyph edge lists hb_map_t> edge_hashes; // Deduplication - hb_vector_t sets; // Ligature sets + hb_vector_t sets; // Ligature and context sets hb_map_t lookup_features[256]; // Feature tracking + hb_codepoint_t current_context_set_index; // Current context during extraction }; ``` **Key features**: - **Deduplication**: Hash-based duplicate detection prevents storing the same edge multiple times -- **Ligature sets**: Tracks alternate glyphs in ligature sets (e.g., for smart quotes) +- **Ligature sets**: Tracks complete set of component glyphs needed for each ligature +- **Context sets**: Records backtrack/lookahead requirements for contextual rules - **Feature tracking**: Records which features activate which lookups - **Persistent storage**: Graph survives beyond initial extraction @@ -93,11 +112,11 @@ This reduces memory footprint while keeping the graph queryable for the lifetime Dependencies are stored per-glyph and indexed sequentially starting from 0. Use `hb_depend_get_glyph_entry()` with incrementing index values to iterate through all dependencies for a given glyph. -## Context Filtering +## Context Filtering and Recording ### Closure Behavior -Closure filters glyphs by context but doesn't track the filtering: +Closure filters glyphs by context during execution: ```c++ // In ContextSubst Format 1 @@ -111,7 +130,7 @@ const auto &input_coverage = this+coverageZ[0]; ### Depend Behavior -Depend filters glyphs AND records which context they came from: +Depend filters glyphs AND records positional requirements: ```c++ // In ContextSubst Format 1 depend method @@ -125,9 +144,26 @@ Depend filters glyphs AND records which context they came from: ; ``` -The `value` parameter identifies which glyph from the coverage was matched, allowing the depend code to filter nested lookups to only glyphs reachable from that specific context. +The `value` parameter identifies which glyph from the coverage was matched, allowing +the depend code to filter nested lookups appropriately. -This filtering is critical for correctness - it prevents recording spurious edges for glyphs that can never actually participate in a particular contextual substitution. +Additionally, for Context and ChainContext rules, depend builds and records context_set: + +```c++ +// Build context set for this rule (in context_depend_recurse_lookups) +hb_codepoint_t context_set_idx = c->depend_data->build_context_set( + &lookahead_sets, nullptr); // or both backtrack and lookahead +c->depend_data->current_context_set_index = context_set_idx; + +// Recurse into lookups - all edges inherit this context_set +context_depend_recurse_lookups(...); + +// Clear context after processing +c->depend_data->current_context_set_index = HB_CODEPOINT_INVALID; +``` + +This records which backtrack and lookahead glyphs must be present for each edge, +enabling accurate closure computation that matches subset behavior in recent testing. ## Feature Tagging @@ -166,18 +202,25 @@ This is stored in the `layout_tag` field of each dependency record and can be qu ## Ligature Sets -For alternate glyphs (like smart quotes or stylistic alternates), depend tracks which alternates belong to the same "set". This is stored as a set of glyphs indexed by a `ligature_set` ID. +For ligature substitutions, depend tracks the complete set of component glyphs needed +to form each ligature. This is stored as a set of glyphs indexed by a `ligature_set` ID. ### Why This Matters -Consider a substitution rule: +Consider a ligature substitution rule: ``` -"ff" → "ff_ligature" +"f" + "f" → "ff_ligature" ``` -If a font contains the ligature, then both 'f' characters depend on the ligature. But more importantly, if you're subsetting and keeping one 'f', you need to keep BOTH 'f' glyphs (even if they're different alternates) to maintain the ability to form the ligature. +The ligature output depends on BOTH component glyphs. During closure computation, the +ligature should only be added when ALL components are present in the closure. -The ligature_set field groups these related glyphs so subsetting tools can handle them correctly. +The ligature_set field identifies which components must be present together. All edges +for the same ligature (from each component to the ligature output) share the same +ligature_set index, allowing closure computation to check if all components are available +before adding the ligature output. + +This enables correct ligature handling during closure computation. ## COLR Cycles and Self-References @@ -269,10 +312,11 @@ This keeps related code together and makes it easier to maintain consistency. | **Goal** | Compute reachable glyphs from a starting set | Extract complete dependency graph | | **Output** | Modified glyph set | Persistent graph with edges | | **Repeated queries** | Must re-walk font each time | O(1) edge lookups in graph | -| **Glyph visibility** | Filters glyphs contextually during lookup processing | Processes all glyphs to capture complete graph | -| **Context filtering** | Filters but doesn't track | Filters AND tracks context | +| **Active glyph filtering** | Filters by active glyphs during traversal | Filters by parent_active_glyphs() during extraction | +| **Sequential accumulation** | Accumulates outputs for next lookup | Does NOT accumulate (records all edges) | +| **Context filtering** | Filters during execution | Records requirements in context_set | | **Feature tagging** | Not tracked | Tracks which features activate edges | -| **Ligature sets** | Not tracked | Groups related alternate glyphs | +| **Ligature sets** | Not tracked | Tracks complete component sets | | **Deduplication** | Not needed | Hash-based to prevent duplicate edges | | **Memory management** | Minimal - no persistent graph | Temporary structures freed after construction | | **Space complexity** | O(1) | O(edges) | diff --git a/src/hb-depend-data.hh b/src/hb-depend-data.hh index 85ca980a3..85c24abd9 100644 --- a/src/hb-depend-data.hh +++ b/src/hb-depend-data.hh @@ -109,33 +109,70 @@ struct hb_lookup_feature_record_t { */ struct hb_depend_data_t { - hb_depend_data_t () {} + hb_depend_data_t () +#ifdef HB_DEPEND_API + : current_context_set_index (HB_CODEPOINT_INVALID) +#endif + {} ~hb_depend_data_t () {} void free_set (hb_codepoint_t set_index) { - // Should only be called to free the most recently allocated set - if (set_index + 1 == sets.length) { - sets.resize(sets.length - 1); // Pop it off - } else { - // Unexpected - freeing a set that's not the last one - DEBUG_MSG (SUBSET, nullptr, "Attempting to free set %u but last set is %u", + if (set_index >= sets.length) + { + DEBUG_MSG (SUBSET, nullptr, "Attempting to free invalid set %u (max is %u)", set_index, sets.length - 1); + return; } + + /* Add to free list for reuse */ + free_set_list.push (set_index); + + /* Clear the set to save memory */ + sets[set_index].clear (); } hb_codepoint_t new_set (hb_codepoint_t cp) { - hb_codepoint_t set_index = sets.length; - sets.push(std::initializer_list({cp})); + hb_codepoint_t set_index; + + if (free_set_list.length > 0) + { + /* Reuse freed set */ + set_index = free_set_list[free_set_list.length - 1]; + free_set_list.resize (free_set_list.length - 1); + sets[set_index].clear (); + sets[set_index].add (cp); + } + else + { + /* Allocate new set */ + set_index = sets.length; + sets.push (std::initializer_list({cp})); + } + return set_index; } hb_codepoint_t new_set (hb_set_t &set) { - hb_codepoint_t set_index = sets.length; - sets.push(set); + hb_codepoint_t set_index; + + if (free_set_list.length > 0) + { + /* Reuse freed set */ + set_index = free_set_list[free_set_list.length - 1]; + free_set_list.resize (free_set_list.length - 1); + sets[set_index].set (set); + } + else + { + /* Allocate new set */ + set_index = sets.length; + sets.push (set); + } + return set_index; } @@ -148,6 +185,140 @@ struct hb_depend_data_t return set_index; } + /* Compute hash of a set's contents (for deduplication) */ + uint64_t compute_set_hash (const hb_set_t &set) const + { + uint64_t hash = 0; + hb_codepoint_t cp = HB_SET_VALUE_INVALID; + /* Need to use non-const iteration since next() modifies internal state */ + hb_set_t temp_set; + temp_set.set (set); + while (temp_set.next (&cp)) + hash = hash * 31 + hb_hash (cp); + return hash; + } + + /* Find existing set with same contents, or create new one (with deduplication). + * Use this for context sets. For ligature sets, call new_set() directly. */ + hb_codepoint_t find_or_create_set (const hb_set_t &set) + { + uint64_t hash = compute_set_hash (set); + + /* Check if we already have this set */ + if (set_hashes.has (hash)) + { + hb_codepoint_t existing_idx = set_hashes.get (hash); + if (sets[existing_idx] == set) + return existing_idx; /* Reuse existing set */ + + /* Hash collision - different set, same hash */ + /* Fall through to create new set */ + } + + /* Create new set and store in hash table */ + hb_codepoint_t new_idx = new_set (const_cast(set)); + set_hashes.set (hash, new_idx); + return new_idx; + } + +#ifdef HB_DEPEND_API + /* Build a context set from context information. + * Encodes backtrack and lookahead requirements as a flattened set. + * Returns HB_CODEPOINT_INVALID if no context. + * + * To ensure canonical encoding and avoid redundancy: + * 1. First pass: collect all direct (single-glyph) requirements + * 2. Second pass: create disjunction sets, subtracting direct requirements + * 3. Combine: add direct requirements and filtered disjunction references + */ + hb_codepoint_t build_context_set (const hb_vector_t *backtrack_sets, + const hb_vector_t *lookahead_sets) + { + if ((!backtrack_sets || backtrack_sets->length == 0) && + (!lookahead_sets || lookahead_sets->length == 0)) + return HB_CODEPOINT_INVALID; + + /* First pass: collect all direct (single-glyph) requirements */ + hb_set_t direct_requirements; + + if (backtrack_sets) + { + for (const auto &back_set : *backtrack_sets) + { + if (back_set.get_population () == 1) + { + hb_codepoint_t gid = back_set.get_min (); + direct_requirements.add (gid); + } + } + } + + if (lookahead_sets) + { + for (const auto &look_set : *lookahead_sets) + { + if (look_set.get_population () == 1) + { + hb_codepoint_t gid = look_set.get_min (); + direct_requirements.add (gid); + } + } + } + + /* Second pass: create disjunction sets, filtering out direct requirements */ + hb_set_t context_elements; + + if (backtrack_sets) + { + for (const auto &back_set : *backtrack_sets) + { + if (back_set.get_population () > 1) + { + /* Multiple glyphs - filter and create set reference */ + hb_set_t filtered_set; + filtered_set.set (back_set); + filtered_set.subtract (direct_requirements); + + if (!filtered_set.is_empty ()) + { + hb_codepoint_t set_idx = find_or_create_set (filtered_set); + context_elements.add (0x80000000 | set_idx); + } + } + } + } + + if (lookahead_sets) + { + for (const auto &look_set : *lookahead_sets) + { + if (look_set.get_population () > 1) + { + /* Multiple glyphs - filter and create set reference */ + hb_set_t filtered_set; + filtered_set.set (look_set); + filtered_set.subtract (direct_requirements); + + if (!filtered_set.is_empty ()) + { + hb_codepoint_t set_idx = find_or_create_set (filtered_set); + context_elements.add (0x80000000 | set_idx); + } + } + } + } + + /* Add direct requirements */ + context_elements.union_ (direct_requirements); + + if (context_elements.is_empty ()) + return HB_CODEPOINT_INVALID; + + /* Store context elements with deduplication */ + return find_or_create_set (context_elements); + } +#endif + void print() { for (unsigned i = 0; i < glyph_dependencies.length; i++) { @@ -174,7 +345,8 @@ struct hb_depend_data_t hb_tag_t table_tag, hb_tag_t layout_tag, hb_codepoint_t dependent, - hb_codepoint_t lig_set_index) + hb_codepoint_t lig_set_index, + hb_codepoint_t context_set_index) { hb_vector_t components; components.push(source); @@ -185,7 +357,21 @@ struct hb_depend_data_t // If there's a ligature set, add its contents in canonical order if (lig_set_index != HB_CODEPOINT_INVALID) { hb_codepoint_t cp = HB_SET_VALUE_INVALID; - while (sets[lig_set_index].next(&cp)) { + /* Need temp copy because next() modifies iterator state */ + hb_set_t temp_set; + temp_set.set (sets[lig_set_index]); + while (temp_set.next(&cp)) { + components.push(cp); + } + } + + // If there's a context set, add its contents in canonical order + if (context_set_index != HB_CODEPOINT_INVALID) { + hb_codepoint_t cp = HB_SET_VALUE_INVALID; + /* Need temp copy because next() modifies iterator state */ + hb_set_t temp_set; + temp_set.set (sets[context_set_index]); + while (temp_set.next(&cp)) { components.push(cp); } } @@ -197,7 +383,8 @@ struct hb_depend_data_t hb_tag_t table_tag, hb_tag_t layout_tag, hb_codepoint_t dependent, - hb_codepoint_t lig_set_index) + hb_codepoint_t lig_set_index, + hb_codepoint_t context_set_index) { // Polynomial rolling hash with prime multiplier (31) uint64_t hash = hb_hash(source); @@ -208,7 +395,21 @@ struct hb_depend_data_t // If there's a ligature set, incorporate its contents in canonical order if (lig_set_index != HB_CODEPOINT_INVALID) { hb_codepoint_t cp = HB_SET_VALUE_INVALID; - while (sets[lig_set_index].next(&cp)) { + /* Need temp copy because next() modifies iterator state */ + hb_set_t temp_set; + temp_set.set (sets[lig_set_index]); + while (temp_set.next(&cp)) { + hash = hash * 31 + hb_hash(cp); + } + } + + // If there's a context set, incorporate its contents in canonical order + if (context_set_index != HB_CODEPOINT_INVALID) { + hb_codepoint_t cp = HB_SET_VALUE_INVALID; + /* Need temp copy because next() modifies iterator state */ + hb_set_t temp_set; + temp_set.set (sets[context_set_index]); + while (temp_set.next(&cp)) { hash = hash * 31 + hb_hash(cp); } } @@ -221,27 +422,40 @@ struct hb_depend_data_t hb_tag_t layout_tag, hb_codepoint_t dependent, hb_codepoint_t lig_set, + hb_codepoint_t context_set, const hb_depend_data_record_t &existing) { - // Check basic fields (not including context_set) + // Check basic fields if (existing.table_tag != table_tag || existing.layout_tag != layout_tag || existing.dependent != dependent) return false; // Check ligature sets - if (existing.ligature_set == HB_CODEPOINT_INVALID && lig_set == HB_CODEPOINT_INVALID) - return true; + if (existing.ligature_set != lig_set) + { + if (existing.ligature_set == HB_CODEPOINT_INVALID || lig_set == HB_CODEPOINT_INVALID) + return false; + if (!(sets[existing.ligature_set] == sets[lig_set])) + return false; + } - if (existing.ligature_set != HB_CODEPOINT_INVALID && lig_set != HB_CODEPOINT_INVALID) - return sets[existing.ligature_set] == sets[lig_set]; + // Check context sets + if (existing.context_set != context_set) + { + if (existing.context_set == HB_CODEPOINT_INVALID || context_set == HB_CODEPOINT_INVALID) + return false; + if (!(sets[existing.context_set] == sets[context_set])) + return false; + } - return false; + return true; } bool get_glyph_entry(hb_codepoint_t gid, hb_codepoint_t index, hb_tag_t *table_tag, hb_codepoint_t *dependent, - hb_tag_t *layout_tag, hb_codepoint_t *ligature_set) + hb_tag_t *layout_tag, hb_codepoint_t *ligature_set, + hb_codepoint_t *context_set) { if (gid < glyph_dependencies.length && index < glyph_dependencies[gid].dependencies.length) { @@ -250,6 +464,7 @@ struct hb_depend_data_t *dependent = d.dependent; *layout_tag = d.layout_tag; *ligature_set = d.ligature_set; + *context_set = d.context_set; return true; } return false; @@ -276,11 +491,11 @@ struct hb_depend_data_t return false; } - // Compute edge hash for deduplication (without context_set) - uint64_t edge_hash = compute_edge_hash(target, table_tag, layout_tag, dependent, lig_set); + // Compute edge hash for deduplication (including context_set) + uint64_t edge_hash = compute_edge_hash(target, table_tag, layout_tag, dependent, lig_set, context_set); // Get component vector for this edge - auto new_components = get_edge_components(target, table_tag, layout_tag, dependent, lig_set); + auto new_components = get_edge_components(target, table_tag, layout_tag, dependent, lig_set, context_set); // Check if we've already seen this hash (O(1) hash table lookup) if (edge_hashes.has(edge_hash)) { @@ -288,22 +503,14 @@ struct hb_depend_data_t const auto &stored_components = edge_hashes.get(edge_hash); bool components_match = (stored_components == new_components); - // DEBUG: Check if vector comparison might be wrong - if (components_match && stored_components.length != new_components.length) { - printf("[BUG] Vector lengths differ but comparison says equal! stored=%u new=%u\n", - (unsigned)stored_components.length, (unsigned)new_components.length); - } - if (components_match) { return false; // True duplicate - same hash, same components } else { // Hash collision! Same hash but different edge components // Fall back to O(n) search through this glyph's dependencies - // real_collisions++; // TODO: Remove before production - auto &gdr = glyph_dependencies[target]; for (auto &dep : gdr.dependencies) { - if (edges_equal(target, table_tag, layout_tag, dependent, lig_set, dep)) { + if (edges_equal(target, table_tag, layout_tag, dependent, lig_set, context_set, dep)) { return false; // Found via fallback } } @@ -328,6 +535,12 @@ struct hb_depend_data_t hb_codepoint_t lig_set = HB_CODEPOINT_INVALID, hb_codepoint_t context_set = HB_CODEPOINT_INVALID) { +#ifdef HB_DEPEND_API + /* Use pre-computed context set index for the current rule */ + if (context_set == HB_CODEPOINT_INVALID) + context_set = current_context_set_index; +#endif + bool any_added = false; for (auto t : lookup_features[lookup_index]) { if (add_depend_layout(target, HB_OT_TAG_GSUB, t, dependent, lig_set, context_set)) @@ -341,7 +554,7 @@ struct hb_depend_data_t hb_codepoint_t lig_set = HB_CODEPOINT_INVALID, hb_codepoint_t context_set = HB_CODEPOINT_INVALID) { - add_depend_layout (target, table_tag, 0, dependent, lig_set, + add_depend_layout (target, table_tag, HB_CODEPOINT_INVALID, dependent, lig_set, context_set); } @@ -354,12 +567,23 @@ struct hb_depend_data_t hb_map_t nominal_glyphs; hb_vector_t sets; + hb_vector_t free_set_list; /* Indices of freed sets for reuse */ hb_vector_t glyph_dependencies; hb_vector_t lookup_features; - // Hash-based deduplication: edge hash -> component vector for collision detection + /* Hash-based deduplication: edge hash -> component vector for collision detection */ hb_hashmap_t> edge_hashes; + + /* Set deduplication: set_hash -> set_index (for context sets) */ + hb_hashmap_t set_hashes; + +#ifdef HB_DEPEND_API + /* Temporary: Context set index for the current rule. + * Built once per rule by context_depend_lookup/chain_context_depend_lookup, + * then used by all edges recorded from nested lookups. */ + hb_codepoint_t current_context_set_index; +#endif }; #endif /* HB_DEPEND_API */ diff --git a/src/hb-depend.cc b/src/hb-depend.cc index 9df101866..debeed93c 100644 --- a/src/hb-depend.cc +++ b/src/hb-depend.cc @@ -85,6 +85,7 @@ hb_depend_t::hb_depend_t (hb_face_t *f) /* Free temporary structures - no longer needed after graph extraction */ data.edge_hashes.fini(); + data.set_hashes.fini(); data.unicodes.fini(); data.nominal_glyphs.fini(); for (auto &fs : data.lookup_features) @@ -335,6 +336,10 @@ hb_depend_from_face (hb_face_t *face) * Otherwise is HB_CODEPOINT_INVALID * @ligature_set: (out): Returns the index of the ligature set for this entry, or * HB_CODEPOINT_INVALID if there is no such set + * @context_set: (out): Returns the index of the context set for this entry, or + * HB_CODEPOINT_INVALID if there is no such set. Context sets specify which + * backtrack and lookahead glyphs must be present for this dependency to apply + * (currently only populated for contextual GSUB substitutions). * * Get the values associated with a dependency entry for a glyph. * Dependencies are indexed sequentially starting from 0. @@ -346,10 +351,11 @@ hb_depend_from_face (hb_face_t *face) * hb_codepoint_t dependent; * hb_tag_t layout_tag; * hb_codepoint_t ligature_set; + * hb_codepoint_t context_set; * * while (hb_depend_get_glyph_entry (depend, gid, index++, * &table_tag, &dependent, - * &layout_tag, &ligature_set)) { + * &layout_tag, &ligature_set, &context_set)) { * // Process dependency... * } * ``` @@ -362,10 +368,10 @@ hb_bool_t hb_depend_get_glyph_entry(hb_depend_t *depend, hb_codepoint_t gid, hb_codepoint_t index, hb_tag_t *table_tag, hb_codepoint_t *dependent, hb_tag_t *layout_tag, - hb_codepoint_t *ligature_set) + hb_codepoint_t *ligature_set, hb_codepoint_t *context_set) { return depend->get_glyph_entry(gid, index, table_tag, dependent, layout_tag, - ligature_set); + ligature_set, context_set); } /** diff --git a/src/hb-depend.h b/src/hb-depend.h index 6042d9e9c..944b57dad 100644 --- a/src/hb-depend.h +++ b/src/hb-depend.h @@ -58,7 +58,8 @@ hb_depend_get_glyph_entry(hb_depend_t *depend, hb_tag_t *table_tag, /* OUT */ hb_codepoint_t *dependent, /* OUT */ hb_tag_t *layout_tag, /* OUT */ - hb_codepoint_t *ligature_set /* OUT */); + hb_codepoint_t *ligature_set, /* OUT */ + hb_codepoint_t *context_set /* OUT */); HB_EXTERN hb_bool_t hb_depend_get_set_from_index(hb_depend_t *depend, hb_codepoint_t index, diff --git a/src/hb-depend.hh b/src/hb-depend.hh index abfef23a8..4911d4f04 100644 --- a/src/hb-depend.hh +++ b/src/hb-depend.hh @@ -63,12 +63,14 @@ struct hb_depend_t } void print() { data.print(); } + bool get_glyph_entry(hb_codepoint_t gid, hb_codepoint_t index, hb_tag_t *table_tag, hb_codepoint_t *dependent, - hb_tag_t *layout_tag, hb_codepoint_t *ligature_set) + hb_tag_t *layout_tag, hb_codepoint_t *ligature_set, + hb_codepoint_t *context_set) { return data.get_glyph_entry(gid, index, table_tag, dependent, layout_tag, - ligature_set); + ligature_set, context_set); } bool get_set_from_index(hb_codepoint_t index, hb_set_t *out) { diff --git a/src/hb-ot-layout-gsub-table.hh b/src/hb-ot-layout-gsub-table.hh index 1378a915b..7e36ead07 100644 --- a/src/hb-ot-layout-gsub-table.hh +++ b/src/hb-ot-layout-gsub-table.hh @@ -62,6 +62,11 @@ template /*static*/ typename hb_depend_context_t::return_t SubstLookup::depend_glyphs_recurse_func (hb_depend_context_t *c, unsigned lookup_index, hb_set_t *covered_seq_indices, unsigned seq_index, unsigned end_index) { const SubstLookup &l = c->face->table.GSUB.get_relaxed ()->table->get_lookup (lookup_index); + /* After a non-1-to-1 lookup (expansion/contraction), subsequent lookups in the same + * contextual rule see all glyphs, not position-specific ones. This matches closure behavior. + * Mark sequence positions as covered so later lookups use the full glyph set. */ + if (l.may_have_non_1to1 ()) + hb_set_add_range (covered_seq_indices, seq_index, end_index); return l.dispatch (c); } #endif diff --git a/src/hb-ot-layout-gsubgpos.hh b/src/hb-ot-layout-gsubgpos.hh index c52bd2375..26849e340 100644 --- a/src/hb-ot-layout-gsubgpos.hh +++ b/src/hb-ot-layout-gsubgpos.hh @@ -418,10 +418,33 @@ struct hb_depend_context_t : return true; } + /* Context information for recording contextual dependencies */ + struct context_info_t { + hb_vector_t backtrack_sets; /* Sets of glyphs in backtrack positions */ + hb_vector_t lookahead_sets; /* Sets of glyphs in lookahead positions */ + }; + + void push_context (context_info_t &&ctx) + { + context_stack.push (std::move (ctx)); + } + + void pop_context () + { + if (context_stack.length > 0) + context_stack.resize (context_stack.length - 1); + } + + const context_info_t* current_context () const + { + return context_stack.length > 0 ? &context_stack[context_stack.length - 1] : nullptr; + } + hb_depend_data_t *depend_data; hb_face_t *face; hb_set_t *glyphs; hb_vector_t active_glyphs_stack; + hb_vector_t context_stack; /* Stack of context information */ recurse_func_t recurse_func; hb_codepoint_t lookup_index; hb_set_t lookups_seen; @@ -1832,32 +1855,65 @@ static void context_depend_recurse_lookups (hb_depend_context_t *c, intersected_glyphs_func_t intersected_glyphs_func, void *cache) { -#ifndef NDEBUG - if (unlikely (getenv ("HB_DEPEND_DEBUG_MODE"))) { - fprintf (stderr, "DEPEND: context_depend_recurse: parent_active=%u, lookups=%u\n", - c->parent_active_glyphs().get_population(), lookupCount); - } -#endif + /* For depend graph extraction, we filter active glyphs by InputCoverage constraints + * (same as closure), but we do NOT do sequential accumulation of outputs. + * + * - Position filtering (InputCoverage): Required. The contextual rule can only match + * glyphs in its InputCoverage. Edges for other glyphs are spurious. + * - Sequential accumulation: Not done. We want ALL possible edges, so we don't + * limit later lookups based on outputs from earlier lookups in the same rule. */ - /* During graph extraction, we use simple recursion without sequential accumulation. - * We want to record ALL possible dependencies, so every lookup sees ALL glyphs - * (parent_active_glyphs). Sequential accumulation would limit the active glyphs - * based on outputs, causing us to miss edges. */ hb_set_t covered_seq_indicies; + hb_set_t pos_glyphs; for (unsigned int i = 0; i < lookupCount; i++) { unsigned seqIndex = lookupRecord[i].sequenceIndex; if (seqIndex >= inputCount) continue; - covered_seq_indicies.add (seqIndex); + bool has_pos_glyphs = false; + if (!covered_seq_indicies.has (seqIndex)) + { + has_pos_glyphs = true; + pos_glyphs.clear (); + if (seqIndex == 0) + { + switch (context_format) { + case ContextFormat::SimpleContext: + pos_glyphs.add (value); + break; + case ContextFormat::ClassBasedContext: + intersected_glyphs_func (&c->parent_active_glyphs (), data, value, &pos_glyphs, cache); + break; + case ContextFormat::CoverageBasedContext: + pos_glyphs.set (c->parent_active_glyphs ()); + break; + } + } + else + { + const void *input_data = input; + unsigned input_value = seqIndex - 1; + if (context_format != ContextFormat::SimpleContext) + { + input_data = data; + input_value = input[seqIndex - 1]; + } + + intersected_glyphs_func (c->glyphs, input_data, input_value, &pos_glyphs, cache); + } + } + + covered_seq_indicies.add (seqIndex); hb_set_t *cur_active_glyphs = c->push_cur_active_glyphs (); if (unlikely (!cur_active_glyphs)) return; - - // For graph extraction, pass ALL glyphs (the universe) to ensure nested contexts see everything - cur_active_glyphs->set (*c->glyphs); + if (has_pos_glyphs) { + *cur_active_glyphs = std::move (pos_glyphs); + } else { + *cur_active_glyphs = *c->glyphs; + } unsigned endIndex = inputCount; if (context_format == ContextFormat::CoverageBasedContext) @@ -2103,17 +2159,57 @@ static inline void context_depend_lookup (hb_depend_context_t *c, { // ContextDependLookupContext has the same layout as ContextClosureLookupContext auto &closure_lookup_context = reinterpret_cast(lookup_context); - if (context_intersects (c->glyphs, - inputCount, input, - closure_lookup_context)) - context_depend_recurse_lookups (c, - inputCount, input, - lookupCount, lookupRecord, - value, - lookup_context.context_format, - lookup_context.intersects_data, - lookup_context.funcs.intersected_glyphs, - lookup_context.intersected_glyphs_cache); + if (!context_intersects (c->glyphs, + inputCount, input, + closure_lookup_context)) + return; + + /* Build context information - for ContextSubst, input positions are lookahead */ + typename hb_depend_context_t::context_info_t ctx_info; + + for (unsigned i = 0; i < inputCount - 1; i++) + { + hb_set_t pos_glyphs; + /* Extract glyphs at this position based on context format */ + switch (lookup_context.context_format) + { + case ContextFormat::SimpleContext: + /* For SimpleContext, input[i] is the glyph ID directly */ + pos_glyphs.add (input[i]); + break; + case ContextFormat::ClassBasedContext: + /* For class-based context, input[i] is a class index */ + collect_class (&pos_glyphs, input[i], lookup_context.intersects_data); + break; + case ContextFormat::CoverageBasedContext: + /* For coverage-based context, input[i] is a coverage offset */ + collect_coverage (&pos_glyphs, input[i], lookup_context.intersects_data); + break; + } + if (!pos_glyphs.is_empty ()) + ctx_info.lookahead_sets.push (pos_glyphs); + } + + /* Build context set once for this rule */ + hb_codepoint_t context_set_idx = c->depend_data->build_context_set ( + &ctx_info.lookahead_sets, nullptr); + c->depend_data->current_context_set_index = context_set_idx; + + /* Push context before recursing (use move to avoid copy issues) */ + c->push_context (std::move (ctx_info)); + + context_depend_recurse_lookups (c, + inputCount, input, + lookupCount, lookupRecord, + value, + lookup_context.context_format, + lookup_context.intersects_data, + lookup_context.funcs.intersected_glyphs, + lookup_context.intersected_glyphs_cache); + + /* Pop context and clear context set index */ + c->pop_context (); + c->depend_data->current_context_set_index = HB_CODEPOINT_INVALID; } #endif @@ -2621,7 +2717,9 @@ struct ContextFormat1_4 struct ContextDependLookupContext lookup_context = { {intersects_glyph, intersected_glyph}, ContextFormat::SimpleContext, - nullptr + nullptr, + nullptr, /* intersects_cache */ + nullptr /* intersected_glyphs_cache */ }; + hb_zip (this+coverage, hb_range ((unsigned) ruleSet.len)) @@ -3099,7 +3197,9 @@ struct ContextFormat3 struct ContextDependLookupContext lookup_context = { {intersects_coverage, intersected_coverage_glyphs}, ContextFormat::CoverageBasedContext, - this + this, + nullptr, /* intersects_cache */ + nullptr /* intersected_glyphs_cache */ }; context_depend_lookup (c, glyphCount, (const HBUINT16 *) (coverageZ.arrayZ + 1), @@ -3385,19 +3485,84 @@ static inline void chain_context_depend_lookup (hb_depend_context_t *c, { // ChainContextDependLookupContext has the same layout as ChainContextClosureLookupContext auto &closure_lookup_context = reinterpret_cast(lookup_context); - if (chain_context_intersects (c->glyphs, - backtrackCount, backtrack, - inputCount, input, - lookaheadCount, lookahead, - closure_lookup_context)) - context_depend_recurse_lookups (c, - inputCount, input, - lookupCount, lookupRecord, - value, - lookup_context.context_format, - lookup_context.intersects_data[1], - lookup_context.funcs.intersected_glyphs, - lookup_context.intersected_glyphs_cache); + if (!chain_context_intersects (c->glyphs, + backtrackCount, backtrack, + inputCount, input, + lookaheadCount, lookahead, + closure_lookup_context)) + return; + + /* Build context information - extract backtrack and lookahead glyphs */ + typename hb_depend_context_t::context_info_t ctx_info; + + /* Extract backtrack glyphs */ + for (unsigned i = 0; i < backtrackCount; i++) + { + hb_set_t pos_glyphs; + /* Extract glyphs based on context format */ + switch (lookup_context.context_format) + { + case ContextFormat::SimpleContext: + /* For SimpleContext, backtrack[i] is the glyph ID directly */ + pos_glyphs.add (backtrack[i]); + break; + case ContextFormat::ClassBasedContext: + /* For class-based context, backtrack[i] is a class index */ + collect_class (&pos_glyphs, backtrack[i], lookup_context.intersects_data[0]); + break; + case ContextFormat::CoverageBasedContext: + /* For coverage-based context, backtrack[i] is a coverage offset */ + collect_coverage (&pos_glyphs, backtrack[i], lookup_context.intersects_data[0]); + break; + } + if (!pos_glyphs.is_empty ()) + ctx_info.backtrack_sets.push (pos_glyphs); + } + + /* Extract lookahead glyphs */ + for (unsigned i = 0; i < lookaheadCount; i++) + { + hb_set_t pos_glyphs; + /* Extract glyphs based on context format */ + switch (lookup_context.context_format) + { + case ContextFormat::SimpleContext: + /* For SimpleContext, lookahead[i] is the glyph ID directly */ + pos_glyphs.add (lookahead[i]); + break; + case ContextFormat::ClassBasedContext: + /* For class-based context, lookahead[i] is a class index */ + collect_class (&pos_glyphs, lookahead[i], lookup_context.intersects_data[2]); + break; + case ContextFormat::CoverageBasedContext: + /* For coverage-based context, lookahead[i] is a coverage offset */ + collect_coverage (&pos_glyphs, lookahead[i], lookup_context.intersects_data[2]); + break; + } + if (!pos_glyphs.is_empty ()) + ctx_info.lookahead_sets.push (pos_glyphs); + } + + /* Build context set once for this rule */ + hb_codepoint_t context_set_idx = c->depend_data->build_context_set ( + &ctx_info.backtrack_sets, &ctx_info.lookahead_sets); + c->depend_data->current_context_set_index = context_set_idx; + + /* Push context before recursing (use move to avoid copy issues) */ + c->push_context (std::move (ctx_info)); + + context_depend_recurse_lookups (c, + inputCount, input, + lookupCount, lookupRecord, + value, + lookup_context.context_format, + lookup_context.intersects_data[1], + lookup_context.funcs.intersected_glyphs, + lookup_context.intersected_glyphs_cache); + + /* Pop context and clear context set index */ + c->pop_context (); + c->depend_data->current_context_set_index = HB_CODEPOINT_INVALID; } #endif @@ -4001,7 +4166,9 @@ struct ChainContextFormat1_4 struct ChainContextDependLookupContext lookup_context = { {intersects_glyph, intersected_glyph}, ContextFormat::SimpleContext, - {nullptr, nullptr, nullptr} + {nullptr, nullptr, nullptr}, + {nullptr, nullptr, nullptr}, /* intersects_cache */ + nullptr /* intersected_glyphs_cache */ }; + hb_zip (this+coverage, hb_range ((unsigned) ruleSet.len)) @@ -4546,7 +4713,9 @@ struct ChainContextFormat3 struct ChainContextDependLookupContext lookup_context = { {intersects_coverage, intersected_coverage_glyphs}, ContextFormat::CoverageBasedContext, - {this, this, this} + {this, this, this}, + {nullptr, nullptr, nullptr}, /* intersects_cache */ + nullptr /* intersected_glyphs_cache */ }; chain_context_depend_lookup (c, backtrack.len, (const HBUINT16 *) backtrack.arrayZ, diff --git a/test/api/test-ot-depend.c b/test/api/test-ot-depend.c index 606b0aede..c00db2103 100644 --- a/test/api/test-ot-depend.c +++ b/test/api/test-ot-depend.c @@ -245,6 +245,7 @@ find_dependency (hb_depend_t *depend, hb_codepoint_t dependent; hb_tag_t layout_tag; hb_codepoint_t ligature_set; + hb_codepoint_t context_set; hb_tag_t filter_layout_tag = (out_layout_tag && *out_layout_tag != HB_TAG_NONE) ? *out_layout_tag : HB_TAG_NONE; @@ -252,7 +253,7 @@ find_dependency (hb_depend_t *depend, { hb_bool_t found = hb_depend_get_glyph_entry (depend, source_glyph, index, &table_tag, &dependent, - &layout_tag, &ligature_set); + &layout_tag, &ligature_set, &context_set); if (!found) break; @@ -549,7 +550,13 @@ test_depend_gsub_formats (void) g_assert_true (find_dependency (depend_notosans, 3948, 4017, HB_OT_TAG_GSUB, &layout_tag, &ligature_set)); g_test_message (" ligature_set = %u", ligature_set); - g_assert_cmpuint (684, ==, ligature_set); + + /* Verify ligature set contains the other component glyph (3969) */ + g_assert_cmpuint (HB_CODEPOINT_INVALID, !=, ligature_set); + hb_set_t *lig_set = hb_set_create (); + g_assert_true (hb_depend_get_set_from_index (depend_notosans, ligature_set, lig_set)); + g_assert_true (hb_set_has (lig_set, 3969)); /* uni0944 is the other ligature component */ + hb_set_destroy (lig_set); hb_depend_destroy (depend_notosans); hb_face_destroy (face_notosans); diff --git a/test/fuzzing/hb-depend-closure-parity-main.cc b/test/fuzzing/hb-depend-closure-parity-main.cc index 9435fa860..c48503e14 100644 --- a/test/fuzzing/hb-depend-closure-parity-main.cc +++ b/test/fuzzing/hb-depend-closure-parity-main.cc @@ -13,6 +13,16 @@ extern uint64_t _hb_depend_fuzzer_single_test; /* Specific test to run, or 0 fo extern unsigned _hb_depend_fuzzer_num_tests; /* Number of tests to run (default 1024) */ extern bool _hb_depend_fuzzer_verbose; extern bool _hb_depend_fuzzer_report_over_approximation; +extern bool _hb_depend_fuzzer_report_under_approximation; +extern float _hb_depend_fuzzer_inject_over_approx; +extern float _hb_depend_fuzzer_inject_under_approx; + +/* Explicit test mode */ +extern bool _hb_depend_fuzzer_explicit_test; +extern const char *_hb_depend_fuzzer_explicit_unicodes; +extern const char *_hb_depend_fuzzer_explicit_gids; +extern bool _hb_depend_fuzzer_explicit_no_layout_closure; +extern const char *_hb_depend_fuzzer_explicit_layout_features; static void print_usage (const char *prog) @@ -21,13 +31,30 @@ print_usage (const char *prog) fprintf (stderr, "\n"); fprintf (stderr, "Validate depend API by comparing closures against subset.\n"); fprintf (stderr, "\n"); - fprintf (stderr, "Options:\n"); + fprintf (stderr, "Test Modes:\n"); + fprintf (stderr, " (default) Run random tests using seed\n"); + fprintf (stderr, " -t, --test HEXSEED Run only specific test (64-bit hex, with or without 0x)\n"); + fprintf (stderr, " --unicodes LIST Run explicit test with given unicodes (U+XXXX or decimal)\n"); + fprintf (stderr, " --gids LIST Run explicit test with given glyph IDs (decimal)\n"); + fprintf (stderr, "\n"); + fprintf (stderr, "Explicit Test Options (used with --unicodes or --gids):\n"); + fprintf (stderr, " --no-layout-closure Skip GSUB closure (use with --gids)\n"); + fprintf (stderr, " --layout-features=LIST\n"); + fprintf (stderr, " Specify GSUB features (comma-separated tags or * for all)\n"); + fprintf (stderr, " Default: subset's default features\n"); + fprintf (stderr, "\n"); + fprintf (stderr, "General Options:\n"); fprintf (stderr, " -s, --seed SEED Use specific random seed (default: random)\n"); fprintf (stderr, " -n, --num-tests N Run N tests per font (default: 1024)\n"); - fprintf (stderr, " -t, --test HEXSEED Run only specific test (64-bit hex, with or without 0x)\n"); fprintf (stderr, " -v, --verbose Enable verbose output (all test details)\n"); fprintf (stderr, " -r, --report-over-approximation\n"); fprintf (stderr, " Report when depend finds more glyphs than subset\n"); + fprintf (stderr, " --report-under-approximation\n"); + fprintf (stderr, " Report under-approximation instead of aborting\n"); + fprintf (stderr, " -o, --inject-over-approx[=RATE]\n"); + fprintf (stderr, " Inject over-approximation errors (default rate: 0.02 = 2%%)\n"); + fprintf (stderr, " -u, --inject-under-approx[=RATE]\n"); + fprintf (stderr, " Inject under-approximation errors (default rate: 0.02 = 2%%)\n"); fprintf (stderr, " -h, --help Show this help message\n"); fprintf (stderr, "\n"); fprintf (stderr, "Examples:\n"); @@ -35,6 +62,9 @@ print_usage (const char *prog) fprintf (stderr, " %s -n 100 fonts/*.ttf # Run 100 tests per font\n", prog); fprintf (stderr, " %s -v fonts/MyFont.ttf # Run with verbose output\n", prog); fprintf (stderr, " %s -t 0x0000303900000015 fonts/MyFont.ttf # Run specific test\n", prog); + fprintf (stderr, " %s --unicodes=U+0041,U+0042 fonts/MyFont.ttf # Test with 'A' and 'B'\n", prog); + fprintf (stderr, " %s --gids=10,20,30 --no-layout-closure fonts/MyFont.ttf # Non-GSUB test\n", prog); + fprintf (stderr, " %s --unicodes=65-90 --layout-features=liga,calt fonts/MyFont.ttf # Custom features\n", prog); } int main (int argc, char **argv) @@ -44,7 +74,17 @@ int main (int argc, char **argv) unsigned num_tests = 1024; bool verbose = false; bool report_over_approximation = false; + bool report_under_approximation = false; bool seed_specified = false; + float inject_over_approx = 0.0f; + float inject_under_approx = 0.0f; + + /* Explicit test mode */ + bool explicit_test = false; + const char *explicit_unicodes = NULL; + const char *explicit_gids = NULL; + bool explicit_no_layout_closure = false; + const char *explicit_layout_features = NULL; /* Parse options */ int arg_idx = 1; @@ -65,6 +105,11 @@ int main (int argc, char **argv) report_over_approximation = true; arg_idx++; } + else if (strcmp (argv[arg_idx], "--report-under-approximation") == 0) + { + report_under_approximation = true; + arg_idx++; + } else if (strcmp (argv[arg_idx], "-s") == 0 || strcmp (argv[arg_idx], "--seed") == 0) { if (arg_idx + 1 >= argc) @@ -100,6 +145,93 @@ int main (int argc, char **argv) single_test = strtoull (argv[arg_idx + 1], NULL, 16); arg_idx += 2; } + else if (strcmp (argv[arg_idx], "-o") == 0 || strcmp (argv[arg_idx], "--inject-over-approx") == 0) + { + /* Check if next arg exists and is a number (not another flag) */ + if (arg_idx + 1 < argc && argv[arg_idx + 1][0] != '-') + { + inject_over_approx = strtof (argv[arg_idx + 1], NULL); + arg_idx += 2; + } + else + { + /* Use default target rate (2%) */ + inject_over_approx = 0.02f; + arg_idx++; + } + } + else if (strcmp (argv[arg_idx], "-u") == 0 || strcmp (argv[arg_idx], "--inject-under-approx") == 0) + { + /* Check if next arg exists and is a number (not another flag) */ + if (arg_idx + 1 < argc && argv[arg_idx + 1][0] != '-') + { + inject_under_approx = strtof (argv[arg_idx + 1], NULL); + arg_idx += 2; + } + else + { + /* Use default target rate (2%) */ + inject_under_approx = 0.02f; + arg_idx++; + } + } + else if (strncmp (argv[arg_idx], "--unicodes=", 11) == 0) + { + explicit_test = true; + explicit_unicodes = argv[arg_idx] + 11; + arg_idx++; + } + else if (strcmp (argv[arg_idx], "--unicodes") == 0) + { + if (arg_idx + 1 >= argc) + { + fprintf (stderr, "Error: --unicodes requires an argument\n"); + print_usage (argv[0]); + return 1; + } + explicit_test = true; + explicit_unicodes = argv[arg_idx + 1]; + arg_idx += 2; + } + else if (strncmp (argv[arg_idx], "--gids=", 7) == 0) + { + explicit_test = true; + explicit_gids = argv[arg_idx] + 7; + arg_idx++; + } + else if (strcmp (argv[arg_idx], "--gids") == 0) + { + if (arg_idx + 1 >= argc) + { + fprintf (stderr, "Error: --gids requires an argument\n"); + print_usage (argv[0]); + return 1; + } + explicit_test = true; + explicit_gids = argv[arg_idx + 1]; + arg_idx += 2; + } + else if (strcmp (argv[arg_idx], "--no-layout-closure") == 0) + { + explicit_no_layout_closure = true; + arg_idx++; + } + else if (strncmp (argv[arg_idx], "--layout-features=", 18) == 0) + { + explicit_layout_features = argv[arg_idx] + 18; + arg_idx++; + } + else if (strcmp (argv[arg_idx], "--layout-features") == 0) + { + if (arg_idx + 1 >= argc) + { + fprintf (stderr, "Error: --layout-features requires an argument\n"); + print_usage (argv[0]); + return 1; + } + explicit_layout_features = argv[arg_idx + 1]; + arg_idx += 2; + } else { fprintf (stderr, "Error: Unknown option %s\n", argv[arg_idx]); @@ -116,6 +248,26 @@ int main (int argc, char **argv) return 1; } + /* Validate test mode combinations */ + if (explicit_test && single_test != 0) + { + fprintf (stderr, "Error: Cannot use both explicit test mode (--unicodes/--gids) and -t/--test\n"); + print_usage (argv[0]); + return 1; + } + + if (explicit_unicodes && explicit_gids) + { + fprintf (stderr, "Error: Cannot specify both --unicodes and --gids\n"); + print_usage (argv[0]); + return 1; + } + + if (!explicit_test && (explicit_no_layout_closure || explicit_layout_features)) + { + fprintf (stderr, "Warning: --no-layout-closure and --layout-features are ignored without --unicodes or --gids\n"); + } + /* Use random seed if not specified */ if (!seed_specified) seed = (unsigned) time (NULL); @@ -126,12 +278,34 @@ int main (int argc, char **argv) _hb_depend_fuzzer_num_tests = num_tests; _hb_depend_fuzzer_verbose = verbose; _hb_depend_fuzzer_report_over_approximation = report_over_approximation; + _hb_depend_fuzzer_report_under_approximation = report_under_approximation; + _hb_depend_fuzzer_inject_over_approx = inject_over_approx; + _hb_depend_fuzzer_inject_under_approx = inject_under_approx; + + /* Set explicit test mode configuration */ + _hb_depend_fuzzer_explicit_test = explicit_test; + _hb_depend_fuzzer_explicit_unicodes = explicit_unicodes; + _hb_depend_fuzzer_explicit_gids = explicit_gids; + _hb_depend_fuzzer_explicit_no_layout_closure = explicit_no_layout_closure; + _hb_depend_fuzzer_explicit_layout_features = explicit_layout_features; /* TAP version must be the first line */ printf ("TAP version 14\n"); - /* Print seed for reproducibility */ - if (single_test == 0) + /* Print test mode info */ + if (explicit_test) + { + if (explicit_unicodes) + printf ("# Explicit test mode: --unicodes=%s\n", explicit_unicodes); + else if (explicit_gids) + printf ("# Explicit test mode: --gids=%s\n", explicit_gids); + + if (explicit_no_layout_closure) + printf ("# Using --no-layout-closure\n"); + if (explicit_layout_features) + printf ("# Using --layout-features=%s\n", explicit_layout_features); + } + else if (single_test == 0) printf ("# Using seed: %u, running %u tests per font\n", seed, num_tests); else printf ("# Running single test: 0x%016llx\n", (unsigned long long) single_test); diff --git a/test/fuzzing/hb-depend-closure-parity.cc b/test/fuzzing/hb-depend-closure-parity.cc index e7650155b..acd08d6a8 100644 --- a/test/fuzzing/hb-depend-closure-parity.cc +++ b/test/fuzzing/hb-depend-closure-parity.cc @@ -11,6 +11,17 @@ #ifdef HB_DEPEND_API #include +/* Debug messaging - follows HarfBuzz DEBUG_MSG pattern but simplified for fuzzer */ +#ifndef HB_DEBUG_DEPEND +#define HB_DEBUG_DEPEND 1 +#endif + +#if HB_DEBUG_DEPEND +#define DEBUG_MSG_DEPEND(msg, ...) fprintf (stderr, "%-10s" msg "\n", "DEPEND", ##__VA_ARGS__) +#else +#define DEBUG_MSG_DEPEND(msg, ...) do {} while (0) +#endif + /* Globals for configuration (set by main() in standalone mode) */ const char *_hb_depend_fuzzer_current_font_path = NULL; unsigned _hb_depend_fuzzer_seed = 0; @@ -18,7 +29,20 @@ uint64_t _hb_depend_fuzzer_single_test = 0; /* Specific test to run, or 0 for a unsigned _hb_depend_fuzzer_num_tests = 1024; /* Number of tests to run */ bool _hb_depend_fuzzer_verbose = false; bool _hb_depend_fuzzer_report_over_approximation = false; -hb_codepoint_t _hb_depend_fuzzer_debug_gid = HB_CODEPOINT_INVALID; /* Specific glyph to debug */ +bool _hb_depend_fuzzer_report_under_approximation = false; +float _hb_depend_fuzzer_inject_over_approx = 0.0f; +float _hb_depend_fuzzer_inject_under_approx = 0.0f; + +/* Explicit test mode */ +bool _hb_depend_fuzzer_explicit_test = false; +const char *_hb_depend_fuzzer_explicit_unicodes = NULL; +const char *_hb_depend_fuzzer_explicit_gids = NULL; +bool _hb_depend_fuzzer_explicit_no_layout_closure = false; +const char *_hb_depend_fuzzer_explicit_layout_features = NULL; + +/* Debug control (only used when HB_DEBUG_DEPEND > 0) */ +static hb_codepoint_t debug_gid = HB_CODEPOINT_INVALID; /* Specific glyph to debug */ +static bool trace_closure = false; /* Trace closure computation in detail */ static bool config_initialized = false; @@ -34,10 +58,15 @@ init_fuzzer_config () /* Initialize RNG with seed (already set by main or defaulted to random) */ srand (_hb_depend_fuzzer_seed); - /* Check for debug glyph ID */ + /* Check for debug glyph ID (only used when HB_DEBUG_DEPEND > 0) */ const char *debug_gid_str = getenv ("HB_DEPEND_FUZZER_DEBUG_GID"); if (debug_gid_str) - _hb_depend_fuzzer_debug_gid = (hb_codepoint_t) strtoul (debug_gid_str, NULL, 0); + debug_gid = (hb_codepoint_t) strtoul (debug_gid_str, NULL, 0); + + /* Check for closure tracing (only used when HB_DEBUG_DEPEND > 0) */ + const char *trace_closure_str = getenv ("HB_DEPEND_FUZZER_TRACE_CLOSURE"); + if (trace_closure_str && strcmp (trace_closure_str, "1") == 0) + trace_closure = true; } /* Simple RNG helpers */ @@ -101,6 +130,175 @@ rng_select_from_set (hb_set_t *out, unsigned n, const hb_set_t *source) free (array); } +/* Forward declaration */ +static hb_set_t * get_gsub_features (hb_face_t *face); + +/* Parse comma-separated list of unicodes (U+XXXX, decimal, or ranges) into a set */ +static bool +parse_unicodes (const char *str, hb_set_t *unicodes) +{ + if (!str || !*str) + return false; + + const char *p = str; + while (*p) + { + /* Skip whitespace and commas */ + while (*p == ' ' || *p == ',') + p++; + if (!*p) + break; + + /* Parse unicode value (U+XXXX or decimal) */ + const char *start = p; + hb_codepoint_t start_cp; + if (*p == 'U' && *(p + 1) == '+') + { + /* U+XXXX format */ + start_cp = (hb_codepoint_t) strtoul (p + 2, (char **) &p, 16); + } + else + { + /* Decimal format */ + start_cp = (hb_codepoint_t) strtoul (p, (char **) &p, 10); + } + + /* Check if parsing advanced (avoid infinite loop on invalid input) */ + if (p == start || (p == start + 2 && start[0] == 'U' && start[1] == '+')) + { + fprintf (stderr, "Error: Invalid unicode value at: %.20s\n", start); + return false; + } + + /* Check for range */ + if (*p == '-') + { + p++; + const char *end_start = p; + hb_codepoint_t end_cp; + if (*p == 'U' && *(p + 1) == '+') + end_cp = (hb_codepoint_t) strtoul (p + 2, (char **) &p, 16); + else + end_cp = (hb_codepoint_t) strtoul (p, (char **) &p, 10); + + /* Check if parsing advanced */ + if (p == end_start || (p == end_start + 2 && end_start[0] == 'U' && end_start[1] == '+')) + { + fprintf (stderr, "Error: Invalid unicode value in range at: %.20s\n", end_start); + return false; + } + + hb_set_add_range (unicodes, start_cp, end_cp); + } + else + { + hb_set_add (unicodes, start_cp); + } + } + + return !hb_set_is_empty (unicodes); +} + +/* Parse comma-separated list of glyph IDs (decimal or ranges) into a set */ +static bool +parse_gids (const char *str, hb_set_t *gids) +{ + if (!str || !*str) + return false; + + const char *p = str; + while (*p) + { + /* Skip whitespace and commas */ + while (*p == ' ' || *p == ',') + p++; + if (!*p) + break; + + /* Parse glyph ID */ + const char *start = p; + hb_codepoint_t start_gid = (hb_codepoint_t) strtoul (p, (char **) &p, 10); + + /* Check if parsing advanced (avoid infinite loop on invalid input) */ + if (p == start) + { + fprintf (stderr, "Error: Invalid glyph ID at: %.20s\n", start); + return false; + } + + /* Check for range */ + if (*p == '-') + { + p++; + const char *end_start = p; + hb_codepoint_t end_gid = (hb_codepoint_t) strtoul (p, (char **) &p, 10); + + /* Check if parsing advanced */ + if (p == end_start) + { + fprintf (stderr, "Error: Invalid glyph ID in range at: %.20s\n", end_start); + return false; + } + + hb_set_add_range (gids, start_gid, end_gid); + } + else + { + hb_set_add (gids, start_gid); + } + } + + return !hb_set_is_empty (gids); +} + +/* Parse comma-separated list of feature tags into a set + * Special value "*" means all features */ +static bool +parse_features (const char *str, hb_set_t *features, hb_face_t *face) +{ + if (!str || !*str) + return false; + + /* Check for special "*" value (all features) */ + if (strcmp (str, "*") == 0) + { + /* Get all GSUB features */ + hb_set_t *all_features = get_gsub_features (face); + hb_set_union (features, all_features); + hb_set_destroy (all_features); + return true; + } + + const char *p = str; + while (*p) + { + /* Skip whitespace and commas */ + while (*p == ' ' || *p == ',') + p++; + if (!*p) + break; + + /* Parse 4-character tag */ + char tag_str[5] = {0}; + int i = 0; + while (*p && *p != ',' && *p != ' ' && i < 4) + tag_str[i++] = *p++; + + if (i > 0) + { + /* Pad with spaces if needed */ + while (i < 4) + tag_str[i++] = ' '; + tag_str[4] = '\0'; + + hb_tag_t tag = HB_TAG (tag_str[0], tag_str[1], tag_str[2], tag_str[3]); + hb_set_add (features, tag); + } + } + + return !hb_set_is_empty (features); +} + /* Test parameters generated from a 64-bit seed */ struct test_params { @@ -139,6 +337,82 @@ get_gsub_features (hb_face_t *face) return features; } +/* Generate test parameters from explicit command-line arguments */ +static test_params +generate_test_from_explicit (hb_face_t *face, hb_font_t *font) +{ + test_params params; + params.glyphs = hb_set_create (); + params.codepoints = NULL; + params.features = NULL; + params.target_size = 0; + + /* Parse unicodes or gids */ + if (_hb_depend_fuzzer_explicit_unicodes) + { + /* GSUB test mode with unicodes */ + params.is_gsub = true; + params.codepoints = hb_set_create (); + + if (!parse_unicodes (_hb_depend_fuzzer_explicit_unicodes, params.codepoints)) + { + fprintf (stderr, "Error: Failed to parse --unicodes argument\n"); + hb_set_destroy (params.glyphs); + hb_set_destroy (params.codepoints); + params.glyphs = NULL; + params.codepoints = NULL; + return params; + } + + /* Map codepoints to glyphs */ + hb_codepoint_t cp = HB_SET_VALUE_INVALID; + while (hb_set_next (params.codepoints, &cp)) + { + hb_codepoint_t gid; + if (hb_font_get_nominal_glyph (font, cp, &gid)) + hb_set_add (params.glyphs, gid); + } + + params.target_size = hb_set_get_population (params.codepoints); + } + else if (_hb_depend_fuzzer_explicit_gids) + { + /* Non-GSUB test mode with glyph IDs */ + params.is_gsub = !_hb_depend_fuzzer_explicit_no_layout_closure; + + if (!parse_gids (_hb_depend_fuzzer_explicit_gids, params.glyphs)) + { + fprintf (stderr, "Error: Failed to parse --gids argument\n"); + hb_set_destroy (params.glyphs); + params.glyphs = NULL; + return params; + } + + params.target_size = hb_set_get_population (params.glyphs); + } + else + { + fprintf (stderr, "Error: Explicit mode requires --unicodes or --gids\n"); + hb_set_destroy (params.glyphs); + params.glyphs = NULL; + return params; + } + + /* Parse layout features if specified */ + if (_hb_depend_fuzzer_explicit_layout_features) + { + params.features = hb_set_create (); + if (!parse_features (_hb_depend_fuzzer_explicit_layout_features, params.features, face)) + { + fprintf (stderr, "Error: Failed to parse --layout-features argument\n"); + hb_set_destroy (params.features); + params.features = NULL; + } + } + + return params; +} + /* Generate a test from a 64-bit seed * * The seed determines: @@ -344,49 +618,128 @@ generate_test_from_seed (uint64_t seed, hb_face_t *face, hb_font_t *font) * battle-tested subset closure implementation. */ -/* Compute closure using depend API by following the dependency graph - * skip_gsub: if true, skip GSUB dependencies to match subset behavior - * (subset doesn't follow GSUB when starting with glyph IDs) - * active_features: if non-NULL, only follow GSUB dependencies from these features */ -static void -compute_depend_closure (hb_depend_t *depend, hb_set_t *glyphs, bool skip_gsub, - hb_set_t *active_features) +/* Check if context requirements are satisfied + * + * Context sets encode which backtrack and lookahead glyphs must be present + * for a dependency edge to apply. Elements in the context set can be: + * - Direct GID reference (value < 0x80000000): that specific glyph must be in closure + * - Indirect set reference (value >= 0x80000000): at least one glyph from that set must be in closure + * + * Returns true if ALL context requirements are met, false otherwise. + */ +static bool +check_context_satisfied (hb_depend_t *depend, + hb_codepoint_t context_set_idx, + hb_set_t *current_closure) { - if (_hb_depend_fuzzer_debug_gid != HB_CODEPOINT_INVALID) + if (context_set_idx == HB_CODEPOINT_INVALID) + return true; /* No context requirements */ + + hb_set_t *context_elements = hb_set_create (); + if (!hb_depend_get_set_from_index (depend, context_set_idx, context_elements)) { - fprintf (stderr, "[DEBUG] Starting closure, glyph %u %s in initial set\n", - _hb_depend_fuzzer_debug_gid, - hb_set_has (glyphs, _hb_depend_fuzzer_debug_gid) ? "IS" : "is NOT"); + hb_set_destroy (context_elements); + return true; /* Invalid context set, don't filter */ } - hb_set_t *to_process = hb_set_create (); - hb_set_union (to_process, glyphs); + bool satisfied = true; + hb_codepoint_t elem = HB_SET_VALUE_INVALID; + while (hb_set_next (context_elements, &elem)) + { + if (elem < 0x80000000) + { + /* Direct GID reference - this specific glyph must be present */ + if (!hb_set_has (current_closure, elem)) + { + satisfied = false; + break; + } + } + else + { + /* Indirect set reference - at least one glyph from this set must be present */ + hb_codepoint_t set_idx = elem & 0x7FFFFFFF; + hb_set_t *required_glyphs = hb_set_create (); - /* Track ligature glyph IDs separately for deferred processing */ - hb_set_t *deferred_ligatures = hb_set_create (); + if (!hb_depend_get_set_from_index (depend, set_idx, required_glyphs)) + { + hb_set_destroy (required_glyphs); + continue; /* Invalid set reference, skip this requirement */ + } + + /* Check if ANY glyph from required_glyphs is in current_closure */ + bool any_present = false; + hb_codepoint_t gid = HB_SET_VALUE_INVALID; + while (hb_set_next (required_glyphs, &gid)) + { + if (hb_set_has (current_closure, gid)) + { + any_present = true; + break; + } + } + + hb_set_destroy (required_glyphs); + + if (!any_present) + { + satisfied = false; + break; + } + } + } + + hb_set_destroy (context_elements); + return satisfied; +} + +/* Helper: Process edges for specific table(s) + * table_filter: if non-NULL, only process edges from these tables + * Returns: true if any new glyphs were added */ +static bool +process_edges_for_tables (hb_depend_t *depend, + hb_set_t *glyphs, + hb_set_t *to_process, + hb_set_t *deferred_ligatures, + hb_set_t *table_filter, + bool skip_gsub, + hb_set_t *active_features) +{ + bool added_any = false; - /* Phase 1: Process non-ligature dependencies normally */ while (!hb_set_is_empty (to_process)) { hb_codepoint_t gid = hb_set_get_min (to_process); hb_set_del (to_process, gid); - if (gid == _hb_depend_fuzzer_debug_gid) - fprintf (stderr, "[DEBUG] Processing glyph %u\n", gid); + if (gid == debug_gid) + DEBUG_MSG_DEPEND("Processing glyph %u", gid); hb_codepoint_t index = 0; hb_tag_t table_tag; hb_codepoint_t dependent; hb_tag_t layout_tag; hb_codepoint_t ligature_set; + hb_codepoint_t context_set; while (hb_depend_get_glyph_entry (depend, gid, index++, &table_tag, &dependent, - &layout_tag, &ligature_set)) + &layout_tag, &ligature_set, &context_set)) { - if (dependent == _hb_depend_fuzzer_debug_gid || gid == _hb_depend_fuzzer_debug_gid) - fprintf (stderr, "[DEBUG] Edge: %u -> %u (table=%c%c%c%c, ligset=%u)\n", - gid, dependent, HB_UNTAG(table_tag), ligature_set); + /* Debug: trace all edges involving our debug glyph */ + if (debug_gid != HB_CODEPOINT_INVALID && + (gid == debug_gid || dependent == debug_gid)) + DEBUG_MSG_DEPEND("Considering edge: %u -> %u (feature=%c%c%c%c, ctx=%u)", + gid, dependent, HB_UNTAG(layout_tag), context_set); + + /* Filter by table if requested */ + if (table_filter && !hb_set_has (table_filter, table_tag)) + continue; + + if (dependent == debug_gid || gid == debug_gid) + DEBUG_MSG_DEPEND("Edge: %u -> %u (table=%c%c%c%c, ligset=%u)", + gid, dependent, HB_UNTAG(table_tag), ligature_set); + /* Skip GSUB dependencies if requested (to match subset behavior) */ if (skip_gsub && table_tag == HB_OT_TAG_GSUB) continue; @@ -395,7 +748,48 @@ compute_depend_closure (hb_depend_t *depend, hb_set_t *glyphs, bool skip_gsub, if (active_features && table_tag == HB_OT_TAG_GSUB) { if (!hb_set_has (active_features, layout_tag)) - continue; /* Skip dependencies from inactive features */ + { + /* Over-approximation injection: sometimes follow inactive feature edges */ + bool skip_feature_check = false; + if (_hb_depend_fuzzer_inject_over_approx > 0.0f) { + float r = (float)rand() / (float)RAND_MAX; + if (r < _hb_depend_fuzzer_inject_over_approx) { + skip_feature_check = true; + } + } + + if (!skip_feature_check) + { + if (dependent == debug_gid) + DEBUG_MSG_DEPEND("Skipping edge %u -> %u: feature %c%c%c%c not active", + gid, dependent, HB_UNTAG(layout_tag)); + continue; /* Skip dependencies from inactive features */ + } + } + } + + /* Check context requirements - skip edge if context not satisfied */ + if (context_set != HB_CODEPOINT_INVALID) + { + if (!check_context_satisfied (depend, context_set, glyphs)) + { + /* Over-approximation injection: sometimes follow edges with unsatisfied context */ + bool skip_context_check = false; + if (_hb_depend_fuzzer_inject_over_approx > 0.0f) { + float r = (float)rand() / (float)RAND_MAX; + if (r < _hb_depend_fuzzer_inject_over_approx) { + skip_context_check = true; + } + } + + if (!skip_context_check) + { + if (dependent == debug_gid) + DEBUG_MSG_DEPEND("Skipping edge %u -> %u: context not satisfied (ctx_set=%u)", + gid, dependent, context_set); + continue; /* Context requirements not met, skip this edge */ + } + } } /* Check if this is a ligature dependency (has a ligature set) */ @@ -408,158 +802,230 @@ compute_depend_closure (hb_depend_t *depend, hb_set_t *glyphs, bool skip_gsub, } else { + /* Under-approximation injection: randomly skip following this edge */ + if (_hb_depend_fuzzer_inject_under_approx > 0.0f) { + float r = (float)rand() / (float)RAND_MAX; + if (r < _hb_depend_fuzzer_inject_under_approx) { + /* Skip this edge to inject under-approximation */ + continue; + } + } + /* Non-ligature dependency: add immediately */ if (!hb_set_has (glyphs, dependent)) { - if (dependent == _hb_depend_fuzzer_debug_gid) - fprintf (stderr, "[DEBUG] Adding glyph %u to closure (from %u)\n", dependent, gid); + if (dependent == debug_gid) + DEBUG_MSG_DEPEND("Adding glyph %u to closure (from %u, feature=%c%c%c%c, ctx=%u)", + dependent, gid, HB_UNTAG(layout_tag), context_set); + if (trace_closure) + DEBUG_MSG_DEPEND("Depend: %u -> %u (table=%c%c%c%c, feature=%c%c%c%c)", + gid, dependent, HB_UNTAG(table_tag), HB_UNTAG(layout_tag)); hb_set_add (glyphs, dependent); hb_set_add (to_process, dependent); + added_any = true; } } } } - /* Phase 2: Process deferred ligatures iteratively - * Only add a ligature when ALL glyphs in its ligature set are in closure */ + return added_any; +} + +/* Compute closure using depend API by following the dependency graph + * skip_gsub: if true, skip GSUB dependencies to match subset behavior + * (subset doesn't follow GSUB when starting with glyph IDs) + * active_features: if non-NULL, only follow GSUB dependencies from these features */ +static void +compute_depend_closure (hb_depend_t *depend, hb_set_t *glyphs, bool skip_gsub, + hb_set_t *active_features) +{ + if (debug_gid != HB_CODEPOINT_INVALID) + { + DEBUG_MSG_DEPEND("Starting closure, glyph %u %s in initial set", + debug_gid, + hb_set_has (glyphs, debug_gid) ? "IS" : "is NOT"); + } + + if (trace_closure) + { + hb_codepoint_t gid = HB_SET_VALUE_INVALID; + fprintf (stderr, "DEPEND: Initial set:"); + while (hb_set_next (glyphs, &gid)) + fprintf (stderr, " %u", gid); + fprintf (stderr, "\n"); + } + + hb_set_t *to_process = hb_set_create (); + hb_set_t *deferred_ligatures = hb_set_create (); + + /* Process edges in stages matching subset's table order: + * 1. GSUB (iterate internally until stable - handles context dependencies) + * 2. MATH (on glyphs from GSUB) + * 3. COLR (on glyphs from GSUB+MATH) + * 4. glyf/CFF (on glyphs from GSUB+MATH+COLR) + * + * No loop back to GSUB after glyf/CFF - glyf components are rendering + * implementation details, not shaping inputs. Subset doesn't loop back either. */ + + /* Helper sets for ligature processing (used in GSUB stage) */ hb_set_t *ligature_set_glyphs = hb_set_create (); hb_set_t *ligatures_to_remove = hb_set_create (); - bool made_progress = true; - while (made_progress && !hb_set_is_empty (deferred_ligatures)) { - made_progress = false; - hb_set_clear (ligatures_to_remove); + /* Stage 1: GSUB closure - iterate until stable (like subset does internally) + * Ligatures are processed HERE as part of GSUB, not after glyf/CFF. + * This matches subset behavior where GSUB closure completes before glyf. */ + if (!skip_gsub) + { + hb_set_t *gsub_filter = hb_set_create (); + hb_set_add (gsub_filter, HB_OT_TAG_GSUB); - hb_codepoint_t lig_glyph = HB_SET_VALUE_INVALID; - while (hb_set_next (deferred_ligatures, &lig_glyph)) - { - /* A ligature may appear in multiple ligature sets (different component combinations). - * We need to check ALL ligature sets and add the ligature if ANY set is satisfied. - * Example: uni1FFC can be formed by Omega+uni1FBE OR Omega+uni0345 */ - bool found_any_ligature_set = false; - bool any_set_satisfied = false; + unsigned prev_gsub_count; + unsigned iteration = 0; + do { + prev_gsub_count = hb_set_get_population (glyphs); - /* Scan through all glyphs to find ALL ligature sets that produce this ligature */ - hb_codepoint_t src_glyph = HB_SET_VALUE_INVALID; - while (hb_set_next (glyphs, &src_glyph)) + /* Process non-ligature GSUB edges */ + hb_set_union (to_process, glyphs); + process_edges_for_tables (depend, glyphs, to_process, deferred_ligatures, + gsub_filter, skip_gsub, active_features); + + /* Process deferred ligatures (GSUB ligatures only) + * Only add a ligature when ALL glyphs in its ligature set are in closure */ + bool lig_made_progress = true; + while (lig_made_progress && !hb_set_is_empty (deferred_ligatures)) { - hb_codepoint_t idx = 0; - hb_tag_t ttag; - hb_codepoint_t dep; - hb_tag_t ltag; - hb_codepoint_t ligset; + lig_made_progress = false; + hb_set_clear (ligatures_to_remove); - while (hb_depend_get_glyph_entry (depend, src_glyph, idx++, - &ttag, &dep, <ag, &ligset)) + hb_codepoint_t lig_glyph = HB_SET_VALUE_INVALID; + while (hb_set_next (deferred_ligatures, &lig_glyph)) { - if (dep == lig_glyph && ligset != HB_CODEPOINT_INVALID) + bool any_set_satisfied = false; + + /* Scan through glyphs in closure to find ligature sets */ + hb_codepoint_t src_glyph = HB_SET_VALUE_INVALID; + while (hb_set_next (glyphs, &src_glyph)) { - found_any_ligature_set = true; - - /* Check if this particular ligature set is satisfied */ - hb_set_clear (ligature_set_glyphs); - hb_depend_get_set_from_index (depend, ligset, ligature_set_glyphs); - - if (hb_set_is_subset (ligature_set_glyphs, glyphs)) - { - /* This ligature set is satisfied - we can add the ligature */ - any_set_satisfied = true; - break; - } - } - } - if (any_set_satisfied) - break; - } - - if (!found_any_ligature_set) - { - /* No ligature set found at all, remove from deferred */ - hb_set_add (ligatures_to_remove, lig_glyph); - continue; - } - - /* If any ligature set was satisfied, add the ligature to closure */ - if (any_set_satisfied) - { - /* All components present - add the ligature to closure */ - if (!hb_set_has (glyphs, lig_glyph)) - { - hb_set_add (glyphs, lig_glyph); - - /* Process the ligature's non-ligature dependencies */ - hb_codepoint_t index = 0; - hb_tag_t table_tag; - hb_codepoint_t dependent; - hb_tag_t layout_tag; - hb_codepoint_t ligature_set_id; - - while (hb_depend_get_glyph_entry (depend, lig_glyph, index++, - &table_tag, &dependent, - &layout_tag, &ligature_set_id)) - { - /* Only process non-ligature dependencies */ - if (ligature_set_id == HB_CODEPOINT_INVALID) - { - if (!hb_set_has (glyphs, dependent)) - { - hb_set_add (glyphs, dependent); - hb_set_add (to_process, dependent); - } - } - } - - /* Process newly added non-ligature dependencies */ - while (!hb_set_is_empty (to_process)) - { - hb_codepoint_t gid = hb_set_get_min (to_process); - hb_set_del (to_process, gid); - hb_codepoint_t idx = 0; hb_tag_t ttag; hb_codepoint_t dep; hb_tag_t ltag; hb_codepoint_t ligset; + hb_codepoint_t ctxset; - while (hb_depend_get_glyph_entry (depend, gid, idx++, - &ttag, &dep, <ag, &ligset)) + while (hb_depend_get_glyph_entry (depend, src_glyph, idx++, + &ttag, &dep, <ag, &ligset, &ctxset)) { - /* Skip GSUB if requested */ - if (skip_gsub && ttag == HB_OT_TAG_GSUB) - continue; - - /* Filter by active features */ - if (active_features && ttag == HB_OT_TAG_GSUB) + if (dep == lig_glyph && ligset != HB_CODEPOINT_INVALID && ttag == HB_OT_TAG_GSUB) { - if (!hb_set_has (active_features, ltag)) - continue; - } + /* Check if this ligature set is satisfied */ + hb_set_clear (ligature_set_glyphs); + hb_depend_get_set_from_index (depend, ligset, ligature_set_glyphs); - /* Only follow non-ligature dependencies */ - if (ligset == HB_CODEPOINT_INVALID) - { - if (!hb_set_has (glyphs, dep)) + if (hb_set_is_subset (ligature_set_glyphs, glyphs)) { - hb_set_add (glyphs, dep); - hb_set_add (to_process, dep); + any_set_satisfied = true; + break; } } } + if (any_set_satisfied) + break; } - made_progress = true; + /* Over-approximation injection: sometimes add ligature even when components NOT satisfied */ + if (!any_set_satisfied && _hb_depend_fuzzer_inject_over_approx > 0.0f) { + float r = (float)rand() / (float)RAND_MAX; + if (r < _hb_depend_fuzzer_inject_over_approx) { + any_set_satisfied = true; /* Pretend it's satisfied */ + } + } + + if (any_set_satisfied) + { + /* Under-approximation injection: randomly skip adding ligature */ + bool skip_ligature = false; + if (_hb_depend_fuzzer_inject_under_approx > 0.0f) { + float r = (float)rand() / (float)RAND_MAX; + if (r < _hb_depend_fuzzer_inject_under_approx) { + skip_ligature = true; + } + } + + if (!skip_ligature && !hb_set_has (glyphs, lig_glyph)) + { + if (trace_closure) + DEBUG_MSG_DEPEND("Adding GSUB ligature %u (components satisfied)", lig_glyph); + hb_set_add (glyphs, lig_glyph); + lig_made_progress = true; + } + hb_set_add (ligatures_to_remove, lig_glyph); + } } - /* Mark this ligature for removal from deferred list */ - hb_set_add (ligatures_to_remove, lig_glyph); + hb_set_subtract (deferred_ligatures, ligatures_to_remove); } - } - /* Remove processed ligatures from deferred set */ - hb_set_subtract (deferred_ligatures, ligatures_to_remove); + iteration++; + } while (hb_set_get_population (glyphs) > prev_gsub_count && iteration < 64); + + hb_set_destroy (gsub_filter); + + if (trace_closure) + DEBUG_MSG_DEPEND("After GSUB (%u iterations): closure has %u glyphs", + iteration, hb_set_get_population (glyphs)); } + /* Stage 2: MATH closure */ + { + hb_set_t *math_filter = hb_set_create (); + hb_set_add (math_filter, HB_OT_TAG_MATH); + + hb_set_union (to_process, glyphs); + process_edges_for_tables (depend, glyphs, to_process, deferred_ligatures, + math_filter, skip_gsub, active_features); + hb_set_destroy (math_filter); + + if (trace_closure) + DEBUG_MSG_DEPEND("After MATH: closure has %u glyphs", + hb_set_get_population (glyphs)); + } + + /* Stage 3: COLR closure */ + { + hb_set_t *colr_filter = hb_set_create (); + hb_set_add (colr_filter, HB_TAG('C','O','L','R')); + + hb_set_union (to_process, glyphs); + process_edges_for_tables (depend, glyphs, to_process, deferred_ligatures, + colr_filter, skip_gsub, active_features); + hb_set_destroy (colr_filter); + + if (trace_closure) + DEBUG_MSG_DEPEND("After COLR: closure has %u glyphs", + hb_set_get_population (glyphs)); + } + + /* Stage 4: glyf/CFF closure */ + { + hb_set_t *glyf_filter = hb_set_create (); + hb_set_add (glyf_filter, HB_TAG('g','l','y','f')); + hb_set_add (glyf_filter, HB_TAG('C','F','F',' ')); + hb_set_add (glyf_filter, HB_TAG('c','m','a','p')); /* Include cmap for UVS */ + + hb_set_union (to_process, glyphs); + process_edges_for_tables (depend, glyphs, to_process, deferred_ligatures, + glyf_filter, skip_gsub, active_features); + hb_set_destroy (glyf_filter); + + if (trace_closure) + DEBUG_MSG_DEPEND("After glyf/CFF: closure has %u glyphs", + hb_set_get_population (glyphs)); + } + } /* End table stages */ + + /* Cleanup */ hb_set_destroy (ligatures_to_remove); hb_set_destroy (ligature_set_glyphs); hb_set_destroy (deferred_ligatures); @@ -634,11 +1100,11 @@ compute_subset_closure (hb_face_t *face, hb_set_t *glyphs, bool skip_gsub, while (hb_map_next (glyph_map, &idx, &key, &value)) hb_set_add (glyphs, key); - if (_hb_depend_fuzzer_debug_gid != HB_CODEPOINT_INVALID) + if (debug_gid != HB_CODEPOINT_INVALID) { - fprintf (stderr, "[DEBUG] Subset closure %s glyph %u\n", - hb_set_has (glyphs, _hb_depend_fuzzer_debug_gid) ? "INCLUDES" : "does not include", - _hb_depend_fuzzer_debug_gid); + DEBUG_MSG_DEPEND("Subset closure %s glyph %u", + hb_set_has (glyphs, debug_gid) ? "INCLUDES" : "does not include", + debug_gid); } hb_subset_plan_destroy (plan); @@ -729,21 +1195,37 @@ compare_closures (hb_face_t *face, hb_depend_t *depend, * This will replace subset_closure with the actual closure. */ compute_subset_closure (face, subset_closure, skip_gsub, codepoints, active_features); + if (trace_closure) + { + DEBUG_MSG_DEPEND("Subset: Final closure has %u glyphs, checking for 82 and 124...", + hb_set_get_population (subset_closure)); + DEBUG_MSG_DEPEND("Subset: glyph 82 ('o') %s in closure", + hb_set_has (subset_closure, 82) ? "IS" : "is NOT"); + DEBUG_MSG_DEPEND("Subset: glyph 124 (ordmasculine) %s in closure", + hb_set_has (subset_closure, 124) ? "IS" : "is NOT"); + } + /* Extract starting glyphs from subset's closure for fair comparison. - * For GSUB tests starting from codepoints, subset internally maps them to glyphs - * which may differ from our fuzzer's mapping. We need to use subset's starting - * glyphs to ensure both methods start from the same point. */ + * For GSUB tests starting from codepoints, we need to map codepoints to glyphs + * using ONLY cmap (not glyf/COLR/MATH), matching subset's order. */ hb_set_t *actual_start_glyphs = hb_set_create (); - /* Find glyphs that are in subset_closure. - * For GSUB tests, this includes glyphs mapped from input codepoints. - * We'll temporarily compute without GSUB to get just the starting glyphs. */ + /* For GSUB tests, map codepoints to glyphs via cmap only */ if (is_gsub && codepoints && !hb_set_is_empty (codepoints)) { - hb_set_t *temp_closure = hb_set_create (); - compute_subset_closure (face, temp_closure, true /* skip GSUB */, codepoints, NULL); - hb_set_union (actual_start_glyphs, temp_closure); - hb_set_destroy (temp_closure); + /* Add .notdef */ + hb_set_add (actual_start_glyphs, 0); + + /* Map each codepoint to its glyph via cmap */ + hb_font_t *font = hb_font_create (face); + hb_codepoint_t cp = HB_SET_VALUE_INVALID; + while (hb_set_next (codepoints, &cp)) + { + hb_codepoint_t gid; + if (hb_font_get_nominal_glyph (font, cp, &gid)) + hb_set_add (actual_start_glyphs, gid); + } + hb_font_destroy (font); } else { @@ -760,6 +1242,24 @@ compare_closures (hb_face_t *face, hb_depend_t *depend, hb_set_add (depend_closure, 0); /* Run depend with same active features as subset */ + if (debug_gid != HB_CODEPOINT_INVALID && active_features) + { + hb_codepoint_t feat = HB_SET_VALUE_INVALID; + fprintf (stderr, "DEPEND: Active features (%u):", hb_set_get_population (active_features)); + while (hb_set_next (active_features, &feat)) + fprintf (stderr, " %c%c%c%c", HB_UNTAG(feat)); + fprintf (stderr, "\n"); + } + + /* Seed RNG for error injection based on test seed to ensure reproducibility. + * For a given test seed and injection probability, the same edges will be + * skipped/added every time. */ + if (_hb_depend_fuzzer_inject_over_approx > 0.0f || + _hb_depend_fuzzer_inject_under_approx > 0.0f) + { + srand ((unsigned) seed); + } + compute_depend_closure (depend, depend_closure, skip_gsub, skip_gsub ? NULL : active_features); @@ -780,143 +1280,32 @@ compare_closures (hb_face_t *face, hb_depend_t *depend, /* CRITICAL ERROR: Subset found glyphs that depend API missed. * This means the depend API is incomplete - it failed to extract - * dependencies that actually exist in the font. - * Abort to signal fuzzer that a bug was found. */ + * dependencies that actually exist in the font. */ if (!hb_set_is_empty (in_subset_not_depend)) { - fprintf (stderr, "\n=== DEPEND API BUG DETECTED ===\n"); - fprintf (stderr, "Font: %s\n", font_path ? font_path : "(unknown)"); - fprintf (stderr, "Test seed: 0x%016llx\n", (unsigned long long) seed); - - /* DEBUG: Check if dependencies exist in the graph for missing glyphs */ - fprintf (stderr, "\n[DEBUG] Checking if dependencies exist in graph for missing glyphs:\n"); - hb_codepoint_t missing_gid = HB_SET_VALUE_INVALID; - unsigned missing_count = 0; - while (hb_set_next (in_subset_not_depend, &missing_gid) && missing_count < 3) + /* Report if verbose or reporting flag is set */ + if (_hb_depend_fuzzer_report_under_approximation || + _hb_depend_fuzzer_report_over_approximation || + _hb_depend_fuzzer_verbose) { - fprintf (stderr, "[DEBUG] Glyph %u: Looking for incoming edges...\n", missing_gid); + fprintf (stderr, "Test 0x%016llx: UNDER-APPROX - Depend missed %u glyphs:", + (unsigned long long) seed, hb_set_get_population (in_subset_not_depend)); - // Check if any glyph in the depend closure has this as a dependency - bool found_edge = false; - hb_codepoint_t src_gid = HB_SET_VALUE_INVALID; - while (hb_set_next (depend_closure, &src_gid)) - { - unsigned int index = 0; - hb_tag_t table_tag; - hb_codepoint_t dependent; - hb_tag_t layout_tag; - hb_codepoint_t ligature_set; - - while (hb_depend_get_glyph_entry (depend, src_gid, index++, - &table_tag, &dependent, - &layout_tag, &ligature_set)) - { - if (dependent == missing_gid) - { - fprintf (stderr, "[DEBUG] Edge found: %u -> %u (table=%c%c%c%c, ligset=%u)\n", - src_gid, missing_gid, HB_UNTAG(table_tag), ligature_set); - found_edge = true; - break; - } - } - if (found_edge) break; - } - - if (!found_edge) - { - fprintf (stderr, "[DEBUG] NO EDGE FOUND from any glyph in depend_closure!\n"); - fprintf (stderr, "[DEBUG] This is a GRAPH EXTRACTION BUG - dependency not recorded.\n"); - } - else - { - fprintf (stderr, "[DEBUG] Edge exists but wasn't followed - CLOSURE BUG!\n"); - } - - missing_count++; - } - fprintf (stderr, "\n"); - - fprintf (stderr, "Starting glyphs (%u): ", hb_set_get_population (start_glyphs)); - hb_codepoint_t gid = HB_SET_VALUE_INVALID; - unsigned gid_count = 0; - while (hb_set_next (start_glyphs, &gid) && gid_count < 20) - { - fprintf (stderr, "%u ", gid); - gid_count++; - } - if (gid_count == 20 && hb_set_get_population (start_glyphs) > 20) - fprintf (stderr, "..."); - fprintf (stderr, "\n"); - - if (codepoints && !hb_set_is_empty (codepoints)) - { - fprintf (stderr, "Starting codepoints (%u): ", hb_set_get_population (codepoints)); - hb_codepoint_t cp = HB_SET_VALUE_INVALID; - unsigned cp_count = 0; - /* Print all codepoints to enable exact reproduction */ - while (hb_set_next (codepoints, &cp)) - { - fprintf (stderr, "U+%04X ", cp); - cp_count++; - if (cp_count % 20 == 0) - fprintf (stderr, "\n "); - } - fprintf (stderr, "\n"); - } - - if (!skip_gsub && !hb_set_is_empty (active_features)) - { - fprintf (stderr, "Active features (%u): ", hb_set_get_population (active_features)); - hb_codepoint_t feature = HB_SET_VALUE_INVALID; + hb_codepoint_t gid = HB_SET_VALUE_INVALID; unsigned count = 0; - while (hb_set_next (active_features, &feature) && count < 10) + while (hb_set_next (in_subset_not_depend, &gid) && count < 20) { - fprintf (stderr, "%c%c%c%c ", - HB_UNTAG ((hb_tag_t) feature)); + fprintf (stderr, " %u", gid); count++; } - if (count == 10 && !hb_set_is_empty (active_features)) - fprintf (stderr, "..."); + if (count == 20) + fprintf (stderr, " ..."); fprintf (stderr, "\n"); } - /* DEBUG: Print both closures */ - fprintf (stderr, "\nDEBUG - Subset closure glyphs: "); - gid = HB_SET_VALUE_INVALID; - while (hb_set_next (subset_closure, &gid)) - fprintf (stderr, "%u ", gid); - fprintf (stderr, "\n"); - - fprintf (stderr, "DEBUG - Depend closure glyphs: "); - gid = HB_SET_VALUE_INVALID; - while (hb_set_next (depend_closure, &gid)) - fprintf (stderr, "%u ", gid); - fprintf (stderr, "\n\n"); - - fprintf (stderr, "Depend closure: %u glyphs\n", hb_set_get_population (depend_closure)); - fprintf (stderr, "Subset closure: %u glyphs\n", hb_set_get_population (subset_closure)); - fprintf (stderr, "\nMissing glyphs in depend (%u):", - hb_set_get_population (in_subset_not_depend)); - gid = HB_SET_VALUE_INVALID; - unsigned count = 0; - while (hb_set_next (in_subset_not_depend, &gid) && count < 50) - { - fprintf (stderr, " %u", gid); - count++; - } - if (count == 50 && !hb_set_is_empty (in_subset_not_depend)) - fprintf (stderr, " ..."); - fprintf (stderr, "\n"); - fprintf (stderr, "===============================\n\n"); - - hb_set_destroy (in_subset_not_depend); - hb_set_destroy (in_depend_not_subset); - hb_set_destroy (active_features); - hb_set_destroy (subset_closure); - hb_set_destroy (depend_closure); - - /* Abort to signal fuzzer about the bug */ - assert (0 && "Depend API missed glyphs that subset found"); + /* Abort unless reporting flag is set */ + if (!_hb_depend_fuzzer_report_under_approximation) + assert (0 && "Depend API missed glyphs that subset found"); } /* Report cases where depend has extra glyphs (over-approximation) @@ -994,11 +1383,106 @@ extern "C" int LLVMFuzzerTestOneInput (const uint8_t *data, size_t size) return 0; } + /* Calculate adaptive probabilities based on graph size if error injection is enabled. + * Goal: achieve ~2% detection rate regardless of font complexity. + * + * Strategy: + * - Count total edges E in the dependency graph + * - Estimate average edges traversed per test: V ≈ E * 0.0025 (0.25%) + * - Calculate per-edge probability: P = target_rate / V + * + * This makes user input (e.g., -u 0.02) specify the TARGET detection rate (2%), + * which is then scaled based on font complexity. Complex fonts like NotoNastaliq + * get lower per-edge probability, simple fonts get higher per-edge probability. + * + * The 0.0025 multiplier was tuned empirically across 8 system fonts to achieve + * avg ~2% detection rate (measured: 2.12% under, 3.20% over on 4000 tests). + */ + if (_hb_depend_fuzzer_inject_over_approx > 0.0f || + _hb_depend_fuzzer_inject_under_approx > 0.0f) + { + /* Count total edges in dependency graph */ + unsigned total_edges = 0; + unsigned num_glyphs = hb_face_get_glyph_count (face); + for (unsigned gid = 0; gid < num_glyphs; gid++) + { + hb_codepoint_t idx = 0; + hb_tag_t table_tag; + hb_codepoint_t dependent; + hb_tag_t layout_tag; + hb_codepoint_t ligature_set; + hb_codepoint_t context_set; + + while (hb_depend_get_glyph_entry (depend, gid, idx++, + &table_tag, &dependent, + &layout_tag, &ligature_set, &context_set)) + { + total_edges++; + } + } + + /* Estimate average edges traversed per test (roughly 0.25% of total edges) */ + float estimated_edges_per_test = total_edges * 0.0025f; + if (estimated_edges_per_test < 5.0f) + estimated_edges_per_test = 5.0f; /* Minimum to avoid division issues */ + + /* Calculate adaptive probabilities to achieve target detection rate + * User input is now interpreted as TARGET_RATE not per-edge probability */ + if (_hb_depend_fuzzer_inject_over_approx > 0.0f) + { + float target_rate = _hb_depend_fuzzer_inject_over_approx; + _hb_depend_fuzzer_inject_over_approx = target_rate / estimated_edges_per_test; + } + + if (_hb_depend_fuzzer_inject_under_approx > 0.0f) + { + float target_rate = _hb_depend_fuzzer_inject_under_approx; + _hb_depend_fuzzer_inject_under_approx = target_rate / estimated_edges_per_test; + } + + if (_hb_depend_fuzzer_verbose) + { + fprintf (stderr, "# Adaptive error injection: %u total edges, est. %.0f edges/test\n", + total_edges, estimated_edges_per_test); + fprintf (stderr, "# Calculated per-edge probabilities: over=%.6f under=%.6f\n", + _hb_depend_fuzzer_inject_over_approx, + _hb_depend_fuzzer_inject_under_approx); + } + } + /* Create font for codepoint-to-glyph mapping */ hb_font_t *font = hb_font_create (face); - /* Check if running single test or full test suite */ - if (_hb_depend_fuzzer_single_test != 0) + /* Check test mode: explicit, single test, or full suite */ + if (_hb_depend_fuzzer_explicit_test) + { + /* Run explicit test with command-line parameters */ + test_params params = generate_test_from_explicit (face, font); + + /* Check if parsing succeeded */ + if (params.glyphs == NULL) + { + fprintf (stderr, "Error: Failed to generate test from explicit parameters\n"); + } + else + { + /* Always print parameters for explicit test */ + print_test_params (0, params.is_gsub, params.glyphs, params.codepoints, params.features); + + /* Run closure comparison (use seed 0 for explicit tests) */ + compare_closures (face, depend, 0, _hb_depend_fuzzer_current_font_path, + params.is_gsub, params.glyphs, params.codepoints, params.features); + } + + /* Clean up test parameters */ + if (params.glyphs) + hb_set_destroy (params.glyphs); + if (params.codepoints) + hb_set_destroy (params.codepoints); + if (params.features) + hb_set_destroy (params.features); + } + else if (_hb_depend_fuzzer_single_test != 0) { /* Run single specific test */ uint64_t test_seed = _hb_depend_fuzzer_single_test; diff --git a/test/fuzzing/hb-depend-fuzzer.cc b/test/fuzzing/hb-depend-fuzzer.cc index 3d8f2820b..442485df8 100644 --- a/test/fuzzing/hb-depend-fuzzer.cc +++ b/test/fuzzing/hb-depend-fuzzer.cc @@ -27,18 +27,21 @@ extern "C" int LLVMFuzzerTestOneInput (const uint8_t *data, size_t size) hb_codepoint_t dependent; hb_tag_t layout_tag; hb_codepoint_t ligature_set; + hb_codepoint_t context_set; // Iterate through up to 10 entries per glyph while (index < 10 && hb_depend_get_glyph_entry (depend, gid, index++, &table_tag, &dependent, - &layout_tag, &ligature_set)) + &layout_tag, &ligature_set, + &context_set)) { // Just access the data; success = no crash (void) table_tag; (void) dependent; (void) layout_tag; (void) ligature_set; + (void) context_set; } }