[subset] Bound shared COLRv0 layer expansion and make V0 closures linear

COLR::subset() gives every retained COLRv0 base glyph a private copy of
its layer range, so N base glyphs sharing L layers cost N x L layer
copies with two hash lookups each, and nothing checked the total against
the 16-bit numLayers header field.  serialize_V0() walks the lazy
iterator twice and _hb_subset_table_try() retries while doubling the
buffer, so a ~200 KB font with all records naming the same range kept
hb-subset busy for about two minutes before failing.

Cap the summed layer count with a cheap walk over base_it before layer_it
is built, and drop COLR (return_trace (false)) when the total exceeds
0xFFFF, the same way this function already handles V0 records it cannot
subset; this also removes the 16-bit wrap in serialize_V0().  Any total
above 65535 could only have produced a failed subset or a wrapped,
invalid numLayers, so no valid output is lost.

The two V0 closure helpers had the same N x L shape during planning;
collect the layer indices of all retained base glyphs into one set first
and visit each layer once.  closure_glyphs() now takes the glyph set in
one call (the accelerator wrapper changes signature with it; no public
API change), and _colr_closure() passes the whole set instead of looping.

Adds test_subset_colr_shared_layers (256 base glyphs all claiming the
same 256 layers): retaining 255 gives a 65280-layer table, retaining all
256 gives an empty COLR.

Discovered by Anthropic's OSS Scanner, as vulnerability ANT-2026-4T7NHCTK.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Behdad EsfahbodandClaude Opus 4.8 committed 2026-10-30 16:04:35 -06:00
1 parent f840e4d018
commit c5dedbef33
5 files changed
+122 -16

No files matched your search

+42 -14
View File
@@ -2591,9 +2591,9 @@ struct COLR
colr->depend (depend_data);
}
void closure_glyphs (hb_codepoint_t glyph,
void closure_glyphs (const hb_set_t *glyphs,
hb_set_t *related_ids /* OUT */) const
{ colr->closure_glyphs (glyph, related_ids); }
{ colr->closure_glyphs (glyphs, related_ids); }
void closure_V0palette_indices (const hb_set_t *glyphs,
hb_set_t *palettes /* OUT */) const
@@ -2681,16 +2681,33 @@ struct COLR
}
}
void closure_glyphs (hb_codepoint_t glyph,
/* Base glyphs may share layers. The V0 closures collect the layer
* indices of all base glyphs first, so each layer is visited once. */
void collect_V0layer_indices (const BaseGlyphRecord &record,
hb_set_t *layer_indices /* OUT */) const
{
unsigned count = (this+layersZ).as_array (numLayers).sub_array (record.firstLayerIdx,
record.numLayers).length;
if (!count) return;
layer_indices->add_range (record.firstLayerIdx, record.firstLayerIdx + count - 1);
}
void closure_glyphs (const hb_set_t *glyphs,
hb_set_t *related_ids /* OUT */) const
{
const BaseGlyphRecord *record = get_base_glyph_record (glyph);
if (!record) return;
if (!numBaseGlyphs || !numLayers) return;
hb_array_t<const LayerRecord> all_layers = (this+layersZ).as_array (numLayers);
auto glyph_layers = (this+layersZ).as_array (numLayers).sub_array (record->firstLayerIdx,
record->numLayers);
if (!glyph_layers.length) return;
related_ids->add_array (&glyph_layers[0].glyphId, glyph_layers.length, LayerRecord::min_size);
hb_set_t layer_indices;
for (hb_codepoint_t glyph : *glyphs)
{
const BaseGlyphRecord *record = get_base_glyph_record (glyph);
if (!record) continue;
collect_V0layer_indices (*record, &layer_indices);
}
for (unsigned idx : layer_indices)
related_ids->add (all_layers[idx].glyphId);
}
void closure_V0palette_indices (const hb_set_t *glyphs,
@@ -2700,14 +2717,15 @@ struct COLR
hb_array_t<const BaseGlyphRecord> baseGlyphs = (this+baseGlyphsZ).as_array (numBaseGlyphs);
hb_array_t<const LayerRecord> all_layers = (this+layersZ).as_array (numLayers);
for (const BaseGlyphRecord record : baseGlyphs)
hb_set_t layer_indices;
for (const BaseGlyphRecord &record : baseGlyphs)
{
if (!glyphs->has (record.glyphId)) continue;
hb_array_t<const LayerRecord> glyph_layers = all_layers.sub_array (record.firstLayerIdx,
record.numLayers);
for (const LayerRecord layer : glyph_layers)
palettes->add (layer.colorIdx);
collect_V0layer_indices (record, &layer_indices);
}
for (unsigned idx : layer_indices)
palettes->add (all_layers[idx].colorIdx);
}
void closure_forV1 (hb_set_t *glyphset,
@@ -2980,6 +2998,16 @@ struct COLR
| hb_map_retains_sorting (hb_second)
;
/* Base glyphs sharing layers each get their own copy in the output.
* Drop the table instead of expanding more layers than numLayers holds. */
unsigned num_layers = 0;
for (const BaseGlyphRecord record : + base_it)
{
num_layers += record.numLayers;
if (unlikely (num_layers > 0xFFFFu))
return_trace (false);
}
auto layer_it =
+ hb_range (c->plan->num_output_glyphs ())
| hb_map (reverse_glyph_map)
+1 -2
View File
@@ -118,8 +118,7 @@ static void _colr_closure (hb_subset_plan_t* plan,
hb_set_t palette_indices, layer_indices;
// Collect all glyphs referenced by COLRv0
hb_set_t glyphset_colrv0;
for (hb_codepoint_t gid : *glyphs_colred)
colr.closure_glyphs (gid, &glyphset_colrv0);
colr.closure_glyphs (glyphs_colred, &glyphset_colrv0);
glyphs_colred->union_ (glyphset_colrv0);
Binary file not shown.
@@ -0,0 +1,34 @@
"""Generate a COLRv0 font whose base glyphs all share the same layer range."""
from pathlib import Path
from fontTools.fontBuilder import FontBuilder
from fontTools.pens.ttGlyphPen import TTGlyphPen
from fontTools.ttLib.tables.DefaultTable import DefaultTable
import struct
# Every base glyph claims all the layers, so retaining 256 of them expands
# to 65536 layer records, one more than the COLR header can count.
count = 256
bases = [f"base{i}" for i in range(count)]
layers = [f"layer{i}" for i in range(count)]
names = [".notdef"] + bases + layers
fb = FontBuilder(1000, isTTF=True)
fb.setupGlyphOrder(names)
fb.setupCharacterMap({0xE000 + i: name for i, name in enumerate(bases)})
fb.setupGlyf({name: TTGlyphPen(None).glyph() for name in names})
fb.setupHorizontalMetrics({name: (1000, 0) for name in names})
fb.setupHorizontalHeader(ascent=800, descent=-200)
fb.setupNameTable({"familyName": "COLR Shared Layers", "styleName": "Regular"})
fb.setupOS2(sTypoAscender=800, sTypoDescender=-200, usWinAscent=800, usWinDescent=200)
fb.setupPost()
fb.setupCPAL([[(0, 0, 0, 1), (1, 1, 1, 1)]])
# fontTools writes each base glyph its own layers; build the table by hand.
header = struct.pack(">HHIIH", 0, count, 14, 14 + count * 6, count)
base_records = b"".join(struct.pack(">HHH", 1 + i, 0, count) for i in range(count))
layer_records = b"".join(struct.pack(">HH", 1 + count + i, i % 2) for i in range(count))
colr = DefaultTable("COLR")
colr.data = header + base_records + layer_records
fb.font["COLR"] = colr
fb.save(Path(__file__).with_name("colr-shared-layers.ttf"))
+45
View File
@@ -647,6 +647,50 @@ test_subset_colr_layers_out_of_range (void)
hb_blob_destroy (colr);
}
static void
test_subset_colr_shared_layers (void)
{
hb_face_t *face = hb_test_open_font_file ("fonts/colr-shared-layers.ttf");
/* Every base glyph claims all 256 layers. Retaining 255 of them expands
* to 65280 layer records; one more does not fit numLayers. */
for (unsigned count = 255; count <= 256; count++)
{
hb_subset_input_t *input = hb_subset_input_create_or_fail ();
hb_set_add_range (hb_subset_input_unicode_set (input), 0xE000, 0xE000 + count - 1);
hb_face_t *subset = hb_subset_test_create_subset (face, input);
hb_blob_t *blob = hb_face_reference_table (subset, HB_TAG ('C','O','L','R'));
unsigned length;
const unsigned char *data = (const unsigned char *) hb_blob_get_data (blob, &length);
if (count == 256)
{
g_assert_cmpuint (length, ==, 0);
hb_blob_destroy (blob);
hb_face_destroy (subset);
continue;
}
g_assert_cmpuint (length, ==, 14 + count * 6 + count * 256 * 4);
g_assert_cmpuint (read_uint_be (data + 2, 2), ==, count);
g_assert_cmpuint (read_uint_be (data + 12, 2), ==, count * 256);
unsigned bases = read_uint_be (data + 4, 4), layers = read_uint_be (data + 8, 4);
for (unsigned i = 0; i < count; i++)
{
g_assert_cmpuint (read_uint_be (data + bases + i * 6, 2), ==, i + 1);
g_assert_cmpuint (read_uint_be (data + bases + i * 6 + 2, 2), ==, i * 256);
g_assert_cmpuint (read_uint_be (data + bases + i * 6 + 4, 2), ==, 256);
for (unsigned j = 0; j < 256; j++)
{
unsigned layer = layers + (i * 256 + j) * 4;
g_assert_cmpuint (read_uint_be (data + layer, 2), ==, count + 1 + j);
g_assert_cmpuint (read_uint_be (data + layer + 2, 2), ==, j % 2);
}
}
hb_blob_destroy (blob);
hb_face_destroy (subset);
}
hb_face_destroy (face);
}
int
main (int argc, char **argv)
{
@@ -664,6 +708,7 @@ main (int argc, char **argv)
hb_test_add (test_subset_colr_sentinel_map_without_variations);
hb_test_add (test_subset_colr_var_overflow);
hb_test_add (test_subset_colr_layers_out_of_range);
hb_test_add (test_subset_colr_shared_layers);
return hb_test_run();
}