[repacker] De-duplicate overflow records without an overflow_record_t hash map

will_overflow() de-duplicated (parent, child) overflow records through
an hb_hashmap_t keyed by overflow_record_t, whose hash() collapses to
2654435761 * (31 * parent + child).  With dense vertex ids, a graph
where many consecutive vertices link the same children puts all records
into a handful of equal-hash chains, so the map degenerates into
quadratic probing and repeated over-growth in every resolution round: a
crafted 60 KB GDEF font with a LigCaretList of overlapping LigGlyphs
took ~65 s of CPU and ~520 MiB before failing.

graph.ordering_ is a permutation of the vertices, so every parent is
visited exactly once and a duplicate record can only come from two
links of the same parent to the same child.  A per-parent hb_set_t of
the children already recorded, cleared when moving to the next parent,
yields exactly the same records in the same order with no hashing of
records at all.  The now-unused operator==/!=/hash() on
overflow_record_t are removed (the struct is only used in vectors
elsewhere).

A regression test, test_will_overflow_records(), asserts one record per
overflowing (parent, child) pair when a parent links the same child
twice.  The attack font drops to ~1.3 s CPU and 133 MiB, the remaining
cost being the 1.8 million links the sanitizer budget admits; no public
API/ABI change.

Discovered by Anthropic's OSS Scanner, as vulnerability ANT-2026-QNM0BP56.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Behdad EsfahbodandClaude Opus 4.8 committed 2026-10-30 15:37:38 -06:00
1 parent 9179b64235
commit 7e28f358ae
2 files changed
+52 -21

No files matched your search

+9 -21
View File
@@ -35,23 +35,6 @@ struct overflow_record_t
{
unsigned parent;
unsigned child;
bool operator != (const overflow_record_t o) const
{ return !(*this == o); }
inline bool operator == (const overflow_record_t& o) const
{
return parent == o.parent &&
child == o.child;
}
inline uint32_t hash () const
{
uint32_t current = 0;
current = current * 31 + hb_hash (parent);
current = current * 31 + hb_hash (child);
return current;
}
};
inline
@@ -113,10 +96,15 @@ will_overflow (graph_t& graph,
if (overflows) overflows->clear ();
graph.update_positions ();
hb_hashmap_t<overflow_record_t, bool> record_set;
// Each parent is visited once (ordering_ is a permutation of the vertices),
// so duplicate records can only come from two links of the same parent to the
// same child. A per-parent set of children avoids hashing (parent, child)
// records, whose hash a font can make collide.
hb_set_t overflowing_children;
const auto& vertices = graph.vertices_;
for (unsigned parent_idx : graph.ordering_)
{
overflowing_children.clear ();
// Don't need to check virtual links for overflow
for (const auto& link : vertices.arrayZ[parent_idx].obj ().real_links)
{
@@ -129,14 +117,14 @@ will_overflow (graph_t& graph,
overflow_record_t r;
r.parent = parent_idx;
r.child = link.objidx;
if (record_set.has(r)) continue; // don't keep duplicate overflows.
if (overflowing_children.has (r.child)) continue; // don't keep duplicate overflows.
overflows->push (r);
record_set.set (r, true);
overflowing_children.add (r.child);
}
}
TRY (graph_result_t<void>::from (record_set, ALLOCATION_FAILURE));
TRY (graph_result_t<void>::from (overflowing_children, ALLOCATION_FAILURE));
if (overflows)
TRY (graph_result_t<void>::from (*overflows, ALLOCATION_FAILURE));
+43
View File
@@ -2187,6 +2187,48 @@ static void test_will_overflow_3 ()
free (buffer);
}
static void test_will_overflow_records ()
{
size_t buffer_size = 160000;
void* buffer = malloc (buffer_size);
hb_serialize_context_t c (buffer, buffer_size);
std::string large_string(70000, 'a');
c.start_serialize<char> ();
unsigned obj_leaf = add_object ("d", 1, &c);
unsigned obj_large = add_object (large_string.c_str(), 70000, &c);
start_object ("c", 1, &c);
add_offset (obj_leaf, &c);
add_offset (obj_leaf, &c);
unsigned obj_c = c.pop_pack (false);
start_object ("b", 1, &c);
add_offset (obj_leaf, &c);
add_offset (obj_leaf, &c);
unsigned obj_b = c.pop_pack (false);
start_object ("a", 1, &c);
add_offset (obj_b, &c);
add_offset (obj_c, &c);
add_offset (obj_large, &c);
c.pop_pack (false);
c.end_serialize();
graph_t graph = graph_t::create (c.object_graph ()).value ();
// One record per overflowing (parent, child) pair, even if linked more than once.
hb_vector_t<graph::overflow_record_t> overflows;
hb_always_assert (graph::will_overflow (graph, &overflows).value ());
hb_always_assert (overflows.length == 2);
hb_always_assert (overflows[0].parent != overflows[1].parent);
hb_always_assert (overflows[0].child == overflows[1].child);
free (buffer);
}
static void test_resolve_overflows_via_sort ()
{
size_t buffer_size = 160000;
@@ -3262,6 +3304,7 @@ main (int argc, char **argv)
test_will_overflow_1 ();
test_will_overflow_2 ();
test_will_overflow_3 ();
test_will_overflow_records ();
test_resolve_overflows_via_sort ();
test_resolve_overflows_via_duplication ();
test_resolve_overflows_via_multiple_duplications ();