Add MSL support for mixed-precision cooperative matrix multiply-add (#9629)

This commit is contained in:
Mike Seddon authored and GitHub committed 2026-06-25 22:14:27 -04:00
1 parent 0d734cb6de
commit cab7d0b47f
7 files changed
+192 -8

No files matched your search

+1
View File
@@ -171,6 +171,7 @@ By @beholdnec in [#8505](https://github.com/gfx-rs/wgpu/pull/8505).
- spirv-out ray tracing pipelines. By @Vecvec in [#9085](https://github.com/gfx-rs/wgpu/pull/9085).
- Add `spirv-out` ray tracing pipelines. By @Vecvec in [#9085](https://github.com/gfx-rs/wgpu/pull/9085).
- Add `naga::front::wgsl::ParseError::notes()`. By @kwillemsen in [#9572](https://github.com/gfx-rs/wgpu/pull/9572).
- Add MSL support for cooperative matrix multiply-add with lower-precision A/B operands and a higher-precision accumulator/result, such as `coopMultiplyAdd(f16, f16, f32) -> f32`. By @seddonm1 in [#9629](https://github.com/gfx-rs/wgpu/pull/9629).
### Changes
+16 -8
View File
@@ -514,7 +514,8 @@ pub(super) enum WrappedFunction {
columns: crate::CooperativeSize,
rows: crate::CooperativeSize,
intermediate: crate::CooperativeSize,
scalar: crate::Scalar,
ab_scalar: crate::Scalar,
c_scalar: crate::Scalar,
},
RayQueryGetIntersection {
committed: bool,
@@ -6453,9 +6454,10 @@ template <typename A>
space: crate::AddressSpace,
a: Handle<crate::Expression>,
b: Handle<crate::Expression>,
c: Handle<crate::Expression>,
) -> BackendResult {
let space_name = space.to_msl_name().unwrap_or_default();
let (a_c, a_r, scalar) = match *func_ctx.resolve_type(a, &module.types) {
let (a_c, a_r, ab_scalar) = match *func_ctx.resolve_type(a, &module.types) {
crate::TypeInner::CooperativeMatrix {
columns,
rows,
@@ -6468,26 +6470,32 @@ template <typename A>
crate::TypeInner::CooperativeMatrix { columns, rows, .. } => (columns, rows),
_ => unreachable!(),
};
let c_scalar = match *func_ctx.resolve_type(c, &module.types) {
crate::TypeInner::CooperativeMatrix { scalar, .. } => scalar,
_ => unreachable!(),
};
let wrapped = WrappedFunction::CooperativeMultiplyAdd {
space_name,
columns: b_c,
rows: a_r,
intermediate: a_c,
scalar,
ab_scalar,
c_scalar,
};
if !self.wrapped_functions.insert(wrapped) {
return Ok(());
}
let scalar_name = scalar.to_msl_name();
let ab_scalar_name = ab_scalar.to_msl_name();
let c_scalar_name = c_scalar.to_msl_name();
writeln!(
self.out,
"{NAMESPACE}::simdgroup_{scalar_name}{}x{} {COOPERATIVE_MULTIPLY_ADD_FUNCTION}(const {space_name} {NAMESPACE}::simdgroup_{scalar_name}{}x{}& a, const {space_name} {NAMESPACE}::simdgroup_{scalar_name}{}x{}& b, const {space_name} {NAMESPACE}::simdgroup_{scalar_name}{}x{}& c) {{",
"{NAMESPACE}::simdgroup_{c_scalar_name}{}x{} {COOPERATIVE_MULTIPLY_ADD_FUNCTION}(const {space_name} {NAMESPACE}::simdgroup_{ab_scalar_name}{}x{}& a, const {space_name} {NAMESPACE}::simdgroup_{ab_scalar_name}{}x{}& b, const {space_name} {NAMESPACE}::simdgroup_{c_scalar_name}{}x{}& c) {{",
b_c as u32, a_r as u32, a_c as u32, a_r as u32, b_c as u32, b_r as u32, b_c as u32, a_r as u32,
)?;
let l1 = back::Level(1);
writeln!(
self.out,
"{l1}{NAMESPACE}::simdgroup_{scalar_name}{}x{} d;",
"{l1}{NAMESPACE}::simdgroup_{c_scalar_name}{}x{} d;",
b_c as u32, a_r as u32
)?;
writeln!(self.out, "{l1}simdgroup_multiply_accumulate(d,a,b,c);")?;
@@ -6585,9 +6593,9 @@ template <typename A>
data.pointer,
)?;
}
crate::Expression::CooperativeMultiplyAdd { a, b, c: _ } => {
crate::Expression::CooperativeMultiplyAdd { a, b, c } => {
let space = crate::AddressSpace::Private;
self.write_wrapped_cooperative_multiply_add(module, func_ctx, space, a, b)?;
self.write_wrapped_cooperative_multiply_add(module, func_ctx, space, a, b, c)?;
}
crate::Expression::RayQueryGetIntersection { committed, .. } => {
self.write_rq_get_intersection_function(module, committed)?;
@@ -0,0 +1,9 @@
targets = "SPIRV | METAL | WGSL"
capabilities = "COOPERATIVE_MATRIX | SHADER_FLOAT16"
[spv]
debug = false
version = [1, 4]
[msl]
lang_version = [2, 3]
@@ -0,0 +1,21 @@
enable f16;
enable wgpu_cooperative_matrix;
@group(0) @binding(0)
var<storage, read_write> ab: array<f16>;
@group(0) @binding(1)
var<storage, read_write> accum: array<f32>;
@compute @workgroup_size(8, 8, 1)
fn main() {
// loading f16 multiplicands from memory
let a: coop_mat8x8<f16, A> = coopLoad<coop_mat8x8<f16, A>>(&ab[0]);
let b: coop_mat8x8<f16, B> = coopLoad<coop_mat8x8<f16, B>>(&ab[0]);
// loading f32 accumulator from memory
var c: coop_mat8x8<f32, C> = coopLoad<coop_mat8x8<f32, C>>(&accum[0]);
// actual mixed-accumulator multiply-add
c = coopMultiplyAdd(a, b, c);
// storing f32 accumulator into memory
coopStore(c, &accum[0]);
}
@@ -0,0 +1,47 @@
// language: metal2.3
#include <metal_stdlib>
#include <simd/simd.h>
using metal::uint;
struct _mslBufferSizes {
uint size0;
uint size1;
};
typedef half type_1[1];
typedef float type_3[1];
metal::simdgroup_half8x8 NagaCooperativeLoad(const device half* ptr, int stride, bool is_row_major) {
metal::simdgroup_half8x8 m;
simdgroup_load(m, ptr, stride, 0, is_row_major);
return m;
}
metal::simdgroup_float8x8 NagaCooperativeLoad(const device float* ptr, int stride, bool is_row_major) {
metal::simdgroup_float8x8 m;
simdgroup_load(m, ptr, stride, 0, is_row_major);
return m;
}
metal::simdgroup_float8x8 NagaCooperativeMultiplyAdd(const thread metal::simdgroup_half8x8& a, const thread metal::simdgroup_half8x8& b, const thread metal::simdgroup_float8x8& c) {
metal::simdgroup_float8x8 d;
simdgroup_multiply_accumulate(d,a,b,c);
return d;
}
[[max_total_threads_per_threadgroup(64)]] kernel void main_(
device type_1 const& ab [[user(fake0)]]
, device type_3& accum [[user(fake0)]]
, constant _mslBufferSizes& _buffer_sizes [[user(fake0)]]
) {
metal::simdgroup_float8x8 c = {};
metal::simdgroup_half8x8 a = NagaCooperativeLoad(&ab[0], 8u, true);
metal::simdgroup_half8x8 b = NagaCooperativeLoad(&ab[0], 8u, true);
c = NagaCooperativeLoad(&accum[0], 8u, true);
metal::simdgroup_float8x8 _e13 = c;
c = NagaCooperativeMultiplyAdd(a, b, _e13);
metal::simdgroup_float8x8 _e15 = c;
simdgroup_store(_e15, &accum[0], 8u, 0, true);
return;
}
@@ -0,0 +1,77 @@
; SPIR-V
; Version: 1.4
; Generator: rspirv
; Bound: 45
OpCapability Shader
OpCapability Float16
OpCapability StorageBuffer16BitAccess
OpCapability UniformAndStorageBuffer16BitAccess
OpCapability StorageInputOutput16
OpCapability CooperativeMatrixKHR
OpCapability VulkanMemoryModel
OpExtension "SPV_KHR_16bit_storage"
OpExtension "SPV_KHR_cooperative_matrix"
OpExtension "SPV_KHR_vulkan_memory_model"
%1 = OpExtInstImport "GLSL.std.450"
OpMemoryModel Logical Vulkan
OpEntryPoint GLCompute %19 "main" %12 %15
OpExecutionMode %19 LocalSize 8 8 1
OpDecorate %4 ArrayStride 2
OpDecorate %6 ArrayStride 4
OpDecorate %12 DescriptorSet 0
OpDecorate %12 Binding 0
OpDecorate %13 Block
OpMemberDecorate %13 0 Offset 0
OpDecorate %15 DescriptorSet 0
OpDecorate %15 Binding 1
OpDecorate %16 Block
OpMemberDecorate %16 0 Offset 0
%2 = OpTypeVoid
%3 = OpTypeFloat 16
%4 = OpTypeRuntimeArray %3
%5 = OpTypeFloat 32
%6 = OpTypeRuntimeArray %5
%9 = OpTypeInt 32 0
%8 = OpConstant %9 3
%10 = OpConstant %9 8
%11 = OpConstant %9 2
%7 = OpTypeCooperativeMatrixKHR %5 %8 %10 %10 %11
%13 = OpTypeStruct %4
%14 = OpTypePointer StorageBuffer %13
%12 = OpVariable %14 StorageBuffer
%16 = OpTypeStruct %6
%17 = OpTypePointer StorageBuffer %16
%15 = OpVariable %17 StorageBuffer
%20 = OpTypeFunction %2
%21 = OpTypePointer StorageBuffer %4
%22 = OpConstant %9 0
%24 = OpTypePointer StorageBuffer %6
%27 = OpTypePointer Function %7
%28 = OpConstantNull %7
%30 = OpTypePointer StorageBuffer %3
%31 = OpTypeCooperativeMatrixKHR %3 %8 %10 %10 %22
%32 = OpConstant %9 1
%35 = OpTypeCooperativeMatrixKHR %3 %8 %10 %10 %32
%38 = OpTypePointer StorageBuffer %5
%19 = OpFunction %2 None %20
%18 = OpLabel
%26 = OpVariable %27 Function %28
%23 = OpAccessChain %21 %12 %22
%25 = OpAccessChain %24 %15 %22
OpBranch %29
%29 = OpLabel
%33 = OpAccessChain %30 %23 %22
%34 = OpCooperativeMatrixLoadKHR %31 %33 %32 %10
%36 = OpAccessChain %30 %23 %22
%37 = OpCooperativeMatrixLoadKHR %35 %36 %32 %10
%39 = OpAccessChain %38 %25 %22
%40 = OpCooperativeMatrixLoadKHR %7 %39 %32 %10
OpStore %26 %40
%41 = OpLoad %7 %26
%42 = OpCooperativeMatrixMulAddKHR %7 %34 %37 %41
OpStore %26 %42
%43 = OpLoad %7 %26
%44 = OpAccessChain %38 %25 %22
OpCooperativeMatrixStoreKHR %44 %43 %32 %10
OpReturn
OpFunctionEnd
@@ -0,0 +1,21 @@
enable f16;
enable wgpu_cooperative_matrix;
@group(0) @binding(0)
var<storage, read_write> ab: array<f16>;
@group(0) @binding(1)
var<storage, read_write> accum: array<f32>;
@compute @workgroup_size(8, 8, 1)
fn main() {
var c: coop_mat8x8<f32,C>;
let a = coopLoad<coop_mat8x8<f16,A>>((&ab[0]), 8u);
let b = coopLoad<coop_mat8x8<f16,B>>((&ab[0]), 8u);
c = coopLoad<coop_mat8x8<f32,C>>((&accum[0]), 8u);
let _e13 = c;
c = coopMultiplyAdd(a, b, _e13);
let _e15 = c;
coopStore(_e15, (&accum[0]), 8u);
return;
}