From cab7d0b47f2f3d9f5900f586a2fb215e355574c8 Mon Sep 17 00:00:00 2001 From: Mike Seddon Date: Fri, 26 Jun 2026 12:14:27 +1000 Subject: [PATCH] Add MSL support for mixed-precision cooperative matrix multiply-add (#9629) --- CHANGELOG.md | 1 + naga/src/back/msl/writer.rs | 24 ++++-- .../cooperative-matrix-mixed-accumulator.toml | 9 +++ .../cooperative-matrix-mixed-accumulator.wgsl | 21 +++++ ...cooperative-matrix-mixed-accumulator.metal | 47 +++++++++++ ...ooperative-matrix-mixed-accumulator.spvasm | 77 +++++++++++++++++++ ...-cooperative-matrix-mixed-accumulator.wgsl | 21 +++++ 7 files changed, 192 insertions(+), 8 deletions(-) create mode 100644 naga/tests/in/wgsl/cooperative-matrix-mixed-accumulator.toml create mode 100644 naga/tests/in/wgsl/cooperative-matrix-mixed-accumulator.wgsl create mode 100644 naga/tests/out/msl/wgsl-cooperative-matrix-mixed-accumulator.metal create mode 100644 naga/tests/out/spv/wgsl-cooperative-matrix-mixed-accumulator.spvasm create mode 100644 naga/tests/out/wgsl/wgsl-cooperative-matrix-mixed-accumulator.wgsl diff --git a/CHANGELOG.md b/CHANGELOG.md index 21406b964..2c39f64bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -171,6 +171,7 @@ By @beholdnec in [#8505](https://github.com/gfx-rs/wgpu/pull/8505). - spirv-out ray tracing pipelines. By @Vecvec in [#9085](https://github.com/gfx-rs/wgpu/pull/9085). - Add `spirv-out` ray tracing pipelines. By @Vecvec in [#9085](https://github.com/gfx-rs/wgpu/pull/9085). - Add `naga::front::wgsl::ParseError::notes()`. By @kwillemsen in [#9572](https://github.com/gfx-rs/wgpu/pull/9572). +- Add MSL support for cooperative matrix multiply-add with lower-precision A/B operands and a higher-precision accumulator/result, such as `coopMultiplyAdd(f16, f16, f32) -> f32`. By @seddonm1 in [#9629](https://github.com/gfx-rs/wgpu/pull/9629). ### Changes diff --git a/naga/src/back/msl/writer.rs b/naga/src/back/msl/writer.rs index 5b2fc2600..de03c76c0 100644 --- a/naga/src/back/msl/writer.rs +++ b/naga/src/back/msl/writer.rs @@ -514,7 +514,8 @@ pub(super) enum WrappedFunction { columns: crate::CooperativeSize, rows: crate::CooperativeSize, intermediate: crate::CooperativeSize, - scalar: crate::Scalar, + ab_scalar: crate::Scalar, + c_scalar: crate::Scalar, }, RayQueryGetIntersection { committed: bool, @@ -6453,9 +6454,10 @@ template space: crate::AddressSpace, a: Handle, b: Handle, + c: Handle, ) -> BackendResult { let space_name = space.to_msl_name().unwrap_or_default(); - let (a_c, a_r, scalar) = match *func_ctx.resolve_type(a, &module.types) { + let (a_c, a_r, ab_scalar) = match *func_ctx.resolve_type(a, &module.types) { crate::TypeInner::CooperativeMatrix { columns, rows, @@ -6468,26 +6470,32 @@ template crate::TypeInner::CooperativeMatrix { columns, rows, .. } => (columns, rows), _ => unreachable!(), }; + let c_scalar = match *func_ctx.resolve_type(c, &module.types) { + crate::TypeInner::CooperativeMatrix { scalar, .. } => scalar, + _ => unreachable!(), + }; let wrapped = WrappedFunction::CooperativeMultiplyAdd { space_name, columns: b_c, rows: a_r, intermediate: a_c, - scalar, + ab_scalar, + c_scalar, }; if !self.wrapped_functions.insert(wrapped) { return Ok(()); } - let scalar_name = scalar.to_msl_name(); + let ab_scalar_name = ab_scalar.to_msl_name(); + let c_scalar_name = c_scalar.to_msl_name(); writeln!( self.out, - "{NAMESPACE}::simdgroup_{scalar_name}{}x{} {COOPERATIVE_MULTIPLY_ADD_FUNCTION}(const {space_name} {NAMESPACE}::simdgroup_{scalar_name}{}x{}& a, const {space_name} {NAMESPACE}::simdgroup_{scalar_name}{}x{}& b, const {space_name} {NAMESPACE}::simdgroup_{scalar_name}{}x{}& c) {{", + "{NAMESPACE}::simdgroup_{c_scalar_name}{}x{} {COOPERATIVE_MULTIPLY_ADD_FUNCTION}(const {space_name} {NAMESPACE}::simdgroup_{ab_scalar_name}{}x{}& a, const {space_name} {NAMESPACE}::simdgroup_{ab_scalar_name}{}x{}& b, const {space_name} {NAMESPACE}::simdgroup_{c_scalar_name}{}x{}& c) {{", b_c as u32, a_r as u32, a_c as u32, a_r as u32, b_c as u32, b_r as u32, b_c as u32, a_r as u32, )?; let l1 = back::Level(1); writeln!( self.out, - "{l1}{NAMESPACE}::simdgroup_{scalar_name}{}x{} d;", + "{l1}{NAMESPACE}::simdgroup_{c_scalar_name}{}x{} d;", b_c as u32, a_r as u32 )?; writeln!(self.out, "{l1}simdgroup_multiply_accumulate(d,a,b,c);")?; @@ -6585,9 +6593,9 @@ template data.pointer, )?; } - crate::Expression::CooperativeMultiplyAdd { a, b, c: _ } => { + crate::Expression::CooperativeMultiplyAdd { a, b, c } => { let space = crate::AddressSpace::Private; - self.write_wrapped_cooperative_multiply_add(module, func_ctx, space, a, b)?; + self.write_wrapped_cooperative_multiply_add(module, func_ctx, space, a, b, c)?; } crate::Expression::RayQueryGetIntersection { committed, .. } => { self.write_rq_get_intersection_function(module, committed)?; diff --git a/naga/tests/in/wgsl/cooperative-matrix-mixed-accumulator.toml b/naga/tests/in/wgsl/cooperative-matrix-mixed-accumulator.toml new file mode 100644 index 000000000..56cda1270 --- /dev/null +++ b/naga/tests/in/wgsl/cooperative-matrix-mixed-accumulator.toml @@ -0,0 +1,9 @@ +targets = "SPIRV | METAL | WGSL" +capabilities = "COOPERATIVE_MATRIX | SHADER_FLOAT16" + +[spv] +debug = false +version = [1, 4] + +[msl] +lang_version = [2, 3] diff --git a/naga/tests/in/wgsl/cooperative-matrix-mixed-accumulator.wgsl b/naga/tests/in/wgsl/cooperative-matrix-mixed-accumulator.wgsl new file mode 100644 index 000000000..bf75fbe1a --- /dev/null +++ b/naga/tests/in/wgsl/cooperative-matrix-mixed-accumulator.wgsl @@ -0,0 +1,21 @@ +enable f16; +enable wgpu_cooperative_matrix; + +@group(0) @binding(0) +var ab: array; +@group(0) @binding(1) +var accum: array; + +@compute @workgroup_size(8, 8, 1) +fn main() { + // loading f16 multiplicands from memory + let a: coop_mat8x8 = coopLoad>(&ab[0]); + let b: coop_mat8x8 = coopLoad>(&ab[0]); + // loading f32 accumulator from memory + var c: coop_mat8x8 = coopLoad>(&accum[0]); + + // actual mixed-accumulator multiply-add + c = coopMultiplyAdd(a, b, c); + // storing f32 accumulator into memory + coopStore(c, &accum[0]); +} diff --git a/naga/tests/out/msl/wgsl-cooperative-matrix-mixed-accumulator.metal b/naga/tests/out/msl/wgsl-cooperative-matrix-mixed-accumulator.metal new file mode 100644 index 000000000..6544e7f68 --- /dev/null +++ b/naga/tests/out/msl/wgsl-cooperative-matrix-mixed-accumulator.metal @@ -0,0 +1,47 @@ +// language: metal2.3 +#include +#include + +using metal::uint; + +struct _mslBufferSizes { + uint size0; + uint size1; +}; + +typedef half type_1[1]; +typedef float type_3[1]; +metal::simdgroup_half8x8 NagaCooperativeLoad(const device half* ptr, int stride, bool is_row_major) { + metal::simdgroup_half8x8 m; + simdgroup_load(m, ptr, stride, 0, is_row_major); + return m; +} + +metal::simdgroup_float8x8 NagaCooperativeLoad(const device float* ptr, int stride, bool is_row_major) { + metal::simdgroup_float8x8 m; + simdgroup_load(m, ptr, stride, 0, is_row_major); + return m; +} + +metal::simdgroup_float8x8 NagaCooperativeMultiplyAdd(const thread metal::simdgroup_half8x8& a, const thread metal::simdgroup_half8x8& b, const thread metal::simdgroup_float8x8& c) { + metal::simdgroup_float8x8 d; + simdgroup_multiply_accumulate(d,a,b,c); + return d; +} + + +[[max_total_threads_per_threadgroup(64)]] kernel void main_( + device type_1 const& ab [[user(fake0)]] +, device type_3& accum [[user(fake0)]] +, constant _mslBufferSizes& _buffer_sizes [[user(fake0)]] +) { + metal::simdgroup_float8x8 c = {}; + metal::simdgroup_half8x8 a = NagaCooperativeLoad(&ab[0], 8u, true); + metal::simdgroup_half8x8 b = NagaCooperativeLoad(&ab[0], 8u, true); + c = NagaCooperativeLoad(&accum[0], 8u, true); + metal::simdgroup_float8x8 _e13 = c; + c = NagaCooperativeMultiplyAdd(a, b, _e13); + metal::simdgroup_float8x8 _e15 = c; + simdgroup_store(_e15, &accum[0], 8u, 0, true); + return; +} diff --git a/naga/tests/out/spv/wgsl-cooperative-matrix-mixed-accumulator.spvasm b/naga/tests/out/spv/wgsl-cooperative-matrix-mixed-accumulator.spvasm new file mode 100644 index 000000000..369ac62b3 --- /dev/null +++ b/naga/tests/out/spv/wgsl-cooperative-matrix-mixed-accumulator.spvasm @@ -0,0 +1,77 @@ +; SPIR-V +; Version: 1.4 +; Generator: rspirv +; Bound: 45 +OpCapability Shader +OpCapability Float16 +OpCapability StorageBuffer16BitAccess +OpCapability UniformAndStorageBuffer16BitAccess +OpCapability StorageInputOutput16 +OpCapability CooperativeMatrixKHR +OpCapability VulkanMemoryModel +OpExtension "SPV_KHR_16bit_storage" +OpExtension "SPV_KHR_cooperative_matrix" +OpExtension "SPV_KHR_vulkan_memory_model" +%1 = OpExtInstImport "GLSL.std.450" +OpMemoryModel Logical Vulkan +OpEntryPoint GLCompute %19 "main" %12 %15 +OpExecutionMode %19 LocalSize 8 8 1 +OpDecorate %4 ArrayStride 2 +OpDecorate %6 ArrayStride 4 +OpDecorate %12 DescriptorSet 0 +OpDecorate %12 Binding 0 +OpDecorate %13 Block +OpMemberDecorate %13 0 Offset 0 +OpDecorate %15 DescriptorSet 0 +OpDecorate %15 Binding 1 +OpDecorate %16 Block +OpMemberDecorate %16 0 Offset 0 +%2 = OpTypeVoid +%3 = OpTypeFloat 16 +%4 = OpTypeRuntimeArray %3 +%5 = OpTypeFloat 32 +%6 = OpTypeRuntimeArray %5 +%9 = OpTypeInt 32 0 +%8 = OpConstant %9 3 +%10 = OpConstant %9 8 +%11 = OpConstant %9 2 +%7 = OpTypeCooperativeMatrixKHR %5 %8 %10 %10 %11 +%13 = OpTypeStruct %4 +%14 = OpTypePointer StorageBuffer %13 +%12 = OpVariable %14 StorageBuffer +%16 = OpTypeStruct %6 +%17 = OpTypePointer StorageBuffer %16 +%15 = OpVariable %17 StorageBuffer +%20 = OpTypeFunction %2 +%21 = OpTypePointer StorageBuffer %4 +%22 = OpConstant %9 0 +%24 = OpTypePointer StorageBuffer %6 +%27 = OpTypePointer Function %7 +%28 = OpConstantNull %7 +%30 = OpTypePointer StorageBuffer %3 +%31 = OpTypeCooperativeMatrixKHR %3 %8 %10 %10 %22 +%32 = OpConstant %9 1 +%35 = OpTypeCooperativeMatrixKHR %3 %8 %10 %10 %32 +%38 = OpTypePointer StorageBuffer %5 +%19 = OpFunction %2 None %20 +%18 = OpLabel +%26 = OpVariable %27 Function %28 +%23 = OpAccessChain %21 %12 %22 +%25 = OpAccessChain %24 %15 %22 +OpBranch %29 +%29 = OpLabel +%33 = OpAccessChain %30 %23 %22 +%34 = OpCooperativeMatrixLoadKHR %31 %33 %32 %10 +%36 = OpAccessChain %30 %23 %22 +%37 = OpCooperativeMatrixLoadKHR %35 %36 %32 %10 +%39 = OpAccessChain %38 %25 %22 +%40 = OpCooperativeMatrixLoadKHR %7 %39 %32 %10 +OpStore %26 %40 +%41 = OpLoad %7 %26 +%42 = OpCooperativeMatrixMulAddKHR %7 %34 %37 %41 +OpStore %26 %42 +%43 = OpLoad %7 %26 +%44 = OpAccessChain %38 %25 %22 +OpCooperativeMatrixStoreKHR %44 %43 %32 %10 +OpReturn +OpFunctionEnd \ No newline at end of file diff --git a/naga/tests/out/wgsl/wgsl-cooperative-matrix-mixed-accumulator.wgsl b/naga/tests/out/wgsl/wgsl-cooperative-matrix-mixed-accumulator.wgsl new file mode 100644 index 000000000..44feabb6d --- /dev/null +++ b/naga/tests/out/wgsl/wgsl-cooperative-matrix-mixed-accumulator.wgsl @@ -0,0 +1,21 @@ +enable f16; +enable wgpu_cooperative_matrix; + +@group(0) @binding(0) +var ab: array; +@group(0) @binding(1) +var accum: array; + +@compute @workgroup_size(8, 8, 1) +fn main() { + var c: coop_mat8x8; + + let a = coopLoad>((&ab[0]), 8u); + let b = coopLoad>((&ab[0]), 8u); + c = coopLoad>((&accum[0]), 8u); + let _e13 = c; + c = coopMultiplyAdd(a, b, _e13); + let _e15 = c; + coopStore(_e15, (&accum[0]), 8u); + return; +}