mirror of
https://github.com/storytold/wgpu.git
synced 2026-11-21 23:11:01 +01:00
Add MSL support for mixed-precision cooperative matrix multiply-add (#9629)
This commit is contained in:
1 parent
0d734cb6de
commit
cab7d0b47f
7 files changed
+192
-8
No files matched your search
@@ -171,6 +171,7 @@ By @beholdnec in [#8505](https://github.com/gfx-rs/wgpu/pull/8505).
|
||||
- spirv-out ray tracing pipelines. By @Vecvec in [#9085](https://github.com/gfx-rs/wgpu/pull/9085).
|
||||
- Add `spirv-out` ray tracing pipelines. By @Vecvec in [#9085](https://github.com/gfx-rs/wgpu/pull/9085).
|
||||
- Add `naga::front::wgsl::ParseError::notes()`. By @kwillemsen in [#9572](https://github.com/gfx-rs/wgpu/pull/9572).
|
||||
- Add MSL support for cooperative matrix multiply-add with lower-precision A/B operands and a higher-precision accumulator/result, such as `coopMultiplyAdd(f16, f16, f32) -> f32`. By @seddonm1 in [#9629](https://github.com/gfx-rs/wgpu/pull/9629).
|
||||
|
||||
### Changes
|
||||
|
||||
|
||||
@@ -514,7 +514,8 @@ pub(super) enum WrappedFunction {
|
||||
columns: crate::CooperativeSize,
|
||||
rows: crate::CooperativeSize,
|
||||
intermediate: crate::CooperativeSize,
|
||||
scalar: crate::Scalar,
|
||||
ab_scalar: crate::Scalar,
|
||||
c_scalar: crate::Scalar,
|
||||
},
|
||||
RayQueryGetIntersection {
|
||||
committed: bool,
|
||||
@@ -6453,9 +6454,10 @@ template <typename A>
|
||||
space: crate::AddressSpace,
|
||||
a: Handle<crate::Expression>,
|
||||
b: Handle<crate::Expression>,
|
||||
c: Handle<crate::Expression>,
|
||||
) -> BackendResult {
|
||||
let space_name = space.to_msl_name().unwrap_or_default();
|
||||
let (a_c, a_r, scalar) = match *func_ctx.resolve_type(a, &module.types) {
|
||||
let (a_c, a_r, ab_scalar) = match *func_ctx.resolve_type(a, &module.types) {
|
||||
crate::TypeInner::CooperativeMatrix {
|
||||
columns,
|
||||
rows,
|
||||
@@ -6468,26 +6470,32 @@ template <typename A>
|
||||
crate::TypeInner::CooperativeMatrix { columns, rows, .. } => (columns, rows),
|
||||
_ => unreachable!(),
|
||||
};
|
||||
let c_scalar = match *func_ctx.resolve_type(c, &module.types) {
|
||||
crate::TypeInner::CooperativeMatrix { scalar, .. } => scalar,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
let wrapped = WrappedFunction::CooperativeMultiplyAdd {
|
||||
space_name,
|
||||
columns: b_c,
|
||||
rows: a_r,
|
||||
intermediate: a_c,
|
||||
scalar,
|
||||
ab_scalar,
|
||||
c_scalar,
|
||||
};
|
||||
if !self.wrapped_functions.insert(wrapped) {
|
||||
return Ok(());
|
||||
}
|
||||
let scalar_name = scalar.to_msl_name();
|
||||
let ab_scalar_name = ab_scalar.to_msl_name();
|
||||
let c_scalar_name = c_scalar.to_msl_name();
|
||||
writeln!(
|
||||
self.out,
|
||||
"{NAMESPACE}::simdgroup_{scalar_name}{}x{} {COOPERATIVE_MULTIPLY_ADD_FUNCTION}(const {space_name} {NAMESPACE}::simdgroup_{scalar_name}{}x{}& a, const {space_name} {NAMESPACE}::simdgroup_{scalar_name}{}x{}& b, const {space_name} {NAMESPACE}::simdgroup_{scalar_name}{}x{}& c) {{",
|
||||
"{NAMESPACE}::simdgroup_{c_scalar_name}{}x{} {COOPERATIVE_MULTIPLY_ADD_FUNCTION}(const {space_name} {NAMESPACE}::simdgroup_{ab_scalar_name}{}x{}& a, const {space_name} {NAMESPACE}::simdgroup_{ab_scalar_name}{}x{}& b, const {space_name} {NAMESPACE}::simdgroup_{c_scalar_name}{}x{}& c) {{",
|
||||
b_c as u32, a_r as u32, a_c as u32, a_r as u32, b_c as u32, b_r as u32, b_c as u32, a_r as u32,
|
||||
)?;
|
||||
let l1 = back::Level(1);
|
||||
writeln!(
|
||||
self.out,
|
||||
"{l1}{NAMESPACE}::simdgroup_{scalar_name}{}x{} d;",
|
||||
"{l1}{NAMESPACE}::simdgroup_{c_scalar_name}{}x{} d;",
|
||||
b_c as u32, a_r as u32
|
||||
)?;
|
||||
writeln!(self.out, "{l1}simdgroup_multiply_accumulate(d,a,b,c);")?;
|
||||
@@ -6585,9 +6593,9 @@ template <typename A>
|
||||
data.pointer,
|
||||
)?;
|
||||
}
|
||||
crate::Expression::CooperativeMultiplyAdd { a, b, c: _ } => {
|
||||
crate::Expression::CooperativeMultiplyAdd { a, b, c } => {
|
||||
let space = crate::AddressSpace::Private;
|
||||
self.write_wrapped_cooperative_multiply_add(module, func_ctx, space, a, b)?;
|
||||
self.write_wrapped_cooperative_multiply_add(module, func_ctx, space, a, b, c)?;
|
||||
}
|
||||
crate::Expression::RayQueryGetIntersection { committed, .. } => {
|
||||
self.write_rq_get_intersection_function(module, committed)?;
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
targets = "SPIRV | METAL | WGSL"
|
||||
capabilities = "COOPERATIVE_MATRIX | SHADER_FLOAT16"
|
||||
|
||||
[spv]
|
||||
debug = false
|
||||
version = [1, 4]
|
||||
|
||||
[msl]
|
||||
lang_version = [2, 3]
|
||||
@@ -0,0 +1,21 @@
|
||||
enable f16;
|
||||
enable wgpu_cooperative_matrix;
|
||||
|
||||
@group(0) @binding(0)
|
||||
var<storage, read_write> ab: array<f16>;
|
||||
@group(0) @binding(1)
|
||||
var<storage, read_write> accum: array<f32>;
|
||||
|
||||
@compute @workgroup_size(8, 8, 1)
|
||||
fn main() {
|
||||
// loading f16 multiplicands from memory
|
||||
let a: coop_mat8x8<f16, A> = coopLoad<coop_mat8x8<f16, A>>(&ab[0]);
|
||||
let b: coop_mat8x8<f16, B> = coopLoad<coop_mat8x8<f16, B>>(&ab[0]);
|
||||
// loading f32 accumulator from memory
|
||||
var c: coop_mat8x8<f32, C> = coopLoad<coop_mat8x8<f32, C>>(&accum[0]);
|
||||
|
||||
// actual mixed-accumulator multiply-add
|
||||
c = coopMultiplyAdd(a, b, c);
|
||||
// storing f32 accumulator into memory
|
||||
coopStore(c, &accum[0]);
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
// language: metal2.3
|
||||
#include <metal_stdlib>
|
||||
#include <simd/simd.h>
|
||||
|
||||
using metal::uint;
|
||||
|
||||
struct _mslBufferSizes {
|
||||
uint size0;
|
||||
uint size1;
|
||||
};
|
||||
|
||||
typedef half type_1[1];
|
||||
typedef float type_3[1];
|
||||
metal::simdgroup_half8x8 NagaCooperativeLoad(const device half* ptr, int stride, bool is_row_major) {
|
||||
metal::simdgroup_half8x8 m;
|
||||
simdgroup_load(m, ptr, stride, 0, is_row_major);
|
||||
return m;
|
||||
}
|
||||
|
||||
metal::simdgroup_float8x8 NagaCooperativeLoad(const device float* ptr, int stride, bool is_row_major) {
|
||||
metal::simdgroup_float8x8 m;
|
||||
simdgroup_load(m, ptr, stride, 0, is_row_major);
|
||||
return m;
|
||||
}
|
||||
|
||||
metal::simdgroup_float8x8 NagaCooperativeMultiplyAdd(const thread metal::simdgroup_half8x8& a, const thread metal::simdgroup_half8x8& b, const thread metal::simdgroup_float8x8& c) {
|
||||
metal::simdgroup_float8x8 d;
|
||||
simdgroup_multiply_accumulate(d,a,b,c);
|
||||
return d;
|
||||
}
|
||||
|
||||
|
||||
[[max_total_threads_per_threadgroup(64)]] kernel void main_(
|
||||
device type_1 const& ab [[user(fake0)]]
|
||||
, device type_3& accum [[user(fake0)]]
|
||||
, constant _mslBufferSizes& _buffer_sizes [[user(fake0)]]
|
||||
) {
|
||||
metal::simdgroup_float8x8 c = {};
|
||||
metal::simdgroup_half8x8 a = NagaCooperativeLoad(&ab[0], 8u, true);
|
||||
metal::simdgroup_half8x8 b = NagaCooperativeLoad(&ab[0], 8u, true);
|
||||
c = NagaCooperativeLoad(&accum[0], 8u, true);
|
||||
metal::simdgroup_float8x8 _e13 = c;
|
||||
c = NagaCooperativeMultiplyAdd(a, b, _e13);
|
||||
metal::simdgroup_float8x8 _e15 = c;
|
||||
simdgroup_store(_e15, &accum[0], 8u, 0, true);
|
||||
return;
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
; SPIR-V
|
||||
; Version: 1.4
|
||||
; Generator: rspirv
|
||||
; Bound: 45
|
||||
OpCapability Shader
|
||||
OpCapability Float16
|
||||
OpCapability StorageBuffer16BitAccess
|
||||
OpCapability UniformAndStorageBuffer16BitAccess
|
||||
OpCapability StorageInputOutput16
|
||||
OpCapability CooperativeMatrixKHR
|
||||
OpCapability VulkanMemoryModel
|
||||
OpExtension "SPV_KHR_16bit_storage"
|
||||
OpExtension "SPV_KHR_cooperative_matrix"
|
||||
OpExtension "SPV_KHR_vulkan_memory_model"
|
||||
%1 = OpExtInstImport "GLSL.std.450"
|
||||
OpMemoryModel Logical Vulkan
|
||||
OpEntryPoint GLCompute %19 "main" %12 %15
|
||||
OpExecutionMode %19 LocalSize 8 8 1
|
||||
OpDecorate %4 ArrayStride 2
|
||||
OpDecorate %6 ArrayStride 4
|
||||
OpDecorate %12 DescriptorSet 0
|
||||
OpDecorate %12 Binding 0
|
||||
OpDecorate %13 Block
|
||||
OpMemberDecorate %13 0 Offset 0
|
||||
OpDecorate %15 DescriptorSet 0
|
||||
OpDecorate %15 Binding 1
|
||||
OpDecorate %16 Block
|
||||
OpMemberDecorate %16 0 Offset 0
|
||||
%2 = OpTypeVoid
|
||||
%3 = OpTypeFloat 16
|
||||
%4 = OpTypeRuntimeArray %3
|
||||
%5 = OpTypeFloat 32
|
||||
%6 = OpTypeRuntimeArray %5
|
||||
%9 = OpTypeInt 32 0
|
||||
%8 = OpConstant %9 3
|
||||
%10 = OpConstant %9 8
|
||||
%11 = OpConstant %9 2
|
||||
%7 = OpTypeCooperativeMatrixKHR %5 %8 %10 %10 %11
|
||||
%13 = OpTypeStruct %4
|
||||
%14 = OpTypePointer StorageBuffer %13
|
||||
%12 = OpVariable %14 StorageBuffer
|
||||
%16 = OpTypeStruct %6
|
||||
%17 = OpTypePointer StorageBuffer %16
|
||||
%15 = OpVariable %17 StorageBuffer
|
||||
%20 = OpTypeFunction %2
|
||||
%21 = OpTypePointer StorageBuffer %4
|
||||
%22 = OpConstant %9 0
|
||||
%24 = OpTypePointer StorageBuffer %6
|
||||
%27 = OpTypePointer Function %7
|
||||
%28 = OpConstantNull %7
|
||||
%30 = OpTypePointer StorageBuffer %3
|
||||
%31 = OpTypeCooperativeMatrixKHR %3 %8 %10 %10 %22
|
||||
%32 = OpConstant %9 1
|
||||
%35 = OpTypeCooperativeMatrixKHR %3 %8 %10 %10 %32
|
||||
%38 = OpTypePointer StorageBuffer %5
|
||||
%19 = OpFunction %2 None %20
|
||||
%18 = OpLabel
|
||||
%26 = OpVariable %27 Function %28
|
||||
%23 = OpAccessChain %21 %12 %22
|
||||
%25 = OpAccessChain %24 %15 %22
|
||||
OpBranch %29
|
||||
%29 = OpLabel
|
||||
%33 = OpAccessChain %30 %23 %22
|
||||
%34 = OpCooperativeMatrixLoadKHR %31 %33 %32 %10
|
||||
%36 = OpAccessChain %30 %23 %22
|
||||
%37 = OpCooperativeMatrixLoadKHR %35 %36 %32 %10
|
||||
%39 = OpAccessChain %38 %25 %22
|
||||
%40 = OpCooperativeMatrixLoadKHR %7 %39 %32 %10
|
||||
OpStore %26 %40
|
||||
%41 = OpLoad %7 %26
|
||||
%42 = OpCooperativeMatrixMulAddKHR %7 %34 %37 %41
|
||||
OpStore %26 %42
|
||||
%43 = OpLoad %7 %26
|
||||
%44 = OpAccessChain %38 %25 %22
|
||||
OpCooperativeMatrixStoreKHR %44 %43 %32 %10
|
||||
OpReturn
|
||||
OpFunctionEnd
|
||||
@@ -0,0 +1,21 @@
|
||||
enable f16;
|
||||
enable wgpu_cooperative_matrix;
|
||||
|
||||
@group(0) @binding(0)
|
||||
var<storage, read_write> ab: array<f16>;
|
||||
@group(0) @binding(1)
|
||||
var<storage, read_write> accum: array<f32>;
|
||||
|
||||
@compute @workgroup_size(8, 8, 1)
|
||||
fn main() {
|
||||
var c: coop_mat8x8<f32,C>;
|
||||
|
||||
let a = coopLoad<coop_mat8x8<f16,A>>((&ab[0]), 8u);
|
||||
let b = coopLoad<coop_mat8x8<f16,B>>((&ab[0]), 8u);
|
||||
c = coopLoad<coop_mat8x8<f32,C>>((&accum[0]), 8u);
|
||||
let _e13 = c;
|
||||
c = coopMultiplyAdd(a, b, _e13);
|
||||
let _e15 = c;
|
||||
coopStore(_e15, (&accum[0]), 8u);
|
||||
return;
|
||||
}
|
||||
Reference in new issue
Block a user