From 007f23ed3b3ee5f9df33496a05131fe538dc8f93 Mon Sep 17 00:00:00 2001 From: keptsecret Date: Tue, 14 Jul 2026 16:57:59 +0700 Subject: [PATCH 01/21] chained scan initial commit w rough outline --- .../nbl/builtin/hlsl/scan/chained_scan.hlsl | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 include/nbl/builtin/hlsl/scan/chained_scan.hlsl diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl new file mode 100644 index 0000000000..b24278fdb8 --- /dev/null +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -0,0 +1,119 @@ +#ifndef _NBL_HLSL_SCAN_CHAINED_SCAN_INCLUDED_ +#define _NBL_HLSL_SCAN_CHAINED_SCAN_INCLUDED_ + +#include "nbl/builtin/hlsl/workgroup2/shared_scan.hlsl" + +namespace nbl +{ +namespace hlsl +{ +namespace scan +{ + +namespace impl +{ +template +struct scan +{ + using scalar_t = typename BinOp::type_t; + using vector_t = vector; // data accessor needs to be this type + + NBL_CONSTEXPR_STATIC_INLINE uint32_t Flag_NotReady = 0; + NBL_CONSTEXPR_STATIC_INLINE uint32_t Flag_Reduction = 1; // workgroup only has local reduction ready + NBL_CONSTEXPR_STATIC_INLINE uint32_t Flag_Inclusive = 2; // workgroup has summed all preceding groups and added to own sum + NBL_CONSTEXPR_STATIC_INLINE uint32_t Flag_Mask = 3; + + NBL_CONSTEXPR_STATIC_INLINE uint16_t Flag_Shift = 2; + + // TODO: I think we need separate DataAccessor classes between device-wide and workgroup scan + + // TODO: double check op, it's all plus right now + + template + void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(DataAccessor) workgroupReduction) // TODO: need different type for workgroupReduction + { + // TODO: what to do with config? + + const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); + const uint16_t workgroupId = glsl::gl_WorkGroupID(); + + // TODO: workgroup scan here, don't want to deal with that now + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + const scalar_t currGroupReduction = // TODO get last elem of curr workgroup; + if (!invocIx) + { + const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, workgroupId > 0u) | currGroupReduction << Flag_Shift; + workgroupReduction.atomicExchange(workgroupId, storeVal); + } + + // lookback + if (workgroupId && !invocIx) + { + scalar_t prevReduction = 0u; + uint16_t lookbackIx = workgroupId - uint16_t(1u); + + while (true) // TODO: check if lookbackIx < 0? + { + scalar_t flagPayload; + workgroupReduction.get(lookbackIx, flagPayload); + + if ((flagPayload & Flag_Mask) > Flag_NotReady) + { + prevReduction += flagPayload >> Flag_Shift; + if ((flagPayload & Flag_Mask) == Flag_Inclusive) + { + const scalar_t groupReduction = // TODO get last elem of workgroup index lookbackIx; + const scalar_t storeVal = Flag_Inclusive | ((prevReduction + groupReduction) << Flag_Shift); + workgroupReduction.atomicExchange(workgroupId, storeVal); + scratchAccessor.set(0u, prevReduction); + break; + } + else + lookbackIx--; + } + else + lookbackIx--; + } + } + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + scalar_t prevReduction; + scratchAccessor.get(0u, prevReduction); + prevReduction += currGroupReduction; + + // TODO: propagate correctly when data accessor done + dataAccessor[invocIx] += prevReduction; + } +}; +} + +template) +struct inclusive_scan +{ + using scalar_t = typename BinOp::type_t; + + template && ArithmeticSharedMemoryAccessor) + static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor) + { + impl::scan fn; + fn.template __call(dataAccessor, scratchAccessor); + } +}; + +template) +struct exclusive_scan +{ + using scalar_t = typename BinOp::type_t; + + template && ArithmeticSharedMemoryAccessor) + static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor) + { + } +}; + +} +} +} + +#endif From 9604ca6005438f9d45b5a22db24474814051d069 Mon Sep 17 00:00:00 2001 From: keptsecret Date: Wed, 15 Jul 2026 16:21:13 +0700 Subject: [PATCH 02/21] implement proper read/write with separate data accessor for workgroup --- .../nbl/builtin/hlsl/scan/chained_scan.hlsl | 104 +++++++++++++++--- 1 file changed, 90 insertions(+), 14 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index b24278fdb8..3903816c17 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -12,8 +12,60 @@ namespace scan namespace impl { -template -struct scan +template +struct WorkgroupDataProxy +{ + using dtype_t = vector; + + NBL_CONSTEXPR_STATIC_INLINE uint16_t WorkgroupSize = uint16_t(1u) << WorkgroupSizeLog2; + NBL_CONSTEXPR_STATIC_INLINE uint16_t PreloadedDataCount = VirtualWorkgroupSize / WorkgroupSize; + + static WorkgroupDataProxy create(const uint64_t inputBuf, const uint64_t outputBuf) + { + WorkgroupDataProxy retval; + const uint32_t workgroupOffset = glsl::gl_WorkGroupID().x * VirtualWorkgroupSize * sizeof(dtype_t); + retval.accessor = DoubleLegacyBdaAccessor::create(inputBuf + workgroupOffset, outputBuf + workgroupOffset); + return retval; + } + + template + void get(const IndexType ix, NBL_REF_ARG(AccessType) value) + { + value = preloaded[ix>>WorkgroupSizeLog2]; + } + template + void set(const IndexType ix, const AccessType value) + { + preloaded[ix>>WorkgroupSizeLog2] = value; + } + + void preload() + { + const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); + NBL_UNROLL + for (uint16_t idx = 0; idx < PreloadedDataCount; idx++) + accessor.get(idx * WorkgroupSize + invocIx, preloaded[idx]); + } + void unload() + { + const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); + NBL_UNROLL + for (uint16_t idx = 0; idx < PreloadedDataCount; idx++) + accessor.set(idx * WorkgroupSize + invocIx, preloaded[idx]); + } + + void workgroupExecutionAndMemoryBarrier() + { + glsl::barrier(); + //glsl::memoryBarrierShared(); implied by the above + } + + DoubleLegacyBdaAccessor accessor; + dtype_t preloaded[PreloadedDataCount]; +}; + +template // TODO: Config is same as workgroup2 stuff? +struct Scan { using scalar_t = typename BinOp::type_t; using vector_t = vector; // data accessor needs to be this type @@ -25,22 +77,38 @@ struct scan NBL_CONSTEXPR_STATIC_INLINE uint16_t Flag_Shift = 2; - // TODO: I think we need separate DataAccessor classes between device-wide and workgroup scan + NBL_CONSTEXPR_STATIC_INLINE uint16_t WorkgroupSize = uint16_t(1u) << Config::WorkgroupSizeLog2; + NBL_CONSTEXPR_STATIC_INLINE uint16_t ItemsPerInvoc = Config::VirtualWorkgroupSize / WorkgroupSize; // TODO: double check op, it's all plus right now template void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(DataAccessor) workgroupReduction) // TODO: need different type for workgroupReduction - { - // TODO: what to do with config? - + { const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); const uint16_t workgroupId = glsl::gl_WorkGroupID(); - // TODO: workgroup scan here, don't want to deal with that now + scalar_t currGroupReduction; + { + using data_proxy_t = WorkgroupDataProxy; + data_proxy_t wgDataAccessor = data_proxy_t::create(pc.pInputBuf, pc.pOutputBuf[Binop::BindingIndex]); + wgDataAccessor.preload(); + + if (Exclusive) + workgroup2::exclusive_scan::template __call(wgDataAccessor, scratchAccessor); + else + workgroup2::inclusive_scan::template __call(wgDataAccessor, scratchAccessor); + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + wgDataAccessor.unload(); // TODO: maybe we don't have to unload, just write once after everything is done + + // TODO: double check this but it should be the last element of the last workgroup thread + // don't know what it's like if virtual workgroup size doesn't divide by workgroup size exactly + if (invocIx == glsl::gl_SubgroupSize() * glsl::gl_NumSubgroups() - 1u) + currGroupReduction = wgDataAccessor.preloaded[data_proxy_t::PreloadedDataCount-1u][config_t::ItemsPerInvocation_0-1u]; + } scratchAccessor.workgroupExecutionAndMemoryBarrier(); - const scalar_t currGroupReduction = // TODO get last elem of curr workgroup; if (!invocIx) { const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, workgroupId > 0u) | currGroupReduction << Flag_Shift; @@ -82,8 +150,14 @@ struct scan scratchAccessor.get(0u, prevReduction); prevReduction += currGroupReduction; - // TODO: propagate correctly when data accessor done - dataAccessor[invocIx] += prevReduction; + NBL_UNROLL + for (uint16_t idx = 0; idx < ItemsPerInvoc; idx++) + { + dtype_t data; + dataAccessor.template get(idx * WorkgroupSize + invocIx, data); + data += prevReduction; + dataAccessor.template set(idx * WorkgroupSize + invocIx, data); + } } }; } @@ -94,10 +168,10 @@ struct inclusive_scan using scalar_t = typename BinOp::type_t; template && ArithmeticSharedMemoryAccessor) - static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor) + static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(DataAccessor) workgroupReduction) { - impl::scan fn; - fn.template __call(dataAccessor, scratchAccessor); + impl::Scan fn; + fn.template __call(dataAccessor, scratchAccessor, workgroupReduction); } }; @@ -107,8 +181,10 @@ struct exclusive_scan using scalar_t = typename BinOp::type_t; template && ArithmeticSharedMemoryAccessor) - static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor) + static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(DataAccessor) workgroupReduction) { + impl::Scan fn; + fn.template __call(dataAccessor, scratchAccessor, workgroupReduction); } }; From 41928263948fb70097c534b09d78995fd3467802 Mon Sep 17 00:00:00 2001 From: keptsecret Date: Wed, 15 Jul 2026 16:45:34 +0700 Subject: [PATCH 03/21] use correct binop --- include/nbl/builtin/hlsl/scan/chained_scan.hlsl | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index 3903816c17..5fbf7aef07 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -69,6 +69,7 @@ struct Scan { using scalar_t = typename BinOp::type_t; using vector_t = vector; // data accessor needs to be this type + using binop_t = BinOp; NBL_CONSTEXPR_STATIC_INLINE uint32_t Flag_NotReady = 0; NBL_CONSTEXPR_STATIC_INLINE uint32_t Flag_Reduction = 1; // workgroup only has local reduction ready @@ -79,14 +80,13 @@ struct Scan NBL_CONSTEXPR_STATIC_INLINE uint16_t WorkgroupSize = uint16_t(1u) << Config::WorkgroupSizeLog2; NBL_CONSTEXPR_STATIC_INLINE uint16_t ItemsPerInvoc = Config::VirtualWorkgroupSize / WorkgroupSize; - - // TODO: double check op, it's all plus right now template void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(DataAccessor) workgroupReduction) // TODO: need different type for workgroupReduction { const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); const uint16_t workgroupId = glsl::gl_WorkGroupID(); + binop_t binop; scalar_t currGroupReduction; { @@ -105,7 +105,7 @@ struct Scan // TODO: double check this but it should be the last element of the last workgroup thread // don't know what it's like if virtual workgroup size doesn't divide by workgroup size exactly if (invocIx == glsl::gl_SubgroupSize() * glsl::gl_NumSubgroups() - 1u) - currGroupReduction = wgDataAccessor.preloaded[data_proxy_t::PreloadedDataCount-1u][config_t::ItemsPerInvocation_0-1u]; + currGroupReduction = wgDataAccessor.preloaded[data_proxy_t::PreloadedDataCount-1u][Config::ItemsPerInvocation_0-1u]; } scratchAccessor.workgroupExecutionAndMemoryBarrier(); @@ -128,11 +128,10 @@ struct Scan if ((flagPayload & Flag_Mask) > Flag_NotReady) { - prevReduction += flagPayload >> Flag_Shift; + prevReduction = binop(prevReduction, flagPayload >> Flag_Shift); if ((flagPayload & Flag_Mask) == Flag_Inclusive) { - const scalar_t groupReduction = // TODO get last elem of workgroup index lookbackIx; - const scalar_t storeVal = Flag_Inclusive | ((prevReduction + groupReduction) << Flag_Shift); + const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); workgroupReduction.atomicExchange(workgroupId, storeVal); scratchAccessor.set(0u, prevReduction); break; @@ -148,14 +147,16 @@ struct Scan scalar_t prevReduction; scratchAccessor.get(0u, prevReduction); - prevReduction += currGroupReduction; + prevReduction = binop(prevReduction, currGroupReduction); NBL_UNROLL for (uint16_t idx = 0; idx < ItemsPerInvoc; idx++) { dtype_t data; dataAccessor.template get(idx * WorkgroupSize + invocIx, data); - data += prevReduction; + NBL_UNROLL + for (uint16_t i = 0; i < Config::ItemsPerInvocation_0; i++) + data[i] = binop(prevReduction, data[i]); dataAccessor.template set(idx * WorkgroupSize + invocIx, data); } } From b132a2ee67dbd2d0858b3a1c4a675324ac7b3eec Mon Sep 17 00:00:00 2001 From: keptsecret Date: Thu, 16 Jul 2026 10:59:46 +0700 Subject: [PATCH 04/21] minor fix to creating wg accessor --- include/nbl/builtin/hlsl/scan/chained_scan.hlsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index 5fbf7aef07..46c5081f7b 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -91,7 +91,7 @@ struct Scan scalar_t currGroupReduction; { using data_proxy_t = WorkgroupDataProxy; - data_proxy_t wgDataAccessor = data_proxy_t::create(pc.pInputBuf, pc.pOutputBuf[Binop::BindingIndex]); + data_proxy_t wgDataAccessor = data_proxy_t::create(dataAccessor.getInputBufAddr(), dataAccessor.getOutputBufAddr()); wgDataAccessor.preload(); if (Exclusive) From b46cad2296c029ff4f296281b24aa06111a759c4 Mon Sep 17 00:00:00 2001 From: keptsecret Date: Thu, 16 Jul 2026 16:51:31 +0700 Subject: [PATCH 05/21] use a separate accessor type for reduction buf --- include/nbl/builtin/hlsl/scan/chained_scan.hlsl | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index 46c5081f7b..2e7e24e740 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -81,8 +81,8 @@ struct Scan NBL_CONSTEXPR_STATIC_INLINE uint16_t WorkgroupSize = uint16_t(1u) << Config::WorkgroupSizeLog2; NBL_CONSTEXPR_STATIC_INLINE uint16_t ItemsPerInvoc = Config::VirtualWorkgroupSize / WorkgroupSize; - template - void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(DataAccessor) workgroupReduction) // TODO: need different type for workgroupReduction + template + void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction) { const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); const uint16_t workgroupId = glsl::gl_WorkGroupID(); @@ -168,8 +168,9 @@ struct inclusive_scan { using scalar_t = typename BinOp::type_t; - template && ArithmeticSharedMemoryAccessor) - static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(DataAccessor) workgroupReduction) + // TODO: might want new concept for ReductionAccessor + template && ArithmeticSharedMemoryAccessor) + static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction) { impl::Scan fn; fn.template __call(dataAccessor, scratchAccessor, workgroupReduction); @@ -181,8 +182,8 @@ struct exclusive_scan { using scalar_t = typename BinOp::type_t; - template && ArithmeticSharedMemoryAccessor) - static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(DataAccessor) workgroupReduction) + template && ArithmeticSharedMemoryAccessor) + static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction) { impl::Scan fn; fn.template __call(dataAccessor, scratchAccessor, workgroupReduction); From 59219c5df5e7d275d1037f4b736266c60f673e1a Mon Sep 17 00:00:00 2001 From: keptsecret Date: Fri, 17 Jul 2026 16:46:29 +0700 Subject: [PATCH 06/21] fix some compilation bugs, still not running correct --- .../nbl/builtin/hlsl/scan/chained_scan.hlsl | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index 2e7e24e740..06523822ac 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -1,7 +1,7 @@ #ifndef _NBL_HLSL_SCAN_CHAINED_SCAN_INCLUDED_ #define _NBL_HLSL_SCAN_CHAINED_SCAN_INCLUDED_ -#include "nbl/builtin/hlsl/workgroup2/shared_scan.hlsl" +#include "nbl/builtin/hlsl/workgroup2/arithmetic.hlsl" namespace nbl { @@ -85,7 +85,7 @@ struct Scan void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction) { const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); - const uint16_t workgroupId = glsl::gl_WorkGroupID(); + const uint16_t workgroupId = uint16_t(glsl::gl_WorkGroupID().x); binop_t binop; scalar_t currGroupReduction; @@ -133,7 +133,7 @@ struct Scan { const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); workgroupReduction.atomicExchange(workgroupId, storeVal); - scratchAccessor.set(0u, prevReduction); + scratchAccessor.template set(0u, prevReduction); break; } else @@ -146,30 +146,30 @@ struct Scan scratchAccessor.workgroupExecutionAndMemoryBarrier(); scalar_t prevReduction; - scratchAccessor.get(0u, prevReduction); + scratchAccessor.template get(0u, prevReduction); prevReduction = binop(prevReduction, currGroupReduction); NBL_UNROLL for (uint16_t idx = 0; idx < ItemsPerInvoc; idx++) { - dtype_t data; - dataAccessor.template get(idx * WorkgroupSize + invocIx, data); + vector_t data; + dataAccessor.template get(idx * WorkgroupSize + invocIx, data); NBL_UNROLL for (uint16_t i = 0; i < Config::ItemsPerInvocation_0; i++) data[i] = binop(prevReduction, data[i]); - dataAccessor.template set(idx * WorkgroupSize + invocIx, data); + dataAccessor.template set(idx * WorkgroupSize + invocIx, data); } } }; } -template) +template) struct inclusive_scan { using scalar_t = typename BinOp::type_t; // TODO: might want new concept for ReductionAccessor - template && ArithmeticSharedMemoryAccessor) + template && workgroup2::ArithmeticSharedMemoryAccessor) static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction) { impl::Scan fn; @@ -177,12 +177,12 @@ struct inclusive_scan } }; -template) +template) struct exclusive_scan { using scalar_t = typename BinOp::type_t; - template && ArithmeticSharedMemoryAccessor) + template && workgroup2::ArithmeticSharedMemoryAccessor) static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction) { impl::Scan fn; From a056aeba3a5c4549d928186676e23a130071757b Mon Sep 17 00:00:00 2001 From: keptsecret Date: Tue, 21 Jul 2026 12:05:22 +0700 Subject: [PATCH 07/21] fix some bugs in chained scan, still need to add lookback fallback --- .../nbl/builtin/hlsl/scan/chained_scan.hlsl | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index 06523822ac..a303825d03 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -88,28 +88,28 @@ struct Scan const uint16_t workgroupId = uint16_t(glsl::gl_WorkGroupID().x); binop_t binop; + using wg_data_proxy_t = WorkgroupDataProxy; + wg_data_proxy_t wgDataAccessor = wg_data_proxy_t::create(dataAccessor.getInputBufAddr(), dataAccessor.getOutputBufAddr()); scalar_t currGroupReduction; { - using data_proxy_t = WorkgroupDataProxy; - data_proxy_t wgDataAccessor = data_proxy_t::create(dataAccessor.getInputBufAddr(), dataAccessor.getOutputBufAddr()); wgDataAccessor.preload(); - if (Exclusive) - workgroup2::exclusive_scan::template __call(wgDataAccessor, scratchAccessor); + NBL_IF_CONSTEXPR(Exclusive) + workgroup2::exclusive_scan::template __call(wgDataAccessor, scratchAccessor); else - workgroup2::inclusive_scan::template __call(wgDataAccessor, scratchAccessor); + workgroup2::inclusive_scan::template __call(wgDataAccessor, scratchAccessor); scratchAccessor.workgroupExecutionAndMemoryBarrier(); - wgDataAccessor.unload(); // TODO: maybe we don't have to unload, just write once after everything is done + // wgDataAccessor.unload(); // TODO: maybe we don't have to unload, just write once after everything is done // TODO: double check this but it should be the last element of the last workgroup thread // don't know what it's like if virtual workgroup size doesn't divide by workgroup size exactly if (invocIx == glsl::gl_SubgroupSize() * glsl::gl_NumSubgroups() - 1u) - currGroupReduction = wgDataAccessor.preloaded[data_proxy_t::PreloadedDataCount-1u][Config::ItemsPerInvocation_0-1u]; + currGroupReduction = wgDataAccessor.preloaded[wg_data_proxy_t::PreloadedDataCount-1u][Config::ItemsPerInvocation_0-1u]; } scratchAccessor.workgroupExecutionAndMemoryBarrier(); - if (!invocIx) + if (invocIx == glsl::gl_SubgroupSize() * glsl::gl_NumSubgroups() - 1u) { const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, workgroupId > 0u) | currGroupReduction << Flag_Shift; workgroupReduction.atomicExchange(workgroupId, storeVal); @@ -121,7 +121,7 @@ struct Scan scalar_t prevReduction = 0u; uint16_t lookbackIx = workgroupId - uint16_t(1u); - while (true) // TODO: check if lookbackIx < 0? + while (lookbackIx > 0) // TODO: check if lookbackIx < 0? { scalar_t flagPayload; workgroupReduction.get(lookbackIx, flagPayload); @@ -139,8 +139,8 @@ struct Scan else lookbackIx--; } - else - lookbackIx--; + // else + // lookbackIx--; } } scratchAccessor.workgroupExecutionAndMemoryBarrier(); @@ -150,9 +150,9 @@ struct Scan prevReduction = binop(prevReduction, currGroupReduction); NBL_UNROLL - for (uint16_t idx = 0; idx < ItemsPerInvoc; idx++) + for (uint16_t idx = 0; idx < wg_data_proxy_t::PreloadedDataCount; idx++) { - vector_t data; + vector_t data = wgDataAccessor.preloaded[idx]; dataAccessor.template get(idx * WorkgroupSize + invocIx, data); NBL_UNROLL for (uint16_t i = 0; i < Config::ItemsPerInvocation_0; i++) From 12b2d9bc5530194a8b6db8eb1241513fac4570e4 Mon Sep 17 00:00:00 2001 From: keptsecret Date: Tue, 21 Jul 2026 16:51:34 +0700 Subject: [PATCH 08/21] lookback fallback to avoid spin --- .../nbl/builtin/hlsl/scan/chained_scan.hlsl | 90 +++++++++++++++---- 1 file changed, 72 insertions(+), 18 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index a303825d03..1b824ffcd9 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -3,6 +3,8 @@ #include "nbl/builtin/hlsl/workgroup2/arithmetic.hlsl" +groupshared bool sIsLocked; + namespace nbl { namespace hlsl @@ -20,10 +22,10 @@ struct WorkgroupDataProxy NBL_CONSTEXPR_STATIC_INLINE uint16_t WorkgroupSize = uint16_t(1u) << WorkgroupSizeLog2; NBL_CONSTEXPR_STATIC_INLINE uint16_t PreloadedDataCount = VirtualWorkgroupSize / WorkgroupSize; - static WorkgroupDataProxy create(const uint64_t inputBuf, const uint64_t outputBuf) + static WorkgroupDataProxy create(const uint64_t inputBuf, const uint64_t outputBuf, const uint16_t workgroupId) { WorkgroupDataProxy retval; - const uint32_t workgroupOffset = glsl::gl_WorkGroupID().x * VirtualWorkgroupSize * sizeof(dtype_t); + const uint32_t workgroupOffset = workgroupId * VirtualWorkgroupSize * sizeof(dtype_t); retval.accessor = DoubleLegacyBdaAccessor::create(inputBuf + workgroupOffset, outputBuf + workgroupOffset); return retval; } @@ -88,8 +90,11 @@ struct Scan const uint16_t workgroupId = uint16_t(glsl::gl_WorkGroupID().x); binop_t binop; + if (!invocIx) + sIsLocked = true; + using wg_data_proxy_t = WorkgroupDataProxy; - wg_data_proxy_t wgDataAccessor = wg_data_proxy_t::create(dataAccessor.getInputBufAddr(), dataAccessor.getOutputBufAddr()); + wg_data_proxy_t wgDataAccessor = wg_data_proxy_t::create(dataAccessor.getInputBufAddr(), dataAccessor.getOutputBufAddr(), workgroupId); scalar_t currGroupReduction; { wgDataAccessor.preload(); @@ -116,31 +121,80 @@ struct Scan } // lookback - if (workgroupId && !invocIx) + if (workgroupId > 0) { + bool locked = sIsLocked; + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + scalar_t prevReduction = 0u; uint16_t lookbackIx = workgroupId - uint16_t(1u); - while (lookbackIx > 0) // TODO: check if lookbackIx < 0? + while (locked) { - scalar_t flagPayload; - workgroupReduction.get(lookbackIx, flagPayload); + // lookback: try to get reduction from previous workgroups + if (!invocIx) + { + scalar_t flagPayload; + workgroupReduction.get(lookbackIx, flagPayload); - if ((flagPayload & Flag_Mask) > Flag_NotReady) + if ((flagPayload & Flag_Mask) > Flag_NotReady) + { + prevReduction = binop(prevReduction, flagPayload >> Flag_Shift); + if ((flagPayload & Flag_Mask) == Flag_Inclusive) + { + const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); + workgroupReduction.atomicExchange(workgroupId, storeVal); + scratchAccessor.template set(0u, prevReduction); + sIsLocked = false; + break; + } + else + lookbackIx--; + } + + // lookback failed + // broadcast id and prepare to do reduction ourselves + scratchAccessor.template set(1u, lookbackIx); + } + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + locked = sIsLocked; + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + if (locked) { - prevReduction = binop(prevReduction, flagPayload >> Flag_Shift); - if ((flagPayload & Flag_Mask) == Flag_Inclusive) + // do reduction for lookbackIx workgroup + uint32_t fallbackGroupId; + scratchAccessor.template get(1u, fallbackGroupId); + + wg_data_proxy_t fallbackDataAccessor = wg_data_proxy_t::create(dataAccessor.getInputBufAddr(), dataAccessor.getOutputBufAddr(), fallbackGroupId); + fallbackDataAccessor.preload(); + scalar_t fallbackReduction = workgroup2::reduction::template __call(fallbackDataAccessor, scratchAccessor); + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + if (!invocIx) { - const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); - workgroupReduction.atomicExchange(workgroupId, storeVal); - scratchAccessor.template set(0u, prevReduction); - break; + scalar_t fallbackPayload; + const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, fallbackGroupId > 0u) | (fallbackReduction << Flag_Shift); + workgroupReduction.atomicMax(workgroupId, storeVal); + + prevReduction = binop(prevReduction, hlsl::max(fallbackReduction, fallbackPayload) >> Flag_Shift); + if (!fallbackGroupId || (fallbackPayload & Flag_Mask) == Flag_Inclusive) + { + const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); + workgroupReduction.atomicExchange(workgroupId, storeVal); + scratchAccessor.template set(0u, prevReduction); + sIsLocked = false; + } + else + { + lookbackIndex--; + } } - else - lookbackIx--; + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + locked = sIsLocked; + scratchAccessor.workgroupExecutionAndMemoryBarrier(); } - // else - // lookbackIx--; } } scratchAccessor.workgroupExecutionAndMemoryBarrier(); From 068776e951a198180a08b8041e64d292ff047399 Mon Sep 17 00:00:00 2001 From: keptsecret Date: Wed, 22 Jul 2026 10:54:47 +0700 Subject: [PATCH 09/21] implement atomic min and max --- .../nbl/builtin/hlsl/glsl_compat/core.hlsl | 104 ++++++++++++++++-- 1 file changed, 94 insertions(+), 10 deletions(-) diff --git a/include/nbl/builtin/hlsl/glsl_compat/core.hlsl b/include/nbl/builtin/hlsl/glsl_compat/core.hlsl index c2450e57aa..f2934fa762 100644 --- a/include/nbl/builtin/hlsl/glsl_compat/core.hlsl +++ b/include/nbl/builtin/hlsl/glsl_compat/core.hlsl @@ -116,16 +116,6 @@ enable_if_t, T> atomicXor(Ptr_T ptr, T value) { return spirv::atomicXor(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); } -/* TODO: @Hazardu struct dispatchers like for `bitfieldExtract` -template -T atomicMin(NBL_REF_ARG(T) ptr, T value) -{ -} -template -T atomicMax(NBL_REF_ARG(T) ptr, T value) -{ -} -*/ template T atomicExchange(NBL_REF_ARG(T) ptr, T value) { @@ -147,6 +137,100 @@ enable_if_t, T> atomicCompSwap(Ptr_T ptr, T comparato return spirv::atomicCompareExchange(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, spv::MemorySemanticsMaskNone, value, comparator); } +namespace impl +{ +template +struct atomicMin; + +template +NBL_PARTIAL_REQ_TOP(concepts::SignedIntegral) +struct atomicMin) > +{ + static T __call(NBL_REF_ARG(T) ptr, T value) + { + return spirv::atomicSMin(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } + + template // DXC Workaround + static T __call(Ptr_T ptr, T value) + { + return spirv::atomicSMin(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } +}; + +template +NBL_PARTIAL_REQ_TOP(concepts::UnsignedIntegral) +struct atomicMin) > +{ + static T __call(NBL_REF_ARG(T) ptr, T value) + { + return spirv::atomicUMin(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } + + template // DXC Workaround + static T __call(Ptr_T ptr, T value) + { + return spirv::atomicUMin(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } +}; + +template +struct atomicMax; + +template +NBL_PARTIAL_REQ_TOP(concepts::SignedIntegral) +struct atomicMax) > +{ + static T __call(NBL_REF_ARG(T) ptr, T value) + { + return spirv::atomicSMax(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } + + template // DXC Workaround + static T __call(Ptr_T ptr, T value) + { + return spirv::atomicSMax(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } +}; + +template +NBL_PARTIAL_REQ_TOP(concepts::UnsignedIntegral) +struct atomicMax) > +{ + static T __call(NBL_REF_ARG(T) ptr, T value) + { + return spirv::atomicUMax(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } + + template // DXC Workaround + static T __call(Ptr_T ptr, T value) + { + return spirv::atomicUMax(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } +}; +} + +template +T atomicMin(NBL_REF_ARG(T) ptr, T value) +{ + return impl::atomicMin.__call(ptr, value); +} +template // DXC Workaround +enable_if_t, T> atomicMin(Ptr_T ptr, T value) +{ + return impl::atomicMin.template __call(ptr, value); +} +template +T atomicMax(NBL_REF_ARG(T) ptr, T value) +{ + return impl::atomicMax.__call(ptr, value); +} +template // DXC Workaround +enable_if_t, T> atomicMax(Ptr_T ptr, T value) +{ + return impl::atomicMax.template __call(ptr, value); +} + /** * GLSL extended math */ From 747d35b0630f85a6ff432345e8061683ebd570b0 Mon Sep 17 00:00:00 2001 From: keptsecret Date: Wed, 22 Jul 2026 11:27:38 +0700 Subject: [PATCH 10/21] some bug fixes to atomic min/max --- include/nbl/builtin/hlsl/glsl_compat/core.hlsl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/nbl/builtin/hlsl/glsl_compat/core.hlsl b/include/nbl/builtin/hlsl/glsl_compat/core.hlsl index f2934fa762..1549ce1990 100644 --- a/include/nbl/builtin/hlsl/glsl_compat/core.hlsl +++ b/include/nbl/builtin/hlsl/glsl_compat/core.hlsl @@ -213,22 +213,22 @@ struct atomicMax) > template T atomicMin(NBL_REF_ARG(T) ptr, T value) { - return impl::atomicMin.__call(ptr, value); + return impl::atomicMin::__call(ptr, value); } template // DXC Workaround enable_if_t, T> atomicMin(Ptr_T ptr, T value) { - return impl::atomicMin.template __call(ptr, value); + return impl::atomicMin::template __call(ptr, value); } template T atomicMax(NBL_REF_ARG(T) ptr, T value) { - return impl::atomicMax.__call(ptr, value); + return impl::atomicMax::__call(ptr, value); } template // DXC Workaround enable_if_t, T> atomicMax(Ptr_T ptr, T value) { - return impl::atomicMax.template __call(ptr, value); + return impl::atomicMax::template __call(ptr, value); } /** From 972a8fbddf9c3d9e0fb7cc7bfd0670d07623dcbe Mon Sep 17 00:00:00 2001 From: keptsecret Date: Wed, 22 Jul 2026 14:41:10 +0700 Subject: [PATCH 11/21] a number of bug fixes to indexing stuff --- .../nbl/builtin/hlsl/scan/chained_scan.hlsl | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index 1b824ffcd9..90e0bf9827 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -109,12 +109,17 @@ struct Scan // TODO: double check this but it should be the last element of the last workgroup thread // don't know what it's like if virtual workgroup size doesn't divide by workgroup size exactly - if (invocIx == glsl::gl_SubgroupSize() * glsl::gl_NumSubgroups() - 1u) - currGroupReduction = wgDataAccessor.preloaded[wg_data_proxy_t::PreloadedDataCount-1u][Config::ItemsPerInvocation_0-1u]; + const uint32_t lastInvocIx = glsl::gl_SubgroupSize() * glsl::gl_NumSubgroups() - 1u; + currGroupReduction = wgDataAccessor.preloaded[wg_data_proxy_t::PreloadedDataCount-1u][Config::ItemsPerInvocation_0-1u]; + if (invocIx == lastInvocIx) + scratchAccessor.template set(0u, currGroupReduction); + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + scratchAccessor.template get(0u, currGroupReduction); } scratchAccessor.workgroupExecutionAndMemoryBarrier(); - if (invocIx == glsl::gl_SubgroupSize() * glsl::gl_NumSubgroups() - 1u) + if (!invocIx) { const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, workgroupId > 0u) | currGroupReduction << Flag_Shift; workgroupReduction.atomicExchange(workgroupId, storeVal); @@ -132,6 +137,7 @@ struct Scan while (locked) { // lookback: try to get reduction from previous workgroups + bool lookbackFailed = true; if (!invocIx) { scalar_t flagPayload; @@ -146,15 +152,15 @@ struct Scan workgroupReduction.atomicExchange(workgroupId, storeVal); scratchAccessor.template set(0u, prevReduction); sIsLocked = false; - break; + lookbackFailed = false; } else lookbackIx--; } - // lookback failed // broadcast id and prepare to do reduction ourselves - scratchAccessor.template set(1u, lookbackIx); + if (lookbackFailed) + scratchAccessor.template set(1u, lookbackIx); } scratchAccessor.workgroupExecutionAndMemoryBarrier(); @@ -163,7 +169,7 @@ struct Scan if (locked) { // do reduction for lookbackIx workgroup - uint32_t fallbackGroupId; + uint16_t fallbackGroupId; scratchAccessor.template get(1u, fallbackGroupId); wg_data_proxy_t fallbackDataAccessor = wg_data_proxy_t::create(dataAccessor.getInputBufAddr(), dataAccessor.getOutputBufAddr(), fallbackGroupId); @@ -173,11 +179,10 @@ struct Scan if (!invocIx) { - scalar_t fallbackPayload; const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, fallbackGroupId > 0u) | (fallbackReduction << Flag_Shift); - workgroupReduction.atomicMax(workgroupId, storeVal); + const scalar_t fallbackPayload = workgroupReduction.atomicMax(fallbackGroupId, storeVal); - prevReduction = binop(prevReduction, hlsl::max(fallbackReduction, fallbackPayload) >> Flag_Shift); + prevReduction = binop(prevReduction, hlsl::mix(fallbackReduction, fallbackPayload >> Flag_Shift, fallbackPayload > scalar_t(0.0))); if (!fallbackGroupId || (fallbackPayload & Flag_Mask) == Flag_Inclusive) { const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); @@ -187,7 +192,7 @@ struct Scan } else { - lookbackIndex--; + lookbackIx--; } } scratchAccessor.workgroupExecutionAndMemoryBarrier(); @@ -199,15 +204,14 @@ struct Scan } scratchAccessor.workgroupExecutionAndMemoryBarrier(); - scalar_t prevReduction; - scratchAccessor.template get(0u, prevReduction); - prevReduction = binop(prevReduction, currGroupReduction); + scalar_t prevReduction = 0u; + if (workgroupId > 0) + scratchAccessor.template get(0u, prevReduction); NBL_UNROLL for (uint16_t idx = 0; idx < wg_data_proxy_t::PreloadedDataCount; idx++) { vector_t data = wgDataAccessor.preloaded[idx]; - dataAccessor.template get(idx * WorkgroupSize + invocIx, data); NBL_UNROLL for (uint16_t i = 0; i < Config::ItemsPerInvocation_0; i++) data[i] = binop(prevReduction, data[i]); From 9085b5dd60cf98d2c70ff73cbef4d5b13b001f1e Mon Sep 17 00:00:00 2001 From: keptsecret Date: Wed, 22 Jul 2026 16:59:54 +0700 Subject: [PATCH 12/21] fix exclusive scan to reduction store --- include/nbl/builtin/hlsl/scan/chained_scan.hlsl | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index 90e0bf9827..4a3f05e7a1 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -99,7 +99,14 @@ struct Scan { wgDataAccessor.preload(); - NBL_IF_CONSTEXPR(Exclusive) + // TODO: double check this but it should be the last element of the last workgroup thread + // don't know what it's like if virtual workgroup size doesn't divide by workgroup size exactly + const uint32_t lastInvocIx = glsl::gl_SubgroupSize() * glsl::gl_NumSubgroups() - 1u; + scalar_t lastElem; + if (invocIx == lastInvocIx) + lastElem = wgDataAccessor.preloaded[wg_data_proxy_t::PreloadedDataCount-1u][Config::ItemsPerInvocation_0-1u]; + + if (Exclusive) workgroup2::exclusive_scan::template __call(wgDataAccessor, scratchAccessor); else workgroup2::inclusive_scan::template __call(wgDataAccessor, scratchAccessor); @@ -107,10 +114,9 @@ struct Scan // wgDataAccessor.unload(); // TODO: maybe we don't have to unload, just write once after everything is done - // TODO: double check this but it should be the last element of the last workgroup thread - // don't know what it's like if virtual workgroup size doesn't divide by workgroup size exactly - const uint32_t lastInvocIx = glsl::gl_SubgroupSize() * glsl::gl_NumSubgroups() - 1u; currGroupReduction = wgDataAccessor.preloaded[wg_data_proxy_t::PreloadedDataCount-1u][Config::ItemsPerInvocation_0-1u]; + if (Exclusive) + currGroupReduction = binop(currGroupReduction, lastElem); if (invocIx == lastInvocIx) scratchAccessor.template set(0u, currGroupReduction); scratchAccessor.workgroupExecutionAndMemoryBarrier(); From 3a59e93ae2be7bd99d44f2b27ed6b17957fd4fa7 Mon Sep 17 00:00:00 2001 From: keptsecret Date: Thu, 23 Jul 2026 17:03:45 +0700 Subject: [PATCH 13/21] use a workgroup counter to determine execution order --- .../nbl/builtin/hlsl/scan/chained_scan.hlsl | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index 4a3f05e7a1..8e8e005bae 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -83,16 +83,23 @@ struct Scan NBL_CONSTEXPR_STATIC_INLINE uint16_t WorkgroupSize = uint16_t(1u) << Config::WorkgroupSizeLog2; NBL_CONSTEXPR_STATIC_INLINE uint16_t ItemsPerInvoc = Config::VirtualWorkgroupSize / WorkgroupSize; - template - void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction) + template + void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction, NBL_REF_ARG(WorkgroupCounter) workgroupCounter) { const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); - const uint16_t workgroupId = uint16_t(glsl::gl_WorkGroupID().x); - binop_t binop; - if (!invocIx) + { + const uint32_t id = workgroupCounter.atomicAdd(0u, 1u); + scratchAccessor.template set(0u, id); sIsLocked = true; + } + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + uint16_t workgroupId; + scratchAccessor.template get(0u, workgroupId); + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + binop_t binop; using wg_data_proxy_t = WorkgroupDataProxy; wg_data_proxy_t wgDataAccessor = wg_data_proxy_t::create(dataAccessor.getInputBufAddr(), dataAccessor.getOutputBufAddr(), workgroupId); scalar_t currGroupReduction; @@ -210,6 +217,8 @@ struct Scan } scratchAccessor.workgroupExecutionAndMemoryBarrier(); + dataAccessor.initAtWorkgroupID(workgroupId); + scalar_t prevReduction = 0u; if (workgroupId > 0) scratchAccessor.template get(0u, prevReduction); @@ -233,11 +242,11 @@ struct inclusive_scan using scalar_t = typename BinOp::type_t; // TODO: might want new concept for ReductionAccessor - template && workgroup2::ArithmeticSharedMemoryAccessor) - static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction) + template && workgroup2::ArithmeticSharedMemoryAccessor) + static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction, NBL_REF_ARG(WorkgroupCounter) counter) { impl::Scan fn; - fn.template __call(dataAccessor, scratchAccessor, workgroupReduction); + fn.template __call(dataAccessor, scratchAccessor, workgroupReduction, counter); } }; @@ -246,11 +255,11 @@ struct exclusive_scan { using scalar_t = typename BinOp::type_t; - template && workgroup2::ArithmeticSharedMemoryAccessor) - static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction) + template && workgroup2::ArithmeticSharedMemoryAccessor) + static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction, NBL_REF_ARG(WorkgroupCounter) counter) { impl::Scan fn; - fn.template __call(dataAccessor, scratchAccessor, workgroupReduction); + fn.template __call(dataAccessor, scratchAccessor, workgroupReduction, counter); } }; From 25e2d0193c0a6a61c5e9c2d29daa806be1e5fc7f Mon Sep 17 00:00:00 2001 From: keptsecret Date: Mon, 27 Jul 2026 12:23:45 +0700 Subject: [PATCH 14/21] fix for indexing in workgroup scan accessor (unsure if temp fix) --- .../nbl/builtin/hlsl/scan/chained_scan.hlsl | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index 8e8e005bae..2c064f0084 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -19,14 +19,15 @@ struct WorkgroupDataProxy { using dtype_t = vector; - NBL_CONSTEXPR_STATIC_INLINE uint16_t WorkgroupSize = uint16_t(1u) << WorkgroupSizeLog2; - NBL_CONSTEXPR_STATIC_INLINE uint16_t PreloadedDataCount = VirtualWorkgroupSize / WorkgroupSize; + NBL_CONSTEXPR_STATIC_INLINE uint32_t WorkgroupSize = 1u << WorkgroupSizeLog2; + NBL_CONSTEXPR_STATIC_INLINE uint16_t PreloadedDataCount = uint16_t(VirtualWorkgroupSize / WorkgroupSize); static WorkgroupDataProxy create(const uint64_t inputBuf, const uint64_t outputBuf, const uint16_t workgroupId) { WorkgroupDataProxy retval; const uint32_t workgroupOffset = workgroupId * VirtualWorkgroupSize * sizeof(dtype_t); - retval.accessor = DoubleLegacyBdaAccessor::create(inputBuf + workgroupOffset, outputBuf + workgroupOffset); + retval.accessor = DoubleLegacyBdaAccessor::create(inputBuf/* + workgroupOffset*/, outputBuf /*+ workgroupOffset*/); + retval.workgroupID = workgroupId; return retval; } @@ -46,14 +47,14 @@ struct WorkgroupDataProxy const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); NBL_UNROLL for (uint16_t idx = 0; idx < PreloadedDataCount; idx++) - accessor.get(idx * WorkgroupSize + invocIx, preloaded[idx]); + accessor.get((workgroupID + idx) * WorkgroupSize + invocIx, preloaded[idx]); } void unload() { const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); NBL_UNROLL for (uint16_t idx = 0; idx < PreloadedDataCount; idx++) - accessor.set(idx * WorkgroupSize + invocIx, preloaded[idx]); + accessor.set((workgroupID + idx) * WorkgroupSize + invocIx, preloaded[idx]); } void workgroupExecutionAndMemoryBarrier() @@ -64,6 +65,7 @@ struct WorkgroupDataProxy DoubleLegacyBdaAccessor accessor; dtype_t preloaded[PreloadedDataCount]; + uint32_t workgroupID; // TODO: remove possibly?, current problem is that adding the workgroup offset directly to address doesn't work }; template // TODO: Config is same as workgroup2 stuff? @@ -80,8 +82,8 @@ struct Scan NBL_CONSTEXPR_STATIC_INLINE uint16_t Flag_Shift = 2; - NBL_CONSTEXPR_STATIC_INLINE uint16_t WorkgroupSize = uint16_t(1u) << Config::WorkgroupSizeLog2; - NBL_CONSTEXPR_STATIC_INLINE uint16_t ItemsPerInvoc = Config::VirtualWorkgroupSize / WorkgroupSize; + NBL_CONSTEXPR_STATIC_INLINE uint32_t WorkgroupSize = 1u << Config::WorkgroupSizeLog2; + NBL_CONSTEXPR_STATIC_INLINE uint16_t ItemsPerInvoc = uint16_t(Config::VirtualWorkgroupSize / WorkgroupSize); template void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction, NBL_REF_ARG(WorkgroupCounter) workgroupCounter) @@ -94,6 +96,7 @@ struct Scan sIsLocked = true; } scratchAccessor.workgroupExecutionAndMemoryBarrier(); + // spirv::memoryBarrier(spv::ScopeDevice, spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsUniformMemoryMask); uint16_t workgroupId; scratchAccessor.template get(0u, workgroupId); @@ -138,7 +141,6 @@ struct Scan workgroupReduction.atomicExchange(workgroupId, storeVal); } - // lookback if (workgroupId > 0) { bool locked = sIsLocked; @@ -230,7 +232,7 @@ struct Scan NBL_UNROLL for (uint16_t i = 0; i < Config::ItemsPerInvocation_0; i++) data[i] = binop(prevReduction, data[i]); - dataAccessor.template set(idx * WorkgroupSize + invocIx, data); + dataAccessor.template set(idx * WorkgroupSize + invocIx, data); } } }; From ba9e151427a8b92d96294bf987fa0314d76f2d83 Mon Sep 17 00:00:00 2001 From: keptsecret Date: Tue, 28 Jul 2026 16:58:42 +0700 Subject: [PATCH 15/21] fix lookback to spin a bit for some more consistency --- .../nbl/builtin/hlsl/scan/chained_scan.hlsl | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index 2c064f0084..d551005448 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -84,6 +84,7 @@ struct Scan NBL_CONSTEXPR_STATIC_INLINE uint32_t WorkgroupSize = 1u << Config::WorkgroupSizeLog2; NBL_CONSTEXPR_STATIC_INLINE uint16_t ItemsPerInvoc = uint16_t(Config::VirtualWorkgroupSize / WorkgroupSize); + NBL_CONSTEXPR_STATIC_INLINE uint16_t MaxSpinCount = uint16_t(4u); template void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction, NBL_REF_ARG(WorkgroupCounter) workgroupCounter) @@ -152,29 +153,36 @@ struct Scan while (locked) { // lookback: try to get reduction from previous workgroups - bool lookbackFailed = true; if (!invocIx) { - scalar_t flagPayload; - workgroupReduction.get(lookbackIx, flagPayload); - - if ((flagPayload & Flag_Mask) > Flag_NotReady) + uint16_t spinCount = uint16_t(0u); + [loop] + while (spinCount < MaxSpinCount) { - prevReduction = binop(prevReduction, flagPayload >> Flag_Shift); - if ((flagPayload & Flag_Mask) == Flag_Inclusive) + scalar_t flagPayload; + workgroupReduction.get(lookbackIx, flagPayload); + + if ((flagPayload & Flag_Mask) > Flag_NotReady) { - const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); - workgroupReduction.atomicExchange(workgroupId, storeVal); - scratchAccessor.template set(0u, prevReduction); - sIsLocked = false; - lookbackFailed = false; + spinCount = uint16_t(0u); + prevReduction = binop(prevReduction, flagPayload >> Flag_Shift); + if ((flagPayload & Flag_Mask) == Flag_Inclusive) + { + const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); + workgroupReduction.atomicExchange(workgroupId, storeVal); + scratchAccessor.template set(0u, prevReduction); + sIsLocked = false; + break; + } + else + lookbackIx--; } else - lookbackIx--; + spinCount++; } // broadcast id and prepare to do reduction ourselves - if (lookbackFailed) + if (spinCount == MaxSpinCount) scratchAccessor.template set(1u, lookbackIx); } scratchAccessor.workgroupExecutionAndMemoryBarrier(); @@ -206,9 +214,7 @@ struct Scan sIsLocked = false; } else - { lookbackIx--; - } } scratchAccessor.workgroupExecutionAndMemoryBarrier(); From dd61813dc3c0890be850767324b9289b84a6a116 Mon Sep 17 00:00:00 2001 From: keptsecret Date: Thu, 30 Jul 2026 15:17:10 +0700 Subject: [PATCH 16/21] can use single scratch address + fixes bug in example, memorybarrier probably not needed but left in for now --- include/nbl/builtin/hlsl/scan/chained_scan.hlsl | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index d551005448..b20add8b7b 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -97,7 +97,6 @@ struct Scan sIsLocked = true; } scratchAccessor.workgroupExecutionAndMemoryBarrier(); - // spirv::memoryBarrier(spv::ScopeDevice, spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsUniformMemoryMask); uint16_t workgroupId; scratchAccessor.template get(0u, workgroupId); @@ -141,8 +140,9 @@ struct Scan const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, workgroupId > 0u) | currGroupReduction << Flag_Shift; workgroupReduction.atomicExchange(workgroupId, storeVal); } + // workgroupReduction.memoryBarrier(); - if (workgroupId > 0) + if (workgroupId) { bool locked = sIsLocked; scratchAccessor.workgroupExecutionAndMemoryBarrier(); @@ -183,9 +183,10 @@ struct Scan // broadcast id and prepare to do reduction ourselves if (spinCount == MaxSpinCount) - scratchAccessor.template set(1u, lookbackIx); + scratchAccessor.template set(0u, lookbackIx); } scratchAccessor.workgroupExecutionAndMemoryBarrier(); + // workgroupReduction.memoryBarrier(); locked = sIsLocked; scratchAccessor.workgroupExecutionAndMemoryBarrier(); @@ -193,7 +194,7 @@ struct Scan { // do reduction for lookbackIx workgroup uint16_t fallbackGroupId; - scratchAccessor.template get(1u, fallbackGroupId); + scratchAccessor.template get(0u, fallbackGroupId); wg_data_proxy_t fallbackDataAccessor = wg_data_proxy_t::create(dataAccessor.getInputBufAddr(), dataAccessor.getOutputBufAddr(), fallbackGroupId); fallbackDataAccessor.preload(); @@ -204,12 +205,14 @@ struct Scan { const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, fallbackGroupId > 0u) | (fallbackReduction << Flag_Shift); const scalar_t fallbackPayload = workgroupReduction.atomicMax(fallbackGroupId, storeVal); + // workgroupReduction.memoryBarrier(); prevReduction = binop(prevReduction, hlsl::mix(fallbackReduction, fallbackPayload >> Flag_Shift, fallbackPayload > scalar_t(0.0))); - if (!fallbackGroupId || (fallbackPayload & Flag_Mask) == Flag_Inclusive) + if (fallbackGroupId == 0u || (fallbackPayload & Flag_Mask) == Flag_Inclusive) { const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); workgroupReduction.atomicExchange(workgroupId, storeVal); + // workgroupReduction.memoryBarrier(); scratchAccessor.template set(0u, prevReduction); sIsLocked = false; } @@ -228,7 +231,7 @@ struct Scan dataAccessor.initAtWorkgroupID(workgroupId); scalar_t prevReduction = 0u; - if (workgroupId > 0) + if (workgroupId) scratchAccessor.template get(0u, prevReduction); NBL_UNROLL From 02d6966fc7b42773b928c7844c616b36e9f2e929 Mon Sep 17 00:00:00 2001 From: keptsecret Date: Thu, 30 Jul 2026 16:40:33 +0700 Subject: [PATCH 17/21] fixes to indexing in preload --- include/nbl/builtin/hlsl/scan/chained_scan.hlsl | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index b20add8b7b..59b8b86726 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -25,8 +25,7 @@ struct WorkgroupDataProxy static WorkgroupDataProxy create(const uint64_t inputBuf, const uint64_t outputBuf, const uint16_t workgroupId) { WorkgroupDataProxy retval; - const uint32_t workgroupOffset = workgroupId * VirtualWorkgroupSize * sizeof(dtype_t); - retval.accessor = DoubleLegacyBdaAccessor::create(inputBuf/* + workgroupOffset*/, outputBuf /*+ workgroupOffset*/); + retval.accessor = DoubleLegacyBdaAccessor::create(inputBuf, outputBuf); retval.workgroupID = workgroupId; return retval; } @@ -47,14 +46,7 @@ struct WorkgroupDataProxy const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); NBL_UNROLL for (uint16_t idx = 0; idx < PreloadedDataCount; idx++) - accessor.get((workgroupID + idx) * WorkgroupSize + invocIx, preloaded[idx]); - } - void unload() - { - const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); - NBL_UNROLL - for (uint16_t idx = 0; idx < PreloadedDataCount; idx++) - accessor.set((workgroupID + idx) * WorkgroupSize + invocIx, preloaded[idx]); + accessor.get(workgroupID * VirtualWorkgroupSize + idx * WorkgroupSize + invocIx, preloaded[idx]); } void workgroupExecutionAndMemoryBarrier() @@ -109,7 +101,7 @@ struct Scan { wgDataAccessor.preload(); - // TODO: double check this but it should be the last element of the last workgroup thread + // should be the last element of the last workgroup thread // don't know what it's like if virtual workgroup size doesn't divide by workgroup size exactly const uint32_t lastInvocIx = glsl::gl_SubgroupSize() * glsl::gl_NumSubgroups() - 1u; scalar_t lastElem; @@ -122,8 +114,6 @@ struct Scan workgroup2::inclusive_scan::template __call(wgDataAccessor, scratchAccessor); scratchAccessor.workgroupExecutionAndMemoryBarrier(); - // wgDataAccessor.unload(); // TODO: maybe we don't have to unload, just write once after everything is done - currGroupReduction = wgDataAccessor.preloaded[wg_data_proxy_t::PreloadedDataCount-1u][Config::ItemsPerInvocation_0-1u]; if (Exclusive) currGroupReduction = binop(currGroupReduction, lastElem); From e5153a9aa09c6e91ebf26f39bae0a8b9e92d994d Mon Sep 17 00:00:00 2001 From: keptsecret Date: Thu, 30 Jul 2026 16:53:52 +0700 Subject: [PATCH 18/21] add chain scan hlsl to cmakelist --- src/nbl/builtin/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nbl/builtin/CMakeLists.txt b/src/nbl/builtin/CMakeLists.txt index df20f6b6b3..a75768c981 100644 --- a/src/nbl/builtin/CMakeLists.txt +++ b/src/nbl/builtin/CMakeLists.txt @@ -362,6 +362,8 @@ LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/workgroup2/impl/items_per_inv LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/workgroup2/impl/arithmetic_config_def.hlsl") LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/workgroup2/arithmetic.hlsl") LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/workgroup2/shared_scan.hlsl") +#device scan +LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/scan/chained_scan.hlsl") #Extensions LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/ext/FullScreenTriangle/SVertexAttributes.hlsl") LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/ext/FullScreenTriangle/default.vert.hlsl") From fbf4b57f82b044feee1729d117c260eca4372bb1 Mon Sep 17 00:00:00 2001 From: keptsecret Date: Thu, 30 Jul 2026 16:54:02 +0700 Subject: [PATCH 19/21] latest example --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index 8515f32fc4..0990b176a1 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 8515f32fc4bb630270dbe3bca83a2835fd2006e2 +Subproject commit 0990b176a17aee6381a767162f7c2d1d31335c8b From c5b30319c1091c5ec6d04ad545afd64245e750d5 Mon Sep 17 00:00:00 2001 From: keptsecret Date: Fri, 31 Jul 2026 12:15:16 +0700 Subject: [PATCH 20/21] concepts for accessors of device scan, removed obsolete comments --- .../concepts/accessors/device_arithmetic.hlsl | 65 +++++++++++++++++++ .../nbl/builtin/hlsl/scan/chained_scan.hlsl | 28 ++++---- src/nbl/builtin/CMakeLists.txt | 1 + 3 files changed, 78 insertions(+), 16 deletions(-) create mode 100644 include/nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl diff --git a/include/nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl b/include/nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl new file mode 100644 index 0000000000..e852f889e6 --- /dev/null +++ b/include/nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl @@ -0,0 +1,65 @@ +#ifndef _NBL_BUILTIN_HLSL_CONCEPTS_ACCESSORS_DEVICE_ARITHMETIC_INCLUDED_ +#define _NBL_BUILTIN_HLSL_CONCEPTS_ACCESSORS_DEVICE_ARITHMETIC_INCLUDED_ + +#include "nbl/builtin/hlsl/concepts/accessors/generic_shared_data.hlsl" + +namespace nbl +{ +namespace hlsl +{ +namespace scan +{ + +template +NBL_BOOL_CONCEPT ArithmeticSharedMemoryAccessor = concepts::accessors::GenericSharedMemoryAccessor; + +template +NBL_BOOL_CONCEPT ArithmeticReadOnlyDataAccessor = concepts::accessors::GenericReadAccessor; + +template +NBL_BOOL_CONCEPT ArithmeticDataAccessor = concepts::accessors::GenericDataAccessor; + +#define NBL_CONCEPT_NAME DeviceReductionsAccessor +#define NBL_CONCEPT_TPLT_PRM_KINDS (typename)(typename) +#define NBL_CONCEPT_TPLT_PRM_NAMES (T)(V) +#define NBL_CONCEPT_PARAM_0 (accessor, T) +#define NBL_CONCEPT_PARAM_1 (val, V) +#define NBL_CONCEPT_PARAM_2 (index, uint64_t) +NBL_CONCEPT_BEGIN(3) +#define accessor NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_0 +#define val NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_1 +#define index NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_2 +NBL_CONCEPT_END( + ((NBL_CONCEPT_REQ_TYPE_ALIAS_CONCEPT)(concepts::accessors::GenericDataAccessor, T, V, uint64_t)) + ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((accessor.atomicMax(index, val)), is_same_v, V)) + ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((accessor.atomicExchange(index, val)), is_same_v, V)) +); +#undef val +#undef index +#undef accessor +#include + +// TODO: as counter, maybe just increment 1 always? +#define NBL_CONCEPT_NAME WorkgroupCounterAccessor +#define NBL_CONCEPT_TPLT_PRM_KINDS (typename) +#define NBL_CONCEPT_TPLT_PRM_NAMES (T) +#define NBL_CONCEPT_PARAM_0 (accessor, T) +#define NBL_CONCEPT_PARAM_1 (val, uint32_t) +#define NBL_CONCEPT_PARAM_2 (index, uint64_t) +NBL_CONCEPT_BEGIN(3) +#define accessor NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_0 +#define val NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_1 +#define index NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_2 +NBL_CONCEPT_END( + ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((accessor.atomicAdd(index, val)), is_same_v, uint32_t)) +); +#undef val +#undef index +#undef accessor +#include + +} +} +} + +#endif diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index 59b8b86726..2f1c10cd95 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -1,6 +1,7 @@ #ifndef _NBL_HLSL_SCAN_CHAINED_SCAN_INCLUDED_ #define _NBL_HLSL_SCAN_CHAINED_SCAN_INCLUDED_ +#include "nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl" #include "nbl/builtin/hlsl/workgroup2/arithmetic.hlsl" groupshared bool sIsLocked; @@ -57,7 +58,7 @@ struct WorkgroupDataProxy DoubleLegacyBdaAccessor accessor; dtype_t preloaded[PreloadedDataCount]; - uint32_t workgroupID; // TODO: remove possibly?, current problem is that adding the workgroup offset directly to address doesn't work + uint32_t workgroupID; }; template // TODO: Config is same as workgroup2 stuff? @@ -118,10 +119,10 @@ struct Scan if (Exclusive) currGroupReduction = binop(currGroupReduction, lastElem); if (invocIx == lastInvocIx) - scratchAccessor.template set(0u, currGroupReduction); + scratchAccessor.template set(0u, currGroupReduction); scratchAccessor.workgroupExecutionAndMemoryBarrier(); - scratchAccessor.template get(0u, currGroupReduction); + scratchAccessor.template get(0u, currGroupReduction); } scratchAccessor.workgroupExecutionAndMemoryBarrier(); @@ -130,7 +131,6 @@ struct Scan const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, workgroupId > 0u) | currGroupReduction << Flag_Shift; workgroupReduction.atomicExchange(workgroupId, storeVal); } - // workgroupReduction.memoryBarrier(); if (workgroupId) { @@ -150,7 +150,7 @@ struct Scan while (spinCount < MaxSpinCount) { scalar_t flagPayload; - workgroupReduction.get(lookbackIx, flagPayload); + workgroupReduction.template get(lookbackIx, flagPayload); if ((flagPayload & Flag_Mask) > Flag_NotReady) { @@ -160,7 +160,7 @@ struct Scan { const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); workgroupReduction.atomicExchange(workgroupId, storeVal); - scratchAccessor.template set(0u, prevReduction); + scratchAccessor.template set(0u, prevReduction); sIsLocked = false; break; } @@ -173,10 +173,9 @@ struct Scan // broadcast id and prepare to do reduction ourselves if (spinCount == MaxSpinCount) - scratchAccessor.template set(0u, lookbackIx); + scratchAccessor.template set(0u, lookbackIx); } scratchAccessor.workgroupExecutionAndMemoryBarrier(); - // workgroupReduction.memoryBarrier(); locked = sIsLocked; scratchAccessor.workgroupExecutionAndMemoryBarrier(); @@ -184,7 +183,7 @@ struct Scan { // do reduction for lookbackIx workgroup uint16_t fallbackGroupId; - scratchAccessor.template get(0u, fallbackGroupId); + scratchAccessor.template get(0u, fallbackGroupId); wg_data_proxy_t fallbackDataAccessor = wg_data_proxy_t::create(dataAccessor.getInputBufAddr(), dataAccessor.getOutputBufAddr(), fallbackGroupId); fallbackDataAccessor.preload(); @@ -195,15 +194,13 @@ struct Scan { const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, fallbackGroupId > 0u) | (fallbackReduction << Flag_Shift); const scalar_t fallbackPayload = workgroupReduction.atomicMax(fallbackGroupId, storeVal); - // workgroupReduction.memoryBarrier(); prevReduction = binop(prevReduction, hlsl::mix(fallbackReduction, fallbackPayload >> Flag_Shift, fallbackPayload > scalar_t(0.0))); if (fallbackGroupId == 0u || (fallbackPayload & Flag_Mask) == Flag_Inclusive) { const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); workgroupReduction.atomicExchange(workgroupId, storeVal); - // workgroupReduction.memoryBarrier(); - scratchAccessor.template set(0u, prevReduction); + scratchAccessor.template set(0u, prevReduction); sIsLocked = false; } else @@ -222,7 +219,7 @@ struct Scan scalar_t prevReduction = 0u; if (workgroupId) - scratchAccessor.template get(0u, prevReduction); + scratchAccessor.template get(0u, prevReduction); NBL_UNROLL for (uint16_t idx = 0; idx < wg_data_proxy_t::PreloadedDataCount; idx++) @@ -242,8 +239,7 @@ struct inclusive_scan { using scalar_t = typename BinOp::type_t; - // TODO: might want new concept for ReductionAccessor - template && workgroup2::ArithmeticSharedMemoryAccessor) + template && ArithmeticSharedMemoryAccessor && DeviceReductionsAccessor && WorkgroupCounterAccessor) static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction, NBL_REF_ARG(WorkgroupCounter) counter) { impl::Scan fn; @@ -256,7 +252,7 @@ struct exclusive_scan { using scalar_t = typename BinOp::type_t; - template && workgroup2::ArithmeticSharedMemoryAccessor) + template && ArithmeticSharedMemoryAccessor && DeviceReductionsAccessor && WorkgroupCounterAccessor) static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction, NBL_REF_ARG(WorkgroupCounter) counter) { impl::Scan fn; diff --git a/src/nbl/builtin/CMakeLists.txt b/src/nbl/builtin/CMakeLists.txt index a75768c981..d6140e2625 100644 --- a/src/nbl/builtin/CMakeLists.txt +++ b/src/nbl/builtin/CMakeLists.txt @@ -387,6 +387,7 @@ LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/concepts/accessors/storable_i LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/concepts/accessors/generic_shared_data.hlsl") LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/concepts/accessors/fft.hlsl") LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/concepts/accessors/workgroup_arithmetic.hlsl") +LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/concepts/accessors/device_arithmetic.hlsl") #tgmath LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/tgmath.hlsl") LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/tgmath/impl.hlsl") From 12a8f560d2a65bbc507829a952c36f6889cdba7e Mon Sep 17 00:00:00 2001 From: keptsecret Date: Fri, 31 Jul 2026 12:16:12 +0700 Subject: [PATCH 21/21] latest example --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index 0990b176a1..5ac0f19fd4 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 0990b176a17aee6381a767162f7c2d1d31335c8b +Subproject commit 5ac0f19fd46cef5680b92b5fa4fd97f3f2b29fe4