diff --git a/examples_tests b/examples_tests index 8515f32fc4..5ac0f19fd4 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 8515f32fc4bb630270dbe3bca83a2835fd2006e2 +Subproject commit 5ac0f19fd46cef5680b92b5fa4fd97f3f2b29fe4 diff --git a/include/nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl b/include/nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl new file mode 100644 index 0000000000..e852f889e6 --- /dev/null +++ b/include/nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl @@ -0,0 +1,65 @@ +#ifndef _NBL_BUILTIN_HLSL_CONCEPTS_ACCESSORS_DEVICE_ARITHMETIC_INCLUDED_ +#define _NBL_BUILTIN_HLSL_CONCEPTS_ACCESSORS_DEVICE_ARITHMETIC_INCLUDED_ + +#include "nbl/builtin/hlsl/concepts/accessors/generic_shared_data.hlsl" + +namespace nbl +{ +namespace hlsl +{ +namespace scan +{ + +template +NBL_BOOL_CONCEPT ArithmeticSharedMemoryAccessor = concepts::accessors::GenericSharedMemoryAccessor; + +template +NBL_BOOL_CONCEPT ArithmeticReadOnlyDataAccessor = concepts::accessors::GenericReadAccessor; + +template +NBL_BOOL_CONCEPT ArithmeticDataAccessor = concepts::accessors::GenericDataAccessor; + +#define NBL_CONCEPT_NAME DeviceReductionsAccessor +#define NBL_CONCEPT_TPLT_PRM_KINDS (typename)(typename) +#define NBL_CONCEPT_TPLT_PRM_NAMES (T)(V) +#define NBL_CONCEPT_PARAM_0 (accessor, T) +#define NBL_CONCEPT_PARAM_1 (val, V) +#define NBL_CONCEPT_PARAM_2 (index, uint64_t) +NBL_CONCEPT_BEGIN(3) +#define accessor NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_0 +#define val NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_1 +#define index NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_2 +NBL_CONCEPT_END( + ((NBL_CONCEPT_REQ_TYPE_ALIAS_CONCEPT)(concepts::accessors::GenericDataAccessor, T, V, uint64_t)) + ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((accessor.atomicMax(index, val)), is_same_v, V)) + ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((accessor.atomicExchange(index, val)), is_same_v, V)) +); +#undef val +#undef index +#undef accessor +#include + +// TODO: as counter, maybe just increment 1 always? +#define NBL_CONCEPT_NAME WorkgroupCounterAccessor +#define NBL_CONCEPT_TPLT_PRM_KINDS (typename) +#define NBL_CONCEPT_TPLT_PRM_NAMES (T) +#define NBL_CONCEPT_PARAM_0 (accessor, T) +#define NBL_CONCEPT_PARAM_1 (val, uint32_t) +#define NBL_CONCEPT_PARAM_2 (index, uint64_t) +NBL_CONCEPT_BEGIN(3) +#define accessor NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_0 +#define val NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_1 +#define index NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_2 +NBL_CONCEPT_END( + ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((accessor.atomicAdd(index, val)), is_same_v, uint32_t)) +); +#undef val +#undef index +#undef accessor +#include + +} +} +} + +#endif diff --git a/include/nbl/builtin/hlsl/glsl_compat/core.hlsl b/include/nbl/builtin/hlsl/glsl_compat/core.hlsl index c2450e57aa..1549ce1990 100644 --- a/include/nbl/builtin/hlsl/glsl_compat/core.hlsl +++ b/include/nbl/builtin/hlsl/glsl_compat/core.hlsl @@ -116,16 +116,6 @@ enable_if_t, T> atomicXor(Ptr_T ptr, T value) { return spirv::atomicXor(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); } -/* TODO: @Hazardu struct dispatchers like for `bitfieldExtract` -template -T atomicMin(NBL_REF_ARG(T) ptr, T value) -{ -} -template -T atomicMax(NBL_REF_ARG(T) ptr, T value) -{ -} -*/ template T atomicExchange(NBL_REF_ARG(T) ptr, T value) { @@ -147,6 +137,100 @@ enable_if_t, T> atomicCompSwap(Ptr_T ptr, T comparato return spirv::atomicCompareExchange(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, spv::MemorySemanticsMaskNone, value, comparator); } +namespace impl +{ +template +struct atomicMin; + +template +NBL_PARTIAL_REQ_TOP(concepts::SignedIntegral) +struct atomicMin) > +{ + static T __call(NBL_REF_ARG(T) ptr, T value) + { + return spirv::atomicSMin(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } + + template // DXC Workaround + static T __call(Ptr_T ptr, T value) + { + return spirv::atomicSMin(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } +}; + +template +NBL_PARTIAL_REQ_TOP(concepts::UnsignedIntegral) +struct atomicMin) > +{ + static T __call(NBL_REF_ARG(T) ptr, T value) + { + return spirv::atomicUMin(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } + + template // DXC Workaround + static T __call(Ptr_T ptr, T value) + { + return spirv::atomicUMin(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } +}; + +template +struct atomicMax; + +template +NBL_PARTIAL_REQ_TOP(concepts::SignedIntegral) +struct atomicMax) > +{ + static T __call(NBL_REF_ARG(T) ptr, T value) + { + return spirv::atomicSMax(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } + + template // DXC Workaround + static T __call(Ptr_T ptr, T value) + { + return spirv::atomicSMax(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } +}; + +template +NBL_PARTIAL_REQ_TOP(concepts::UnsignedIntegral) +struct atomicMax) > +{ + static T __call(NBL_REF_ARG(T) ptr, T value) + { + return spirv::atomicUMax(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } + + template // DXC Workaround + static T __call(Ptr_T ptr, T value) + { + return spirv::atomicUMax(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } +}; +} + +template +T atomicMin(NBL_REF_ARG(T) ptr, T value) +{ + return impl::atomicMin::__call(ptr, value); +} +template // DXC Workaround +enable_if_t, T> atomicMin(Ptr_T ptr, T value) +{ + return impl::atomicMin::template __call(ptr, value); +} +template +T atomicMax(NBL_REF_ARG(T) ptr, T value) +{ + return impl::atomicMax::__call(ptr, value); +} +template // DXC Workaround +enable_if_t, T> atomicMax(Ptr_T ptr, T value) +{ + return impl::atomicMax::template __call(ptr, value); +} + /** * GLSL extended math */ diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl new file mode 100644 index 0000000000..2f1c10cd95 --- /dev/null +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -0,0 +1,267 @@ +#ifndef _NBL_HLSL_SCAN_CHAINED_SCAN_INCLUDED_ +#define _NBL_HLSL_SCAN_CHAINED_SCAN_INCLUDED_ + +#include "nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl" +#include "nbl/builtin/hlsl/workgroup2/arithmetic.hlsl" + +groupshared bool sIsLocked; + +namespace nbl +{ +namespace hlsl +{ +namespace scan +{ + +namespace impl +{ +template +struct WorkgroupDataProxy +{ + using dtype_t = vector; + + NBL_CONSTEXPR_STATIC_INLINE uint32_t WorkgroupSize = 1u << WorkgroupSizeLog2; + NBL_CONSTEXPR_STATIC_INLINE uint16_t PreloadedDataCount = uint16_t(VirtualWorkgroupSize / WorkgroupSize); + + static WorkgroupDataProxy create(const uint64_t inputBuf, const uint64_t outputBuf, const uint16_t workgroupId) + { + WorkgroupDataProxy retval; + retval.accessor = DoubleLegacyBdaAccessor::create(inputBuf, outputBuf); + retval.workgroupID = workgroupId; + return retval; + } + + template + void get(const IndexType ix, NBL_REF_ARG(AccessType) value) + { + value = preloaded[ix>>WorkgroupSizeLog2]; + } + template + void set(const IndexType ix, const AccessType value) + { + preloaded[ix>>WorkgroupSizeLog2] = value; + } + + void preload() + { + const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); + NBL_UNROLL + for (uint16_t idx = 0; idx < PreloadedDataCount; idx++) + accessor.get(workgroupID * VirtualWorkgroupSize + idx * WorkgroupSize + invocIx, preloaded[idx]); + } + + void workgroupExecutionAndMemoryBarrier() + { + glsl::barrier(); + //glsl::memoryBarrierShared(); implied by the above + } + + DoubleLegacyBdaAccessor accessor; + dtype_t preloaded[PreloadedDataCount]; + uint32_t workgroupID; +}; + +template // TODO: Config is same as workgroup2 stuff? +struct Scan +{ + using scalar_t = typename BinOp::type_t; + using vector_t = vector; // data accessor needs to be this type + using binop_t = BinOp; + + NBL_CONSTEXPR_STATIC_INLINE uint32_t Flag_NotReady = 0; + NBL_CONSTEXPR_STATIC_INLINE uint32_t Flag_Reduction = 1; // workgroup only has local reduction ready + NBL_CONSTEXPR_STATIC_INLINE uint32_t Flag_Inclusive = 2; // workgroup has summed all preceding groups and added to own sum + NBL_CONSTEXPR_STATIC_INLINE uint32_t Flag_Mask = 3; + + NBL_CONSTEXPR_STATIC_INLINE uint16_t Flag_Shift = 2; + + NBL_CONSTEXPR_STATIC_INLINE uint32_t WorkgroupSize = 1u << Config::WorkgroupSizeLog2; + NBL_CONSTEXPR_STATIC_INLINE uint16_t ItemsPerInvoc = uint16_t(Config::VirtualWorkgroupSize / WorkgroupSize); + NBL_CONSTEXPR_STATIC_INLINE uint16_t MaxSpinCount = uint16_t(4u); + + template + void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction, NBL_REF_ARG(WorkgroupCounter) workgroupCounter) + { + const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); + if (!invocIx) + { + const uint32_t id = workgroupCounter.atomicAdd(0u, 1u); + scratchAccessor.template set(0u, id); + sIsLocked = true; + } + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + uint16_t workgroupId; + scratchAccessor.template get(0u, workgroupId); + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + binop_t binop; + using wg_data_proxy_t = WorkgroupDataProxy; + wg_data_proxy_t wgDataAccessor = wg_data_proxy_t::create(dataAccessor.getInputBufAddr(), dataAccessor.getOutputBufAddr(), workgroupId); + scalar_t currGroupReduction; + { + wgDataAccessor.preload(); + + // should be the last element of the last workgroup thread + // don't know what it's like if virtual workgroup size doesn't divide by workgroup size exactly + const uint32_t lastInvocIx = glsl::gl_SubgroupSize() * glsl::gl_NumSubgroups() - 1u; + scalar_t lastElem; + if (invocIx == lastInvocIx) + lastElem = wgDataAccessor.preloaded[wg_data_proxy_t::PreloadedDataCount-1u][Config::ItemsPerInvocation_0-1u]; + + if (Exclusive) + workgroup2::exclusive_scan::template __call(wgDataAccessor, scratchAccessor); + else + workgroup2::inclusive_scan::template __call(wgDataAccessor, scratchAccessor); + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + currGroupReduction = wgDataAccessor.preloaded[wg_data_proxy_t::PreloadedDataCount-1u][Config::ItemsPerInvocation_0-1u]; + if (Exclusive) + currGroupReduction = binop(currGroupReduction, lastElem); + if (invocIx == lastInvocIx) + scratchAccessor.template set(0u, currGroupReduction); + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + scratchAccessor.template get(0u, currGroupReduction); + } + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + if (!invocIx) + { + const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, workgroupId > 0u) | currGroupReduction << Flag_Shift; + workgroupReduction.atomicExchange(workgroupId, storeVal); + } + + if (workgroupId) + { + bool locked = sIsLocked; + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + scalar_t prevReduction = 0u; + uint16_t lookbackIx = workgroupId - uint16_t(1u); + + while (locked) + { + // lookback: try to get reduction from previous workgroups + if (!invocIx) + { + uint16_t spinCount = uint16_t(0u); + [loop] + while (spinCount < MaxSpinCount) + { + scalar_t flagPayload; + workgroupReduction.template get(lookbackIx, flagPayload); + + if ((flagPayload & Flag_Mask) > Flag_NotReady) + { + spinCount = uint16_t(0u); + prevReduction = binop(prevReduction, flagPayload >> Flag_Shift); + if ((flagPayload & Flag_Mask) == Flag_Inclusive) + { + const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); + workgroupReduction.atomicExchange(workgroupId, storeVal); + scratchAccessor.template set(0u, prevReduction); + sIsLocked = false; + break; + } + else + lookbackIx--; + } + else + spinCount++; + } + + // broadcast id and prepare to do reduction ourselves + if (spinCount == MaxSpinCount) + scratchAccessor.template set(0u, lookbackIx); + } + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + locked = sIsLocked; + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + if (locked) + { + // do reduction for lookbackIx workgroup + uint16_t fallbackGroupId; + scratchAccessor.template get(0u, fallbackGroupId); + + wg_data_proxy_t fallbackDataAccessor = wg_data_proxy_t::create(dataAccessor.getInputBufAddr(), dataAccessor.getOutputBufAddr(), fallbackGroupId); + fallbackDataAccessor.preload(); + scalar_t fallbackReduction = workgroup2::reduction::template __call(fallbackDataAccessor, scratchAccessor); + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + if (!invocIx) + { + const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, fallbackGroupId > 0u) | (fallbackReduction << Flag_Shift); + const scalar_t fallbackPayload = workgroupReduction.atomicMax(fallbackGroupId, storeVal); + + prevReduction = binop(prevReduction, hlsl::mix(fallbackReduction, fallbackPayload >> Flag_Shift, fallbackPayload > scalar_t(0.0))); + if (fallbackGroupId == 0u || (fallbackPayload & Flag_Mask) == Flag_Inclusive) + { + const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); + workgroupReduction.atomicExchange(workgroupId, storeVal); + scratchAccessor.template set(0u, prevReduction); + sIsLocked = false; + } + else + lookbackIx--; + } + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + locked = sIsLocked; + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + } + } + } + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + dataAccessor.initAtWorkgroupID(workgroupId); + + scalar_t prevReduction = 0u; + if (workgroupId) + scratchAccessor.template get(0u, prevReduction); + + NBL_UNROLL + for (uint16_t idx = 0; idx < wg_data_proxy_t::PreloadedDataCount; idx++) + { + vector_t data = wgDataAccessor.preloaded[idx]; + NBL_UNROLL + for (uint16_t i = 0; i < Config::ItemsPerInvocation_0; i++) + data[i] = binop(prevReduction, data[i]); + dataAccessor.template set(idx * WorkgroupSize + invocIx, data); + } + } +}; +} + +template) +struct inclusive_scan +{ + using scalar_t = typename BinOp::type_t; + + template && ArithmeticSharedMemoryAccessor && DeviceReductionsAccessor && WorkgroupCounterAccessor) + static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction, NBL_REF_ARG(WorkgroupCounter) counter) + { + impl::Scan fn; + fn.template __call(dataAccessor, scratchAccessor, workgroupReduction, counter); + } +}; + +template) +struct exclusive_scan +{ + using scalar_t = typename BinOp::type_t; + + template && ArithmeticSharedMemoryAccessor && DeviceReductionsAccessor && WorkgroupCounterAccessor) + static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction, NBL_REF_ARG(WorkgroupCounter) counter) + { + impl::Scan fn; + fn.template __call(dataAccessor, scratchAccessor, workgroupReduction, counter); + } +}; + +} +} +} + +#endif diff --git a/src/nbl/builtin/CMakeLists.txt b/src/nbl/builtin/CMakeLists.txt index df20f6b6b3..d6140e2625 100644 --- a/src/nbl/builtin/CMakeLists.txt +++ b/src/nbl/builtin/CMakeLists.txt @@ -362,6 +362,8 @@ LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/workgroup2/impl/items_per_inv LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/workgroup2/impl/arithmetic_config_def.hlsl") LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/workgroup2/arithmetic.hlsl") LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/workgroup2/shared_scan.hlsl") +#device scan +LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/scan/chained_scan.hlsl") #Extensions LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/ext/FullScreenTriangle/SVertexAttributes.hlsl") LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/ext/FullScreenTriangle/default.vert.hlsl") @@ -385,6 +387,7 @@ LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/concepts/accessors/storable_i LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/concepts/accessors/generic_shared_data.hlsl") LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/concepts/accessors/fft.hlsl") LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/concepts/accessors/workgroup_arithmetic.hlsl") +LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/concepts/accessors/device_arithmetic.hlsl") #tgmath LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/tgmath.hlsl") LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/tgmath/impl.hlsl")