Add the GPU rigid body pipeline from https://github.com/erwincoumans/experiments as a Bullet 3.x preview for Bullet 2.80
This commit is contained in:
erwin.coumans
@@ -0,0 +1,73 @@
|
||||
/*
|
||||
Copyright (c) 2012 Advanced Micro Devices, Inc.
|
||||
|
||||
This software is provided 'as-is', without any express or implied warranty.
|
||||
In no event will the authors be held liable for any damages arising from the use of this software.
|
||||
Permission is granted to anyone to use this software for any purpose,
|
||||
including commercial applications, and to alter it and redistribute it freely,
|
||||
subject to the following restrictions:
|
||||
|
||||
1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required.
|
||||
2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software.
|
||||
3. This notice may not be removed or altered from any source distribution.
|
||||
*/
|
||||
//Originally written by Takahiro Harada
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <Adl/Adl.h>
|
||||
#include <AdlPrimitives/Math/Math.h>
|
||||
|
||||
namespace adl
|
||||
{
|
||||
|
||||
class PrefixScanBase
|
||||
{
|
||||
public:
|
||||
enum Option
|
||||
{
|
||||
INCLUSIVE,
|
||||
EXCLUSIVE
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
template<DeviceType TYPE>
|
||||
class PrefixScan : public PrefixScanBase
|
||||
{
|
||||
public:
|
||||
typedef Launcher::BufferInfo BufferInfo;
|
||||
|
||||
enum
|
||||
{
|
||||
BLOCK_SIZE = 128
|
||||
};
|
||||
|
||||
struct Data
|
||||
{
|
||||
Option m_option;
|
||||
const Device* m_device;
|
||||
Kernel* m_localScanKernel;
|
||||
Kernel* m_blockSumKernel;
|
||||
Kernel* m_propagationKernel;
|
||||
Buffer<u32>* m_workBuffer;
|
||||
Buffer<int4>* m_constBuffer[3];// todo. dx need one for each
|
||||
int m_maxSize;
|
||||
};
|
||||
|
||||
static
|
||||
Data* allocate(const Device* deviceData, int maxSize, Option option = EXCLUSIVE);
|
||||
|
||||
static
|
||||
void deallocate(Data* data);
|
||||
|
||||
static
|
||||
void execute(Data* data, Buffer<u32>& src, Buffer<u32>& dst, int n, u32* sum = 0);
|
||||
};
|
||||
|
||||
|
||||
|
||||
#include <AdlPrimitives/Scan/PrefixScanHost.inl>
|
||||
#include <AdlPrimitives/Scan/PrefixScan.inl>
|
||||
|
||||
};
|
||||
@@ -0,0 +1,125 @@
|
||||
/*
|
||||
Copyright (c) 2012 Advanced Micro Devices, Inc.
|
||||
|
||||
This software is provided 'as-is', without any express or implied warranty.
|
||||
In no event will the authors be held liable for any damages arising from the use of this software.
|
||||
Permission is granted to anyone to use this software for any purpose,
|
||||
including commercial applications, and to alter it and redistribute it freely,
|
||||
subject to the following restrictions:
|
||||
|
||||
1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required.
|
||||
2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software.
|
||||
3. This notice may not be removed or altered from any source distribution.
|
||||
*/
|
||||
//Originally written by Takahiro Harada
|
||||
|
||||
|
||||
#define PATH "..\\..\\opencl\\primitives\\AdlPrimitives\\Scan\\PrefixScanKernels"
|
||||
#define KERNEL0 "LocalScanKernel"
|
||||
#define KERNEL1 "TopLevelScanKernel"
|
||||
#define KERNEL2 "AddOffsetKernel"
|
||||
|
||||
#include <AdlPrimitives/Scan/PrefixScanKernelsCL.h>
|
||||
#include <AdlPrimitives/Scan/PrefixScanKernelsDX11.h>
|
||||
|
||||
template<DeviceType TYPE>
|
||||
typename PrefixScan<TYPE>::Data* PrefixScan<TYPE>::allocate(const Device* device, int maxSize, Option option)
|
||||
{
|
||||
ADLASSERT( TYPE == device->m_type );
|
||||
|
||||
ADLASSERT( maxSize <= BLOCK_SIZE*2*2048 );
|
||||
|
||||
const char* src[] =
|
||||
#if defined(ADL_LOAD_KERNEL_FROM_STRING)
|
||||
{prefixScanKernelsCL, prefixScanKernelsDX11};
|
||||
#else
|
||||
{0,0};
|
||||
#endif
|
||||
Data* data = new Data;
|
||||
data->m_device = device;
|
||||
data->m_localScanKernel = device->getKernel( PATH, KERNEL0, 0, src[TYPE] );
|
||||
data->m_blockSumKernel = device->getKernel( PATH, KERNEL1, 0, src[TYPE] );
|
||||
data->m_propagationKernel = device->getKernel( PATH, KERNEL2, 0, src[TYPE] );
|
||||
|
||||
int bufSize = (NEXTMULTIPLEOF( max2( maxSize/BLOCK_SIZE, (int)BLOCK_SIZE ), BLOCK_SIZE )+1);
|
||||
data->m_workBuffer = new Buffer<u32>( device, bufSize );
|
||||
data->m_constBuffer[0] = new Buffer<int4>( device, 1, BufferBase::BUFFER_CONST );
|
||||
data->m_constBuffer[1] = new Buffer<int4>( device, 1, BufferBase::BUFFER_CONST );
|
||||
data->m_constBuffer[2] = new Buffer<int4>( device, 1, BufferBase::BUFFER_CONST );
|
||||
|
||||
data->m_maxSize = maxSize;
|
||||
data->m_option = option;
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
template<DeviceType TYPE>
|
||||
void PrefixScan<TYPE>::deallocate(Data* data)
|
||||
{
|
||||
delete data->m_workBuffer;
|
||||
delete data->m_constBuffer[0];
|
||||
delete data->m_constBuffer[1];
|
||||
delete data->m_constBuffer[2];
|
||||
delete data;
|
||||
}
|
||||
|
||||
template<DeviceType TYPE>
|
||||
void PrefixScan<TYPE>::execute(Data* data, Buffer<u32>& src, Buffer<u32>& dst, int n, u32* sum)
|
||||
{
|
||||
ADLASSERT( data );
|
||||
ADLASSERT( n <= data->m_maxSize );
|
||||
ADLASSERT( data->m_option == EXCLUSIVE );
|
||||
const u32 numBlocks = u32( (n+BLOCK_SIZE*2-1)/(BLOCK_SIZE*2) );
|
||||
|
||||
|
||||
int4 constBuffer;
|
||||
constBuffer.x = n;
|
||||
constBuffer.y = numBlocks;
|
||||
constBuffer.z = (int)nextPowerOf2( numBlocks );
|
||||
|
||||
Buffer<u32>* srcNative = BufferUtils::map<TYPE, true>( data->m_device, &src );
|
||||
Buffer<u32>* dstNative = BufferUtils::map<TYPE, false>( data->m_device, &dst );
|
||||
|
||||
{
|
||||
BufferInfo bInfo[] = { BufferInfo( dstNative ), BufferInfo( srcNative ), BufferInfo( data->m_workBuffer ) };
|
||||
|
||||
Launcher launcher( data->m_device, data->m_localScanKernel );
|
||||
launcher.setBuffers( bInfo, sizeof(bInfo)/sizeof(Launcher::BufferInfo) );
|
||||
launcher.setConst( *data->m_constBuffer[0], constBuffer );
|
||||
launcher.launch1D( numBlocks*BLOCK_SIZE, BLOCK_SIZE );
|
||||
}
|
||||
|
||||
{
|
||||
BufferInfo bInfo[] = { BufferInfo( data->m_workBuffer ) };
|
||||
|
||||
Launcher launcher( data->m_device, data->m_blockSumKernel );
|
||||
launcher.setBuffers( bInfo, sizeof(bInfo)/sizeof(Launcher::BufferInfo) );
|
||||
launcher.setConst( *data->m_constBuffer[1], constBuffer );
|
||||
launcher.launch1D( BLOCK_SIZE, BLOCK_SIZE );
|
||||
}
|
||||
|
||||
|
||||
if( numBlocks > 1 )
|
||||
{
|
||||
BufferInfo bInfo[] = { BufferInfo( dstNative ), BufferInfo( data->m_workBuffer ) };
|
||||
Launcher launcher( data->m_device, data->m_propagationKernel );
|
||||
launcher.setBuffers( bInfo, sizeof(bInfo)/sizeof(Launcher::BufferInfo) );
|
||||
launcher.setConst( *data->m_constBuffer[2], constBuffer );
|
||||
launcher.launch1D( (numBlocks-1)*BLOCK_SIZE, BLOCK_SIZE );
|
||||
}
|
||||
|
||||
DeviceUtils::waitForCompletion( data->m_device );
|
||||
if( sum )
|
||||
{
|
||||
dstNative->read( sum, 1, n-1);
|
||||
}
|
||||
DeviceUtils::waitForCompletion( data->m_device );
|
||||
|
||||
BufferUtils::unmap<false>( srcNative, &src );
|
||||
BufferUtils::unmap<true>( dstNative, &dst );
|
||||
}
|
||||
|
||||
#undef PATH
|
||||
#undef KERNEL0
|
||||
#undef KERNEL1
|
||||
#undef KERNEL2
|
||||
@@ -0,0 +1,74 @@
|
||||
/*
|
||||
Copyright (c) 2012 Advanced Micro Devices, Inc.
|
||||
|
||||
This software is provided 'as-is', without any express or implied warranty.
|
||||
In no event will the authors be held liable for any damages arising from the use of this software.
|
||||
Permission is granted to anyone to use this software for any purpose,
|
||||
including commercial applications, and to alter it and redistribute it freely,
|
||||
subject to the following restrictions:
|
||||
|
||||
1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required.
|
||||
2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software.
|
||||
3. This notice may not be removed or altered from any source distribution.
|
||||
*/
|
||||
//Originally written by Takahiro Harada
|
||||
|
||||
|
||||
template<>
|
||||
class PrefixScan<TYPE_HOST> : public PrefixScanBase
|
||||
{
|
||||
public:
|
||||
struct Data
|
||||
{
|
||||
Option m_option;
|
||||
};
|
||||
|
||||
static
|
||||
Data* allocate(const Device* deviceData, int maxSize, Option option = EXCLUSIVE)
|
||||
{
|
||||
ADLASSERT( deviceData->m_type == TYPE_HOST );
|
||||
|
||||
Data* data = new Data;
|
||||
data->m_option = option;
|
||||
return data;
|
||||
}
|
||||
|
||||
static
|
||||
void deallocate(Data* data)
|
||||
{
|
||||
delete data;
|
||||
}
|
||||
|
||||
static
|
||||
void execute(Data* data, Buffer<u32>& src, Buffer<u32>& dst, int n, u32* sum = 0)
|
||||
{
|
||||
ADLASSERT( src.getType() == TYPE_HOST && dst.getType() == TYPE_HOST );
|
||||
HostBuffer<u32>& hSrc = (HostBuffer<u32>&)src;
|
||||
HostBuffer<u32>& hDst = (HostBuffer<u32>&)dst;
|
||||
|
||||
u32 s = 0;
|
||||
if( data->m_option == EXCLUSIVE )
|
||||
{
|
||||
for(int i=0; i<n; i++)
|
||||
{
|
||||
hDst[i] = s;
|
||||
s += hSrc[i];
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
for(int i=0; i<n; i++)
|
||||
{
|
||||
s += hSrc[i];
|
||||
hDst[i] = s;
|
||||
}
|
||||
}
|
||||
|
||||
if( sum )
|
||||
{
|
||||
*sum = hDst[n-1];
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
};
|
||||
@@ -0,0 +1,153 @@
|
||||
/*
|
||||
Copyright (c) 2012 Advanced Micro Devices, Inc.
|
||||
|
||||
This software is provided 'as-is', without any express or implied warranty.
|
||||
In no event will the authors be held liable for any damages arising from the use of this software.
|
||||
Permission is granted to anyone to use this software for any purpose,
|
||||
including commercial applications, and to alter it and redistribute it freely,
|
||||
subject to the following restrictions:
|
||||
|
||||
1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required.
|
||||
2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software.
|
||||
3. This notice may not be removed or altered from any source distribution.
|
||||
*/
|
||||
//Originally written by Takahiro Harada
|
||||
|
||||
|
||||
typedef unsigned int u32;
|
||||
#define GET_GROUP_IDX get_group_id(0)
|
||||
#define GET_LOCAL_IDX get_local_id(0)
|
||||
#define GET_GLOBAL_IDX get_global_id(0)
|
||||
#define GET_GROUP_SIZE get_local_size(0)
|
||||
#define GROUP_LDS_BARRIER barrier(CLK_LOCAL_MEM_FENCE)
|
||||
|
||||
// takahiro end
|
||||
#define WG_SIZE 128
|
||||
|
||||
typedef struct
|
||||
{
|
||||
uint m_numElems;
|
||||
uint m_numBlocks;
|
||||
uint m_numScanBlocks;
|
||||
uint m_padding[1];
|
||||
} ConstBuffer;
|
||||
|
||||
|
||||
u32 ScanExclusive(__local u32* data, u32 n, int lIdx, int lSize)
|
||||
{
|
||||
u32 blocksum;
|
||||
int offset = 1;
|
||||
for(int nActive=n>>1; nActive>0; nActive>>=1, offset<<=1)
|
||||
{
|
||||
GROUP_LDS_BARRIER;
|
||||
for(int iIdx=lIdx; iIdx<nActive; iIdx+=lSize)
|
||||
{
|
||||
int ai = offset*(2*iIdx+1)-1;
|
||||
int bi = offset*(2*iIdx+2)-1;
|
||||
data[bi] += data[ai];
|
||||
}
|
||||
}
|
||||
|
||||
GROUP_LDS_BARRIER;
|
||||
|
||||
if( lIdx == 0 )
|
||||
{
|
||||
blocksum = data[ n-1 ];
|
||||
data[ n-1 ] = 0;
|
||||
}
|
||||
|
||||
GROUP_LDS_BARRIER;
|
||||
|
||||
offset >>= 1;
|
||||
for(int nActive=1; nActive<n; nActive<<=1, offset>>=1 )
|
||||
{
|
||||
GROUP_LDS_BARRIER;
|
||||
for( int iIdx = lIdx; iIdx<nActive; iIdx += lSize )
|
||||
{
|
||||
int ai = offset*(2*iIdx+1)-1;
|
||||
int bi = offset*(2*iIdx+2)-1;
|
||||
u32 temp = data[ai];
|
||||
data[ai] = data[bi];
|
||||
data[bi] += temp;
|
||||
}
|
||||
}
|
||||
GROUP_LDS_BARRIER;
|
||||
|
||||
return blocksum;
|
||||
}
|
||||
|
||||
|
||||
__attribute__((reqd_work_group_size(WG_SIZE,1,1)))
|
||||
__kernel
|
||||
void LocalScanKernel(__global u32* dst, __global u32 *src, __global u32 *sumBuffer,
|
||||
ConstBuffer cb)
|
||||
{
|
||||
__local u32 ldsData[WG_SIZE*2];
|
||||
|
||||
int gIdx = GET_GLOBAL_IDX;
|
||||
int lIdx = GET_LOCAL_IDX;
|
||||
|
||||
ldsData[2*lIdx] = ( 2*gIdx < cb.m_numElems )? src[2*gIdx]: 0;
|
||||
ldsData[2*lIdx + 1] = ( 2*gIdx+1 < cb.m_numElems )? src[2*gIdx + 1]: 0;
|
||||
|
||||
u32 sum = ScanExclusive(ldsData, WG_SIZE*2, GET_LOCAL_IDX, GET_GROUP_SIZE);
|
||||
|
||||
if( lIdx == 0 ) sumBuffer[GET_GROUP_IDX] = sum;
|
||||
|
||||
if( (2*gIdx) < cb.m_numElems )
|
||||
{
|
||||
dst[2*gIdx] = ldsData[2*lIdx];
|
||||
}
|
||||
if( (2*gIdx + 1) < cb.m_numElems )
|
||||
{
|
||||
dst[2*gIdx + 1] = ldsData[2*lIdx + 1];
|
||||
}
|
||||
}
|
||||
|
||||
__attribute__((reqd_work_group_size(WG_SIZE,1,1)))
|
||||
__kernel
|
||||
void AddOffsetKernel(__global u32 *dst, __global u32 *blockSum, ConstBuffer cb)
|
||||
{
|
||||
const u32 blockSize = WG_SIZE*2;
|
||||
|
||||
int myIdx = GET_GROUP_IDX+1;
|
||||
int lIdx = GET_LOCAL_IDX;
|
||||
|
||||
u32 iBlockSum = blockSum[myIdx];
|
||||
|
||||
int endValue = min((myIdx+1)*(blockSize), cb.m_numElems);
|
||||
for(int i=myIdx*blockSize+lIdx; i<endValue; i+=GET_GROUP_SIZE)
|
||||
{
|
||||
dst[i] += iBlockSum;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
__attribute__((reqd_work_group_size(WG_SIZE,1,1)))
|
||||
__kernel
|
||||
void TopLevelScanKernel(__global u32* dst, ConstBuffer cb)
|
||||
{
|
||||
__local u32 ldsData[2048];
|
||||
int gIdx = GET_GLOBAL_IDX;
|
||||
int lIdx = GET_LOCAL_IDX;
|
||||
int lSize = GET_GROUP_SIZE;
|
||||
|
||||
for(int i=lIdx; i<cb.m_numScanBlocks; i+=lSize )
|
||||
{
|
||||
ldsData[i] = (i<cb.m_numBlocks)? dst[i]:0;
|
||||
}
|
||||
|
||||
GROUP_LDS_BARRIER;
|
||||
|
||||
u32 sum = ScanExclusive(ldsData, cb.m_numScanBlocks, GET_LOCAL_IDX, GET_GROUP_SIZE);
|
||||
|
||||
for(int i=lIdx; i<cb.m_numBlocks; i+=lSize )
|
||||
{
|
||||
dst[i] = ldsData[i];
|
||||
}
|
||||
|
||||
if( gIdx == 0 )
|
||||
{
|
||||
dst[cb.m_numBlocks] = sum;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
/*
|
||||
Copyright (c) 2012 Advanced Micro Devices, Inc.
|
||||
|
||||
This software is provided 'as-is', without any express or implied warranty.
|
||||
In no event will the authors be held liable for any damages arising from the use of this software.
|
||||
Permission is granted to anyone to use this software for any purpose,
|
||||
including commercial applications, and to alter it and redistribute it freely,
|
||||
subject to the following restrictions:
|
||||
|
||||
1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required.
|
||||
2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software.
|
||||
3. This notice may not be removed or altered from any source distribution.
|
||||
*/
|
||||
//Originally written by Takahiro Harada
|
||||
|
||||
|
||||
typedef uint u32;
|
||||
|
||||
#define GET_GROUP_IDX groupIdx.x
|
||||
#define GET_LOCAL_IDX localIdx.x
|
||||
#define GET_GLOBAL_IDX globalIdx.x
|
||||
#define GROUP_LDS_BARRIER GroupMemoryBarrierWithGroupSync()
|
||||
|
||||
// takahiro end
|
||||
#define WG_SIZE 128
|
||||
|
||||
#define GET_GROUP_SIZE WG_SIZE
|
||||
|
||||
|
||||
cbuffer SortCB : register( b0 )
|
||||
{
|
||||
int m_numElems;
|
||||
int m_numBlocks;
|
||||
int m_numScanBlocks;
|
||||
};
|
||||
|
||||
RWStructuredBuffer<uint> dst : register( u0 );
|
||||
RWStructuredBuffer<uint> src : register( u1 );
|
||||
RWStructuredBuffer<uint> sumBuffer : register( u2 );
|
||||
|
||||
|
||||
groupshared u32 ldsData[2048];
|
||||
|
||||
u32 ScanExclusive(u32 n, int lIdx, int lSize)
|
||||
{
|
||||
u32 blocksum;
|
||||
int offset = 1;
|
||||
for(int nActive=n>>1; nActive>0; nActive>>=1, offset<<=1)
|
||||
{
|
||||
GROUP_LDS_BARRIER;
|
||||
for(int iIdx=lIdx; iIdx<nActive; iIdx+=lSize)
|
||||
{
|
||||
int ai = offset*(2*iIdx+1)-1;
|
||||
int bi = offset*(2*iIdx+2)-1;
|
||||
ldsData[bi] += ldsData[ai];
|
||||
}
|
||||
}
|
||||
|
||||
GROUP_LDS_BARRIER;
|
||||
|
||||
if( lIdx == 0 )
|
||||
{
|
||||
blocksum = ldsData[ n-1 ];
|
||||
ldsData[ n-1 ] = 0;
|
||||
}
|
||||
|
||||
GROUP_LDS_BARRIER;
|
||||
|
||||
offset >>= 1;
|
||||
for(int nActive=1; nActive<n; nActive<<=1, offset>>=1 )
|
||||
{
|
||||
GROUP_LDS_BARRIER;
|
||||
for( int iIdx = lIdx; iIdx<nActive; iIdx += lSize )
|
||||
{
|
||||
int ai = offset*(2*iIdx+1)-1;
|
||||
int bi = offset*(2*iIdx+2)-1;
|
||||
u32 temp = ldsData[ai];
|
||||
ldsData[ai] = ldsData[bi];
|
||||
ldsData[bi] += temp;
|
||||
}
|
||||
}
|
||||
GROUP_LDS_BARRIER;
|
||||
|
||||
return blocksum;
|
||||
}
|
||||
|
||||
[numthreads(WG_SIZE, 1, 1)]
|
||||
void LocalScanKernel(uint3 globalIdx : SV_DispatchThreadID, uint3 localIdx : SV_GroupThreadID, uint3 groupIdx : SV_GroupID)
|
||||
{
|
||||
int gIdx = GET_GLOBAL_IDX;
|
||||
int lIdx = GET_LOCAL_IDX;
|
||||
|
||||
ldsData[2*lIdx] = ( 2*gIdx < m_numElems )? src[2*gIdx]: 0;
|
||||
ldsData[2*lIdx + 1] = ( 2*gIdx+1 < m_numElems )? src[2*gIdx + 1]: 0;
|
||||
|
||||
u32 sum = ScanExclusive(WG_SIZE*2, GET_LOCAL_IDX, GET_GROUP_SIZE);
|
||||
|
||||
if( lIdx == 0 ) sumBuffer[GET_GROUP_IDX] = sum;
|
||||
|
||||
if( (2*gIdx) < m_numElems )
|
||||
{
|
||||
dst[2*gIdx] = ldsData[2*lIdx];
|
||||
}
|
||||
if( (2*gIdx + 1) < m_numElems )
|
||||
{
|
||||
dst[2*gIdx + 1] = ldsData[2*lIdx + 1];
|
||||
}
|
||||
}
|
||||
|
||||
[numthreads(WG_SIZE, 1, 1)]
|
||||
void TopLevelScanKernel(uint3 globalIdx : SV_DispatchThreadID, uint3 localIdx : SV_GroupThreadID, uint3 groupIdx : SV_GroupID)
|
||||
{
|
||||
int gIdx = GET_GLOBAL_IDX;
|
||||
int lIdx = GET_LOCAL_IDX;
|
||||
int lSize = GET_GROUP_SIZE;
|
||||
|
||||
for(int i=lIdx; i<m_numScanBlocks; i+=lSize )
|
||||
{
|
||||
ldsData[i] = (i<m_numBlocks)? dst[i]:0;
|
||||
}
|
||||
|
||||
GROUP_LDS_BARRIER;
|
||||
|
||||
u32 sum = ScanExclusive(m_numScanBlocks, GET_LOCAL_IDX, GET_GROUP_SIZE);
|
||||
|
||||
for(int i=lIdx; i<m_numBlocks; i+=lSize )
|
||||
{
|
||||
dst[i] = ldsData[i];
|
||||
}
|
||||
|
||||
if( gIdx == 0 )
|
||||
{
|
||||
dst[m_numBlocks] = sum;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
RWStructuredBuffer<uint> blockSum2 : register( u1 );
|
||||
|
||||
[numthreads(WG_SIZE, 1, 1)]
|
||||
void AddOffsetKernel(uint3 globalIdx : SV_DispatchThreadID, uint3 localIdx : SV_GroupThreadID, uint3 groupIdx : SV_GroupID)
|
||||
{
|
||||
const u32 blockSize = WG_SIZE*2;
|
||||
|
||||
int myIdx = GET_GROUP_IDX+1;
|
||||
int llIdx = GET_LOCAL_IDX;
|
||||
|
||||
u32 iBlockSum = blockSum2[myIdx];
|
||||
|
||||
int endValue = min((myIdx+1)*(blockSize), m_numElems);
|
||||
for(int i=myIdx*blockSize+llIdx; i<endValue; i+=GET_GROUP_SIZE)
|
||||
{
|
||||
dst[i] += iBlockSum;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
static const char* prefixScanKernelsCL= \
|
||||
"/*\n"
|
||||
" 2011 Takahiro Harada\n"
|
||||
"*/\n"
|
||||
"\n"
|
||||
"typedef unsigned int u32;\n"
|
||||
"#define GET_GROUP_IDX get_group_id(0)\n"
|
||||
"#define GET_LOCAL_IDX get_local_id(0)\n"
|
||||
"#define GET_GLOBAL_IDX get_global_id(0)\n"
|
||||
"#define GET_GROUP_SIZE get_local_size(0)\n"
|
||||
"#define GROUP_LDS_BARRIER barrier(CLK_LOCAL_MEM_FENCE)\n"
|
||||
"\n"
|
||||
"// takahiro end\n"
|
||||
"#define WG_SIZE 128\n"
|
||||
"\n"
|
||||
"typedef struct\n"
|
||||
"{\n"
|
||||
" uint m_numElems;\n"
|
||||
" uint m_numBlocks;\n"
|
||||
" uint m_numScanBlocks;\n"
|
||||
" uint m_padding[1];\n"
|
||||
"} ConstBuffer;\n"
|
||||
"\n"
|
||||
"\n"
|
||||
"u32 ScanExclusive(__local u32* data, u32 n, int lIdx, int lSize)\n"
|
||||
"{\n"
|
||||
" u32 blocksum;\n"
|
||||
" int offset = 1;\n"
|
||||
" for(int nActive=n>>1; nActive>0; nActive>>=1, offset<<=1)\n"
|
||||
" {\n"
|
||||
" GROUP_LDS_BARRIER;\n"
|
||||
" for(int iIdx=lIdx; iIdx<nActive; iIdx+=lSize)\n"
|
||||
" {\n"
|
||||
" int ai = offset*(2*iIdx+1)-1;\n"
|
||||
" int bi = offset*(2*iIdx+2)-1;\n"
|
||||
" data[bi] += data[ai];\n"
|
||||
" }\n"
|
||||
" }\n"
|
||||
"\n"
|
||||
" GROUP_LDS_BARRIER;\n"
|
||||
"\n"
|
||||
" if( lIdx == 0 )\n"
|
||||
" {\n"
|
||||
" blocksum = data[ n-1 ];\n"
|
||||
" data[ n-1 ] = 0;\n"
|
||||
" }\n"
|
||||
"\n"
|
||||
" GROUP_LDS_BARRIER;\n"
|
||||
"\n"
|
||||
" offset >>= 1;\n"
|
||||
" for(int nActive=1; nActive<n; nActive<<=1, offset>>=1 )\n"
|
||||
" {\n"
|
||||
" GROUP_LDS_BARRIER;\n"
|
||||
" for( int iIdx = lIdx; iIdx<nActive; iIdx += lSize )\n"
|
||||
" {\n"
|
||||
" int ai = offset*(2*iIdx+1)-1;\n"
|
||||
" int bi = offset*(2*iIdx+2)-1;\n"
|
||||
" u32 temp = data[ai];\n"
|
||||
" data[ai] = data[bi];\n"
|
||||
" data[bi] += temp;\n"
|
||||
" }\n"
|
||||
" }\n"
|
||||
" GROUP_LDS_BARRIER;\n"
|
||||
"\n"
|
||||
" return blocksum;\n"
|
||||
"}\n"
|
||||
"\n"
|
||||
"\n"
|
||||
"__attribute__((reqd_work_group_size(WG_SIZE,1,1)))\n"
|
||||
"__kernel\n"
|
||||
"void LocalScanKernel(__global u32* dst, __global u32 *src, __global u32 *sumBuffer,\n"
|
||||
" ConstBuffer cb)\n"
|
||||
"{\n"
|
||||
" __local u32 ldsData[WG_SIZE*2];\n"
|
||||
"\n"
|
||||
" int gIdx = GET_GLOBAL_IDX;\n"
|
||||
" int lIdx = GET_LOCAL_IDX;\n"
|
||||
"\n"
|
||||
" ldsData[2*lIdx] = ( 2*gIdx < cb.m_numElems )? src[2*gIdx]: 0;\n"
|
||||
" ldsData[2*lIdx + 1] = ( 2*gIdx+1 < cb.m_numElems )? src[2*gIdx + 1]: 0;\n"
|
||||
"\n"
|
||||
" u32 sum = ScanExclusive(ldsData, WG_SIZE*2, GET_LOCAL_IDX, GET_GROUP_SIZE);\n"
|
||||
"\n"
|
||||
" if( lIdx == 0 ) sumBuffer[GET_GROUP_IDX] = sum;\n"
|
||||
"\n"
|
||||
" if( (2*gIdx) < cb.m_numElems )\n"
|
||||
" {\n"
|
||||
" dst[2*gIdx] = ldsData[2*lIdx];\n"
|
||||
" }\n"
|
||||
" if( (2*gIdx + 1) < cb.m_numElems )\n"
|
||||
" {\n"
|
||||
" dst[2*gIdx + 1] = ldsData[2*lIdx + 1];\n"
|
||||
" }\n"
|
||||
"}\n"
|
||||
"\n"
|
||||
"__attribute__((reqd_work_group_size(WG_SIZE,1,1)))\n"
|
||||
"__kernel\n"
|
||||
"void AddOffsetKernel(__global u32 *dst, __global u32 *blockSum, ConstBuffer cb)\n"
|
||||
"{\n"
|
||||
" const u32 blockSize = WG_SIZE*2;\n"
|
||||
"\n"
|
||||
" int myIdx = GET_GROUP_IDX+1;\n"
|
||||
" int lIdx = GET_LOCAL_IDX;\n"
|
||||
"\n"
|
||||
" u32 iBlockSum = blockSum[myIdx];\n"
|
||||
"\n"
|
||||
" int endValue = min((myIdx+1)*(blockSize), cb.m_numElems);\n"
|
||||
" for(int i=myIdx*blockSize+lIdx; i<endValue; i+=GET_GROUP_SIZE)\n"
|
||||
" {\n"
|
||||
" dst[i] += iBlockSum;\n"
|
||||
" }\n"
|
||||
"}\n"
|
||||
"\n"
|
||||
"\n"
|
||||
"__attribute__((reqd_work_group_size(WG_SIZE,1,1)))\n"
|
||||
"__kernel\n"
|
||||
"void TopLevelScanKernel(__global u32* dst, ConstBuffer cb)\n"
|
||||
"{\n"
|
||||
" __local u32 ldsData[2048];\n"
|
||||
" int gIdx = GET_GLOBAL_IDX;\n"
|
||||
" int lIdx = GET_LOCAL_IDX;\n"
|
||||
" int lSize = GET_GROUP_SIZE;\n"
|
||||
"\n"
|
||||
" for(int i=lIdx; i<cb.m_numScanBlocks; i+=lSize )\n"
|
||||
" {\n"
|
||||
" ldsData[i] = (i<cb.m_numBlocks)? dst[i]:0;\n"
|
||||
" }\n"
|
||||
"\n"
|
||||
" GROUP_LDS_BARRIER;\n"
|
||||
"\n"
|
||||
" u32 sum = ScanExclusive(ldsData, cb.m_numScanBlocks, GET_LOCAL_IDX, GET_GROUP_SIZE);\n"
|
||||
"\n"
|
||||
" for(int i=lIdx; i<cb.m_numBlocks; i+=lSize )\n"
|
||||
" {\n"
|
||||
" dst[i] = ldsData[i];\n"
|
||||
" }\n"
|
||||
"\n"
|
||||
" if( gIdx == 0 )\n"
|
||||
" {\n"
|
||||
" dst[cb.m_numBlocks] = sum;\n"
|
||||
" }\n"
|
||||
"}\n"
|
||||
;
|
||||
@@ -0,0 +1,147 @@
|
||||
static const char* prefixScanKernelsDX11= \
|
||||
"/*\n"
|
||||
" 2011 Takahiro Harada\n"
|
||||
"*/\n"
|
||||
"\n"
|
||||
"typedef uint u32;\n"
|
||||
"\n"
|
||||
"#define GET_GROUP_IDX groupIdx.x\n"
|
||||
"#define GET_LOCAL_IDX localIdx.x\n"
|
||||
"#define GET_GLOBAL_IDX globalIdx.x\n"
|
||||
"#define GROUP_LDS_BARRIER GroupMemoryBarrierWithGroupSync()\n"
|
||||
"\n"
|
||||
"// takahiro end\n"
|
||||
"#define WG_SIZE 128\n"
|
||||
"\n"
|
||||
"#define GET_GROUP_SIZE WG_SIZE\n"
|
||||
"\n"
|
||||
"\n"
|
||||
"cbuffer SortCB : register( b0 )\n"
|
||||
"{\n"
|
||||
" int m_numElems;\n"
|
||||
" int m_numBlocks;\n"
|
||||
" int m_numScanBlocks;\n"
|
||||
"};\n"
|
||||
" \n"
|
||||
"RWStructuredBuffer<uint> dst : register( u0 );\n"
|
||||
"RWStructuredBuffer<uint> src : register( u1 );\n"
|
||||
"RWStructuredBuffer<uint> sumBuffer : register( u2 );\n"
|
||||
"\n"
|
||||
"\n"
|
||||
"groupshared u32 ldsData[2048];\n"
|
||||
"\n"
|
||||
"u32 ScanExclusive(u32 n, int lIdx, int lSize)\n"
|
||||
"{\n"
|
||||
" u32 blocksum;\n"
|
||||
" int offset = 1;\n"
|
||||
" for(int nActive=n>>1; nActive>0; nActive>>=1, offset<<=1)\n"
|
||||
" {\n"
|
||||
" GROUP_LDS_BARRIER;\n"
|
||||
" for(int iIdx=lIdx; iIdx<nActive; iIdx+=lSize)\n"
|
||||
" {\n"
|
||||
" int ai = offset*(2*iIdx+1)-1;\n"
|
||||
" int bi = offset*(2*iIdx+2)-1;\n"
|
||||
" ldsData[bi] += ldsData[ai];\n"
|
||||
" }\n"
|
||||
" }\n"
|
||||
"\n"
|
||||
" GROUP_LDS_BARRIER;\n"
|
||||
"\n"
|
||||
" if( lIdx == 0 )\n"
|
||||
" {\n"
|
||||
" blocksum = ldsData[ n-1 ];\n"
|
||||
" ldsData[ n-1 ] = 0;\n"
|
||||
" }\n"
|
||||
"\n"
|
||||
" GROUP_LDS_BARRIER;\n"
|
||||
"\n"
|
||||
" offset >>= 1;\n"
|
||||
" for(int nActive=1; nActive<n; nActive<<=1, offset>>=1 )\n"
|
||||
" {\n"
|
||||
" GROUP_LDS_BARRIER;\n"
|
||||
" for( int iIdx = lIdx; iIdx<nActive; iIdx += lSize )\n"
|
||||
" {\n"
|
||||
" int ai = offset*(2*iIdx+1)-1;\n"
|
||||
" int bi = offset*(2*iIdx+2)-1;\n"
|
||||
" u32 temp = ldsData[ai];\n"
|
||||
" ldsData[ai] = ldsData[bi];\n"
|
||||
" ldsData[bi] += temp;\n"
|
||||
" }\n"
|
||||
" }\n"
|
||||
" GROUP_LDS_BARRIER;\n"
|
||||
"\n"
|
||||
" return blocksum;\n"
|
||||
"}\n"
|
||||
"\n"
|
||||
"[numthreads(WG_SIZE, 1, 1)]\n"
|
||||
"void LocalScanKernel(uint3 globalIdx : SV_DispatchThreadID, uint3 localIdx : SV_GroupThreadID, uint3 groupIdx : SV_GroupID)\n"
|
||||
"{\n"
|
||||
" int gIdx = GET_GLOBAL_IDX;\n"
|
||||
" int lIdx = GET_LOCAL_IDX;\n"
|
||||
"\n"
|
||||
" ldsData[2*lIdx] = ( 2*gIdx < m_numElems )? src[2*gIdx]: 0;\n"
|
||||
" ldsData[2*lIdx + 1] = ( 2*gIdx+1 < m_numElems )? src[2*gIdx + 1]: 0;\n"
|
||||
"\n"
|
||||
" u32 sum = ScanExclusive(WG_SIZE*2, GET_LOCAL_IDX, GET_GROUP_SIZE);\n"
|
||||
"\n"
|
||||
" if( lIdx == 0 ) sumBuffer[GET_GROUP_IDX] = sum;\n"
|
||||
"\n"
|
||||
" if( (2*gIdx) < m_numElems )\n"
|
||||
" {\n"
|
||||
" dst[2*gIdx] = ldsData[2*lIdx];\n"
|
||||
" }\n"
|
||||
" if( (2*gIdx + 1) < m_numElems )\n"
|
||||
" {\n"
|
||||
" dst[2*gIdx + 1] = ldsData[2*lIdx + 1];\n"
|
||||
" }\n"
|
||||
"}\n"
|
||||
"\n"
|
||||
"[numthreads(WG_SIZE, 1, 1)]\n"
|
||||
"void TopLevelScanKernel(uint3 globalIdx : SV_DispatchThreadID, uint3 localIdx : SV_GroupThreadID, uint3 groupIdx : SV_GroupID)\n"
|
||||
"{\n"
|
||||
" int gIdx = GET_GLOBAL_IDX;\n"
|
||||
" int lIdx = GET_LOCAL_IDX;\n"
|
||||
" int lSize = GET_GROUP_SIZE;\n"
|
||||
"\n"
|
||||
" for(int i=lIdx; i<m_numScanBlocks; i+=lSize )\n"
|
||||
" {\n"
|
||||
" ldsData[i] = (i<m_numBlocks)? dst[i]:0;\n"
|
||||
" }\n"
|
||||
"\n"
|
||||
" GROUP_LDS_BARRIER;\n"
|
||||
"\n"
|
||||
" u32 sum = ScanExclusive(m_numScanBlocks, GET_LOCAL_IDX, GET_GROUP_SIZE);\n"
|
||||
"\n"
|
||||
" for(int i=lIdx; i<m_numBlocks; i+=lSize )\n"
|
||||
" {\n"
|
||||
" dst[i] = ldsData[i];\n"
|
||||
" }\n"
|
||||
"\n"
|
||||
" if( gIdx == 0 )\n"
|
||||
" {\n"
|
||||
" dst[m_numBlocks] = sum;\n"
|
||||
" }\n"
|
||||
"}\n"
|
||||
"\n"
|
||||
"\n"
|
||||
" \n"
|
||||
"RWStructuredBuffer<uint> blockSum2 : register( u1 );\n"
|
||||
"\n"
|
||||
"[numthreads(WG_SIZE, 1, 1)]\n"
|
||||
"void AddOffsetKernel(uint3 globalIdx : SV_DispatchThreadID, uint3 localIdx : SV_GroupThreadID, uint3 groupIdx : SV_GroupID)\n"
|
||||
"{\n"
|
||||
" const u32 blockSize = WG_SIZE*2;\n"
|
||||
"\n"
|
||||
" int myIdx = GET_GROUP_IDX+1;\n"
|
||||
" int llIdx = GET_LOCAL_IDX;\n"
|
||||
"\n"
|
||||
" u32 iBlockSum = blockSum2[myIdx];\n"
|
||||
"\n"
|
||||
" int endValue = min((myIdx+1)*(blockSize), m_numElems);\n"
|
||||
" for(int i=myIdx*blockSize+llIdx; i<endValue; i+=GET_GROUP_SIZE)\n"
|
||||
" {\n"
|
||||
" dst[i] += iBlockSum;\n"
|
||||
" }\n"
|
||||
"}\n"
|
||||
"\n"
|
||||
;
|
||||
Reference in New Issue
Block a user