How can I write a D3D12 texture in cuda?

Question

One point about such pipeline is that everything happens in GPU and bypass the CPU.

The texture is created as such:

ThrowIfFailed(m_device->CreateCommittedResource(&CD3DX12_HEAP_PROPERTIES(D3D12_HEAP_TYPE_DEFAULT), D3D12_HEAP_FLAG_SHARED,
            &texDesc, D3D12_RESOURCE_STATE_PIXEL_SHADER_RESOURCE, nullptr, IID_PPV_ARGS(&TextureArray)));
// texture pixel format: DXGI_FORMAT_R32G32B32A32_FLOAT

How can I access in read+write to TextureArray (ComPtr<ID3D12Resource> TextureArray;) in cuda ?

Answer 1

When creating the resources, one can export and map the texture resource to a cuda surface, subsequently available for direct read and write. The export and map needs only to happen once. The key steps are the following:

HANDLE sharedHandle{};
WindowsSecurityAttributes secAttr{};
ThrowIfFailed(m_device->CreateSharedHandle(TextureArray.Get(), &secAttr, GENERIC_ALL, 0, &sharedHandle));
const auto texAllocInfo = m_device->GetResourceAllocationInfo(m_nodeMask, 1, &texDesc);

cudaExternalMemoryHandleDesc cuExtmemHandleDesc{};
cuExtmemHandleDesc.type = cudaExternalMemoryHandleTypeD3D12Heap;
cuExtmemHandleDesc.handle.win32.handle = sharedHandle;
cuExtmemHandleDesc.size = texAllocInfo.SizeInBytes;
cuExtmemHandleDesc.flags = cudaExternalMemoryDedicated;
CheckCudaErrors(cudaImportExternalMemory(&m_externalMemory, &cuExtmemHandleDesc));

cudaExternalMemoryMipmappedArrayDesc cuExtmemMipDesc{};
cuExtmemMipDesc.extent = make_cudaExtent(texDesc.Width, texDesc.Height, 0);
cuExtmemMipDesc.formatDesc = cudaCreateChannelDesc<float4>();
cuExtmemMipDesc.numLevels = 1;
cuExtmemMipDesc.flags = cudaArraySurfaceLoadStore;

cudaMipmappedArray_t cuMipArray{};
CheckCudaErrors(cudaExternalMemoryGetMappedMipmappedArray(&cuMipArray, m_externalMemory, &cuExtmemMipDesc));

cudaArray_t cuArray{};
CheckCudaErrors(cudaGetMipmappedArrayLevel(&cuArray, cuMipArray, 0));

cudaResourceDesc cuResDesc{};
cuResDesc.resType = cudaResourceTypeArray;
cuResDesc.res.array.array = cuArray;
checkCudaErrors(cudaCreateSurfaceObject(&cuSurface, &cuResDesc));
// where: cudaSurfaceObject_t cuSurface{};

The cuda kernel can then be launched to write the texture. A semaphore also need to be created and shared:

ThrowIfFailed(m_device->CreateFence(m_fenceValues[m_frameIndex], D3D12_FENCE_FLAG_SHARED, IID_PPV_ARGS(&m_fence)));
NAME_D3D12_OBJECT(m_fence);

cudaExternalSemaphoreHandleDesc externalSemaphoreHandleDesc{};
memset(&externalSemaphoreHandleDesc, 0, sizeof(externalSemaphoreHandleDesc));
WindowsSecurityAttributes windowsSecurityAttributes;
LPCWSTR name{};
HANDLE sharedHandle{};
externalSemaphoreHandleDesc.type = cudaExternalSemaphoreHandleTypeD3D12Fence;
m_device->CreateSharedHandle(m_fence.Get(), &windowsSecurityAttributes, GENERIC_ALL, name, &sharedHandle);
externalSemaphoreHandleDesc.handle.win32.handle = sharedHandle;
externalSemaphoreHandleDesc.flags = 0;
CheckCudaErrors(cudaImportExternalSemaphore(&m_externalSemaphore, &externalSemaphoreHandleDesc));
m_fenceValues[m_frameIndex]++;

m_fenceEvent = CreateEvent(nullptr, false, false, nullptr);
if (m_fenceEvent == nullptr)
    ThrowIfFailed(HRESULT_FROM_WIN32(GetLastError()));
WaitForGpu();

A full demonstration of it is available at https://github.com/mprevot/CudaD3D12Update