Texture does not draw with ATI Radeon RX, but does with NVIDIA/no GPU

Question

I am getting a problem when trying to draw a texture on the following hardware (configuration 1):

ATi Radeon RX 6600 XT

Screen (OS Windows 11):

I have other hardware (configuration 2):

nVidia Geforce GTX950M

And it has no problems. Screen (OS Windows 10):

My code (I made a minimal example):

#include <Windows.h>

#include "Resource.h"

#include <d3d9.h>
#include <d3dx9math.h>
#include <dxerr9.h>

#include "stb_image.h"
#include <cassert>
#include <fstream>
#include <vector>

typedef intptr_t HTEXTURE;

struct vertex
{
    vertex() {}
    vertex(float x, float y, float z, DWORD col, float tx, float ty, float tx2, float ty2) :x(x), y(y), z(z), col(col), tx(tx), ty(ty),
        tx2(tx2), ty2(ty2) {}
    float           x, y;   
    float           z;          
    DWORD           col;        
    float           tx, ty;     
    float           tx2, ty2;
};

struct Quad
{
    vertex      v[4];
    HTEXTURE        tex;
    int             blend;
};


#define D3DFVF_HGEVERTEX (D3DFVF_XYZ | D3DFVF_DIFFUSE | D3DFVF_TEX1 | D3DFVF_TEX2)          
#define D3DFVF_HGE_MASKEDVERTEX (D3DFVF_XYZ | D3DFVF_DIFFUSE | D3DFVF_TEX1 | D3DFVF_TEX2)   
#define VERTEX_BUFFER_SIZE 4000
#define BLEND_COLORMUL      0
#define BLEND_ALPHABLEND    2
#define BLEND_NOZWRITE      0
#define BLEND_DEFAULT       (BLEND_COLORMUL | BLEND_ALPHABLEND | BLEND_NOZWRITE)

HINSTANCE hInst;
WCHAR szTitle[] = L"test";
WCHAR szWindowClass[] = L"test_win";
HWND hWnd;
IDirect3D9* pD3D;
IDirect3DDevice9* pD3DDevice;
D3DPRESENT_PARAMETERS* d3dpp;
D3DPRESENT_PARAMETERS                       d3dppW;
D3DPRESENT_PARAMETERS                       d3dppFS;
int                                         nScreenBPP;
D3DXMATRIX                                  matProj;
D3DXMATRIX                                  matView;
vertex* VertArray;
IDirect3DVertexBuffer9* pVB;
IDirect3DIndexBuffer9* pIB;
int                                         nPrim = 0;
D3DVIEWPORT9                                rCurrentViewPort;
RECT                    rectW;
LONG                    styleW;
Quad quad;
IDirect3DTexture9* pD3DXTex = 0;

BOOL                InitWin(HINSTANCE, int);
LRESULT CALLBACK    WndProc(HWND, UINT, WPARAM, LPARAM);


int _format_id(D3DFORMAT fmt)
{
    switch (fmt)
    {
    case D3DFMT_R5G6B5:     return 1;
    case D3DFMT_X1R5G5B5:   return 2;
    case D3DFMT_A1R5G5B5:   return 3;
    case D3DFMT_X8R8G8B8:   return 4;
    case D3DFMT_A8R8G8B8:   return 5;
    default:                return 0;
    }
}

void _SetProjectionMatrix(int width, int height)
{
    D3DXMATRIX tmp;
    D3DXMatrixScaling(&matProj, 1.0f, -1.0f, 1.0f);
    D3DXMatrixTranslation(&tmp, -0.5f, height + 0.5f, 0.0f);
    D3DXMatrixMultiply(&matProj, &matProj, &tmp);
    D3DXMatrixOrthoOffCenterLH(&tmp, 0.f, (float)width, 0, (float)height, 0.0f, 1.0f);
    D3DXMatrixMultiply(&matProj, &matProj, &tmp);
}

bool initD3d()
{
    static const char* szFormats[] = { "UNKNOWN", "R5G6B5", "X1R5G5B5", "A1R5G5B5", "X8R8G8B8", "A8R8G8B8" };
    D3DDISPLAYMODE          Mode;
    D3DFORMAT               Format = D3DFMT_UNKNOWN;
    UINT nModes, i;

    pD3D = Direct3DCreate9(D3D_SDK_VERSION); 

    pD3D->GetAdapterDisplayMode(D3DADAPTER_DEFAULT, &Mode);

    ZeroMemory(&d3dppW, sizeof(d3dppW));

    d3dppW.BackBufferWidth = 1366;
    d3dppW.BackBufferHeight = 768;
    d3dppW.BackBufferFormat = Mode.Format;
    d3dppW.BackBufferCount = 1;
    d3dppW.MultiSampleType = D3DMULTISAMPLE_NONE;
    d3dppW.hDeviceWindow = hWnd;
    d3dppW.Windowed = TRUE;

    d3dppW.SwapEffect = D3DSWAPEFFECT_DISCARD;
    d3dppW.PresentationInterval = D3DPRESENT_INTERVAL_ONE;

    nModes = pD3D->GetAdapterModeCount(D3DADAPTER_DEFAULT, Mode.Format);

    int nFullscreenWidth = GetSystemMetrics(SM_CXSCREEN);
    int nFullscreenHeight = GetSystemMetrics(SM_CYSCREEN);

    for (i = 0; i < nModes; ++i)
    {
        pD3D->EnumAdapterModes(D3DADAPTER_DEFAULT, Mode.Format, i, &Mode);

        if (Mode.Width != (UINT)nFullscreenWidth || Mode.Height != (UINT)nFullscreenHeight)
            continue;

        if (nScreenBPP == 16 && (_format_id(Mode.Format) > _format_id(D3DFMT_A1R5G5B5)))
            continue;

        if (_format_id(Mode.Format) > _format_id(Format))
            Format = Mode.Format;
    }

    if (Format == D3DFMT_UNKNOWN)
    {
        return false;
    }

    ZeroMemory(&d3dppFS, sizeof(d3dppFS));

    d3dppFS.BackBufferWidth = 1366;
    d3dppFS.BackBufferHeight = 768;
    d3dppFS.BackBufferFormat = Mode.Format;
    d3dppFS.BackBufferCount = 1;
    d3dppFS.MultiSampleType = D3DMULTISAMPLE_NONE;
    d3dppFS.hDeviceWindow = hWnd;
    d3dppFS.Windowed = TRUE;
    d3dppFS.SwapEffect = D3DSWAPEFFECT_DISCARD;
    d3dppFS.FullScreen_RefreshRateInHz = D3DPRESENT_RATE_DEFAULT;

    d3dppFS.PresentationInterval = D3DPRESENT_INTERVAL_ONE;

    d3dpp = true ? &d3dppW : &d3dppFS;

    if (_format_id(d3dpp->BackBufferFormat) < 4)
        nScreenBPP = 16;
    else
        nScreenBPP = 32;

    // Create D3D Device
    UINT       nAdapter = D3DADAPTER_DEFAULT;
    D3DDEVTYPE eDeviceType = D3DDEVTYPE_HAL;

    D3DCAPS9 caps;
    pD3D->GetDeviceCaps(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, &caps);

    DWORD dwVP;
    if ((caps.VertexShaderVersion < D3DVS_VERSION(1, 1)) || !(caps.DevCaps & D3DDEVCAPS_HWTRANSFORMANDLIGHT))
    {
        dwVP = D3DCREATE_SOFTWARE_VERTEXPROCESSING | D3DCREATE_MULTITHREADED;
    }
    else
    {
        dwVP = D3DCREATE_HARDWARE_VERTEXPROCESSING | D3DCREATE_MULTITHREADED;
    }

    dwVP = D3DCREATE_SOFTWARE_VERTEXPROCESSING | D3DCREATE_MULTITHREADED;

    pD3D->CreateDevice(nAdapter, eDeviceType, hWnd, dwVP, d3dpp, &pD3DDevice);

    _SetProjectionMatrix(1366, 768);
    D3DXMatrixIdentity(&matView);

    return true;
}

IDirect3DTexture9* loadTexture(IDirect3DTexture9* texture, const std::string& path)
{
    if (!pD3DDevice)
        return nullptr;

    bool bResult = false;

    DWORD               dwSizeFile = 0;
    D3DFORMAT           fmt1;
    D3DFORMAT           fmt2;

    std::ifstream file(path, std::ios::binary | std::ios::ate);
    dwSizeFile = file.tellg();
    file.seekg(0, std::ios::beg);
    std::vector<char> pDataTex(dwSizeFile);
    if (!file.read(pDataTex.data(), dwSizeFile))
        return nullptr;

    fmt1 = D3DFMT_A8R8G8B8;
    fmt2 = D3DFMT_UNKNOWN;

    int nWidth, nHeight, nBPP;
    int nRet = stbi_info_from_memory((stbi_uc*)pDataTex.data(), dwSizeFile, &nWidth, &nHeight, &nBPP);

    HRESULT hr = D3DXCreateTextureFromFileInMemoryEx(pD3DDevice, pDataTex.data(), dwSizeFile,
        D3DX_DEFAULT_NONPOW2, D3DX_DEFAULT_NONPOW2,
        0,  // Mip levels
        0,                              // Usage
        fmt1,                           // Format
        D3DPOOL_MANAGED,                // Memory pool
        D3DX_FILTER_NONE,               // Filter
        D3DX_DEFAULT,                   // Mip filter
        0,                              // Color key
        NULL, NULL, &texture);

    return texture;
}

bool initQuad()
{
    quad.v[0] = vertex(0, 0, 1, -1, 0, 0, 0, 0);
    quad.v[1] = vertex(1100, 0, 1, -1, 1, 0, 0, 0);
    quad.v[2] = vertex(1100, 650, 1, -1, 1, 1, 0, 0);
    quad.v[3] = vertex(0, 650, 1, -1, 0, 1, 0, 0);
    quad.tex = -1;
    quad.blend = 2;

    return true;
}

bool initGeometry()
{
    VertArray = 0;

    pD3DDevice->CreateVertexBuffer(VERTEX_BUFFER_SIZE * sizeof(vertex), D3DUSAGE_DYNAMIC | D3DUSAGE_WRITEONLY, D3DFVF_HGEVERTEX, D3DPOOL_DEFAULT, &pVB, NULL);

    pD3DDevice->SetVertexShader(NULL);
    pD3DDevice->SetFVF(D3DFVF_HGEVERTEX);
    pD3DDevice->SetStreamSource(0, pVB, 0, sizeof(vertex));

    pD3DDevice->CreateIndexBuffer(VERTEX_BUFFER_SIZE * 6 / 4 * sizeof(WORD), D3DUSAGE_WRITEONLY, D3DFMT_INDEX16, D3DPOOL_DEFAULT, &pIB, NULL);

    WORD* pIndices, n = 0;
    if (FAILED(pIB->Lock(0, 0, (void**)&pIndices, 0)))
    {
        return false;
    }

    for (int i = 0; i < VERTEX_BUFFER_SIZE / 4; i++)
    {
        *pIndices++ = n;
        *pIndices++ = n + 1;
        *pIndices++ = n + 2;
        *pIndices++ = n + 2;
        *pIndices++ = n + 3;
        *pIndices++ = n;
        n += 4;
    }

    pIB->Unlock();
    pD3DDevice->SetIndices(pIB);

    pD3DDevice->SetRenderState(D3DRS_CULLMODE, D3DCULL_NONE);
    pD3DDevice->SetRenderState(D3DRS_LIGHTING, FALSE);

    pD3DDevice->SetRenderState(D3DRS_ALPHABLENDENABLE, TRUE);
    pD3DDevice->SetRenderState(D3DRS_SRCBLEND, D3DBLEND_SRCALPHA);
    pD3DDevice->SetRenderState(D3DRS_DESTBLEND, D3DBLEND_INVSRCALPHA);

    pD3DDevice->SetRenderState(D3DRS_ALPHATESTENABLE, FALSE);
    pD3DDevice->SetRenderState(D3DRS_ALPHAREF, 0x01);
    pD3DDevice->SetRenderState(D3DRS_ALPHAFUNC, D3DCMP_GREATEREQUAL);

    pD3DDevice->SetTextureStageState(0, D3DTSS_COLOROP, D3DTOP_MODULATE);
    pD3DDevice->SetTextureStageState(0, D3DTSS_COLORARG1, D3DTA_TEXTURE);
    pD3DDevice->SetTextureStageState(0, D3DTSS_COLORARG2, D3DTA_DIFFUSE);
    pD3DDevice->SetTextureStageState(0, D3DTSS_ALPHAOP, D3DTOP_MODULATE);
    pD3DDevice->SetTextureStageState(0, D3DTSS_ALPHAARG1, D3DTA_TEXTURE);
    pD3DDevice->SetTextureStageState(0, D3DTSS_ALPHAARG2, D3DTA_DIFFUSE);

    pD3DDevice->SetTextureStageState(1, D3DTSS_COLOROP, D3DTOP_SELECTARG1);
    pD3DDevice->SetTextureStageState(1, D3DTSS_COLORARG1, D3DTA_CURRENT);
    pD3DDevice->SetTextureStageState(1, D3DTSS_ALPHAOP, D3DTOP_MODULATE); // TODO: problem
    pD3DDevice->SetTextureStageState(1, D3DTSS_ALPHAARG1, D3DTA_TEXTURE);
    pD3DDevice->SetTextureStageState(1, D3DTSS_ALPHAARG2, D3DTA_CURRENT);
    pD3DDevice->SetTextureStageState(1, D3DTSS_TEXTURETRANSFORMFLAGS, D3DTTFF_COUNT2);

    pD3DDevice->SetSamplerState(0, D3DSAMP_MIPFILTER, D3DTEXF_POINT);

    pD3DDevice->SetSamplerState(0, D3DSAMP_MAGFILTER, D3DTEXF_LINEAR);
    pD3DDevice->SetSamplerState(0, D3DSAMP_MINFILTER, D3DTEXF_LINEAR);

    pD3DDevice->SetSamplerState(0, D3DSAMP_ADDRESSU, D3DTADDRESS_CLAMP);
    pD3DDevice->SetSamplerState(0, D3DSAMP_ADDRESSV, D3DTADDRESS_CLAMP);
    pD3DDevice->SetSamplerState(0, D3DSAMP_ADDRESSW, D3DTADDRESS_CLAMP);

    pD3DDevice->SetSamplerState(1, D3DSAMP_ADDRESSU, D3DTADDRESS_BORDER);
    pD3DDevice->SetSamplerState(1, D3DSAMP_ADDRESSV, D3DTADDRESS_BORDER);
    pD3DDevice->SetSamplerState(1, D3DSAMP_BORDERCOLOR, D3DCOLOR_ARGB(0x0, 0, 0, 0));

    pD3DDevice->SetTransform(D3DTS_VIEW, &matView);
    pD3DDevice->SetTransform(D3DTS_PROJECTION, &matProj);

    D3DVIEWPORT9 vp;
    HRESULT hr = pD3DDevice->GetViewport(&vp);
    if (hr == D3D_OK)
        rCurrentViewPort = vp;

    pD3DXTex = loadTexture(pD3DXTex, "Tiger-PNG-Image.png");
    initQuad();

    return true;
}

void SetTexture()
{
    pD3DDevice->SetTexture(0, pD3DXTex);
}

bool beginScene()
{
    pD3DDevice->BeginScene();
    pVB->Lock(0, 0, (VOID**)&VertArray, D3DLOCK_DISCARD);
    return true;
}

void endScene()
{
    pVB->Unlock();
    VertArray = 0;
    nPrim = 0;
    pD3DDevice->EndScene();

    pD3DDevice->Present(NULL, NULL, NULL, NULL);
}

void clearScene()
{
    int color = D3DCOLOR_ARGB(255, 0, 125, 0);
    pD3DDevice->Clear(0, NULL, D3DCLEAR_TARGET, color, 1.0f, 0);
}

void renderQuad(Quad* quad)
{
    SetTexture();
    memcpy(&VertArray[nPrim * 4], quad->v, sizeof(vertex) * 4);
    nPrim++;
    pD3DDevice->DrawIndexedPrimitive(D3DPT_TRIANGLELIST, 0, 0, nPrim << 2, 0, nPrim << 1);
}

void _UpdateRects()
{
    int width = 1366;
    int height = 768;

    rectW.left = (GetSystemMetrics(SM_CXSCREEN) - width) / 2;
    rectW.top = (GetSystemMetrics(SM_CYSCREEN) - height) / 2;
    rectW.right = rectW.left + width;
    rectW.bottom = rectW.top + height;
    styleW = WS_POPUP | WS_CAPTION | WS_SYSMENU | WS_MINIMIZEBOX | WS_VISIBLE;
    AdjustWindowRect(&rectW, styleW, FALSE);

}

BOOL InitWin(HINSTANCE hInstance, int nCmdShow)
{
    WNDCLASSEXW wcex;

    wcex.cbSize = sizeof(WNDCLASSEX);

    wcex.style = CS_DBLCLKS | CS_OWNDC | CS_HREDRAW | CS_VREDRAW;
    wcex.lpfnWndProc = WndProc;
    wcex.cbClsExtra = 0;
    wcex.cbWndExtra = 0;
    wcex.hInstance = hInstance;
    wcex.hIcon = LoadIcon(hInstance, MAKEINTRESOURCE(IDI_TESTRXBUG));
    wcex.hCursor = LoadCursor(nullptr, IDC_ARROW);
    wcex.hbrBackground = (HBRUSH)GetStockObject(BLACK_BRUSH);
    wcex.lpszMenuName = NULL;
    wcex.lpszClassName = szWindowClass;
    wcex.hIconSm = LoadIcon(wcex.hInstance, MAKEINTRESOURCE(IDI_SMALL));

    RegisterClassExW(&wcex);

    _UpdateRects();
    hWnd = CreateWindowEx(0, szWindowClass, szTitle, styleW,
        rectW.left, rectW.top, rectW.right - rectW.left, rectW.bottom - rectW.top,
        NULL, NULL, hInstance, NULL);

    if (!hWnd)
        return FALSE;

    ShowWindow(hWnd, nCmdShow);

    return TRUE;
}

LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam)
{
    switch (message)
    {
    case WM_DESTROY:
        PostQuitMessage(0);
        break;
    default:
        return DefWindowProc(hWnd, message, wParam, lParam);
    }
    return 0;
}


int APIENTRY wWinMain(_In_ HINSTANCE hInstance,
    _In_opt_ HINSTANCE hPrevInstance,
    _In_ LPWSTR    lpCmdLine,
    _In_ int       nCmdShow)
{
    if (!InitWin(hInstance, nCmdShow))
        return FALSE;


    MSG msg = { 0 };

    initD3d();
    initGeometry();

    while (msg.message != WM_QUIT)
    {
        if (PeekMessage(&msg, NULL, 0, 0, PM_NOREMOVE))
        {
            if (GetMessage(&msg, NULL, 0, 0))
            {
                TranslateMessage(&msg);
                DispatchMessage(&msg);
            }
        }
        else
        {
            clearScene();
            beginScene();
            renderQuad(&quad);
            endScene();
        }
    }

    return (int)msg.wParam;
}

The problem is on this line:

pD3DDevice->SetTextureStageState(1, D3DTSS_ALPHAOP, D3DTOP_MODULATE); // TODO: problem

Also, if I set another texture for rendering at index 1, then everything will work (not correctly, but the rendering will work):

pD3DDevice->SetTexture(0, pD3DXTex);
pD3DDevice->SetTexture(1, pD3DXTex); // this

How to fix it?

P.S.: I tested on a Windows 11 virtual machine with configuration 1, but without a video card - everything worked correctly.

Answer 1

I can see that you're checking the HRESULTs returned from D3D functions in some cases (but not all of them), and where you're checking them you just silently handle the error. I'd suggest adding debug logging and/or assert()s for every D3D function call that returns an HRESULT.

Ideally you'd also enable the D3D9 debug runtime using the DirectX Control Panel, and see what errors get reported with it enabled. In my experience, if you're doing something that works on one GPU and fails on another, you're probably using the API in a way that's not quite correct, and one of the drivers is letting you get away with not using the API correctly. The debug runtimes will normally catch those errors. Unfortunately, you'll need Windows 7 / XP to use the debug runtimes. See https://stackoverflow.com/questions/73295505/how-to-enable-direct3d-9-debug-output-on-windows-10

However, D3D9 is so old now that I wouldn't recommend using it for anything, unless you really need to work with very old hardware / software. For anything else, I'd suggest starting with D3D11 instead. D3D11 has much better tools available to help you debug any problems you encounter.