0
\$\begingroup\$

I have been working on an Infinity Grid shader for my engine.

It was already implemented to Vulkan’s GLSL, so it worked great with OpenGL’s (4.6) GLSL.

Here's a reference.

This basic shader with simple XY plane that projected in the world and apply fragment/pixel shader calculation to look as a grid. So what could have gone wrong? With OpenGL it works good as expected, but with DirectX 11 it doesn't.

That's how the grid looks in OpenGL. This is how it looks like in OpenGL

But in DirectX 11, that grid starts to glitch or even disappear. It is only visible while I'm actively resizing the window.

I'm sure the problem is not at DirectX 11 Pipeline (Blend, Depth-Stencil, Rasterizer states) because I already 100 times disabled, re-enabled it and changed its descriptions (D3D11_BLEND_DESC, D3D11_DEPTH_STENCIL_DESC). I tried different combinations but the result is still the same. So I assume that the problem is actually in the pixel/fragment shader, I think with some math functions or even with alpha.

GLSL Vertex Shader:

#version 450 core

// Input VAO data.
layout (location = 0) in vec3 attrib_Position;
layout (location = 1) in vec4 attrib_Color;
layout (location = 2) in vec2 attrib_TextureCoord;
layout (location = 3) in vec3 attrib_Normal;
layout (location = 4) in vec3 attrib_Tangent;
layout (location = 5) in vec3 attrib_Bitangent;

layout(std140, binding = 0) uniform UB_GridVS_MVP {
    mat4 viewMatrix;
    mat4 projectionMatrix;
    mat4 invViewMatrix;
    mat4 invProjectionMatrix;
} view;

// Out to Framgment Shader.
layout(location = 1) out vec3 nearPoint;
layout(location = 2) out vec3 farPoint;
layout(location = 3) out mat4 fragView;
layout(location = 7) out mat4 fragProj;

vec3 UnprojectPoint(float x, float y, float z) {
    vec4 unprojectedPoint =  view.invViewMatrix * view.invProjectionMatrix * vec4(x, y, z, 1.0);
    return unprojectedPoint.xyz / unprojectedPoint.w;
}

void main(void) {

    // Unprojecting the near/far plane.
    vec3 p = attrib_Position;

    fragView    = view.viewMatrix;
    fragProj    = view.projectionMatrix;
    nearPoint   = UnprojectPoint(p.x, p.y, 0.0);
    farPoint    = UnprojectPoint(p.x, p.y, 1.0);
    gl_Position = vec4(p, 1.0);
}

GLSL Fragment Shader:

#version 450 core

layout(location = 1) in vec3 nearPoint; // nearPoint calculated in vertex shader
layout(location = 2) in vec3 farPoint;  // farPoint calculated in vertex shader
layout(location = 3) in mat4 fragView;
layout(location = 7) in mat4 fragProj;

layout(std140, binding = 1) uniform UB_GridFS {
    vec4  gridColor1;
    vec4  gridColor2;
    vec4  cameraPos;
    vec4  cameraForward;
    float near;
    float far;
    float maxDistance;
    float offset;
} gridUBO;

layout (location = 0) out vec4 pixelBuffer;

const float subdivisions = 10.0f;

vec4 Grid(vec3 fragPos3D, float scale, bool drawAxis, vec3 gridColor) {
    vec2 coord = fragPos3D.xz * scale; // use the scale variable to set the distance between the lines
    vec2 derivative = fwidth(coord);
    vec2 grid = abs(fract(coord - 0.5) - 0.5) / derivative;
    float line = min(grid.x, grid.y);
    float minimumz = min(derivative.y, 1);
    float minimumx = min(derivative.x, 1);
    vec4 color = vec4(gridColor, 1.0 - min(line, 1.0));
    // z axis
    if(fragPos3D.x > -0.1 * minimumx && fragPos3D.x < 0.1 * minimumx)
        color = vec4(0.0, 0.0, 1.0, color.w);//1.0;
    // x axis
    if(fragPos3D.z > -0.1 * minimumz && fragPos3D.z < 0.1 * minimumz)
        color = vec4(1.0, 0.0, 0.0, color.w);//1.0;
    return color;
}

float ComputeDepth(vec3 pos) {
    vec4 clip_space_pos = fragProj * fragView * vec4(pos.xyz, 1.0);
    return (clip_space_pos.z / clip_space_pos.w);
}

float ComputeLinearDepth(vec3 pos, float near, float far) {
    vec4 clip_space_pos = fragProj * fragView * vec4(pos.xyz, 1.0);
    float clip_space_depth = (clip_space_pos.z / clip_space_pos.w) * 2.0 - 1.0; // put back between -1 and 1
    float linearDepth = (2.0 * near * far) / (far + near - clip_space_depth * (far - near)); // get linear value between 0.01 and 100
    return linearDepth / far; // normalize
}

int RoundToPowerOfTen(float n) {
    return int(pow(10.0, floor( (1 / log(10)) * log(n))));
}

void main(void) {

    float t = -nearPoint.y / (farPoint.y - nearPoint.y);
    if (t < 0.)
        discard;
    
    // Temp
    float _far = 200.0f;
        
    vec3 fragPos3D = nearPoint + t * (farPoint - nearPoint);
    gl_FragDepth = ComputeDepth(fragPos3D);

    float linearDepth = ComputeLinearDepth(fragPos3D, gridUBO.near, _far);
    float fading = max(0, (0.5 - linearDepth));
    float decreaseDistance = _far * 1.5;
    vec3 pseudoViewPos = vec3(gridUBO.cameraPos.x, fragPos3D.y, gridUBO.cameraPos.z);

    float dist, angleFade;
    if (fragProj[3][3] == 0.0) {
        
        // Perspective
        vec3 viewvec = gridUBO.cameraPos.xyz - fragPos3D;
        dist = length(viewvec);
        viewvec /= dist;
        
        float angle;
        angle = viewvec.y;
        angle = 1.0 - abs(angle);
        angle *= angle;
        angleFade = 1.0 - angle * angle;
        angleFade *= 1.0 - smoothstep(0.0, gridUBO.maxDistance, dist - gridUBO.maxDistance);
    
    } else {
    
        // Orthographic
        dist = gl_FragCoord.z * 2.0 - 1.0;
        
        // Avoid fading in +Z direction in camera view.
        dist = clamp(dist, 0.0, 1.0);// abs(dist);
        angleFade = 1.0 - smoothstep(0.0, 0.5, dist - 0.5);
        dist = 1.0; /* avoid branch after */
    }

    float distanceToCamera = abs(gridUBO.cameraPos.y - fragPos3D.y);
    int powerOfTen = RoundToPowerOfTen(distanceToCamera);
    powerOfTen = max(1, powerOfTen);
    float divs = 1.0 / float(powerOfTen);
    float secondFade = smoothstep(subdivisions / divs, 1 / divs, distanceToCamera);

    vec4 grid1 = Grid(fragPos3D, divs / subdivisions, true, gridUBO.gridColor1.xyz);
    vec4 grid2 = Grid(fragPos3D, divs, true, gridUBO.gridColor2.xyz);

    grid2.a *= secondFade;
    grid1.a *= (1 - secondFade);

    // Adding multiple resolution for the grid.
    pixelBuffer   = grid1 + grid2;
    pixelBuffer.a = max(grid1.a, grid2.a);
    
    pixelBuffer *= float(t > 0);
    pixelBuffer.a *= fading;
    pixelBuffer.a *= angleFade;
}

HLSL Vertex Shader:

// Input VBO data.
struct VsInput {
    float3 position     : attrib_Position;
    float4 color        : attrib_Color;
    float2 textureCoord : attrib_TextureCoord;
    float3 normal       : attrib_Normal;
    float3 tangent      : attrib_Tangent;
    float3 bitangent    : attrib_Bitangent;
};

// Output VBO data to Pixel Shader.
struct VsOut {
    float3   nearPoint : v_NearPoint;
    float3   farPoint  : v_FarPoint;
    float4x4 fragView  : v_FragView;
    float4x4 fragProj  : v_FragProj;
    float4   position  : SV_Position;
};

// View Projection + MVP buffer.
cbuffer UB_GridVS_MVP : register(b0) {
    float4x4 viewMatrix;
    float4x4 projectionMatrix;
    float4x4 invViewMatrix;
    float4x4 invProjectionMatrix;
};

// Grid position are in xy clipped space
// Grid mesh loaded by Force on CPU as attrib_Position.

float3 UnprojectPoint(float x, float y, float z) {
    float4 unprojectedPoint = mul(mul(invViewMatrix, invProjectionMatrix), float4(x, y, z, 1.0f));
    return unprojectedPoint.xyz / unprojectedPoint.w;
}

VsOut main(VsInput input) {

    // Unprojecting the near/far plane.
    float3 p = input.position;

    VsOut vsOut;
    vsOut.fragView  = viewMatrix;
    vsOut.fragProj  = projectionMatrix;
    vsOut.nearPoint = UnprojectPoint(p.x, p.y, 0.0);
    vsOut.farPoint  = UnprojectPoint(p.x, p.y, 1.0);
    vsOut.position  = float4(p, 1.0);
    return vsOut;
}

HLSL Pixel Shader:

// Input VAO data from Vertex Shader.
struct FsInput {
    float3   nearPoint : v_NearPoint;
    float3   farPoint  : v_FarPoint;
    float4x4 fragView  : v_FragView;
    float4x4 fragProj  : v_FragProj;
    float4   position  : SV_Position;
};

cbuffer UB_GridFS : register(b1) {
    float4 gridColor1;
    float4 gridColor2;
    float4 cameraPos;
    float4 cameraForward;
    float  near;
    float  far;
    float  maxDistance;
    float  offset;
};

struct OMOut {
    float4  pixelBuffer : SV_Target0;
    float   depthBuffer : SV_Depth;
};

// Constants.
const static float step = 100.0f;
const static float subdivisions = 10.0f;

float4 Grid(float3 fragPos3D, float scale, float3 gridColor) { // No cause glitching OK
    float2 coord = float2(fragPos3D.x * scale, fragPos3D.z * scale);
    float2 derivative = fwidth(coord);
    float2 grid = abs(frac(coord - 0.5) - 0.5) / derivative;
    float _line = min(grid.x, grid.y);
    float minimumz = min(derivative.y, 1);
    float minimumx = min(derivative.x, 1);
    float4 color = float4(gridColor, 1.0 - min(_line, 1.0));
    // z axis
    if(fragPos3D.x > -0.1 * minimumx && fragPos3D.x < 0.1 * minimumx)
        color = float4(0.0, 0.0, 1.0, color.w);
    // x axis
    if(fragPos3D.z > -0.1 * minimumz && fragPos3D.z < 0.1 * minimumz)
        color = float4(1.0, 0.0, 0.0, color.w);
    return color;
}

float ComputeDepth(float3 pos, float4x4 fragProj, float4x4 fragView) {
    float4 clip_space_pos = mul(mul(fragProj, fragView), float4(pos, 1.0));
    return (clip_space_pos.z / clip_space_pos.w);
}

float ComputeLinearDepth(float3 pos, float4x4 fragProj, float4x4 fragView, float near, float far) {
    float4 clip_space_pos = mul(mul(fragProj, fragView), float4(pos, 1.0));
    float clip_space_depth = (clip_space_pos.z / clip_space_pos.w) * 2.0 - 1.0; // put back between -1 and 1
    float linearDepth = (2.0 * near * far) / (far + near - clip_space_depth * (far - near)); // get linear value between 0.01 and 100
    return linearDepth / far; // normalize
}

int RoundToPowerOfTen(float n) { // Cause litte glitch
    return int(pow(10.0, floor( (1.0f / log(10.0)) * log(n))));
}

OMOut main(FsInput input) {

    OMOut omOut;

    // Temp
    float _far = 100.0f;
    
    float t = -input.nearPoint.y / (input.farPoint.y - input.nearPoint.y);
    //if (t < 0.)
    //  discard;
    
    float3 fragPos3D = input.nearPoint + t * (input.farPoint - input.nearPoint);
    omOut.depthBuffer = ComputeDepth(fragPos3D, input.fragProj, input.fragView);
    
    float linearDepth = ComputeLinearDepth(fragPos3D, input.fragProj, input.fragView, near, _far);
    float fading = max(0, (0.5 - linearDepth));
    float decreaseDistance = _far * 1.5;
    float3 pseudoViewPos = float3(cameraPos.x, fragPos3D.y, cameraPos.z);
    
    float dist, angleFade;
    if (input.fragProj[3][3] == 0.0) {
    
        // Perspective
        float3 viewvec = cameraPos.xyz - fragPos3D;
        dist = length(viewvec);
        viewvec /= dist;
        
        float angle;
        angle = viewvec.y;
        angle = 1.0 - abs(angle);
        angle = mul(angle, angle);
        angleFade = 1.0 - angle * angle;
        angleFade = mul(angleFade, 1.0 - smoothstep(0.0, maxDistance, dist - maxDistance));
    } else {
        
        // Orthographic
        dist = input.position.z * 2.0 - 1.0; // fragCoord.z is unprojectedPoint position or direct position?
        
        // Avoid fading in +Z direction in camera view.
        dist = saturate(dist);// abs(dist);
        angleFade = 1.0 - smoothstep(0.0, 0.5, dist - 0.5);
        dist = 1.0; /* avoid branch after */
    }

    float distanceToCamera = abs(cameraPos.y - fragPos3D.y);
    int powerOfTen = RoundToPowerOfTen(distanceToCamera);
    powerOfTen = max(1, powerOfTen);
    float divs = 1.0 / float(powerOfTen);
    float secondFade = smoothstep(subdivisions / divs, 1 / divs, distanceToCamera);

    float4 grid1 = Grid(fragPos3D, divs / subdivisions, gridColor1.xyz);
    float4 grid2 = Grid(fragPos3D, divs, gridColor2.xyz);

    grid2.a = mul(grid2.a, secondFade);
    grid1.a = mul(grid1.a, (1 - secondFade));

    // Adding multiple resolution for the grid.
    omOut.pixelBuffer   = grid1 + grid2;
    omOut.pixelBuffer.a = max(grid1.a, grid2.a);
    
    omOut.pixelBuffer = mul(omOut.pixelBuffer, float(t > 0));
    omOut.pixelBuffer.a = mul(omOut.pixelBuffer.a, fading);
    omOut.pixelBuffer.a = mul(omOut.pixelBuffer.a, angleFade);
    return omOut;
}

All Uniform/Constant buffers data were uploaded correctly. What is causing the glitch?

Here video showing whats happens.

EDIT1: I have been notices that when i remove this lines from HLSL Pixel Shader

    //omOut.pixelBuffer.a = max(grid1.a, grid2.a);
    //
    //omOut.pixelBuffer = mul(omOut.pixelBuffer, float(t > 0));
    //omOut.pixelBuffer.a = mul(omOut.pixelBuffer.a, fading);
    //omOut.pixelBuffer.a = mul(omOut.pixelBuffer.a, angleFade);

I can see normal white grid almost without glitches (e.g it became less). enter image description here

And if i output red color to pixelBuffer

omOut.pixelBuffer   = float4(1, 0, 0, 1);

All glitches are gone. enter image description here

So i can guess that maybe something wrong with one of math function or all of them. Maybe some of it internally behaves little different that GLSL does.

Here all math functions that use HLSL Pixel Shader.

  • fwidth
  • abs
  • frac
  • min
  • max
  • pow
  • log
  • floor
  • length
  • smoothstep
  • saturate
\$\endgroup\$
7
  • \$\begingroup\$ But in DirectX 11 grid starting to glitch or even disappears, but when i resize window (during resizing) it visible but then again disappears. Could you provide a video of this? When it comes to shader issues, the visuals can give a lot of hints on what's wrong. \$\endgroup\$ Feb 12 at 17:17
  • \$\begingroup\$ Yes, I will update the question now. \$\endgroup\$ Feb 12 at 17:34
  • \$\begingroup\$ I cant say for sure but that don't look like any shader I've ever seen in hlsl. \$\endgroup\$
    – Pow
    Feb 12 at 17:41
  • \$\begingroup\$ @TomTsagkatos I've now update question. Video example down below. \$\endgroup\$ Feb 12 at 17:50
  • \$\begingroup\$ @Pow Whats wrong with shader ? In vertex shader is simple input vertex data from vertex buffer layout and from bounded Constant Buffer (where i mark UB_ it means for Uniform/Constant buffer data), and calculate near/far points and send to Pixel shader. And in pixel shader where is magic happens. Sorry i cannot provide much more info because to test this shader need setup DirectX pipeline, load mesh, create VB, IB etc and run. And my engine supports OpenGL/DirectX with another level of abstraction so paste every line from C++ code is hard. \$\endgroup\$ Feb 12 at 17:58

1 Answer 1

2
\$\begingroup\$

Given that it works when resizing the window, I wonder if there's some kind of issue outside of the shader code.

Resizing the window will probably be re-creating some render targets, so maybe you need to clear some or more of them every frame?

Also given that there's a non-opaque alpha output from the shader, you should double check all of the blending render state setup. You could also try outputting opaque pixels from the shader.

Having said that, those are just educated guesses. I'd recommend using a GPU debugger like https://renderdoc.org/ to analyse what's going on.

EDIT: I'd also recommend using D3D11_CREATE_DEVICE_DEBUG as that can catch some types of error.

\$\endgroup\$
6
  • \$\begingroup\$ This is very strange because it only happens with this shader, because when I draw other models, even with textures, everything is fine with them. As for RenderTarget, I definitely clear it every frame regardless if RenderTargetView is a Swap Chain Target or it is an additional RenderTargetView + Texture2D + ShaderResourceView (OpenGL Framebuffer concept). \$\endgroup\$ Feb 13 at 11:41
  • \$\begingroup\$ for(int i = 0; i < colorTextures.size(); i++) deviceContect->ClearRenderTargetView(renderTargets[i].Get(), &color.x); deviceContect->ClearDepthStencilView(depthTarget.Get(), D3D11_CLEAR_DEPTH | D3D11_CLEAR_STENCIL, 1.0f, 0); \$\endgroup\$ Feb 13 at 11:43
  • \$\begingroup\$ So first i clear each render target textures (ID3D11Texture2D) and then clear depth stencil. Im not using stencil buffer but add D3D11_CLEAR_STENCIL but that still not resolve the problem. Here also next variable FLOAT depth witch i set to 1.0f maybe something wrong with it, but Docs says Clear the depth buffer with this value. This value will be clamped between 0 and 1. What is the meaning of this? \$\endgroup\$ Feb 13 at 11:49
  • \$\begingroup\$ Also i've try use RenderDoc with DX11 but my example app or engine crashes when i try run with it. But OpenGL captures good. \$\endgroup\$ Feb 13 at 11:52
  • \$\begingroup\$ Here are my current blend desc settings, I tried to set other parameters, but the result is the same, it's very strange. SrcBlend = D3D11_BLEND_SRC_ALPHA; DestBlend = D3D11_BLEND_INV_SRC_ALPHA; BlendOp = D3D11_BLEND_OP_ADD; SrcBlendAlpha = D3D11_BLEND_ONE; DestBlendAlpha = D3D11_BLEND_INV_SRC_ALPHA; BlendOpAlpha = D3D11_BLEND_OP_ADD; RenderTargetWriteMask = D3D11_COLOR_WRITE_ENABLE_ALL; \$\endgroup\$ Feb 13 at 11:58

You must log in to answer this question.

Not the answer you're looking for? Browse other questions tagged .