XNA / MonoGame - render transparent background using stencil buffer problem (3d)

Question

I'm trying to render 3d texture quads with the following properties:

1. With lighting (rule out "AlphaTestEffect").
2. Transparent background (no opacity levels, just binary opaque or transparent).
3. No ordering, eg, I don't want to sort objects by camera distance.

If I just try to render the quads regularly with their transparent background, the result is that even the transparent pixels are written to the depth buffer, which cause a background bug see picture:

So I try to do it with a different way, using the stencil buffer. What I do is the following:

1. Render just the transparent pixels on the stencil buffer, to create a mask.
2. Render the texture itself, again, using the stencil buffer I created before.

Now it works with the background renderings, but the problem is that different quads with transparent background hide each other in transparent pixels, as seen here:

I'm not sure if it's because the stencil buffer is not cleared between renders or maybe in one of my steps I mistakenly write to depth?

Anyway here's my code (at least the relevant parts of it):

// set effect and settings for the stencil mask phase
_alphaMaskEffect = new AlphaTestEffect(device);
_alphaMaskEffect.World = Matrix.Identity;
_alphaMaskEffect.AlphaFunction = CompareFunction.Equal;
_alphaMaskEffect.ReferenceAlpha = 0;
_alphaMaskEffect.VertexColorEnabled = true;
_alphaMaskEffect.DiffuseColor = Color.White.ToVector3();
_alphaRenderStencilMaskSettings = new DepthStencilState
{
    StencilEnable = true,
    StencilFunction = CompareFunction.Always,
    StencilPass = StencilOperation.Replace,
    ReferenceStencil = 1,
    DepthBufferEnable = true,
    DepthBufferWriteEnable = false,
};

// set settings for the regular rendering phase
_alphaRenderTextureWithStencilSettings = new DepthStencilState
{
    StencilEnable = true,
    StencilFunction = CompareFunction.Equal,
    StencilPass = StencilOperation.Keep,
    ReferenceStencil = 0,
    DepthBufferEnable = true,
    DepthBufferWriteEnable = true,
};

And here's the draw function itself:

// set settings for the stencil buffer rendering
device.DepthStencilState = _alphaRenderStencilMaskSettings;

// render stencil buffer
foreach (EffectPass pass in _alphaMaskEffect.CurrentTechnique.Passes)
{
    // draw current pass
    pass.Apply();
    device.DrawUserIndexedPrimitives
        <VertexPositionNormalTexture>(
        PrimitiveType.TriangleList,
        Vertices, 0, 4,
        Indexes, 0, 2);
}

// now render the quad with the stencil buffer
device.DepthStencilState = _alphaRenderTextureWithStencilSettings;

// render effect itself
foreach (EffectPass pass in _effect.CurrentTechnique.Passes)
{
    // draw current pass
    pass.Apply();
    device.DrawUserIndexedPrimitives
        <VertexPositionNormalTexture>(
        PrimitiveType.TriangleList,
        Vertices, 0, 4,
        Indexes, 0, 2);
}

tl;dr what's the right way to render 3d quads with transparent background, lighting and without sorting objects by distance from camera? And if it's the code I tried - how do I make the renderings stop hiding each other?

Answer 1

Hey mate the best way to do it is with alpha test, that way no sorting but it still helps to sort and u can do lighting, render with opaque blend state, cull mode to none.

I do my grass like that, its just a quad with alpha test and then I build all the normal data in the shade. I also do use the vface semantic to flip the normal because its double sided.

alpha test in the pixel shader:

 diffuseLayer = DiffuseMaps.Sample(samANISOTROPICClamp, float3(input.TexCoord, input.TextureID));

clip(diffuseLayer.a - 0.18); // 0.18 is the alpha cut off, play with this

flipping the normal based on vface:

   float3 normalViewSpace = NormalMapToSpaceNormal(NormalLayer.rgb, TBN[2], TBN[1], TBN[0]);
normalViewSpace = input.vFace ? normalViewSpace : -normalViewSpace; // ignore the viewsapce stuff that is for my deferred render

Tangents and binormals in the shader based on a normal(N should be float3(0,1,0), UV should be the clip space xy, P should be world space positon:

float3x3 cotangent_frame(float3 N, float3 p, float2 uv)
{
// get edge vectors of the pixel triangle
float3 dp1 = ddx_fine(p);
float3 dp2 = ddy_fine(p);
float2 duv1 = ddx_fine(uv);
float2 duv2 = ddy_fine(uv);

// solve the linear system
float3 dp2perp = cross(dp2, N);
float3 dp1perp = cross(N, dp1);
float3 T = dp2perp * duv1.x + dp1perp * duv2.x;
float3 B = dp2perp * duv1.y + dp1perp * duv2.y;

// construct a scale-invariant frame 
float invmax =   -sqrt(max(dot(T, T), dot(B, B)));
return float3x3(T * invmax, B * invmax, N);
}

here is what the grass can look like:

CPU draw code(with dx 11 I don't have to set vertex buffers(runtime gens the points):

            _Context.InputAssembler.SetVertexBuffers(0, Nothing, Nothing)
        _Context.InputAssembler.InputLayout = Nothing

        _Context.OutputMerger.DepthStencilState = _device.RenderStates.Default
        _Context.Rasterizer.State = _device.RenderStates.CullNone
        _Context.SetBlendState(_device.RenderStates.Opaque)

        GrassEffectVars.Set("View", _camera.EyeTransform)
        GrassEffectVars.Set("Projection", _camera.ProjectionTransform)
        GrassEffectVars.Set("GrassNodes", GrassNodesView)
        GrassEffectVars.Set("DiffuseMaps", GrassTextureArraySRV)
        GrassEffectVars.Set("NormalMaps", GrassNormalTextureArraySRV)
        GrassEffectVars.Set("CameraTransform", _camera.Transform)

        GrassEffectVars.Set("gEyePosW", _camera.CameraBounds.Center)
        GrassEffectVars.Set("FarClip", _camera.FarClip)

        _Context.InputAssembler.SetIndexBuffer(indexBuffer, Format.R32_UInt, 0)
        _Context.InputAssembler.PrimitiveTopology = PrimitiveTopology.TriangleList

        GrassEffectVars.Set("InvView", _camera.Transform)
        GrassEffectVars.Set("WorldViewProj", _camera.EyeProjectionTransform)

        _Context.ApplyShader(GrassEffectPass2)
        _Context.DrawIndexedInstanced(6, GrassCount, 0, 0, 0)

Full grass shader as asked(builds quads from points):

struct PS_INPUT3
{
float4 Position : SV_POSITION;
float2 TexCoord : TEXCOORD0;
uint TextureID : TextureID;
float4 Depth : VS_DEPTH;
bool vFace : SV_IsFrontFace;
};

PS_INPUT2 VS_Main2(uint id : SV_VertexID, uint instid : SV_InstanceId)
{
PS_INPUT2 output = (PS_INPUT2) 0;

uint particleIndex = id / 4;
uint vertexInQuad = id % 4;

GrassNode Grass = GrassNodes[instid];
float3 WS_Position = Grass.Pos.xyz;
float4 Data = Grass.Data;

float3 position;
position.x = (vertexInQuad % 2) ? 1.0 : -1.0;
position.y = (vertexInQuad & 2) ? -1.0 : 1.0;
position.z = 0.0;
position.xy *= Data.y;   

WS_Position.y += Data.y;
WS_Position.x += Data.z;
WS_Position.z += Data.w; 

position = mul(position, (float3x3) InvView) ;

WS_Position = WS_Position.xyz + position;

output.Depth.xyz = WS_Position;

output.Depth.w = mul(float4(WS_Position, 1), View).z;
output.Position = mul(float4(WS_Position.xyz, 1.0), WorldViewProj);    
output.TexCoord.x = (vertexInQuad % 2) ? 1.0 : 0.0;
output.TexCoord.y = (vertexInQuad & 2) ? 1.0 : 0.0;
output.TextureID = Data.x;
return output;
}

GBufferPixelShaderOutput PS_Main2(PS_INPUT3 input)
{

GBufferPixelShaderOutput output;
float4 diffuseLayer = 0;
float4 NormalLayer = 0;   

diffuseLayer += DiffuseMaps.Sample(samANISOTROPICClamp, float3(input.TexCoord, input.TextureID));

clip(diffuseLayer.a - 0.18);

NormalLayer += NormalMaps.Sample( samANISOTROPICClamp, float3(input.TexCoord, input.TextureID)) ;

float3x3 TBN = cotangent_frame(float3(0, 0.957, 0), input.Depth.xyz, input.Position.xy);

TBN[2] = normalize(mul(TBN[2], (float3x3) View));
TBN[0] = normalize(mul(TBN[0], (float3x3) View));
TBN[1] = normalize(mul(TBN[1], (float3x3) View));

float3 normalViewSpace = NormalMapToSpaceNormal(NormalLayer.rgb, TBN[2], TBN[1], TBN[0]);
normalViewSpace = input.vFace ? normalViewSpace : -normalViewSpace;
output.Color = float4(diffuseLayer.rgb,  diffuseLayer.r);

output.RenderMaterialID = matID;

output.Depth = float4(-input.Depth.w / FarClip, 0, EncodeNormal(normalViewSpace.xyz));

return output;

}