XNA: Non-overlapping shadows

Question

I want to create an effect similar to the following in XNA / MonoGame:

There's a background and a semi-transparent shadow consisting of several objects (two circles in this example). However, the shadows merge without darkening their overlapping region!

My current approach was the following:

• Render all required shadow objects using a DepthStencilState and a AlphaTestEffect.
• Then, render a single-pixel semitransparent black texture over the entire screen using another DepthStencilState to display shadows.

However, it didn't work as expected: the shadow objects get rendered to the target, and the stencil buffer seems unaffected.

Here's what I want:

• A way to render a sprite to Stencil Buffer only, with absolutely no visible modifications to the target.
• A way to render a semitransparent texture using the stencil buffer.

Stencil states and effects are initialized as follows:

StencilCreator = new DepthStencilState
{
    StencilEnable = true,
    StencilFunction = CompareFunction.Always,
    StencilPass = StencilOperation.Replace,
    ReferenceStencil = 1
};

StencilRenderer = new DepthStencilState
{
    StencilEnable = true,
    StencilFunction = CompareFunction.Equal,
    ReferenceStencil = 1,
    StencilPass = StencilOperation.Keep
};

var projection = Matrix.CreateOrthographicOffCenter(0, GraphicsDevice.Viewport.Width, GraphicsDevice.Viewport.Height, 0, 0, 1); 
var halfPixel = Matrix.CreateTranslation(-0.5f, -0.5f, 0);

AlphaEffect = new AlphaTestEffect(GraphicsDevice)
{
    DiffuseColor = Color.White.ToVector3(),
    AlphaFunction = CompareFunction.Greater,
    ReferenceAlpha = 0,
    World = Matrix.Identity,
    View = Matrix.Identity,
    Projection = halfPixel * projection
};

MaskingTexture = new Texture2D(GameEngine.GraphicsDevice, 1, 1);
MaskingTexture.SetData(new[] { new Color(0f, 0f, 0f, 0.3f) });

What am I doing wrong?

Answer 1

I think this could be solved using render targets. I propose the following technique

Create two render targets, one for the colors in the scene and one for the lighting in your scene

colorMap = new RenderTarget2D(GraphicsDevice, windowWidth, windowHeight, false, SurfaceFormat.Color, DepthFormat.Depth16, 16, RenderTargetUsage.DiscardContents);
lightMap = new RenderTarget2D(GraphicsDevice, windowWidth, windowHeight, false, SurfaceFormat.Color, DepthFormat.None, 16, RenderTargetUsage.DiscardContents);

Set your colorMap and draw your game, without lighting

GraphicsDevice.SetRenderTarget(colorMap);
GraphicsDevice.Clear(Color.CornflowerBlue);
DrawMyAwesomeGame();

Set your lightMap and draw your lighting (or inversely your shadows)
Note that you have to clear your lightmap to white if you want to paint the shadows or to black if you want to paint he lit place

GraphicsDevice.SetRenderTarget(lightMap);
GraphicsDevice.Clear(Color.White); 
DrawShadows();

Now let's return rendering to the screen

GraphicsDevice.SetRenderTarget(null);

Now we're going to need a shader to combine these two. Let assume we already have one (we'll see the source later)

combineEffect.Parameters["colorMap"].SetValue(colorMap);
combineEffect.Parameters["lightMap"].SetValue(lightMap);    

combineEffect.Techniques[0].Passes[0].Apply();           

Of course the shader needs something to work with, so lets draw a full screen quad (code at the end of this post)

quad.Render(GraphicsDevice, Vector2.One * -1.0f, Vector2.One);  

Ok so let's get back to the shader, how does it combine the two

float3 diffuseColor = tex2D(colorSampler, input.TexCoord).rgb;
float4 light = tex2D(lightSampler, input.TexCoord);
float ambient = 0.1f; // ambient lighting, everything is atleast lit so much

diffuseLight.r = max(diffuseLight.r, ambient);
diffuseLight.g = max(diffuseLight.g, ambient);
diffuseLight.b = max(diffuseLight.b, ambient);

return float4((diffuseColor * diffuseLight), 1);

This should do the trick!

Other code used in this post

The full shader:

texture colorMap;
texture lightMap;

sampler colorSampler = sampler_state
{
  Texture = (colorMap);
  AddressU = CLAMP;
  AddressV = CLAMP;
  MagFilter = LINEAR;
  MinFilter = LINEAR;
  Mipfilter = LINEAR;
};
sampler lightSampler = sampler_state
{
  Texture = (lightMap);
  AddressU = CLAMP;
  AddressV = CLAMP;
  MagFilter = LINEAR;
  MinFilter = LINEAR;
  Mipfilter = LINEAR;
};

struct VertexShaderInput
{
  float3 Position : POSITION0;
  float2 TexCoord : TEXCOORD0;
};

struct VertexShaderOutput
{
  float4 Position : POSITION0;
  float2 TexCoord : TEXCOORD0;
};

VertexShaderOutput VertexShaderFunction(VertexShaderInput input)
{
    VertexShaderOutput output;
    output.Position = float4(input.Position, 1);
    output.TexCoord = input.TexCoord;
    return output;
}

float4 PixelShaderFunction(VertexShaderOutput input) : COLOR0
{
    float3 diffuseColor = tex2D(colorSampler, input.TexCoord).rgb;
    float4 light = tex2D(lightSampler, input.TexCoord);
    float ambient = 0.1f; // ambient lighting, everything is atleast lit so much

    diffuseLight.r = max(diffuseLight.r, ambient);
    diffuseLight.g = max(diffuseLight.g, ambient);
    diffuseLight.b = max(diffuseLight.b, ambient);

    return float4((diffuseColor * diffuseLight), 1);
}

technique Technique1
{
  pass Pass1
  {
    VertexShader = compile vs_2_0 VertexShaderFunction();
    PixelShader = compile ps_2_0 PixelShaderFunction();
  }
}

The quad primitive:

internal class Quad
{
    private VertexPositionTexture[] vertices;
    private short[] indexBuffer;

    internal Quad()
    {
        vertices = new VertexPositionTexture[]
                    {
                        new VertexPositionTexture(
                            new Vector3(0, 0, 0),
                            new Vector2(1, 1)),
                        new VertexPositionTexture(
                            new Vector3(0, 0, 0),
                            new Vector2(0, 1)),
                        new VertexPositionTexture(
                            new Vector3(0, 0, 0),
                            new Vector2(0, 0)),
                        new VertexPositionTexture(
                            new Vector3(0, 0, 0),
                            new Vector2(1, 0))
                    };

        indexBuffer = new short[] { 0, 1, 2, 2, 3, 0 };
    }

    /// <summary>
    /// Renders the four vertices for a quad directly without touching render states,
    /// setting shaders, etc...
    /// </summary>        
    internal void Render(GraphicsDevice device, Vector2 bottomLeft, Vector2 topRight)
    {
        vertices[0].Position.X = topRight.X;
        vertices[0].Position.Y = bottomLeft.Y;

        vertices[1].Position.X = bottomLeft.X;
        vertices[1].Position.Y = bottomLeft.Y;

        vertices[2].Position.X = bottomLeft.X;
        vertices[2].Position.Y = topRight.Y;

        vertices[3].Position.X = topRight.X;
        vertices[3].Position.Y = topRight.Y;

        device.DrawUserIndexedPrimitives<VertexPositionTexture>
        (
            PrimitiveType.TriangleList,
            vertices,
            0,
            4,
            indexBuffer,
            0,
            2
        );
    }

Answer 2

I actually found a much easier easier way that does not require any shaders. The trick is simple:

1. Create a RenderTarget2D and clear it with Colors.Transparent.
2. Render all the shadows into the render target using solid black color. This way shadows will not darken each other, as each of them is pitch black already.
3. Draw the render target into the main scene using BlendingMode.AlphaBlend.

The transparency of draw color will determine the overall shadow darkness.