Use 2 values of a mask

Question

Let's say one has a black/gray/white mask and wants to draw picture A over the white/gray values and picture B over the black/gray values (the blacker the more B picture). The best I come up with would be something like:

1. Set to RenderTargetA
2. Draw Mask
3. Draw PictureA over Mask (using DepthStencilState)
4. Set to RenderTargetB
5. Draw Mask
6. Draw PictureB over Mask
7. Draw both RenderTargets over each other somehow

But that just seems a) overly complicated and b) both render targets still have the mask drawn on them so it wouldn't have worked anyway. So how would one achieve this?

Answer 1

Yes. Shaders are the way. Just a simple shader worked. Here's the shader:

texture activeTexture;
sampler2D activeSampler = sampler_state {
    Texture = (activeTexture);
};

texture passiveTexture;
sampler2D passiveSampler = sampler_state {
    Texture = (passiveTexture);
};

texture visionMask;
sampler2D visionSampler = sampler_state {
    Texture = (visionMask);
};

sampler s0;
sampler ms0 : register(s0);

float4 MainPS(float2 coords: TEXCOORD0) : COLOR0
{
    float4 visionColor = tex2D(visionSampler, coords);
    float4 activeColor = tex2D(activeSampler, coords);
    float4 passiveColor = tex2D(passiveSampler, coords);

    float4 color = float4(0, 0, 0, 1);
    color.rgb = activeColor.rgb * visionColor.rgb + ((float3(1, 1, 1) - visionColor.rgb) * passiveColor.rgb);

    return color;
}

technique Technique0
{
    pass Pass0
    {
        PixelShader = compile ps_3_0 MainPS();
    }
}