HLSL Manual Alpha Blending

Question

I'm trying to do alpha blending manually because I only want to apply alpha blending on certain pixels. Underlying is the texture I'm writing to. This is what I got so far, but it doesn't give the same result as when AlphaBlendEnable = TRUE

What formula does default alpha blend in HLSL use?

matrix WorldViewProjection;

texture2D OverlyingShadingMap;
sampler2D OverlyingShadingMapSampler = sampler_state
{
    texture = <OverlyingShadingMap>;
};

texture2D UnderlyingShadingMap;
sampler2D UnderlyingShadingMapSampler = sampler_state
{
    texture = <UnderlyingShadingMap>;
};

struct VertexShaderOutput
{
    float4 Position : SV_POSITION;
    float4 Color : COLOR0;
    float2 TextureCoordinates : TEXCOORD0;
};

VertexShaderOutput SpriteVertexShader(float4 position : POSITION0, float4 color : COLOR0, float2 texCoord : TEXCOORD0)
{
    VertexShaderOutput output;
    output.Position = mul(position, WorldViewProjection);
    output.Color = color;
    output.TextureCoordinates = texCoord;
    return output;
}

float4 SpritePixelShader(VertexShaderOutput input) : COLOR
{
    float4 overlying = tex2D(OverlyingShadingMapSampler, input.TextureCoordinates);
    clip(overlying.a - 0.001);
    float4 underlying = tex2D(UnderlyingShadingMapSampler, input.TextureCoordinates);
    // Alphablending
    float3 rgb = overlying.rgb * overlying.a + (1 - overlying.a) * underlying.a * underlying.rgb;
    float alpha = underlying.a + (1 - underlying.a) * overlying.a;
    return float4(rgb, alpha);
}

technique SpriteDrawing
{
    pass P0
    {
        AlphaBlendEnable = FALSE;
        VertexShader = compile vs_2_0 SpriteVertexShader();
        PixelShader = compile ps_2_0 SpritePixelShader();
    }
};

Answer 1

Change this:

float3 rgb = overlying.rgb * overlying.a + (1 - overlying.a) * underlying.a * underlying.rgb;
float alpha = underlying.a + (1 - underlying.a) * overlying.a;
return float4(rgb, alpha);

To this:

return underlying + (1 - underlying.a) * overlying;

Formula for alpha blending is: $$pixelColour=src\_pixel+(1−src_α).dest\_pixel$$ Taken from here, which also lists formulas for other blendstates in XNA: http://glasnost.itcarlow.ie/~powerk/technology/xna/blending/blending.html