Take the 2-minute tour ×
Game Development Stack Exchange is a question and answer site for professional and independent game developers. It's 100% free, no registration required.

I'm currently (attempting) to add SSAO to my engine, except it's...not really work, to say the least. I use a deferred renderer to render my scene. I have four render targets: Albedo, Light, Normal, and Depth.

Here are the parameters for all of them (Surface Format, Depth Format):

Albedo: 32-bit ARGB, Depth24Stencil8

Light: 32-bit ARGB, None

Normal: 32-bit ARGB, None

Depth: 8-bit R (Single), Depth24Stencil8

To generate my random noise map for the SSAO, I do the following for each pixel in the noise map:

            Vector3 v3 = Vector3.Zero;
            double z = rand.NextDouble() * 2.0 - 1.0;
            double r = Math.Sqrt(1.0 - z * z);
            double angle = rand.NextDouble() * MathHelper.TwoPi;
            v3.X = (float)(r * Math.Cos(angle));
            v3.Y = (float)(r * Math.Sin(angle));
            v3.Z = (float)z;

            v3 += offset;
            v3 *= 0.5f;

            result[i] = new Color(v3);

This is my GBuffer rendering effect:

PixelInput RenderGBufferColorVertexShader(VertexInput input)
{
    PixelInput pi = ( PixelInput ) 0;

    pi.Position = mul(input.Position, WorldViewProjection);
    pi.Normal = mul(input.Normal, WorldInverseTranspose);
    pi.Color = input.Color;
    pi.TPosition = pi.Position;
    pi.WPosition = input.Position;


return pi;  
}

GBufferTarget RenderGBufferColorPixelShader(PixelInput input)
{
    GBufferTarget output = ( GBufferTarget ) 0;

    float3 position = input.TPosition.xyz / input.TPosition.w;

    output.Albedo = lerp(float4(1.0f, 1.0f, 1.0f, 1.0f), input.Color, ColorFactor);
    output.Normal = EncodeNormal(input.Normal);
    output.Depth = position.z;

    return output;
}

And here is the SSAO effect:

float4 EncodeNormal(float3 normal)
{
    return float4((normal.xyz * 0.5f) + 0.5f, 0.0f);
}

float3 DecodeNormal(float4 encoded)
{
    return encoded * 2.0 - 1.0f;
}

float Intensity;
float Size;

float2 NoiseOffset;

float4x4 ViewProjection;
float4x4 ViewProjectionInverse;

texture DepthMap;
texture NormalMap;
texture RandomMap;

const float3 samples[16] =
{
    float3(0.01537562, 0.01389096, 0.02276565),
    float3(-0.0332658, -0.2151698, -0.0660736),
    float3(-0.06420016, -0.1919067, 0.5329634),
    float3(-0.05896204, -0.04509097, -0.03611697),
    float3(-0.1302175, 0.01034653, 0.01543675),
    float3(0.3168565, -0.182557, -0.01421785),
    float3(-0.02134448, -0.1056605, 0.00576055),
    float3(-0.3502164, 0.281433, -0.2245609),
    float3(-0.00123525, 0.00151868, 0.02614773),
    float3(0.1814744, 0.05798516, -0.02362876),
    float3(0.07945167, -0.08302628, 0.4423518),
    float3(0.321987, -0.05670302, -0.05418307),
    float3(-0.00165138, -0.00410309, 0.00537362),
    float3(0.01687791, 0.03189049, -0.04060405),
    float3(-0.04335613, -0.00530749, 0.06443053),
float3(0.8474263, -0.3590308, -0.02318038),
};

sampler DepthSampler = sampler_state
{
    Texture = DepthMap;
    MipFilter = Point;
    MinFilter = Point;
    MagFilter = Point;
    AddressU = Clamp;
    AddressV = Clamp;
    AddressW = Clamp;
};

sampler NormalSampler = sampler_state
{
    Texture = NormalMap;
    MipFilter = Linear;
    MinFilter = Linear;
    MagFilter = Linear;
    AddressU = Clamp;
    AddressV = Clamp;
    AddressW = Clamp;
};

sampler RandomSampler = sampler_state
{
    Texture = RandomMap;
    MipFilter = Linear;
    MinFilter = Linear;
    MagFilter = Linear;
};

struct VertexInput
{
    float4 Position : POSITION0;
    float2 TextureCoordinates : TEXCOORD0;
};

struct PixelInput
{
    float4 Position : POSITION0;
    float2 TextureCoordinates : TEXCOORD0;
};

PixelInput SSAOVertexShader(VertexInput input)
{
    PixelInput pi = ( PixelInput ) 0;

    pi.Position = input.Position;
    pi.TextureCoordinates = input.TextureCoordinates;

    return pi;
}

float3 GetXYZ(float2 uv)
{
    float depth = tex2D(DepthSampler, uv);
    float2 xy = uv * 2.0f - 1.0f;
    xy.y *= -1;

    float4 p = float4(xy, depth, 1);
    float4 q = mul(p, ViewProjectionInverse);
    return q.xyz / q.w;
}

float3 GetNormal(float2 uv)
{
    return DecodeNormal(tex2D(NormalSampler, uv));
}

float4 SSAOPixelShader(PixelInput input) : COLOR0
{
    float depth = tex2D(DepthSampler, input.TextureCoordinates);

    float3 position = GetXYZ(input.TextureCoordinates);
    float3 normal = GetNormal(input.TextureCoordinates);

    float occlusion = 1.0f;

    float3 reflectionRay = DecodeNormal(tex2D(RandomSampler, input.TextureCoordinates + NoiseOffset));

    for (int i = 0; i < 16; i++)
    {
        float3 sampleXYZ = position + reflect(samples[i], reflectionRay) * Size;

        float4 screenXYZW = mul(float4(sampleXYZ, 1.0f), ViewProjection);
        float3 screenXYZ = screenXYZW.xyz / screenXYZW.w;
        float2 sampleUV = float2(screenXYZ.x * 0.5f + 0.5f, 1.0f - (screenXYZ.y * 0.5f + 0.5f));

        float frontMostDepthAtSample = tex2D(DepthSampler, sampleUV);

        if (frontMostDepthAtSample < screenXYZ.z)
        {
                occlusion -= 1.0f / 16.0f;
        }
    }

    return float4(occlusion * Intensity * float3(1.0, 1.0, 1.0), 1.0);
}

technique SSAO
{
    pass Pass0
    {
        VertexShader = compile vs_3_0 SSAOVertexShader();
        PixelShader = compile ps_3_0 SSAOPixelShader();
    }
}

However, when I use the effect, I get some pretty bad distortion: First image

Here's the light map that goes with it -- is the static-like effect supposed to be like that? I've noticed that even if I'm looking at nothing, I still get the static-like effect. (you can see it in the screenshot; the top half doesn't have any geometry yet it still has the static-like effect) Second image

Also, does anyone have any advice on how to effectively debug shaders?

share|improve this question

closed as off-topic by Anko, Seth Battin, Josh Petrie Feb 26 at 4:04

This question appears to be off-topic. The users who voted to close gave this specific reason:

  • "Questions about debugging a problem in your project must present a concise selection of code and context so as to allow a reader to diagnose the issue without needing to read all of your code or to engage in extensive back-and-forth dialog. For more information, see this meta thread." – Anko, Seth Battin, Josh Petrie
If this question can be reworded to fit the rules in the help center, please edit the question.

1  
Try using PIX for debugging D3D9 shaders. It's a quite powerful tool, and the tutorial linked shows you how to use it with XNA. –  Nathan Reed Jun 29 '13 at 7:17