Game Development Stack Exchange is a question and answer site for professional and independent game developers. Join them; it only takes a minute:

Sign up
Here's how it works:
  1. Anybody can ask a question
  2. Anybody can answer
  3. The best answers are voted up and rise to the top

I'm currently (attempting) to add SSAO to my engine, except it's...not really work, to say the least. I use a deferred renderer to render my scene. I have four render targets: Albedo, Light, Normal, and Depth.

Here are the parameters for all of them (Surface Format, Depth Format):

Albedo: 32-bit ARGB, Depth24Stencil8

Light: 32-bit ARGB, None

Normal: 32-bit ARGB, None

Depth: 8-bit R (Single), Depth24Stencil8

To generate my random noise map for the SSAO, I do the following for each pixel in the noise map:

            Vector3 v3 = Vector3.Zero;
            double z = rand.NextDouble() * 2.0 - 1.0;
            double r = Math.Sqrt(1.0 - z * z);
            double angle = rand.NextDouble() * MathHelper.TwoPi;
            v3.X = (float)(r * Math.Cos(angle));
            v3.Y = (float)(r * Math.Sin(angle));
            v3.Z = (float)z;

            v3 += offset;
            v3 *= 0.5f;

            result[i] = new Color(v3);

This is my GBuffer rendering effect:

PixelInput RenderGBufferColorVertexShader(VertexInput input)
{
    PixelInput pi = ( PixelInput ) 0;

    pi.Position = mul(input.Position, WorldViewProjection);
    pi.Normal = mul(input.Normal, WorldInverseTranspose);
    pi.Color = input.Color;
    pi.TPosition = pi.Position;
    pi.WPosition = input.Position;


return pi;  
}

GBufferTarget RenderGBufferColorPixelShader(PixelInput input)
{
    GBufferTarget output = ( GBufferTarget ) 0;

    float3 position = input.TPosition.xyz / input.TPosition.w;

    output.Albedo = lerp(float4(1.0f, 1.0f, 1.0f, 1.0f), input.Color, ColorFactor);
    output.Normal = EncodeNormal(input.Normal);
    output.Depth = position.z;

    return output;
}

And here is the SSAO effect:

float4 EncodeNormal(float3 normal)
{
    return float4((normal.xyz * 0.5f) + 0.5f, 0.0f);
}

float3 DecodeNormal(float4 encoded)
{
    return encoded * 2.0 - 1.0f;
}

float Intensity;
float Size;

float2 NoiseOffset;

float4x4 ViewProjection;
float4x4 ViewProjectionInverse;

texture DepthMap;
texture NormalMap;
texture RandomMap;

const float3 samples[16] =
{
    float3(0.01537562, 0.01389096, 0.02276565),
    float3(-0.0332658, -0.2151698, -0.0660736),
    float3(-0.06420016, -0.1919067, 0.5329634),
    float3(-0.05896204, -0.04509097, -0.03611697),
    float3(-0.1302175, 0.01034653, 0.01543675),
    float3(0.3168565, -0.182557, -0.01421785),
    float3(-0.02134448, -0.1056605, 0.00576055),
    float3(-0.3502164, 0.281433, -0.2245609),
    float3(-0.00123525, 0.00151868, 0.02614773),
    float3(0.1814744, 0.05798516, -0.02362876),
    float3(0.07945167, -0.08302628, 0.4423518),
    float3(0.321987, -0.05670302, -0.05418307),
    float3(-0.00165138, -0.00410309, 0.00537362),
    float3(0.01687791, 0.03189049, -0.04060405),
    float3(-0.04335613, -0.00530749, 0.06443053),
float3(0.8474263, -0.3590308, -0.02318038),
};

sampler DepthSampler = sampler_state
{
    Texture = DepthMap;
    MipFilter = Point;
    MinFilter = Point;
    MagFilter = Point;
    AddressU = Clamp;
    AddressV = Clamp;
    AddressW = Clamp;
};

sampler NormalSampler = sampler_state
{
    Texture = NormalMap;
    MipFilter = Linear;
    MinFilter = Linear;
    MagFilter = Linear;
    AddressU = Clamp;
    AddressV = Clamp;
    AddressW = Clamp;
};

sampler RandomSampler = sampler_state
{
    Texture = RandomMap;
    MipFilter = Linear;
    MinFilter = Linear;
    MagFilter = Linear;
};

struct VertexInput
{
    float4 Position : POSITION0;
    float2 TextureCoordinates : TEXCOORD0;
};

struct PixelInput
{
    float4 Position : POSITION0;
    float2 TextureCoordinates : TEXCOORD0;
};

PixelInput SSAOVertexShader(VertexInput input)
{
    PixelInput pi = ( PixelInput ) 0;

    pi.Position = input.Position;
    pi.TextureCoordinates = input.TextureCoordinates;

    return pi;
}

float3 GetXYZ(float2 uv)
{
    float depth = tex2D(DepthSampler, uv);
    float2 xy = uv * 2.0f - 1.0f;
    xy.y *= -1;

    float4 p = float4(xy, depth, 1);
    float4 q = mul(p, ViewProjectionInverse);
    return q.xyz / q.w;
}

float3 GetNormal(float2 uv)
{
    return DecodeNormal(tex2D(NormalSampler, uv));
}

float4 SSAOPixelShader(PixelInput input) : COLOR0
{
    float depth = tex2D(DepthSampler, input.TextureCoordinates);

    float3 position = GetXYZ(input.TextureCoordinates);
    float3 normal = GetNormal(input.TextureCoordinates);

    float occlusion = 1.0f;

    float3 reflectionRay = DecodeNormal(tex2D(RandomSampler, input.TextureCoordinates + NoiseOffset));

    for (int i = 0; i < 16; i++)
    {
        float3 sampleXYZ = position + reflect(samples[i], reflectionRay) * Size;

        float4 screenXYZW = mul(float4(sampleXYZ, 1.0f), ViewProjection);
        float3 screenXYZ = screenXYZW.xyz / screenXYZW.w;
        float2 sampleUV = float2(screenXYZ.x * 0.5f + 0.5f, 1.0f - (screenXYZ.y * 0.5f + 0.5f));

        float frontMostDepthAtSample = tex2D(DepthSampler, sampleUV);

        if (frontMostDepthAtSample < screenXYZ.z)
        {
                occlusion -= 1.0f / 16.0f;
        }
    }

    return float4(occlusion * Intensity * float3(1.0, 1.0, 1.0), 1.0);
}

technique SSAO
{
    pass Pass0
    {
        VertexShader = compile vs_3_0 SSAOVertexShader();
        PixelShader = compile ps_3_0 SSAOPixelShader();
    }
}

However, when I use the effect, I get some pretty bad distortion: First image

Here's the light map that goes with it -- is the static-like effect supposed to be like that? I've noticed that even if I'm looking at nothing, I still get the static-like effect. (you can see it in the screenshot; the top half doesn't have any geometry yet it still has the static-like effect) Second image

Also, does anyone have any advice on how to effectively debug shaders?

share|improve this question

closed as off-topic by Anko, Seth Battin, Josh Petrie Feb 26 '14 at 4:04

This question appears to be off-topic. The users who voted to close gave this specific reason:

  • "Questions about debugging a problem in your project must present a concise selection of code and context so as to allow a reader to diagnose the issue without needing to read all of your code or to engage in extensive back-and-forth dialog. For more information, see this meta thread." – Anko, Seth Battin, Josh Petrie
If this question can be reworded to fit the rules in the help center, please edit the question.

1  
Try using PIX for debugging D3D9 shaders. It's a quite powerful tool, and the tutorial linked shows you how to use it with XNA. – Nathan Reed Jun 29 '13 at 7:17