Unwanted darkening at polygon edges when using normal maps with SSAO

Question

I'm using a SSAO alghorithm called "SAO", seen here: Scalable Ambient Obscurance.

I'm currently having issues with "smooth" normals, for example when using normal/height maps.

To give an example, here is the SAO and the normals for a chair rendered with "raw" normals, i.e not using the accompanied normal map:

Looks pretty OK overall.

now.. when I apply the chair's normal map:

What could be wrong? Is there anything obvious wrong with the normals?

I'm using the SAO shader, with only very minor changes. The normal/position textures are in world space, and then converted to view space. I know this is not optimal, but for the sake of the alghorithm it should not matter.

#ifndef SSAO_PIXEL_HLSL
#define SSAO_PIXEL_HLSL

#include "Constants.h"
#include "Common.hlsl"

static const float gNumSamples = 11.0;
static const float gRadius = 0.2;
static const float gRadius2 = gRadius * gRadius;
static const float gProjScale = 500.0;
static const float gNumSpiralTurns = 7;
static const float gBias = 0.01;
static const float gIntensity = 1.0;


cbuffer SSAOCBuffer : register(CBUFFER_REGISTER_PIXEL)
{
    float4x4 gViewProjMatrix;
    float4x4 gProjMatrix;
    float4x4 gViewMatrix;
    float2 gScreenSize;
};

Texture2D gPositionTexture : register(TEXTURE_REGISTER_POSITION);
Texture2D gNormalTexture : register(TEXTURE_REGISTER_NORMAL);
SamplerState gPointSampler : register(SAMPLER_REGISTER_POINT);


float3 reconstructNormal(float3 positionWorldSpace)
{
    return normalize(cross(ddx(positionWorldSpace), ddy(positionWorldSpace)));
}

/** Read the camera - space position of the point at screen - space pixel ssP + unitOffset * ssR.Assumes length(unitOffset) == 1 */
float3 getOffsetPosition(int2 ssC, float2 unitOffset, float ssR) {
    // Derivation:
    //  mipLevel = floor(log(ssR / MAX_OFFSET));

    // TODO: mip levels
    int mipLevel = 0; //TODO: clamp((int)floor(log2(ssR)) - LOG_MAX_OFFSET, 0, MAX_MIP_LEVEL);

    int2 ssP = int2(ssR*unitOffset) + ssC;

    float3 P = gPositionTexture[ssP].xyz;

    P = mul(gViewMatrix, float4(P, 1.0)).xyz;

    return P;
}

float2 tapLocation(int sampleNumber, float spinAngle, out float ssR)
{
    // Radius relative to ssR
    float alpha = float(sampleNumber + 0.5) * (1.0 / gNumSamples);
    float angle = alpha * (gNumSpiralTurns * 6.28) + spinAngle;

    ssR = alpha;
    return float2(cos(angle), sin(angle));
}

float sampleAO(uint2 screenSpacePos, float3 originPos, float3 normal, float ssDiskRadius, int tapIndex, float randomPatternRotationAngle)
{
    float ssR;
    float2 unitOffset = tapLocation(tapIndex, randomPatternRotationAngle, ssR);
    ssR *= ssDiskRadius;

    // The occluding point in camera space
    float3 Q = getOffsetPosition(screenSpacePos, unitOffset, ssR);

    float3 v = Q - originPos;

    float vv = dot(v, v);
    float vn = dot(v, normal);

    const float epsilon = 0.01;
    float f = max(gRadius2 - vv, 0.0); 
    
    return f * f * f * max((vn - gBias) / (epsilon + vv), 0.0);
}

float4 ps_main(float4 position : SV_Position) : SV_Target0
{
    uint2 screenSpacePos = (uint2)position.xy;

    float3 originPos = gPositionTexture[screenSpacePos].xyz;
    originPos = mul(gViewMatrix, float4(originPos, 1.0)).xyz;
    float3 normal = gNormalTexture[screenSpacePos].xyz;//reconstructNormal(originPos);
    normal = mul(gViewMatrix, float4(normal, 0.0)).xyz;

    // Hash function used in the HPG12 AlchemyAO paper
    float randomPatternRotationAngle = (3 * screenSpacePos.x ^ screenSpacePos.y + screenSpacePos.x * screenSpacePos.y) * 10;
    float ssDiskRadius = -gProjScale * gRadius / originPos.z;

    float ao = 0.0;
    for (int i = 0; i < gNumSamples; i++)
    {
        ao += sampleAO(screenSpacePos, originPos, normal, ssDiskRadius, i, randomPatternRotationAngle);
    }

    float temp = gRadius2 * gRadius;
    ao /= temp * temp;

    float A = max(0.0, 1.0 - ao * gIntensity * (5.0 / gNumSamples));

    return A;
}

#endif

EDIT: update, using reconstructNormals() to reconstruct normals - for simpler models it looks okay (the chair) but the more advanced models like the alien looks really bad/blocky. Any way to fix this?

Answer 1

I'm not the expert to answer this question, but perhaps:

Points on your detailed normal map are skewing some of your offset samples back into the geometry?

The details on your normal map contain dark spots/crevices, which are being read by the shader as corners?