0
\$\begingroup\$

I'm writing an OpenGL/DX11 Windows/Linux "engine" and I've encountered a confusing behavior in DX, specifically. For math, I'm using GLM, which means all of my HLSL mull() calls relating to transformations have the matrix as the first parameter.

All that being said, here's a screenshot with the current setup, running OpenGL.

Phong lighting using OpenGL 4.6

And here's nothing changed, but switching to DX11.

Phong lighting using Direct3D 11

The terrain is still fine, but the ship's normals are obviously in disagreement. If I transpose the TBN matrix, I get correct ship lighting and incorrect terrain lighting, like this.

Phong lighting using Direct3D 11 with a transposed TBN matrix

Of course the first thing that comes to mind is column versus row major (hence me even trying a transposed matrix in the first place). I've not had issues with matrix/vector notation since everything uses GLM CPU-side. Even if that were the issue though, it wouldn't explain why one asset requires a transposition and the other does not.

I'll provide my fragment and pixel shaders for sanity checking if anyone would like to peruse them, but I'm hoping (guessing?) this is more of an asset pipeline issue than a code issue. Regardless, I'm eager to solve it. =) Thank you in advance!

GLSL fragment code:

#version 460 core

layout(location = 0) in VS_OUT {
    vec4 world_pos;
    vec3 normal;
    vec3 tangent;
    vec3 bitangent;
    vec2 texcoords;
} ps_in;

struct DirectionalLight {
    vec4 direction;

    vec4 ambient;
    vec4 diffuse;
    vec4 specular;
    float shine;
    float attenuation;
};

struct PointLight {
    vec4  position;

    vec4  ambient;
    vec4  diffuse;
    vec4  specular;
    float shine;
    float attenuation;
};

struct SpotLight {
    vec4  position;
    vec4  heading;

    vec4  ambient;
    vec4  diffuse;
    vec4  specular;
    float shine;
    float attenuation;

    float inner_cone;
    float outer_cone;
};

struct Material {
    vec4  ambient;
    vec4  diffuse;
    vec4  specular;
    float shine;
};

layout(std140, binding = 1) uniform WorldMaterial {
    mat4  world_matrix;
    Material material;
};

layout(std140, binding = 2) uniform LightParams {
    DirectionalLight directional;
    PointLight point;
    SpotLight  spot;
    vec4 camera_pos;
};

layout (binding = 0) uniform sampler2D diffuse_map;
layout (binding = 1) uniform sampler2D normal_map;
layout (binding = 2) uniform sampler2D specular_map;
layout (binding = 3) uniform sampler2D emission_map;

layout(location = 0) out vec4 final_color;

vec4 phong_model(vec3 normal, vec4 light_pos, vec4 ambient, vec4 diffuse,
                vec4 specular, float attenuation) {
    vec4 surface_to_light  = normalize(light_pos - ps_in.world_pos);
    vec4 surface_to_camera = normalize(camera_pos - ps_in.world_pos);
    
    //ambient
    vec4 ambient_result = ambient * material.ambient;

    //diffuse
    float diffuse_coefficient = max(0.0, dot(normal.xyz, surface_to_light.xyz));
    vec4 diffuse_result = diffuse_coefficient * diffuse;
    
    //specular
    float shine = 0.0;
    if(diffuse_coefficient > 0.0) {
        vec3 reflected = reflect(-surface_to_light.xyz, normal);
        shine = pow(max(0.0, dot(surface_to_camera.xyz, reflected)),
                    material.shine);
    }

    vec4 specular_result = specular * material.specular * shine;
    
    //attenuation
    float to_light = length(light_pos - ps_in.world_pos);
    float attenuation_result = 1.0 / dot(
        vec3(0.0, attenuation, 0.0),
        vec3(1.0, to_light, pow(to_light, 2))
    );

    return ambient_result + attenuation_result *
                            (diffuse_result + specular_result);
}

vec4 directional_calc(vec3 normal) {
    // ambient
    vec4 ambient = directional.ambient * material.ambient;

    // diffuse 
    vec4  direction = normalize(-directional.direction);
    float angle     = max(dot(normal, direction.xyz), 0.0);
    vec4  diffuse   = directional.diffuse * angle * material.diffuse;
    
    // specular
    vec3  view_dir    = normalize(camera_pos.xyz - ps_in.world_pos.xyz);
    vec3  reflect_dir = reflect(-direction.xyz, normal);  
    float shine       = pow(max(dot(view_dir, reflect_dir), 0.0),
                            material.shine);
    vec4  specular    = directional.specular * material.specular * shine;

    return vec4(ambient + diffuse + specular);
}

vec4 point_calc(vec3 normal) {
    return phong_model(normal, point.position, point.ambient,
                    point.diffuse, point.specular, point.attenuation);
}

vec4 spot_calc(vec3 normal) {
    vec4 light_to_surface = normalize(ps_in.world_pos - spot.position);
    float pixel_angle = dot(light_to_surface, spot.heading);

    if(pixel_angle > spot.outer_cone) {
        vec4 light_intensity = phong_model(normal, spot.position,
                                        spot.ambient, spot.diffuse,
                                        spot.specular, spot.attenuation);

        float theta   = pixel_angle;
        float epsilon = spot.inner_cone - spot.outer_cone;
        float fade    = clamp((theta - spot.outer_cone) / epsilon, 0.0, 1.0); 

        return light_intensity * fade;
    }
    else {
        return vec4(0.0);
    }
}

void main() {
    mat3 TBN = mat3(ps_in.tangent,
                    ps_in.bitangent,
                    ps_in.normal);

    vec3 normal = texture(normal_map, ps_in.texcoords).rgb;
    normal = normal * 2.0 - 1.0;
    normal = normalize(TBN * normal);

    // vec3 normal = normalize(ps_in.normal_mat * ps_in.normal);

    // combination!
    vec4 directional = directional_calc(normal);
    vec4 point = point_calc(normal);
    vec4 spot = spot_calc(normal);

    vec4 light_intensity = directional + point + spot;
    vec4 texel = texture(diffuse_map, ps_in.texcoords);

    final_color = light_intensity * texel;
}

HLSL pixel code:

struct Material {
    float4 ambient;
    float4 diffuse;
    float4 specular;
    float  shine;
};

struct DirectionalLight {
    float4 direction;

    float4 ambient;
    float4 diffuse;
    float4 specular;
    float  shine;
    float  attenuation;
};

struct PointLight {
    float4 position;

    float4 ambient;
    float4 diffuse;
    float4 specular;
    float  shine;
    float  attenuation;
};

struct SpotLight {
    float4 position;
    float4 heading;

    float4 ambient;
    float4 diffuse;
    float4 specular;
    float  shine;
    float  attenuation;

    float inner_cone;
    float outer_cone;
};

cbuffer WorldMaterial : register(b1) {
    float4x4 world_matrix;
    Material material;
}

cbuffer LightParams : register(b2) {
    DirectionalLight directional;
    PointLight       pointlight;
    SpotLight        spotlight;
    float4           camera_pos;
};

Texture2D    diffuse_map       : register(t0);
SamplerState diffuse_map_samp  : register(s0);
Texture2D    normal_map        : register(t1);
SamplerState normal_map_samp   : register(s1);
Texture2D    specular_map      : register(t2);
SamplerState specular_map_samp : register(s2);
Texture2D    emission_map      : register(t3);
SamplerState emission_map_samp : register(s3);

struct VS_OUT {
    float4 sv_pos    : SV_POSITION;
    float4 world_pos : POSITION;
    float4 normal    : NORMAL;
    float4 tangent   : TANGENT;
    float4 bitangent : BINORMAL;
    float2 texcoords : TEXCOORD;
};

float4 phong_model(float3 normal, float4 light_pos, float4 ambient, float4 diffuse,
                float4 specular, float attenuation, float4 world_pos) {
    float4 surface_to_light  = normalize(light_pos - world_pos);
    float4 surface_to_camera = normalize(camera_pos - world_pos);
    
    //ambient
    float4 ambient_result = ambient * material.ambient;

    //diffuse
    float diffuse_coefficient = max(0.0, dot(normal.xyz, surface_to_light.xyz));
    float4 diffuse_result = diffuse_coefficient * diffuse;
    
    //specular
    float shine = 0.0;
    if(diffuse_coefficient > 0.0) {
        float3 reflected = reflect(-surface_to_light.xyz, normal);
        shine = pow(max(0.0, dot(surface_to_camera.xyz, reflected)),
                    material.shine);
    }

    float4 specular_result = specular * material.specular * shine;
    
    //attenuation
    float to_light = length(light_pos - world_pos);
    float attenuation_result = 1.0 / dot(
        float3(0.0, attenuation, 0.0),
        float3(1.0, to_light, pow(to_light, 2))
    );

    return ambient_result + attenuation_result *
        (diffuse_result + specular_result);
}

float4 directional_calc(float3 normal, float4 world_pos) {
    // ambient
    float4 ambient = directional.ambient * material.ambient;

    // diffuse 
    float4 direction = normalize(-directional.direction);
    float  angle     = max(dot(normal, direction.xyz), 0.0);
    float4 diffuse   = directional.diffuse * angle * material.diffuse;
    
    // specular
    float3 view_dir    = normalize(camera_pos.xyz - world_pos.xyz);
    float3 reflect_dir = reflect(-direction.xyz, normal);  
    float  shine       = pow(max(dot(view_dir, reflect_dir), 0.0),
                            material.shine);
    float4 specular    = directional.specular * material.specular * shine;

    return float4(ambient + diffuse + specular);
}

float4 point_calc(float3 normal, float4 world_pos) {
    return phong_model(normal, pointlight.position, pointlight.ambient,
                    pointlight.diffuse, pointlight.specular,
                    pointlight.attenuation, world_pos);
}

float4 spot_calc(float3 normal, float4 world_pos) {
    float4 light_to_surface = normalize(world_pos - spotlight.position);
    float pixel_angle = dot(light_to_surface, spotlight.heading);

    if(pixel_angle > spotlight.outer_cone) {
        float4 light_intensity =
            phong_model(normal, spotlight.position,
                        spotlight.ambient, spotlight.diffuse,
                        spotlight.specular, spotlight.attenuation,
                        world_pos);

        float theta = pixel_angle;
        float epsilon = spotlight.inner_cone - spotlight.outer_cone;
        float fade = clamp((theta - spotlight.outer_cone) / epsilon, 0.0, 1.0); 

        return light_intensity * fade;
    }
    else {
        return float4(0.0, 0.0, 0.0, 0.0);
    }
}

float4 main(VS_OUT input) : SV_TARGET
{
    float3x3 TBN = float3x3(
        input.tangent.xyz,
        input.bitangent.xyz,
        input.normal.xyz
    );

    float3 normal = normal_map.Sample(normal_map_samp, input.texcoords).rgb;
    normal = normal * 2.0 - 1.0;
    normal = normalize(mul(TBN, normal));

    float4 dir_val   = directional_calc(normal, input.world_pos);
    float4 point_val = point_calc(normal, input.world_pos);
    float4 spot_val  = spot_calc(normal, input.world_pos);

    float4 light_intensity = dir_val + point_val + spot_val;
    float4 texel = diffuse_map.Sample(diffuse_map_samp, input.texcoords);

    return light_intensity * texel;
}
\$\endgroup\$

1 Answer 1

0
\$\begingroup\$

Well, that was silly. =) It turned out I was normalizing normals and tangents in my GLSL shader and not in my DX shader. Naturally, that mean the normals I was calculating myself (for the terrain; the ship already had its normals baked in) were whacky and GL just happened to be instructed to be more cautious.

Easy peasy!

\$\endgroup\$
1
  • 1
    \$\begingroup\$ Don't forget to mark this answer as Accepted if it solved your problem. \$\endgroup\$
    – DMGregory
    Jun 9, 2022 at 0:42

You must log in to answer this question.

Not the answer you're looking for? Browse other questions tagged .