Unity ignoring the deferred shader

Question

I've been following this tutorial to get a cel-shader working in Unity 5 along with the standard shaders (explained in the last third of the article).

My issue is that when I got to replacing the built-in deferred shader, Unity seems to ignore it! I've been trying really hard to get it to work but it simply seems to be either ignoring my modified shader, which is actually just the default deferred shader with a small change in how the ndotl is calculated.

Here's a picture of what I've configured/modified in order to try to get it working:

Here's the full code for the modified Internal-DeferredShading file:

    Shader "Hidden/Celshading-DeferredShading" {
Properties {
    _LightTexture0 ("", any) = "" {}
    _LightTextureB0 ("", 2D) = "" {}
    _ShadowMapTexture ("", any) = "" {}
    _SrcBlend ("", Float) = 1
    _DstBlend ("", Float) = 1
}
SubShader {

// Pass 1: Lighting pass
//  LDR case - Lighting encoded into a subtractive ARGB8 buffer
//  HDR case - Lighting additively blended into floating point buffer
Pass {
    ZWrite Off
    Blend [_SrcBlend] [_DstBlend]

CGPROGRAM
#pragma target 3.0
#pragma vertex vert_deferred
#pragma fragment frag
#pragma multi_compile_lightpass
#pragma multi_compile ___ UNITY_HDR_ON

#pragma exclude_renderers nomrt

#include "UnityCG.cginc"
#include "UnityDeferredLibrary.cginc"
#include "UnityPBSLighting.cginc"
#include "UnityStandardUtils.cginc"
#include "UnityStandardBRDF.cginc"

sampler2D _CameraGBufferTexture0;
sampler2D _CameraGBufferTexture1;
sampler2D _CameraGBufferTexture2;

half4 CalculateLight (unity_v2f_deferred i)
{
    float3 wpos;
    float2 uv;
    float atten, fadeDist;
    UnityLight light;
    UNITY_INITIALIZE_OUTPUT(UnityLight, light);
    UnityDeferredCalculateLightParams (i, wpos, uv, light.dir, atten, fadeDist);

    half4 gbuffer0 = tex2D (_CameraGBufferTexture0, uv);
    half4 gbuffer1 = tex2D (_CameraGBufferTexture1, uv);
    half4 gbuffer2 = tex2D (_CameraGBufferTexture2, uv);

    light.color = _LightColor.rgb * atten;
    half3 baseColor = gbuffer0.rgb;
    half3 specColor = gbuffer1.rgb;
    half oneMinusRoughness = gbuffer1.a;
    half3 normalWorld = gbuffer2.rgb * 2 - 1;
    normalWorld = normalize(normalWorld);
    float3 eyeVec = normalize(wpos-_WorldSpaceCameraPos);
    half oneMinusReflectivity = 1 - SpecularStrength(specColor.rgb);

    if (light.ndotl <= 0.0) light.ndotl = 0;
    else light.ndotl = 1;

    //light.ndotl = 0;

    UnityIndirect ind;
    UNITY_INITIALIZE_OUTPUT(UnityIndirect, ind);
    ind.diffuse = 0;
    ind.specular = 0;

    half4 res = UNITY_BRDF_PBS (baseColor, specColor, oneMinusReflectivity, oneMinusRoughness, normalWorld, -eyeVec, light, ind);

    return res;
}

#ifdef UNITY_HDR_ON
half4
#else
fixed4
#endif
frag (unity_v2f_deferred i) : SV_Target
{
    half4 c = CalculateLight(i);
    #ifdef UNITY_HDR_ON
    return c;
    #else
    return exp2(-c);
    #endif
}

ENDCG
}


// Pass 2: Final decode pass.
// Used only with HDR off, to decode the logarithmic buffer into the main RT
Pass {
    ZTest Always Cull Off ZWrite Off
    Stencil {
        ref [_StencilNonBackground]
        readmask [_StencilNonBackground]
        // Normally just comp would be sufficient, but there's a bug and only front face stencil state is set (case 583207)
        compback equal
        compfront equal
    }

CGPROGRAM
#pragma target 3.0
#pragma vertex vert
#pragma fragment frag
#pragma exclude_renderers nomrt

sampler2D _LightBuffer;
struct v2f {
    float4 vertex : SV_POSITION;
    float2 texcoord : TEXCOORD0;
};

v2f vert (float4 vertex : POSITION, float2 texcoord : TEXCOORD0)
{
    v2f o;
    o.vertex = mul(UNITY_MATRIX_MVP, vertex);
    o.texcoord = texcoord.xy;
    return o;
}

fixed4 frag (v2f i) : SV_Target
{
    return -log2(tex2D(_LightBuffer, i.texcoord));
}
ENDCG 
}

}
Fallback Off
}

Does anyone have an idea why this isn't working? I've even tried replacing the if/else that sets the ndotl to always 0 or always 1 but it almost seems like Unity is ignoring that line altogether, or something is overriding it later down the line and I'm not sure what.

Other than these modifications, the Unity Project only contains a handful of standard assets.

Answer 1

Update for Unity 5.5

See @TWickz's answer.

---

TL;DR

Add #define UNITY_BRDF_PBS BRDF2_Unity_PBS right before #pragma's and #define's in the shader.

---

Finding the solution was time-consuming, but I gained useful knowledge and so will you if you follow my “investigation log” below. I hope it's suitable answer format in this case.

Step 1

Tinkering with light.ndotl doesn't change anything, as described by question author. Let's do a good old color change then:

half3 baseColor = gbuffer0.rgb; // find this line
baseColor.g = 0; // add this line after it

And we get nice purple shade:

Step 2

Now we know that it's not shader that is ignored, but a light.ndotl specifically.

UNITY_BRDF_PBS() is pretty much the only possible suspect, because light.ndotl is used only there out of all code in the function after light.ndotl initialization.

No choice left other than remove the no longer useful light.ndotl and explore UNITY_BRDF_PBS() definition.

Step 3

Let's find UNITY_BRDF_PBS in the files that are #included into our shader.

In UnityPBSLighting.cginc:

//-------------------------------------------------------------------------------------
// Default BRDF to use:
#if !defined (UNITY_BRDF_PBS) // allow to explicitly override BRDF in custom shader
    // still add safe net for low shader models, otherwise we might end up with shaders failing to compile
    // the only exception is WebGL in 5.3 - it will be built with shader target 2.0 but we want it to get rid of constraints, as it is effectively desktop
    #if SHADER_TARGET < 30 && !UNITY_53_SPECIFIC_TARGET_WEBGL
        #define UNITY_BRDF_PBS BRDF3_Unity_PBS
    #elif UNITY_PBS_USE_BRDF3
        #define UNITY_BRDF_PBS BRDF3_Unity_PBS
    #elif UNITY_PBS_USE_BRDF2
        #define UNITY_BRDF_PBS BRDF2_Unity_PBS
    #elif UNITY_PBS_USE_BRDF1
        #define UNITY_BRDF_PBS BRDF1_Unity_PBS
    #elif defined(SHADER_TARGET_SURFACE_ANALYSIS)
        // we do preprocess pass during shader analysis and we dont actually care about brdf as we need only inputs/outputs
        #define UNITY_BRDF_PBS BRDF1_Unity_PBS
    #else
        #error something broke in auto-choosing BRDF
    #endif
#endif

It's clear that real work is done by one of BRDFn_Unity_PBS() functions. We should start by finding the best one.

Step 4

Again, there is nothing more about BRDFn_Unity_PBS() in this file, so we should find it in the #included files. UnityStandardBRDF.cginc contains all three variants of this function. №1 is apparently the best since comment of №3 starts with:

// Old school, not microfacet based Modified Normalized Blinn-Phong BRDF

Variant №1 from the UnityStandardBRDF.cginc:

half4 BRDF1_Unity_PBS (half3 diffColor, half3 specColor, half oneMinusReflectivity, half oneMinusRoughness,
    half3 normal, half3 viewDir,
    UnityLight light, UnityIndirect gi)
{
    half roughness = 1-oneMinusRoughness;
    half3 halfDir = Unity_SafeNormalize (light.dir + viewDir);

#if UNITY_BRDF_GGX 
    // NdotV should not be negative for visible pixels, but it can happen due to perspective projection and normal mapping
    // In this case we will modify the normal so it become valid and not cause weird artifact (other game try to clamp or abs the NdotV to prevent this trouble).
    // The amount we shift the normal toward the view vector is define by the dot product.
    // This correction is only apply with smithJoint visibility function because artifact are more visible in this case due to highlight edge of rough surface
    half shiftAmount = dot(normal, viewDir);
    normal = shiftAmount < 0.0f ? normal + viewDir * (-shiftAmount + 1e-5f) : normal;
    // A re-normalization should be apply here but as the shift is small we don't do it to save ALU.
    //normal = normalize(normal);

    // As we have modify the normal we need to recalculate the dot product nl. 
    // Note that  light.ndotl is a clamped cosine and only the ForwardSimple mode use a specific ndotL with BRDF3
    half nl = DotClamped(normal, light.dir);
#else
    half nl = light.ndotl;
#endif
    half nh = BlinnTerm (normal, halfDir);

    // ... more code ...
}

“As we have modify the normal we need to recalculate the dot product nl. Note that light.ndotl is a clamped cosine and only the ForwardSimple mode use a specific ndotL with BRDF3”

This comment looks suspicious. Everything suggests that this function blatantly ignores the light.ndotl value.

Step 5

Apparently, function №1 is too fancy for us, better use a more old and faithful variant. Earlier in UNITY_BRDF_PBS() definition we read:

#if !defined (UNITY_BRDF_PBS) // allow to explicitly override BRDF in custom shader

Let's override and use a bit less fancy variant then, i.e. №2. To our shader file after the first CGPROGRAM add:

#define UNITY_BRDF_PBS BRDF2_Unity_PBS

And it works!

Further development

It's a good enough solution for tinkering, but in production code it's probably a bad idea to force variant №2 unconditionally. Better mimic standard approach:

// Override UNITY_BRDF_PBS to exclude BRDF1_Unity_PBS variant,
// otherwise, it mimics the definition from UnityPBSLighting.cginc.
#if SHADER_TARGET < 30 && !UNITY_53_SPECIFIC_TARGET_WEBGL
#define UNITY_BRDF_PBS BRDF3_Unity_PBS
#elif UNITY_PBS_USE_BRDF3
#define UNITY_BRDF_PBS BRDF3_Unity_PBS
#else
#define UNITY_BRDF_PBS BRDF2_Unity_PBS
#endif

Answer 2

Note that some of the functions used here have been deprecated in Unity 5.5, but I have tested it to work.

---

In Unity 5.5, many changes have been made. The most significant, being that the dot product is now calculated on the fly, and the light.ndotl field is no longer taken into consideration anywhere in code.

I spoke to the original author of the blog David Leon and he mentioned that using the LambertTerm function with a slight modification, in the standard BRDF instead of the dot product would help.

Based on that input, these are the steps I followed to get the shader working.

Step 1 :

Download the built in shaders for Unity 5.5

Step 2 :

Copy the following files into your Assets/Resources folder (Create the folder if not available)

1. UnityStandardBRDF.cginc
2. UnityDeprecated.cginc

Rename the two files as UnityStandardBRDFCustom.cginc and UnityDeprecatedEX.cginc

Step 3 :

The LambertTerm function has been moved to the Deprecated section of code, so we make the change in the UnityDepcreatedEX.cginc file.

Remove all code in the UnityDeprecatedEX.cginc file and replace with this.

inline half CustomDotClamped ( half3 a, half3 b ){ 
 #if (SHADER_TARGET < 30)
    return saturate(dot(a,b));
 #else
    return max(0.0h, dot(a,b));
#endif
}

inline half CustomLambertTerm ( half3 normal, half3 lightDir ) {
  return smoothstep(0.0,0.05f, CustomDotClamped (normal, lightDir));
}

Step 4 :

Remove all content of the UnityStandardBRDFCustom file and replace with the following custom implementation of BRDF_1_Unity_PBS method. Basically what I've done to the method is renamed it to BRDF_CUSTOM_Unity_PBS and replaced the calculation of nl with the custom LambertTerm function.

#include "UnityCG.cginc"
#include "UnityStandardConfig.cginc"
#include "UnityLightingCommon.cginc"
#include "UnityDeprecatedEx.cginc"

half4 BRDF_CUSTOM_Unity_PBS (half3 diffColor, half3 specColor, half oneMinusReflectivity, half smoothness,
    half3 normal, half3 viewDir,
    UnityLight light, UnityIndirect gi)
{
    half perceptualRoughness = SmoothnessToPerceptualRoughness (smoothness);
    half3 halfDir = Unity_SafeNormalize (light.dir + viewDir);

// NdotV should not be negative for visible pixels, but it can happen due to perspective projection and normal mapping
// In this case normal should be modified to become valid (i.e facing camera) and not cause weird artifacts.
// but this operation adds few ALU and users may not want it. Alternative is to simply take the abs of NdotV (less correct but works too).
// Following define allow to control this. Set it to 0 if ALU is critical on your platform.
// This correction is interesting for GGX with SmithJoint visibility function because artifacts are more visible in this case due to highlight edge of rough surface
// Edit: Disable this code by default for now as it is not compatible with two sided lighting used in SpeedTree.
#define UNITY_HANDLE_CORRECTLY_NEGATIVE_NDOTV 0 

#if UNITY_HANDLE_CORRECTLY_NEGATIVE_NDOTV
    // The amount we shift the normal toward the view vector is defined by the dot product.
    half shiftAmount = dot(normal, viewDir);
    normal = shiftAmount < 0.0f ? normal + viewDir * (-shiftAmount + 1e-5f) : normal;
    // A re-normalization should be applied here but as the shift is small we don't do it to save ALU.
    //normal = normalize(normal);

    half nv = saturate(dot(normal, viewDir)); // TODO: this saturate should no be necessary here
#else
    half nv = abs(dot(normal, viewDir));    // This abs allow to limit artifact
#endif

    //half nl = saturate(dot(normal, light.dir));
    //celshading
    half nl = CustomLambertTerm(normal, light.dir);

    half nh = saturate(dot(normal, halfDir));

    half lv = saturate(dot(light.dir, viewDir));
    half lh = saturate(dot(light.dir, halfDir));

    // Diffuse term
    half diffuseTerm = DisneyDiffuse(nv, nl, lh, perceptualRoughness) * nl;

    // Specular term
    // HACK: theoretically we should divide diffuseTerm by Pi and not multiply specularTerm!
    // BUT 1) that will make shader look significantly darker than Legacy ones
    // and 2) on engine side "Non-important" lights have to be divided by Pi too in cases when they are injected into ambient SH
    half roughness = PerceptualRoughnessToRoughness(perceptualRoughness);
#if UNITY_BRDF_GGX
    half V = SmithJointGGXVisibilityTerm (nl, nv, roughness);
    half D = GGXTerm (nh, roughness);
#else
    // Legacy
    half V = SmithBeckmannVisibilityTerm (nl, nv, roughness);
    half D = NDFBlinnPhongNormalizedTerm (nh, PerceptualRoughnessToSpecPower(perceptualRoughness));
#endif

    half specularTerm = V*D * UNITY_PI; // Torrance-Sparrow model, Fresnel is applied later

#   ifdef UNITY_COLORSPACE_GAMMA
        specularTerm = sqrt(max(1e-4h, specularTerm));
#   endif

    // specularTerm * nl can be NaN on Metal in some cases, use max() to make sure it's a sane value
    specularTerm = max(0, specularTerm * nl);
#if defined(_SPECULARHIGHLIGHTS_OFF)
    specularTerm = 0.0;
#endif

    // surfaceReduction = Int D(NdotH) * NdotH * Id(NdotL>0) dH = 1/(roughness^2+1)
    half surfaceReduction;
#   ifdef UNITY_COLORSPACE_GAMMA
        surfaceReduction = 1.0-0.28*roughness*perceptualRoughness;      // 1-0.28*x^3 as approximation for (1/(x^4+1))^(1/2.2) on the domain [0;1]
#   else
        surfaceReduction = 1.0 / (roughness*roughness + 1.0);           // fade \in [0.5;1]
#   endif

    // To provide true Lambert lighting, we need to be able to kill specular completely.
    specularTerm *= any(specColor) ? 1.0 : 0.0;

    half grazingTerm = saturate(smoothness + (1-oneMinusReflectivity));
    half3 color =   diffColor * (gi.diffuse + light.color * diffuseTerm)
                    + specularTerm * light.color * FresnelTerm (specColor, lh)
                    + surfaceReduction * gi.specular * FresnelLerp (specColor, grazingTerm, nv);

    return half4(color, 1);
}

Step 5 :

Create the shader file. Name it whatever you like. The shader code is at the end of the answer :)

Key things to note on the shader implementation.

1. We override the definition of the UNITY_BRDF_PBS macro with the customized function we implemented in the UnityStandardBRDFCustom.cginc (Be sure to do this right after CGPROGRAM and before the pragmas).

2. Secondly, since we override the UNITY_BRDF_PBS function completely, we no longer need a reference to the UnityPBSLighting.cginc. Therefore it's commented out.

3. We use both UnityStandardBRDF and UnityStandardBRDFCustom cginc files in the shader declared in the mentioned order respectively.

Step 6:

Go to Edit->Project Settings-> Graphics and update the Deffered shader to your custom shader. Make sure that your camera's rendering path is set to deffered.

Shader "Hidden/Internal-DeferredShading" {
Properties {
    _LightTexture0 ("", any) = "" {}
    _LightTextureB0 ("", 2D) = "" {}
    _ShadowMapTexture ("", any) = "" {}
    _SrcBlend ("", Float) = 1
    _DstBlend ("", Float) = 1

}
SubShader {

// Pass 1: Lighting pass
//  LDR case - Lighting encoded into a subtractive ARGB8 buffer
//  HDR case - Lighting additively blended into floating point buffer
Pass {
    ZWrite Off
    Blend [_SrcBlend] [_DstBlend]

CGPROGRAM

#define UNITY_BRDF_PBS BRDF_CUSTOM_Unity_PBS

#pragma multi_compile FANCY_STUFF_OFF FANCY_STUFF_ON
#pragma target 3.0
#pragma vertex vert_deferred
#pragma fragment frag
#pragma multi_compile_lightpass
#pragma multi_compile ___ UNITY_HDR_ON

#pragma exclude_renderers nomrt

#include "UnityCG.cginc"
#include "UnityDeferredLibrary.cginc"
//#include "UnityPBSLighting.cginc"
#include "UnityStandardUtils.cginc"
#include "UnityGBuffer.cginc"
#include "UnityStandardBRDF.cginc"
#include "UnityStandardBRDFCustom.cginc"

sampler2D _CameraGBufferTexture0;
sampler2D _CameraGBufferTexture1;
sampler2D _CameraGBufferTexture2;

half4 CalculateLight (unity_v2f_deferred i)
{
    float3 wpos;
    float2 uv;
    float atten, fadeDist;
    UnityLight light;
    UNITY_INITIALIZE_OUTPUT(UnityLight, light);
    UnityDeferredCalculateLightParams (i, wpos, uv, light.dir, atten, fadeDist);

    light.color = _LightColor.rgb * atten;



    // unpack Gbuffer
    half4 gbuffer0 = tex2D (_CameraGBufferTexture0, uv);
    half4 gbuffer1 = tex2D (_CameraGBufferTexture1, uv);
    half4 gbuffer2 = tex2D (_CameraGBufferTexture2, uv);
    UnityStandardData data = UnityStandardDataFromGbuffer(gbuffer0, gbuffer1, gbuffer2);

    data.diffuseColor.rgb   = gbuffer0.rgb;

    float3 eyeVec = normalize(wpos-_WorldSpaceCameraPos);
    half oneMinusReflectivity = 1 - SpecularStrength(data.specularColor.rgb);

    UnityIndirect ind;
    UNITY_INITIALIZE_OUTPUT(UnityIndirect, ind);
    ind.diffuse = 0;
    ind.specular = 0;

    if (light.ndotl <= 0.0) 
     light.ndotl = 0;
    else 
     light.ndotl = 1;

    half4 res = UNITY_BRDF_PBS (data.diffuseColor, data.specularColor, oneMinusReflectivity, data.smoothness, data.normalWorld, -eyeVec, light, ind);

    return res;
}

#ifdef UNITY_HDR_ON
half4
#else
fixed4
#endif
frag (unity_v2f_deferred i) : SV_Target
{
    half4 c = CalculateLight(i);
    #ifdef UNITY_HDR_ON
    return c;
    #else
    return exp2(-c);
    #endif
}

ENDCG
}


// Pass 2: Final decode pass.
// Used only with HDR off, to decode the logarithmic buffer into the main RT
Pass {
    ZTest Always Cull Off ZWrite Off
    Stencil {
        ref [_StencilNonBackground]
        readmask [_StencilNonBackground]
        // Normally just comp would be sufficient, but there's a bug and only front face stencil state is set (case 583207)
        compback equal
        compfront equal
    }

CGPROGRAM
#pragma target 3.0
#pragma vertex vert
#pragma fragment frag
#pragma exclude_renderers nomrt

#include "UnityCG.cginc"

sampler2D _LightBuffer;
struct v2f {
    float4 vertex : SV_POSITION;
    float2 texcoord : TEXCOORD0;
};

v2f vert (float4 vertex : POSITION, float2 texcoord : TEXCOORD0)
{
    v2f o;
    o.vertex = UnityObjectToClipPos(vertex);
    o.texcoord = texcoord.xy;
#ifdef UNITY_SINGLE_PASS_STEREO
    o.texcoord = TransformStereoScreenSpaceTex(o.texcoord, 1.0f);
#endif
    return o;
}

fixed4 frag (v2f i) : SV_Target
{
    return -log2(tex2D(_LightBuffer, i.texcoord));
}
ENDCG 
}

}
Fallback Off
}