UPDATE: I've narrowed it down to the domain shader - the line that physically changes the output triangles' position. When I comment out this line, it solves the problem, although then I have no tessellation. Do I need to do some kind of manual depth writing after changes the position?
Domain Shader:
cbuffer cbDomainInfo
{
float3 g_vEyePosition;
float _padding;
};
struct PatchTess
{
float flEdges[ 3 ] : SV_TessFactor;
float flInside : SV_InsideTessFactor;
};
struct DomainInput
{
float4 vPosition : POSITION;
float2 vTexcoord : TEXCOORD0;
float3 vNormal : NORMAL;
float3 vViewVec : TEXCOORD1;
float flAlpha : ALPHA;
};
struct DomainOutput
{
float4 vPosition : SV_POSITION;
float2 vTexcoord : TEXCOORD0;
float3 vNormal : NORMAL;
float3 vViewVec : TEXCOORD1;
float flAlpha : ALPHA;
};
Texture2D g_texDispMap;
SamplerState g_samSampler;
[ domain( "tri" ) ]
DomainOutput DS( PatchTess input, float3 uvwCoord : SV_DomainLocation, const OutputPatch<DomainInput, 3> patch )
{
DomainOutput output;
output.vPosition = uvwCoord.x * patch[ 0 ].vPosition + uvwCoord.y * patch[ 1 ].vPosition + uvwCoord.z * patch[ 2 ].vPosition;
output.vNormal = uvwCoord.x * patch[ 0 ].vNormal + uvwCoord.y * patch[ 1 ].vNormal + uvwCoord.z * patch[ 2 ].vNormal;
output.vTexcoord = uvwCoord.x * patch[ 0 ].vTexcoord + uvwCoord.y * patch[ 1 ].vTexcoord + uvwCoord.z * patch[ 2 ].vTexcoord;
output.vViewVec = uvwCoord.x * patch[ 0 ].vViewVec + uvwCoord.y * patch[ 1 ].vViewVec + uvwCoord.z * patch[ 2 ].vViewVec;
output.flAlpha = uvwCoord.x * patch[ 0 ].flAlpha + uvwCoord.y * patch[ 1 ].flAlpha + uvwCoord.z * patch[ 2 ].flAlpha;
output.vNormal = normalize( output.vNormal );
float flMipInterval = 20.f;
float flMipLevel = clamp( ( distance( output.vPosition, float4( g_vEyePosition, 1.f ) - flMipInterval ) ) / flMipInterval, 0.f, 6.f );
float flHeightMap = g_texDispMap.SampleLevel( g_samSampler, output.vTexcoord, flMipLevel ).r;
//TODO: Pass in height scale via cbuffer
//This next line causes the problem:
output.vPosition += float4( ( 20.f *( flHeightMap - 1.f ) ) * output.vNormal, 1.f );
return output;
}