Shader / Reconstructing position from DEPTH in VR through Projection Matrix

Question

In Unity you can access the Depth-Buffer in the Pixel Shader. With that Depth-Buffer and the Nearplane, Farplane, CamToWorldMatrix and FOV you can calculate every pixel position of the objects the camera is seeing.

in VR though I get the wrong positions - because a VR frustum is slightly skewed. (http://doc-ok.org/?p=77)

So now we dont use the variables above to determine the Position - Now should be using the Projection Matrix.

Unitys "OnRenderImage()" is rendering alternately left and right eye of the VR device. So we "take" the projection matrix from each different eye. Also the FOV , Near and Far Planes, should be correct. We have also the screen Resolution. - which is exactly "540x600"

The Big Question here:

How to get that pixel Position with the help of those values.?

This is working for non-VR for those who need this in a computeshader:

Script:

        computeShader.SetTexture(_Kernel, "_DepthTexture", depthRenderTexture);
        computeShader.SetFloat("_CamFOV", camFOV);
        computeShader.SetFloat("_CamAspect", camAspect);
        computeShader.SetFloat("_CamNear", camNear);
        computeShader.SetFloat("_CamFar", camFar);
        computeShader.SetFloat("_ScreenWidth", Camera.main.pixelWidth);
        computeShader.SetFloat("_ScreenHeight", Camera.main.pixelHeight);
        computeShader.SetVector("_CamWorldMatrix0", camWorldMatrix.GetRow(0));
        computeShader.SetVector("_CamWorldMatrix1", camWorldMatrix.GetRow(1));
        computeShader.SetVector("_CamWorldMatrix2", camWorldMatrix.GetRow(2));
        computeShader.SetVector("_CamWorldMatrix3", camWorldMatrix.GetRow(3));
        computeShader.Dispatch(_Kernel, (int)(Camera.main.pixelWidth*Camera.main.pixelHeight)/128, 1, 1);

---

ComputeShader:

    [numthreads(128, 1, 1)]
    void CSMain(uint3 id : SV_DispatchThreadID)
    {
        if (id.x >= (uint)(_ScreenWidth*_ScreenHeight))
        return;

        int y = id.x / int(_ScreenWidth);
        int x = id.x % int(_ScreenWidth);
        float4 depthInfo = _DepthTexture[uint2(x, y)];

        float depthValue = depthInfo.z * _CamFar;
        // world X and Y components of our target vector
        float tanFov = tan(radians(_CamFOV / 2));
        float screenDimY = tanFov * _CamNear;
        float screenDimX = screenDimY * _CamAspect;
        // normalize screenpos from range 0..1 to range -1..1
        float4 normPos = depthInfo * 2 - 1;
        float screenPosX = screenDimX * normPos.x;
        float screenPosY = screenDimY * normPos.y;
        float screenPosZ = -_CamNear;
        float4 objInEyeSpaceVector;
        objInEyeSpaceVector.xyz = float3(screenPosX, screenPosY, screenPosZ) * depthValue / _CamNear;
        objInEyeSpaceVector.w = 1;
        float4x4 camWorldMat = float4x4(_CamWorldMatrix0, _CamWorldMatrix1, _CamWorldMatrix2, _CamWorldMatrix3);
        float4 objInWorldSpace =  mul(camWorldMat, objInEyeSpaceVector);

}

^ but this has to work now in VR too.

There are already the answers for a unity-shader. But i Need that as a computeshader

I dont have a v2f vert (appdata v) and a fixed4 frag (v2f i) : SV_Target in the compute shader. In the end of the day, i would like ... Input: Matrixes, floats, everything i Need incl. depth. I have already the current X and Y coordinate of the Screen-Pixel which gets rendered. So... the last line of Code should be: float4 pixelPositionWorld; Just like in the first example i had.

Answer 1

The approach we'll take depends on whether we're rendering an object in the world (like a screenspace decal volume) or a full-screen blit pass (like a post effect).

Object in World

struct v2f
{
    float2 uv : TEXCOORD0;
    float3 worldDirection : TEXCOORD1;
    float4 screenPosition : TEXCOORD2;
    float4 vertex : SV_POSITION;
};

sampler2D _CameraDepthTexture;

v2f vert (appdata v)
{
    v2f o;

    // Subtract camera position from vertex position in world
    // to get a ray pointing from the camera to this vertex.
    o.worldDirection = mul(unity_ObjectToWorld, v.vertex).xyz - _WorldSpaceCameraPos;

    // Typical boilerplate.
    o.vertex = UnityObjectToClipPos(v.vertex);
    o.uv = v.uv;

    // Save the clip space position so we can use it later.
    // (There are more efficient ways to do this in SM 3.0+, 
    // but here I'm aiming for the simplest version I can.
    // Optimized versions welcome in additional answers!)
    o.screenPosition = o.vertex;

    // Done.
    return o;
}

fixed4 frag (v2f i) : SV_Target
{
    // Compute projective scaling factor...
    float perspectiveDivide = 1.0f / i.screenPosition.w;

    // Scale our view ray to unit depth.
    float3 direction = i.worldDirection * perspectiveDivide;

    // Calculate our UV within the screen (for reading depth buffer)
    float2 screenUV = (i.screenPosition.xy * perspectiveDivide) * 0.5f + 0.5f;

    // Read depth, linearizing into worldspace units.    
    float depth = LinearEyeDepth(UNITY_SAMPLE_DEPTH(tex2D(_CameraDepthTexture, screenUV)));

    // Advance by depth along our view ray from the camera position.
    // This is the worldspace coordinate of the corresponding fragment
    // we retrieved from the depth buffer.
    float3 worldspace = direction * depth + _WorldSpaceCameraPos;

    // Draw a worldspace tartan pattern over the scene to demonstrate.
    return float4(frac((worldspace)), 1.0f);
}

Blit / Post Effect

Unfortunately, when rendering a post effect as with Graphics.Blit it appears that the scene camera view & projection matrices aren't populated correctly, so we'll need to compute the needed parameters in a script and pass them to the shader as a property.

Script

void LateUpdate() {
   // To investigate: do we need to use non-jittered version for antialiasing effects?
   var p = _camera.projectionMatrix;
   // Undo some of the weird projection-y things so it's more intuitive to work with.
   p[2, 3] = p[3, 2] = 0.0f;
   p[3, 3] = 1.0f;

   // I'll confess I don't understand entirely why this is right,
   // I just kept fiddling with numbers until it worked.
   p = Matrix4x4.Inverse(p * _camera.worldToCameraMatrix) 
      * Matrix4x4.TRS(new Vector3(0, 0, -p[2,2]), Quaternion.identity, Vector3.one);

   _material.SetMatrix("_ClipToWorld", p);
}

Shader

sampler2D _CameraDepthTexture;
float4x4 _ClipToWorld;

v2f vert (appdata v)
{
    v2f o;

    // No need for a matrix multiply here when a FMADD will do.
    o.vertex = v.vertex * float4(2, 2, 1, 1) - float4(1, 1, 0, 0);

    // Construct a vector on the Z = 0 plane corresponding to our screenspace location.
    float4 clip = float4((v.uv.xy * 2.0f - 1.0f) * float2(1, -1), 0.0f, 1.0f);
    // Use matrix computed in script to convert to worldspace.
    o.worldDirection = mul(_ClipToWorld, clip) -_WorldSpaceCameraPos;

    // UV passthrough.
    // Flipped Y may be a platform-specific difference - check OpenGL version.
    o.uv = v.uv;
    o.uv.y = 1.0f - o.uv.y;

    return o;
}

fixed4 frag (v2f i) : SV_Target
{   
    // Read depth, linearizing into worldspace units.
    float depth = LinearEyeDepth(UNITY_SAMPLE_DEPTH(tex2D(_CameraDepthTexture, i.uv)));

    // Multiply by worldspace direction (no perspective divide needed).
    float3 worldspace = i.worldDirection * depth + _WorldSpaceCameraPos;

    // Draw a worldspace tartan pattern over the scene to demonstrate.  
    return float4(frac((worldspace)) + float3(0, 0, 0.1), 1.0f);
}

Answer 2

I wanted to drop a note here since I referred to this thread multiple times during my process of solving this issue. Linked is my Unity Forum post about this. My implementation differs quite a bit from @DMGregory to circumvent the issues unique to being in Single-Pass Stereo (SPS) VR.

In short there are a couple of issues that needed to be worked around:

1. CommandBuffer.Blit (and possibly Graphics.Blit?) have weird UVs in SPS
2. The Unity CBuffer values are wrong. You need to calculate your own.
3. The Clip-To-View math needs to be done a very particular way to correctly reverse the projection.
4. The Unity API Stereo Projection Matrix is wrong and needs to be slightly modified. I verified this by using RenderDoc and peeking at Unity's CBuffer during the FowardOpaque queue.

I'll recap the code from my forum post:

In your C#

    private void OnPreRender() // This is just a Later-Than-LateUpdate
    {

        if (cam.stereoEnabled)
        {
            // Left and Right Eye inverse View Matrices
            leftToWorld = cam.GetStereoViewMatrix(Camera.StereoscopicEye.Left).inverse;
            rightToWorld = cam.GetStereoViewMatrix(Camera.StereoscopicEye.Right).inverse;
            mat.SetMatrix("_LeftEyeToWorld", leftToWorld);
            mat.SetMatrix("_RightEyeToWorld", rightToWorld);

            leftEye = cam.GetStereoProjectionMatrix(Camera.StereoscopicEye.Left);
            rightEye = cam.GetStereoProjectionMatrix(Camera.StereoscopicEye.Right);

            // Compensate for RenderTexture...
            leftEye = GL.GetGPUProjectionMatrix(leftEye, true).inverse;
            rightEye = GL.GetGPUProjectionMatrix(rightEye,true).inverse;
            // Negate [1,1] to reflect Unity's CBuffer state
            leftEye[1, 1] *= -1;
            rightEye[1, 1] *= -1;

            mat.SetMatrix("_LeftEyeProjection", leftEye);
            mat.SetMatrix("_RightEyeProjection", rightEye);
        }
    }

In your Shader

fixed4 frag (v2f i) : SV_Target
{
    float d = SAMPLE_DEPTH_TEXTURE(_CameraDepthTexture, i.uv); // non-linear Z
    float2 uv = i.uv;

    float4x4 proj, eyeToWorld;

    if (uv.x < .5) // Left Eye
    {
        uv.x = saturate(uv.x * 2); // 0..1 for left side of buffer
        proj = _LeftEyeProjection;
        eyeToWorld = _LeftEyeToWorld;
    }
    else // Right Eye
    {
        uv.x = saturate((uv.x - 0.5) * 2); // 0..1 for right side of buffer
        proj = _RightEyeProjection;
        eyeToWorld = _RightEyeToWorld;
    }

    float2 uvClip = uv * 2.0 - 1.0;
    float4 clipPos = float4(uvClip, d, 1.0);
    float4 viewPos = mul(proj, clipPos); // inverse projection by clip position
    viewPos /= viewPos.w; // perspective division
    float3 worldPos = mul(eyeToWorld, viewPos).xyz;

    fixed3 color = pow(abs(cos(worldPos * UNITY_PI * 4)), 20); // visualize grid
    return fixed4(color, 1);
}

And the result should be something like:

Cheers!

Answer 3

I'd like to provide an updated version of @colourmath's solution for single pass stereo since unity has changed since then and you can use unity_StereoEyeIndex to your advantage. And since this thread is the easiest thing to find when searching for VR world position in post effect shader, I figured it would be good to help anyone searching for a solution. I should note this is a solution for a post processing shader and not an object shader.

C# Script:

Matrix4x4[] matrices = new Matrix4x4[4];
void SetMatrix() {

    //cache matrices so they can be used in render image step
    if (mainCam.stereoEnabled) {
        // Both stereo eye inverse view matrices
        matrices[0] = mainCam.GetStereoViewMatrix(Camera.StereoscopicEye.Left).inverse;
        matrices[2] = mainCam.GetStereoViewMatrix(Camera.StereoscopicEye.Right).inverse;

        // Both stereo eye inverse projection matrices, plumbed through GetGPUProjectionMatrix to compensate for render texture
        matrices[1] = GL.GetGPUProjectionMatrix(mainCam.GetStereoProjectionMatrix(Camera.StereoscopicEye.Left), true).inverse;
        matrices[3] = GL.GetGPUProjectionMatrix(mainCam.GetStereoProjectionMatrix(Camera.StereoscopicEye.Right), true).inverse;

        // Negate [1,1] to reflect Unity's CBuffer state
        matrices[1][1, 1] *= -1;
        matrices[3][1, 1] *= -1;

    } else {
        // Main eye inverse view matrix
        matrices[0] = mainCam.cameraToWorldMatrix;

        // Inverse projection matrices, plumbed through GetGPUProjectionMatrix to compensate for render texture
        matrices[1] = GL.GetGPUProjectionMatrix(mainCam.projectionMatrix, true).inverse;

        // Negate [1,1] to reflect Unity's CBuffer state
        matrices[1][1, 1] *= -1;
    }

    renderMat.SetMatrixArray("stereoMatrices", matrices);
}

Shader:

float4x4 stereoMatrices[4];

#define IVIEW_MATRIX stereoMatrices[unity_StereoEyeIndex * 2]
#define IPROJ_MATRIX stereoMatrices[unity_StereoEyeIndex * 2 + 1]

v2f vert(appdata v) {
    v2f o;

    o.pos = v.vertex * float4(2, 2, 1, 1) - float4(1, 1, 0, 0);
    o.uv = UnityStereoScreenSpaceUVAdjust(v.uv, _MainTex_ST);

    //_ProjectionParams.y is camera near plane
    float4 viewPos = mul(IPROJ_MATRIX, float4(v.uv * 2.0 - 1.0, _ProjectionParams.y, 1));
    viewPos /= viewPos.w;
    float3 wPos = mul(IVIEW_MATRIX, viewPos).xyz;

    o.ray = wPos - _WorldSpaceCameraPos;

    return o;
}

fixed4 frag(v2f i) : SV_Target {

    float zdepth = DECODE_EYEDEPTH(UNITY_SAMPLE_DEPTH(tex2D(_CameraDepthTexture, i.uv)));
    float3 wPos = i.ray * zdepth + _WorldSpaceCameraPos;

    ...

}

Answer 4

A version that does't require additional script.

Based on Unity built in shader "Particle Add.shader".

#pragma vertex vert
#pragma fragment frag
#include "UnityCG.cginc"

struct appdata
{
    float4 vertex : POSITION;
};
struct v2f
{
    float4 vertex : SV_POSITION;
    float4 projPos : TEXCOORD0;
    float3 ray : TEXCOORD1;
};

UNITY_DECLARE_DEPTH_TEXTURE(_CameraDepthTexture);

v2f vert (appdata v)
{
    v2f o;
    float4 worldPos = mul(UNITY_MATRIX_M, v.vertex);
    o.ray = worldPos.xyz - _WorldSpaceCameraPos;
    o.vertex = mul(UNITY_MATRIX_VP, worldPos);
    o.projPos = ComputeScreenPos (o.vertex);
    o.projPos.z = -mul(UNITY_MATRIX_V, worldPos).z;
    return o;
}

float4 frag (v2f i) : SV_Target
{
    float sceneDepth = LinearEyeDepth (SAMPLE_DEPTH_TEXTURE_PROJ(_CameraDepthTexture, UNITY_PROJ_COORD(i.projPos)));
    float3 worldPosition = sceneDepth * i.ray / i.projPos.z + _WorldSpaceCameraPos;

    // Draw a worldspace tartan pattern over the scene to demonstrate.  
    return float4(frac(worldPosition), 1.0f);
}