1
\$\begingroup\$

I'm doing filtering on a depth image and I would like to convert that piece of code into job system of unity, because it's so slow. The image update is called each frame

 public unsafe  void DRSFilter( byte *  src, byte * dst, uint data_delta, uint history_scan_depth)
     {
         if (history_scan_depth > history_capacity)
         {
             history_scan_depth = history_capacity;
         }
 
         for (int i = 0; i < layer_size; i++)
         {
             byte b = src[i];
             bool is_color_pixel = false;
             uint avrg = 0;
             for (int k = 0; k < history_scan_depth; k++)
             {
                 avrg += history[k * layer_size + i];
                 if (history[k * layer_size + i] > 0)
                 {
                     is_color_pixel = true;
                    // break;
                 }
             }
             average[i] = (byte)(avrg / history_scan_depth);
 
            //if (b > 0 || b == 0 && !is_color_pixel) {
             if (Abs(b - average[i]) < data_delta)
             {
                 dst[i] = b;
             }
            // }
 
             history[layer * layer_size + i] = b;
         }
 
         layer++;
         if (layer == history_scan_depth)
         {
             layer = 0;
         }
     }
 
 
  private unsafe void ImageUpdated(Camera cam,  ref Mat cvMatDepth)
     {
 
         long addrsrc = cvMatDepth.dataAddr();
         long addrDst = dstResult.dataAddr();
         DRSFilter((byte*)addrsrc, (byte*)addrDst, a, b);
  
         Utils.matToTexture2D(dstResult, texture);
         canvas.GetComponent<RawImage>().texture = texture;
 
 
     }

Update Sketch:

 [BurstCompile] // <-- For this you need to add the Burst package using the Unity package manager, this attribute is completely optional but it improves performance by a lot (like 10 times faster)
    public struct ConvertJobParallel : IJobParallelFor
    {
        public NativeArray<byte> input;
        public NativeArray<byte> output;
        public NativeArray<byte> history;
        public NativeArray<byte> average;
        public int _history_scan_depth;
        public int _history_capacity;
        public int _layer_size;
        public int _data_delta;
        public int _layer;

        // This gets called by the job system. index ranges between 0 and the 'arrayLength' specified in the first parameter of Schedule()
        public void Execute(int index)
        {
            if (_history_scan_depth > _history_capacity)
            {
                _history_scan_depth = _history_capacity;
            }

            byte b = input[index];
            uint avrg = 0;
            for (int k = 0; k < _history_scan_depth; k++)
            {
                avrg += history[k * _layer_size + index];

            }
            average[index] = (byte)(avrg / _history_scan_depth);

            if (Math.Abs(b - average[index]) < _data_delta)
            {
                output[index] = b;
            }


            history[_layer * _layer_size + index] = b;


            _layer++;
            if (_layer == _history_scan_depth)
            {
                _layer = 0;
            }
        }

    }
   [SerializeField] private Texture2D _inputTexture; // Texture that we are reading, should have read/write enabled set to true and Compression set to None
    [SerializeField] private Texture2D rawImage;
    private Texture2D _outputTexture;

    private NativeArray<byte> inputArray;
    private NativeArray<byte> outputArray;

    private NativeArray<byte> historyArray;
    private NativeArray<byte> averageArray;

    private int _history_scan_depth;
    private int _history_capacity;
    private int _data_delta;
    private int _layer;
    private void Awake()
    {
        historyArray = new NativeArray<byte>(history_capacity * layer_size, Allocator.Persistent);
        averageArray = new NativeArray<byte>(layer_size, Allocator.Persistent);
        outputArray = new NativeArray<byte>(layer_size, Allocator.Persistent);

    }

    private void OnDestroy()
    {
        historyArray.Dispose(); // We need to manually dispose our NativeArrays. The arrays retrieved from the Textures are disposed by the textures so we don't care here
        averageArray.Dispose();
        outputArray.Dispose();

    }

    private void ImageUpdated(ref Texture2D zedTextureDepth)
    {

        inputArray = zedTextureDepth.GetRawTextureData<byte>(); // Could use GetPixelData instead when working with mipmaps


        var job = new ConvertJobParallel()
        {
            input = inputArray,
            output = outputArray,
            history = historyArray,
            average = averageArray,
            _history_scan_depth = history_scan_depth,
            _history_capacity = history_capacity,
            _layer_size = layer_size,
            _data_delta = data_delta,
            _layer = layer
        };

        JobHandle handle1 = job.Schedule(inputArray.Length, 100); // The 100 here is a magic number, it is basically a number that says how many items (Color32 in our example) are handled in one 'batch' where each batch can run async on any thread
        JobHandle.ScheduleBatchedJobs(); // Start running (all) our previously scheduled jobs
        handle1.Complete(); // Wait on the main thread fot this job to complete

        _outputTexture.LoadRawTextureData(outputArray);
        _outputTexture.Apply(); // Copy changes to gpu

        canvas.GetComponent<RawImage>().texture = rawImage;


    }
\$\endgroup\$
9
  • 1
    \$\begingroup\$ Can you tell us more about what this code is doing for you, and the kind of data you're using it on? At a glance, this looks like it might be even better suited to GPU parallelization, rather than jobs. Why constrain yourself to 2-16 CPU threads chewing away at this when you could have hundreds on the GPU! This can also save some of the work of re-uploading the resulting buffer to the GPU as a texture when you're done, if the data's able to just live there the whole time. \$\endgroup\$ – DMGregory Nov 6 '20 at 9:43
  • \$\begingroup\$ @DMGregory Thanks for your suggestion. Basically that code it's input is an OpenCV Matrix which has a depth buffer 4 Channel Unsigned char, it does averaging over a history and store the result in destination. Can you elaborate more about the GPU approach or any approach as an answer. Thanks so muc \$\endgroup\$ – Ahmed Saleh Nov 6 '20 at 11:32
  • \$\begingroup\$ Looking at this, I think that if it is possible to do on the GPU it is probably the way to go. But in case you also want answers using the Job system, the current example code is missing some information needed to convert it. For example, what is Mat? A custom class you wrote, class imported from a C library? Does the 'addrsrs' ever change? if so, where/when? My first idea is that DRSFilter() should alter the NativeArray<Color32> (Or something similar, depending on Utils.matToTexture2D implementation) of a Texture2D directly, removing the need of converting Mat to texture every frame \$\endgroup\$ – troien Nov 6 '20 at 13:45
  • \$\begingroup\$ @troien I found a way to remove opencv so there is no need for Mat to texture or getting the input as OpenCV Mat. Consider now the input is a Texture and the Output should be a Texture, how would I parallelize it on GPU ? \$\endgroup\$ – Ahmed Saleh Nov 6 '20 at 14:16
  • \$\begingroup\$ I don't have time for a full answer just now, so here's a sketch: Make a new unlit shader. Give it a texture parameter for each "page" of history, and one for the new data src (call this one _MainTex for blitting convenience). In the fragment shader, sample the corresponding texel from each history page to compute the average, and use that to determine your output value. Use Graphics.Blit to run this shader on the src texture, outputting its results into a RenderTexture of the same dimensions. Then replace one of the history pages with the new data. \$\endgroup\$ – DMGregory Nov 6 '20 at 15:01
2
\$\begingroup\$

Here is a vertex-fragment shader that imitates the action of your job:

Shader "Unlit/DepthFilter"
{
    Properties
    {
        _MainTex  ("Texture", 2D)   = "white" {}
        _HistoryA ("History A", 2D) = "white" {}
        _HistoryB ("History B", 2D) = "white" {}
        _HistoryC ("History C", 2D) = "white" {}
        _dataDelta ("Data Delta", range(0, 256)) = 256
    }
    SubShader
    {
        Tags { "RenderType"="Opaque" }
        LOD 100

        Pass
        {
            CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag
        
            #include "UnityCG.cginc"

            struct appdata
            {
                float4 vertex : POSITION;
                float2 uv : TEXCOORD0;
            };

            struct v2f
            {
                float2 uv : TEXCOORD0;
                float4 vertex : SV_POSITION;
            };

            sampler2D _MainTex;
            sampler2D _HistoryA;
            sampler2D _HistoryB;
            sampler2D _HistoryC;
            float     _dataDelta;

            v2f vert (appdata v)
            {
                v2f o;
                o.vertex = UnityObjectToClipPos(v.vertex);
                o.uv = v.uv;
                return o;
            }

            fixed4 frag (v2f i) : SV_Target
            {
                fixed source = tex2D(_MainTex, i.uv).r;

                // Accumulate the history into a total.
                float average = tex2D(_HistoryA, i.uv).r;
                average      += tex2D(_HistoryB, i.uv).r;               
                average      += tex2D(_HistoryC, i.uv).r;
                
                // Divide by our sample count to get an average.
                average /= 3.0f;
                
                // Write into the output texture if and only if
                // abs(source - average) < _dataDelta.
                clip(_dataDelta/256.0f - abs(source - average));
                return source;
            }
            ENDCG
        }
    }
}

Assign this shader to a material and you'll see three slots to assign textures: one representing your new "source" data, and three others representing your history buffer. You'll also have a slider property where you can assign your data delta.

We can ask the GPU to perform this filter on a given source and destination buffer like so:

public class DepthFilter : MonoBehaviour
{
    const int HISTORY_CAPACITY = 3;

    readonly static int _dataDeltaID = Shader.PropertyToID("_dataDelta");
    readonly static int[] _historyIDs = new int[] {
        Shader.PropertyToID("_HistoryA"),
        Shader.PropertyToID("_HistoryB"),
        Shader.PropertyToID("_HistoryC"),
    };

    public Material filterMaterialAsset;
    Material _filterMaterialInstance;
    RenderTexture[] _historyPages;    
    int _oldestPage = 0;

    public void Filter(ref RenderTexture source, RenderTexture destination, uint dataDelta) {

        // If we haven't initialized our history buffer, do so.
        // Match the size/format of whatever render target
        // we were passed as a source.
        if (_historyPages == null) {
            _filterMaterialInstance = Instantiate(filterMaterialAsset);
            _historyPages = new RenderTexture[HISTORY_CAPACITY];
            for(int i = 0; i < HISTORY_CAPACITY; i++) {
                _historyPages[i] = new RenderTexture(source);
                _filterMaterialInstance.SetTexture(_historyIDs[i], _historyPages[i]);
            }
        }

        _filterMaterialInstance.SetFloat(_dataDeltaID, dataDelta);

        // Apply the shader filter to the source and current history,
        // and write the result into destination.
        Graphics.Blit(source, destination, _filterMaterialInstance);

        // Our source becomes the newest sample in our history.
        // (Or, if it's supposed to be your destination, use that instead).
        // The oldest history page takes its place as a new writable source texture.
        var newPage = source;
        source = _historyPages[_oldestPage];
        _historyPages[_oldestPage] = newPage;
        _filterMaterialInstance.SetTexture(_historyIDs[_oldestPage], newPage);
        _oldestPage = (_oldestPage + 1) % HISTORY_CAPACITY;        
    }
}

Call Filter() to write the filtered result into your destination texture, and update the history buffer with the latest source data, returning you the oldest history page to overwrite with your new source data next frame.

This approach keeps all of this data GPU-side at all times. We never download that data back to the CPU to process it and upload it back to the GPU afterward, we just tell the GPU what to do with the data it already has in its memory.

\$\endgroup\$
8
  • \$\begingroup\$ WORKS LIKE A CHARRRRRRRRRRRM ! Many THANKS!!! \$\endgroup\$ – Ahmed Saleh Nov 10 '20 at 5:52
  • \$\begingroup\$ is it possible to change this Graphics.Blit(source, destination, _filterMaterialInstance); to write directly to a texture ? because I have to convert back the render to texture to a texture \$\endgroup\$ – Ahmed Saleh Nov 10 '20 at 9:56
  • \$\begingroup\$ No. RenderTextures are "textures that GPU shaders can write to" so it's already exactly the kind of texture we need for this purpose. If you're using it on the GPU, you don't have to do any extra conversion: you can use a RenderTexture anywhere you could use a texture, just not the reverse. If you need to read it back to the CPU side, you can copy the contents into a CPU-accessible Texture2D with ReadPixels() — just note that this step is slower than the rest. \$\endgroup\$ – DMGregory Nov 10 '20 at 10:54
  • \$\begingroup\$ Hi, The filter is working fine with low delta values like 10 there are no flickering at all, but there is ghosting effect that is seen. I increased the history pages to 12 with no effect still there is ghosting \$\endgroup\$ – Ahmed Saleh Nov 11 '20 at 13:53
  • \$\begingroup\$ Want to post a new question about removing the ghosting effect, showing your modified shader, script, input, and output images? \$\endgroup\$ – DMGregory Nov 11 '20 at 13:59
2
\$\begingroup\$

Although a GPU solution is probably better and faster in your case, I don't have enough knowledge about this so I'll leave it to someone else to write that answer :)

Here I'll show you an example of how to edit textures using Unity's job system. This example is not tailored to your needs, but I think it should be enough of an example to help you along implement your functionality yourself should you choose not to use a GPU solution for any reason.

In the below example I made a Parallel job that swaps the red and green channels, this can however be altered to do whetever you want ofcourse.

public class Example : MonoBehaviour
{
    [SerializeField] private Texture2D _inputTexture; // Texture that we are reading, should have read/write enabled set to true and Compression set to None
    [SerializeField] private RawImage rawImage;
    private Texture2D _outputTexture;

    private NativeArray<Color32> inputArray;
    private NativeArray<Color32> outputArray;

    private NativeArray<byte> yourCustomArray; // Can be any (struct)type So byte, int, Color32, etc. all work

    private void Awake()
    {
        // Copy original texture to a new Texture2D for testing
        _outputTexture = new Texture2D(_inputTexture.width, _inputTexture.height, _inputTexture.format, _inputTexture.mipmapCount > 1);
        Graphics.CopyTexture(_inputTexture, _outputTexture);

        // Get the underlying NativeArrays which can be used by the jobs system
        inputArray = _inputTexture.GetRawTextureData<Color32>(); // Could use GetPixelData instead when working with mipmaps
        outputArray = _outputTexture.GetRawTextureData<Color32>();

        // Assign outputtexture to a rawimage
        rawImage.texture = _outputTexture;

        yourCustomArray = new NativeArray<byte>(inputArray.Length, Allocator.Persistent);
    }

    private void OnDestroy()
    {
        yourCustomArray.Dispose(); // We need to manually dispose our NativeArrays. The arrays retrieved from the Textures are disposed by the textures so we don't care here
    }

    private void Update()
    {
        var job = new ConvertJobParallel()
        {
            input = inputArray,
            output = outputArray,
            yourData = yourCustomArray
        };

        JobHandle handle1 = job.Schedule(inputArray.Length, 100); // The 100 here is a magic number, it is basically a number that says how many items (Color32 in our example) are handled in one 'batch' where each batch can run async on any thread
        JobHandle.ScheduleBatchedJobs(); // Start running (all) our previously scheduled jobs
        handle1.Complete(); // Wait on the main thread fot this job to complete
        _outputTexture.Apply(); // Copy changes to gpu
    }

    [BurstCompile] // <-- For this you need to add the Burst package using the Unity package manager, this attribute is completely optional but it improves performance by a lot (like 10 times faster)
    public struct ConvertJobParallel : IJobParallelFor
    {
        [ReadOnly] public NativeArray<Color32> input;
        public NativeArray<Color32> output;
        public NativeArray<byte> yourData;

        // This gets called by the job system. index ranges between 0 and the 'arrayLength' specified in the first parameter of Schedule()
        public void Execute(int index)
        {
            // This is simply a demo test, it swaps the red and green channels, here you put your logic
            Color32 inputColor = input[index];
            output[index] = new Color32(inputColor.g, inputColor.r, inputColor.b, inputColor.a);
        }
    }
}

So in your case you can change the yourCustomArray and yourData to whatever you want, you can add more NativeArrays if you want to in a similar way, you can also add variables to your job that contain only one value (like public uint data_delta; for example). Note however that you should consider any variable that is not a NativeArray as readonly in your job.

EDIT Forgot to mention this, in this test I immediatly call Complete() after scheduling the job. If possible you should always make sure that you do other things in between scheduling and awaiting for the job to complete to fully utilize multithreading. So for instance, schedule at the start of the frame and complete at the end of the frame or in a later frame depending on your needs.

Edit2 Based on comments and the updated code you posted. The Execute(int index) method of the ConvertJobParallel will be run multithreaded by Unity. When you call job.Schedule(inputArray.Length, 100); you are telling Unity you want to call Execute inputArray.Length times in batches of 100. Unity will then handle that these batches of 100 are distributed among different threads. This also means that altering the variables of ConvertJobParallel inside the Execute job is problematic. The only thing which you can write to and persists after the job are the contents of the NativeArray's you pass along. That layer++ you do won't work as you expect it, as it won't increment the actual layer outside the job. So these parts should be moved outside the job (Actually, anything inside your original code outside this loop for (int i = 0; i < layer_size; i++) should be moved outside the job

You should also try to cache the NativeArrays of the Textures you are using if possible. These NativeArrays contain the actual bytes of this Texture on CPU side. Meaning if you change any value of that nativeArray, you change the value of the Texture itself and vice-versa. But it won't be copied to the GPU untill you call texture.Apply(). Note that the url you posted of how you retrieve the texture, it uses a cached texture if possible, so if it returns a cached texture whose NativeArray you retrieved in the past, you don't need to update your cached NativeArray, your NativeArray should contain the correct bytes already.

So my change to you current posted code would be something like this: The Job itself:

[BurstCompile] // <-- For this you need to add the Burst package using the Unity package manager, this attribute is completely optional but it improves performance by a lot (like 10 times faster)
public struct ConvertJobParallel : IJobParallelFor
{
    [ReadOnly] public NativeArray<byte> input;
    public NativeArray<byte> output;
    [NativeDisableParallelForRestriction] public NativeArray<byte> history;
    public NativeArray<byte> average;

    // These should always be considered read only, as writing to them is problematic. Execute gets called async on many threads.
    // So editing these values inside execute does not yield the wanted results
    public int _history_scan_depth;
    public int _layer_size;
    public int _data_delta;
    public int _layer;

    // This gets called by the job system. index ranges between 0 and the 'arrayLength' specified in the first parameter of Schedule()
    public void Execute(int index)
    {
        byte b = input[index];
        uint avrg = 0;
        for (int k = 0; k < _history_scan_depth; k++)
        {
            avrg += history[k * _layer_size + index];
        }
        average[index] = (byte)(avrg / _history_scan_depth);

        if (math.abs(b - average[index]) < _data_delta)
        {
            output[index] = b;
        }

        history[_layer * _layer_size + index] = b;
    }
}

Scheduling the job:

if (history_scan_depth > history_capacity)
    history_scan_depth = history_capacity;

var job = new ConvertJobParallel()
{
    input = inputArray,
    output = outputArray,
    history = historyArray,
    average = averageArray,
    _history_scan_depth = history_scan_depth,
    _layer_size = layer_size,
    _data_delta = data_delta,
    _layer = layer
};

layer++;
if (layer >= history_scan_depth)
    layer = 0;

JobHandle handle1 = job.Schedule(inputArray.Length, 100); // The 100 here is a magic number, it is basically a number that says how many items (Color32 in our example) are handled in one 'batch' where each batch can run async on any thread
JobHandle.ScheduleBatchedJobs(); // Start running (all) our previously scheduled jobs
handle1.Complete(); // Wait on the main thread fot this job to complete

_outputTexture.Apply(); // Copy changes to gpu

In this example, I'm not sure what _data_delta should be, in your comment to the original post I read 500, but since you are comparing it to a byte which max value is 250 I tested this at 50. _outputTexture.LoadRawTextureData(outputArray); You shouldn't really call this, as you just want to be Editing the NativeArray of that texture directly, rather then copying another one to it every frame, I do this in my original post by creating a new Texture2D with correct width, height and format and then use GetRawTextureData to get the outputArray that is directly mapped to the Texture2D.

\$\endgroup\$
12
  • \$\begingroup\$ The current problem right now with your approach is inputArray = zedTextureDepth.GetRawTextureData<byte>(); // Could use GetPixelData instead when working with mipmaps I call this each frame, and that drops the frame alot. second question, how do I parallize the threads so that each one does scan one row, write to history buffer, as the original algorithm \$\endgroup\$ – Ahmed Saleh Nov 9 '20 at 8:51
  • \$\begingroup\$ here is full script pastebin.com/k2BsyL9u \$\endgroup\$ – Ahmed Saleh Nov 9 '20 at 8:56
  • \$\begingroup\$ Well, notice that I call GetRawTextureData only once in awake. And then re-use it every frame, editing the content. The moment you call Apply(); it sends a copy of this NativeArray to the gpu, but it keeps the NativeArray in cpu memory for as long as the Texture you retrieved it from exists. Meaning you don't have to call GetRawTextureData every frame \$\endgroup\$ – troien Nov 9 '20 at 13:13
  • \$\begingroup\$ I still don't know how to pack the threads to get the average and write into history buffer, please help me :( \$\endgroup\$ – Ahmed Saleh Nov 9 '20 at 13:15
  • \$\begingroup\$ regading GetRawTextureData, in your sample it assumes that the input is in awake avaiable, in my case I get the input from a call back \$\endgroup\$ – Ahmed Saleh Nov 9 '20 at 13:17

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.