Modify textures with opencv

[Federica]

Hi, 

I'm working on a project in Unity with HTC Vive Pro Eye headset and the SRWorks SDK. 

I would like to process the textures of camera image created in ViveSR_DaulCameraImageCapture.cs script with OpenCV. 

For example I would like to apply a Gaussian filter to see a blurred effect through the lenses of the headset.

Do you know how and where in the scripts provided by the SRWorks SDK I can work to obtain this effect? Thank you.

[Slim]

Hi,

The forum seems not very active. I am also struggling, but maybe what I got so far can help someone.

The problem with OpenCV is that you need the image to be in CPU memory. So first you need to copy it somehow from GPU.

I used AsyncGPUReadbackRequest for it. Now I am stuck on how to Apply this modified Texture back to SRWorks.

Here is my code:

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using Vive.Plugin.SR;
using OpenCvSharp;
using UnityEngine.Rendering;

public class OpenCVMat : MonoBehaviour
{
    Texture2D left;
    public Texture2D leftCPU;
    public Texture2D gray;
    Texture2D right;
    int frameIndex;
    int timeIndex;
    Matrix4x4 poseLeft;
    Matrix4x4 poseRight;

    AsyncGPUReadbackRequest request;
    private bool initialized;


    void OnCompleteReadback(AsyncGPUReadbackRequest request)
    {
        if (request.hasError)
        {
            Debug.Log("GPU readback error detected.");
            return;
        }

        var tex = new Texture2D(ViveSR_DualCameraImageCapture.UndistortedImageWidth, ViveSR_DualCameraImageCapture.UndistortedImageHeight, TextureFormat.RGBA32, false);
        tex.LoadRawTextureData(request.GetData<uint>());
        tex.Apply();
        Graphics.CopyTexture(tex, leftCPU);
        Destroy(tex);
    }

    void Update()
    {
        if(ViveSR.FrameworkStatus == FrameworkStatus.WORKING)
        {
            if (!initialized)
            {
                InitTextures();
                initialized = true;
            }

            if(initialized && request.done)
            {
                Mat mat = OpenCvSharp.Unity.TextureToMat(leftCPU);
                Mat grayMat = new Mat();
                Cv2.CvtColor(mat, grayMat, ColorConversionCodes.BGRA2GRAY);
                gray = OpenCvSharp.Unity.MatToTexture(grayMat);

                grayMat.Dispose();
                mat.Dispose();

                ViveSR_DualCameraImageCapture.GetUndistortedTexture(out left, out right, out frameIndex, out timeIndex, out poseLeft, out poseRight);
                request = AsyncGPUReadback.Request(left, 0, TextureFormat.RGBA32, OnCompleteReadback);
            }
        }
    }

    void InitTextures()
    {
        left = new Texture2D(ViveSR_DualCameraImageCapture.UndistortedImageWidth, ViveSR_DualCameraImageCapture.UndistortedImageHeight, TextureFormat.RGBA32, false);
        leftCPU = new Texture2D(ViveSR_DualCameraImageCapture.UndistortedImageWidth, ViveSR_DualCameraImageCapture.UndistortedImageHeight, TextureFormat.RGBA32, false);
        gray = new Texture2D(ViveSR_DualCameraImageCapture.UndistortedImageWidth, ViveSR_DualCameraImageCapture.UndistortedImageHeight, TextureFormat.RGBA32, false);
        right = new Texture2D(ViveSR_DualCameraImageCapture.UndistortedImageWidth, ViveSR_DualCameraImageCapture.UndistortedImageHeight, TextureFormat.RGBA32, false);
    }
}

 

[Frank_D]

On 6/11/2021 at 5:23 AM, Slim said:

Hi,

The forum seems not very active. I am also struggling, but maybe what I got so far can help someone.

The problem with OpenCV is that you need the image to be in CPU memory. So first you need to copy it somehow from GPU.

I used AsyncGPUReadbackRequest for it. Now I am stuck on how to Apply this modified Texture back to SRWorks.

Here is my code:

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using Vive.Plugin.SR;
using OpenCvSharp;
using UnityEngine.Rendering;

public class OpenCVMat : MonoBehaviour
{
    Texture2D left;
    public Texture2D leftCPU;
    public Texture2D gray;
    Texture2D right;
    int frameIndex;
    int timeIndex;
    Matrix4x4 poseLeft;
    Matrix4x4 poseRight;

    AsyncGPUReadbackRequest request;
    private bool initialized;


    void OnCompleteReadback(AsyncGPUReadbackRequest request)
    {
        if (request.hasError)
        {
            Debug.Log("GPU readback error detected.");
            return;
        }

        var tex = new Texture2D(ViveSR_DualCameraImageCapture.UndistortedImageWidth, ViveSR_DualCameraImageCapture.UndistortedImageHeight, TextureFormat.RGBA32, false);
        tex.LoadRawTextureData(request.GetData<uint>());
        tex.Apply();
        Graphics.CopyTexture(tex, leftCPU);
        Destroy(tex);
    }

    void Update()
    {
        if(ViveSR.FrameworkStatus == FrameworkStatus.WORKING)
        {
            if (!initialized)
            {
                InitTextures();
                initialized = true;
            }

            if(initialized && request.done)
            {
                Mat mat = OpenCvSharp.Unity.TextureToMat(leftCPU);
                Mat grayMat = new Mat();
                Cv2.CvtColor(mat, grayMat, ColorConversionCodes.BGRA2GRAY);
                gray = OpenCvSharp.Unity.MatToTexture(grayMat);

                grayMat.Dispose();
                mat.Dispose();

                ViveSR_DualCameraImageCapture.GetUndistortedTexture(out left, out right, out frameIndex, out timeIndex, out poseLeft, out poseRight);
                request = AsyncGPUReadback.Request(left, 0, TextureFormat.RGBA32, OnCompleteReadback);
            }
        }
    }

    void InitTextures()
    {
        left = new Texture2D(ViveSR_DualCameraImageCapture.UndistortedImageWidth, ViveSR_DualCameraImageCapture.UndistortedImageHeight, TextureFormat.RGBA32, false);
        leftCPU = new Texture2D(ViveSR_DualCameraImageCapture.UndistortedImageWidth, ViveSR_DualCameraImageCapture.UndistortedImageHeight, TextureFormat.RGBA32, false);
        gray = new Texture2D(ViveSR_DualCameraImageCapture.UndistortedImageWidth, ViveSR_DualCameraImageCapture.UndistortedImageHeight, TextureFormat.RGBA32, false);
        right = new Texture2D(ViveSR_DualCameraImageCapture.UndistortedImageWidth, ViveSR_DualCameraImageCapture.UndistortedImageHeight, TextureFormat.RGBA32, false);
    }
}

 

Hi~

Thank you for your great work!

Yet this script make the texture transparent and 180 degree rotation.