How To Access the Camera Image in Unity

This article describes two approaches for obtaining the camera image (without augmentation) in Unity:

  • Use the Vuforia.Image class
  • Use the image as an OpenGL texture

Use the Vuforia.Image class

Using the Vuforia.Image class works much like the native version.

Register for the desired image format using the CameraDevice.SetFrameFormat() method:
 
CameraDevice.Instance.SetFrameFormat(Image.PIXEL_FORMAT.RGB888, true);

Note: available pixel formats include RGBA8888, RGB888, RGB565, YUV and GRAYSCALE; the number of bytes per pixel varies from 4 bytes (32-bit) in the RGBA8888 mode, to 1 byte (8-bit) for the GRAYSCALE mode. 

Call this method after Vuforia has been initialized and started; to this aim, it is recommended to register an OnVuforiaStarted callback in the Start() method of your MonoBehaviour script, e.g.:
 
void Start() 
{

    VuforiaARController.Instance.RegisterVuforiaStartedCallback(OnVuforiaStarted);

}

private void OnVuforiaStarted()
{
    // Vuforia has started, now register camera image format

    if (CameraDevice.Instance.SetFrameFormat(mPixelFormat, true))
    {
        Debug.Log("Successfully registered pixel format " + mPixelFormat.ToString());

        mFormatRegistered = true;
    }
    else
    {
        Debug.LogError(
            "Failed to register pixel format " + mPixelFormat.ToString() +
            "\n the format may be unsupported by your device;" +
            "\n consider using a different pixel format.");

        mFormatRegistered = false;
    }
}

Retrieve the camera image using the CameraDevice.GetCameraImage() method.
  • Take this action from the OnTrackablesUpdated() callback.  That way you can ensure that you retrieve the latest camera image that matches the current frame.
  • Always make sure that the camera image is not null, since it can take a few frames for the image to become available after registering for an image format.
  • Make sure to unregister the camera image format whenever the application is paused, and to register it again when the application is resumed.
Instructions: Attach this script to a GameObject in your AR scene and check the logs in the Console to see the captured Image information:
 
using UnityEngine;
using System.Collections;

using Vuforia;

public class CameraImageAccess : MonoBehaviour
{

    #region PRIVATE_MEMBERS

    private Image.PIXEL_FORMAT mPixelFormat = Image.PIXEL_FORMAT.UNKNOWN_FORMAT;

    private bool mAccessCameraImage = true;
    private bool mFormatRegistered = false;

    #endregion // PRIVATE_MEMBERS

    #region MONOBEHAVIOUR_METHODS

    void Start()
    {

        #if UNITY_EDITOR
        mPixelFormat = Image.PIXEL_FORMAT.GRAYSCALE; // Need Grayscale for Editor
        #elif
        m_PixelFormat = Image.PIXEL_FORMAT.RGB888; // Use RGB888 for mobile
        #endif

        // Register Vuforia life-cycle callbacks:
        VuforiaARController.Instance.RegisterVuforiaStartedCallback(OnVuforiaStarted);
        VuforiaARController.Instance.RegisterTrackablesUpdatedCallback(OnTrackablesUpdated);
        VuforiaARController.Instance.RegisterOnPauseCallback(OnPause);

    }

    #endregion // MONOBEHAVIOUR_METHODS

    #region PRIVATE_METHODS

    void OnVuforiaStarted()
    {

        // Try register camera image format
        if (CameraDevice.Instance.SetFrameFormat(mPixelFormat, true))
        {
            Debug.Log("Successfully registered pixel format " + mPixelFormat.ToString());

            mFormatRegistered = true;
        }
        else
        {
            Debug.LogError(
                "\nFailed to register pixel format: " + mPixelFormat.ToString() +
                "\nThe format may be unsupported by your device." +
                "\nConsider using a different pixel format.\n");

            mFormatRegistered = false;
        }

    }

    /// <summary>
    /// Called each time the Vuforia state is updated
    /// </summary>
    void OnTrackablesUpdated()
    {
        if (mFormatRegistered)
        {
            if (mAccessCameraImage)
            {
                Vuforia.Image image = CameraDevice.Instance.GetCameraImage(mPixelFormat);

                if (image != null)
                {
                    Debug.Log(
                        "\nImage Format: " + image.PixelFormat +
                        "\nImage Size:   " + image.Width + "x" + image.Height +
                        "\nBuffer Size:  " + image.BufferWidth + "x" + image.BufferHeight +
                        "\nImage Stride: " + image.Stride + "\n"
                    );

                    byte[] pixels = image.Pixels;

                    if (pixels != null && pixels.Length > 0)
                    {
                        Debug.Log(
                            "\nImage pixels: " + 
                            pixels[0] + ", " + 
                            pixels[1] + ", " + 
                            pixels[2] + ", ...\n"
                        );
                    }
                }
            }
        }
    }

    /// <summary>
    /// Called when app is paused / resumed
    /// </summary>
    void OnPause(bool paused)
    {
        if (paused)
        {
            Debug.Log("App was paused");
            UnregisterFormat();
        }
        else
        {
            Debug.Log("App was resumed");
            RegisterFormat();
        }
    }

    /// <summary>
    /// Register the camera pixel format
    /// </summary>
    void RegisterFormat()
    {
        if (CameraDevice.Instance.SetFrameFormat(mPixelFormat, true))
        {
            Debug.Log("Successfully registered camera pixel format " + mPixelFormat.ToString());
            mFormatRegistered = true;
        }
        else
        {
            Debug.LogError("Failed to register camera pixel format " + mPixelFormat.ToString());
            mFormatRegistered = false;
        }
    }

    /// <summary>
    /// Unregister the camera pixel format (e.g. call this when app is paused)
    /// </summary>
    void UnregisterFormat()
    {
        Debug.Log("Unregistering camera pixel format " + mPixelFormat.ToString());
        CameraDevice.Instance.SetFrameFormat(mPixelFormat, false);
        mFormatRegistered = false;
    }

    #endregion //PRIVATE_METHODS
}
 

Use an OpenGL texture

The Image class provides the camera pixels as a byte array. That approach is useful for some image processing tasks, but sometimes it is preferable to obtain the image as an OpenGL texture, for example if you wish to use the texture in a Material applied to a game object and/or to process the texture a shader. You can obtain the image as an OpenGL texture using the approach demonstrated in the OcclusionManagement sample.

  1. Register a Texture2D object to be filled with the camera pixels at each frame instead of letting Vuforia render the camera image natively at each frame, using the VuforiaRenderer.VideoBackgroundTexture API
  2. See the OcclusionManagement sample scripts for an example of this technique.