[FIXED] Share wikitude AR using openvidu

Issue

I'm working on a video conference app using openvidu. We are trying to include wikitude AR session in the call. The problem is that both of them requires access to the camera so I have the next scenario: if I instantiate the local participant video first I can't start the wikitude AR session because video don't load. If I instantiate the wikitude session firstly the other participants of the call don't see the device video. I was able to create a custom video capturer for openvidu, that imitates the camera. It is required to send every frame for it to works.

package org.webrtc;

import android.content.Context;
import android.graphics.Bitmap;

import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.atomic.AtomicReference;

public class CustomVideoCapturer implements VideoCapturer {
    private final static String TAG = "FileVideoCapturer";

    //private final FileVideoCapturer.VideoReader videoReader;
    private final Timer timer = new Timer();
    private CapturerObserver capturerObserver;

    private AtomicReference<Bitmap> image = new AtomicReference<Bitmap>();

    private final TimerTask tickTask = new TimerTask() {
        @Override
        public void run() {
            tick();
        }
    };

    public CustomVideoCapturer() {

    }

    public void tick() {
        Bitmap frame = image.get();
        if (frame != null && !frame.isRecycled()) {
            NV21Buffer nv21Buffer = new NV21Buffer(getNV21(frame),frame.getWidth(),frame.getHeight(), null);
            VideoFrame videoFrame = new VideoFrame(nv21Buffer, 0, System.nanoTime());
            capturerObserver.onFrameCaptured(videoFrame);
        }
    }

    byte [] getNV21(Bitmap image) {
        int [] argb = new int[image.getWidth() * image.getHeight()];

        image.getPixels(argb, 0, image.getWidth(), 0, 0, image.getWidth(), image.getHeight());

        byte [] yuv = new byte[image.getWidth()*image.getHeight()*3/2];
        encodeYUV420SP(yuv, argb, image.getWidth(), image.getHeight());

        image.recycle();

        return yuv;
    }

    void encodeYUV420SP(byte[] yuv420sp, int[] argb, int width, int height) {
        final int frameSize = width * height;

        int yIndex = 0;
        int uvIndex = frameSize;

        int a, R, G, B, Y, U, V;
        int index = 0;
        for (int j = 0; j < height; j++) {
            for (int i = 0; i < width; i++) {

                a = (argb[index] & 0xff000000) >> 24; // a is not used obviously
                R = (argb[index] & 0xff0000) >> 16;
                G = (argb[index] & 0xff00) >> 8;
                B = (argb[index] & 0xff) >> 0;

                // well known RGB to YUV algorithm
                Y = ( (  66 * R + 129 * G +  25 * B + 128) >> 8) +  16;
                U = ( ( -38 * R -  74 * G + 112 * B + 128) >> 8) + 128;
                V = ( ( 112 * R -  94 * G -  18 * B + 128) >> 8) + 128;

                // NV21 has a plane of Y and interleaved planes of VU each sampled by a factor of 2
                //    meaning for every 4 Y pixels there are 1 V and 1 U.  Note the sampling is every other
                //    pixel AND every other scanline.
                yuv420sp[yIndex++] = (byte) ((Y < 0) ? 0 : ((Y > 255) ? 255 : Y));
                if (j % 2 == 0 && index % 2 == 0) {
                    yuv420sp[uvIndex++] = (byte)((V<0) ? 0 : ((V > 255) ? 255 : V));
                    yuv420sp[uvIndex++] = (byte)((U<0) ? 0 : ((U > 255) ? 255 : U));
                }

                index ++;
            }
        }
    }

    public void sendFrame(Bitmap bitmap) {
        image.set(bitmap);
    }

    @Override
    public void initialize(SurfaceTextureHelper surfaceTextureHelper, Context applicationContext,
                           CapturerObserver capturerObserver) {
        this.capturerObserver = capturerObserver;
    }

    @Override
    public void startCapture(int width, int height, int framerate) {
        //timer.schedule(tickTask, 0, 1000 / framerate);
        threadCV().start();
    }

    Thread threadCV() {
        return new Thread() {
            @Override
            public void run() {
                while (true) {
                    if (image.get() != null) {
                        tick();
                    }

                    try {
                        Thread.sleep(10);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                }
            }
        };
    }

    @Override
    public void stopCapture() throws InterruptedException {
        timer.cancel();
    }

    @Override
    public void changeCaptureFormat(int width, int height, int framerate) {
        // Empty on purpose
    }

    @Override
    public void dispose() {
        //videoReader.close();
    }

    @Override
    public boolean isScreencast() {
        return false;
    }

    private interface VideoReader {
        VideoFrame getNextFrame();

        void close();
    }

    /**
     * Read video data from file for the .y4m container.
     */

}

On the local participant I than use this function to send the frame:

    public void sendFrame(Bitmap frame) {
        customVideoCapturer.sendFrame(frame);
    }

But I wasn't be able to thake the frames from the wikitude Camera. There is a way to access the frames and resend them?

Solution

Such as of the Native Api sdk, version 9.10.0, according to answer from wikitude support https://support.wikitude.com/support/discussions/topics/5000096719?page=1 , to access the camera frames a custom plugin should be created: https://www.wikitude.com/external/doc/documentation/latest/androidnative/pluginsapi.html#plugins-api

Answered By - vnapastiuk