<!doctype html>
<!--
Copyright 2022 The Immersive Web Community Group
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-->
<html>
<head>
<meta charset='utf-8'>
<meta name='viewport' content='width=device-width, initial-scale=1, user-scalable=no'>
<meta name='mobile-web-app-capable' content='yes'>
<meta name='apple-mobile-web-app-capable' content='yes'>
<title>WebXR Camera Access Marker Tracking</title>
<link href='../css/common.css' rel='stylesheet'></link>
<style>
#text-info {
position: absolute;
bottom: 5%;
left: 2%;
font-family: monospace;
color: white;
background-color: rgba(0, 0, 0, 0.5);
}
canvas {
position: relative;
left: initial;
top: initial;
right: initial;
bottom: initial;
/*width: initial;*/
width: 50%;
height: initial;
}
</style>
<!--
This needs a version of opencv.js that's built including opencv_contrib to ensure
modules/aruco/ is present. The default distribution doesn't include that.
Here are the build instructions for a non-multithreaded WASM build with SIMD enabled, based
on https://docs.opencv.org/4.x/d4/da1/tutorial_js_setup.html . The specific instructions
are for a Debian/Ubuntu system, please adjust as needed for other platforms.
sudo apt-get install emscripten
git clone https://github.com/opencv/opencv.git
git clone https://github.com/opencv/opencv_contrib.git
python3 ./opencv/platforms/js/build_js.py \
--emscripten_dir /usr/share/emscripten \
--cmake_option="-DOPENCV_EXTRA_MODULES_PATH=../opencv_contrib/modules" \
--build_wasm --simd build_wasm_simd
The resulting output file is: build_wasm_simd/bin/opencv.js
-->
<script src="opencv/opencv.js"></script>
</head>
<body>
<header>
<details open>
<summary>Camera Access Marker Tracking</summary>
This sample tracks <a href="https://docs.opencv.org/4.x/d5/dae/tutorial_aruco_detection.html">ArUco markers</a> using OpenCV.js with WASM and SIMD enabled.
<p>
<input id="asyncRead" type="checkbox" checked>
<label for="asyncRead">Read pixel data asynchronously<br/>
<input id="delayImage" type="checkbox" checked>
<label for="delayImage">In async mode, delay camera image to match poses<br/>
<input id="debugOpenCV" type="checkbox">
<label for="debugOpenCV">Show debug image for OpenCV</label><br/>
<input type="range" id="cvDownscale" min="1" max="8" value="4">
<label for="cvDownscale">Downscale image <span id="cvDownscaleValue">4</span>x before marker detection.</label><br/>
<a class="back" href="./index.html">Back</a>
</p>
<div id="warning-zone"></div>
<button id="xr-button" class="barebones-button" disabled>XR not found</button>
</details>
</header>
<div id="text-overlay">
<div id="text-info"></div>
<canvas id="out-canvas" width="25%" height="25%" style='display: none; top: 10px; left: 10px; border: 2px solid green;'>
</div>
<main style='text-align: center;'>
<p>Click 'Enter AR' to see content</p>
</main>
<script type="module">
import * as mat4 from "../js/third-party/gl-matrix/mat4.js"
import * as vec3 from "../js/third-party/gl-matrix/vec3.js"
import * as vec4 from "../js/third-party/gl-matrix/vec4.js"
import {QueryArgs} from '../js/cottontail/src/util/query-args.js';
// XR globals.
let xrButton = document.getElementById('xr-button');
let xrSession = null;
let xrRefSpace = null;
// WebGL scene globals.
let gl = null;
let glBinding = null;
let cubeRotation = 0.0;
let rotationSpeed = 0.01;
let shaderProgram = null;
let programInfo = null;
let buffers = null;
let readback_framebuffer = null;
let readback_pixels = null;
let framebufferCompletenessChecked = false;
let glErrorsChecked = false;
let scaledWidth = 0;
let scaledHeight = 0;
let quadCopyShaderProgram = null;
let quadCopyProgramInfo = null;
let quadCopyBuffers = null;
let quadCopyTextures = [];
let quadCopyTexIdx = 0;
let backgroundTexture = null;
let readPixelBuf = null;
let readPixelsStarted = false;
let readPixelsDone = false;
let pendingMarkerDetection = null;
let frameNum = 0;
let arc = null;
let markersInFrame = [];
let imgScale = parseInt(document.getElementById('cvDownscale').value);
// If requested, use DOM overlay to provide information about the read color:
const use_dom_overlay = QueryArgs.getBool('useDomOverlay', true);
// Optionally, enable time traces for use with chrome://tracing . This
// is off by default since it spams the console log with timing data.
const use_timers = QueryArgs.getBool('useTimers', false);
const textOverlayElement = document.querySelector("#text-overlay");
if (!textOverlayElement) {
console.error("#text-overlay element not found!");
throw new Error("#text-overlay element not found!");
}
const textInfoElement = document.querySelector("#text-info");
if (!textInfoElement) {
console.error("#text-info element not found!");
throw new Error("#text-info element not found!");
}
document.getElementById('cvDownscale').onchange = (ev) => {
document.getElementById('cvDownscaleValue').innerText = ev.target.value;
};
document.getElementById('asyncRead').onchange = (ev) => {
// Disable the "delay image" input if in synchronous mode since it's
// not applicable in that case.
document.getElementById('delayImage').disabled =
ev.target.checked ? '' : 'disabled';
};
let isWebXRSupported = false;
let isOpenCVLoaded = false;
cv.then(() => {
console.log('OpenCV load complete.');
// Hack: the cv module doesn't seem to be loading properly?
if (!cv.imread) cv = Module;
isOpenCVLoaded = true;
updateXRButton();
});
function updateXRButton() {
console.log('isWebXRSupported=' + isWebXRSupported + " isOpenCVLoaded=" + isOpenCVLoaded);
if (isWebXRSupported && isOpenCVLoaded) {
xrButton.innerHTML = 'Enter AR';
xrButton.disabled = false;
} else {
xrButton.innerHTML = 'AR not found';
xrButton.disabled = true;
}
}
function checkSupportedState() {
navigator.xr.isSessionSupported('immersive-ar').then((supported) => {
console.log('isSessionSupported returned ' + supported);
isWebXRSupported = supported;
updateXRButton();
});
}
function initXR() {
if (!window.isSecureContext) {
let message = "WebXR unavailable due to insecure context";
document.getElementById("warning-zone").innerText = message;
}
if (navigator.xr) {
xrButton.addEventListener('click', onButtonClicked);
navigator.xr.addEventListener('devicechange', checkSupportedState);
checkSupportedState();
}
}
function onButtonClicked() {
if (!xrSession) {
const sessionOptions = {
requiredFeatures: ['camera-access'],
trackedImages: [],
//optionalFeatures: ['image-tracking'], // for autofocus
optionalFeatures: [],
};
if (use_dom_overlay) {
sessionOptions.requiredFeatures.push('dom-overlay');
//sessionOptions.domOverlay = { root: document.body };
sessionOptions.domOverlay = { root: textOverlayElement };
}
navigator.xr.requestSession('immersive-ar', sessionOptions)
.then(onSessionStarted, onRequestSessionError);
} else {
xrSession.end();
}
}
function onSessionStarted(session) {
xrSession = session;
xrButton.innerHTML = 'Exit AR';
session.addEventListener('end', onSessionEnded);
let canvas = document.createElement('canvas');
gl = canvas.getContext('webgl2', {
xrCompatible: true
});
glBinding = new XRWebGLBinding(session, gl);
// Init cube geometry and cube's default texture.
initializeGLCube(gl);
initializeGLQuadCopy(gl);
document.getElementById('out-canvas').style.display =
document.getElementById('debugOpenCV').checked ?
'initial' : 'none';
for (let i = 0; i < 2; ++i) {
quadCopyTextures[i] = gl.createTexture();
}
readback_framebuffer = gl.createFramebuffer();
readPixelBuf = gl.createBuffer();
session.updateRenderState({ baseLayer: new XRWebGLLayer(session, gl) });
session.requestReferenceSpace('viewer').then((refSpace) => {
xrRefSpace = refSpace;
session.requestAnimationFrame(onXRFrame);
});
imgScale = parseInt(document.getElementById('cvDownscale').value);
initializeMarkerTracking();
}
function onRequestSessionError(ex) {
alert("Failed to start immersive AR session.");
console.error(ex.message);
}
function onEndSession(session) {
session.end();
}
function onSessionEnded(event) {
xrSession = null;
xrButton.innerHTML = 'Enter AR';
gl = null;
framebufferCompletenessChecked = false;
glErrorsChecked = false;
lastTime = 0;
readback_pixels = null;
backgroundTexture = null;
readPixelsStarted = false;
readPixelsDone = false;
pendingMarkerDetection = null;
markersInFrame = [];
}
// Only print each unique intrinsic string once.
const intrinsicsPrinted = {};
// Calculates the camera intrinsics matrix from a projection matrix and viewport.
// See camera-access-barebones.html in this directory for a detailed comment about
// this function.
function getCameraIntrinsics(projectionMatrix, viewport) {
const p = projectionMatrix;
// Principal point in pixels (typically at or near the center of the viewport)
let u0 = (1 - p[8]) * viewport.width / 2 + viewport.x;
let v0 = (1 - p[9]) * viewport.height / 2 + viewport.y;
// Focal lengths in pixels (these are equal for square pixels)
let ax = viewport.width / 2 * p[0];
let ay = viewport.height / 2 * p[5];
// Skew factor in pixels (nonzero for rhomboid pixels)
let gamma = viewport.width / 2 * p[4];
// Print the calculated intrinsics, but once per unique value to
// avoid log spam. These can change every frame for some XR devices.
const intrinsicString = (
"intrinsics: u0=" +u0 + " v0=" + v0 + " ax=" + ax + " ay=" + ay +
" gamma=" + gamma + " for viewport {width=" +
viewport.width + ",height=" + viewport.height + ",x=" +
viewport.x + ",y=" + viewport.y + "}");
if (!intrinsicsPrinted[intrinsicString]) {
console.log("projection:", Array.from(projectionMatrix).join(", "));
console.log(intrinsicString);
intrinsicsPrinted[intrinsicString] = true;
}
return {u0, v0, ax, ay, gamma};
}
let lastTime = 0;
const frameTimes = [];
const frameTimeMax = 10;
let frameTimeIdx = 0;
function showFps(timestamp) {
if (lastTime) {
const delta = timestamp - lastTime;
frameTimes[frameTimeIdx] = delta;
frameTimeIdx = (frameTimeIdx + 1) % frameTimeMax;
if (frameTimes.length >= frameTimeMax - 1) {
const fps = Math.round(1000 * frameTimeMax / frameTimes.reduce((sum, v) => sum + v));
const msg = '' + fps + ' fps';
console.debug(msg);
if (use_dom_overlay) {
document.getElementById('text-info').innerText = msg;
}
}
}
lastTime = timestamp;
}
function consoleTimeStart(name) {
if (use_timers) console.time(name);
}
function consoleTimeEnd(name) {
if (use_timers) console.timeEnd(name);
}
function onXRFrame(frameTimestamp, frame) {
++frameNum;
let session = frame.session;
session.requestAnimationFrame(onXRFrame);
let pose = frame.getViewerPose(xrRefSpace);
if (!pose) return;
for (let view of pose.views) {
let viewport = session.renderState.baseLayer.getViewport(view);
let asyncRead = document.getElementById('asyncRead').checked;
if (asyncRead) {
// Use asynchronous readPixels and draw the augmented scene based on the
// last-detected marker poses, effectively adding a frame of delay to the
// session. If the scene also had elements that are based on poses from the XR
// session, such as hit test results, those poses would also need to be saved
// to match the marker poses.
// WebXR rAF for frame N
// start async readPixels for frame N
// run marker detection for frame N-1
// draw scene for frame N-1
//
// WebXR rAF for frame N+1
// start async readPixels for frame N+1
// run marker detection for frame N
// draw scene for frame N
//
// ... etc. There may be be rAF calls where the async readPixels for the
// previous frame hasn't completed yet. In that case, just re-draw the
// scene based on the last-available marker poses, reusing the corresponding
// old camera image as background as appropriate.
if (pendingMarkerDetection) {
// We have read the pixels for a previous frame, but haven't started marker
// detection on it yet. It's now time to grab a fresh frame. Marker processing
// will start below.
readPixelsDone = false;
}
if (!readPixelsStarted && !readPixelsDone) {
// We're not currently running marker detection. Copy the camera frame and kick
// off the asynchronous readPixels + marker detection for it.
console.debug('start frame ' + frameNum + ' readPixels');
if (!view.camera) return;
if (!copyCameraTexture(view)) return;
readPixelsAndScheduleMarkerDetectionAsync(view);
readPixelsStarted = true;
// Update the FPS counter whenever we start processing a fresh image.
showFps(frameTimestamp);
}
if (pendingMarkerDetection) {
// Run marker detection on the previous frame synchronously, while the
// asynchronous readPixels for the current frame is happening in the
// background.
pendingMarkerDetection();
pendingMarkerDetection = null;
}
// Draw the scene based on the last-available camera image and marker poses.
console.debug('draw frame ' + frameNum);
const delayImage = document.getElementById('delayImage').checked;
drawMarkerScene(session, viewport, view, delayImage);
} else {
// Simple version - copy the camera texture, readPixels, detect markers, draw them.
if (!view.camera) return;
if (!copyCameraTexture(view)) return;
consoleTimeStart("readPixelsSync");
gl.readPixels(0, 0, scaledWidth, scaledHeight,
gl.RGBA, gl.UNSIGNED_BYTE, readback_pixels);
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
consoleTimeEnd("readPixelsSync");
detectMarkers(new Uint8ClampedArray(readback_pixels.buffer),
scaledWidth, scaledHeight, view);
drawMarkerScene(session, viewport, view, false);
// Update the FPS counter. This is the simple case, there's one processed camera frame
// per rAF callback.
showFps(frameTimestamp);
}
}
// Once per session, check for GL errors and report them. Then turn
// off this expensive check.
if (!glErrorsChecked) {
const e = gl.getError();
if (e != 0) {
console.warn("Got a GL error:", e);
} else {
glErrorsChecked = true;
}
}
}
function copyCameraTexture(view) {
const cameraTexture = glBinding.getCameraImage(view.camera);
scaledWidth = Math.floor(view.camera.width / imgScale);
scaledHeight = Math.floor(view.camera.height / imgScale);
const texture_bytes = scaledWidth * scaledHeight * 4;
if (!readback_pixels || readback_pixels.length != texture_bytes) {
readback_pixels = new Uint8Array(texture_bytes);
for (let i = 0; i < quadCopyTextures.length; ++i) {
gl.bindTexture(gl.TEXTURE_2D, quadCopyTextures[i]);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, scaledWidth, scaledHeight,
0, gl.RGBA, gl.UNSIGNED_BYTE, null);
}
}
console.debug('frame ' + frameNum + ' update texture ' + quadCopyTexIdx);
gl.bindTexture(gl.TEXTURE_2D, quadCopyTextures[quadCopyTexIdx]);
gl.bindFramebuffer(gl.FRAMEBUFFER, readback_framebuffer);
gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0,
gl.TEXTURE_2D, quadCopyTextures[quadCopyTexIdx], 0);
// Once per session, check if the framebuffer setup worked.
if (!framebufferCompletenessChecked &&
gl.checkFramebufferStatus(gl.FRAMEBUFFER) != gl.FRAMEBUFFER_COMPLETE) {
console.warn("Framebuffer incomplete!");
return false;
}
framebufferCompletenessChecked = true;
gl.viewport(0, 0, scaledWidth, scaledHeight);
quadCopyDraw(gl, quadCopyProgramInfo, quadCopyBuffers, cameraTexture);
return true;
}
function readPixelsAndScheduleMarkerDetectionAsync(view) {
const subtaskFrameNum = frameNum;
const subtaskQuadCopyTexIdx = quadCopyTexIdx;
consoleTimeStart("readPixelsAsync");
//readback_pixels.fill(0);
readPixelsAsync(gl, 0, 0, scaledWidth, scaledHeight, gl.RGBA, gl.UNSIGNED_BYTE,
readback_pixels, readPixelBuf).then(() => {
consoleTimeEnd("readPixelsAsync");
console.debug('finished frame ' + subtaskFrameNum + ' readPixels');
readPixelsStarted = false;
// session ended?
if (!readback_pixels) return;
readPixelsDone = true;
pendingMarkerDetection = () => {
console.debug('async do frame ' + subtaskFrameNum + ' marker detection, texture ' + subtaskQuadCopyTexIdx);
detectMarkers(new Uint8ClampedArray(readback_pixels.buffer),
scaledWidth, scaledHeight, view);
backgroundTexture = quadCopyTextures[subtaskQuadCopyTexIdx];
console.debug('finished frame ' + subtaskFrameNum + ' marker detection, new background texture ' + subtaskQuadCopyTexIdx);
};
});
quadCopyTexIdx = (quadCopyTexIdx + 1) % quadCopyTextures.length;
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
}
function drawMarkerScene(session, viewport, view, drawBackground) {
consoleTimeStart("drawMarkerScene");
gl.bindFramebuffer(gl.FRAMEBUFFER, session.renderState.baseLayer.framebuffer);
gl.clearColor(0, 0, 0, 0);
gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
gl.enable(gl.DEPTH_TEST);
gl.enable(gl.CULL_FACE);
gl.cullFace(gl.BACK);
gl.viewport(viewport.x, viewport.y,
viewport.width, viewport.height);
if (drawBackground && backgroundTexture) {
// Draw the stored camera image matching the scene's pose to the screen. Note that
// the backgroundTexture was flipped vertically (to match OpenCV expectations), and
// is now flipped vertically again to ensure a correctly oriented image.
gl.disable(gl.DEPTH_TEST);
quadCopyDraw(gl, quadCopyProgramInfo, quadCopyBuffers, backgroundTexture);
gl.enable(gl.DEPTH_TEST);
}
if (drawBackground) {
gl.enable(gl.BLEND);
gl.blendFunc(gl.SRC_COLOR, gl.DST_COLOR);
}
const markerColor = vec4.fromValues(1, 0, 0, 0.75);
for (let i = 0; i < markersInFrame.length; ++i) {
let pose = markersInFrame[i].pose;
let id = markersInFrame[i].id;
//console.log('i=', i, 'id=', id, 'pose=', pose);
markerColor[0] = (1 + Math.cos(id * 2.5)) / 2;
markerColor[1] = (1 + Math.sin(id * 2.5)) / 2;
markerColor[2] = (1 + Math.sin(id * 4.1)) / 2;
drawCube(pose, gl, programInfo, buffers, view, markerColor);
}
gl.disable(gl.BLEND);
consoleTimeEnd("drawMarkerScene");
}
let markerImage = null;
let dictionary = null;
let markerCorners = null;
let markerIds = null;
let rvecs = null;
let tvecs = null;
let detectorParams = null;
function initializeMarkerTracking() {
markerImage = new cv.Mat();
dictionary = new cv.aruco_Dictionary(cv.DICT_6X6_250);
markerCorners = new cv.MatVector();
markerIds = new cv.Mat();
rvecs = new cv.Mat();
tvecs = new cv.Mat();
detectorParams = new cv.aruco_DetectorParameters();
detectorParams.useAruco3Detection = true;
}
function detectMarkers(rawData, scaledWidth, scaledHeight, view) {
consoleTimeStart("detectMarkers");
const debugOpenCV = document.getElementById('debugOpenCV').checked;
const img = new cv.Mat(scaledHeight, scaledWidth, cv.CV_8UC4);
img.data.set(rawData);
// OpenCV needs either a three-channel RGB image or a grayscale image. The source is
// four-channel RGBA and needs to be converted.
cv.cvtColor(img, img, cv.COLOR_RGBA2GRAY, 0);
// Run ArUco marker detection
cv.detectMarkers(img, dictionary, markerCorners, markerIds, detectorParams);
// Assume that the camera image is undistorted. We don't have access to distortion
// parameters. The AR implementation should have taken care of any required corrections.
let distCoeffs = cv.matFromArray(5, 1, cv.CV_64F, [0, 0, 0, 0, 0]);
// Calculate camera intrinsics based on the scaled texture size. Note that the image
// data was vertically flipped by the quad copy, so the data starts with the top left
// pixel as expected by OpenCV. The marker pose transformation will convert from
// OpenCV's camera space to WebXR viewer space.
let intrinsics = getCameraIntrinsics(view.projectionMatrix,
{width: scaledWidth, height: scaledHeight, x: 0, y: 0});
let cameraMatrix = cv.matFromArray(3, 3, cv.CV_64F,
[intrinsics.ax, 0, intrinsics.u0,
0, intrinsics.ay, intrinsics.v0,
0, 0, 1]);
// Marker detection needs to know the physical marker size in meters. If the real size
// is different, a 3D rendering based on the tracked marker will still look correct on a
// 2D screen, but the depth will be wrong. This could lead to occlusion or physics
// issues where a correct depth would be necessary.
const markerSize = 0.1;
cv.estimatePoseSingleMarkers(markerCorners, markerSize, cameraMatrix, distCoeffs, rvecs, tvecs);
markersInFrame = [];
for (let i = 0; i < markerIds.rows; ++i) {
// Rotation vector pointing along the rotation axis. Its length
// provides the rotation angle in radians.
let rvec = cv.matFromArray(3, 1, cv.CV_64F,
[rvecs.doublePtr(0, i)[0],
rvecs.doublePtr(0, i)[1],
rvecs.doublePtr(0, i)[2]]);
// Translation vector in OpenCV camera space.
let tvec = cv.matFromArray(3, 1, cv.CV_64F,
[tvecs.doublePtr(0, i)[0],
tvecs.doublePtr(0, i)[1],
tvecs.doublePtr(0, i)[2]]);
if (debugOpenCV) {
cv.drawFrameAxes(img, cameraMatrix, distCoeffs, rvec, tvec, markerSize);
}
// Convert rvec/tvec to a pose in XR space. (For clarity, read these operations from
// the bottom up to see how the transform from model space to world space is
// constructed.)
let m = mat4.create();
// OpenCV's camera coordinate system has +x right, +y down, +z away.
// We need +y down and -z away, this is a 180 degree rotation around the X axis.
mat4.fromRotation(m, Math.PI, [1, 0, 0]);
// Apply the tvec translation.
mat4.translate(m, m, tvecs.doublePtr(0, i));
// Rotate the cube based on rvec's length and axis.
mat4.rotate(m, m, vec3.length(rvecs.doublePtr(0, i)), rvecs.doublePtr(0, i))
// Resize the cube to match the marker size.
mat4.scale(m, m, [markerSize / 2, markerSize / 2, markerSize / 2]);
// The drawn cube has a 2 unit (meter) edge length. Translate it so that
// its origin is in the center of the bottom face.
mat4.translate(m, m, [0, 0, 1]);
// Save the marker for rendering.
markersInFrame.push({id: markerIds.intAt(i), pose: m});
}
if (debugOpenCV) {
cv.drawDetectedMarkers(img, markerCorners, markerIds);
cv.imshow('out-canvas', img);
}
img.delete();
consoleTimeEnd("detectMarkers");
}
// This GL rendering code is adapted from a publicly provided code sample found at
// https://developer.mozilla.org/en-US/docs/Web/API/WebGL_API/Tutorial/Using_textures_in_WebGL
function initializeGLQuadCopy(gl) {
quadCopyShaderProgram = initShaderProgram(gl, `
attribute vec2 a_position;
varying vec2 v_uv;
void main() {
gl_Position = vec4(a_position, 0.0, 1.0);
// Rescale UV from [-1, 1] to [0, 1]. This also adds
// a Y flip since OpenCV expects the image data to start
// at the top left corner. When drawing the resulting
// texture to the screen, it'll be Y flipped again to
// appear in its normal orientation.
v_uv.x = (a_position.x + 1.0) * 0.5;
v_uv.y = (1.0 - a_position.y) * 0.5;
}
`, `
precision mediump float;
varying vec2 v_uv;
uniform sampler2D u_texture;
void main() {
gl_FragColor = texture2D(u_texture, v_uv);
gl_FragColor.a = 1.0;
}
`);
quadCopyProgramInfo = {
program: quadCopyShaderProgram,
attribLocations: {
vertexPosition: gl.getAttribLocation(quadCopyShaderProgram, 'a_position'),
},
uniformLocations: {
uSampler: gl.getUniformLocation(quadCopyShaderProgram, 'u_texture'),
},
};
quadCopyBuffers = initQuadCopyBuffers(gl);
}
function initQuadCopyBuffers(gl) {
// Create a buffer for the cube's vertex positions.
const positionBuffer = gl.createBuffer();
// Select the positionBuffer as the one to apply buffer
// operations to from here out.
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
// Now create an array of positions for the cube.
const positions = [
-1, 1,
-1, -1,
1, -1,
1, -1,
1, 1,
-1, 1,
]
// Now pass the list of positions into WebGL to build the
// shape. We do this by creating a Float32Array from the
// JavaScript array, then use it to fill the current buffer.
gl.bufferData(
gl.ARRAY_BUFFER,
new Float32Array(positions),
gl.STATIC_DRAW
);
return {
position: positionBuffer,
};
}
function quadCopyDraw(gl, programInfo, buffers, texture, view) {
// Tell WebGL how to pull out the positions from the position
// buffer into the vertexPosition attribute
{
const numComponents = 2;
const type = gl.FLOAT;
const normalize = false;
const stride = 0;
const offset = 0;
gl.bindBuffer(gl.ARRAY_BUFFER, buffers.position);
gl.vertexAttribPointer(
programInfo.attribLocations.vertexPosition,
numComponents,
type,
normalize,
stride,
offset
);
gl.enableVertexAttribArray(
programInfo.attribLocations.vertexPosition
);
}
gl.useProgram(programInfo.program);
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
// Mipmaps might be useful for >2x downscaling, but this
// apparently doesn't work for the opaque camera texture. It
// requires a WebGL2 context since it's non-power-of-2, but
// still fails silently (framebuffer incomplete).
//gl.generateMipmap(gl.TEXTURE_2D);
gl.uniform1i(programInfo.uniformLocations.uSampler, 0);
gl.drawArrays(gl.TRIANGLES, 0, 6);
}
// Vertex shader program
const vsSource = `
attribute vec4 aVertexPosition;
uniform mat4 uModelViewMatrix;
uniform mat4 uProjectionMatrix;
void main(void) {
gl_Position = uProjectionMatrix * uModelViewMatrix * aVertexPosition;
}
`;
// Fragment shader program
const fsSource = `
uniform mediump vec4 uBaseColor;
precision mediump float;
void main(void) {
gl_FragColor = uBaseColor;
}
`;
function initializeGLCube(gl) {
// Initialize a shader program; this is where all the lighting
// for the vertices and so forth is established.
shaderProgram = initShaderProgram(gl, vsSource, fsSource);
// Collect all the info needed to use the shader program.
// Look up which attributes our shader program is using
// for aVertexPosition, aTextureCoord and also
// look up uniform locations.
programInfo = {
program: shaderProgram,
attribLocations: {
vertexPosition: gl.getAttribLocation(shaderProgram, 'aVertexPosition'),
},
uniformLocations: {
projectionMatrix: gl.getUniformLocation(shaderProgram, 'uProjectionMatrix'),
modelViewMatrix: gl.getUniformLocation(shaderProgram, 'uModelViewMatrix'),
uBaseColor: gl.getUniformLocation(shaderProgram, 'uBaseColor'),
},
};
// Here's where we call the routine that builds all the
// objects we'll be drawing.
buffers = initBuffers(gl);
}
// Initialize the buffers we'll need. For this demo, we just
// have one object -- a simple three-dimensional cube.
function initBuffers(gl) {
// Create a buffer for the cube's vertex positions.
const positionBuffer = gl.createBuffer();
// Select the positionBuffer as the one to apply buffer
// operations to from here out.
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
// Now create an array of positions for the cube.
const positions = [
// Front face
-1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0,
// Back face
-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0,
// Top face
-1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, -1.0,
// Bottom face
-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, 1.0,
// Right face
1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0,
// Left face
-1.0, -1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0,
];
// Now pass the list of positions into WebGL to build the
// shape. We do this by creating a Float32Array from the
// JavaScript array, then use it to fill the current buffer.
gl.bufferData(
gl.ARRAY_BUFFER,
new Float32Array(positions),
gl.STATIC_DRAW
);
// Build the element array buffer; this specifies the indices
// into the vertex arrays for each face's vertices.
const indexBuffer = gl.createBuffer();
gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, indexBuffer);
// This array defines each face as two triangles, using the
// indices into the vertex array to specify each triangle's
// position.
const indices = [
0, 1, 2, 0, 2, 3, // front
4, 5, 6, 4, 6, 7, // back
8, 9, 10, 8, 10, 11, // top
12, 13, 14, 12, 14, 15, // bottom
16, 17, 18, 16, 18, 19, // right
20, 21, 22, 20, 22, 23, // left
];
// Now send the element array to GL
gl.bufferData(
gl.ELEMENT_ARRAY_BUFFER,
new Uint16Array(indices),
gl.STATIC_DRAW
);
return {
position: positionBuffer,
indices: indexBuffer,
};
}
function drawCube(modelViewMatrix, gl, programInfo, buffers, view, baseColor) {
// Tell WebGL how to pull out the positions from the position
// buffer into the vertexPosition attribute
{
const numComponents = 3;
const type = gl.FLOAT;
const normalize = false;
const stride = 0;
const offset = 0;
gl.bindBuffer(gl.ARRAY_BUFFER, buffers.position);
gl.vertexAttribPointer(
programInfo.attribLocations.vertexPosition,
numComponents,
type,
normalize,
stride,
offset
);
gl.enableVertexAttribArray(
programInfo.attribLocations.vertexPosition
);
}
// Tell WebGL which indices to use to index the vertices
gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, buffers.indices);
// Tell WebGL to use our program when drawing
gl.useProgram(programInfo.program);
// Set the shader uniforms
gl.uniformMatrix4fv(
programInfo.uniformLocations.projectionMatrix,
false,
view.projectionMatrix
);
gl.uniformMatrix4fv(
programInfo.uniformLocations.modelViewMatrix,
false,
modelViewMatrix
);
gl.uniform4fv(programInfo.uniformLocations.uBaseColor, baseColor);
{
const vertexCount = 36;
const type = gl.UNSIGNED_SHORT;
const offset = 0;
gl.drawElements(gl.TRIANGLES, vertexCount, type, offset);
}
}
// Initialize a shader program, so WebGL knows how to draw our data
//
function initShaderProgram(gl, vsSource, fsSource) {
const vertexShader = loadShader(gl, gl.VERTEX_SHADER, vsSource);
const fragmentShader = loadShader(gl, gl.FRAGMENT_SHADER, fsSource);
// Create the shader program
const shaderProgram = gl.createProgram();
gl.attachShader(shaderProgram, vertexShader);
gl.attachShader(shaderProgram, fragmentShader);
gl.linkProgram(shaderProgram);
// If creating the shader program failed, alert
if (!gl.getProgramParameter(shaderProgram, gl.LINK_STATUS)) {
alert(
"Unable to initialize the shader program: " +
gl.getProgramInfoLog(shaderProgram)
);
return null;
}
return shaderProgram;
}
// Creates a shader of the given type, uploads the source and
// compiles it.
//
function loadShader(gl, type, source) {
const shader = gl.createShader(type);
// Send the source to the shader object
gl.shaderSource(shader, source);
// Compile the shader program
gl.compileShader(shader);
// See if it compiled successfully
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
alert(
"An error occurred compiling the shaders: " +
gl.getShaderInfoLog(shader)
);
gl.deleteShader(shader);
return null;
}
return shader;
}
// ... end of GL rendering code adapted from a publicly provided code sample found at
// https://developer.mozilla.org/en-US/docs/Web/API/WebGL_API/Tutorial/Using_textures_in_WebGL
// This non-blocking async data readback implementation is adapted from a publicly provided code sample found at
// https://developer.mozilla.org/en-US/docs/Web/API/WebGL_API/WebGL_best_practices#use_non-blocking_async_data_readback
function clientWaitAsync(gl, sync, flags, interval_ms) {
return new Promise((resolve, reject) => {
function test() {
const res = gl.clientWaitSync(sync, flags, 0);
if (res === gl.WAIT_FAILED) {
reject();
return;
}
if (res === gl.TIMEOUT_EXPIRED) {
setTimeout(test, interval_ms);
return;
}
resolve();
}
test();
});
}
async function getBufferSubDataAsync(
gl, target, buffer, srcByteOffset, dstBuffer,
/* optional */ dstOffset, /* optional */ length) {
const sync = gl.fenceSync(gl.SYNC_GPU_COMMANDS_COMPLETE, 0);
gl.flush();
await clientWaitAsync(gl, sync, 0, 1);
gl.deleteSync(sync);
gl.bindBuffer(target, buffer);
gl.getBufferSubData(target, srcByteOffset, dstBuffer, dstOffset, length);
gl.bindBuffer(target, null);
return dstBuffer;
}
async function readPixelsAsync(gl, x, y, w, h, format, type, dest, buf) {
gl.bindBuffer(gl.PIXEL_PACK_BUFFER, buf);
gl.bufferData(gl.PIXEL_PACK_BUFFER, dest.byteLength, gl.STREAM_READ);
gl.readPixels(x, y, w, h, format, type, 0);
gl.bindBuffer(gl.PIXEL_PACK_BUFFER, null);
await getBufferSubDataAsync(gl, gl.PIXEL_PACK_BUFFER, buf, 0, dest);
return dest;
}
// ... end of non-blocking async data readback implementation adapted from a publicly provided code sample found at
// https://developer.mozilla.org/en-US/docs/Web/API/WebGL_API/WebGL_best_practices#use_non-blocking_async_data_readback
initXR();
</script>
</body>
</html>