fix: rotation matrix to threejs

OpenHPS · Nov 27, 2023 · c19b3d7 · c19b3d7
1 parent 42112c9
commit c19b3d7
Show file tree

Hide file tree

Showing 4 changed files with 38 additions and 21 deletions.
diff --git a/examples/eswc2024/src/nodes/ArUcoMarkerDetection.ts b/examples/eswc2024/src/nodes/ArUcoMarkerDetection.ts
@@ -1,7 +1,7 @@
 import { ImageFrame } from "@openhps/video";
-import { Absolute3DPosition } from '@openhps/core';
+import { Absolute3DPosition, Matrix3, Matrix4, Orientation } from '@openhps/core';
 import { ImageProcessingNode, cv as OpenCV, ImageProcessingOptions } from '@openhps/opencv/web';
-import { ArUcoMarker } from "@/models";
+import { ArUcoMarker } from "../models";
 
 const { cv } = require('@openhps/opencv/web'); // eslint-disable-line
 
@@ -53,12 +53,25 @@ export class ArUcoMarkerDetection<InOut extends ImageFrame> extends ImageProcess
                     // Calculate pose for each marker
                     for (let i = 0; i < markerIds.rows; i++) {
                         cv.solvePnP(objPoints, corners.get(i), cameraMatrix, distCoeffs, rvec, tvec);
+                        cv.drawFrameAxes(image, cameraMatrix, distCoeffs, rvec, tvec, 0.1);
                         // Get rotation matrix
                         cv.Rodrigues(rvec, rotationMat);
-                        cv.drawFrameAxes(image, cameraMatrix, distCoeffs, rvec, tvec, 0.1);
                         const marker = new ArUcoMarker();
                         marker.identifier = markerIds.data[i];
                         marker.setPosition(new Absolute3DPosition());
+                        const matrix = new Matrix4().setFromMatrix3(new Matrix3(
+                            rotationMat.data32F[0],
+                            rotationMat.data32F[1],
+                            rotationMat.data32F[2],
+                            rotationMat.data32F[3],
+                            rotationMat.data32F[4],
+                            rotationMat.data32F[5],
+                            rotationMat.data32F[6],
+                            rotationMat.data32F[7],
+                            rotationMat.data32F[8],
+                        ));
+                        marker.position.orientation = Orientation.fromRotationMatrix(matrix);
+                        frame.addObject(marker);
                     }
 
                     rvec.delete();
@@ -76,6 +89,7 @@ export class ArUcoMarkerDetection<InOut extends ImageFrame> extends ImageProcess
                 dictionary.delete();
                 detector.delete();
                 refineParams.delete();
+                gray.delete();
                 resolve(image);
             } catch (ex) {
                 if (typeof ex === 'number') {

diff --git a/examples/eswc2024/src/nodes/ThreeJSNode.ts b/examples/eswc2024/src/nodes/ThreeJSNode.ts
@@ -1,5 +1,7 @@
+import { ArUcoMarker } from "@/models";
+import { DataFrame } from "@openhps/core";
 import { cv, ImageProcessingNode, ImageProcessingOptions } from "@openhps/opencv/web";
-import { ImageFrame } from "@openhps/video";
+import { ImageFrame, PerspectiveCameraObject } from "@openhps/video";
 import * as THREE from 'three';
 
 export class ThreeJSNode<InOut extends ImageFrame> extends ImageProcessingNode<InOut> {
@@ -8,9 +10,15 @@ export class ThreeJSNode<InOut extends ImageFrame> extends ImageProcessingNode<I
 
     constructor(options?: ThreeJSNodeOptions) {
         super(options);
+        const camera = new PerspectiveCameraObject("camera");
+        camera.fov = 40;
+        camera.width = 1024;
+        camera.height = 720;
+        camera.near = 0.01;
+        camera.far = 1000;
     }
 
-    processImage(image: cv.Mat): Promise<cv.Mat> {
+    processImage(image: cv.Mat, frame: DataFrame): Promise<cv.Mat> {
         return new Promise((resolve) => {
             this.options.canvas.width = image.cols;
             this.options.canvas.height = image.rows;
@@ -20,21 +28,18 @@ export class ThreeJSNode<InOut extends ImageFrame> extends ImageProcessingNode<I
 
             const scene = new THREE.Scene();
 
-            const geometry = new THREE.BoxGeometry(0.2, 0.2, 0.2);
-            const material = new THREE.MeshNormalMaterial();
+            frame.getObjects(ArUcoMarker).forEach(marker => {
+                const geometry = new THREE.BoxGeometry(0.2, 0.2, 0.2);
+                const material = new THREE.MeshNormalMaterial();
+                const mesh = new THREE.Mesh(geometry, material);
+                mesh.rotation.setFromRotationMatrix(marker.position.orientation.toRotationMatrix());
+                scene.add(mesh);
+            });
 
-            const mesh = new THREE.Mesh(geometry, material);
-            scene.add(mesh);
+            const renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true, canvas: this.options.canvas });
+            renderer.setSize(this.options.canvas.width, this.options.canvas.height);
+            renderer.render(scene, camera);
 
-            // const renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true, canvas: this.options.canvas });
-            // renderer.setSize(this.options.canvas.width, this.options.canvas.height);
-            // renderer.render(scene, camera);
-
-            try {
-                cv.imshow(this.options.canvas, image);
-            } catch(ex) {
-                console.log(ex)
-            }
             resolve(image);
         });
     }

diff --git a/examples/eswc2024/src/views/CameraPage.vue b/examples/eswc2024/src/views/CameraPage.vue
@@ -44,9 +44,6 @@ export default class CameraPage extends Vue {
 </script>
 
 <style scoped>
-#camera {
-  display: none;
-}
 .ar {
   position: absolute;
   top: 0;

diff --git a/examples/eswc2024/tsconfig.json b/examples/eswc2024/tsconfig.json
@@ -14,6 +14,7 @@
     "sourceMap": true,
     "baseUrl": ".",
     "types": [
+      "node",
       "webpack-env",
       "jest"
     ],