xr now networking hands

Author: EliCDavis

generator/html/js/index.js

@@ -48,6 +48,7 @@ const viewportSettings = {
     ground: "0xcbcbcb"
 }
 
+const representationManager = new WebSocketRepresentationManager();
 const viewportManager = new ViewportManager(viewportSettings);
 const updateLoop = new UpdateManager();
 
@@ -58,6 +59,8 @@ const clock = new THREE.Clock();
 const camera = new THREE.PerspectiveCamera(75, window.innerWidth / window.innerHeight, 0.1, 1000);
 camera.position.set(0, 2, 3);
 
+representationManager.AddRepresentation("player", camera)
+
 const scene = new THREE.Scene();
 scene.background = new THREE.Color(viewportSettings.background);
 
@@ -150,7 +153,7 @@ const nodeManager = new NodeManager(App);
 const schemaManager = new SchemaManager(requestManager, nodeManager);
 
 if (RenderingConfiguration.XrEnabled) {
-    InitXR(scene, renderer);
+    InitXR(scene, renderer, updateLoop, representationManager, groundMesh);
 }
 
 nodeManager.subscribeToParameterChange((param) => {
@@ -517,8 +520,7 @@ function onWindowResize() {
 window.addEventListener('resize', onWindowResize);
 
 
-const representationManager = new WebSocketRepresentationManager();
-representationManager.addRepresentation("player", camera)
+
 
 const websocketManager = new WebSocketManager(
     representationManager,

generator/html/js/websocket.js

@@ -6,37 +6,68 @@ export class WebSocketRepresentationManager {
         this.rerpresentations = new Map();
     }
 
-    addRepresentation(key, threeObj) {
+    AddRepresentation(key, threeObj) {
         if (this.rerpresentations.has(key)) {
-            throw new Error("representation manager already has a rep for key: " + key)
+            console.warn("representation manager already has a rep for key: " + key)
+            // throw new Error("representation manager already has a rep for key: " + key)
         }
         this.rerpresentations.set(key, threeObj)
     }
 
-    removeRepresentation(key, threeObj) {
-        if (this.rerpresentations.has(key)) {
-            throw new Error("representation manager already has a rep for key: " + key)
+    RemoveRepresentation(key, threeObj) {
+        if (!this.rerpresentations.has(key)) {
+            throw new Error("representation manager does not have a rep for key: " + key)
         }
         this.rerpresentations.set(key, threeObj)
     }
 
-    toMessage() {
+    ToMessage() {
         const message = [];
+
+        let pos = new THREE.Vector3();
+        let rot = new THREE.Quaternion();
+
         this.rerpresentations.forEach((threeObj, key) => {
-            message.push({
-                type: key,
-                "position": {
-                    "x": threeObj.position.x,
-                    "y": threeObj.position.y,
-                    "z": threeObj.position.z,
-                },
-                "rotation": {
-                    "x": threeObj.quaternion.x,
-                    "y": threeObj.quaternion.y,
-                    "z": threeObj.quaternion.z,
-                    "w": threeObj.quaternion.w,
+            if (!threeObj) {
+                return;
+            }
+
+            if (!threeObj.matrixWorld) {
+                return;
+            }
+
+            // Somethings wrong with the matrix world, let's not include this
+            // representation
+            const eles = threeObj.matrixWorld.elements;
+            for(let i = 0; i < eles.length; i ++) {
+                if (isNaN(eles[i]) || !isFinite(eles[i])) {
+                    return;
                 }
-            })
+            }
+
+            try {
+                const worldMatrix = threeObj.matrixWorld;
+                pos.setFromMatrixPosition(worldMatrix)
+                rot.setFromRotationMatrix(worldMatrix)
+                message.push({
+                    type: key,
+                    "position": {
+                        "x": pos.x,
+                        "y": pos.y,
+                        "z": pos.z,
+                    },
+                    "rotation": {
+                        "x": rot.x,
+                        "y": rot.y,
+                        "z": rot.z,
+                        "w": rot.w,
+                    }
+                })
+            } catch (error) {
+                console.error(error);
+                // Expected output: ReferenceError: nonExistentFunction is not defined
+                // (Note: the exact output may be browser-dependent)
+            }
         });
         return message;
     }
@@ -103,6 +134,7 @@ export class WebSocketManager {
 
     createPlayerObject(name, playerData) {
         const newPlayer = new THREE.Group();
+        newPlayer.name = "player";
 
         const sphere = new THREE.Mesh(
             this.playerConfiguration.playerGeometry,
@@ -152,6 +184,8 @@ export class WebSocketManager {
         newPlayer.position.x = playerData.position.x;
         newPlayer.position.y = playerData.position.y;
         newPlayer.position.z = playerData.position.z;
+
+        newPlayer.scale.set(0.25, 0.25, 0.25)
         this.scene.add(newPlayer);
 
         return {
@@ -165,18 +199,53 @@ export class WebSocketManager {
         };
     }
 
+    createHandObject(handData) {
+        console.log("creating hand")
+        const hand = new THREE.Group();
+        hand.name = "hand";
+
+        const sphere = new THREE.Mesh(
+            this.playerConfiguration.playerGeometry,
+            this.playerConfiguration.playerMaterial
+        );
+        sphere.scale.set(0.1, 0.1, 0.1)
+        hand.add(sphere);
+
+        hand.position.x = handData.position.x;
+        hand.position.y = handData.position.y;
+        hand.position.z = handData.position.z;
+
+        this.scene.add(hand);
+
+        return {
+            desiredPosition: handData.position,
+            desiredRotation: handData.rotation,
+            obj: hand,
+            cleanup: () => {
+                this.scene.remove(hand);
+            }
+        };
+    }
+
+
     setupPlayer(key, playerData) {
         const resps = []
 
-        console.log(playerData);
-
         if (playerData.representation) {
             playerData.representation.forEach((rep) => {
                 switch (rep.type) {
                     case "player":
                         resps.push(this.createPlayerObject(playerData.name, rep))
                         break;
 
+                    case "left-hand":
+                        resps.push(this.createHandObject(rep));
+                        break;
+
+                    case "right-hand":
+                        resps.push(this.createHandObject(rep));
+                        break;
+
                     default:
                         break;
                 }
@@ -203,7 +272,7 @@ export class WebSocketManager {
 
                 const q = new THREE.Quaternion(dr.x, dr.y, dr.z, dr.w);
                 if (!rep.obj.quaternion.equals(q)) {
-                    rep.obj.quaternion.rotateTowards(q, delta *2);
+                    rep.obj.quaternion.rotateTowards(q, delta * 2);
                 }
 
                 const pp = rep.obj.position;
@@ -260,9 +329,9 @@ export class WebSocketManager {
             playersUpdated[playerID] = true;
 
             if (playerID in this.connectedPlayers && this.connectedPlayers[playerID].representation.length === serverPlayer.representation.length) {
-                console.log("updating...")
                 const player = this.connectedPlayers[playerID];
                 for (let i = 0; i < player.representation.length; i++) {
+                    console.log("updating " + serverPlayer.representation[i].type)
                     player.representation[i].desiredPosition.x = serverPlayer.representation[i].position.x;
                     player.representation[i].desiredPosition.y = serverPlayer.representation[i].position.y;
                     player.representation[i].desiredPosition.z = serverPlayer.representation[i].position.z;
@@ -274,6 +343,10 @@ export class WebSocketManager {
                 }
 
             } else {
+                if (playerID in this.connectedPlayers) {
+                    this.removePlayer(playerID);
+                }
+
                 // Create a new Player!
                 this.setupPlayer(playerID, serverPlayer);
             }
@@ -284,12 +357,15 @@ export class WebSocketManager {
             if (updated) {
                 continue;
             }
-
-            this.connectedPlayers[playerID].representation.forEach(rep => rep.cleanup());
-            delete this.connectedPlayers[playerID];
+            this.removePlayer(playerID);
         }
     }
 
+    removePlayer(playerID) {
+        this.connectedPlayers[playerID].representation.forEach(rep => rep.cleanup());
+        delete this.connectedPlayers[playerID];
+    }
+
     onMessage(evt) {
         const message = JSON.parse(evt.data);
 
@@ -317,7 +393,7 @@ export class WebSocketManager {
         this.conn.send(JSON.stringify({
             "type": "Client-SetOrientation",
             "data": {
-                "representation": this.representationManager.toMessage(),
+                "representation": this.representationManager.ToMessage(),
             }
         }));
     }

generator/html/js/xr.js

@@ -11,26 +11,27 @@ let INTERSECTION;
 const tempMatrix = new THREE.Matrix4();
 
 
-export const InitXR = (scene, renderer, updateManager) => {
+export const InitXR = (scene, renderer, updateManager, representationManager, ground) => {
 
+    floor = ground;
     raycaster = new THREE.Raycaster();
 
     console.log(renderer.xr)
     renderer.xr.addEventListener('sessionstart', () => {
         baseReferenceSpace = renderer.xr.getReferenceSpace();
 
-        if(!marker) {
+        if (!marker) {
             marker = new THREE.Mesh(
                 new THREE.CircleGeometry(0.25, 32).rotateX(- Math.PI / 2),
                 new THREE.MeshBasicMaterial({ color: 0xbcbcbc })
             );
             scene.add(marker);
         }
-        updateManager.addToUpdate(render);
+        updateManager.addToUpdate(intersectionUpdate);
     });
 
     renderer.xr.addEventListener('sessionend', () => {
-        updateManager.removeFromUpdate(render);
+        updateManager.removeFromUpdate(intersectionUpdate);
         scene.remove(marker);
         marker = null;
     });
@@ -59,22 +60,28 @@ export const InitXR = (scene, renderer, updateManager) => {
     controller1 = renderer.xr.getController(0);
     controller1.addEventListener('selectstart', onSelectStart);
     controller1.addEventListener('selectend', onSelectEnd);
-    controller1.addEventListener('connected', function (event) {
-        this.add(buildController(event.data));
+    controller1.addEventListener('connected', (event) => {
+        const rep = buildController(event.data);
+        representationManager.AddRepresentation("left-hand", controller1);
+        controller1.add(rep);
     });
-    controller1.addEventListener('disconnected', function () {
-        this.remove(this.children[0]);
+    controller1.addEventListener('disconnected', () => {
+        representationManager.RemoveRepresentation("left-hand", controller1);
+        controller1.remove(controller1.children[0]);
     });
     scene.add(controller1);
 
     controller2 = renderer.xr.getController(1);
     controller2.addEventListener('selectstart', onSelectStart);
     controller2.addEventListener('selectend', onSelectEnd);
-    controller2.addEventListener('connected', function (event) {
-        this.add(buildController(event.data));
+    controller2.addEventListener('connected', (event) => {
+        const rep = buildController(event.data);
+        representationManager.AddRepresentation("right-hand", controller2);
+        controller2.add(rep);
     });
-    controller2.addEventListener('disconnected', function () {
-        this.remove(this.children[0]);
+    controller2.addEventListener('disconnected', () => {
+        representationManager.RemoveRepresentation("right-hand", controller2);
+        controller2.remove(controller2.children[0]);
     });
     scene.add(controller2);
 
@@ -97,6 +104,7 @@ export const InitXR = (scene, renderer, updateManager) => {
 function buildController(data) {
     let geometry, material;
 
+    console.log(data)
     switch (data.targetRayMode) {
         case 'tracked-pointer':
             geometry = new THREE.BufferGeometry();
@@ -111,10 +119,14 @@ function buildController(data) {
             geometry = new THREE.RingGeometry(0.02, 0.04, 32).translate(0, 0, - 1);
             material = new THREE.MeshBasicMaterial({ opacity: 0.5, transparent: true });
             return new THREE.Mesh(geometry, material);
+
+        default:
+            console.warn("unrecognized target ray mode: " + data.targetRayMode);
+            return new THREE.Group();
     }
 }
 
-function render() {
+function intersectionUpdate() {
     INTERSECTION = undefined;
 
     if (controller1.userData.isSelecting === true) {

generator/room/client.go

@@ -19,7 +19,7 @@ const (
 	pingPeriod = (pongWait * 9) / 10
 
 	// Maximum message size allowed from peer.
-	maxMessageSize = 512
+	maxMessageSize = 1024 * 10
 )
 
 var (