import * as THREE from 'three'; import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js'; // Import GLTFLoader import { HandLandmarker, FilesetResolver } from 'https://esm.sh/@mediapipe/tasks-vision@0.10.14'; import { AudioManager } from 'audioManager'; // Import the AudioManager import { SpeechManager } from 'SpeechManager'; // Import SpeechManager export class Game { // Added export here constructor(renderDiv) { this.renderDiv = renderDiv; this.scene = null; this.camera = null; this.renderer = null; this.videoElement = null; this.handLandmarker = null; this.lastVideoTime = -1; this.hands = []; // Stores data about detected hands (landmarks, anchor position, line group) this.handLineMaterial = null; // Material for hand lines this.fingertipMaterialHand1 = null; // Material for first hand's fingertip circles (blue) this.fingertipMaterialHand2 = null; // Material for second hand's fingertip circles (green) this.fingertipLandmarkIndices = [0, 4, 8, 12, 16, 20]; // WRIST + TIP landmarks this.handConnections = null; // Landmark connection definitions // this.handCollisionRadius = 30; // Conceptual radius for hand collision, was 25 (sphere radius) - Not needed for template this.gameState = 'loading'; // loading, ready, tracking, error this.gameOverText = null; // Will be repurposed or simplified this.startButton = null; // Add reference for start button this.startScreenOverlay = null; // Reference for the start screen overlay this.clock = new THREE.Clock(); this.audioManager = new AudioManager(); // Create an instance of AudioManager this.lastLandmarkPositions = [[], []]; // Store last known smoothed positions for each hand's landmarks this.smoothingFactor = 0.4; // Alpha for exponential smoothing (0 < alpha <= 1). Smaller = more smoothing. this.loadedModels = {}; // To store loaded models if any (e.g. a generic hand model in future) this.speechManager = null; this.speechBubble = null; this.speechBubbleTimeout = null; this.isSpeechActive = false; // Track if speech recognition is active for styling // Initialize asynchronously this._init().catch(error => { console.error("Initialization failed:", error); this._showError("Initialization failed. Check console."); }); } async _init() { this._setupDOM(); // Sets up basic DOM, including speech bubble container this._setupThree(); this._setupSpeechRecognition(); // Initialize SpeechManager await this._loadAssets(); // Add asset loading step await this._setupHandTracking(); // This needs to complete before we can proceed // Ensure webcam is playing before starting game logic dependent on it await this.videoElement.play(); window.addEventListener('resize', this._onResize.bind(this)); this.gameState = 'ready'; // Change state to ready, not playing // Show start button now that loading is complete this._showStartButton(); this._animate(); // Start the animation loop (it will check state) } _setupDOM() { this.renderDiv.style.position = 'relative'; this.renderDiv.style.width = '100vw'; // Use viewport units for fullscreen this.renderDiv.style.height = '100vh'; this.renderDiv.style.overflow = 'hidden'; this.renderDiv.style.background = '#111'; // Fallback background // --- Start Screen Overlay --- this.startScreenOverlay = document.createElement('div'); this.startScreenOverlay.style.position = 'absolute'; this.startScreenOverlay.style.top = '0'; this.startScreenOverlay.style.left = '0'; this.startScreenOverlay.style.width = '100%'; this.startScreenOverlay.style.height = '100%'; this.startScreenOverlay.style.background = `url('https://play.rosebud.ai/assets/nerdCover.png?r0G7') no-repeat center center / cover`; this.startScreenOverlay.style.zIndex = '30'; // Above everything else initially this.startScreenOverlay.style.display = 'flex'; // Use flexbox for centering content this.startScreenOverlay.style.flexDirection = 'column'; this.startScreenOverlay.style.justifyContent = 'center'; this.startScreenOverlay.style.alignItems = 'center'; this.startScreenOverlay.style.color = '#fff'; this.startScreenOverlay.style.textAlign = 'center'; this.startScreenOverlay.style.fontFamily = '"Arial Black", Gadget, sans-serif'; // Text shadow will be applied to individual elements for more control // Create a container for the text elements to apply a background const textContainer = document.createElement('div'); textContainer.style.backgroundColor = 'rgba(0, 0, 0, 0.4)'; // Further reduced opacity textContainer.style.padding = '10px 25px'; // Further reduced padding textContainer.style.borderRadius = '15px'; textContainer.style.maxWidth = '90%'; textContainer.style.boxSizing = 'border-box'; // Ensure padding is included in maxWidth const title = document.createElement('h1'); title.innerText = "Hand Tracking Demo"; // New title title.style.fontSize = 'clamp(40px, 10vw, 80px)'; title.style.marginBottom = '20px'; title.style.textShadow = '3px 3px 8px rgba(0,0,0,0.9)'; textContainer.appendChild(title); const instructions = document.createElement('p'); instructions.innerHTML = `Show your hands to the camera to see the landmarks.
Click START to begin.`; // New instructions instructions.style.fontSize = 'clamp(18px, 4vw, 30px)'; instructions.style.lineHeight = '1.5'; instructions.style.marginBottom = '30px'; instructions.style.textShadow = '2px 2px 6px rgba(0,0,0,0.8)'; textContainer.appendChild(instructions); // Add loading text, will be hidden later const loadingText = document.createElement('p'); loadingText.id = 'loading-text'; loadingText.innerText = 'Loading assets and webcam...'; loadingText.style.fontSize = 'clamp(16px, 3vw, 24px)'; loadingText.style.fontStyle = 'italic'; loadingText.style.marginTop = '10px'; loadingText.style.textShadow = '1px 1px 4px rgba(0,0,0,0.7)'; textContainer.appendChild(loadingText); this.startScreenOverlay.appendChild(textContainer); this.renderDiv.appendChild(this.startScreenOverlay); // --- End Start Screen Overlay --- this.videoElement = document.createElement('video'); this.videoElement.style.position = 'absolute'; this.videoElement.style.top = '0'; this.videoElement.style.left = '0'; this.videoElement.style.width = '100%'; this.videoElement.style.height = '100%'; this.videoElement.style.objectFit = 'cover'; this.videoElement.style.transform = 'scaleX(-1)'; // Mirror view for intuitive control this.videoElement.autoplay = true; this.videoElement.muted = true; // Mute video to avoid feedback loops if audio was captured this.videoElement.playsInline = true; this.videoElement.style.zIndex = '0'; // Ensure video is behind THREE canvas this.renderDiv.appendChild(this.videoElement); // Container for Status text (formerly Game Over) and restart hint this.gameOverContainer = document.createElement('div'); this.gameOverContainer.style.position = 'absolute'; this.gameOverContainer.style.top = '50%'; this.gameOverContainer.style.left = '50%'; this.gameOverContainer.style.transform = 'translate(-50%, -50%)'; this.gameOverContainer.style.zIndex = '10'; this.gameOverContainer.style.display = 'none'; // Hidden initially this.gameOverContainer.style.pointerEvents = 'none'; // Don't block clicks this.gameOverContainer.style.textAlign = 'center'; // Center text elements within this.gameOverContainer.style.color = 'white'; // Default color, can be changed by _showError this.gameOverContainer.style.textShadow = '2px 2px 4px black'; this.gameOverContainer.style.fontFamily = '"Arial Black", Gadget, sans-serif'; // Main Status Text (formerly Game Over Text) this.gameOverText = document.createElement('div'); // Will be 'gameOverText' internally this.gameOverText.innerText = 'STATUS'; // Generic placeholder this.gameOverText.style.fontSize = 'clamp(36px, 10vw, 72px)'; // Responsive font size this.gameOverText.style.fontWeight = 'bold'; this.gameOverText.style.marginBottom = '10px'; // Space below main text this.gameOverContainer.appendChild(this.gameOverText); // Restart Hint Text (may or may not be shown depending on context) this.restartHintText = document.createElement('div'); this.restartHintText.innerText = '(click to restart tracking)'; this.restartHintText.style.fontSize = 'clamp(16px, 3vw, 24px)'; this.restartHintText.style.fontWeight = 'normal'; this.restartHintText.style.opacity = '0.8'; // Slightly faded this.gameOverContainer.appendChild(this.restartHintText); this.renderDiv.appendChild(this.gameOverContainer); // ScoreDisplay removed // Watermelon (Center Emoji Marker) setup removed // Chad Image Marker setup removed // Start Button - Now positioned inside the overlay initially this.startButton = document.createElement('button'); this.startButton.innerText = 'START GAME'; // Remove absolute positioning, let flexbox handle it in overlay // this.startButton.style.position = 'absolute'; // this.startButton.style.top = '50%'; // this.startButton.style.left = '50%'; // this.startButton.style.transform = 'translate(-50%, -50%)'; this.startButton.style.marginTop = '20px'; // Add some space above button this.startButton.style.padding = '20px 40px'; this.startButton.style.fontSize = 'clamp(20px, 5vw, 40px)'; this.startButton.style.fontFamily = '"Arial Black", Gadget, sans-serif'; this.startButton.style.fontWeight = 'bold'; this.startButton.style.color = '#fff'; this.startButton.style.backgroundColor = '#4CAF50'; // Green this.startButton.style.border = '5px solid #FF476F'; this.startButton.style.borderRadius = '10px'; this.startButton.style.cursor = 'pointer'; this.startButton.style.boxShadow = '0 5px 15px rgba(0,0,0,0.3)'; // this.startButton.style.zIndex = '20'; // zIndex handled by parent overlay this.startButton.style.display = 'none'; // Hidden initially, shown after loading this.startButton.onclick = this._startGame.bind(this); // Bind click handler // Append startButton to the textContainer as well, or directly to overlay if preferred // For now, keeping it in the overlay but after the textContainer for logical flow this.startScreenOverlay.appendChild(this.startButton); // --- Speech Bubble --- this.speechBubble = document.createElement('div'); this.speechBubble.id = 'speech-bubble'; this.speechBubble.style.position = 'absolute'; this.speechBubble.style.top = '20px'; // Changed from bottom this.speechBubble.style.left = '50%'; this.speechBubble.style.transform = 'translateX(-50%)'; this.speechBubble.style.padding = '15px 25px'; this.speechBubble.style.backgroundColor = 'rgba(255, 255, 255, 0.9)'; this.speechBubble.style.border = '2px solid #007bff'; this.speechBubble.style.borderRadius = '15px'; this.speechBubble.style.boxShadow = '0 4px 12px rgba(0,0,0,0.2)'; this.speechBubble.style.color = '#333'; this.speechBubble.style.fontFamily = '"Arial", sans-serif'; this.speechBubble.style.fontSize = 'clamp(16px, 3vw, 22px)'; this.speechBubble.style.maxWidth = '80%'; this.speechBubble.style.textAlign = 'center'; this.speechBubble.style.zIndex = '25'; // Above most things but below modal popups if any this.speechBubble.style.opacity = '0'; // Hidden initially, fade in // Added boxShadow, border, padding, fontSize, top to transition for smooth active state changes this.speechBubble.style.transition = 'opacity 0.5s ease-in-out, transform 0.3s ease-in-out, box-shadow 0.3s ease-in-out, border 0.3s ease-in-out, padding 0.3s ease-in-out, font-size 0.3s ease-in-out, top 0.3s ease-in-out'; this.speechBubble.style.pointerEvents = 'none'; // Not interactive this.speechBubble.innerHTML = "..."; // Default text this.renderDiv.appendChild(this.speechBubble); } _setupThree() { const width = this.renderDiv.clientWidth; const height = this.renderDiv.clientHeight; this.scene = new THREE.Scene(); // Using OrthographicCamera for a 2D-like overlay effect this.camera = new THREE.OrthographicCamera(width / -2, width / 2, height / 2, height / -2, 1, 1000); this.camera.position.z = 100; // Position along Z doesn't change scale in Ortho this.renderer = new THREE.WebGLRenderer({ alpha: true, antialias: true }); this.renderer.setSize(width, height); this.renderer.setPixelRatio(window.devicePixelRatio); this.renderer.domElement.style.position = 'absolute'; this.renderer.domElement.style.top = '0'; this.renderer.domElement.style.left = '0'; this.renderer.domElement.style.zIndex = '1'; // Canvas on top of video this.renderDiv.appendChild(this.renderer.domElement); const ambientLight = new THREE.AmbientLight(0xffffff, 0.7); this.scene.add(ambientLight); const directionalLight = new THREE.DirectionalLight(0xffffff, 0.9); directionalLight.position.set(0, 0, 100); // Pointing from behind camera this.scene.add(directionalLight); // Setup hand visualization (palm circles removed, lines will be added later) for (let i = 0; i < 2; i++) { // Support up to 2 hands const lineGroup = new THREE.Group(); lineGroup.visible = false; this.scene.add(lineGroup); this.hands.push({ landmarks: null, anchorPos: new THREE.Vector3(), lineGroup: lineGroup // removed pickedUpGhost, isPinching, pinchPointWorld }); } this.handLineMaterial = new THREE.LineBasicMaterial({ color: 0x00ccff, linewidth: 8 }); this.fingertipMaterialHand1 = new THREE.MeshBasicMaterial({ color: 0x0000ff, side: THREE.DoubleSide }); // Blue this.fingertipMaterialHand2 = new THREE.MeshBasicMaterial({ color: 0x00ff00, side: THREE.DoubleSide }); // Green // Define connections for MediaPipe hand landmarks // See: https://developers.google.com/mediapipe/solutions/vision/hand_landmarker#hand_landmarks this.handConnections = [ // Thumb [0, 1], [1, 2], [2, 3], [3, 4], // Index finger [0, 5], [5, 6], [6, 7], [7, 8], // Middle finger [0, 9], [9, 10], [10, 11], [11, 12], // Ring finger [0, 13], [13, 14], [14, 15], [15, 16], // Pinky [0, 17], [17, 18], [18, 19], [19, 20], // Palm [5, 9], [9, 13], [13, 17] // Connect base of fingers ]; // Particle resources removed // Ground line removed } async _loadAssets() { console.log("Loading assets..."); // const textureLoader = new THREE.TextureLoader(); // Only if loading other textures // const gltfLoader = new GLTFLoader(); // Only if loading other models try { // Ghost Textures loading removed // Ghost GLTF Model loading removed (was already commented out) console.log("No game-specific assets to load for template."); } catch (error) { console.error("Error loading assets:", error); this._showError("Failed to load assets."); // Generic message throw error; // Stop initialization } } async _setupHandTracking() { try { console.log("Setting up Hand Tracking..."); const vision = await FilesetResolver.forVisionTasks( 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.14/wasm' ); this.handLandmarker = await HandLandmarker.createFromOptions(vision, { baseOptions: { modelAssetPath: `https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task`, delegate: 'GPU', // Use GPU if available }, numHands: 2, runningMode: 'VIDEO', // Process video stream }); console.log("HandLandmarker created."); console.log("Requesting webcam access..."); const stream = await navigator.mediaDevices.getUserMedia({ video: { facingMode: 'user', width: { ideal: 1280 }, height: { ideal: 720 } }, audio: false, }); this.videoElement.srcObject = stream; console.log("Webcam stream obtained."); // Wait for video metadata to load to ensure dimensions are available return new Promise((resolve) => { this.videoElement.onloadedmetadata = () => { console.log("Webcam metadata loaded."); // Adjust video size slightly after metadata is loaded if needed, but CSS handles most this.videoElement.style.width = this.renderDiv.clientWidth + 'px'; this.videoElement.style.height = this.renderDiv.clientHeight + 'px'; resolve(); }; }); } catch (error) { console.error('Error setting up Hand Tracking or Webcam:', error); this._showError(`Webcam/Hand Tracking Error: ${error.message}. Please allow camera access.`); throw error; // Re-throw to stop initialization } } // _startSpawning, _scheduleNextSpawn, _stopSpawning, _spawnGhost methods removed. _updateHands() { if (!this.handLandmarker || !this.videoElement.srcObject || this.videoElement.readyState < 2 || this.videoElement.videoWidth === 0) return; const videoTime = this.videoElement.currentTime; if (videoTime > this.lastVideoTime) { this.lastVideoTime = videoTime; try { const results = this.handLandmarker.detectForVideo(this.videoElement, performance.now()); const videoParams = this._getVisibleVideoParameters(); if (!videoParams) return; const canvasWidth = this.renderDiv.clientWidth; const canvasHeight = this.renderDiv.clientHeight; for (let i = 0; i < this.hands.length; i++) { const hand = this.hands[i]; if (results.landmarks && results.landmarks[i]) { let currentRawLandmarks = results.landmarks[i]; if (!this.lastLandmarkPositions[i] || this.lastLandmarkPositions[i].length !== currentRawLandmarks.length) { this.lastLandmarkPositions[i] = currentRawLandmarks.map(lm => ({ ...lm })); } const smoothedLandmarks = currentRawLandmarks.map((lm, lmIndex) => { const prevLm = this.lastLandmarkPositions[i][lmIndex]; return { x: this.smoothingFactor * lm.x + (1 - this.smoothingFactor) * prevLm.x, y: this.smoothingFactor * lm.y + (1 - this.smoothingFactor) * prevLm.y, z: this.smoothingFactor * lm.z + (1 - this.smoothingFactor) * prevLm.z, // visibility and presence are not typically part of MediaPipe landmark objects directly, // but if they were, you might carry them over or smooth them if appropriate. }; }); this.lastLandmarkPositions[i] = smoothedLandmarks.map(lm => ({...lm})); // Update last positions with new smoothed ones hand.landmarks = smoothedLandmarks; const palm = smoothedLandmarks[9]; // MIDDLE_FINGER_MCP const lmOriginalX = palm.x * videoParams.videoNaturalWidth; const lmOriginalY = palm.y * videoParams.videoNaturalHeight; let normX_visible = (lmOriginalX - videoParams.offsetX) / videoParams.visibleWidth; let normY_visible = (lmOriginalY - videoParams.offsetY) / videoParams.visibleHeight; const handX = (1 - normX_visible) * canvasWidth - canvasWidth / 2; const handY = (1 - normY_visible) * canvasHeight - canvasHeight / 2; hand.anchorPos.set(handX, handY, 1); this._updateHandLines(i, smoothedLandmarks, videoParams, canvasWidth, canvasHeight); hand.lineGroup.visible = true; } else { // No landmarks for this hand hand.landmarks = null; if (hand.lineGroup) hand.lineGroup.visible = false; } } } catch (error) { console.error("Error during hand detection:", error); } } } _getVisibleVideoParameters() { if (!this.videoElement || this.videoElement.videoWidth === 0 || this.videoElement.videoHeight === 0) { return null; } const vNatW = this.videoElement.videoWidth; const vNatH = this.videoElement.videoHeight; const rW = this.renderDiv.clientWidth; const rH = this.renderDiv.clientHeight; if (vNatW === 0 || vNatH === 0 || rW === 0 || rH === 0) return null; const videoAR = vNatW / vNatH; const renderDivAR = rW / rH; let finalVideoPixelX, finalVideoPixelY; let visibleVideoPixelWidth, visibleVideoPixelHeight; if (videoAR > renderDivAR) { // Video is wider than renderDiv, scaled to fit renderDiv height, cropped horizontally. const scale = rH / vNatH; // Scale factor based on height. const scaledVideoWidth = vNatW * scale; // Width of video if scaled to fit renderDiv height. // Total original video pixels cropped horizontally (from both sides combined). const totalCroppedPixelsX = (scaledVideoWidth - rW) / scale; finalVideoPixelX = totalCroppedPixelsX / 2; // Pixels cropped from the left of original video. finalVideoPixelY = 0; // No vertical cropping. visibleVideoPixelWidth = vNatW - totalCroppedPixelsX; // Width of the visible part in original video pixels. visibleVideoPixelHeight = vNatH; // Full height is visible. } else { // Video is taller than renderDiv (or same AR), scaled to fit renderDiv width, cropped vertically. const scale = rW / vNatW; // Scale factor based on width. const scaledVideoHeight = vNatH * scale; // Height of video if scaled to fit renderDiv width. // Total original video pixels cropped vertically (from top and bottom combined). const totalCroppedPixelsY = (scaledVideoHeight - rH) / scale; finalVideoPixelX = 0; // No horizontal cropping. finalVideoPixelY = totalCroppedPixelsY / 2; // Pixels cropped from the top of original video. visibleVideoPixelWidth = vNatW; // Full width is visible. visibleVideoPixelHeight = vNatH - totalCroppedPixelsY; // Height of the visible part in original video pixels. } // Safety check for degenerate cases (e.g., extreme aspect ratios leading to zero visible dimension) if (visibleVideoPixelWidth <= 0 || visibleVideoPixelHeight <= 0) { // Fallback or log error, this shouldn't happen in normal scenarios console.warn("Calculated visible video dimension is zero or negative.", {visibleVideoPixelWidth, visibleVideoPixelHeight}); return { // Return full video dimensions as a fallback to prevent division by zero offsetX: 0, offsetY: 0, visibleWidth: vNatW, visibleHeight: vNatH, videoNaturalWidth: vNatW, videoNaturalHeight: vNatH }; } return { offsetX: finalVideoPixelX, offsetY: finalVideoPixelY, visibleWidth: visibleVideoPixelWidth, visibleHeight: visibleVideoPixelHeight, videoNaturalWidth: vNatW, videoNaturalHeight: vNatH }; } // _updateGhosts method removed. _showStatusScreen(message, color = 'white', showRestartHint = false) { // Generalized from _showGameOver this.gameOverContainer.style.display = 'block'; this.gameOverText.innerText = message; this.gameOverText.style.color = color; this.restartHintText.style.display = showRestartHint ? 'block' : 'none'; // No spawning to stop for template } _showError(message) { this.gameOverContainer.style.display = 'block'; this.gameOverText.innerText = `ERROR: ${message}`; this.gameOverText.style.color = 'orange'; this.restartHintText.style.display = 'true'; // Show restart hint on error this.gameState = 'error'; // No spawning to stop this.hands.forEach(hand => { if (hand.lineGroup) hand.lineGroup.visible = false; }); if (this.startButton) this.startButton.style.display = 'none'; } _startGame() { console.log("Starting tracking from button..."); if (this.gameState !== 'ready') return; this.audioManager.resumeContext(); // For general audio this.speechManager.requestPermissionAndStart(); // For speech recognition if (this.startScreenOverlay) { this.startScreenOverlay.style.display = 'none'; } this.startButton.style.display = 'none'; this.gameState = 'tracking'; // Changed from 'playing' this.lastVideoTime = -1; this.clock.start(); // Removed display of score, castle, chad // Removed _startSpawning() } _restartGame() { // Renamed to _restartTracking or similar if preferred console.log("Restarting tracking..."); this.gameOverContainer.style.display = 'none'; if (this.startScreenOverlay) { this.startScreenOverlay.style.display = 'none'; } this.hands.forEach(hand => { if (hand.lineGroup) { hand.lineGroup.visible = false; } }); // Ghost removal removed // Score reset removed // Visibility of game elements removed this.gameState = 'tracking'; // Changed from 'playing' this.lastVideoTime = -1; this.clock.start(); // Removed _startSpawning() } // _updateScoreDisplay method removed. _showStartButton() { // This is called when assets and webcam are ready if (this.startScreenOverlay) { // Hide loading text inside the overlay const loadingText = this.startScreenOverlay.querySelector('#loading-text'); if (loadingText) loadingText.style.display = 'none'; // Show the start button inside the overlay if (this.startButton) { this.startButton.style.display = 'block'; } } else { // Fallback if overlay wasn't created correctly if (this.startButton) { this.startButton.style.display = 'block'; // Position it centrally if overlay doesn't exist this.startButton.style.position = 'absolute'; this.startButton.style.top = '50%'; this.startButton.style.left = '50%'; this.startButton.style.transform = 'translate(-50%, -50%)'; this.startButton.style.zIndex = '20'; } } } _onResize() { const width = this.renderDiv.clientWidth; const height = this.renderDiv.clientHeight; // Update camera perspective this.camera.left = width / -2; this.camera.right = width / 2; this.camera.top = height / 2; this.camera.bottom = height / -2; this.camera.updateProjectionMatrix(); // Update renderer size this.renderer.setSize(width, height); // Update video element size this.videoElement.style.width = width + 'px'; this.videoElement.style.height = height + 'px'; // Watermelon, Chad, GroundLine updates removed. } _updateHandLines(handIndex, landmarks, videoParams, canvasWidth, canvasHeight) { const hand = this.hands[handIndex]; const lineGroup = hand.lineGroup; while (lineGroup.children.length) { const child = lineGroup.children[0]; lineGroup.remove(child); if (child.geometry) child.geometry.dispose(); // Materials are shared, no need to dispose them here unless they are unique per line/circle } if (!landmarks || landmarks.length === 0 || !videoParams) { lineGroup.visible = false; return; } const points3D = landmarks.map(lm => { const lmOriginalX = lm.x * videoParams.videoNaturalWidth; const lmOriginalY = lm.y * videoParams.videoNaturalHeight; let normX_visible = (lmOriginalX - videoParams.offsetX) / videoParams.visibleWidth; let normY_visible = (lmOriginalY - videoParams.offsetY) / videoParams.visibleHeight; // Clamp values to [0, 1] range to avoid issues if landmarks fall slightly outside the calculated visible area // due to precision or edge cases. This ensures they map correctly to the canvas. normX_visible = Math.max(0, Math.min(1, normX_visible)); normY_visible = Math.max(0, Math.min(1, normY_visible)); const x = (1 - normX_visible) * canvasWidth - canvasWidth / 2; const y = (1 - normY_visible) * canvasHeight - canvasHeight / 2; return new THREE.Vector3(x, y, 1.1); // Z for fingertip circles, slightly in front of lines }); const lineZ = 1; // Z for connection lines this.handConnections.forEach(conn => { const p1 = points3D[conn[0]]; const p2 = points3D[conn[1]]; if (p1 && p2) { // Ensure landmarks exist // Create points for the line with the correct Z const lineP1 = p1.clone().setZ(lineZ); const lineP2 = p2.clone().setZ(lineZ); const geometry = new THREE.BufferGeometry().setFromPoints([lineP1, lineP2]); const line = new THREE.Line(geometry, this.handLineMaterial); lineGroup.add(line); } }); // Draw fingertip circles const fingertipRadius = 8; // Radius of the circle for fingertips const wristRadius = 12; // Larger radius for the wrist const circleSegments = 16; // Smoothness of the circle this.fingertipLandmarkIndices.forEach(index => { const landmarkPosition = points3D[index]; if (landmarkPosition) { const radius = (index === 0) ? wristRadius : fingertipRadius; // Use wristRadius for landmark 0 const circleGeometry = new THREE.CircleGeometry(radius, circleSegments); const material = handIndex === 0 ? this.fingertipMaterialHand1 : this.fingertipMaterialHand2; const landmarkCircle = new THREE.Mesh(circleGeometry, material); landmarkCircle.position.copy(landmarkPosition); // Already has Z=1.1 lineGroup.add(landmarkCircle); } }); lineGroup.visible = true; } _animate() { requestAnimationFrame(this._animate.bind(this)); // const deltaTime = this.clock.getDelta(); // deltaTime not used currently // Update hands if tracking if (this.gameState === 'tracking') { this._updateHands(); } // _updateGhosts and _updateParticles calls removed. // Always render the scene this.renderer.render(this.scene, this.camera); } start() { // Add click listener for resuming audio context and potentially restarting on error this.renderDiv.addEventListener('click', () => { this.audioManager.resumeContext(); if (this.gameState === 'error' || this.gameState === 'paused') { // Assuming 'paused' might be a state where restart is desired this._restartGame(); // Restart tracking } }); console.log('Game setup initiated. Waiting for async operations...'); // Note: Game interaction now starts via the Start Button after _init completes. } _updateSpeechBubbleAppearance() { if (!this.speechBubble) return; const isPlaceholder = this.speechBubble.innerHTML === "..." || this.speechBubble.innerText === "..."; // Apply active styling only if recognition is generally active AND we are not displaying the placeholder. // This means interim/final text will get the active style, but the "..." placeholder will not, // even if the recognition service itself is still running in the background. const showActiveStyling = this.isSpeechActive && !isPlaceholder; const translateY = isPlaceholder ? '-5px' : '0px'; const scale = showActiveStyling ? '1.15' : '1.0'; this.speechBubble.style.transform = `translateX(-50%) translateY(${translateY}) scale(${scale})`; if (showActiveStyling) { // Cyan glow, blue drop shadow, enhanced original shadow this.speechBubble.style.boxShadow = '0 0 18px 6px rgba(0, 255, 255, 0.75), 0 7px 14px rgba(0, 123, 255, 0.6), 0 4px 10px rgba(0,0,0,0.25)'; this.speechBubble.style.border = '3px solid #00ccff'; // Brighter cyan border this.speechBubble.style.padding = '18px 28px'; // Slightly larger padding this.speechBubble.style.fontSize = 'clamp(20px, 3.5vw, 26px)'; // Larger font when active this.speechBubble.style.top = '30px'; // Increased top margin when active } else { // Default/inactive styling this.speechBubble.style.boxShadow = '0 4px 12px rgba(0,0,0,0.2)'; this.speechBubble.style.border = '2px solid #007bff'; // Original blue border this.speechBubble.style.padding = '15px 25px'; this.speechBubble.style.fontSize = 'clamp(16px, 3vw, 22px)'; // Original font size this.speechBubble.style.top = '20px'; // Original top margin } } _setupSpeechRecognition() { this.speechManager = new SpeechManager( (finalTranscript, interimTranscript) => { // onTranscript callback if (this.speechBubble) { clearTimeout(this.speechBubbleTimeout); if (finalTranscript) { this.speechBubble.innerHTML = finalTranscript; this.speechBubble.style.opacity = '1'; this.speechBubbleTimeout = setTimeout(() => { this.speechBubble.innerHTML = "..."; this.speechBubble.style.opacity = '0.7'; this._updateSpeechBubbleAppearance(); // Update appearance for "..." }, 2000); } else if (interimTranscript) { this.speechBubble.innerHTML = `${interimTranscript}`; this.speechBubble.style.opacity = '1'; } else { // No final, no interim (e.g., recognition just started, ended, or error) this.speechBubbleTimeout = setTimeout(() => { // Check if it's not already "..." to avoid redundant sets if called multiple times if (this.speechBubble.innerHTML !== "...") { this.speechBubble.innerHTML = "..."; } this.speechBubble.style.opacity = '0.7'; this._updateSpeechBubbleAppearance(); // Update appearance for "..." }, 500); } this._updateSpeechBubbleAppearance(); // Always update appearance after content change } }, (isActive) => { // onRecognitionActive callback this.isSpeechActive = isActive; this._updateSpeechBubbleAppearance(); } ); // Initialize speech bubble with "..." and apply initial appearance if (this.speechBubble) { this.speechBubble.innerHTML = "..."; this.speechBubble.style.opacity = '0.7'; this._updateSpeechBubbleAppearance(); // Apply initial styles (isSpeechActive will be false) } // We will call requestPermissionAndStart() on user interaction (e.g., start button) } }