import * as THREE from 'three'; import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js'; // Import GLTFLoader import { HandLandmarker, FilesetResolver } from 'https://esm.sh/@mediapipe/tasks-vision@0.10.14'; import { EffectComposer, RenderPass, EffectPass, BloomEffect, ToneMappingEffect, SMAAEffect } from 'postprocessing'; import { AudioManager } from 'audioManager'; // Import the AudioManager import { SpeechManager } from 'SpeechManager'; // Import SpeechManager export class Game { // Added export here constructor(renderDiv) { this.renderDiv = renderDiv; this.scene = null; this.camera = null; this.renderer = null; this.composer = null; // For post-processing this.videoElement = null; this.handLandmarker = null; this.lastVideoTime = -1; this.hands = []; // Stores data about detected hands (landmarks, anchor position, line group) this.handLineMaterial = null; // Material for hand lines this.fingertipMaterialHand1 = null; // Material for first hand's fingertip circles (blue) this.fingertipMaterialHand2 = null; // Material for second hand's fingertip circles (green) this.fingertipLandmarkIndices = [0, 4, 8, 12, 16, 20]; // WRIST + TIP landmarks this.handConnections = null; // Landmark connection definitions // this.handCollisionRadius = 30; // Conceptual radius for hand collision, was 25 (sphere radius) - Not needed for template this.gameState = 'loading'; // loading, ready, tracking, error this.gameOverText = null; // Will be repurposed or simplified this.startButton = null; // Add reference for start button this.startScreenOverlay = null; // Reference for the start screen overlay this.clock = new THREE.Clock(); this.audioManager = new AudioManager(); // Create an instance of AudioManager this.waveLine = null; // THREE.Line for visualizing the audio waveform this.lastLandmarkPositions = [[], []]; // Store last known smoothed positions for each hand's landmarks this.smoothingFactor = 0.4; // Alpha for exponential smoothing (0 < alpha <= 1). Smaller = more smoothing. this.loadedModels = {}; // To store loaded models if any (e.g. a generic hand model in future) this.speechManager = null; this.hand2PinchState = false; // To track if the second hand is currently pinching this.speechBubble = null; this.speechBubbleTimeout = null; this.isSpeechActive = false; // Track if speech recognition is active for styling this.beatToggleButton = null; this.instrumentDisplay = null; // For showing the current instrument name this.instrumentCycleButton = null; // Button to cycle instruments // Initialize asynchronously this._init().catch(error => { console.error("Initialization failed:", error); this._showError("Initialization failed. Check console."); }); } async _init() { this._setupDOM(); // Sets up basic DOM, including speech bubble container this._setupThree(); this._setupSpeechRecognition(); // Initialize SpeechManager await this._loadAssets(); // Add asset loading step await this._setupHandTracking(); // This needs to complete before we can proceed // Ensure webcam is playing before starting game logic dependent on it await this.videoElement.play(); window.addEventListener('resize', this._onResize.bind(this)); this.gameState = 'ready'; // Change state to ready, not playing // Show start button now that loading is complete this._showStartButton(); this._animate(); // Start the animation loop (it will check state) } _setupDOM() { this.renderDiv.style.position = 'relative'; this.renderDiv.style.width = '100vw'; // Use viewport units for fullscreen this.renderDiv.style.height = '100vh'; this.renderDiv.style.overflow = 'hidden'; this.renderDiv.style.background = '#111'; // Fallback background // --- Start Screen Overlay --- this.startScreenOverlay = document.createElement('div'); this.startScreenOverlay.style.position = 'absolute'; this.startScreenOverlay.style.top = '0'; this.startScreenOverlay.style.left = '0'; this.startScreenOverlay.style.width = '100%'; this.startScreenOverlay.style.height = '100%'; this.startScreenOverlay.style.background = '#000'; this.startScreenOverlay.style.zIndex = '30'; // Above everything else initially this.startScreenOverlay.style.display = 'flex'; // Use flexbox for centering content this.startScreenOverlay.style.flexDirection = 'column'; this.startScreenOverlay.style.justifyContent = 'center'; this.startScreenOverlay.style.alignItems = 'center'; this.startScreenOverlay.style.color = '#fff'; this.startScreenOverlay.style.textAlign = 'center'; this.startScreenOverlay.style.fontFamily = '"Arial Black", Gadget, sans-serif'; // Text shadow will be applied to individual elements for more control // Create a container for the text elements to apply a background const textContainer = document.createElement('div'); textContainer.style.backgroundColor = 'rgba(0, 0, 0, 0.4)'; // Further reduced opacity textContainer.style.padding = '10px 25px'; // Further reduced padding textContainer.style.borderRadius = '15px'; textContainer.style.maxWidth = '90%'; textContainer.style.boxSizing = 'border-box'; // Ensure padding is included in maxWidth const image = document.createElement('img'); image.src = 'https://play.rosebud.ai/assets/readyplayer9999_Hand-Waving_Music_desktop_app_using_motion_of_2c54c421-9e5d-4884-ad9e-dba5788b8534_3.png?O7pj'; image.style.width = 'clamp(150px, 30vw, 250px)'; image.style.height = 'auto'; image.style.borderRadius = '50%'; image.style.marginBottom = '20px'; image.style.boxShadow = '0 0 25px rgba(0, 255, 255, 0.4)'; image.style.border = '3px solid rgba(0, 255, 255, 0.6)'; textContainer.appendChild(image); const title = document.createElement('h1'); title.innerText = "Hand-Waving Music App"; // New title title.style.fontSize = 'clamp(40px, 10vw, 80px)'; title.style.marginBottom = '20px'; title.style.textShadow = '3px 3px 8px rgba(0,0,0,0.9)'; textContainer.appendChild(title); const instructions = document.createElement('p'); instructions.innerHTML = `Right Hand: X-axis for Pitch, Y-axis for Filter, Pinch for Volume.
Left Hand: Y-axis to control Beat Tempo.`; // New instructions instructions.style.fontSize = 'clamp(18px, 4vw, 30px)'; instructions.style.lineHeight = '1.5'; instructions.style.marginBottom = '30px'; instructions.style.textShadow = '2px 2px 6px rgba(0,0,0,0.8)'; textContainer.appendChild(instructions); // Add loading text, will be hidden later const loadingText = document.createElement('p'); loadingText.id = 'loading-text'; loadingText.innerText = 'Loading assets and webcam...'; loadingText.style.fontSize = 'clamp(16px, 3vw, 24px)'; loadingText.style.fontStyle = 'italic'; loadingText.style.marginTop = '10px'; loadingText.style.textShadow = '1px 1px 4px rgba(0,0,0,0.7)'; textContainer.appendChild(loadingText); this.startScreenOverlay.appendChild(textContainer); this.renderDiv.appendChild(this.startScreenOverlay); // --- End Start Screen Overlay --- this.videoElement = document.createElement('video'); this.videoElement.style.position = 'absolute'; this.videoElement.style.top = '0'; this.videoElement.style.left = '0'; this.videoElement.style.width = '100%'; this.videoElement.style.height = '100%'; this.videoElement.style.objectFit = 'cover'; this.videoElement.style.transform = 'scaleX(-1)'; // Mirror view for intuitive control this.videoElement.autoplay = true; this.videoElement.muted = true; // Mute video to avoid feedback loops if audio was captured this.videoElement.playsInline = true; this.videoElement.style.zIndex = '0'; // Ensure video is behind THREE canvas this.renderDiv.appendChild(this.videoElement); // Container for Status text (formerly Game Over) and restart hint this.gameOverContainer = document.createElement('div'); this.gameOverContainer.style.position = 'absolute'; this.gameOverContainer.style.top = '50%'; this.gameOverContainer.style.left = '50%'; this.gameOverContainer.style.transform = 'translate(-50%, -50%)'; this.gameOverContainer.style.zIndex = '10'; this.gameOverContainer.style.display = 'none'; // Hidden initially this.gameOverContainer.style.pointerEvents = 'none'; // Don't block clicks this.gameOverContainer.style.textAlign = 'center'; // Center text elements within this.gameOverContainer.style.color = 'white'; // Default color, can be changed by _showError this.gameOverContainer.style.textShadow = '2px 2px 4px black'; this.gameOverContainer.style.fontFamily = '"Arial Black", Gadget, sans-serif'; // Main Status Text (formerly Game Over Text) this.gameOverText = document.createElement('div'); // Will be 'gameOverText' internally this.gameOverText.innerText = 'STATUS'; // Generic placeholder this.gameOverText.style.fontSize = 'clamp(36px, 10vw, 72px)'; // Responsive font size this.gameOverText.style.fontWeight = 'bold'; this.gameOverText.style.marginBottom = '10px'; // Space below main text this.gameOverContainer.appendChild(this.gameOverText); // Restart Hint Text (may or may not be shown depending on context) this.restartHintText = document.createElement('div'); this.restartHintText.innerText = '(click to restart tracking)'; this.restartHintText.style.fontSize = 'clamp(16px, 3vw, 24px)'; this.restartHintText.style.fontWeight = 'normal'; this.restartHintText.style.opacity = '0.8'; // Slightly faded this.gameOverContainer.appendChild(this.restartHintText); this.renderDiv.appendChild(this.gameOverContainer); // ScoreDisplay removed // Watermelon (Center Emoji Marker) setup removed // Chad Image Marker setup removed // Start Button - Now positioned inside the overlay initially this.startButton = document.createElement('button'); this.startButton.innerText = 'START'; // Remove absolute positioning, let flexbox handle it in overlay // this.startButton.style.position = 'absolute'; // this.startButton.style.top = '50%'; // this.startButton.style.left = '50%'; // this.startButton.style.transform = 'translate(-50%, -50%)'; this.startButton.style.marginTop = '20px'; // Add some space above button this.startButton.style.padding = '20px 40px'; this.startButton.style.fontSize = 'clamp(20px, 5vw, 40px)'; this.startButton.style.fontFamily = '"Arial Black", Gadget, sans-serif'; this.startButton.style.fontWeight = 'bold'; this.startButton.style.color = '#fff'; this.startButton.style.backgroundColor = '#4CAF50'; // Green this.startButton.style.border = '5px solid #FF476F'; this.startButton.style.borderRadius = '10px'; this.startButton.style.cursor = 'pointer'; this.startButton.style.boxShadow = '0 5px 15px rgba(0,0,0,0.3)'; // this.startButton.style.zIndex = '20'; // zIndex handled by parent overlay this.startButton.style.display = 'none'; // Hidden initially, shown after loading this.startButton.onclick = this._startGame.bind(this); // Bind click handler // Append startButton to the textContainer as well, or directly to overlay if preferred // For now, keeping it in the overlay but after the textContainer for logical flow this.startScreenOverlay.appendChild(this.startButton); // --- Speech Bubble --- this.speechBubble = document.createElement('div'); this.speechBubble.id = 'speech-bubble'; this.speechBubble.style.position = 'absolute'; this.speechBubble.style.top = '20px'; // Changed from bottom this.speechBubble.style.left = '50%'; this.speechBubble.style.transform = 'translateX(-50%)'; this.speechBubble.style.padding = '15px 25px'; this.speechBubble.style.backgroundColor = 'rgba(255, 255, 255, 0.9)'; this.speechBubble.style.border = '2px solid #007bff'; this.speechBubble.style.borderRadius = '15px'; this.speechBubble.style.boxShadow = '0 4px 12px rgba(0,0,0,0.2)'; this.speechBubble.style.color = '#333'; this.speechBubble.style.fontFamily = '"Arial", sans-serif'; this.speechBubble.style.fontSize = 'clamp(16px, 3vw, 22px)'; this.speechBubble.style.maxWidth = '80%'; this.speechBubble.style.textAlign = 'center'; this.speechBubble.style.zIndex = '25'; // Above most things but below modal popups if any this.speechBubble.style.opacity = '0'; // Hidden initially, fade in // Added boxShadow, border, padding, fontSize, top to transition for smooth active state changes this.speechBubble.style.transition = 'opacity 0.5s ease-in-out, transform 0.3s ease-in-out, box-shadow 0.3s ease-in-out, border 0.3s ease-in-out, padding 0.3s ease-in-out, font-size 0.3s ease-in-out, top 0.3s ease-in-out'; this.speechBubble.style.pointerEvents = 'none'; // Not interactive this.speechBubble.innerHTML = "..."; // Default text this.renderDiv.appendChild(this.speechBubble); // --- Instrument Name Display --- this.instrumentDisplay = document.createElement('div'); this.instrumentDisplay.id = 'instrument-display'; this.instrumentDisplay.style.position = 'absolute'; this.instrumentDisplay.style.bottom = '80px'; // Move up to make space for the new button this.instrumentDisplay.style.left = '50%'; // Center horizontally this.instrumentDisplay.style.transform = 'translateX(-50%)'; this.instrumentDisplay.style.padding = '10px 20px'; this.instrumentDisplay.style.backgroundColor = 'rgba(0, 0, 0, 0.6)'; this.instrumentDisplay.style.color = '#FFA500'; // Orange color this.instrumentDisplay.style.fontFamily = '"Courier New", Courier, monospace'; this.instrumentDisplay.style.fontSize = 'clamp(16px, 3vw, 24px)'; this.instrumentDisplay.style.border = '2px solid #FFA500'; this.instrumentDisplay.style.borderRadius = '8px'; this.instrumentDisplay.style.zIndex = '20'; this.instrumentDisplay.style.opacity = '0'; // Hidden until game starts this.instrumentDisplay.style.transition = 'opacity 0.5s ease-in-out'; this.instrumentDisplay.innerText = 'VOCAL SYNTH'; // Initial value this.renderDiv.appendChild(this.instrumentDisplay); // --- Instrument Cycle Button --- this.instrumentCycleButton = document.createElement('button'); this.instrumentCycleButton.id = 'instrument-cycle-button'; this.instrumentCycleButton.style.position = 'absolute'; this.instrumentCycleButton.style.bottom = '20px'; this.instrumentCycleButton.style.left = '50%'; this.instrumentCycleButton.style.transform = 'translateX(-50%)'; this.instrumentCycleButton.style.padding = '10px 20px'; this.instrumentCycleButton.style.backgroundColor = 'rgba(0, 0, 0, 0.6)'; this.instrumentCycleButton.style.color = '#FFA500'; // Orange color this.instrumentCycleButton.style.fontFamily = '"Courier New", Courier, monospace'; this.instrumentCycleButton.style.fontSize = 'clamp(16px, 3vw, 24px)'; this.instrumentCycleButton.style.border = '2px solid #FFA500'; this.instrumentCycleButton.style.borderRadius = '8px'; this.instrumentCycleButton.style.zIndex = '20'; this.instrumentCycleButton.style.cursor = 'pointer'; this.instrumentCycleButton.style.opacity = '0'; // Hidden until game starts this.instrumentCycleButton.style.transition = 'opacity 0.5s ease-in-out'; this.instrumentCycleButton.innerText = 'CHANGE INSTRUMENT'; this.instrumentCycleButton.onclick = (e) => { e.stopPropagation(); const displayName = this.audioManager.cycleInstrument(); if (this.instrumentDisplay && displayName) { this.instrumentDisplay.innerText = displayName.toUpperCase(); } }; this.renderDiv.appendChild(this.instrumentCycleButton); // --- Beat Toggle Button --- this.beatToggleButton = document.createElement('button'); this.beatToggleButton.id = 'beat-toggle-button'; this.beatToggleButton.style.position = 'absolute'; this.beatToggleButton.style.bottom = '20px'; this.beatToggleButton.style.right = '20px'; this.beatToggleButton.style.padding = '10px 20px'; this.beatToggleButton.style.backgroundColor = 'rgba(0, 0, 0, 0.6)'; this.beatToggleButton.style.color = '#00ffff'; // Start with ON color this.beatToggleButton.style.fontFamily = '"Courier New", Courier, monospace'; this.beatToggleButton.style.fontSize = 'clamp(16px, 3vw, 24px)'; this.beatToggleButton.style.border = '2px solid #00ffff'; // Start with ON color this.beatToggleButton.style.borderRadius = '8px'; this.beatToggleButton.style.zIndex = '20'; this.beatToggleButton.style.cursor = 'pointer'; this.beatToggleButton.style.opacity = '0'; // Hidden until game starts this.beatToggleButton.style.transition = 'opacity 0.5s ease-in-out, background-color 0.3s, color 0.3s, border-color 0.3s'; this.beatToggleButton.innerText = 'BEAT: ON'; // Initial state since beat auto-starts this.beatToggleButton.onclick = (e) => { e.stopPropagation(); // Prevent the main 'restart' click handler on the renderDiv from firing const isPlaying = this.audioManager.toggleBeat(); this.beatToggleButton.innerText = `BEAT: ${isPlaying ? 'ON' : 'OFF'}`; if (isPlaying) { this.beatToggleButton.style.color = '#00ffff'; this.beatToggleButton.style.borderColor = '#00ffff'; } else { this.beatToggleButton.style.color = '#FF476F'; this.beatToggleButton.style.borderColor = '#FF476F'; } }; this.renderDiv.appendChild(this.beatToggleButton); } _setupThree() { const width = this.renderDiv.clientWidth; const height = this.renderDiv.clientHeight; this.scene = new THREE.Scene(); // Using OrthographicCamera for a 2D-like overlay effect this.camera = new THREE.OrthographicCamera(width / -2, width / 2, height / 2, height / -2, 1, 1000); this.camera.position.z = 100; // Position along Z doesn't change scale in Ortho this.renderer = new THREE.WebGLRenderer({ alpha: true, antialias: true }); this.renderer.setSize(width, height); this.renderer.setPixelRatio(window.devicePixelRatio); this.renderer.domElement.style.position = 'absolute'; this.renderer.domElement.style.top = '0'; this.renderer.domElement.style.left = '0'; this.renderer.domElement.style.zIndex = '1'; // Canvas on top of video this.renderDiv.appendChild(this.renderer.domElement); // Post-processing setup this.composer = new EffectComposer(this.renderer); this.composer.addPass(new RenderPass(this.scene, this.camera)); const bloomEffect = new BloomEffect({ luminanceThreshold: 0.1, // Lower threshold to make more things glow luminanceSmoothing: 0.2, intensity: 1.2, // A bit more intense bloom mipmapBlur: true, }); // Add an anti-aliasing pass for smoother lines overall const smaaPass = new EffectPass(this.camera, new SMAAEffect()); const bloomPass = new EffectPass(this.camera, bloomEffect); this.composer.addPass(bloomPass); this.composer.addPass(smaaPass); const ambientLight = new THREE.AmbientLight(0xffffff, 0.7); this.scene.add(ambientLight); const directionalLight = new THREE.DirectionalLight(0xffffff, 0.9); directionalLight.position.set(0, 0, 100); // Pointing from behind camera this.scene.add(directionalLight); this._setupWaveformVisualizer(); // Setup hand visualization (palm circles removed, lines will be added later) for (let i = 0; i < 2; i++) { // Support up to 2 hands const lineGroup = new THREE.Group(); lineGroup.visible = false; this.scene.add(lineGroup); this.hands.push({ landmarks: null, anchorPos: new THREE.Vector3(), lineGroup: lineGroup // removed pickedUpGhost, isPinching, pinchPointWorld }); } this.handLineMaterial = new THREE.LineBasicMaterial({ color: 0x00ccff, linewidth: 8 }); // Make colors brighter to pop more with the bloom effect this.fingertipMaterialHand1 = new THREE.MeshBasicMaterial({ color: 0x0088ff, side: THREE.DoubleSide }); // Brighter Blue this.fingertipMaterialHand2 = new THREE.MeshBasicMaterial({ color: 0x33ff99, side: THREE.DoubleSide }); // Brighter Green // Define connections for MediaPipe hand landmarks // See: https://developers.google.com/mediapipe/solutions/vision/hand_landmarker#hand_landmarks this.handConnections = [ // Thumb [0, 1], [1, 2], [2, 3], [3, 4], // Index finger [0, 5], [5, 6], [6, 7], [7, 8], // Middle finger [0, 9], [9, 10], [10, 11], [11, 12], // Ring finger [0, 13], [13, 14], [14, 15], [15, 16], // Pinky [0, 17], [17, 18], [18, 19], [19, 20], // Palm [5, 9], [9, 13], [13, 17] // Connect base of fingers ]; // Particle resources removed // Ground line removed } async _loadAssets() { console.log("Loading assets..."); // const textureLoader = new THREE.TextureLoader(); // Only if loading other textures // const gltfLoader = new GLTFLoader(); // Only if loading other models try { // Ghost Textures loading removed // Ghost GLTF Model loading removed (was already commented out) console.log("No game-specific assets to load for template."); } catch (error) { console.error("Error loading assets:", error); this._showError("Failed to load assets."); // Generic message throw error; // Stop initialization } } async _setupHandTracking() { try { console.log("Setting up Hand Tracking..."); const vision = await FilesetResolver.forVisionTasks( 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.14/wasm' ); this.handLandmarker = await HandLandmarker.createFromOptions(vision, { baseOptions: { modelAssetPath: `https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task`, delegate: 'GPU', // Use GPU if available }, numHands: 2, runningMode: 'VIDEO', // Process video stream }); console.log("HandLandmarker created."); console.log("Requesting webcam access..."); const stream = await navigator.mediaDevices.getUserMedia({ video: { facingMode: 'user', width: { ideal: 1280 }, height: { ideal: 720 } }, audio: false, }); this.videoElement.srcObject = stream; console.log("Webcam stream obtained."); // Wait for video metadata to load to ensure dimensions are available return new Promise((resolve) => { this.videoElement.onloadedmetadata = () => { console.log("Webcam metadata loaded."); // Adjust video size slightly after metadata is loaded if needed, but CSS handles most this.videoElement.style.width = this.renderDiv.clientWidth + 'px'; this.videoElement.style.height = this.renderDiv.clientHeight + 'px'; resolve(); }; }); } catch (error) { console.error('Error setting up Hand Tracking or Webcam:', error); this._showError(`Webcam/Hand Tracking Error: ${error.message}. Please allow camera access.`); throw error; // Re-throw to stop initialization } } // _startSpawning, _scheduleNextSpawn, _stopSpawning, _spawnGhost methods removed. _updateHands() { if (!this.handLandmarker || !this.videoElement.srcObject || this.videoElement.readyState < 2 || this.videoElement.videoWidth === 0) return; const videoTime = this.videoElement.currentTime; if (videoTime > this.lastVideoTime) { this.lastVideoTime = videoTime; try { const results = this.handLandmarker.detectForVideo(this.videoElement, performance.now()); const videoParams = this._getVisibleVideoParameters(); if (!videoParams) return; const canvasWidth = this.renderDiv.clientWidth; const canvasHeight = this.renderDiv.clientHeight; for (let i = 0; i < this.hands.length; i++) { const hand = this.hands[i]; if (results.landmarks && results.landmarks[i]) { let currentRawLandmarks = results.landmarks[i]; if (!this.lastLandmarkPositions[i] || this.lastLandmarkPositions[i].length !== currentRawLandmarks.length) { this.lastLandmarkPositions[i] = currentRawLandmarks.map(lm => ({ ...lm })); } const smoothedLandmarks = currentRawLandmarks.map((lm, lmIndex) => { const prevLm = this.lastLandmarkPositions[i][lmIndex]; return { x: this.smoothingFactor * lm.x + (1 - this.smoothingFactor) * prevLm.x, y: this.smoothingFactor * lm.y + (1 - this.smoothingFactor) * prevLm.y, z: this.smoothingFactor * lm.z + (1 - this.smoothingFactor) * prevLm.z, // visibility and presence are not typically part of MediaPipe landmark objects directly, // but if they were, you might carry them over or smooth them if appropriate. }; }); this.lastLandmarkPositions[i] = smoothedLandmarks.map(lm => ({...lm})); // Update last positions with new smoothed ones hand.landmarks = smoothedLandmarks; const palm = smoothedLandmarks[9]; // MIDDLE_FINGER_MCP const lmOriginalX = palm.x * videoParams.videoNaturalWidth; const lmOriginalY = palm.y * videoParams.videoNaturalHeight; let normX_visible = (lmOriginalX - videoParams.offsetX) / videoParams.visibleWidth; let normY_visible = (lmOriginalY - videoParams.offsetY) / videoParams.visibleHeight; const handX = (1 - normX_visible) * canvasWidth - canvasWidth / 2; const handY = (1 - normY_visible) * canvasHeight - canvasHeight / 2; hand.anchorPos.set(handX, handY, 1); this._updateHandLines(i, smoothedLandmarks, videoParams, canvasWidth, canvasHeight); hand.lineGroup.visible = true; // Control Theremin with the first hand (right hand on screen) if (i === 0) { // Right Hand Controls // --- PITCH (X-axis) --- // Invert X for intuitive control (right is higher pitch) const pitchNorm = (1 - normX_visible); // --- FILTER (Y-axis) --- // Invert Y for intuitive control (up is brighter) const filterNorm = (1 - normY_visible); // --- VOLUME (Pinch Distance) --- const thumbTip = smoothedLandmarks[4]; const indexTip = smoothedLandmarks[8]; // Use raw video coordinates for a more stable distance metric const pinchDist = Math.hypot(thumbTip.x - indexTip.x, thumbTip.y - indexTip.y); // Normalize the distance. These min/max values may need tweaking. // 0.02 is very close, 0.2 is quite far apart. const minPinch = 0.02; const maxPinch = 0.2; const volumeNorm = (pinchDist - minPinch) / (maxPinch - minPinch); const clampedVolume = Math.max(0, Math.min(1, volumeNorm)); // Clamp between 0 and 1 this.audioManager.updateSynth(pitchNorm, filterNorm, clampedVolume); } // Control waveform type with the second hand's pinch gesture if (i === 1) { // Left Hand Controls // --- BEAT TEMPO (Y-axis) --- // We need the normalized Y of the left hand. // We can reuse the same logic as the right hand's Y-axis. const palm = smoothedLandmarks[9]; // Middle finger MCP as anchor const lmOriginalY = palm.y * videoParams.videoNaturalHeight; const normY_visible = (lmOriginalY - videoParams.offsetY) / videoParams.visibleHeight; // Invert Y for intuitive control (up is faster) const tempoNorm = 1 - Math.max(0, Math.min(1, normY_visible)); // Clamp to 0-1 this.audioManager.setBPM(tempoNorm); } } else { // No landmarks for this hand hand.landmarks = null; if (hand.lineGroup) hand.lineGroup.visible = false; // If it's the controlling hand that disappears, silence the theremin if (i === 0) { this.audioManager.updateSynth(0.5, 0.5, 0); // Center pitch, neutral filter, zero volume } } } } catch (error) { console.error("Error during hand detection:", error); } } } _getVisibleVideoParameters() { if (!this.videoElement || this.videoElement.videoWidth === 0 || this.videoElement.videoHeight === 0) { return null; } const vNatW = this.videoElement.videoWidth; const vNatH = this.videoElement.videoHeight; const rW = this.renderDiv.clientWidth; const rH = this.renderDiv.clientHeight; if (vNatW === 0 || vNatH === 0 || rW === 0 || rH === 0) return null; const videoAR = vNatW / vNatH; const renderDivAR = rW / rH; let finalVideoPixelX, finalVideoPixelY; let visibleVideoPixelWidth, visibleVideoPixelHeight; if (videoAR > renderDivAR) { // Video is wider than renderDiv, scaled to fit renderDiv height, cropped horizontally. const scale = rH / vNatH; // Scale factor based on height. const scaledVideoWidth = vNatW * scale; // Width of video if scaled to fit renderDiv height. // Total original video pixels cropped horizontally (from both sides combined). const totalCroppedPixelsX = (scaledVideoWidth - rW) / scale; finalVideoPixelX = totalCroppedPixelsX / 2; // Pixels cropped from the left of original video. finalVideoPixelY = 0; // No vertical cropping. visibleVideoPixelWidth = vNatW - totalCroppedPixelsX; // Width of the visible part in original video pixels. visibleVideoPixelHeight = vNatH; // Full height is visible. } else { // Video is taller than renderDiv (or same AR), scaled to fit renderDiv width, cropped vertically. const scale = rW / vNatW; // Scale factor based on width. const scaledVideoHeight = vNatH * scale; // Height of video if scaled to fit renderDiv width. // Total original video pixels cropped vertically (from top and bottom combined). const totalCroppedPixelsY = (scaledVideoHeight - rH) / scale; finalVideoPixelX = 0; // No horizontal cropping. finalVideoPixelY = totalCroppedPixelsY / 2; // Pixels cropped from the top of original video. visibleVideoPixelWidth = vNatW; // Full width is visible. visibleVideoPixelHeight = vNatH - totalCroppedPixelsY; // Height of the visible part in original video pixels. } // Safety check for degenerate cases (e.g., extreme aspect ratios leading to zero visible dimension) if (visibleVideoPixelWidth <= 0 || visibleVideoPixelHeight <= 0) { // Fallback or log error, this shouldn't happen in normal scenarios console.warn("Calculated visible video dimension is zero or negative.", {visibleVideoPixelWidth, visibleVideoPixelHeight}); return { // Return full video dimensions as a fallback to prevent division by zero offsetX: 0, offsetY: 0, visibleWidth: vNatW, visibleHeight: vNatH, videoNaturalWidth: vNatW, videoNaturalHeight: vNatH }; } return { offsetX: finalVideoPixelX, offsetY: finalVideoPixelY, visibleWidth: visibleVideoPixelWidth, visibleHeight: visibleVideoPixelHeight, videoNaturalWidth: vNatW, videoNaturalHeight: vNatH }; } // _updateGhosts method removed. _showStatusScreen(message, color = 'white', showRestartHint = false) { // Generalized from _showGameOver this.gameOverContainer.style.display = 'block'; this.gameOverText.innerText = message; this.gameOverText.style.color = color; this.restartHintText.style.display = showRestartHint ? 'block' : 'none'; // No spawning to stop for template } _showError(message) { this.gameOverContainer.style.display = 'block'; this.gameOverText.innerText = `ERROR: ${message}`; this.gameOverText.style.color = 'orange'; this.restartHintText.style.display = 'true'; // Show restart hint on error this.gameState = 'error'; // No spawning to stop this.hands.forEach(hand => { if (hand.lineGroup) hand.lineGroup.visible = false; }); if (this.startButton) this.startButton.style.display = 'none'; } _startGame() { console.log("Starting tracking from button..."); if (this.gameState !== 'ready') return; this.audioManager.resumeContext(); // For general audio // this.audioManager.startTheremin(); // This is now handled by resumeContext this.audioManager.toggleBeat(); // Start the beat this.speechManager.requestPermissionAndStart(); // For speech recognition if (this.startScreenOverlay) { this.startScreenOverlay.style.display = 'none'; } this.startButton.style.display = 'none'; if (this.instrumentDisplay) this.instrumentDisplay.style.opacity = '1'; // Show instrument display if (this.instrumentCycleButton) this.instrumentCycleButton.style.opacity = '1'; // Show instrument cycle button if (this.beatToggleButton) this.beatToggleButton.style.opacity = '1'; // Show beat toggle button this.gameState = 'tracking'; // Changed from 'playing' this.lastVideoTime = -1; this.clock.start(); // Removed display of score, castle, chad // Removed _startSpawning() } _restartGame() { // Renamed to _restartTracking or similar if preferred console.log("Restarting tracking..."); this.gameOverContainer.style.display = 'none'; if (this.startScreenOverlay) { this.startScreenOverlay.style.display = 'none'; } this.hands.forEach(hand => { if (hand.lineGroup) { hand.lineGroup.visible = false; } }); if (this.instrumentDisplay) this.instrumentDisplay.style.opacity = '0'; // Hide instrument display if (this.instrumentCycleButton) this.instrumentCycleButton.style.opacity = '0'; // Hide instrument cycle button if (this.beatToggleButton) this.beatToggleButton.style.opacity = '0'; // Hide beat toggle button // Ghost removal removed // Score reset removed // Visibility of game elements removed this.gameState = 'tracking'; // Changed from 'playing' this.lastVideoTime = -1; this.clock.start(); // Removed _startSpawning() } _setupWaveformVisualizer() { // Brighter color for the waveform to make it bloom const material = new THREE.LineBasicMaterial({ color: 0x00ffff, linewidth: 3 }); // Correctly get buffer size from audioManager const bufferLength = this.audioManager.getAnalyserBufferSize(); if (bufferLength === 0) { console.warn("Analyser buffer size is 0, cannot create waveform visualizer yet."); return; } const points = new Float32Array(bufferLength * 3); // x, y, z for each point const geometry = new THREE.BufferGeometry(); geometry.setAttribute('position', new THREE.BufferAttribute(points, 3)); this.waveLine = new THREE.Line(geometry, material); this.waveLine.position.z = 50; // Bring it forward this.scene.add(this.waveLine); } _updateWaveform() { if (!this.waveLine || !this.audioManager.isInitialized || !this.audioManager.analyserNode) { if (this.audioManager.isInitialized && !this.waveLine) { // If audio is ready but line isn't, try setting it up now. this._setupWaveformVisualizer(); } return; } const dataArray = this.audioManager.getAnalyserData(); if (!dataArray) return; const positionAttribute = this.waveLine.geometry.getAttribute('position'); const bufferLength = dataArray.length; const width = this.renderDiv.clientWidth; const height = this.renderDiv.clientHeight; for (let i = 0; i < bufferLength; i++) { const v = dataArray[i] / 128.0; // Normalize to 0-2 range const x = (i / (bufferLength - 1)) * width - (width / 2); const y = (v - 1) * (height / 4); // Scale and center vertically positionAttribute.setXYZ(i, x, y, 0); } positionAttribute.needsUpdate = true; } // _updateScoreDisplay method removed. _showStartButton() { // This is called when assets and webcam are ready if (this.startScreenOverlay) { // Hide loading text inside the overlay const loadingText = this.startScreenOverlay.querySelector('#loading-text'); if (loadingText) loadingText.style.display = 'none'; // Show the start button inside the overlay if (this.startButton) { this.startButton.style.display = 'block'; } } else { // Fallback if overlay wasn't created correctly if (this.startButton) { this.startButton.style.display = 'block'; // Position it centrally if overlay doesn't exist this.startButton.style.position = 'absolute'; this.startButton.style.top = '50%'; this.startButton.style.left = '50%'; this.startButton.style.transform = 'translate(-50%, -50%)'; this.startButton.style.zIndex = '20'; } } } _onResize() { const width = this.renderDiv.clientWidth; const height = this.renderDiv.clientHeight; // Update camera perspective this.camera.left = width / -2; this.camera.right = width / 2; this.camera.top = height / 2; this.camera.bottom = height / -2; this.camera.updateProjectionMatrix(); // Update renderer size this.renderer.setSize(width, height); this.composer.setSize(width, height); // Also update composer size // Update video element size this.videoElement.style.width = width + 'px'; this.videoElement.style.height = height + 'px'; // Watermelon, Chad, GroundLine updates removed. } _updateHandLines(handIndex, landmarks, videoParams, canvasWidth, canvasHeight) { const hand = this.hands[handIndex]; const lineGroup = hand.lineGroup; while (lineGroup.children.length) { const child = lineGroup.children[0]; lineGroup.remove(child); if (child.geometry) child.geometry.dispose(); // Materials are shared, no need to dispose them here unless they are unique per line/circle } if (!landmarks || landmarks.length === 0 || !videoParams) { lineGroup.visible = false; return; } const points3D = landmarks.map(lm => { const lmOriginalX = lm.x * videoParams.videoNaturalWidth; const lmOriginalY = lm.y * videoParams.videoNaturalHeight; let normX_visible = (lmOriginalX - videoParams.offsetX) / videoParams.visibleWidth; let normY_visible = (lmOriginalY - videoParams.offsetY) / videoParams.visibleHeight; // Clamp values to [0, 1] range to avoid issues if landmarks fall slightly outside the calculated visible area // due to precision or edge cases. This ensures they map correctly to the canvas. normX_visible = Math.max(0, Math.min(1, normX_visible)); normY_visible = Math.max(0, Math.min(1, normY_visible)); const x = (1 - normX_visible) * canvasWidth - canvasWidth / 2; const y = (1 - normY_visible) * canvasHeight - canvasHeight / 2; return new THREE.Vector3(x, y, 1.1); // Z for fingertip circles, slightly in front of lines }); const lineZ = 1; // Z for connection lines this.handConnections.forEach(conn => { const p1 = points3D[conn[0]]; const p2 = points3D[conn[1]]; if (p1 && p2) { // Ensure landmarks exist // Create points for the line with the correct Z const lineP1 = p1.clone().setZ(lineZ); const lineP2 = p2.clone().setZ(lineZ); const geometry = new THREE.BufferGeometry().setFromPoints([lineP1, lineP2]); const line = new THREE.Line(geometry, this.handLineMaterial); lineGroup.add(line); } }); // Draw fingertip circles const fingertipRadius = 8; // Radius of the circle for fingertips const wristRadius = 12; // Larger radius for the wrist const circleSegments = 16; // Smoothness of the circle this.fingertipLandmarkIndices.forEach(index => { const landmarkPosition = points3D[index]; if (landmarkPosition) { const radius = (index === 0) ? wristRadius : fingertipRadius; // Use wristRadius for landmark 0 const circleGeometry = new THREE.CircleGeometry(radius, circleSegments); const material = handIndex === 0 ? this.fingertipMaterialHand1 : this.fingertipMaterialHand2; const landmarkCircle = new THREE.Mesh(circleGeometry, material); landmarkCircle.position.copy(landmarkPosition); // Already has Z=1.1 lineGroup.add(landmarkCircle); } }); lineGroup.visible = true; } _animate() { requestAnimationFrame(this._animate.bind(this)); const deltaTime = this.clock.getDelta(); // deltaTime now used by composer // Update hands if tracking if (this.gameState === 'tracking') { this._updateHands(); this._updateWaveform(); } // _updateGhosts and _updateParticles calls removed. // Always render the scene using the composer this.composer.render(deltaTime); } start() { // Add click listener for resuming audio context and potentially restarting on error this.renderDiv.addEventListener('click', () => { this.audioManager.resumeContext(); if (this.gameState === 'error' || this.gameState === 'paused') { // Assuming 'paused' might be a state where restart is desired this._restartGame(); // Restart tracking } }); console.log('Game setup initiated. Waiting for async operations...'); // Note: Game interaction now starts via the Start Button after _init completes. } _updateSpeechBubbleAppearance() { if (!this.speechBubble) return; const isPlaceholder = this.speechBubble.innerHTML === "..." || this.speechBubble.innerText === "..."; // Apply active styling only if recognition is generally active AND we are not displaying the placeholder. // This means interim/final text will get the active style, but the "..." placeholder will not, // even if the recognition service itself is still running in the background. const showActiveStyling = this.isSpeechActive && !isPlaceholder; const translateY = isPlaceholder ? '-5px' : '0px'; const scale = showActiveStyling ? '1.15' : '1.0'; this.speechBubble.style.transform = `translateX(-50%) translateY(${translateY}) scale(${scale})`; if (showActiveStyling) { // Cyan glow, blue drop shadow, enhanced original shadow this.speechBubble.style.boxShadow = '0 0 18px 6px rgba(0, 255, 255, 0.75), 0 7px 14px rgba(0, 123, 255, 0.6), 0 4px 10px rgba(0,0,0,0.25)'; this.speechBubble.style.border = '3px solid #00ccff'; // Brighter cyan border this.speechBubble.style.padding = '18px 28px'; // Slightly larger padding this.speechBubble.style.fontSize = 'clamp(20px, 3.5vw, 26px)'; // Larger font when active this.speechBubble.style.top = '30px'; // Increased top margin when active } else { // Default/inactive styling this.speechBubble.style.boxShadow = '0 4px 12px rgba(0,0,0,0.2)'; this.speechBubble.style.border = '2px solid #007bff'; // Original blue border this.speechBubble.style.padding = '15px 25px'; this.speechBubble.style.fontSize = 'clamp(16px, 3vw, 22px)'; // Original font size this.speechBubble.style.top = '20px'; // Original top margin } } _setupSpeechRecognition() { this.speechManager = new SpeechManager( (finalTranscript, interimTranscript) => { // onTranscript callback if (this.speechBubble) { clearTimeout(this.speechBubbleTimeout); if (finalTranscript) { this.speechBubble.innerHTML = finalTranscript; this.speechBubble.style.opacity = '1'; this.speechBubbleTimeout = setTimeout(() => { this.speechBubble.innerHTML = "..."; this.speechBubble.style.opacity = '0.7'; this._updateSpeechBubbleAppearance(); // Update appearance for "..." }, 2000); } else if (interimTranscript) { this.speechBubble.innerHTML = `${interimTranscript}`; this.speechBubble.style.opacity = '1'; } else { // No final, no interim (e.g., recognition just started, ended, or error) this.speechBubbleTimeout = setTimeout(() => { // Check if it's not already "..." to avoid redundant sets if called multiple times if (this.speechBubble.innerHTML !== "...") { this.speechBubble.innerHTML = "..."; } this.speechBubble.style.opacity = '0.7'; this._updateSpeechBubbleAppearance(); // Update appearance for "..." }, 500); } this._updateSpeechBubbleAppearance(); // Always update appearance after content change } }, (isActive) => { // onRecognitionActive callback this.isSpeechActive = isActive; this._updateSpeechBubbleAppearance(); } ); // Initialize speech bubble with "..." and apply initial appearance if (this.speechBubble) { this.speechBubble.innerHTML = "..."; this.speechBubble.style.opacity = '0.7'; this._updateSpeechBubbleAppearance(); // Apply initial styles (isSpeechActive will be false) } // We will call requestPermissionAndStart() on user interaction (e.g., start button) } }

Other games like this

About Hand-waving Music App