import * as THREE from 'three'; import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js'; // Import GLTFLoader import { HandLandmarker, FilesetResolver } from 'https://esm.sh/@mediapipe/tasks-vision@0.10.14'; import { MusicManager } from 'MusicManager'; // Import the MusicManager import * as Tone from 'https://esm.sh/tone'; // Import Tone to access Transport import * as drumManager from 'DrumManager'; // Import the new drum manager module import { WaveformVisualizer } from 'WaveformVisualizer'; // Import the new waveform visualizer export class Game { // Added export here constructor(renderDiv) { this.renderDiv = renderDiv; this.scene = null; this.camera = null; this.renderer = null; this.videoElement = null; this.handLandmarker = null; this.lastVideoTime = -1; this.hands = []; // Stores data about detected hands (landmarks, anchor position, line group) this.handLineMaterial = null; // Material for hand lines this.fingertipMaterialHand1 = null; // Material for first hand's fingertip circles (blue) this.fingertipMaterialHand2 = null; // Material for second hand's fingertip circles (green) this.fingertipLandmarkIndices = [0, 4, 8, 12, 16, 20]; // WRIST + TIP landmarks this.handConnections = null; // Landmark connection definitions // this.handCollisionRadius = 30; // Conceptual radius for hand collision, was 25 (sphere radius) - Not needed for template this.gameState = 'loading'; // loading, ready, tracking, error this.gameOverText = null; // Will be repurposed or simplified this.clock = new THREE.Clock(); this.musicManager = new MusicManager(); // Create an instance of MusicManager this.waveformVisualizer = null; // To be initialized // this.drumManager = new DrumManager(); // DrumManager is now a static module, no instance needed this.lastLandmarkPositions = [[], []]; // Store last known smoothed positions for each hand's landmarks this.smoothingFactor = 0.4; // Alpha for exponential smoothing (0 < alpha <= 1). Smaller = more smoothing. this.loadedModels = {}; // To store loaded models if any (e.g. a generic hand model in future) this.beatIndicators = []; // Array to hold the 16 beat indicator meshes this.beatIndicatorMaterials = []; // Array to hold the base material for each indicator this.beatIndicatorColors = { // Define colors for each drum kick: new THREE.Color("#D72828"), // Eva Red snare: new THREE.Color("#F36E2F"), // Eva Orange clap: new THREE.Color("#7B4394"), // Eva Purple hihat: new THREE.Color("#84C34E"), // Eva Green off: new THREE.Color("#ffffff") // Off state remains white }; this.beatIndicatorGroup = null; // Group to hold all indicators for easy repositioning this.labelColors = { evaPurple: { r: 123, g: 67, b: 148, a: 0.9 }, evaGreen: { r: 132, g: 195, b: 78, a: 0.9 }, evaOrange: { r: 243, g: 110, b: 47, a: 0.9 }, evaRed: { r: 215, g: 40, b: 40, a: 0.9 }, white: { r: 255, g: 255, b: 255, a: 1.0 }, black: { r: 0, g: 0, b: 0, a: 1.0 } }; this.waveformColors = [ new THREE.Color("#7B4394"), // Eva Purple new THREE.Color("#84C34E"), // Eva Green new THREE.Color("#F36E2F"), // Eva Orange new THREE.Color("#D72828"), // Eva Red new THREE.Color("#66ffff"), // Cyan Blue ]; // Initialize asynchronously this._init().catch(error => { console.error("Initialization failed:", error); this._showError("Initialization failed. Check console."); }); } async _init() { this._setupDOM(); // Sets up basic DOM, including speech bubble container this._setupThree(); await this._loadAssets(); // Add asset loading step await this._setupHandTracking(); // This needs to complete before we can proceed // Ensure webcam is playing before starting game logic dependent on it await this.videoElement.play(); window.addEventListener('resize', this._onResize.bind(this)); this._startGame(); // Start the game directly this._setupEventListeners(); // Set up interaction listeners this._animate(); // Start the animation loop (it will check state) } _setupDOM() { this.renderDiv.style.position = 'relative'; this.renderDiv.style.width = '100vw'; // Use viewport units for fullscreen this.renderDiv.style.height = '100vh'; this.renderDiv.style.overflow = 'hidden'; this.renderDiv.style.background = '#111'; // Fallback background this.videoElement = document.createElement('video'); this.videoElement.style.position = 'absolute'; this.videoElement.style.top = '0'; this.videoElement.style.left = '0'; this.videoElement.style.width = '100%'; this.videoElement.style.height = '100%'; this.videoElement.style.objectFit = 'cover'; this.videoElement.style.transform = 'scaleX(-1)'; // Mirror view for intuitive control this.videoElement.style.filter = 'grayscale(100%)'; // Make it black and white this.videoElement.autoplay = true; this.videoElement.muted = true; // Mute video to avoid feedback loops if audio was captured this.videoElement.playsInline = true; this.videoElement.style.zIndex = '0'; // Ensure video is behind THREE canvas this.renderDiv.appendChild(this.videoElement); // Container for Status text (formerly Game Over) and restart hint this.gameOverContainer = document.createElement('div'); this.gameOverContainer.style.position = 'absolute'; this.gameOverContainer.style.top = '50%'; this.gameOverContainer.style.left = '50%'; this.gameOverContainer.style.transform = 'translate(-50%, -50%)'; this.gameOverContainer.style.zIndex = '10'; this.gameOverContainer.style.display = 'none'; // Hidden initially this.gameOverContainer.style.pointerEvents = 'none'; // Don't block clicks this.gameOverContainer.style.textAlign = 'center'; // Center text elements within this.gameOverContainer.style.color = 'white'; // Default color, can be changed by _showError this.gameOverContainer.style.textShadow = '2px 2px 4px black'; this.gameOverContainer.style.fontFamily = '"Arial Black", Gadget, sans-serif'; // Main Status Text (formerly Game Over Text) this.gameOverText = document.createElement('div'); // Will be 'gameOverText' internally this.gameOverText.innerText = 'STATUS'; // Generic placeholder this.gameOverText.style.fontSize = 'clamp(36px, 10vw, 72px)'; // Responsive font size this.gameOverText.style.fontWeight = 'bold'; this.gameOverText.style.marginBottom = '10px'; // Space below main text this.gameOverContainer.appendChild(this.gameOverText); // Restart Hint Text (may or may not be shown depending on context) this.restartHintText = document.createElement('div'); this.restartHintText.innerText = '(click to restart tracking)'; this.restartHintText.style.fontSize = 'clamp(16px, 3vw, 24px)'; this.restartHintText.style.fontWeight = 'normal'; this.restartHintText.style.opacity = '0.8'; // Slightly faded this.gameOverContainer.appendChild(this.restartHintText); this.renderDiv.appendChild(this.gameOverContainer); // ScoreDisplay removed // Watermelon (Center Emoji Marker) setup removed // Chad Image Marker setup removed } _setupThree() { const width = this.renderDiv.clientWidth; const height = this.renderDiv.clientHeight; this.scene = new THREE.Scene(); // Using OrthographicCamera for a 2D-like overlay effect this.camera = new THREE.OrthographicCamera(width / -2, width / 2, height / 2, height / -2, 1, 1000); this.camera.position.z = 100; // Position along Z doesn't change scale in Ortho this.renderer = new THREE.WebGLRenderer({ alpha: true, antialias: true }); this.renderer.setSize(width, height); this.renderer.setPixelRatio(window.devicePixelRatio); this.renderer.domElement.style.position = 'absolute'; this.renderer.domElement.style.top = '0'; this.renderer.domElement.style.left = '0'; this.renderer.domElement.style.zIndex = '1'; // Canvas on top of video this.renderDiv.appendChild(this.renderer.domElement); const ambientLight = new THREE.AmbientLight(0xffffff, 0.7); this.scene.add(ambientLight); const directionalLight = new THREE.DirectionalLight(0xffffff, 0.9); directionalLight.position.set(0, 0, 100); // Pointing from behind camera this.scene.add(directionalLight); // Setup hand visualization (palm circles removed, lines will be added later) for (let i = 0; i < 2; i++) { // Support up to 2 hands const lineGroup = new THREE.Group(); lineGroup.visible = false; this.scene.add(lineGroup); this.hands.push({ landmarks: null, anchorPos: new THREE.Vector3(), lineGroup: lineGroup, isFist: false // Track if the hand is currently in a fist // removed pickedUpGhost, isPinching, pinchPointWorld }); } this.handLineMaterial = new THREE.LineBasicMaterial({ color: 0x00ccff, linewidth: 8 }); this.fingertipMaterialHand1 = new THREE.MeshBasicMaterial({ color: 0xffffff, side: THREE.DoubleSide }); // White this.fingertipMaterialHand2 = new THREE.MeshBasicMaterial({ color: 0xffffff, side: THREE.DoubleSide }); // White // Define connections for MediaPipe hand landmarks // See: https://developers.google.com/mediapipe/solutions/vision/hand_landmarker#hand_landmarks this.handConnections = [ // Thumb [0, 1], [1, 2], [2, 3], [3, 4], // Index finger [0, 5], [5, 6], [6, 7], [7, 8], // Middle finger [0, 9], [9, 10], [10, 11], [11, 12], // Ring finger [0, 13], [13, 14], [14, 15], [15, 16], // Pinky [0, 17], [17, 18], [18, 19], [19, 20], // Palm [5, 9], [9, 13], [13, 17] // Connect base of fingers ]; // Particle resources removed // Ground line removed // --- Beat Indicator --- this.beatIndicatorGroup = new THREE.Group(); this.scene.add(this.beatIndicatorGroup); this._setupBeatIndicatorMaterials(); // Create materials based on drum pattern const indicatorSize = 20; const indicatorGeometry = new THREE.PlaneGeometry(indicatorSize, indicatorSize); for (let i = 0; i < 16; i++) { // Use the pre-calculated material for this beat index const indicator = new THREE.Mesh(indicatorGeometry, this.beatIndicatorMaterials[i]); this.beatIndicatorGroup.add(indicator); this.beatIndicators.push(indicator); } this._positionBeatIndicators(); // Position them right after creation } async _loadAssets() { console.log("Loading assets..."); // const textureLoader = new THREE.TextureLoader(); // Only if loading other textures // const gltfLoader = new GLTFLoader(); // Only if loading other models try { // Ghost Textures loading removed // Ghost GLTF Model loading removed (was already commented out) await drumManager.loadSamples(); // Load drum sounds console.log("No game-specific assets to load for template."); } catch (error) { console.error("Error loading assets:", error); this._showError("Failed to load assets."); // Generic message throw error; // Stop initialization } } async _setupHandTracking() { try { console.log("Setting up Hand Tracking..."); const vision = await FilesetResolver.forVisionTasks( 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.14/wasm' ); this.handLandmarker = await HandLandmarker.createFromOptions(vision, { baseOptions: { modelAssetPath: `https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task`, delegate: 'GPU', // Use GPU if available }, numHands: 2, runningMode: 'VIDEO', // Process video stream }); console.log("HandLandmarker created."); console.log("Requesting webcam access..."); const stream = await navigator.mediaDevices.getUserMedia({ video: { facingMode: 'user', width: { ideal: 1280 }, height: { ideal: 720 } }, audio: false, }); this.videoElement.srcObject = stream; console.log("Webcam stream obtained."); // Wait for video metadata to load to ensure dimensions are available return new Promise((resolve) => { this.videoElement.onloadedmetadata = () => { console.log("Webcam metadata loaded."); // Adjust video size slightly after metadata is loaded if needed, but CSS handles most this.videoElement.style.width = this.renderDiv.clientWidth + 'px'; this.videoElement.style.height = this.renderDiv.clientHeight + 'px'; resolve(); }; }); } catch (error) { console.error('Error setting up Hand Tracking or Webcam:', error); this._showError(`Webcam/Hand Tracking Error: ${error.message}. Please allow camera access.`); throw error; // Re-throw to stop initialization } } // _startSpawning, _scheduleNextSpawn, _stopSpawning, _spawnGhost methods removed. _updateHands() { if (!this.handLandmarker || !this.videoElement.srcObject || this.videoElement.readyState < 2 || this.videoElement.videoWidth === 0) return; const videoTime = this.videoElement.currentTime; if (videoTime > this.lastVideoTime) { this.lastVideoTime = videoTime; try { const results = this.handLandmarker.detectForVideo(this.videoElement, performance.now()); const videoParams = this._getVisibleVideoParameters(); if (!videoParams) return; const canvasWidth = this.renderDiv.clientWidth; const canvasHeight = this.renderDiv.clientHeight; // C Minor Pentatonic Scale const scale = ['C3', 'Eb3', 'F3', 'G3', 'Bb3', 'C4', 'Eb4', 'F4', 'G4', 'Bb4', 'C5', 'Eb5']; for (let i = 0; i < this.hands.length; i++) { const hand = this.hands[i]; const wasVisible = hand.landmarks !== null; if (results.landmarks && results.landmarks[i]) { let currentRawLandmarks = results.landmarks[i]; if (!this.lastLandmarkPositions[i] || this.lastLandmarkPositions[i].length !== currentRawLandmarks.length) { this.lastLandmarkPositions[i] = currentRawLandmarks.map(lm => ({ ...lm })); } const smoothedLandmarks = currentRawLandmarks.map((lm, lmIndex) => { const prevLm = this.lastLandmarkPositions[i][lmIndex]; return { x: this.smoothingFactor * lm.x + (1 - this.smoothingFactor) * prevLm.x, y: this.smoothingFactor * lm.y + (1 - this.smoothingFactor) * prevLm.y, z: this.smoothingFactor * lm.z + (1 - this.smoothingFactor) * prevLm.z, }; }); this.lastLandmarkPositions[i] = smoothedLandmarks.map(lm => ({...lm})); hand.landmarks = smoothedLandmarks; const palm = smoothedLandmarks[9]; // MIDDLE_FINGER_MCP const lmOriginalX = palm.x * videoParams.videoNaturalWidth; const lmOriginalY = palm.y * videoParams.videoNaturalHeight; let normX_visible = (lmOriginalX - videoParams.offsetX) / videoParams.visibleWidth; let normY_visible = (lmOriginalY - videoParams.offsetY) / videoParams.visibleHeight; const handX = (1 - normX_visible) * canvasWidth - canvasWidth / 2; const handY = (1 - normY_visible) * canvasHeight - canvasHeight / 2; hand.anchorPos.set(handX, handY, 1); if (i === 0) { // Hand 0: Arpeggio Control // --- Music & Gesture Control --- const isFistNow = this._isFist(smoothedLandmarks); if (isFistNow && !hand.isFist) { // Fist gesture was just made this.musicManager.cycleSynth(); this.musicManager.stopArpeggio(i); // Stop any old arpeggio } hand.isFist = isFistNow; const noteIndex = Math.floor((1 - normY_visible) * scale.length); const note = scale[Math.max(0, Math.min(scale.length - 1, noteIndex))]; if (this.waveformVisualizer) { const colorIndex = noteIndex % this.waveformColors.length; const newColor = this.waveformColors[colorIndex]; this.waveformVisualizer.updateColor(newColor); } const thumbTip = smoothedLandmarks[4]; const indexTip = smoothedLandmarks[8]; const dx = thumbTip.x - indexTip.x; const dy = thumbTip.y - indexTip.y; const distance = Math.sqrt(dx * dx + dy * dy); const velocity = Math.max(0, Math.min(1.0, distance * 5)); this._updateHandLines(i, smoothedLandmarks, videoParams, canvasWidth, canvasHeight, { note, velocity, isFist: isFistNow }); if (!isFistNow) { // Only play arpeggio if not in a fist // Start/Restart arpeggio if the hand just appeared OR if it just opened from a fist. const arpeggioIsActive = this.musicManager.activePatterns.has(i); if (!wasVisible || !arpeggioIsActive) { this.musicManager.startArpeggio(i, note); } else { this.musicManager.updateArpeggio(i, note); } this.musicManager.updateArpeggioVolume(i, velocity); } else { // If it is a fist, make sure the arpeggio is stopped this.musicManager.stopArpeggio(i); } } else if (i === 1) { // Hand 1: Drum Control const fingerStates = this._getFingerStates(smoothedLandmarks); drumManager.updateActiveDrums(fingerStates); this._updateHandLines(i, smoothedLandmarks, videoParams, canvasWidth, canvasHeight, { fingerStates }); } hand.lineGroup.visible = true; } else { // No landmarks for this hand if (wasVisible) { // Hand just disappeared if (i === 0) { this.musicManager.stopArpeggio(i); } else if (i === 1) { // Disable all drums when hand is gone drumManager.updateActiveDrums({}); } } hand.landmarks = null; if (hand.lineGroup) hand.lineGroup.visible = false; } } } catch (error) { console.error("Error during hand detection:", error); } } } _getVisibleVideoParameters() { if (!this.videoElement || this.videoElement.videoWidth === 0 || this.videoElement.videoHeight === 0) { return null; } const vNatW = this.videoElement.videoWidth; const vNatH = this.videoElement.videoHeight; const rW = this.renderDiv.clientWidth; const rH = this.renderDiv.clientHeight; if (vNatW === 0 || vNatH === 0 || rW === 0 || rH === 0) return null; const videoAR = vNatW / vNatH; const renderDivAR = rW / rH; let finalVideoPixelX, finalVideoPixelY; let visibleVideoPixelWidth, visibleVideoPixelHeight; if (videoAR > renderDivAR) { // Video is wider than renderDiv, scaled to fit renderDiv height, cropped horizontally. const scale = rH / vNatH; // Scale factor based on height. const scaledVideoWidth = vNatW * scale; // Width of video if scaled to fit renderDiv height. // Total original video pixels cropped horizontally (from both sides combined). const totalCroppedPixelsX = (scaledVideoWidth - rW) / scale; finalVideoPixelX = totalCroppedPixelsX / 2; // Pixels cropped from the left of original video. finalVideoPixelY = 0; // No vertical cropping. visibleVideoPixelWidth = vNatW - totalCroppedPixelsX; // Width of the visible part in original video pixels. visibleVideoPixelHeight = vNatH; // Full height is visible. } else { // Video is taller than renderDiv (or same AR), scaled to fit renderDiv width, cropped vertically. const scale = rW / vNatW; // Scale factor based on width. const scaledVideoHeight = vNatH * scale; // Height of video if scaled to fit renderDiv width. // Total original video pixels cropped vertically (from top and bottom combined). const totalCroppedPixelsY = (scaledVideoHeight - rH) / scale; finalVideoPixelX = 0; // No horizontal cropping. finalVideoPixelY = totalCroppedPixelsY / 2; // Pixels cropped from the top of original video. visibleVideoPixelWidth = vNatW; // Full width is visible. visibleVideoPixelHeight = vNatH - totalCroppedPixelsY; // Height of the visible part in original video pixels. } // Safety check for degenerate cases (e.g., extreme aspect ratios leading to zero visible dimension) if (visibleVideoPixelWidth <= 0 || visibleVideoPixelHeight <= 0) { // Fallback or log error, this shouldn't happen in normal scenarios console.warn("Calculated visible video dimension is zero or negative.", {visibleVideoPixelWidth, visibleVideoPixelHeight}); return { // Return full video dimensions as a fallback to prevent division by zero offsetX: 0, offsetY: 0, visibleWidth: vNatW, visibleHeight: vNatH, videoNaturalWidth: vNatW, videoNaturalHeight: vNatH }; } return { offsetX: finalVideoPixelX, offsetY: finalVideoPixelY, visibleWidth: visibleVideoPixelWidth, visibleHeight: visibleVideoPixelHeight, videoNaturalWidth: vNatW, videoNaturalHeight: vNatH }; } // _updateGhosts method removed. _showStatusScreen(message, color = 'white', showRestartHint = false) { // Generalized from _showGameOver this.gameOverContainer.style.display = 'block'; this.gameOverText.innerText = message; this.gameOverText.style.color = color; this.restartHintText.style.display = showRestartHint ? 'block' : 'none'; // No spawning to stop for template } _showError(message) { this.gameOverContainer.style.display = 'block'; this.gameOverText.innerText = `ERROR: ${message}`; this.gameOverText.style.color = 'orange'; this.restartHintText.style.display = 'true'; // Show restart hint on error this.gameState = 'error'; // No spawning to stop this.hands.forEach(hand => { if (hand.lineGroup) hand.lineGroup.visible = false; }); // if (this.startButton) this.startButton.style.display = 'none'; // No longer exists } _startGame() { console.log("Starting tracking..."); // This is now called automatically, so no need to check gameState this.musicManager.start().then(() => { drumManager.startSequence(); // Start drums *after* audio context is ready. // Setup the waveform visualizer after the music manager is ready const analyser = this.musicManager.getAnalyser(); if (analyser) { this.waveformVisualizer = new WaveformVisualizer(this.scene, analyser, this.renderDiv.clientWidth, this.renderDiv.clientHeight); } }); this.gameState = 'tracking'; // Changed from 'playing' this.lastVideoTime = -1; this.clock.start(); // Removed display of score, castle, chad // Removed _startSpawning() } _restartGame() { // Renamed to _restartTracking or similar if preferred console.log("Restarting tracking..."); this.gameOverContainer.style.display = 'none'; this.hands.forEach(hand => { if (hand.lineGroup) { hand.lineGroup.visible = false; } }); // Ghost removal removed // Score reset removed // Visibility of game elements removed this.gameState = 'tracking'; // Changed from 'playing' this.lastVideoTime = -1; this.clock.start(); // Removed _startSpawning() } // _updateScoreDisplay method removed. _onResize() { const width = this.renderDiv.clientWidth; const height = this.renderDiv.clientHeight; // Update camera perspective this.camera.left = width / -2; this.camera.right = width / 2; this.camera.top = height / 2; this.camera.bottom = height / -2; this.camera.updateProjectionMatrix(); // Update renderer size this.renderer.setSize(width, height); // Update video element size this.videoElement.style.width = width + 'px'; this.videoElement.style.height = height + 'px'; // Watermelon, Chad, GroundLine updates removed. this._positionBeatIndicators(); if (this.waveformVisualizer) { this.waveformVisualizer.updatePosition(width, height); } } _positionBeatIndicators() { const width = this.renderDiv.clientWidth; const height = this.renderDiv.clientHeight; const totalWidth = width * 0.8; // Occupy 80% of screen width to match the waveform const spacing = totalWidth / 16; const startX = -totalWidth / 2 + spacing / 2; const yPos = -height / 2 + 150; // Positioned a bit higher from the bottom this.beatIndicators.forEach((indicator, i) => { indicator.position.set(startX + i * spacing, yPos, 1); }); } _setupBeatIndicatorMaterials() { // All indicators start as 'off' (white) for (let i = 0; i < 16; i++) { // We just need one material definition now and will copy it. this.beatIndicatorMaterials[i] = new THREE.MeshBasicMaterial({ color: this.beatIndicatorColors.off, transparent: true, opacity: 0.5 }); } } _createTextSprite(message, parameters) { parameters = parameters || {}; const fontface = parameters.fontface || 'Arial'; const fontsize = parameters.fontsize || 24; // borderColor is no longer needed const backgroundColor = parameters.backgroundColor || { r: 255, g: 255, b: 255, a: 0.8 }; const textColor = parameters.textColor || { r: 0, g: 0, b: 0, a: 1.0 }; const canvas = document.createElement('canvas'); const context = canvas.getContext('2d'); context.font = `Bold ${fontsize}px ${fontface}`; // get size data (height depends only on font size) const metrics = context.measureText(message); const textWidth = metrics.width; const padding = 10; const canvasWidth = textWidth + padding * 2; const canvasHeight = fontsize * 1.4 + padding; canvas.width = canvasWidth; canvas.height = canvasHeight; // Font needs to be re-applied after resizing canvas context.font = `Bold ${fontsize}px ${fontface}`; // background color context.fillStyle = `rgba(${backgroundColor.r},${backgroundColor.g},${backgroundColor.b},${backgroundColor.a})`; context.fillRect(0, 0, canvasWidth, canvasHeight); // text color and position context.fillStyle = `rgba(${textColor.r}, ${textColor.g}, ${textColor.b}, 1.0)`; context.textAlign = 'center'; context.textBaseline = 'middle'; context.fillText(message, canvasWidth / 2, canvasHeight / 2); // canvas contents will be used for a texture const texture = new THREE.CanvasTexture(canvas); texture.needsUpdate = true; const spriteMaterial = new THREE.SpriteMaterial({ map: texture }); const sprite = new THREE.Sprite(spriteMaterial); sprite.scale.set(canvas.width, canvas.height, 1.0); return sprite; } _getFingerStates(landmarks) { // Landmark indices for fingertips const fingertips = { index: 8, middle: 12, ring: 16, pinky: 20 }; // Stricter check using the joint below the tip (PIP joint) to avoid false positives. const fingerJointsBelowTip = { index: 6, middle: 10, ring: 14, pinky: 18 }; const states = {}; for (const [finger, tipIndex] of Object.entries(fingertips)) { const jointIndex = fingerJointsBelowTip[finger]; if (landmarks[tipIndex] && landmarks[jointIndex]) { // A finger is "up" if its tip is higher than the joint just below it. states[finger] = landmarks[tipIndex].y < landmarks[jointIndex].y; } else { states[finger] = false; } } return states; } _isFist(landmarks) { if (!landmarks || landmarks.length < 21) return false; // Use the middle finger's MCP joint as a proxy for the palm center const palmCenter = landmarks[9]; const fingertipsIndices = [4, 8, 12, 16, 20]; // Thumb, Index, Middle, Ring, Pinky // Threshold for normalized landmark distance. If fingertips are further than this from palm, it's not a fist. // This value may need tuning. A smaller value makes the fist detection stricter. const fistThreshold = 0.1; for (const tipIndex of fingertipsIndices) { const tip = landmarks[tipIndex]; const dx = tip.x - palmCenter.x; const dy = tip.y - palmCenter.y; const distance = Math.sqrt(dx * dx + dy * dy); if (distance > fistThreshold) { return false; // At least one finger is open } } return true; // All fingertips are close to the palm } _updateHandLines(handIndex, landmarks, videoParams, canvasWidth, canvasHeight, controlData) { const hand = this.hands[handIndex]; const lineGroup = hand.lineGroup; // Clean up previous frame's objects while (lineGroup.children.length) { const child = lineGroup.children[0]; lineGroup.remove(child); if (child.geometry) child.geometry.dispose(); if (child.material) { // For sprites, we need to dispose the texture map as well if (child.material.map) child.material.map.dispose(); child.material.dispose(); } } if (!landmarks || landmarks.length === 0 || !videoParams) { lineGroup.visible = false; return; } const points3D = landmarks.map(lm => { const lmOriginalX = lm.x * videoParams.videoNaturalWidth; const lmOriginalY = lm.y * videoParams.videoNaturalHeight; let normX_visible = (lmOriginalX - videoParams.offsetX) / videoParams.visibleWidth; let normY_visible = (lmOriginalY - videoParams.offsetY) / videoParams.visibleHeight; normX_visible = Math.max(0, Math.min(1, normX_visible)); normY_visible = Math.max(0, Math.min(1, normY_visible)); const x = (1 - normX_visible) * canvasWidth - canvasWidth / 2; const y = (1 - normY_visible) * canvasHeight - canvasHeight / 2; return new THREE.Vector3(x, y, 1.1); // Z for fingertip circles }); // --- Draw Skeleton Lines --- const lineZ = 1; this.handConnections.forEach(conn => { const p1 = points3D[conn[0]]; const p2 = points3D[conn[1]]; if (p1 && p2) { const lineP1 = p1.clone().setZ(lineZ); const lineP2 = p2.clone().setZ(lineZ); const geometry = new THREE.BufferGeometry().setFromPoints([lineP1, lineP2]); const line = new THREE.Line(geometry, this.handLineMaterial); lineGroup.add(line); } }); // --- Draw Fingertip & Wrist Circles --- const fingertipRadius = 8, wristRadius = 12, circleSegments = 16; this.fingertipLandmarkIndices.forEach(index => { const landmarkPosition = points3D[index]; if (landmarkPosition) { const radius = (index === 0) ? wristRadius : fingertipRadius; const circleGeometry = new THREE.CircleGeometry(radius, circleSegments); const material = handIndex === 0 ? this.fingertipMaterialHand1 : this.fingertipMaterialHand2; const landmarkCircle = new THREE.Mesh(circleGeometry, material); landmarkCircle.position.copy(landmarkPosition); lineGroup.add(landmarkCircle); } }); // --- Draw Thumb-to-Index line and Labels --- const thumbPos = points3D[4]; const indexPos = points3D[8]; const wristPos = points3D[0]; if (wristPos) { // Ensure wrist position is valid before proceeding // Labels depend on which hand it is if (handIndex === 0 && thumbPos && indexPos) { // Arpeggio Hand - requires thumb and index // Connecting line const lineGeom = new THREE.BufferGeometry().setFromPoints([thumbPos, indexPos]); const line = new THREE.Line(lineGeom, new THREE.LineBasicMaterial({ color: 0xffffff, linewidth: 3 })); lineGroup.add(line); // Volume and Pitch labels const { note, velocity, isFist } = controlData; if (isFist) { const fistLabel = this._createTextSprite(`SYNTH ${this.musicManager.currentSynthIndex + 1}`, { fontsize: 22, backgroundColor: this.labelColors.evaPurple, textColor: this.labelColors.evaGreen }); fistLabel.position.set(wristPos.x, wristPos.y + 60, 2); lineGroup.add(fistLabel); } else { const midPoint = new THREE.Vector3().lerpVectors(thumbPos, indexPos, 0.5); const volumeLabel = this._createTextSprite(`Volume: ${velocity.toFixed(2)}`, { fontsize: 18, backgroundColor: this.labelColors.evaOrange, textColor: this.labelColors.white }); volumeLabel.position.set(midPoint.x, midPoint.y, 2); lineGroup.add(volumeLabel); const pitchLabel = this._createTextSprite(`Pitch: ${note}`, { fontsize: 18, backgroundColor: this.labelColors.evaGreen, textColor: this.labelColors.black }); pitchLabel.position.set(wristPos.x, wristPos.y + 60, 2); // Position above the wrist lineGroup.add(pitchLabel); } } else if (handIndex === 1) { // Drum Hand - no thumb interaction needed const { fingerStates } = controlData; const activeDrums = Object.entries(fingerStates) .filter(([_, isUp]) => isUp) .map(([finger, _]) => drumManager.getFingerToDrumMap()[finger]) .join(', '); const drumLabel = this._createTextSprite(`Drums: ${activeDrums || 'None'}`, { fontsize: 18, backgroundColor: this.labelColors.evaRed, textColor: this.labelColors.white }); drumLabel.position.set(wristPos.x, wristPos.y + 60, 2); lineGroup.add(drumLabel); } } lineGroup.visible = true; } _animate() { requestAnimationFrame(this._animate.bind(this)); if (this.gameState === 'tracking') { const deltaTime = this.clock.getDelta(); this._updateHands(); this._updateBeatIndicator(); if (this.waveformVisualizer) { this.waveformVisualizer.update(); } } this.renderer.render(this.scene, this.camera); } _updateBeatIndicator() { const currentBeat = drumManager.getCurrentBeat(); const progress = Tone.Transport.progress; const beatProgress = (progress * 16) % 1; const pulse = 1.5 + (0.5 * Math.cos(beatProgress * Math.PI * 2)); const activeDrums = drumManager.getActiveDrums(); const drumPattern = drumManager.getDrumPattern(); const drumPriority = ['kick', 'snare', 'clap', 'hihat']; this.beatIndicators.forEach((indicator, i) => { // Determine the color for this step based on active drums let stepColor = this.beatIndicatorColors.off; let isHit = false; for (const drum of drumPriority) { if (activeDrums.has(drum) && drumPattern[drum][i]) { stepColor = this.beatIndicatorColors[drum]; isHit = true; break; } } indicator.material.color.set(stepColor); indicator.material.opacity = isHit ? 0.9 : 0.5; // Apply pulse only to the current beat marker if (i === currentBeat) { indicator.scale.set(pulse, pulse, 1); } else { indicator.scale.set(1, 1, 1); } }); } _setupEventListeners() { // Add click listener for resuming audio context and potentially restarting on error this.renderDiv.addEventListener('click', () => { this.musicManager.start(); // Resume audio context on any click if (this.gameState === 'error') { // A click will restart if there was an error this._restartGame(); } }); console.log('Game event listeners set up.'); } }

Other games like this

About arpeggiator