// MotionBricks G1 playground: three.js scene, input, HUD and the main loop. import * as THREE from 'three'; import { OrbitControls } from 'three/addons/controls/OrbitControls.js'; import { loadRobotModel, Robot, heroMaterials, ghostMaterials } from './robot.js'; import { loadWorld, LAYOUTS } from './assets.js'; import { Inference } from './inference.js'; import { Character, InteractionDirector, MODES, CROUCH_MODE } from './character.js'; import { NQ, PPT, FPS } from './motion.js'; import { quat } from './mathx.js'; import { icon } from './icons.js'; const params = new URLSearchParams(location.search); const CAPTURE = params.has('capture'); const PROMO = CAPTURE && params.has('promo'); const LOCAL = params.get('src') === 'local'; const SPACE = window.MB_SPACE || ''; const MODEL_FILE = 'mmm_v6_g1_fp16.onnx'; const MODEL_URLS = LOCAL ? ['models/g1_fp16.onnx'] : [SPACE ? `https://huggingface.co/spaces/${SPACE}/resolve/main/models/${MODEL_FILE}` : null, `models/${MODEL_FILE}`].filter(Boolean); const $ = (id) => document.getElementById(id); // Hue only for what the model sees (context, targets and the generated path) and the yellow click target; the rest is neutral. const COLORS = { context: 0xa78bfa, target: 0x5eead4, plan: 0x5eead4, trail: 0xffffff, goal: 0xfbbf24, contact: 0xededed, box: 0xa3a3a3 }; // ---------------------------------------------------------------------------------------------- renderer const renderer = new THREE.WebGLRenderer({ antialias: true, preserveDrawingBuffer: CAPTURE }); renderer.setPixelRatio(CAPTURE ? (PROMO ? devicePixelRatio : 1) : Math.min(devicePixelRatio, 2)); renderer.setSize(innerWidth, innerHeight); renderer.shadowMap.enabled = true; renderer.shadowMap.type = THREE.PCFShadowMap; renderer.toneMapping = THREE.ACESFilmicToneMapping; renderer.toneMappingExposure = 1.05; $('stage').appendChild(renderer.domElement); const scene = new THREE.Scene(); scene.background = new THREE.Color(0x0a0a0a); scene.fog = new THREE.Fog(0x0a0a0a, 14, 42); const camera = new THREE.PerspectiveCamera(38, innerWidth / innerHeight, 0.05, 200); camera.position.set(-10.4, 2.5, -0.6); const controls = new OrbitControls(camera, renderer.domElement); controls.enableDamping = true; controls.dampingFactor = 0.08; controls.minDistance = 1.5; controls.maxDistance = 18; controls.maxPolarAngle = Math.PI * 0.49; controls.target.set(-6.3, 0.75, -3.1); scene.add(new THREE.HemisphereLight(0xeef2f8, 0x1a1a1a, 0.9)); const sun = new THREE.DirectionalLight(0xffffff, 2.4); sun.castShadow = true; sun.shadow.mapSize.set(2048, 2048); sun.shadow.camera.left = -6; sun.shadow.camera.right = 6; sun.shadow.camera.top = 6; sun.shadow.camera.bottom = -6; sun.shadow.camera.near = 1; sun.shadow.camera.far = 30; sun.shadow.bias = -0.0004; sun.shadow.normalBias = 0.02; scene.add(sun, sun.target); const rim = new THREE.DirectionalLight(0xc8d6ec, 1.1); rim.position.set(-6, 5, -8); scene.add(rim); // Z-up world: everything MotionBricks produces lives in here. const world = new THREE.Group(); world.rotation.x = -Math.PI / 2; scene.add(world); function makeFloor() { const size = 1024, c = document.createElement('canvas'); c.width = c.height = size; const g = c.getContext('2d'); g.fillStyle = '#141414'; g.fillRect(0, 0, size, size); g.strokeStyle = 'rgba(255,255,255,0.045)'; g.lineWidth = 2; for (let i = 0; i <= 8; i++) { const p = (i * size) / 8; g.beginPath(); g.moveTo(p, 0); g.lineTo(p, size); g.moveTo(0, p); g.lineTo(size, p); g.stroke(); } g.strokeStyle = 'rgba(255,255,255,0.09)'; g.lineWidth = 3; g.strokeRect(0, 0, size, size); const tex = new THREE.CanvasTexture(c); tex.wrapS = tex.wrapT = THREE.RepeatWrapping; tex.repeat.set(30, 30); tex.anisotropy = 8; tex.colorSpace = THREE.SRGBColorSpace; const floor = new THREE.Mesh(new THREE.PlaneGeometry(120, 120), new THREE.MeshStandardMaterial({ map: tex, roughness: 0.92, metalness: 0 })); floor.receiveShadow = true; floor.name = 'floor'; world.add(floor); return floor; } const floor = makeFloor(); function label(text, color = '#ededed') { const c = document.createElement('canvas'); c.width = 512; c.height = 128; const g = c.getContext('2d'); g.font = '500 54px Geist, system-ui, sans-serif'; g.textAlign = 'center'; g.textBaseline = 'middle'; g.fillStyle = 'rgba(18,18,18,0.82)'; const w = g.measureText(text).width + 64; g.beginPath(); g.roundRect(256 - w / 2, 20, w, 88, 44); g.fill(); g.fillStyle = color; g.fillText(text, 256, 66); const tex = new THREE.CanvasTexture(c); tex.colorSpace = THREE.SRGBColorSpace; const s = new THREE.Sprite(new THREE.SpriteMaterial({ map: tex, depthWrite: false, transparent: true })); s.scale.set(1.0, 0.25, 1); return s; } // ---------------------------------------------------------------------------------------------- state let model, liteModel, hero, ctxGhost, tgtGhost, character, director, worldData; const instanceGhosts = []; const inference = CAPTURE ? null : new Inference('js/worker.js'); const ui = { ghosts: true, plan: true, boxes: true, probe: false, contacts: true, trail: true, auto: true, run: false, keys: new Set(), goal: null, goalRun: false, interact: false, mode: null, joystick: null, }; const log = []; // ---------------------------------------------------------------------------------------------- overlays const planLine = new THREE.Line(new THREE.BufferGeometry(), new THREE.LineBasicMaterial({ color: COLORS.plan, transparent: true, opacity: 0.9 })); const trailLine = new THREE.Line(new THREE.BufferGeometry(), new THREE.LineBasicMaterial({ color: COLORS.trail, transparent: true, opacity: 0.25 })); const transitionDot = new THREE.Mesh(new THREE.CircleGeometry(0.06, 24), new THREE.MeshBasicMaterial({ color: COLORS.plan })); const goalMarker = new THREE.Mesh(new THREE.RingGeometry(0.22, 0.31, 48), new THREE.MeshBasicMaterial({ color: COLORS.goal, transparent: true, opacity: 0.95 })); const probeMesh = new THREE.LineSegments(new THREE.EdgesGeometry(new THREE.BoxGeometry(1, 1, 1)), new THREE.LineBasicMaterial({ color: COLORS.box, transparent: true, opacity: 0.5 })); probeMesh.visible = false; world.add(planLine, trailLine, transitionDot, goalMarker, probeMesh); goalMarker.visible = false; const contactDiscs = [0, 1, 2, 3].map(() => { const m = new THREE.Mesh(new THREE.CircleGeometry(0.05, 20), new THREE.MeshBasicMaterial({ color: COLORS.contact, transparent: true, opacity: 0 })); world.add(m); return m; }); const trail = []; const boxMeshes = []; const props = new Map(); // solid cube name -> its mesh (the promo frames shots around them) function buildProps(data) { const propMat = (c) => { const col = new THREE.Color(c[0], c[1], c[2]).multiplyScalar(0.55).addScalar(0.12); const grey = col.r * 0.3 + col.g * 0.59 + col.b * 0.11; return new THREE.MeshStandardMaterial({ color: col.lerp(new THREE.Color(grey, grey, grey), 0.75), roughness: 0.6, metalness: 0.05 }); }; for (const cube of data.cubes) { let mesh; if (cube.name === 'Bar') { // Visual only: a beam across the path at the authored clearance, on two posts. const g = new THREE.Group(); const bottom = cube.t.p[2] - cube.size[2] / 2; const beam = new THREE.Mesh(new THREE.BoxGeometry(0.18, 2.6, 0.14), propMat(cube.color)); beam.position.set(0, 0, bottom + 0.07); g.add(beam); for (const s of [-1, 1]) { const post = new THREE.Mesh(new THREE.BoxGeometry(0.12, 0.12, bottom + 0.14), propMat([0.35, 0.37, 0.4])); post.position.set(0, s * 1.3, (bottom + 0.14) / 2); g.add(post); } g.traverse((o) => { o.castShadow = o.receiveShadow = true; }); // The authored Bar has no net rotation, so the beam's local Y runs across the approach (+X). g.position.set(cube.t.p[0], cube.t.p[1], 0); g.quaternion.set(...cube.t.q); mesh = g; } else { mesh = new THREE.Mesh(new THREE.BoxGeometry(...cube.size), propMat(cube.color)); mesh.position.set(...cube.t.p); mesh.quaternion.set(...cube.t.q); mesh.castShadow = mesh.receiveShadow = true; } mesh.name = cube.name; props.set(cube.name, mesh); world.add(mesh); } const names = { SmallJumpSP: 'Jump', MidJumpSP: 'Vault', SitSP: 'Sit', AvoidanceSP: 'Duck under' }; for (const inst of data.instances) { const s = label(names[inst.name] || inst.name); s.position.set(inst.position[0], inst.position[1], 2.05); world.add(s); for (const b of inst.sp.worldBoxes()) { const m = new THREE.LineSegments(new THREE.EdgesGeometry(new THREE.BoxGeometry(b.half[0] * 2, b.half[1] * 2, b.half[2] * 2)), new THREE.LineBasicMaterial({ color: COLORS.box, transparent: true, opacity: 0.35 })); m.userData.inst = inst; world.add(m); boxMeshes.push({ mesh: m, inst, box: b }); } } } function updateBoxes() { for (const bm of boxMeshes) { bm.mesh.visible = ui.boxes; const b = bm.inst.sp.worldBoxes()[bm.inst.sp.worldBoxes().indexOf(bm.box)] || bm.box; bm.mesh.position.set(...b.t.p); bm.mesh.quaternion.set(...b.t.q); const active = bm.inst.interactable; bm.mesh.material.color.set(active ? COLORS.target : COLORS.box); bm.mesh.material.opacity = active ? 0.9 : 0.35; } const p = director && director.probe; probeMesh.visible = ui.probe && !!p; if (p) { probeMesh.position.set(...p.t.p); probeMesh.quaternion.set(...p.t.q); probeMesh.scale.set(p.half[0] * 2, p.half[1] * 2, p.half[2] * 2); } } function setLine(line, pts, z) { const arr = new Float32Array(pts.length * 3); pts.forEach((p, i) => { arr[i * 3] = p[0]; arr[i * 3 + 1] = p[1]; arr[i * 3 + 2] = z; }); line.geometry.setAttribute('position', new THREE.BufferAttribute(arr, 3)); line.geometry.computeBoundingSphere(); } const _v = new THREE.Vector3(); const _m = new THREE.Matrix4(); function updateOverlays(dt) { const ch = character; hero.setPose(ch.pose); const req = ch.lastRequest; ctxGhost.visible = tgtGhost.visible = ui.ghosts && !!req && ch.source !== 'clip'; if (req && ui.ghosts) { ctxGhost.setPose(req.context.subarray((PPT - 1) * NQ, PPT * NQ)); tgtGhost.setPose(req.targets.subarray(0, NQ)); } const path = ui.plan ? ch.plannedPath() : []; planLine.visible = path.length > 1; if (path.length > 1) setLine(planLine, path, 0.012); transitionDot.visible = ui.plan && !!ch.seg && ch.source !== 'clip'; if (transitionDot.visible) { const f = Math.max(0, ch.seg.n - 1 - ch.seg.transitionEarly); transitionDot.position.set(ch.seg.poses[f * NQ], ch.seg.poses[f * NQ + 1], 0.013); } trail.push([ch.pose[0], ch.pose[1]]); if (trail.length > 360) trail.shift(); trailLine.visible = ui.trail && trail.length > 1; if (trailLine.visible) setLine(trailLine, trail, 0.01); // Foot contacts at the ankle roll joints (heel) and 0.14 m ahead, 0.035 m below (toe). hero.root.updateMatrixWorld(true); const feet = ['left_ankle_roll_link', 'left_ankle_roll_link', 'right_ankle_roll_link', 'right_ankle_roll_link']; for (let i = 0; i < 4; i++) { const body = hero.byName.get(feet[i]); _v.set(i % 2 ? 0.14 : 0, 0, i % 2 ? -0.035 : 0).applyMatrix4(_m.copy(world.matrixWorld).invert().multiply(body.matrixWorld)); const d = contactDiscs[i]; d.position.set(_v.x, _v.y, 0.008); d.material.opacity = ui.contacts ? 0.6 * ch.contacts[i] : 0; } goalMarker.visible = !!ui.goal; if (ui.goal) { goalMarker.position.set(ui.goal[0], ui.goal[1], 0.015); goalMarker.rotation.z += dt * 1.5; } updateBoxes(); } // ---------------------------------------------------------------------------------------------- camera follow let lastFollow = null; function followCamera() { const p = new THREE.Vector3(character.pose[0], character.pose[2] * 0.9, -character.pose[1]); if (CAPTURE) { sun.position.set(p.x + 4, 9, p.z + 3); sun.target.position.copy(p); return; } if (!lastFollow) { lastFollow = p.clone(); return; } const delta = p.clone().sub(lastFollow); delta.y *= 0.3; camera.position.add(delta); controls.target.add(delta); lastFollow.add(delta); sun.position.set(p.x + 4, 9, p.z + 3); sun.target.position.copy(p); } function cameraFacing() { const d = new THREE.Vector3().subVectors(controls.target, camera.position); const f = [d.x, -d.z]; const l = Math.hypot(f[0], f[1]) || 1; return [f[0] / l, f[1] / l, 0]; } // ---------------------------------------------------------------------------------------------- input const MOVE_KEYS = { KeyW: [0, 1], ArrowUp: [0, 1], KeyS: [0, -1], ArrowDown: [0, -1], KeyA: [-1, 0], ArrowLeft: [-1, 0], KeyD: [1, 0], ArrowRight: [1, 0] }; const STYLE_MODES = MODES.slice(3); let lastStyle = STYLE_MODES[0].name; function setMode(name) { if (!character) return; ui.mode = name; if (STYLE_MODES.some((m) => m.name === name)) lastStyle = name; renderModeButtons(name); } function currentModeName() { return character ? character.sm.loco.mode.name : 'linear_1d'; } addEventListener('keydown', (e) => { if (e.target.tagName === 'INPUT') return; ui.keys.add(e.code); if (MOVE_KEYS[e.code]) { ui.goal = null; e.preventDefault(); } if (e.code === 'KeyE' || e.code === 'KeyI') ui.interact = true; if (e.code === 'KeyC' || e.code === 'KeyP') setMode(currentModeName() === CROUCH_MODE ? 'linear_1d' : CROUCH_MODE); if (e.code === 'KeyQ' || e.code === 'KeyO') setMode(currentModeName() === 'directional_2d' ? 'linear_1d' : 'directional_2d'); if (e.code === 'BracketLeft') { const i = STYLE_MODES.findIndex((m) => m.name === currentModeName()); setMode(i < 0 ? lastStyle : STYLE_MODES[(i + 1) % STYLE_MODES.length].name); } if (e.code === 'Digit1') setMode('linear_1d'); if (e.code === 'Digit2') setMode('directional_2d'); if (e.code === 'Digit3') setMode(CROUCH_MODE); const n = Number(e.key); if (n >= 4 && n <= 9) setMode(STYLE_MODES[n - 4].name); if (e.code === 'Space') e.preventDefault(); }); addEventListener('keyup', (e) => ui.keys.delete(e.code)); addEventListener('blur', () => ui.keys.clear()); // Click / tap on the floor to walk there (double click to run). const raycaster = new THREE.Raycaster(); let downAt = null; renderer.domElement.addEventListener('pointerdown', (e) => { downAt = [e.clientX, e.clientY, performance.now()]; }); renderer.domElement.addEventListener('pointerup', (e) => { if (!downAt || !character) return; const moved = Math.hypot(e.clientX - downAt[0], e.clientY - downAt[1]); const quick = performance.now() - downAt[2] < 350; downAt = null; if (moved > 6 || !quick) return; const ndc = new THREE.Vector2((e.clientX / innerWidth) * 2 - 1, -(e.clientY / innerHeight) * 2 + 1); raycaster.setFromCamera(ndc, camera); const hit = raycaster.intersectObject(floor)[0]; if (!hit) return; const local = world.worldToLocal(hit.point.clone()); const now = performance.now(); ui.goalRun = !!(ui.lastClick && now - ui.lastClick < 400); ui.lastClick = now; ui.goal = [local.x, local.y]; }); function buildCommand() { const facing = cameraFacing(); const right = [facing[1], -facing[0], 0]; let ix = 0, iy = 0; for (const k of ui.keys) if (MOVE_KEYS[k]) { ix += MOVE_KEYS[k][0]; iy += MOVE_KEYS[k][1]; } if (ui.joystick) { ix += ui.joystick[0]; iy += ui.joystick[1]; } let movement = null; let run = ui.run || ui.keys.has('ShiftLeft') || ui.keys.has('ShiftRight') || ui.keys.has('Space') || (ui.joystick && Math.hypot(...ui.joystick) > 0.92); if (ix || iy) { const l = Math.max(1, Math.hypot(ix, iy)); movement = [(facing[0] * iy + right[0] * ix) / l, (facing[1] * iy + right[1] * ix) / l, 0]; } else if (ui.goal) { const d = [ui.goal[0] - character.ground[0], ui.goal[1] - character.ground[1], 0]; const dist = Math.hypot(d[0], d[1]); const speed = Math.hypot(...character.movement.desiredVelocity); if (dist < Math.max(0.35, speed * 0.75)) ui.goal = null; else { movement = [d[0] / dist, d[1] / dist, 0]; run = run || ui.goalRun; } } if (!inferenceReady()) movement = null; const cmd = { facing, movement, run, mode: ui.mode }; ui.mode = null; return cmd; } // Touch joystick (function joystick() { const pad = $('joystick'), knob = $('knob'); let id = null, cx = 0, cy = 0; pad.addEventListener('pointerdown', (e) => { id = e.pointerId; const r = pad.getBoundingClientRect(); cx = r.left + r.width / 2; cy = r.top + r.height / 2; pad.setPointerCapture(id); move(e); }); pad.addEventListener('pointermove', (e) => { if (e.pointerId === id) move(e); }); const end = () => { id = null; ui.joystick = null; knob.style.transform = ''; }; pad.addEventListener('pointerup', end); pad.addEventListener('pointercancel', end); function move(e) { const dx = e.clientX - cx, dy = e.clientY - cy, r = 48, l = Math.hypot(dx, dy), k = l > r ? r / l : 1; knob.style.transform = `translate(${dx * k}px, ${dy * k}px)`; ui.joystick = [(dx * k) / r, (-dy * k) / r]; ui.goal = null; } })(); // ---------------------------------------------------------------------------------------------- UI function renderModeButtons(active = currentModeName()) { for (const b of document.querySelectorAll('[data-mode]')) b.classList.toggle('on', b.dataset.mode === active); } function buildPanel() { for (const el of document.querySelectorAll('[data-icon]')) el.innerHTML = icon(el.dataset.icon, Number(el.dataset.size) || 16); // The three gaits are a segmented control; the styles show their number key. MODES.forEach((m, i) => { const b = document.createElement('button'); b.dataset.mode = m.name; b.title = `${m.label} (key ${i + 1})`; b.innerHTML = i < 3 ? m.label : `${m.label}${i + 1}`; b.onclick = () => setMode(m.name); $(i < 3 ? 'modes' : 'styles').appendChild(b); }); const toggles = [ ['overlays', 'ghosts', 'Context & target poses'], ['overlays', 'plan', 'Generated path'], ['overlays', 'contacts', 'Foot contacts'], ['overlays', 'boxes', 'Interaction boxes'], ['overlays', 'probe', 'Probe box'], ['overlays', 'trail', 'Trail'], ['behavior', 'auto', 'Auto interact'], ['behavior', 'run', 'Always run'], ]; for (const [group, k, text] of toggles) { const l = document.createElement('label'); l.innerHTML = `${text}`; l.querySelector('input').onchange = (e) => { ui[k] = e.target.checked; if (k === 'auto' && director) director.auto = ui.auto; }; $(group).appendChild(l); } $('interact').onclick = () => { ui.interact = true; }; $('runbtn').onpointerdown = () => { ui.run = true; }; $('runbtn').onpointerup = $('runbtn').onpointerleave = () => { ui.run = false; }; $('panel-toggle').onclick = () => document.body.classList.toggle('panel-collapsed'); // On a phone the panel is a sheet over the scene, so it starts closed. if (matchMedia('(max-width: 760px)').matches) document.body.classList.add('panel-collapsed'); // On short windows the help would be cut off at the bottom of the panel: start it folded. if (innerHeight < 820) $('help').open = false; const about = $('about'); $('about-open').onclick = () => about.showModal(); $('about-close').onclick = () => about.close(); about.onclick = (e) => { if (e.target === about) about.close(); }; renderModeButtons('linear_1d'); } // The log is a trace: one line per generation, newest first, older lines fading out. function pushLog(iconName, title, detail) { log.unshift({ iconName, title, detail }); if (log.length > 6) log.pop(); $('log').innerHTML = log.map((l, i) => `