From 5d94e8101fc40b054f273b0b968b5d13097f60d7 Mon Sep 17 00:00:00 2001 From: mgt Date: Sun, 11 Jan 2026 17:12:39 +0800 Subject: [PATCH] refactor: split ultra long component --- .../src/modules/media/ThreeDSceneViewer.tsx | 763 +----------------- .../src/modules/media/threeDSceneCamera.ts | 304 +++++++ .../src/modules/media/threeDSceneMetadata.ts | 248 ++++++ .../modules/media/useThreeDSceneParallax.ts | 213 +++++ 4 files changed, 784 insertions(+), 744 deletions(-) create mode 100644 apps/web/src/modules/media/threeDSceneCamera.ts create mode 100644 apps/web/src/modules/media/threeDSceneMetadata.ts create mode 100644 apps/web/src/modules/media/useThreeDSceneParallax.ts diff --git a/apps/web/src/modules/media/ThreeDSceneViewer.tsx b/apps/web/src/modules/media/ThreeDSceneViewer.tsx index cabaf4a9..730c3666 100644 --- a/apps/web/src/modules/media/ThreeDSceneViewer.tsx +++ b/apps/web/src/modules/media/ThreeDSceneViewer.tsx @@ -1,588 +1,27 @@ import { clsxm } from '@afilmory/utils' import { SparkRenderer, SplatFileType, SplatMesh } from '@sparkjsdev/spark' -import type { MutableRefObject } from 'react' import { useCallback, useEffect, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' import * as THREE from 'three' import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls.js' -import { isMobileDevice } from '~/lib/device-viewport' import type { LoadingIndicatorRef } from '~/modules/inspector' import type { PhotoManifest } from '~/types/photo' +import { + type ActiveCameraState, + applyCameraProjection, + applyMetadataCamera, + clearMetadataCamera, + type DefaultCameraState, + type DefaultControlsState, + fitViewToMesh, +} from './threeDSceneCamera' +import { readSogMetadata } from './threeDSceneMetadata' +import { useThreeDSceneParallax } from './useThreeDSceneParallax' + type ThreeDSceneSource = NonNullable -type CameraIntrinsics = { - fx: number - fy: number - cx: number - cy: number - imageWidth: number - imageHeight: number -} - -type CameraMetadata = { - intrinsics: CameraIntrinsics - extrinsicCv: number[] - colorSpaceIndex?: number - headerComments?: string[] -} - -type ActiveCameraState = CameraMetadata & { - near: number - far: number -} - -type DefaultCameraState = { - fov: number - near: number - far: number -} - -type DefaultControlsState = { - dampingFactor: number - rotateSpeed: number - zoomSpeed: number - panSpeed: number -} - -type ParallaxState = { - basePosition: THREE.Vector3 - baseTarget: THREE.Vector3 - baseRight: THREE.Vector3 - baseUp: THREE.Vector3 - strength: number - current: THREE.Vector2 - target: THREE.Vector2 -} - -const PARALLAX_DEADZONE_MOUSE = 0.06 -const PARALLAX_DEADZONE_MOTION = 0.12 -const PARALLAX_STRENGTH_MOBILE_MULTIPLIER = 1.5 -const ORIENTATION_RANGE = { - gamma: 30, - beta: 30, -} - -// --- Metadata helpers (ported from ../browser-sharp) --- - -const toExtrinsic4x4RowMajor = (raw?: number[] | Float32Array | null): number[] => { - if (!raw) { - return [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1] - } - - const values = Array.from(raw) - - if (values.length === 16) return values - - if (values.length === 12) { - const m = [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1] - - m[0] = values[0] - m[1] = values[1] - m[2] = values[2] - m[3] = values[3] - - m[4] = values[4] - m[5] = values[5] - m[6] = values[6] - m[7] = values[7] - - m[8] = values[8] - m[9] = values[9] - m[10] = values[10] - m[11] = values[11] - - const r00 = m[0] - const r01 = m[1] - const r02 = m[2] - const r10 = m[4] - const r11 = m[5] - const r12 = m[6] - const r20 = m[8] - const r21 = m[9] - const r22 = m[10] - - m[0] = r00 - m[1] = r10 - m[2] = r20 - m[4] = r01 - m[5] = r11 - m[6] = r21 - m[8] = r02 - m[9] = r12 - m[10] = r22 - - return m - } - - throw new Error(`Unrecognized extrinsic element length: ${values.length}`) -} - -const parseIntrinsics = (raw: number[] | Float32Array | undefined, imageWidth?: number, imageHeight?: number) => { - if (!raw) return null - const values = Array.from(raw) - - if (values.length === 9) { - if (!(typeof imageWidth === 'number' && Number.isFinite(imageWidth))) return null - if (!(typeof imageHeight === 'number' && Number.isFinite(imageHeight))) return null - const width = imageWidth - const height = imageHeight - return { - fx: values[0], - fy: values[4], - cx: values[2], - cy: values[5], - imageWidth: width, - imageHeight: height, - } - } - - if (values.length === 16) { - if (!(typeof imageWidth === 'number' && Number.isFinite(imageWidth))) return null - if (!(typeof imageHeight === 'number' && Number.isFinite(imageHeight))) return null - const width = imageWidth - const height = imageHeight - return { - fx: values[0], - fy: values[5], - cx: values[2], - cy: values[6], - imageWidth: width, - imageHeight: height, - } - } - - if (values.length === 4) { - const legacyWidth = Number.parseInt(`${values[2]}`) - const legacyHeight = Number.parseInt(`${values[3]}`) - const width = typeof imageWidth === 'number' && Number.isFinite(imageWidth) ? imageWidth : legacyWidth - const height = typeof imageHeight === 'number' && Number.isFinite(imageHeight) ? imageHeight : legacyHeight - if (!Number.isFinite(width) || !Number.isFinite(height)) return null - return { - fx: values[0], - fy: values[1], - cx: (width - 1) * 0.5, - cy: (height - 1) * 0.5, - imageWidth: width, - imageHeight: height, - } - } - - return null -} - -const normalizeColorSpaceIndex = (value: unknown) => { - if (Array.isArray(value)) { - return Number.isFinite(value[0]) ? Number(value[0]) : undefined - } - return Number.isFinite(value as number) ? Number(value) : undefined -} - -const buildCameraMetadata = (raw: Record = {}): CameraMetadata | null => { - const imageSize = raw.image_size - const imageWidth = Array.isArray(imageSize) ? imageSize[0] : undefined - const imageHeight = Array.isArray(imageSize) ? imageSize[1] : undefined - const intrinsics = parseIntrinsics(raw.intrinsic, imageWidth, imageHeight) - if (!intrinsics) return null - - return { - intrinsics, - extrinsicCv: toExtrinsic4x4RowMajor(raw.extrinsic), - colorSpaceIndex: normalizeColorSpaceIndex(raw.color_space), - headerComments: raw.headerComments ?? [], - } -} - -// --- SOG metadata reader (ported from ../browser-sharp) --- - -const ZIP_LOCAL_FILE_HEADER = 0x04034b50 -const ZIP_CENTRAL_DIR_HEADER = 0x02014b50 -const ZIP_END_OF_CENTRAL_DIR = 0x06054b50 -const ZIP_END_OF_CENTRAL_DIR_MIN_SIZE = 22 -const ZIP_MAX_COMMENT_SIZE = 0xffff - -const textDecoder = new TextDecoder('utf-8') - -const findZipEndOfCentralDir = (view: DataView, length: number) => { - const start = Math.max(0, length - (ZIP_END_OF_CENTRAL_DIR_MIN_SIZE + ZIP_MAX_COMMENT_SIZE)) - for (let offset = length - ZIP_END_OF_CENTRAL_DIR_MIN_SIZE; offset >= start; offset -= 1) { - if (view.getUint32(offset, true) === ZIP_END_OF_CENTRAL_DIR) return offset - } - return -1 -} - -const findZipEntry = (bytes: Uint8Array, targetName: string) => { - const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) - const length = bytes.byteLength - const eocdOffset = findZipEndOfCentralDir(view, length) - if (eocdOffset < 0) return null - - const centralDirSize = view.getUint32(eocdOffset + 12, true) - const centralDirOffset = view.getUint32(eocdOffset + 16, true) - const centralDirEnd = centralDirOffset + centralDirSize - if (centralDirEnd > length) return null - - let offset = centralDirOffset - while (offset + 46 <= centralDirEnd) { - if (view.getUint32(offset, true) !== ZIP_CENTRAL_DIR_HEADER) break - - const compression = view.getUint16(offset + 10, true) - const compressedSize = view.getUint32(offset + 20, true) - const nameLength = view.getUint16(offset + 28, true) - const extraLength = view.getUint16(offset + 30, true) - const commentLength = view.getUint16(offset + 32, true) - const localHeaderOffset = view.getUint32(offset + 42, true) - - const nameStart = offset + 46 - const nameEnd = nameStart + nameLength - const name = textDecoder.decode(bytes.subarray(nameStart, nameEnd)) - const entryName = name.split(/[\\/]/).pop()?.toLowerCase() - - if (entryName === targetName) { - return { - compression, - compressedSize, - localHeaderOffset, - } - } - - offset = nameEnd + extraLength + commentLength - } - - return null -} - -const inflateRaw = async (data: Uint8Array) => { - if (typeof DecompressionStream === 'undefined') { - throw new TypeError('DecompressionStream is not available in this browser') - } - const slice = data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer - const stream = new Blob([slice]).stream().pipeThrough(new DecompressionStream('deflate-raw')) - const buffer = await new Response(stream).arrayBuffer() - return new Uint8Array(buffer) -} - -const readZipEntryData = async ( - bytes: Uint8Array, - entry: { compression: number; compressedSize: number; localHeaderOffset: number }, -) => { - const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) - const headerOffset = entry.localHeaderOffset - if (headerOffset + 30 > bytes.byteLength) return null - if (view.getUint32(headerOffset, true) !== ZIP_LOCAL_FILE_HEADER) return null - - const nameLength = view.getUint16(headerOffset + 26, true) - const extraLength = view.getUint16(headerOffset + 28, true) - const dataStart = headerOffset + 30 + nameLength + extraLength - const dataEnd = dataStart + entry.compressedSize - if (dataEnd > bytes.byteLength) return null - - const compressed = bytes.subarray(dataStart, dataEnd) - if (entry.compression === 0) return compressed - if (entry.compression === 8) return inflateRaw(compressed) - - throw new Error(`Unsupported zip compression method: ${entry.compression}`) -} - -const readSogMetaJson = async (bytes: Uint8Array) => { - const entry = findZipEntry(bytes, 'meta.json') - if (!entry) return null - const data = await readZipEntryData(bytes, entry) - if (!data) return null - return JSON.parse(textDecoder.decode(data)) -} - -const readSogMetadata = async (bytes: Uint8Array) => { - const meta = await readSogMetaJson(bytes) - if (!meta || typeof meta !== 'object') return null - const sharpMetadata = meta.sharp_metadata - if (!sharpMetadata || typeof sharpMetadata !== 'object') return null - return buildCameraMetadata(sharpMetadata) -} - -// --- Camera + view helpers (ported from ../browser-sharp) --- - -const makeAxisFlipCvToGl = () => new THREE.Matrix4().set(1, 0, 0, 0, 0, -1, 0, 0, 0, 0, -1, 0, 0, 0, 0, 1) - -const quantileSorted = (sorted: number[], q: number) => { - if (sorted.length === 0) return null - const clampedQ = Math.max(0, Math.min(1, q)) - const pos = (sorted.length - 1) * clampedQ - const lower = Math.floor(pos) - const upper = Math.ceil(pos) - if (lower === upper) return sorted[lower] - const weight = pos - lower - return sorted[lower] * (1 - weight) + sorted[upper] * weight -} - -const computeMlSharpDepthFocus = (mesh: SplatMesh, { qFocus = 0.1, minDepthFocus = 2, maxSamples = 50_000 } = {}) => { - const numSplats = mesh?.packedSplats?.numSplats ?? 0 - if (!numSplats) return minDepthFocus - - const step = Math.max(1, Math.floor(numSplats / maxSamples)) - const depths: number[] = [] - for (let i = 0; i < numSplats; i += step) { - const { center } = mesh.packedSplats.getSplat(i) - const { z } = center - if (Number.isFinite(z) && z > 0) depths.push(z) - } - - if (depths.length === 0) return minDepthFocus - depths.sort((a, b) => a - b) - const q = quantileSorted(depths, qFocus) - if (!Number.isFinite(q)) return minDepthFocus - return Math.max(minDepthFocus, q!) -} - -const makeProjectionFromIntrinsics = ({ - fx, - fy, - cx, - cy, - width, - height, - near, - far, -}: { - fx: number - fy: number - cx: number - cy: number - width: number - height: number - near: number - far: number -}) => { - const left = (-cx * near) / fx - const right = ((width - cx) * near) / fx - const top = (cy * near) / fy - const bottom = (-(height - cy) * near) / fy - - return new THREE.Matrix4().set( - (2 * near) / (right - left), - 0, - (right + left) / (right - left), - 0, - 0, - (2 * near) / (top - bottom), - (top + bottom) / (top - bottom), - 0, - 0, - 0, - -(far + near) / (far - near), - (-2 * far * near) / (far - near), - 0, - 0, - -1, - 0, - ) -} - -const applyCameraProjection = ({ - cameraMetadata, - camera, - controls, - viewportWidth, - viewportHeight, - defaultCamera, - defaultControls, -}: { - cameraMetadata: ActiveCameraState - camera: THREE.PerspectiveCamera - controls: OrbitControls - viewportWidth: number - viewportHeight: number - defaultCamera: DefaultCameraState - defaultControls: DefaultControlsState -}) => { - const { intrinsics, near, far } = cameraMetadata - const sx = viewportWidth / intrinsics.imageWidth - const sy = viewportHeight / intrinsics.imageHeight - const s = Math.min(sx, sy) - const scaledWidth = intrinsics.imageWidth * s - const scaledHeight = intrinsics.imageHeight * s - const offsetX = (viewportWidth - scaledWidth) * 0.5 - const offsetY = (viewportHeight - scaledHeight) * 0.5 - - const fx = intrinsics.fx * s - const fy = intrinsics.fy * s - const cx = intrinsics.cx * s + offsetX - const cy = intrinsics.cy * s + offsetY - - camera.aspect = viewportWidth / Math.max(1, viewportHeight) - camera.fov = THREE.MathUtils.radToDeg(2 * Math.atan(viewportHeight / (2 * Math.max(1e-6, fy)))) - camera.near = near - camera.far = far - - const fovScale = THREE.MathUtils.clamp(camera.fov / defaultCamera.fov, 0.05, 2) - controls.rotateSpeed = Math.max(0.02, defaultControls.rotateSpeed * fovScale * 0.45) - controls.zoomSpeed = Math.max(0.05, defaultControls.zoomSpeed * fovScale * 0.8) - controls.panSpeed = Math.max(0.05, defaultControls.panSpeed * fovScale * 0.8) - - const projection = makeProjectionFromIntrinsics({ - fx, - fy, - cx, - cy, - width: viewportWidth, - height: viewportHeight, - near, - far, - }) - camera.projectionMatrix.copy(projection) - camera.projectionMatrixInverse.copy(projection).invert() -} - -const clearMetadataCamera = ({ - camera, - controls, - defaultCamera, - defaultControls, - activeCameraRef, -}: { - camera: THREE.PerspectiveCamera - controls: OrbitControls - defaultCamera: DefaultCameraState - defaultControls: DefaultControlsState - activeCameraRef: MutableRefObject -}) => { - activeCameraRef.current = null - controls.enabled = true - controls.dampingFactor = defaultControls.dampingFactor - controls.rotateSpeed = defaultControls.rotateSpeed - controls.zoomSpeed = defaultControls.zoomSpeed - controls.panSpeed = defaultControls.panSpeed - camera.fov = defaultCamera.fov - camera.near = defaultCamera.near - camera.far = defaultCamera.far - camera.updateProjectionMatrix() -} - -const applyMetadataCamera = ({ - mesh, - cameraMetadata, - camera, - controls, - container, - defaultCamera, - defaultControls, - activeCameraRef, -}: { - mesh: SplatMesh - cameraMetadata: CameraMetadata - camera: THREE.PerspectiveCamera - controls: OrbitControls - container: HTMLElement - defaultCamera: DefaultCameraState - defaultControls: DefaultControlsState - activeCameraRef: MutableRefObject -}) => { - const meshObject = mesh as unknown as THREE.Object3D & { userData: Record } - const cvToThree = makeAxisFlipCvToGl() - if (!meshObject.userData.__cvToThreeApplied) { - meshObject.applyMatrix4(cvToThree) - meshObject.userData.__cvToThreeApplied = true - } - meshObject.updateMatrixWorld(true) - - const e = cameraMetadata.extrinsicCv - const extrinsicCv = new THREE.Matrix4().set( - e[0], - e[1], - e[2], - e[3], - e[4], - e[5], - e[6], - e[7], - e[8], - e[9], - e[10], - e[11], - e[12], - e[13], - e[14], - e[15], - ) - - const view = new THREE.Matrix4().multiplyMatrices(cvToThree, extrinsicCv).multiply(cvToThree) - const cameraWorld = new THREE.Matrix4().copy(view).invert() - - camera.matrixAutoUpdate = true - camera.matrixWorld.copy(cameraWorld) - camera.matrixWorld.decompose(camera.position, camera.quaternion, camera.scale) - camera.updateMatrix() - camera.updateMatrixWorld(true) - - let { near } = defaultCamera - let { far } = defaultCamera - - if (mesh?.getBoundingBox) { - const box = mesh.getBoundingBox() - const worldBox = box.clone().applyMatrix4(meshObject.matrixWorld) - const size = new THREE.Vector3() - const center = new THREE.Vector3() - worldBox.getSize(size) - worldBox.getCenter(center) - const radius = Math.max(size.length() * 0.5, 0.25) - - const camPos = camera.position.clone() - const dist = camPos.distanceTo(center) - - near = Math.max(0.01, dist - radius * 2) - far = Math.max(near + 1, dist + radius * 6) - - const depthFocusCv = computeMlSharpDepthFocus(mesh) - const lookAtCv = new THREE.Vector3(0, 0, depthFocusCv) - const lookAtThree = lookAtCv.applyMatrix4(meshObject.matrixWorld) - controls.target.copy(lookAtThree) - } - - activeCameraRef.current = { ...cameraMetadata, near, far } - - applyCameraProjection({ - cameraMetadata: activeCameraRef.current, - camera, - controls, - viewportWidth: container.clientWidth || 1, - viewportHeight: container.clientHeight || 1, - defaultCamera, - defaultControls, - }) - - controls.enabled = true - controls.update() -} - -const fitViewToMesh = ({ - mesh, - camera, - controls, -}: { - mesh: SplatMesh - camera: THREE.PerspectiveCamera - controls: OrbitControls -}) => { - if (!mesh.getBoundingBox) return - const box = mesh.getBoundingBox() - const size = new THREE.Vector3() - const center = new THREE.Vector3() - box.getSize(size) - box.getCenter(center) - - const radius = Math.max(size.length() * 0.5, 0.5) - const dist = radius / Math.tan((camera.fov * Math.PI) / 360) - - camera.position.copy(center).add(new THREE.Vector3(dist, dist, dist)) - camera.near = Math.max(0.01, radius * 0.01) - camera.far = Math.max(dist * 4, radius * 8) - camera.updateProjectionMatrix() - - controls.target.copy(center) - controls.update() -} - // --- Viewer component --- interface ThreeDSceneViewerProps { @@ -625,10 +64,12 @@ export const ThreeDSceneViewer = ({ const activeCameraRef = useRef(null) const resizeRef = useRef<(() => void) | null>(null) const resizeObserverRef = useRef(null) - const parallaxRef = useRef(null) - const orientationZeroRef = useRef<{ beta: number; gamma: number } | null>(null) - const orientationPermissionRef = useRef<'unknown' | 'granted' | 'denied'>('unknown') - const tempPositionRef = useRef(new THREE.Vector3()) + + const { applyParallaxFrame, updateParallaxState } = useThreeDSceneParallax({ + containerRef, + cameraRef, + controlsRef, + }) const [canvasBounds, setCanvasBounds] = useState<{ width: number; height: number } | null>(null) const [isReady, setIsReady] = useState(false) @@ -667,163 +108,6 @@ export const ThreeDSceneViewer = ({ updateBounds() }, [updateBounds]) - const updateParallaxState = useCallback((mesh?: SplatMesh) => { - const camera = cameraRef.current - const controls = controlsRef.current - if (!camera || !controls) return - - const basePosition = camera.position.clone() - const baseTarget = controls.target.clone() - const baseQuaternion = camera.quaternion.clone() - const baseRight = new THREE.Vector3(1, 0, 0).applyQuaternion(baseQuaternion) - const baseUp = new THREE.Vector3(0, 1, 0).applyQuaternion(baseQuaternion) - - let radius = basePosition.distanceTo(baseTarget) - if (mesh?.getBoundingBox) { - const meshObject = mesh as unknown as THREE.Object3D - meshObject.updateMatrixWorld(true) - const box = mesh.getBoundingBox() - const worldBox = box.clone().applyMatrix4(meshObject.matrixWorld) - const size = new THREE.Vector3() - worldBox.getSize(size) - radius = Math.max(size.length() * 0.5, 0.25) - } - - const distance = basePosition.distanceTo(baseTarget) - const baseStrength = Math.max(0.003, Math.min(distance * 0.04, radius * 0.12)) - const strength = baseStrength * (isMobileDevice ? PARALLAX_STRENGTH_MOBILE_MULTIPLIER : 1) - - parallaxRef.current = { - basePosition, - baseTarget, - baseRight, - baseUp, - strength, - current: new THREE.Vector2(0, 0), - target: new THREE.Vector2(0, 0), - } - }, []) - - const applyParallaxInput = useCallback((rawX: number, rawY: number, deadzone: number) => { - const parallax = parallaxRef.current - if (!parallax) return - const clampValue = (value: number) => THREE.MathUtils.clamp(value, -1, 1) - const applyDeadzone = (value: number) => { - const abs = Math.abs(value) - if (abs <= deadzone) return 0 - const scaled = (abs - deadzone) / Math.max(1 - deadzone, 1e-6) - return Math.sign(value) * scaled - } - const x = applyDeadzone(clampValue(rawX)) - const y = applyDeadzone(clampValue(rawY)) - parallax.target.set(x, y) - }, []) - - const resetParallaxTarget = useCallback(() => { - parallaxRef.current?.target.set(0, 0) - }, []) - - useEffect(() => { - if (isMobileDevice) return - const container = containerRef.current - if (!container) return - - const handleMove = (event: MouseEvent) => { - const rect = container.getBoundingClientRect() - if (!rect.width || !rect.height) return - const x = (event.clientX - rect.left) / rect.width - const y = (event.clientY - rect.top) / rect.height - const nx = (x - 0.5) * 2 - const ny = (0.5 - y) * 2 - applyParallaxInput(nx, ny, PARALLAX_DEADZONE_MOUSE) - } - - const handleLeave = () => { - resetParallaxTarget() - } - - container.addEventListener('mousemove', handleMove) - container.addEventListener('mouseleave', handleLeave) - - return () => { - container.removeEventListener('mousemove', handleMove) - container.removeEventListener('mouseleave', handleLeave) - } - }, [applyParallaxInput, resetParallaxTarget]) - - useEffect(() => { - if (!isMobileDevice) return - if (typeof window === 'undefined' || typeof DeviceOrientationEvent === 'undefined') return - const container = containerRef.current - if (!container) return - - let subscribed = false - const handleOrientation = (event: DeviceOrientationEvent) => { - if (event.beta == null || event.gamma == null) return - if (!orientationZeroRef.current) { - orientationZeroRef.current = { beta: event.beta, gamma: event.gamma } - } - const base = orientationZeroRef.current - const gamma = event.gamma - base.gamma - const beta = event.beta - base.beta - const nx = THREE.MathUtils.clamp(gamma / ORIENTATION_RANGE.gamma, -1, 1) - const ny = THREE.MathUtils.clamp(-beta / ORIENTATION_RANGE.beta, -1, 1) - applyParallaxInput(nx, ny, PARALLAX_DEADZONE_MOTION) - } - - const subscribe = () => { - if (subscribed) return - window.addEventListener('deviceorientation', handleOrientation, true) - subscribed = true - } - - const unsubscribe = () => { - if (!subscribed) return - window.removeEventListener('deviceorientation', handleOrientation, true) - subscribed = false - } - - const { requestPermission } = DeviceOrientationEvent as typeof DeviceOrientationEvent & { - requestPermission?: () => Promise<'granted' | 'denied'> - } - - const enableOrientation = async () => { - if (orientationPermissionRef.current === 'granted') return - if (typeof requestPermission !== 'function') { - orientationPermissionRef.current = 'granted' - subscribe() - return - } - try { - const result = await requestPermission() - orientationPermissionRef.current = result === 'granted' ? 'granted' : 'denied' - if (result === 'granted') { - orientationZeroRef.current = null - subscribe() - } - } catch { - orientationPermissionRef.current = 'denied' - } - } - - if (typeof requestPermission !== 'function') { - orientationPermissionRef.current = 'granted' - subscribe() - } - - const handlePointerDown = () => { - void enableOrientation() - } - container.addEventListener('pointerdown', handlePointerDown) - - return () => { - container.removeEventListener('pointerdown', handlePointerDown) - unsubscribe() - orientationZeroRef.current = null - resetParallaxTarget() - } - }, [applyParallaxInput, resetParallaxTarget]) - // Initialize Three + Spark renderer once useEffect(() => { const container = containerRef.current @@ -919,16 +203,7 @@ export const ThreeDSceneViewer = ({ } const animate = () => { - const parallax = parallaxRef.current - if (parallax) { - parallax.current.lerp(parallax.target, 0.08) - const tempPosition = tempPositionRef.current - tempPosition.copy(parallax.basePosition) - tempPosition.addScaledVector(parallax.baseRight, parallax.current.x * parallax.strength) - tempPosition.addScaledVector(parallax.baseUp, parallax.current.y * parallax.strength) - camera.position.copy(tempPosition) - camera.lookAt(parallax.baseTarget) - } + applyParallaxFrame() controls.update() renderer.render(sceneObj, camera) frameRef.current = requestAnimationFrame(animate) diff --git a/apps/web/src/modules/media/threeDSceneCamera.ts b/apps/web/src/modules/media/threeDSceneCamera.ts new file mode 100644 index 00000000..63d4d4d4 --- /dev/null +++ b/apps/web/src/modules/media/threeDSceneCamera.ts @@ -0,0 +1,304 @@ +import type { SplatMesh } from '@sparkjsdev/spark' +import type { MutableRefObject } from 'react' +import * as THREE from 'three' +import type { OrbitControls } from 'three/examples/jsm/controls/OrbitControls.js' + +import type { CameraMetadata } from './threeDSceneMetadata' + +export type ActiveCameraState = CameraMetadata & { + near: number + far: number +} + +export type DefaultCameraState = { + fov: number + near: number + far: number +} + +export type DefaultControlsState = { + dampingFactor: number + rotateSpeed: number + zoomSpeed: number + panSpeed: number +} + +const makeAxisFlipCvToGl = () => new THREE.Matrix4().set(1, 0, 0, 0, 0, -1, 0, 0, 0, 0, -1, 0, 0, 0, 0, 1) + +const quantileSorted = (sorted: number[], q: number) => { + if (sorted.length === 0) return null + const clampedQ = Math.max(0, Math.min(1, q)) + const pos = (sorted.length - 1) * clampedQ + const lower = Math.floor(pos) + const upper = Math.ceil(pos) + if (lower === upper) return sorted[lower] + const weight = pos - lower + return sorted[lower] * (1 - weight) + sorted[upper] * weight +} + +const computeMlSharpDepthFocus = (mesh: SplatMesh, { qFocus = 0.1, minDepthFocus = 2, maxSamples = 50_000 } = {}) => { + const numSplats = mesh?.packedSplats?.numSplats ?? 0 + if (!numSplats) return minDepthFocus + + const step = Math.max(1, Math.floor(numSplats / maxSamples)) + const depths: number[] = [] + for (let i = 0; i < numSplats; i += step) { + const { center } = mesh.packedSplats.getSplat(i) + const { z } = center + if (Number.isFinite(z) && z > 0) depths.push(z) + } + + if (depths.length === 0) return minDepthFocus + depths.sort((a, b) => a - b) + const q = quantileSorted(depths, qFocus) + if (!Number.isFinite(q)) return minDepthFocus + return Math.max(minDepthFocus, q!) +} + +const makeProjectionFromIntrinsics = ({ + fx, + fy, + cx, + cy, + width, + height, + near, + far, +}: { + fx: number + fy: number + cx: number + cy: number + width: number + height: number + near: number + far: number +}) => { + const left = (-cx * near) / fx + const right = ((width - cx) * near) / fx + const top = (cy * near) / fy + const bottom = (-(height - cy) * near) / fy + + return new THREE.Matrix4().set( + (2 * near) / (right - left), + 0, + (right + left) / (right - left), + 0, + 0, + (2 * near) / (top - bottom), + (top + bottom) / (top - bottom), + 0, + 0, + 0, + -(far + near) / (far - near), + (-2 * far * near) / (far - near), + 0, + 0, + -1, + 0, + ) +} + +export const applyCameraProjection = ({ + cameraMetadata, + camera, + controls, + viewportWidth, + viewportHeight, + defaultCamera, + defaultControls, +}: { + cameraMetadata: ActiveCameraState + camera: THREE.PerspectiveCamera + controls: OrbitControls + viewportWidth: number + viewportHeight: number + defaultCamera: DefaultCameraState + defaultControls: DefaultControlsState +}) => { + const { intrinsics, near, far } = cameraMetadata + const sx = viewportWidth / intrinsics.imageWidth + const sy = viewportHeight / intrinsics.imageHeight + const s = Math.min(sx, sy) + const scaledWidth = intrinsics.imageWidth * s + const scaledHeight = intrinsics.imageHeight * s + const offsetX = (viewportWidth - scaledWidth) * 0.5 + const offsetY = (viewportHeight - scaledHeight) * 0.5 + + const fx = intrinsics.fx * s + const fy = intrinsics.fy * s + const cx = intrinsics.cx * s + offsetX + const cy = intrinsics.cy * s + offsetY + + camera.aspect = viewportWidth / Math.max(1, viewportHeight) + camera.fov = THREE.MathUtils.radToDeg(2 * Math.atan(viewportHeight / (2 * Math.max(1e-6, fy)))) + camera.near = near + camera.far = far + + const fovScale = THREE.MathUtils.clamp(camera.fov / defaultCamera.fov, 0.05, 2) + controls.rotateSpeed = Math.max(0.02, defaultControls.rotateSpeed * fovScale * 0.45) + controls.zoomSpeed = Math.max(0.05, defaultControls.zoomSpeed * fovScale * 0.8) + controls.panSpeed = Math.max(0.05, defaultControls.panSpeed * fovScale * 0.8) + + const projection = makeProjectionFromIntrinsics({ + fx, + fy, + cx, + cy, + width: viewportWidth, + height: viewportHeight, + near, + far, + }) + camera.projectionMatrix.copy(projection) + camera.projectionMatrixInverse.copy(projection).invert() +} + +export const clearMetadataCamera = ({ + camera, + controls, + defaultCamera, + defaultControls, + activeCameraRef, +}: { + camera: THREE.PerspectiveCamera + controls: OrbitControls + defaultCamera: DefaultCameraState + defaultControls: DefaultControlsState + activeCameraRef: MutableRefObject +}) => { + activeCameraRef.current = null + controls.enabled = true + controls.dampingFactor = defaultControls.dampingFactor + controls.rotateSpeed = defaultControls.rotateSpeed + controls.zoomSpeed = defaultControls.zoomSpeed + controls.panSpeed = defaultControls.panSpeed + camera.fov = defaultCamera.fov + camera.near = defaultCamera.near + camera.far = defaultCamera.far + camera.updateProjectionMatrix() +} + +export const applyMetadataCamera = ({ + mesh, + cameraMetadata, + camera, + controls, + container, + defaultCamera, + defaultControls, + activeCameraRef, +}: { + mesh: SplatMesh + cameraMetadata: CameraMetadata + camera: THREE.PerspectiveCamera + controls: OrbitControls + container: HTMLElement + defaultCamera: DefaultCameraState + defaultControls: DefaultControlsState + activeCameraRef: MutableRefObject +}) => { + const meshObject = mesh as unknown as THREE.Object3D & { userData: Record } + const cvToThree = makeAxisFlipCvToGl() + if (!meshObject.userData.__cvToThreeApplied) { + meshObject.applyMatrix4(cvToThree) + meshObject.userData.__cvToThreeApplied = true + } + meshObject.updateMatrixWorld(true) + + const e = cameraMetadata.extrinsicCv + const extrinsicCv = new THREE.Matrix4().set( + e[0], + e[1], + e[2], + e[3], + e[4], + e[5], + e[6], + e[7], + e[8], + e[9], + e[10], + e[11], + e[12], + e[13], + e[14], + e[15], + ) + + const view = new THREE.Matrix4().multiplyMatrices(cvToThree, extrinsicCv).multiply(cvToThree) + const cameraWorld = new THREE.Matrix4().copy(view).invert() + + camera.matrixAutoUpdate = true + camera.matrixWorld.copy(cameraWorld) + camera.matrixWorld.decompose(camera.position, camera.quaternion, camera.scale) + camera.updateMatrix() + camera.updateMatrixWorld(true) + + let { near } = defaultCamera + let { far } = defaultCamera + + if (mesh?.getBoundingBox) { + const box = mesh.getBoundingBox() + const worldBox = box.clone().applyMatrix4(meshObject.matrixWorld) + const size = new THREE.Vector3() + const center = new THREE.Vector3() + worldBox.getSize(size) + worldBox.getCenter(center) + const radius = Math.max(size.length() * 0.5, 0.25) + + const camPos = camera.position.clone() + const dist = camPos.distanceTo(center) + + near = Math.max(0.01, dist - radius * 2) + far = Math.max(near + 1, dist + radius * 6) + + const depthFocusCv = computeMlSharpDepthFocus(mesh) + const lookAtCv = new THREE.Vector3(0, 0, depthFocusCv) + const lookAtThree = lookAtCv.applyMatrix4(meshObject.matrixWorld) + controls.target.copy(lookAtThree) + } + + activeCameraRef.current = { ...cameraMetadata, near, far } + + applyCameraProjection({ + cameraMetadata: activeCameraRef.current, + camera, + controls, + viewportWidth: container.clientWidth || 1, + viewportHeight: container.clientHeight || 1, + defaultCamera, + defaultControls, + }) + + controls.enabled = true + controls.update() +} + +export const fitViewToMesh = ({ + mesh, + camera, + controls, +}: { + mesh: SplatMesh + camera: THREE.PerspectiveCamera + controls: OrbitControls +}) => { + if (!mesh.getBoundingBox) return + const box = mesh.getBoundingBox() + const size = new THREE.Vector3() + const center = new THREE.Vector3() + box.getSize(size) + box.getCenter(center) + + const radius = Math.max(size.length() * 0.5, 0.5) + const dist = radius / Math.tan((camera.fov * Math.PI) / 360) + + camera.position.copy(center).add(new THREE.Vector3(dist, dist, dist)) + camera.near = Math.max(0.01, radius * 0.01) + camera.far = Math.max(dist * 4, radius * 8) + camera.updateProjectionMatrix() + + controls.target.copy(center) + controls.update() +} diff --git a/apps/web/src/modules/media/threeDSceneMetadata.ts b/apps/web/src/modules/media/threeDSceneMetadata.ts new file mode 100644 index 00000000..4d0bfe1c --- /dev/null +++ b/apps/web/src/modules/media/threeDSceneMetadata.ts @@ -0,0 +1,248 @@ +type CameraIntrinsics = { + fx: number + fy: number + cx: number + cy: number + imageWidth: number + imageHeight: number +} + +export type CameraMetadata = { + intrinsics: CameraIntrinsics + extrinsicCv: number[] + colorSpaceIndex?: number + headerComments?: string[] +} + +const toExtrinsic4x4RowMajor = (raw?: number[] | Float32Array | null): number[] => { + if (!raw) { + return [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1] + } + + const values = Array.from(raw) + + if (values.length === 16) return values + + if (values.length === 12) { + const m = [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1] + + m[0] = values[0] + m[1] = values[1] + m[2] = values[2] + m[3] = values[3] + + m[4] = values[4] + m[5] = values[5] + m[6] = values[6] + m[7] = values[7] + + m[8] = values[8] + m[9] = values[9] + m[10] = values[10] + m[11] = values[11] + + const r00 = m[0] + const r01 = m[1] + const r02 = m[2] + const r10 = m[4] + const r11 = m[5] + const r12 = m[6] + const r20 = m[8] + const r21 = m[9] + const r22 = m[10] + + m[0] = r00 + m[1] = r10 + m[2] = r20 + m[4] = r01 + m[5] = r11 + m[6] = r21 + m[8] = r02 + m[9] = r12 + m[10] = r22 + + return m + } + + throw new Error(`Unrecognized extrinsic element length: ${values.length}`) +} + +const parseIntrinsics = (raw: number[] | Float32Array | undefined, imageWidth?: number, imageHeight?: number) => { + if (!raw) return null + const values = Array.from(raw) + + if (values.length === 9) { + if (!(typeof imageWidth === 'number' && Number.isFinite(imageWidth))) return null + if (!(typeof imageHeight === 'number' && Number.isFinite(imageHeight))) return null + const width = imageWidth + const height = imageHeight + return { + fx: values[0], + fy: values[4], + cx: values[2], + cy: values[5], + imageWidth: width, + imageHeight: height, + } + } + + if (values.length === 16) { + if (!(typeof imageWidth === 'number' && Number.isFinite(imageWidth))) return null + if (!(typeof imageHeight === 'number' && Number.isFinite(imageHeight))) return null + const width = imageWidth + const height = imageHeight + return { + fx: values[0], + fy: values[5], + cx: values[2], + cy: values[6], + imageWidth: width, + imageHeight: height, + } + } + + if (values.length === 4) { + const legacyWidth = Number.parseInt(`${values[2]}`) + const legacyHeight = Number.parseInt(`${values[3]}`) + const width = typeof imageWidth === 'number' && Number.isFinite(imageWidth) ? imageWidth : legacyWidth + const height = typeof imageHeight === 'number' && Number.isFinite(imageHeight) ? imageHeight : legacyHeight + if (!Number.isFinite(width) || !Number.isFinite(height)) return null + return { + fx: values[0], + fy: values[1], + cx: (width - 1) * 0.5, + cy: (height - 1) * 0.5, + imageWidth: width, + imageHeight: height, + } + } + + return null +} + +const normalizeColorSpaceIndex = (value: unknown) => { + if (Array.isArray(value)) { + return Number.isFinite(value[0]) ? Number(value[0]) : undefined + } + return Number.isFinite(value as number) ? Number(value) : undefined +} + +const buildCameraMetadata = (raw: Record = {}): CameraMetadata | null => { + const imageSize = raw.image_size + const imageWidth = Array.isArray(imageSize) ? imageSize[0] : undefined + const imageHeight = Array.isArray(imageSize) ? imageSize[1] : undefined + const intrinsics = parseIntrinsics(raw.intrinsic, imageWidth, imageHeight) + if (!intrinsics) return null + + return { + intrinsics, + extrinsicCv: toExtrinsic4x4RowMajor(raw.extrinsic), + colorSpaceIndex: normalizeColorSpaceIndex(raw.color_space), + headerComments: raw.headerComments ?? [], + } +} + +const ZIP_LOCAL_FILE_HEADER = 0x04034b50 +const ZIP_CENTRAL_DIR_HEADER = 0x02014b50 +const ZIP_END_OF_CENTRAL_DIR = 0x06054b50 +const ZIP_END_OF_CENTRAL_DIR_MIN_SIZE = 22 +const ZIP_MAX_COMMENT_SIZE = 0xffff + +const textDecoder = new TextDecoder('utf-8') + +const findZipEndOfCentralDir = (view: DataView, length: number) => { + const start = Math.max(0, length - (ZIP_END_OF_CENTRAL_DIR_MIN_SIZE + ZIP_MAX_COMMENT_SIZE)) + for (let offset = length - ZIP_END_OF_CENTRAL_DIR_MIN_SIZE; offset >= start; offset -= 1) { + if (view.getUint32(offset, true) === ZIP_END_OF_CENTRAL_DIR) return offset + } + return -1 +} + +const findZipEntry = (bytes: Uint8Array, targetName: string) => { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) + const length = bytes.byteLength + const eocdOffset = findZipEndOfCentralDir(view, length) + if (eocdOffset < 0) return null + + const centralDirSize = view.getUint32(eocdOffset + 12, true) + const centralDirOffset = view.getUint32(eocdOffset + 16, true) + const centralDirEnd = centralDirOffset + centralDirSize + if (centralDirEnd > length) return null + + let offset = centralDirOffset + while (offset + 46 <= centralDirEnd) { + if (view.getUint32(offset, true) !== ZIP_CENTRAL_DIR_HEADER) break + + const compression = view.getUint16(offset + 10, true) + const compressedSize = view.getUint32(offset + 20, true) + const nameLength = view.getUint16(offset + 28, true) + const extraLength = view.getUint16(offset + 30, true) + const commentLength = view.getUint16(offset + 32, true) + const localHeaderOffset = view.getUint32(offset + 42, true) + + const nameStart = offset + 46 + const nameEnd = nameStart + nameLength + const name = textDecoder.decode(bytes.subarray(nameStart, nameEnd)) + const entryName = name.split(/[\\/]/).pop()?.toLowerCase() + + if (entryName === targetName) { + return { + compression, + compressedSize, + localHeaderOffset, + } + } + + offset = nameEnd + extraLength + commentLength + } + + return null +} + +const inflateRaw = async (data: Uint8Array) => { + if (typeof DecompressionStream === 'undefined') { + throw new TypeError('DecompressionStream is not available in this browser') + } + const slice = data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer + const stream = new Blob([slice]).stream().pipeThrough(new DecompressionStream('deflate-raw')) + const buffer = await new Response(stream).arrayBuffer() + return new Uint8Array(buffer) +} + +const readZipEntryData = async ( + bytes: Uint8Array, + entry: { compression: number; compressedSize: number; localHeaderOffset: number }, +) => { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) + const headerOffset = entry.localHeaderOffset + if (headerOffset + 30 > bytes.byteLength) return null + if (view.getUint32(headerOffset, true) !== ZIP_LOCAL_FILE_HEADER) return null + + const nameLength = view.getUint16(headerOffset + 26, true) + const extraLength = view.getUint16(headerOffset + 28, true) + const dataStart = headerOffset + 30 + nameLength + extraLength + const dataEnd = dataStart + entry.compressedSize + if (dataEnd > bytes.byteLength) return null + + const compressed = bytes.subarray(dataStart, dataEnd) + if (entry.compression === 0) return compressed + if (entry.compression === 8) return inflateRaw(compressed) + + throw new Error(`Unsupported zip compression method: ${entry.compression}`) +} + +const readSogMetaJson = async (bytes: Uint8Array) => { + const entry = findZipEntry(bytes, 'meta.json') + if (!entry) return null + const data = await readZipEntryData(bytes, entry) + if (!data) return null + return JSON.parse(textDecoder.decode(data)) +} + +export const readSogMetadata = async (bytes: Uint8Array) => { + const meta = await readSogMetaJson(bytes) + if (!meta || typeof meta !== 'object') return null + const sharpMetadata = meta.sharp_metadata + if (!sharpMetadata || typeof sharpMetadata !== 'object') return null + return buildCameraMetadata(sharpMetadata) +} diff --git a/apps/web/src/modules/media/useThreeDSceneParallax.ts b/apps/web/src/modules/media/useThreeDSceneParallax.ts new file mode 100644 index 00000000..36787e05 --- /dev/null +++ b/apps/web/src/modules/media/useThreeDSceneParallax.ts @@ -0,0 +1,213 @@ +import type { SplatMesh } from '@sparkjsdev/spark' +import type { RefObject } from 'react' +import { useCallback, useEffect, useRef } from 'react' +import * as THREE from 'three' +import type { OrbitControls } from 'three/examples/jsm/controls/OrbitControls.js' + +import { isMobileDevice } from '~/lib/device-viewport' + +type ParallaxState = { + basePosition: THREE.Vector3 + baseTarget: THREE.Vector3 + baseRight: THREE.Vector3 + baseUp: THREE.Vector3 + strength: number + current: THREE.Vector2 + target: THREE.Vector2 +} + +const PARALLAX_DEADZONE_MOUSE = 0.06 +const PARALLAX_DEADZONE_MOTION = 0.12 +const PARALLAX_STRENGTH_MOBILE_MULTIPLIER = 1.5 +const ORIENTATION_RANGE = { + gamma: 30, + beta: 30, +} + +type UseThreeDSceneParallaxParams = { + containerRef: RefObject + cameraRef: RefObject + controlsRef: RefObject +} + +export const useThreeDSceneParallax = ({ containerRef, cameraRef, controlsRef }: UseThreeDSceneParallaxParams) => { + const parallaxRef = useRef(null) + const orientationZeroRef = useRef<{ beta: number; gamma: number } | null>(null) + const orientationPermissionRef = useRef<'unknown' | 'granted' | 'denied'>('unknown') + const tempPositionRef = useRef(new THREE.Vector3()) + + const updateParallaxState = useCallback((mesh?: SplatMesh) => { + const camera = cameraRef.current + const controls = controlsRef.current + if (!camera || !controls) return + + const basePosition = camera.position.clone() + const baseTarget = controls.target.clone() + const baseQuaternion = camera.quaternion.clone() + const baseRight = new THREE.Vector3(1, 0, 0).applyQuaternion(baseQuaternion) + const baseUp = new THREE.Vector3(0, 1, 0).applyQuaternion(baseQuaternion) + + let radius = basePosition.distanceTo(baseTarget) + if (mesh?.getBoundingBox) { + const meshObject = mesh as unknown as THREE.Object3D + meshObject.updateMatrixWorld(true) + const box = mesh.getBoundingBox() + const worldBox = box.clone().applyMatrix4(meshObject.matrixWorld) + const size = new THREE.Vector3() + worldBox.getSize(size) + radius = Math.max(size.length() * 0.5, 0.25) + } + + const distance = basePosition.distanceTo(baseTarget) + const baseStrength = Math.max(0.003, Math.min(distance * 0.04, radius * 0.12)) + const strength = baseStrength * (isMobileDevice ? PARALLAX_STRENGTH_MOBILE_MULTIPLIER : 1) + + parallaxRef.current = { + basePosition, + baseTarget, + baseRight, + baseUp, + strength, + current: new THREE.Vector2(0, 0), + target: new THREE.Vector2(0, 0), + } + }, []) + + const applyParallaxInput = useCallback((rawX: number, rawY: number, deadzone: number) => { + const parallax = parallaxRef.current + if (!parallax) return + const clampValue = (value: number) => THREE.MathUtils.clamp(value, -1, 1) + const applyDeadzone = (value: number) => { + const abs = Math.abs(value) + if (abs <= deadzone) return 0 + const scaled = (abs - deadzone) / Math.max(1 - deadzone, 1e-6) + return Math.sign(value) * scaled + } + const x = applyDeadzone(clampValue(rawX)) + const y = applyDeadzone(clampValue(rawY)) + parallax.target.set(x, y) + }, []) + + const resetParallaxTarget = useCallback(() => { + parallaxRef.current?.target.set(0, 0) + }, []) + + useEffect(() => { + if (isMobileDevice) return + const container = containerRef.current + if (!container) return + + const handleMove = (event: MouseEvent) => { + const rect = container.getBoundingClientRect() + if (!rect.width || !rect.height) return + const x = (event.clientX - rect.left) / rect.width + const y = (event.clientY - rect.top) / rect.height + const nx = (x - 0.5) * 2 + const ny = (0.5 - y) * 2 + applyParallaxInput(nx, ny, PARALLAX_DEADZONE_MOUSE) + } + + const handleLeave = () => { + resetParallaxTarget() + } + + container.addEventListener('mousemove', handleMove) + container.addEventListener('mouseleave', handleLeave) + + return () => { + container.removeEventListener('mousemove', handleMove) + container.removeEventListener('mouseleave', handleLeave) + } + }, [applyParallaxInput, containerRef, resetParallaxTarget]) + + useEffect(() => { + if (!isMobileDevice) return + if (typeof window === 'undefined' || typeof DeviceOrientationEvent === 'undefined') return + const container = containerRef.current + if (!container) return + + let subscribed = false + const handleOrientation = (event: DeviceOrientationEvent) => { + if (event.beta == null || event.gamma == null) return + if (!orientationZeroRef.current) { + orientationZeroRef.current = { beta: event.beta, gamma: event.gamma } + } + const base = orientationZeroRef.current + const gamma = event.gamma - base.gamma + const beta = event.beta - base.beta + const nx = THREE.MathUtils.clamp(gamma / ORIENTATION_RANGE.gamma, -1, 1) + const ny = THREE.MathUtils.clamp(-beta / ORIENTATION_RANGE.beta, -1, 1) + applyParallaxInput(nx, ny, PARALLAX_DEADZONE_MOTION) + } + + const subscribe = () => { + if (subscribed) return + window.addEventListener('deviceorientation', handleOrientation, true) + subscribed = true + } + + const unsubscribe = () => { + if (!subscribed) return + window.removeEventListener('deviceorientation', handleOrientation, true) + subscribed = false + } + + const { requestPermission } = DeviceOrientationEvent as typeof DeviceOrientationEvent & { + requestPermission?: () => Promise<'granted' | 'denied'> + } + + const enableOrientation = async () => { + if (orientationPermissionRef.current === 'granted') return + if (typeof requestPermission !== 'function') { + orientationPermissionRef.current = 'granted' + subscribe() + return + } + try { + const result = await requestPermission() + orientationPermissionRef.current = result === 'granted' ? 'granted' : 'denied' + if (result === 'granted') { + orientationZeroRef.current = null + subscribe() + } + } catch { + orientationPermissionRef.current = 'denied' + } + } + + if (typeof requestPermission !== 'function') { + orientationPermissionRef.current = 'granted' + subscribe() + } + + const handlePointerDown = () => { + void enableOrientation() + } + container.addEventListener('pointerdown', handlePointerDown) + + return () => { + container.removeEventListener('pointerdown', handlePointerDown) + unsubscribe() + orientationZeroRef.current = null + resetParallaxTarget() + } + }, [applyParallaxInput, containerRef, resetParallaxTarget]) + + const applyParallaxFrame = useCallback(() => { + const parallax = parallaxRef.current + const camera = cameraRef.current + if (!parallax || !camera) return + parallax.current.lerp(parallax.target, 0.08) + const tempPosition = tempPositionRef.current + tempPosition.copy(parallax.basePosition) + tempPosition.addScaledVector(parallax.baseRight, parallax.current.x * parallax.strength) + tempPosition.addScaledVector(parallax.baseUp, parallax.current.y * parallax.strength) + camera.position.copy(tempPosition) + camera.lookAt(parallax.baseTarget) + }, []) + + return { + updateParallaxState, + applyParallaxFrame, + } +}