feat: support hdr motion photo (#155)

apps/web/src/components/ui/photo-viewer/LivePhotoVideo.tsx

@@ -48,22 +48,11 @@ export const LivePhotoVideo = ({
 
   const videoRef = useRef<HTMLVideoElement>(null)
   const videoAnimateController = useAnimationControls()
-  const presentationTimestampRef = useRef<number | undefined>(undefined)
 
   useEffect(() => {
     onPlayingChange?.(isPlayingLivePhoto)
   }, [isPlayingLivePhoto, onPlayingChange])
 
-  // Extract and track presentationTimestamp for Motion Photo
-  useEffect(() => {
-    if (videoSource.type === 'motion-photo' && videoSource.presentationTimestamp) {
-      // Convert microseconds to seconds
-      presentationTimestampRef.current = videoSource.presentationTimestamp / 1_000_000
-    } else {
-      presentationTimestampRef.current = undefined
-    }
-  }, [videoSource])
-
   useEffect(() => {
     if (!isCurrentImage || livePhotoVideoLoaded || isConvertingVideo || !videoRef.current) {
       return
@@ -158,22 +147,6 @@ export const LivePhotoVideo = ({
     stop()
   }, [stop])
 
-  // Handle Motion Photo presentation timestamp
-  const handleTimeUpdate = useCallback(() => {
-    const video = videoRef.current
-    const timestamp = presentationTimestampRef.current
-
-    // Only handle Motion Photo with valid timestamp
-    if (!video || timestamp === undefined || videoSource.type !== 'motion-photo') {
-      return
-    }
-
-    // Stop playback when reaching or passing the presentation timestamp
-    if (video.currentTime >= timestamp) {
-      stop()
-    }
-  }, [videoSource, stop])
-
   return (
     <m.video
       ref={videoRef}
@@ -184,7 +157,6 @@ export const LivePhotoVideo = ({
       }}
       muted
       playsInline
-      onTimeUpdate={handleTimeUpdate}
       onEnded={handleVideoEnded}
       initial={{ opacity: 0 }}
       animate={videoAnimateController}

packages/builder/src/image/exif.ts

@@ -124,6 +124,16 @@ const pickKeys: Array<keyof Tags | (string & {})> = [
   'GPSLongitudeRef',
   // HDR相关字段
   'MPImageType',
+  'UniformResourceName',
+  // Motion Photo 相关字段
+  'MotionPhoto',
+  'MotionPhotoVersion',
+  'MotionPhotoPresentationTimestampUs',
+  'ContainerDirectory',
+  'MicroVideo',
+  'MicroVideoVersion',
+  'MicroVideoOffset',
+  'MicroVideoPresentationTimestampUs',
 ]
 function handleExifData(exifData: Tags, metadata: Metadata): PickedExif {
   const date = {

packages/builder/src/photo/gainmap-detector.ts

@@ -0,0 +1,36 @@
+import type { ConsolaInstance } from 'consola'
+import type { ContainerDirectoryItem } from 'exiftool-vendored'
+
+interface GainMapDetectParams {
+  exifData?: Record<string, unknown> | null
+  logger?: ConsolaInstance
+}
+
+/**
+ * Detects HDR GainMap from ContainerDirectory metadata.
+ * Works for both static Ultra HDR images and Motion Photo + HDR combinations.
+ */
+export const detectGainMap = ({ exifData, logger }: GainMapDetectParams): boolean => {
+  try {
+    const containerDirectory = exifData?.ContainerDirectory as ContainerDirectoryItem[] | undefined
+    if (!containerDirectory || !Array.isArray(containerDirectory)) {
+      return false
+    }
+
+    // Find GainMap item
+    for (const entry of containerDirectory) {
+      const item = entry.Item
+      if (!item) continue
+
+      if (item.Semantic === 'GainMap' && item.Length) {
+        logger?.info('[gainmap] Found HDR GainMap in ContainerDirectory')
+        return true
+      }
+    }
+
+    return false
+  } catch (error) {
+    logger?.error('[gainmap] Unexpected error while detecting', error)
+    return false
+  }
+}

packages/builder/src/photo/image-pipeline.ts

@@ -18,6 +18,7 @@ import type { S3ObjectLike } from '../types/s3.js'
 import { shouldProcessPhoto } from './cache-manager.js'
 import { processExifData, processThumbnailAndBlurhash, processToneAnalysis } from './data-processors.js'
 import { getPhotoExecutionContext } from './execution-context.js'
+import { detectGainMap } from './gainmap-detector.js'
 import { extractPhotoInfo } from './info-extractor.js'
 import { processLivePhoto } from './live-photo-handler.js'
 import { getGlobalLoggers } from './logger-adapter.js'
@@ -178,13 +179,18 @@ export async function executePhotoProcessingPipeline(
     // 4. 处理 EXIF 数据
     const exifData = await processExifData(imageBuffer, imageData.rawBuffer, photoKey, existingItem, options)
 
-    // 5. 检测 Motion Photo（从图片中提取嵌入视频的元数据）
+    // 5. 检测 HDR GainMap（Ultra HDR 图片）
+    const hasGainMap = detectGainMap({
+      exifData: exifData as Record<string, unknown> | null,
+    })
+
+    // 6. 检测 Motion Photo（从图片中提取嵌入视频的元数据）
     const motionPhotoMetadata = detectMotionPhoto({
       rawImageBuffer: imageData.rawBuffer,
       exifData: exifData as Record<string, unknown> | null,
     })
 
-    // 6. 处理 Live Photo（独立的视频文件）
+    // 7. 处理 Live Photo（独立的视频文件）
     const livePhotoResult = await processLivePhoto(photoKey, livePhotoMap, storageManager)
 
     // 检测冲突：不允许同时存在 Motion Photo 和 Live Photo
@@ -202,7 +208,6 @@ export async function executePhotoProcessingPipeline(
 
     // 9. 构建照片清单项
     const aspectRatio = metadata.width / metadata.height
-
     const photoItem: PhotoManifestItem = {
       id: photoId,
       title: photoInfo.title,
@@ -224,20 +229,23 @@ export async function executePhotoProcessingPipeline(
       video:
         motionPhotoMetadata?.isMotionPhoto && motionPhotoMetadata.motionPhotoOffset !== undefined
           ? {
-              type: 'motion-photo',
+            type: 'motion-photo',
-              offset: motionPhotoMetadata.motionPhotoOffset,
+            offset: motionPhotoMetadata.motionPhotoOffset,
-              size: motionPhotoMetadata.motionPhotoVideoSize,
+            size: motionPhotoMetadata.motionPhotoVideoSize,
-              presentationTimestamp: motionPhotoMetadata.presentationTimestampUs,
+            presentationTimestamp: motionPhotoMetadata.presentationTimestampUs,
-            }
+          }
           : livePhotoResult.isLivePhoto
             ? {
-                type: 'live-photo',
+              type: 'live-photo',
-                videoUrl: livePhotoResult.livePhotoVideoUrl!,
+              videoUrl: livePhotoResult.livePhotoVideoUrl!,
-                s3Key: livePhotoResult.livePhotoVideoS3Key!,
+              s3Key: livePhotoResult.livePhotoVideoS3Key!,
-              }
+            }
             : undefined,
       // HDR 相关字段
-      isHDR: exifData?.MPImageType === 'Gain Map Image',
+      isHDR:
+        exifData?.MPImageType === 'Gain Map Image' ||
+        exifData?.UniformResourceName === 'urn:iso:std:iso:ts:21496:-1' ||
+        hasGainMap,
     }
 
     loggers.image.success(`✅ 处理完成：${photoKey}`)

packages/builder/src/photo/motion-photo-detector.ts

@@ -1,4 +1,5 @@
 import type { ConsolaInstance } from 'consola'
+import type { ContainerDirectoryItem } from 'exiftool-vendored'
 
 export interface MotionPhotoMetadata {
   isMotionPhoto: boolean
@@ -13,7 +14,6 @@ interface MotionPhotoDetectParams {
   logger?: ConsolaInstance
 }
 
-const MAX_XMP_SCAN_BYTES = 512 * 1024 // 512KB should cover standard XMP blocks
 const MIN_VIDEO_SIZE_BYTES = 8 * 1024 // 8KB minimal sanity check
 const MP4_FTYP = Buffer.from('ftyp')
 
@@ -40,64 +40,6 @@ const toNumber = (value: unknown): number | null => {
   return null
 }
 
-const extractXmpSegment = (buffer: Buffer): string | null => {
-  const scanSize = Math.min(buffer.length, MAX_XMP_SCAN_BYTES)
-  if (scanSize === 0) {
-    return null
-  }
-
-  const header = buffer.toString('utf8', 0, scanSize)
-  const startIndex = header.indexOf('<x:xmpmeta')
-  if (startIndex === -1) {
-    return null
-  }
-
-  const endIndex = header.indexOf('</x:xmpmeta>')
-  if (endIndex === -1) {
-    return null
-  }
-
-  return header.slice(startIndex, endIndex + '</x:xmpmeta>'.length)
-}
-
-const extractXmpBoolean = (xmp: string, tagName: string): boolean | null => {
-  const regex = new RegExp(`<[^:>]*:${tagName}>([^<]+)</[^>]+>`, 'i')
-  const match = xmp.match(regex)
-  if (!match) return null
-  return toBoolean(match[1])
-}
-
-const extractXmpNumber = (xmp: string, tagName: string): number | null => {
-  const regex = new RegExp(`<[^:>]*:${tagName}>([^<]+)</[^>]+>`, 'i')
-  const match = xmp.match(regex)
-  if (!match) return null
-  return toNumber(match[1])
-}
-
-const escapeRegExp = (value: string) => value.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&')
-
-const buildAttrPattern = (attrName: string) => {
-  const escaped = escapeRegExp(attrName)
-  if (attrName.includes(':')) {
-    return escaped
-  }
-  return `(?:[\\w-]+:)?${escaped}`
-}
-
-const extractXmpAttributeBoolean = (xmp: string, attrName: string): boolean | null => {
-  const regex = new RegExp(`${buildAttrPattern(attrName)}="([^"]+)"`, 'i')
-  const match = xmp.match(regex)
-  if (!match) return null
-  return toBoolean(match[1])
-}
-
-const extractXmpAttributeNumber = (xmp: string, attrName: string): number | null => {
-  const regex = new RegExp(`${buildAttrPattern(attrName)}="([^"]+)"`, 'i')
-  const match = xmp.match(regex)
-  if (!match) return null
-  return toNumber(match[1])
-}
-
 const validateMp4Buffer = (buffer: Buffer): boolean => {
   if (buffer.length < MIN_VIDEO_SIZE_BYTES) {
     return false
@@ -109,8 +51,8 @@ const validateMp4Buffer = (buffer: Buffer): boolean => {
 }
 
 /**
- * Detects Motion Photo metadata from image buffer without extracting the video file.
+ * Detects Motion Photo metadata using Android Motion Photo format 1.0 specification.
- * Returns offset and size information for frontend to extract video on-demand.
+ * Supports both standard ContainerDirectory format and legacy MicroVideo format.
  */
 export const detectMotionPhoto = ({
   rawImageBuffer,
@@ -120,126 +62,79 @@ export const detectMotionPhoto = ({
   try {
     const rawLength = rawImageBuffer.length
 
-    const exifIndicatesMotion = toBoolean(exifData?.MotionPhoto) || toBoolean(exifData?.MicroVideo)
+    // Check Motion Photo flags (standard and legacy)
-    let detectedMotion = exifIndicatesMotion
+    const isMotionPhotoFlag = toBoolean(exifData?.MotionPhoto) || toBoolean(exifData?.MicroVideo)
 
-    let presentationTimestampUs = toNumber(
+    const presentationTimestampUs = toNumber(
       exifData?.MotionPhotoPresentationTimestampUs ?? exifData?.MicroVideoPresentationTimestampUs,
     )
 
-    const offsetCandidates = new Set<number>()
+    let videoOffset: number | null = null
-    const addOffsetCandidate = (value: number | null | undefined) => {
-      if (value === null || value === undefined) return
-      if (!Number.isFinite(value)) return
-      const numeric = Number(value)
-      if (numeric <= 0) return
-      offsetCandidates.add(numeric)
-    }
-
-    addOffsetCandidate(toNumber(exifData?.MicroVideoOffset))
-
-    const xmpSegment = extractXmpSegment(rawImageBuffer)
-    if (xmpSegment) {
-      if (!detectedMotion) {
-        const motionFlags = [
-          extractXmpBoolean(xmpSegment, 'MotionPhoto'),
-          extractXmpBoolean(xmpSegment, 'GCamera:MotionPhoto'),
-          extractXmpBoolean(xmpSegment, 'MicroVideo'),
-          extractXmpBoolean(xmpSegment, 'GCamera:MicroVideo'),
-          extractXmpAttributeBoolean(xmpSegment, 'MotionPhoto'),
-          extractXmpAttributeBoolean(xmpSegment, 'GCamera:MotionPhoto'),
-          extractXmpAttributeBoolean(xmpSegment, 'MicroVideo'),
-          extractXmpAttributeBoolean(xmpSegment, 'GCamera:MicroVideo'),
-        ].filter((flag) => flag !== null) as boolean[]
-
-        if (motionFlags.some(Boolean)) {
-          detectedMotion = true
-          logger?.info('[motion-photo] XMP detected MotionPhoto flags')
-        }
-      }
-
-      ;[
-        extractXmpNumber(xmpSegment, 'MicroVideoOffset'),
-        extractXmpNumber(xmpSegment, 'GCamera:MicroVideoOffset'),
-        extractXmpAttributeNumber(xmpSegment, 'MicroVideoOffset'),
-        extractXmpAttributeNumber(xmpSegment, 'GCamera:MicroVideoOffset'),
-      ].forEach((candidate) => addOffsetCandidate(candidate))
-
-      if (presentationTimestampUs === null) {
-        presentationTimestampUs =
-          extractXmpNumber(xmpSegment, 'MotionPhotoPresentationTimestampUs') ??
-          extractXmpNumber(xmpSegment, 'MicroVideoPresentationTimestampUs') ??
-          extractXmpAttributeNumber(xmpSegment, 'MotionPhotoPresentationTimestampUs') ??
-          extractXmpAttributeNumber(xmpSegment, 'MicroVideoPresentationTimestampUs') ??
-          null
-      }
-    }
-
-    if (!detectedMotion && offsetCandidates.size === 0) {
-      return null
-    }
-
-    let resolvedOffset: number | null = null
     let videoSize: number | null = null
 
-    const candidateList = Array.from(offsetCandidates)
+    // Try standard format (Motion Photo 1.0 with ContainerDirectory)
-    for (const candidate of candidateList) {
+    const containerDirectory = exifData?.ContainerDirectory as ContainerDirectoryItem[] | undefined
-      const possibleStarts = new Set<number>()
+    if (containerDirectory && Array.isArray(containerDirectory)) {
-      possibleStarts.add(candidate)
+      logger?.info('[motion-photo] Found ContainerDirectory, using standard format')
-      if (candidate < rawLength) {
-        possibleStarts.add(rawLength - candidate)
-      }
 
-      for (const start of possibleStarts) {
+      // Find video item
-        if (start <= 0 || start >= rawLength - MIN_VIDEO_SIZE_BYTES) {
+      for (const entry of containerDirectory) {
-          continue
+        const item = entry.Item
-        }
+        if (!item) continue
 
-        const chunk = rawImageBuffer.subarray(start)
+        if (item.Semantic === 'MotionPhoto' && item.Length) {
-        if (validateMp4Buffer(chunk)) {
+          // Video is stored at the end of file, Length bytes from the end
-          resolvedOffset = start
+          const offset = rawLength - item.Length
-          videoSize = chunk.length
+          if (offset > 0 && offset < rawLength - MIN_VIDEO_SIZE_BYTES) {
-          if (start !== candidate && logger?.debug) {
+            const chunk = rawImageBuffer.subarray(offset)
-            logger.debug(`[motion-photo] Interpreted offset ${candidate} as start ${start} from file end`)
+            if (validateMp4Buffer(chunk)) {
+              videoOffset = offset
+              videoSize = item.Length
+              logger?.success(
+                `[motion-photo] Found video via ContainerDirectory: offset=${offset}, size=${item.Length}`,
+              )
+            } else {
+              logger?.warn(`[motion-photo] Invalid MP4 at ContainerDirectory offset ${offset}`)
+            }
           }
-          break
         }
       }
-
-      if (resolvedOffset !== null) {
-        break
-      }
     }
 
-    // Fallback: scan for MP4 signature
+    // Fallback to legacy format (MicroVideo with MicroVideoOffset)
-    if (resolvedOffset === null) {
+    if (videoOffset === null && isMotionPhotoFlag) {
-      const searchWindowStart = Math.max(0, rawLength - 8 * 1024 * 1024)
+      const legacyOffset = toNumber(exifData?.MicroVideoOffset)
-      let cursor = rawImageBuffer.indexOf(MP4_FTYP, searchWindowStart)
+      if (legacyOffset !== null) {
-      while (cursor !== -1) {
+        logger?.info('[motion-photo] Using legacy MicroVideoOffset format')
-        const potentialStart = cursor - 4
+
-        if (potentialStart > 0 && potentialStart < rawLength - MIN_VIDEO_SIZE_BYTES) {
+        // Try both interpretations: from start and from end
-          const chunk = rawImageBuffer.subarray(potentialStart)
+        const candidates = [legacyOffset, rawLength - legacyOffset].filter(
+          (offset) => offset > 0 && offset < rawLength - MIN_VIDEO_SIZE_BYTES,
+        )
+
+        for (const offset of candidates) {
+          const chunk = rawImageBuffer.subarray(offset)
           if (validateMp4Buffer(chunk)) {
-            resolvedOffset = potentialStart
+            videoOffset = offset
             videoSize = chunk.length
-            logger?.info(`[motion-photo] Located MP4 via fallback scan at offset ${potentialStart}`)
+            logger?.success(`[motion-photo] Found video via legacy offset: ${offset}`)
             break
           }
         }
-        cursor = rawImageBuffer.indexOf(MP4_FTYP, cursor + 1)
       }
     }
 
-    if (resolvedOffset === null || videoSize === null) {
+    // No motion photo found
-      logger?.warn(`[motion-photo] Unable to locate MP4 after trying offsets ${candidateList.join(', ') || 'none'}`)
+    if (videoOffset === null || videoSize === null) {
+      if (isMotionPhotoFlag) {
+        logger?.warn('[motion-photo] MotionPhoto flag set but no valid video found')
+      }
       return null
     }
 
-    logger?.success(`[motion-photo] Detected Motion Photo at offset ${resolvedOffset}, video size ${videoSize} bytes`)
-
     return {
       isMotionPhoto: true,
-      motionPhotoOffset: resolvedOffset,
+      motionPhotoOffset: videoOffset,
       motionPhotoVideoSize: videoSize,
       presentationTimestampUs: presentationTimestampUs ?? undefined,
     }

packages/builder/src/types/photo.ts

@@ -155,9 +155,20 @@ export interface PickedExif {
 
   // HDR 相关
   MPImageType?: Tags['MPImageType']
+  UniformResourceName?: string
 
   // 评分
   Rating?: number
+
+  // Motion Photo (XMP) related fields
+  MotionPhoto?: Tags['MotionPhoto']
+  MotionPhotoVersion?: Tags['MotionPhotoVersion']
+  MotionPhotoPresentationTimestampUs?: Tags['MotionPhotoPresentationTimestampUs']
+  ContainerDirectory?: Tags['ContainerDirectory']
+  MicroVideo?: Tags['MicroVideo']
+  MicroVideoVersion?: Tags['MicroVideoVersion']
+  MicroVideoOffset?: Tags['MicroVideoOffset']
+  MicroVideoPresentationTimestampUs?: Tags['MicroVideoPresentationTimestampUs']
 }
 
 export interface ThumbnailResult {