feat: add support for motion photo (#153)

packages/builder/src/manifest/migrate.ts

@@ -54,6 +54,53 @@ const MIGRATION_STEPS: MigrationStep[] = [
       return raw
     },
   },
+  {
+    from: 'v7',
+    to: 'v8',
+    exec: (raw) => {
+      logger.main.info('🔄 迁移 v7 -> v8: 将 Live Photo/Motion Photo 字段转换为 VideoSource sum type')
+
+      raw.data.forEach((item: any) => {
+        // 转换为 VideoSource sum type
+        if (item.motionPhotoOffset !== undefined && item.motionPhotoOffset > 0) {
+          // Motion Photo: 嵌入视频
+          item.video = {
+            type: 'motion-photo',
+            offset: item.motionPhotoOffset,
+            ...(item.motionPhotoVideoSize && { size: item.motionPhotoVideoSize }),
+            ...(item.presentationTimestampUs && { presentationTimestamp: item.presentationTimestampUs }),
+          }
+        } else if (item.isLivePhoto && item.livePhotoVideoUrl) {
+          // Live Photo: 独立视频文件
+          // 仅在 s3Key 存在时创建 video 对象，避免无效元数据
+          if (item.livePhotoVideoS3Key) {
+            item.video = {
+              type: 'live-photo',
+              videoUrl: item.livePhotoVideoUrl,
+              s3Key: item.livePhotoVideoS3Key,
+            }
+          } else {
+            logger.main.warn(
+              `⚠️ 照片 ${item.id || item.url} 的 Live Photo 数据不完整（缺少 s3Key），跳过 video 字段生成`,
+            )
+          }
+        }
+        // 如果两者都不是，video 字段保持 undefined
+
+        // 删除旧字段
+        delete item.isLivePhoto
+        delete item.livePhotoVideoUrl
+        delete item.livePhotoVideoS3Key
+        delete item.motionPhotoOffset
+        delete item.motionPhotoVideoSize
+        delete item.presentationTimestampUs
+      })
+
+      // 更新版本号为目标版本
+      ;(raw as any).version = 'v8'
+      return raw
+    },
+  },
 ]
 
 function noOpBumpVersion(raw: any, _target: ManifestVersion): AfilmoryManifest {

packages/builder/src/manifest/version.ts

@@ -1,3 +1,3 @@
 export type ManifestVersion = `v${number}`
 
-export const CURRENT_MANIFEST_VERSION: ManifestVersion = 'v7'
+export const CURRENT_MANIFEST_VERSION: ManifestVersion = 'v8'

packages/builder/src/photo/image-pipeline.ts

@@ -21,6 +21,7 @@ import { getPhotoExecutionContext } from './execution-context.js'
 import { extractPhotoInfo } from './info-extractor.js'
 import { processLivePhoto } from './live-photo-handler.js'
 import { getGlobalLoggers } from './logger-adapter.js'
+import { detectMotionPhoto } from './motion-photo-detector.js'
 import type { PhotoProcessorOptions } from './processor.js'
 
 export interface ProcessedImageData {
@@ -174,16 +175,29 @@ export async function executePhotoProcessingPipeline(
     // 4. 处理 EXIF 数据
     const exifData = await processExifData(imageBuffer, imageData.rawBuffer, photoKey, existingItem, options)
 
-    // 5. 处理影调分析
-    const toneAnalysis = await processToneAnalysis(sharpInstance, photoKey, existingItem, options)
+    // 5. 检测 Motion Photo（从图片中提取嵌入视频的元数据）
+    const motionPhotoMetadata = detectMotionPhoto({
+      rawImageBuffer: imageData.rawBuffer,
+      exifData: exifData as Record<string, unknown> | null,
+    })
 
-    // 6. 提取照片信息
-    const photoInfo = extractPhotoInfo(photoKey, exifData)
-
-    // 7. 处理 Live Photo
+    // 6. 处理 Live Photo（独立的视频文件）
     const livePhotoResult = await processLivePhoto(photoKey, livePhotoMap, storageManager)
 
-    // 8. 构建照片清单项
+    // 检测冲突：不允许同时存在 Motion Photo 和 Live Photo
+    if (motionPhotoMetadata?.isMotionPhoto && livePhotoResult.isLivePhoto) {
+      const errorMsg = `❌ 检测到同时存在 Motion Photo (嵌入视频) 和 Live Photo (独立视频文件)：${photoKey}。这是不允许的，请只保留一种格式。`
+      loggers.image.error(errorMsg)
+      throw new Error(errorMsg)
+    }
+
+    // 7. 处理影调分析
+    const toneAnalysis = await processToneAnalysis(sharpInstance, photoKey, existingItem, options)
+
+    // 8. 提取照片信息
+    const photoInfo = extractPhotoInfo(photoKey, exifData)
+
+    // 9. 构建照片清单项
     const aspectRatio = metadata.width / metadata.height
 
     const photoItem: PhotoManifestItem = {
@@ -203,10 +217,22 @@ export async function executePhotoProcessingPipeline(
       size: obj.Size || 0,
       exif: exifData,
       toneAnalysis,
-      // Live Photo 相关字段
-      isLivePhoto: livePhotoResult.isLivePhoto,
-      livePhotoVideoUrl: livePhotoResult.livePhotoVideoUrl,
-      livePhotoVideoS3Key: livePhotoResult.livePhotoVideoS3Key,
+      // Video source (Motion Photo or Live Photo)
+      video:
+        motionPhotoMetadata?.isMotionPhoto && motionPhotoMetadata.motionPhotoOffset !== undefined
+          ? {
+              type: 'motion-photo',
+              offset: motionPhotoMetadata.motionPhotoOffset,
+              size: motionPhotoMetadata.motionPhotoVideoSize,
+              presentationTimestamp: motionPhotoMetadata.presentationTimestampUs,
+            }
+          : livePhotoResult.isLivePhoto
+            ? {
+                type: 'live-photo',
+                videoUrl: livePhotoResult.livePhotoVideoUrl!,
+                s3Key: livePhotoResult.livePhotoVideoS3Key!,
+              }
+            : undefined,
       // HDR 相关字段
       isHDR: exifData?.MPImageType === 'Gain Map Image',
     }

packages/builder/src/photo/motion-photo-detector.ts

@@ -0,0 +1,250 @@
+import type { ConsolaInstance } from 'consola'
+
+export interface MotionPhotoMetadata {
+  isMotionPhoto: boolean
+  motionPhotoOffset?: number
+  motionPhotoVideoSize?: number
+  presentationTimestampUs?: number
+}
+
+interface MotionPhotoDetectParams {
+  rawImageBuffer: Buffer
+  exifData?: Record<string, unknown> | null
+  logger?: ConsolaInstance
+}
+
+const MAX_XMP_SCAN_BYTES = 512 * 1024 // 512KB should cover standard XMP blocks
+const MIN_VIDEO_SIZE_BYTES = 8 * 1024 // 8KB minimal sanity check
+const MP4_FTYP = Buffer.from('ftyp')
+
+const toBoolean = (value: unknown): boolean => {
+  if (value === null || value === undefined) return false
+  if (typeof value === 'boolean') return value
+  if (typeof value === 'number') return value !== 0
+  if (typeof value === 'bigint') return value !== 0n
+  if (typeof value === 'string') {
+    const normalized = value.trim().toLowerCase()
+    return normalized === '1' || normalized === 'true' || normalized === 'yes'
+  }
+  return false
+}
+
+const toNumber = (value: unknown): number | null => {
+  if (value === null || value === undefined) return null
+  if (typeof value === 'number' && Number.isFinite(value)) return value
+  if (typeof value === 'bigint') return Number(value)
+  if (typeof value === 'string') {
+    const parsed = Number.parseInt(value.trim(), 10)
+    return Number.isFinite(parsed) ? parsed : null
+  }
+  return null
+}
+
+const extractXmpSegment = (buffer: Buffer): string | null => {
+  const scanSize = Math.min(buffer.length, MAX_XMP_SCAN_BYTES)
+  if (scanSize === 0) {
+    return null
+  }
+
+  const header = buffer.toString('utf8', 0, scanSize)
+  const startIndex = header.indexOf('<x:xmpmeta')
+  if (startIndex === -1) {
+    return null
+  }
+
+  const endIndex = header.indexOf('</x:xmpmeta>')
+  if (endIndex === -1) {
+    return null
+  }
+
+  return header.slice(startIndex, endIndex + '</x:xmpmeta>'.length)
+}
+
+const extractXmpBoolean = (xmp: string, tagName: string): boolean | null => {
+  const regex = new RegExp(`<[^:>]*:${tagName}>([^<]+)</[^>]+>`, 'i')
+  const match = xmp.match(regex)
+  if (!match) return null
+  return toBoolean(match[1])
+}
+
+const extractXmpNumber = (xmp: string, tagName: string): number | null => {
+  const regex = new RegExp(`<[^:>]*:${tagName}>([^<]+)</[^>]+>`, 'i')
+  const match = xmp.match(regex)
+  if (!match) return null
+  return toNumber(match[1])
+}
+
+const escapeRegExp = (value: string) => value.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&')
+
+const buildAttrPattern = (attrName: string) => {
+  const escaped = escapeRegExp(attrName)
+  if (attrName.includes(':')) {
+    return escaped
+  }
+  return `(?:[\\w-]+:)?${escaped}`
+}
+
+const extractXmpAttributeBoolean = (xmp: string, attrName: string): boolean | null => {
+  const regex = new RegExp(`${buildAttrPattern(attrName)}="([^"]+)"`, 'i')
+  const match = xmp.match(regex)
+  if (!match) return null
+  return toBoolean(match[1])
+}
+
+const extractXmpAttributeNumber = (xmp: string, attrName: string): number | null => {
+  const regex = new RegExp(`${buildAttrPattern(attrName)}="([^"]+)"`, 'i')
+  const match = xmp.match(regex)
+  if (!match) return null
+  return toNumber(match[1])
+}
+
+const validateMp4Buffer = (buffer: Buffer): boolean => {
+  if (buffer.length < MIN_VIDEO_SIZE_BYTES) {
+    return false
+  }
+
+  // MP4 should contain 'ftyp' brand within the first few bytes
+  const searchWindow = buffer.subarray(0, 32)
+  return searchWindow.includes(MP4_FTYP)
+}
+
+/**
+ * Detects Motion Photo metadata from image buffer without extracting the video file.
+ * Returns offset and size information for frontend to extract video on-demand.
+ */
+export const detectMotionPhoto = ({
+  rawImageBuffer,
+  exifData,
+  logger,
+}: MotionPhotoDetectParams): MotionPhotoMetadata | null => {
+  try {
+    const rawLength = rawImageBuffer.length
+
+    const exifIndicatesMotion = toBoolean(exifData?.MotionPhoto) || toBoolean(exifData?.MicroVideo)
+    let detectedMotion = exifIndicatesMotion
+
+    let presentationTimestampUs = toNumber(
+      exifData?.MotionPhotoPresentationTimestampUs ?? exifData?.MicroVideoPresentationTimestampUs,
+    )
+
+    const offsetCandidates = new Set<number>()
+    const addOffsetCandidate = (value: number | null | undefined) => {
+      if (value === null || value === undefined) return
+      if (!Number.isFinite(value)) return
+      const numeric = Number(value)
+      if (numeric <= 0) return
+      offsetCandidates.add(numeric)
+    }
+
+    addOffsetCandidate(toNumber(exifData?.MicroVideoOffset))
+
+    const xmpSegment = extractXmpSegment(rawImageBuffer)
+    if (xmpSegment) {
+      if (!detectedMotion) {
+        const motionFlags = [
+          extractXmpBoolean(xmpSegment, 'MotionPhoto'),
+          extractXmpBoolean(xmpSegment, 'GCamera:MotionPhoto'),
+          extractXmpBoolean(xmpSegment, 'MicroVideo'),
+          extractXmpBoolean(xmpSegment, 'GCamera:MicroVideo'),
+          extractXmpAttributeBoolean(xmpSegment, 'MotionPhoto'),
+          extractXmpAttributeBoolean(xmpSegment, 'GCamera:MotionPhoto'),
+          extractXmpAttributeBoolean(xmpSegment, 'MicroVideo'),
+          extractXmpAttributeBoolean(xmpSegment, 'GCamera:MicroVideo'),
+        ].filter((flag) => flag !== null) as boolean[]
+
+        if (motionFlags.some(Boolean)) {
+          detectedMotion = true
+          logger?.info('[motion-photo] XMP detected MotionPhoto flags')
+        }
+      }
+
+      ;[
+        extractXmpNumber(xmpSegment, 'MicroVideoOffset'),
+        extractXmpNumber(xmpSegment, 'GCamera:MicroVideoOffset'),
+        extractXmpAttributeNumber(xmpSegment, 'MicroVideoOffset'),
+        extractXmpAttributeNumber(xmpSegment, 'GCamera:MicroVideoOffset'),
+      ].forEach((candidate) => addOffsetCandidate(candidate))
+
+      if (presentationTimestampUs === null) {
+        presentationTimestampUs =
+          extractXmpNumber(xmpSegment, 'MotionPhotoPresentationTimestampUs') ??
+          extractXmpNumber(xmpSegment, 'MicroVideoPresentationTimestampUs') ??
+          extractXmpAttributeNumber(xmpSegment, 'MotionPhotoPresentationTimestampUs') ??
+          extractXmpAttributeNumber(xmpSegment, 'MicroVideoPresentationTimestampUs') ??
+          null
+      }
+    }
+
+    if (!detectedMotion && offsetCandidates.size === 0) {
+      return null
+    }
+
+    let resolvedOffset: number | null = null
+    let videoSize: number | null = null
+
+    const candidateList = Array.from(offsetCandidates)
+    for (const candidate of candidateList) {
+      const possibleStarts = new Set<number>()
+      possibleStarts.add(candidate)
+      if (candidate < rawLength) {
+        possibleStarts.add(rawLength - candidate)
+      }
+
+      for (const start of possibleStarts) {
+        if (start <= 0 || start >= rawLength - MIN_VIDEO_SIZE_BYTES) {
+          continue
+        }
+
+        const chunk = rawImageBuffer.subarray(start)
+        if (validateMp4Buffer(chunk)) {
+          resolvedOffset = start
+          videoSize = chunk.length
+          if (start !== candidate && logger?.debug) {
+            logger.debug(`[motion-photo] Interpreted offset ${candidate} as start ${start} from file end`)
+          }
+          break
+        }
+      }
+
+      if (resolvedOffset !== null) {
+        break
+      }
+    }
+
+    // Fallback: scan for MP4 signature
+    if (resolvedOffset === null) {
+      const searchWindowStart = Math.max(0, rawLength - 8 * 1024 * 1024)
+      let cursor = rawImageBuffer.indexOf(MP4_FTYP, searchWindowStart)
+      while (cursor !== -1) {
+        const potentialStart = cursor - 4
+        if (potentialStart > 0 && potentialStart < rawLength - MIN_VIDEO_SIZE_BYTES) {
+          const chunk = rawImageBuffer.subarray(potentialStart)
+          if (validateMp4Buffer(chunk)) {
+            resolvedOffset = potentialStart
+            videoSize = chunk.length
+            logger?.info(`[motion-photo] Located MP4 via fallback scan at offset ${potentialStart}`)
+            break
+          }
+        }
+        cursor = rawImageBuffer.indexOf(MP4_FTYP, cursor + 1)
+      }
+    }
+
+    if (resolvedOffset === null || videoSize === null) {
+      logger?.warn(`[motion-photo] Unable to locate MP4 after trying offsets ${candidateList.join(', ') || 'none'}`)
+      return null
+    }
+
+    logger?.success(`[motion-photo] Detected Motion Photo at offset ${resolvedOffset}, video size ${videoSize} bytes`)
+
+    return {
+      isMotionPhoto: true,
+      motionPhotoOffset: resolvedOffset,
+      motionPhotoVideoSize: videoSize,
+      presentationTimestampUs: presentationTimestampUs ?? undefined,
+    }
+  } catch (error) {
+    logger?.error('[motion-photo] Unexpected error while detecting', error)
+    return null
+  }
+}

packages/builder/src/types/photo.ts

@@ -28,6 +28,11 @@ export interface ToneAnalysis {
   highlightRatio: number // 0-1，高光区域占比
 }
 
+// Video source sum type: Live Photo or Motion Photo
+export type VideoSource =
+  | { type: 'live-photo'; videoUrl: string; s3Key: string }
+  | { type: 'motion-photo'; offset: number; size?: number; presentationTimestamp?: number }
+
 export interface PhotoInfo {
   title: string
   dateTaken: string
@@ -54,10 +59,9 @@ export interface PhotoManifestItem extends PhotoInfo {
   size: number
   exif: PickedExif | null
   toneAnalysis: ToneAnalysis | null // 影调分析结果
-  isLivePhoto?: boolean
   isHDR?: boolean
-  livePhotoVideoUrl?: string
-  livePhotoVideoS3Key?: string
+  // Video source (Live Photo or Motion Photo)
+  video?: VideoSource
 }
 
 export interface ProcessPhotoResult {