diff --git a/apps/web/src/components/ui/photo-viewer/LivePhotoVideo.tsx b/apps/web/src/components/ui/photo-viewer/LivePhotoVideo.tsx index d7154718..537ff1a0 100644 --- a/apps/web/src/components/ui/photo-viewer/LivePhotoVideo.tsx +++ b/apps/web/src/components/ui/photo-viewer/LivePhotoVideo.tsx @@ -48,22 +48,11 @@ export const LivePhotoVideo = ({ const videoRef = useRef(null) const videoAnimateController = useAnimationControls() - const presentationTimestampRef = useRef(undefined) useEffect(() => { onPlayingChange?.(isPlayingLivePhoto) }, [isPlayingLivePhoto, onPlayingChange]) - // Extract and track presentationTimestamp for Motion Photo - useEffect(() => { - if (videoSource.type === 'motion-photo' && videoSource.presentationTimestamp) { - // Convert microseconds to seconds - presentationTimestampRef.current = videoSource.presentationTimestamp / 1_000_000 - } else { - presentationTimestampRef.current = undefined - } - }, [videoSource]) - useEffect(() => { if (!isCurrentImage || livePhotoVideoLoaded || isConvertingVideo || !videoRef.current) { return @@ -158,22 +147,6 @@ export const LivePhotoVideo = ({ stop() }, [stop]) - // Handle Motion Photo presentation timestamp - const handleTimeUpdate = useCallback(() => { - const video = videoRef.current - const timestamp = presentationTimestampRef.current - - // Only handle Motion Photo with valid timestamp - if (!video || timestamp === undefined || videoSource.type !== 'motion-photo') { - return - } - - // Stop playback when reaching or passing the presentation timestamp - if (video.currentTime >= timestamp) { - stop() - } - }, [videoSource, stop]) - return ( = [ 'GPSLongitudeRef', // HDR相关字段 'MPImageType', + 'UniformResourceName', + // Motion Photo 相关字段 + 'MotionPhoto', + 'MotionPhotoVersion', + 'MotionPhotoPresentationTimestampUs', + 'ContainerDirectory', + 'MicroVideo', + 'MicroVideoVersion', + 'MicroVideoOffset', + 'MicroVideoPresentationTimestampUs', ] function handleExifData(exifData: Tags, metadata: Metadata): PickedExif { const date = { diff --git a/packages/builder/src/photo/gainmap-detector.ts b/packages/builder/src/photo/gainmap-detector.ts new file mode 100644 index 00000000..5a7d4eee --- /dev/null +++ b/packages/builder/src/photo/gainmap-detector.ts @@ -0,0 +1,36 @@ +import type { ConsolaInstance } from 'consola' +import type { ContainerDirectoryItem } from 'exiftool-vendored' + +interface GainMapDetectParams { + exifData?: Record | null + logger?: ConsolaInstance +} + +/** + * Detects HDR GainMap from ContainerDirectory metadata. + * Works for both static Ultra HDR images and Motion Photo + HDR combinations. + */ +export const detectGainMap = ({ exifData, logger }: GainMapDetectParams): boolean => { + try { + const containerDirectory = exifData?.ContainerDirectory as ContainerDirectoryItem[] | undefined + if (!containerDirectory || !Array.isArray(containerDirectory)) { + return false + } + + // Find GainMap item + for (const entry of containerDirectory) { + const item = entry.Item + if (!item) continue + + if (item.Semantic === 'GainMap' && item.Length) { + logger?.info('[gainmap] Found HDR GainMap in ContainerDirectory') + return true + } + } + + return false + } catch (error) { + logger?.error('[gainmap] Unexpected error while detecting', error) + return false + } +} diff --git a/packages/builder/src/photo/image-pipeline.ts b/packages/builder/src/photo/image-pipeline.ts index 17c80967..810201af 100644 --- a/packages/builder/src/photo/image-pipeline.ts +++ b/packages/builder/src/photo/image-pipeline.ts @@ -18,6 +18,7 @@ import type { S3ObjectLike } from '../types/s3.js' import { shouldProcessPhoto } from './cache-manager.js' import { processExifData, processThumbnailAndBlurhash, processToneAnalysis } from './data-processors.js' import { getPhotoExecutionContext } from './execution-context.js' +import { detectGainMap } from './gainmap-detector.js' import { extractPhotoInfo } from './info-extractor.js' import { processLivePhoto } from './live-photo-handler.js' import { getGlobalLoggers } from './logger-adapter.js' @@ -178,13 +179,18 @@ export async function executePhotoProcessingPipeline( // 4. 处理 EXIF 数据 const exifData = await processExifData(imageBuffer, imageData.rawBuffer, photoKey, existingItem, options) - // 5. 检测 Motion Photo（从图片中提取嵌入视频的元数据） + // 5. 检测 HDR GainMap（Ultra HDR 图片） + const hasGainMap = detectGainMap({ + exifData: exifData as Record | null, + }) + + // 6. 检测 Motion Photo（从图片中提取嵌入视频的元数据） const motionPhotoMetadata = detectMotionPhoto({ rawImageBuffer: imageData.rawBuffer, exifData: exifData as Record | null, }) - // 6. 处理 Live Photo（独立的视频文件） + // 7. 处理 Live Photo（独立的视频文件） const livePhotoResult = await processLivePhoto(photoKey, livePhotoMap, storageManager) // 检测冲突：不允许同时存在 Motion Photo 和 Live Photo @@ -202,7 +208,6 @@ export async function executePhotoProcessingPipeline( // 9. 构建照片清单项 const aspectRatio = metadata.width / metadata.height - const photoItem: PhotoManifestItem = { id: photoId, title: photoInfo.title, @@ -224,20 +229,23 @@ export async function executePhotoProcessingPipeline( video: motionPhotoMetadata?.isMotionPhoto && motionPhotoMetadata.motionPhotoOffset !== undefined ? { - type: 'motion-photo', - offset: motionPhotoMetadata.motionPhotoOffset, - size: motionPhotoMetadata.motionPhotoVideoSize, - presentationTimestamp: motionPhotoMetadata.presentationTimestampUs, - } + type: 'motion-photo', + offset: motionPhotoMetadata.motionPhotoOffset, + size: motionPhotoMetadata.motionPhotoVideoSize, + presentationTimestamp: motionPhotoMetadata.presentationTimestampUs, + } : livePhotoResult.isLivePhoto ? { - type: 'live-photo', - videoUrl: livePhotoResult.livePhotoVideoUrl!, - s3Key: livePhotoResult.livePhotoVideoS3Key!, - } + type: 'live-photo', + videoUrl: livePhotoResult.livePhotoVideoUrl!, + s3Key: livePhotoResult.livePhotoVideoS3Key!, + } : undefined, // HDR 相关字段 - isHDR: exifData?.MPImageType === 'Gain Map Image', + isHDR: + exifData?.MPImageType === 'Gain Map Image' || + exifData?.UniformResourceName === 'urn:iso:std:iso:ts:21496:-1' || + hasGainMap, } loggers.image.success(`✅ 处理完成：${photoKey}`) diff --git a/packages/builder/src/photo/motion-photo-detector.ts b/packages/builder/src/photo/motion-photo-detector.ts index 45f7a012..bd2dcdf6 100644 --- a/packages/builder/src/photo/motion-photo-detector.ts +++ b/packages/builder/src/photo/motion-photo-detector.ts @@ -1,4 +1,5 @@ import type { ConsolaInstance } from 'consola' +import type { ContainerDirectoryItem } from 'exiftool-vendored' export interface MotionPhotoMetadata { isMotionPhoto: boolean @@ -13,7 +14,6 @@ interface MotionPhotoDetectParams { logger?: ConsolaInstance } -const MAX_XMP_SCAN_BYTES = 512 * 1024 // 512KB should cover standard XMP blocks const MIN_VIDEO_SIZE_BYTES = 8 * 1024 // 8KB minimal sanity check const MP4_FTYP = Buffer.from('ftyp') @@ -40,64 +40,6 @@ const toNumber = (value: unknown): number | null => { return null } -const extractXmpSegment = (buffer: Buffer): string | null => { - const scanSize = Math.min(buffer.length, MAX_XMP_SCAN_BYTES) - if (scanSize === 0) { - return null - } - - const header = buffer.toString('utf8', 0, scanSize) - const startIndex = header.indexOf('') - if (endIndex === -1) { - return null - } - - return header.slice(startIndex, endIndex + ''.length) -} - -const extractXmpBoolean = (xmp: string, tagName: string): boolean | null => { - const regex = new RegExp(`<[^:>]*:${tagName}>([^<]+)]+>`, 'i') - const match = xmp.match(regex) - if (!match) return null - return toBoolean(match[1]) -} - -const extractXmpNumber = (xmp: string, tagName: string): number | null => { - const regex = new RegExp(`<[^:>]*:${tagName}>([^<]+)]+>`, 'i') - const match = xmp.match(regex) - if (!match) return null - return toNumber(match[1]) -} - -const escapeRegExp = (value: string) => value.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&') - -const buildAttrPattern = (attrName: string) => { - const escaped = escapeRegExp(attrName) - if (attrName.includes(':')) { - return escaped - } - return `(?:[\\w-]+:)?${escaped}` -} - -const extractXmpAttributeBoolean = (xmp: string, attrName: string): boolean | null => { - const regex = new RegExp(`${buildAttrPattern(attrName)}="([^"]+)"`, 'i') - const match = xmp.match(regex) - if (!match) return null - return toBoolean(match[1]) -} - -const extractXmpAttributeNumber = (xmp: string, attrName: string): number | null => { - const regex = new RegExp(`${buildAttrPattern(attrName)}="([^"]+)"`, 'i') - const match = xmp.match(regex) - if (!match) return null - return toNumber(match[1]) -} - const validateMp4Buffer = (buffer: Buffer): boolean => { if (buffer.length < MIN_VIDEO_SIZE_BYTES) { return false @@ -109,8 +51,8 @@ const validateMp4Buffer = (buffer: Buffer): boolean => { } /** - * Detects Motion Photo metadata from image buffer without extracting the video file. - * Returns offset and size information for frontend to extract video on-demand. + * Detects Motion Photo metadata using Android Motion Photo format 1.0 specification. + * Supports both standard ContainerDirectory format and legacy MicroVideo format. */ export const detectMotionPhoto = ({ rawImageBuffer, @@ -120,126 +62,79 @@ export const detectMotionPhoto = ({ try { const rawLength = rawImageBuffer.length - const exifIndicatesMotion = toBoolean(exifData?.MotionPhoto) || toBoolean(exifData?.MicroVideo) - let detectedMotion = exifIndicatesMotion + // Check Motion Photo flags (standard and legacy) + const isMotionPhotoFlag = toBoolean(exifData?.MotionPhoto) || toBoolean(exifData?.MicroVideo) - let presentationTimestampUs = toNumber( + const presentationTimestampUs = toNumber( exifData?.MotionPhotoPresentationTimestampUs ?? exifData?.MicroVideoPresentationTimestampUs, ) - const offsetCandidates = new Set() - const addOffsetCandidate = (value: number | null | undefined) => { - if (value === null || value === undefined) return - if (!Number.isFinite(value)) return - const numeric = Number(value) - if (numeric <= 0) return - offsetCandidates.add(numeric) - } - - addOffsetCandidate(toNumber(exifData?.MicroVideoOffset)) - - const xmpSegment = extractXmpSegment(rawImageBuffer) - if (xmpSegment) { - if (!detectedMotion) { - const motionFlags = [ - extractXmpBoolean(xmpSegment, 'MotionPhoto'), - extractXmpBoolean(xmpSegment, 'GCamera:MotionPhoto'), - extractXmpBoolean(xmpSegment, 'MicroVideo'), - extractXmpBoolean(xmpSegment, 'GCamera:MicroVideo'), - extractXmpAttributeBoolean(xmpSegment, 'MotionPhoto'), - extractXmpAttributeBoolean(xmpSegment, 'GCamera:MotionPhoto'), - extractXmpAttributeBoolean(xmpSegment, 'MicroVideo'), - extractXmpAttributeBoolean(xmpSegment, 'GCamera:MicroVideo'), - ].filter((flag) => flag !== null) as boolean[] - - if (motionFlags.some(Boolean)) { - detectedMotion = true - logger?.info('[motion-photo] XMP detected MotionPhoto flags') - } - } - - ;[ - extractXmpNumber(xmpSegment, 'MicroVideoOffset'), - extractXmpNumber(xmpSegment, 'GCamera:MicroVideoOffset'), - extractXmpAttributeNumber(xmpSegment, 'MicroVideoOffset'), - extractXmpAttributeNumber(xmpSegment, 'GCamera:MicroVideoOffset'), - ].forEach((candidate) => addOffsetCandidate(candidate)) - - if (presentationTimestampUs === null) { - presentationTimestampUs = - extractXmpNumber(xmpSegment, 'MotionPhotoPresentationTimestampUs') ?? - extractXmpNumber(xmpSegment, 'MicroVideoPresentationTimestampUs') ?? - extractXmpAttributeNumber(xmpSegment, 'MotionPhotoPresentationTimestampUs') ?? - extractXmpAttributeNumber(xmpSegment, 'MicroVideoPresentationTimestampUs') ?? - null - } - } - - if (!detectedMotion && offsetCandidates.size === 0) { - return null - } - - let resolvedOffset: number | null = null + let videoOffset: number | null = null let videoSize: number | null = null - const candidateList = Array.from(offsetCandidates) - for (const candidate of candidateList) { - const possibleStarts = new Set() - possibleStarts.add(candidate) - if (candidate < rawLength) { - possibleStarts.add(rawLength - candidate) - } + // Try standard format (Motion Photo 1.0 with ContainerDirectory) + const containerDirectory = exifData?.ContainerDirectory as ContainerDirectoryItem[] | undefined + if (containerDirectory && Array.isArray(containerDirectory)) { + logger?.info('[motion-photo] Found ContainerDirectory, using standard format') - for (const start of possibleStarts) { - if (start <= 0 || start >= rawLength - MIN_VIDEO_SIZE_BYTES) { - continue - } + // Find video item + for (const entry of containerDirectory) { + const item = entry.Item + if (!item) continue - const chunk = rawImageBuffer.subarray(start) - if (validateMp4Buffer(chunk)) { - resolvedOffset = start - videoSize = chunk.length - if (start !== candidate && logger?.debug) { - logger.debug(`[motion-photo] Interpreted offset ${candidate} as start ${start} from file end`) + if (item.Semantic === 'MotionPhoto' && item.Length) { + // Video is stored at the end of file, Length bytes from the end + const offset = rawLength - item.Length + if (offset > 0 && offset < rawLength - MIN_VIDEO_SIZE_BYTES) { + const chunk = rawImageBuffer.subarray(offset) + if (validateMp4Buffer(chunk)) { + videoOffset = offset + videoSize = item.Length + logger?.success( + `[motion-photo] Found video via ContainerDirectory: offset=${offset}, size=${item.Length}`, + ) + } else { + logger?.warn(`[motion-photo] Invalid MP4 at ContainerDirectory offset ${offset}`) + } } - break } } - - if (resolvedOffset !== null) { - break - } } - // Fallback: scan for MP4 signature - if (resolvedOffset === null) { - const searchWindowStart = Math.max(0, rawLength - 8 * 1024 * 1024) - let cursor = rawImageBuffer.indexOf(MP4_FTYP, searchWindowStart) - while (cursor !== -1) { - const potentialStart = cursor - 4 - if (potentialStart > 0 && potentialStart < rawLength - MIN_VIDEO_SIZE_BYTES) { - const chunk = rawImageBuffer.subarray(potentialStart) + // Fallback to legacy format (MicroVideo with MicroVideoOffset) + if (videoOffset === null && isMotionPhotoFlag) { + const legacyOffset = toNumber(exifData?.MicroVideoOffset) + if (legacyOffset !== null) { + logger?.info('[motion-photo] Using legacy MicroVideoOffset format') + + // Try both interpretations: from start and from end + const candidates = [legacyOffset, rawLength - legacyOffset].filter( + (offset) => offset > 0 && offset < rawLength - MIN_VIDEO_SIZE_BYTES, + ) + + for (const offset of candidates) { + const chunk = rawImageBuffer.subarray(offset) if (validateMp4Buffer(chunk)) { - resolvedOffset = potentialStart + videoOffset = offset videoSize = chunk.length - logger?.info(`[motion-photo] Located MP4 via fallback scan at offset ${potentialStart}`) + logger?.success(`[motion-photo] Found video via legacy offset: ${offset}`) break } } - cursor = rawImageBuffer.indexOf(MP4_FTYP, cursor + 1) } } - if (resolvedOffset === null || videoSize === null) { - logger?.warn(`[motion-photo] Unable to locate MP4 after trying offsets ${candidateList.join(', ') || 'none'}`) + // No motion photo found + if (videoOffset === null || videoSize === null) { + if (isMotionPhotoFlag) { + logger?.warn('[motion-photo] MotionPhoto flag set but no valid video found') + } return null } - logger?.success(`[motion-photo] Detected Motion Photo at offset ${resolvedOffset}, video size ${videoSize} bytes`) - return { isMotionPhoto: true, - motionPhotoOffset: resolvedOffset, + motionPhotoOffset: videoOffset, motionPhotoVideoSize: videoSize, presentationTimestampUs: presentationTimestampUs ?? undefined, } diff --git a/packages/builder/src/types/photo.ts b/packages/builder/src/types/photo.ts index 9a668138..310bb3f1 100644 --- a/packages/builder/src/types/photo.ts +++ b/packages/builder/src/types/photo.ts @@ -155,9 +155,20 @@ export interface PickedExif { // HDR 相关 MPImageType?: Tags['MPImageType'] + UniformResourceName?: string // 评分 Rating?: number + + // Motion Photo (XMP) related fields + MotionPhoto?: Tags['MotionPhoto'] + MotionPhotoVersion?: Tags['MotionPhotoVersion'] + MotionPhotoPresentationTimestampUs?: Tags['MotionPhotoPresentationTimestampUs'] + ContainerDirectory?: Tags['ContainerDirectory'] + MicroVideo?: Tags['MicroVideo'] + MicroVideoVersion?: Tags['MicroVideoVersion'] + MicroVideoOffset?: Tags['MicroVideoOffset'] + MicroVideoPresentationTimestampUs?: Tags['MicroVideoPresentationTimestampUs'] } export interface ThumbnailResult {