feat: add manifest migration CLI command and related functionality

- Introduced a new CLI command `manifest:migrate` for handling photo asset manifest migrations.
- Implemented parsing and execution logic for the new command in the CLI.
- Created helper functions to ensure current manifest versions and handle migration logic.
- Updated the `ManifestService` to support manifest upgrades and persist changes to the database.
- Refactored the database provider to streamline the DbAccessor constructor.

Signed-off-by: Innei <tukon479@gmail.com>
This commit is contained in:
Innei
2025-11-26 13:55:53 +08:00
parent b035883184
commit 9118bb9fb3
15 changed files with 507 additions and 97 deletions

View File

@@ -12,7 +12,8 @@
"db:migrate": "pnpm -C ../../packages/db db:migrate",
"db:studio": "pnpm -C ../../packages/db db:studio",
"dev": "nodemon",
"dev:reset-superadmin-password": "vite-node src/index.ts --reset-superadmin-password"
"dev:reset-superadmin-password": "vite-node src/index.ts --reset-superadmin-password",
"manifest:migrate": "vite-node src/index.ts manifest:migrate"
},
"dependencies": {
"@afilmory/be-utils": "workspace:*",

View File

@@ -1,3 +1,5 @@
import type { ManifestMigrationCliOptions } from './manifest-migrate'
import { handleManifestMigrationCli, parseManifestMigrationCliArgs } from './manifest-migrate'
import type { MigrationCliOptions } from './migrate'
import { handleMigrationCli, parseMigrationCliArgs } from './migrate'
import type { ResetCliOptions } from './reset-superadmin'
@@ -19,6 +21,14 @@ const cliCommands: Array<CliCommand<unknown>> = [
console.error('Database migration failed', error)
},
},
{
name: 'manifest:migrate',
parse: parseManifestMigrationCliArgs,
execute: (options) => handleManifestMigrationCli(options as ManifestMigrationCliOptions),
onError: (error) => {
console.error('Manifest migration failed', error)
},
},
{
name: 'reset-superadmin-password',
parse: parseResetCliArgs,

View File

@@ -0,0 +1,181 @@
import { photoAssets } from '@afilmory/db'
import { createLogger } from '@afilmory/framework'
import { eq, gt, sql } from 'drizzle-orm'
import { APP_GLOBAL_PREFIX } from '../app.constants'
import { createConfiguredApp } from '../app.factory'
import { DbAccessor, PgPoolProvider } from '../database/database.provider'
import { ensureCurrentPhotoAssetManifest } from '../modules/content/manifest/manifest-migration.helper'
import { RedisProvider } from '../redis/redis.provider'
const logger = createLogger('CLI:ManifestMigrate')
const COMMAND_NAME = 'manifest:migrate'
const DRY_RUN_FLAG = '--dry-run'
const BATCH_FLAG = '--batch'
const DEFAULT_BATCH_SIZE = 200
export interface ManifestMigrationCliOptions {
readonly command: typeof COMMAND_NAME
readonly dryRun: boolean
readonly batchSize: number
}
export function parseManifestMigrationCliArgs(argv: readonly string[]): ManifestMigrationCliOptions | null {
if (argv.length === 0 || argv[0] !== COMMAND_NAME) {
return null
}
let dryRun = false
let batchSize = DEFAULT_BATCH_SIZE
for (let index = 1; index < argv.length; index++) {
const token = argv[index]
if (!token) continue
if (token === DRY_RUN_FLAG) {
dryRun = true
continue
}
if (token.startsWith(`${DRY_RUN_FLAG}=`)) {
const inline = token.slice(DRY_RUN_FLAG.length + 1)
dryRun = inline !== 'false'
continue
}
if (token === BATCH_FLAG) {
const value = argv[index + 1]
if (!value || value.startsWith('--')) {
throw new Error('Missing value for --batch')
}
batchSize = parseBatchSize(value)
index++
continue
}
if (token.startsWith(`${BATCH_FLAG}=`)) {
const inline = token.slice(BATCH_FLAG.length + 1)
batchSize = parseBatchSize(inline)
continue
}
}
return {
command: COMMAND_NAME,
dryRun,
batchSize,
}
}
function parseBatchSize(value: string): number {
const parsed = Number.parseInt(value, 10)
if (Number.isNaN(parsed) || parsed <= 0) {
throw new Error(`Invalid --batch value: "${value}"`)
}
return parsed
}
export async function handleManifestMigrationCli(options: ManifestMigrationCliOptions): Promise<void> {
const app = await createConfiguredApp({
globalPrefix: APP_GLOBAL_PREFIX,
})
const container = app.getContainer()
const dbAccessor = container.resolve(DbAccessor)
const poolProvider = container.resolve(PgPoolProvider)
const redisProvider = container.resolve(RedisProvider)
try {
const db = dbAccessor.get()
const total = await countPhotoAssets(db)
logger.info(
`Scanning ${total} photo assets in batches of ${options.batchSize}${options.dryRun ? ' (dry run)' : ''}`,
)
const summary = await migratePhotoAssets(db, options)
logger.info(
`${options.dryRun ? 'Dry run' : 'Migration'} completed. Processed ${summary.processed} assets, ${
options.dryRun ? 'would update' : 'updated'
} ${summary.updated} manifest records.`,
)
} finally {
await app.close(COMMAND_NAME)
try {
const pool = poolProvider.getPool()
await pool.end()
} catch (error) {
logger.warn(`Failed to close PostgreSQL pool cleanly: ${String(error)}`)
}
try {
const redis = redisProvider.getClient()
redis.disconnect()
} catch (error) {
logger.warn(`Failed to disconnect Redis client cleanly: ${String(error)}`)
}
}
}
type DbClient = ReturnType<DbAccessor['get']>
async function countPhotoAssets(db: DbClient): Promise<number> {
const [row] = await db.select({ total: sql<number>`count(*)` }).from(photoAssets)
return row?.total ?? 0
}
async function migratePhotoAssets(
db: DbClient,
options: ManifestMigrationCliOptions,
): Promise<{ processed: number; updated: number }> {
let processed = 0
let updated = 0
let lastId: string | null = null
while (true) {
let query: any = db
.select({
id: photoAssets.id,
manifest: photoAssets.manifest,
manifestVersion: photoAssets.manifestVersion,
})
.from(photoAssets)
.orderBy(photoAssets.id)
.limit(options.batchSize)
if (lastId) {
query = query.where(gt(photoAssets.id, lastId))
}
const rows = await query
if (rows.length === 0) {
break
}
lastId = rows.at(-1)?.id ?? null
for (const row of rows) {
processed++
const { manifest: resolved, changed } = ensureCurrentPhotoAssetManifest(row.manifest)
if (!resolved || !changed) {
continue
}
updated++
if (options.dryRun) {
logger.info(`Would update photo asset ${row.id} -> version ${resolved.version}`)
continue
}
await db
.update(photoAssets)
.set({
manifest: resolved,
manifestVersion: resolved.version,
})
.where(eq(photoAssets.id, row.id))
}
}
return { processed, updated }
}

View File

@@ -121,10 +121,7 @@ export class DrizzleProvider {
@injectable()
export class DbAccessor {
constructor(
private readonly provider: DrizzleProvider,
private readonly poolProvider: PgPoolProvider,
) {}
constructor(private readonly provider: DrizzleProvider) {}
get(): DrizzleDb {
const store = getOptionalDbContext()

View File

@@ -0,0 +1,51 @@
import type { AfilmoryManifest } from '@afilmory/builder'
import { CURRENT_MANIFEST_VERSION, migrateManifest } from '@afilmory/builder'
import type { ManifestVersion } from '@afilmory/builder/manifest/version.js'
import type { PhotoAssetManifest } from '@afilmory/db'
export type PhotoAssetManifestPayload = PhotoAssetManifest | null
export function ensureCurrentPhotoAssetManifest(manifest: PhotoAssetManifestPayload): {
manifest: PhotoAssetManifest | null
changed: boolean
} {
if (!manifest?.data) {
return { manifest: null, changed: false }
}
const requiresMigration = manifest.version !== CURRENT_MANIFEST_VERSION || !hasValidFormat(manifest.data.format)
if (!requiresMigration) {
return { manifest, changed: false }
}
const migrated = migrateSingleManifestItem(manifest)
if (!migrated) {
return { manifest, changed: false }
}
return { manifest: migrated, changed: true }
}
function migrateSingleManifestItem(input: PhotoAssetManifest): PhotoAssetManifest | null {
const wrapper: AfilmoryManifest = {
version: input.version as ManifestVersion,
data: [structuredClone(input.data)],
cameras: [],
lenses: [],
}
const migrated = migrateManifest(wrapper, CURRENT_MANIFEST_VERSION)
const migratedItem = migrated.data[0]
if (!migratedItem) {
return null
}
return {
version: CURRENT_MANIFEST_VERSION,
data: migratedItem,
}
}
function hasValidFormat(format: string | undefined | null): boolean {
return typeof format === 'string' && format.trim().length > 0
}

View File

@@ -1,5 +1,9 @@
import type { AfilmoryManifest, CameraInfo, LensInfo, PhotoManifestItem } from '@afilmory/builder'
import { CURRENT_MANIFEST_VERSION, migrateManifest } from '@afilmory/builder'
import type { ManifestVersion } from '@afilmory/builder/manifest/version.js'
import type { PhotoAssetManifest } from '@afilmory/db'
import { CURRENT_PHOTO_MANIFEST_VERSION, photoAssets } from '@afilmory/db'
import { createLogger } from '@afilmory/framework'
import { DbAccessor } from 'core/database/database.provider'
import { StorageAccessService } from 'core/modules/content/photo/access/storage-access.service'
import { createProxyUrl } from 'core/modules/content/photo/access/storage-access.utils'
@@ -8,8 +12,12 @@ import { requireTenantContext } from 'core/modules/platform/tenant/tenant.contex
import { and, eq, inArray } from 'drizzle-orm'
import { injectable } from 'tsyringe'
import { ensureCurrentPhotoAssetManifest } from './manifest-migration.helper'
@injectable()
export class ManifestService {
private readonly logger = createLogger('ManifestService')
constructor(
private readonly dbAccessor: DbAccessor,
private readonly photoStorageService: PhotoStorageService,
@@ -22,6 +30,7 @@ export class ManifestService {
const records = await db
.select({
id: photoAssets.id,
manifest: photoAssets.manifest,
storageProvider: photoAssets.storageProvider,
})
@@ -43,13 +52,19 @@ export class ManifestService {
tenant.tenant.id,
)
const items: PhotoManifestItem[] = []
const upgrades: Array<{ id: string; manifest: PhotoAssetManifest }> = []
for (const record of records) {
const item = record.manifest?.data
if (!item) {
const { manifest, changed } = ensureCurrentPhotoAssetManifest(record.manifest)
if (!manifest) {
continue
}
const normalized = structuredClone(item)
if (changed) {
upgrades.push({ id: record.id, manifest })
}
const normalized = structuredClone(manifest.data)
if (secureAccessEnabled && (record.storageProvider === 'managed' || record.storageProvider === 's3')) {
if (normalized.s3Key) {
normalized.originalUrl = createProxyUrl(normalized.s3Key)
@@ -61,30 +76,74 @@ export class ManifestService {
items.push(normalized)
}
if (upgrades.length > 0) {
await this.persistManifestUpgrades(upgrades)
}
const sorted = this.sortByDateDesc(items)
const cameras = this.buildCameraCollection(sorted)
const lenses = this.buildLensCollection(sorted)
const manifest = this.ensureCurrentManifestVersion({
version: upgrades.length > 0 ? CURRENT_MANIFEST_VERSION : this.resolveManifestVersion(records),
data: sorted,
cameras: [],
lenses: [],
})
const cameras = this.buildCameraCollection(manifest.data)
const lenses = this.buildLensCollection(manifest.data)
return {
version: this.resolveManifestVersion(records),
data: sorted,
...manifest,
cameras,
lenses,
}
}
private resolveManifestVersion(
records: Array<{ manifest: { version: typeof CURRENT_PHOTO_MANIFEST_VERSION } | null }>,
): typeof CURRENT_PHOTO_MANIFEST_VERSION {
records: Array<{ manifest: { version: ManifestVersion | string } | null }>,
): ManifestVersion {
for (const record of records) {
const version = record.manifest?.version
if (version) {
return version
if (typeof version === 'string' && version.length > 0) {
return version as ManifestVersion
}
}
return CURRENT_PHOTO_MANIFEST_VERSION
}
private ensureCurrentManifestVersion(manifest: AfilmoryManifest): AfilmoryManifest {
if (manifest.version === CURRENT_MANIFEST_VERSION) {
return manifest
}
try {
return migrateManifest(manifest, CURRENT_MANIFEST_VERSION)
} catch (error) {
this.logger.warn('Manifest migration failed; returning original payload', { error })
return manifest
}
}
private async persistManifestUpgrades(upgrades: Array<{ id: string; manifest: PhotoAssetManifest }>): Promise<void> {
if (upgrades.length === 0) {
return
}
const db = this.dbAccessor.get()
for (const entry of upgrades) {
try {
await db
.update(photoAssets)
.set({
manifest: entry.manifest,
manifestVersion: entry.manifest.version,
})
.where(eq(photoAssets.id, entry.id))
} catch (error) {
this.logger.warn('Failed to persist manifest upgrade', { photoAssetId: entry.id, error })
}
}
}
private sortByDateDesc(items: PhotoManifestItem[]): PhotoManifestItem[] {
return [...items].sort((a, b) => this.toTimestamp(b.dateTaken) - this.toTimestamp(a.dateTaken))
}

View File

@@ -1,4 +1,6 @@
import type { PhotoManifestItem } from '@afilmory/builder'
import type { ManifestVersion } from '@afilmory/builder/manifest/version'
import { CURRENT_MANIFEST_VERSION } from '@afilmory/builder/manifest/version'
import {
bigint,
boolean,
@@ -29,7 +31,7 @@ export const userRoleEnum = pgEnum('user_role', ['user', 'admin', 'superadmin'])
export const tenantStatusEnum = pgEnum('tenant_status', ['active', 'inactive', 'suspended'])
export const tenantDomainStatusEnum = pgEnum('tenant_domain_status', ['pending', 'verified', 'disabled'])
export const photoSyncStatusEnum = pgEnum('photo_sync_status', ['pending', 'synced', 'conflict'])
export const CURRENT_PHOTO_MANIFEST_VERSION = 'v7' as const
export const CURRENT_PHOTO_MANIFEST_VERSION: ManifestVersion = CURRENT_MANIFEST_VERSION
export type PhotoAssetConflictType = 'missing-in-storage' | 'metadata-mismatch' | 'photo-id-conflict'
/**
@@ -52,7 +54,7 @@ export interface PhotoAssetConflictPayload {
}
export interface PhotoAssetManifest {
version: typeof CURRENT_PHOTO_MANIFEST_VERSION
version: ManifestVersion
data: PhotoManifestItem
}

View File

@@ -58,3 +58,7 @@ export type { BuilderConfig, BuilderConfigInput } from './types/config.js'
export type { AfilmoryManifest, CameraInfo, LensInfo } from './types/manifest.js'
export type { FujiRecipe, PhotoManifestItem, PickedExif, ToneAnalysis } from './types/photo.js'
export type { S3ObjectLike } from './types/s3.js'
///// Mirgation
export { migrateManifest } from './manifest/migrate.js'
export { CURRENT_MANIFEST_VERSION } from './manifest/version.js'

View File

@@ -5,6 +5,7 @@ import { workdir } from '@afilmory/builder/path.js'
import { logger } from '../logger/index.js'
import type { AfilmoryManifest } from '../types/manifest.js'
import { MIGRATION_STEPS } from './migrations/index.js'
import type { ManifestVersion } from './version.js'
import { CURRENT_MANIFEST_VERSION } from './version.js'
@@ -25,84 +26,6 @@ export type MigrationStep = {
exec: ManifestMigrator
}
// Registry of ordered migration steps. Keep empty until concrete steps are added.
const MIGRATION_STEPS: MigrationStep[] = [
{
from: 'v1',
to: 'v6',
exec: () => {
logger.fs.error('🔍 无效的 manifest 版本,创建新的 manifest 文件...')
return {
version: 'v6',
data: [],
cameras: [],
lenses: [],
}
},
},
{
from: 'v6',
to: 'v7',
exec: (raw) => {
raw.data.forEach((item) => {
if (typeof item.thumbnailUrl === 'string') {
item.thumbnailUrl = item.thumbnailUrl.replace(/\.webp$/, '.jpg')
}
})
// 更新版本号为目标版本
;(raw as any).version = 'v7'
return raw
},
},
{
from: 'v7',
to: 'v8',
exec: (raw) => {
logger.main.info('🔄 迁移 v7 -> v8: 将 Live Photo/Motion Photo 字段转换为 VideoSource sum type')
raw.data.forEach((item: any) => {
// 转换为 VideoSource sum type
if (item.motionPhotoOffset !== undefined && item.motionPhotoOffset > 0) {
// Motion Photo: 嵌入视频
item.video = {
type: 'motion-photo',
offset: item.motionPhotoOffset,
...(item.motionPhotoVideoSize && { size: item.motionPhotoVideoSize }),
...(item.presentationTimestampUs && { presentationTimestamp: item.presentationTimestampUs }),
}
} else if (item.isLivePhoto && item.livePhotoVideoUrl) {
// Live Photo: 独立视频文件
// 仅在 s3Key 存在时创建 video 对象,避免无效元数据
if (item.livePhotoVideoS3Key) {
item.video = {
type: 'live-photo',
videoUrl: item.livePhotoVideoUrl,
s3Key: item.livePhotoVideoS3Key,
}
} else {
logger.main.warn(
`⚠️ 照片 ${item.id || item.url} 的 Live Photo 数据不完整(缺少 s3Key跳过 video 字段生成`,
)
}
}
// 如果两者都不是video 字段保持 undefined
// 删除旧字段
delete item.isLivePhoto
delete item.livePhotoVideoUrl
delete item.livePhotoVideoS3Key
delete item.motionPhotoOffset
delete item.motionPhotoVideoSize
delete item.presentationTimestampUs
})
// 更新版本号为目标版本
;(raw as any).version = 'v8'
return raw
},
},
]
function noOpBumpVersion(raw: any, _target: ManifestVersion): AfilmoryManifest {
return raw
}

View File

@@ -0,0 +1,32 @@
import type { MigrationStep } from '../migrate.js'
import { migrateV1ToV6 } from './v1-to-v6.js'
import { migrateV6ToV7 } from './v6-to-v7.js'
import { migrateV7ToV8 } from './v7-to-v8.js'
import { migrateV8ToV9 } from './v8-to-v9.js'
/**
* Registry of ordered migration steps.
* Each step defines a migration from one version to the next.
*/
export const MIGRATION_STEPS: MigrationStep[] = [
{
from: 'v1',
to: 'v6',
exec: migrateV1ToV6,
},
{
from: 'v6',
to: 'v7',
exec: migrateV6ToV7,
},
{
from: 'v7',
to: 'v8',
exec: migrateV7ToV8,
},
{
from: 'v8',
to: 'v9',
exec: migrateV8ToV9,
},
]

View File

@@ -0,0 +1,17 @@
import { logger } from '../../logger/index.js'
import type { AfilmoryManifest } from '../../types/manifest.js'
import type { ManifestMigrator, MigrationContext } from '../migrate.js'
/**
* Migration: v1 -> v6
* 无效的 manifest 版本,创建新的 manifest 文件
*/
export const migrateV1ToV6: ManifestMigrator = (_raw: AfilmoryManifest, _ctx: MigrationContext) => {
logger.fs.error('🔍 无效的 manifest 版本,创建新的 manifest 文件...')
return {
version: 'v6',
data: [],
cameras: [],
lenses: [],
}
}

View File

@@ -0,0 +1,17 @@
import type { AfilmoryManifest } from '../../types/manifest.js'
import type { ManifestMigrator, MigrationContext } from '../migrate.js'
/**
* Migration: v6 -> v7
* 将缩略图 URL 从 .webp 替换为 .jpg
*/
export const migrateV6ToV7: ManifestMigrator = (raw: AfilmoryManifest, _ctx: MigrationContext) => {
raw.data.forEach((item) => {
if (typeof item.thumbnailUrl === 'string') {
item.thumbnailUrl = item.thumbnailUrl.replace(/\.webp$/, '.jpg')
}
})
// 更新版本号为目标版本
;(raw as any).version = 'v7'
return raw
}

View File

@@ -0,0 +1,49 @@
import { logger } from '../../logger/index.js'
import type { AfilmoryManifest } from '../../types/manifest.js'
import type { ManifestMigrator, MigrationContext } from '../migrate.js'
/**
* Migration: v7 -> v8
* 将 Live Photo/Motion Photo 字段转换为 VideoSource sum type
*/
export const migrateV7ToV8: ManifestMigrator = (raw: AfilmoryManifest, _ctx: MigrationContext) => {
logger.main.info('🔄 迁移 v7 -> v8: 将 Live Photo/Motion Photo 字段转换为 VideoSource sum type')
raw.data.forEach((item: any) => {
// 转换为 VideoSource sum type
if (item.motionPhotoOffset !== undefined && item.motionPhotoOffset > 0) {
// Motion Photo: 嵌入视频
item.video = {
type: 'motion-photo',
offset: item.motionPhotoOffset,
...(item.motionPhotoVideoSize && { size: item.motionPhotoVideoSize }),
...(item.presentationTimestampUs && { presentationTimestamp: item.presentationTimestampUs }),
}
} else if (item.isLivePhoto && item.livePhotoVideoUrl) {
// Live Photo: 独立视频文件
// 仅在 s3Key 存在时创建 video 对象,避免无效元数据
if (item.livePhotoVideoS3Key) {
item.video = {
type: 'live-photo',
videoUrl: item.livePhotoVideoUrl,
s3Key: item.livePhotoVideoS3Key,
}
} else {
logger.main.warn(`⚠️ 照片 ${item.id || item.url} 的 Live Photo 数据不完整(缺少 s3Key跳过 video 字段生成`)
}
}
// 如果两者都不是video 字段保持 undefined
// 删除旧字段
delete item.isLivePhoto
delete item.livePhotoVideoUrl
delete item.livePhotoVideoS3Key
delete item.motionPhotoOffset
delete item.motionPhotoVideoSize
delete item.presentationTimestampUs
})
// 更新版本号为目标版本
;(raw as any).version = 'v8'
return raw
}

View File

@@ -0,0 +1,67 @@
import path from 'node:path'
import type { AfilmoryManifest } from '../../types/manifest.js'
import type { ManifestMigrator, MigrationContext } from '../migrate.js'
/**
* Migration: v8 -> v9
* 补全 format 字段
*/
export const migrateV8ToV9: ManifestMigrator = (raw: AfilmoryManifest, _ctx: MigrationContext) => {
if (!Array.isArray(raw.data)) {
raw.data = []
}
raw.data.forEach((item: Record<string, any>) => {
if (!item || typeof item !== 'object') return
const existing = typeof item.format === 'string' ? item.format.trim() : ''
if (existing) {
item.format = existing.toUpperCase()
return
}
const inferred = inferFormatFromManifestItem(item)
item.format = inferred ?? 'UNKNOWN'
})
;(raw as any).version = 'v9'
return raw
}
function inferFormatFromManifestItem(item: Record<string, any>): string | null {
const s3Format = extractFormatFromPath(item.s3Key)
if (s3Format) return s3Format
const originalFormat = extractFormatFromPath(item.originalUrl)
if (originalFormat) return originalFormat
const thumbFormat = extractFormatFromPath(item.thumbnailUrl)
if (thumbFormat) return thumbFormat
return null
}
function extractFormatFromPath(input?: string | null): string | null {
if (!input || typeof input !== 'string') {
return null
}
let target = input.trim()
if (!target) {
return null
}
try {
const parsed = new URL(target)
target = parsed.pathname || target
} catch {
// Not a URL string keep original path
}
const ext = path.extname(target)
if (!ext) {
return null
}
const normalized = ext.slice(1).toUpperCase()
return normalized || null
}

View File

@@ -1,3 +1,3 @@
export type ManifestVersion = `v${number}`
export const CURRENT_MANIFEST_VERSION: ManifestVersion = 'v8'
export const CURRENT_MANIFEST_VERSION: ManifestVersion = 'v9'