mirror of
https://github.com/Afilmory/afilmory
synced 2026-02-01 22:48:17 +00:00
chore: init
This commit is contained in:
@@ -24,6 +24,11 @@
|
||||
window.__SITE_CONFIG__ = {}
|
||||
</script>
|
||||
<script id="manifest"></script>
|
||||
<script type="module">
|
||||
if (import.meta.env.DEV) {
|
||||
import('react-grab')
|
||||
}
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root">
|
||||
|
||||
@@ -58,6 +58,7 @@
|
||||
"react-dom": "19.2.0",
|
||||
"react-error-boundary": "6.0.0",
|
||||
"react-freeze": "1.0.4",
|
||||
"react-grab": "0.0.39",
|
||||
"react-i18next": "16.3.1",
|
||||
"react-image-gallery": "1.4.0",
|
||||
"react-intersection-observer": "10.0.0",
|
||||
|
||||
@@ -51,7 +51,8 @@
|
||||
"vite": "7.2.2",
|
||||
"vite-bundle-analyzer": "1.2.3",
|
||||
"vite-node": "5.0.0",
|
||||
"vite-plugin-node-worker": "1.0.5",
|
||||
"vite-tsconfig-paths": "5.1.4",
|
||||
"vitest": "4.0.8"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -351,7 +351,7 @@ const enUiSchema = {
|
||||
providers: {
|
||||
types: {
|
||||
s3: 'AWS S3 Compatible Object Storage',
|
||||
github: 'GitHub repository',
|
||||
github: 'GitHub Repository',
|
||||
b2: 'Backblaze B2 cloud storage',
|
||||
},
|
||||
fields: {
|
||||
|
||||
@@ -12,7 +12,6 @@ import { CURRENT_PHOTO_MANIFEST_VERSION, DATABASE_ONLY_PROVIDER, photoAssets } f
|
||||
import { EventEmitterService } from '@afilmory/framework'
|
||||
import { DbAccessor } from 'core/database/database.provider'
|
||||
import { BizException, ErrorCode } from 'core/errors'
|
||||
import { runWithBuilderLogRelay } from 'core/modules/infrastructure/data-sync/builder-log-relay'
|
||||
import type {
|
||||
DataSyncAction,
|
||||
DataSyncLogLevel,
|
||||
@@ -25,6 +24,8 @@ import { BILLING_USAGE_EVENT } from 'core/modules/platform/billing/billing.const
|
||||
import { BillingPlanService } from 'core/modules/platform/billing/billing-plan.service'
|
||||
import { BillingUsageService } from 'core/modules/platform/billing/billing-usage.service'
|
||||
import { requireTenantContext } from 'core/modules/platform/tenant/tenant.context'
|
||||
import { BuilderWorkerHost } from 'core/workers/builder/builder-worker.host'
|
||||
import type { BuilderWorkerLogEvent } from 'core/workers/builder/builder-worker.types'
|
||||
import { and, eq, inArray, sql } from 'drizzle-orm'
|
||||
import { injectable } from 'tsyringe'
|
||||
|
||||
@@ -75,6 +76,7 @@ export class PhotoAssetService {
|
||||
private readonly photoStorageService: PhotoStorageService,
|
||||
private readonly billingPlanService: BillingPlanService,
|
||||
private readonly billingUsageService: BillingUsageService,
|
||||
private readonly builderWorkerHost: BuilderWorkerHost,
|
||||
) {}
|
||||
|
||||
private async emitManifestChanged(tenantId: string): Promise<void> {
|
||||
@@ -449,7 +451,6 @@ export class PhotoAssetService {
|
||||
const processedItems = await this.processPendingPhotos({
|
||||
pendingPhotoPlans: allPendingPhotoPlans,
|
||||
videoObjectsByBaseName,
|
||||
builder,
|
||||
builderConfig,
|
||||
storageManager,
|
||||
storageConfig,
|
||||
@@ -865,7 +866,6 @@ export class PhotoAssetService {
|
||||
private async processPendingPhotos(params: {
|
||||
pendingPhotoPlans: PreparedUploadPlan[]
|
||||
videoObjectsByBaseName: Map<string, StorageObject>
|
||||
builder: ReturnType<PhotoBuilderService['createBuilder']>
|
||||
builderConfig: BuilderConfig
|
||||
storageManager: StorageManager
|
||||
storageConfig: StorageConfig
|
||||
@@ -884,7 +884,6 @@ export class PhotoAssetService {
|
||||
const {
|
||||
pendingPhotoPlans,
|
||||
videoObjectsByBaseName,
|
||||
builder,
|
||||
builderConfig,
|
||||
storageManager,
|
||||
storageConfig,
|
||||
@@ -941,19 +940,21 @@ export class PhotoAssetService {
|
||||
}
|
||||
}
|
||||
|
||||
const processed = await runWithBuilderLogRelay(builderLogEmitter, () =>
|
||||
this.photoBuilderService.processPhotoFromStorageObject(storageObject, {
|
||||
builder,
|
||||
builderConfig,
|
||||
processorOptions: {
|
||||
isForceMode: true,
|
||||
isForceManifest: true,
|
||||
isForceThumbnails: true,
|
||||
},
|
||||
livePhotoMap,
|
||||
prefetchedBuffers,
|
||||
}),
|
||||
)
|
||||
const processed = await this.builderWorkerHost.processPhoto({
|
||||
builderConfig,
|
||||
storageConfig,
|
||||
storageObject,
|
||||
livePhotoMap,
|
||||
prefetchedBuffers,
|
||||
processorOptions: {
|
||||
isForceMode: true,
|
||||
isForceManifest: true,
|
||||
isForceThumbnails: true,
|
||||
},
|
||||
logHandler: builderLogEmitter
|
||||
? (event) => this.forwardUploadBuilderLog(builderLogEmitter, event, storageObject.key ?? resolvedPhotoKey)
|
||||
: undefined,
|
||||
})
|
||||
|
||||
const item = processed?.item
|
||||
if (!item) {
|
||||
@@ -1050,6 +1051,32 @@ export class PhotoAssetService {
|
||||
return results
|
||||
}
|
||||
|
||||
private forwardUploadBuilderLog(
|
||||
emitter: DataSyncProgressEmitter | undefined,
|
||||
event: BuilderWorkerLogEvent,
|
||||
storageKey: string,
|
||||
): void {
|
||||
if (!emitter) {
|
||||
return
|
||||
}
|
||||
|
||||
void emitter({
|
||||
type: 'log',
|
||||
payload: {
|
||||
level: event.level,
|
||||
message: event.message,
|
||||
stage: 'missing-in-db',
|
||||
storageKey,
|
||||
details: {
|
||||
...(event.details ?? {}),
|
||||
tag: event.tag ?? undefined,
|
||||
source: event.details?.source ?? 'builder',
|
||||
},
|
||||
timestamp: event.timestamp,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
async generatePublicUrl(storageKey: string): Promise<string> {
|
||||
const tenant = requireTenantContext()
|
||||
const { builderConfig, storageConfig } = await this.photoStorageService.resolveConfigForTenant(tenant.tenant.id)
|
||||
|
||||
@@ -2,6 +2,7 @@ import { Module } from '@afilmory/framework'
|
||||
import { BuilderConfigService } from 'core/modules/configuration/builder-config/builder-config.service'
|
||||
import { SystemSettingModule } from 'core/modules/configuration/system-setting/system-setting.module'
|
||||
import { BillingModule } from 'core/modules/platform/billing/billing.module'
|
||||
import { BuilderWorkerHost } from 'core/workers/builder/builder-worker.host'
|
||||
|
||||
import { PhotoController } from './assets/photo.controller'
|
||||
import { PhotoAssetService } from './assets/photo-asset.service'
|
||||
@@ -11,6 +12,6 @@ import { PhotoStorageService } from './storage/photo-storage.service'
|
||||
@Module({
|
||||
imports: [SystemSettingModule, BillingModule],
|
||||
controllers: [PhotoController],
|
||||
providers: [PhotoBuilderService, PhotoStorageService, PhotoAssetService, BuilderConfigService],
|
||||
providers: [PhotoBuilderService, PhotoStorageService, PhotoAssetService, BuilderConfigService, BuilderWorkerHost],
|
||||
})
|
||||
export class PhotoModule {}
|
||||
|
||||
@@ -1,82 +0,0 @@
|
||||
import { format as utilFormat } from 'node:util'
|
||||
|
||||
import type { LogMessage } from '@afilmory/builder/logger/index.js'
|
||||
import { setLogListener } from '@afilmory/builder/logger/index.js'
|
||||
|
||||
import type { DataSyncLogLevel, DataSyncProgressEmitter } from './data-sync.types'
|
||||
|
||||
const LEVEL_MAP: Record<string, DataSyncLogLevel> = {
|
||||
log: 'info',
|
||||
info: 'info',
|
||||
start: 'info',
|
||||
success: 'success',
|
||||
warn: 'warn',
|
||||
error: 'error',
|
||||
fatal: 'error',
|
||||
debug: 'info',
|
||||
trace: 'info',
|
||||
}
|
||||
|
||||
export async function runWithBuilderLogRelay<T>(
|
||||
emitter: DataSyncProgressEmitter | undefined,
|
||||
task: () => Promise<T>,
|
||||
): Promise<T> {
|
||||
if (!emitter) {
|
||||
return await task()
|
||||
}
|
||||
|
||||
const listener = (message: LogMessage): void => {
|
||||
forwardBuilderLog(emitter, message)
|
||||
}
|
||||
|
||||
setLogListener(listener, { forwardToConsole: true })
|
||||
|
||||
try {
|
||||
return await task()
|
||||
} finally {
|
||||
setLogListener(null, { forwardToConsole: true })
|
||||
}
|
||||
}
|
||||
|
||||
function forwardBuilderLog(emitter: DataSyncProgressEmitter, message: LogMessage): void {
|
||||
const formatted = formatBuilderMessage(message)
|
||||
if (!formatted) {
|
||||
return
|
||||
}
|
||||
|
||||
const level = LEVEL_MAP[message.level] ?? 'info'
|
||||
|
||||
try {
|
||||
void emitter({
|
||||
type: 'log',
|
||||
payload: {
|
||||
level,
|
||||
message: formatted,
|
||||
timestamp: message.timestamp.toISOString(),
|
||||
stage: null,
|
||||
storageKey: undefined,
|
||||
details: {
|
||||
source: 'builder',
|
||||
tag: message.tag,
|
||||
},
|
||||
},
|
||||
})
|
||||
} catch {
|
||||
// Relay should never break builder logging
|
||||
}
|
||||
}
|
||||
|
||||
function formatBuilderMessage(message: LogMessage): string {
|
||||
const prefix = message.tag ? `[${message.tag}] ` : ''
|
||||
|
||||
if (!message.args?.length) {
|
||||
return prefix.trim()
|
||||
}
|
||||
|
||||
try {
|
||||
return `${prefix}${utilFormat(...message.args)}`.trim()
|
||||
} catch {
|
||||
const fallback = message.args[0] ? String(message.args[0]) : ''
|
||||
return `${prefix}${fallback}`.trim()
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,6 @@ import { Roles } from 'core/guards/roles.decorator'
|
||||
import { createProgressSseResponse } from 'core/modules/shared/http/sse'
|
||||
import type { Context } from 'hono'
|
||||
|
||||
import { runWithBuilderLogRelay } from './builder-log-relay'
|
||||
import type { ResolveConflictInput, RunDataSyncInput } from './data-sync.dto'
|
||||
import { ResolveConflictDto, RunDataSyncDto } from './data-sync.dto'
|
||||
import { DataSyncService } from './data-sync.service'
|
||||
@@ -33,15 +32,13 @@ export class DataSyncController {
|
||||
}
|
||||
|
||||
try {
|
||||
await runWithBuilderLogRelay(progressHandler, () =>
|
||||
this.dataSyncService.runSync(
|
||||
{
|
||||
builderConfig: payload.builderConfig as BuilderConfig | undefined,
|
||||
storageConfig: payload.storageConfig as StorageConfig | undefined,
|
||||
dryRun: payload.dryRun ?? false,
|
||||
},
|
||||
progressHandler,
|
||||
),
|
||||
await this.dataSyncService.runSync(
|
||||
{
|
||||
builderConfig: payload.builderConfig as BuilderConfig | undefined,
|
||||
storageConfig: payload.storageConfig as StorageConfig | undefined,
|
||||
dryRun: payload.dryRun ?? false,
|
||||
},
|
||||
progressHandler,
|
||||
)
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'Unknown error'
|
||||
|
||||
@@ -3,12 +3,13 @@ import { BuilderConfigService } from 'core/modules/configuration/builder-config/
|
||||
import { SystemSettingModule } from 'core/modules/configuration/system-setting/system-setting.module'
|
||||
import { BillingModule } from 'core/modules/platform/billing/billing.module'
|
||||
|
||||
import { BuilderWorkerHost } from 'core/workers/builder/builder-worker.host'
|
||||
import { DataSyncController } from './data-sync.controller'
|
||||
import { DataSyncService } from './data-sync.service'
|
||||
|
||||
@Module({
|
||||
imports: [SystemSettingModule, BillingModule],
|
||||
controllers: [DataSyncController],
|
||||
providers: [DataSyncService, BuilderConfigService],
|
||||
providers: [DataSyncService, BuilderConfigService, BuilderWorkerHost],
|
||||
})
|
||||
export class DataSyncModule {}
|
||||
|
||||
@@ -1,6 +1,13 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
|
||||
import type { BuilderConfig, PhotoManifestItem, StorageConfig, StorageManager, StorageObject } from '@afilmory/builder'
|
||||
import type {
|
||||
BuilderConfig,
|
||||
PhotoManifestItem,
|
||||
PhotoProcessorOptions,
|
||||
StorageConfig,
|
||||
StorageManager,
|
||||
StorageObject,
|
||||
} from '@afilmory/builder'
|
||||
import type { PhotoAssetConflictPayload, PhotoAssetConflictSnapshot, PhotoAssetManifest } from '@afilmory/db'
|
||||
import { CURRENT_PHOTO_MANIFEST_VERSION, DATABASE_ONLY_PROVIDER, photoAssets, photoSyncRuns } from '@afilmory/db'
|
||||
import { createLogger, EventEmitterService } from '@afilmory/framework'
|
||||
@@ -12,6 +19,8 @@ import { BILLING_USAGE_EVENT } from 'core/modules/platform/billing/billing.const
|
||||
import { BillingPlanService } from 'core/modules/platform/billing/billing-plan.service'
|
||||
import { BillingUsageService } from 'core/modules/platform/billing/billing-usage.service'
|
||||
import { requireTenantContext } from 'core/modules/platform/tenant/tenant.context'
|
||||
import { BuilderWorkerHost } from 'core/workers/builder/builder-worker.host'
|
||||
import type { BuilderWorkerLogEvent } from 'core/workers/builder/builder-worker.types'
|
||||
import { and, desc, eq } from 'drizzle-orm'
|
||||
import { injectable } from 'tsyringe'
|
||||
|
||||
@@ -57,6 +66,7 @@ type PhotoSyncRunRow = typeof photoSyncRuns.$inferSelect
|
||||
interface SyncPreparation {
|
||||
tenantId: string
|
||||
builder: ReturnType<PhotoBuilderService['createBuilder']>
|
||||
builderConfig: BuilderConfig
|
||||
storageManager: StorageManager
|
||||
effectiveStorageConfig: StorageConfig
|
||||
storageObjects: StorageObject[]
|
||||
@@ -78,6 +88,7 @@ export class DataSyncService {
|
||||
private readonly eventEmitter: EventEmitterService,
|
||||
private readonly dbAccessor: DbAccessor,
|
||||
private readonly photoBuilderService: PhotoBuilderService,
|
||||
private readonly builderWorkerHost: BuilderWorkerHost,
|
||||
private readonly photoStorageService: PhotoStorageService,
|
||||
private readonly billingPlanService: BillingPlanService,
|
||||
private readonly billingUsageService: BillingUsageService,
|
||||
@@ -382,6 +393,7 @@ export class DataSyncService {
|
||||
return {
|
||||
tenantId,
|
||||
builder,
|
||||
builderConfig,
|
||||
storageManager,
|
||||
effectiveStorageConfig,
|
||||
storageObjects,
|
||||
@@ -433,7 +445,7 @@ export class DataSyncService {
|
||||
}
|
||||
|
||||
const livePhotoMap = await this.ensureLivePhotoMap(context, dryRun)
|
||||
const { db, tenantId, effectiveStorageConfig, builder } = context
|
||||
const { db, tenantId, effectiveStorageConfig, builderConfig } = context
|
||||
let processed = 0
|
||||
|
||||
for (const storageObject of context.missingInDb) {
|
||||
@@ -465,13 +477,20 @@ export class DataSyncService {
|
||||
continue
|
||||
}
|
||||
|
||||
const result = await this.safeProcessStorageObject(storageObject, builder, {
|
||||
livePhotoMap,
|
||||
progress: {
|
||||
emitter: onProgress,
|
||||
stage: 'missing-in-db',
|
||||
const result = await this.safeProcessStorageObject(
|
||||
storageObject,
|
||||
{
|
||||
builderConfig,
|
||||
storageConfig: effectiveStorageConfig,
|
||||
},
|
||||
})
|
||||
{
|
||||
livePhotoMap,
|
||||
progress: {
|
||||
emitter: onProgress,
|
||||
stage: 'missing-in-db',
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
if (!result?.item) {
|
||||
summary.errors += 1
|
||||
@@ -1254,10 +1273,14 @@ export class DataSyncService {
|
||||
|
||||
private async safeProcessStorageObject(
|
||||
storageObject: StorageObject,
|
||||
builder: ReturnType<PhotoBuilderService['createBuilder']>,
|
||||
builderContext: {
|
||||
builderConfig: BuilderConfig
|
||||
storageConfig: StorageConfig
|
||||
},
|
||||
options: {
|
||||
existing?: PhotoManifestItem | null
|
||||
livePhotoMap?: Map<string, StorageObject>
|
||||
processorOptions?: Partial<PhotoProcessorOptions>
|
||||
progress?: {
|
||||
emitter?: DataSyncProgressEmitter
|
||||
stage?: DataSyncProgressStage | null
|
||||
@@ -1279,15 +1302,25 @@ export class DataSyncService {
|
||||
},
|
||||
})
|
||||
|
||||
const baseProcessorOptions: Partial<PhotoProcessorOptions> = {
|
||||
isForceMode: true,
|
||||
isForceManifest: true,
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await this.photoBuilderService.processPhotoFromStorageObject(storageObject, {
|
||||
const result = await this.builderWorkerHost.processPhoto({
|
||||
builderConfig: builderContext.builderConfig,
|
||||
storageConfig: builderContext.storageConfig,
|
||||
storageObject,
|
||||
existingItem: options.existing ?? undefined,
|
||||
livePhotoMap: options.livePhotoMap,
|
||||
processorOptions: {
|
||||
isForceMode: true,
|
||||
isForceManifest: true,
|
||||
...baseProcessorOptions,
|
||||
...options.processorOptions,
|
||||
},
|
||||
builder,
|
||||
logHandler: emitter
|
||||
? (event) => this.forwardBuilderLog(event, emitter, stage, storageObject.key)
|
||||
: undefined,
|
||||
})
|
||||
|
||||
if (result?.item) {
|
||||
@@ -1330,6 +1363,29 @@ export class DataSyncService {
|
||||
}
|
||||
}
|
||||
|
||||
private forwardBuilderLog(
|
||||
event: BuilderWorkerLogEvent,
|
||||
emitter: DataSyncProgressEmitter | undefined,
|
||||
stage: DataSyncProgressStage | null,
|
||||
storageKey: string,
|
||||
): void {
|
||||
if (!emitter) {
|
||||
return
|
||||
}
|
||||
|
||||
void this.emitLog(emitter, {
|
||||
level: event.level as DataSyncLogLevel,
|
||||
message: event.message,
|
||||
stage,
|
||||
storageKey,
|
||||
details: {
|
||||
...(event.details ?? {}),
|
||||
tag: event.tag ?? undefined,
|
||||
source: event.details?.source ?? 'builder',
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
private createStorageSnapshot(object: StorageObject): SyncObjectSnapshot {
|
||||
const lastModified = this.toIso(object.lastModified)
|
||||
const metadataHash = this.computeMetadataHash({
|
||||
@@ -1614,6 +1670,10 @@ export class DataSyncService {
|
||||
|
||||
await builder.ensurePluginsReady()
|
||||
const storageManager = builder.getStorageManager()
|
||||
const builderContext = {
|
||||
builderConfig,
|
||||
storageConfig: effectiveStorageConfig,
|
||||
}
|
||||
|
||||
if (payload.type === 'missing-in-storage') {
|
||||
if (dryRun) {
|
||||
@@ -1666,9 +1726,13 @@ export class DataSyncService {
|
||||
})
|
||||
}
|
||||
|
||||
const processResult = await this.safeProcessStorageObject(storageObject, builder, {
|
||||
existing: record.manifest?.data as PhotoManifestItem | undefined,
|
||||
})
|
||||
const processResult = await this.safeProcessStorageObject(
|
||||
storageObject,
|
||||
builderContext,
|
||||
{
|
||||
existing: record.manifest?.data as PhotoManifestItem | undefined,
|
||||
},
|
||||
)
|
||||
if (!processResult?.item) {
|
||||
throw new BizException(ErrorCode.IMAGE_PROCESSING_FAILED, {
|
||||
message: 'Failed to reprocess incoming storage object.',
|
||||
@@ -1726,9 +1790,13 @@ export class DataSyncService {
|
||||
})
|
||||
}
|
||||
|
||||
const processResult = await this.safeProcessStorageObject(storageObject, builder, {
|
||||
existing: record.manifest?.data as PhotoManifestItem | undefined,
|
||||
})
|
||||
const processResult = await this.safeProcessStorageObject(
|
||||
storageObject,
|
||||
builderContext,
|
||||
{
|
||||
existing: record.manifest?.data as PhotoManifestItem | undefined,
|
||||
},
|
||||
)
|
||||
if (!processResult?.item) {
|
||||
throw new BizException(ErrorCode.IMAGE_PROCESSING_FAILED, { message: 'Failed to reprocess storage object.' })
|
||||
}
|
||||
|
||||
@@ -55,7 +55,15 @@ export type StorageResolution = {
|
||||
builderConfig: BuilderConfig
|
||||
storageConfig: StorageConfig
|
||||
storageManager: StorageManager
|
||||
storageProvider: InMemoryDebugStorageProvider
|
||||
}
|
||||
|
||||
export type DebugStorageFileRecord = {
|
||||
key: string
|
||||
metadata: StorageObject
|
||||
buffer: Buffer
|
||||
}
|
||||
|
||||
export class InMemoryDebugStorageProvider implements StorageProvider {
|
||||
private readonly files = new Map<
|
||||
string,
|
||||
@@ -65,6 +73,17 @@ export class InMemoryDebugStorageProvider implements StorageProvider {
|
||||
}
|
||||
>()
|
||||
|
||||
constructor(initialFiles?: DebugStorageFileRecord[]) {
|
||||
if (initialFiles?.length) {
|
||||
for (const file of initialFiles) {
|
||||
this.files.set(file.key, {
|
||||
buffer: Buffer.from(file.buffer),
|
||||
metadata: { ...file.metadata },
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async getFile(key: string): Promise<Buffer | null> {
|
||||
return this.files.get(key)?.buffer ?? null
|
||||
}
|
||||
@@ -141,4 +160,12 @@ export class InMemoryDebugStorageProvider implements StorageProvider {
|
||||
private normalizeKey(key: string): string {
|
||||
return key.replaceAll('\\', '/').replaceAll(/^\/+|\/+$/g, '')
|
||||
}
|
||||
|
||||
exportSnapshot(): DebugStorageFileRecord[] {
|
||||
return Array.from(this.files.entries()).map(([key, entry]) => ({
|
||||
key,
|
||||
buffer: Buffer.from(entry.buffer),
|
||||
metadata: { ...entry.metadata },
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,9 +13,10 @@ import { BizException, ErrorCode } from 'core/errors'
|
||||
import { Roles } from 'core/guards/roles.decorator'
|
||||
import { BypassResponseTransform } from 'core/interceptors/response-transform.decorator'
|
||||
import { PhotoBuilderService } from 'core/modules/content/photo/builder/photo-builder.service'
|
||||
import { runWithBuilderLogRelay } from 'core/modules/infrastructure/data-sync/builder-log-relay'
|
||||
import type { DataSyncProgressEmitter } from 'core/modules/infrastructure/data-sync/data-sync.types'
|
||||
import { createProgressSseResponse } from 'core/modules/shared/http/sse'
|
||||
import { BuilderWorkerHost } from 'core/workers/builder/builder-worker.host'
|
||||
import type { BuilderWorkerProviderSnapshotInput } from 'core/workers/builder/builder-worker.host'
|
||||
import type { Context } from 'hono'
|
||||
|
||||
import type { BuilderDebugProgressEvent, StorageResolution, UploadedDebugFile } from './InMemoryDebugStorageProvider'
|
||||
@@ -29,7 +30,10 @@ const DEBUG_STORAGE_PROVIDER = 'super-admin-debug-storage'
|
||||
export class SuperAdminBuilderDebugController {
|
||||
private readonly logger = createLogger('SuperAdminBuilderDebugController')
|
||||
|
||||
constructor(private readonly photoBuilderService: PhotoBuilderService) {}
|
||||
constructor(
|
||||
private readonly photoBuilderService: PhotoBuilderService,
|
||||
private readonly builderWorkerHost: BuilderWorkerHost,
|
||||
) {}
|
||||
|
||||
@Post('debug')
|
||||
@BypassResponseTransform()
|
||||
@@ -98,7 +102,8 @@ export class SuperAdminBuilderDebugController {
|
||||
builderConfig.user.storage = storageConfig
|
||||
|
||||
const builder = this.photoBuilderService.createBuilder(builderConfig)
|
||||
builder.registerStorageProvider(DEBUG_STORAGE_PROVIDER, () => new InMemoryDebugStorageProvider(), {
|
||||
const storageProvider = new InMemoryDebugStorageProvider()
|
||||
builder.registerStorageProvider(DEBUG_STORAGE_PROVIDER, () => storageProvider, {
|
||||
category: 'local',
|
||||
})
|
||||
this.photoBuilderService.applyStorageConfig(builder, storageConfig)
|
||||
@@ -108,6 +113,7 @@ export class SuperAdminBuilderDebugController {
|
||||
builderConfig,
|
||||
storageConfig,
|
||||
storageManager: builder.getStorageManager(),
|
||||
storageProvider,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -118,7 +124,7 @@ export class SuperAdminBuilderDebugController {
|
||||
logEmitter: DataSyncProgressEmitter
|
||||
},
|
||||
) {
|
||||
const { builder, builderConfig, storageManager, file, sendEvent, logEmitter } = params
|
||||
const { builder, builderConfig, storageManager, storageConfig, storageProvider, file, sendEvent, logEmitter } = params
|
||||
const cleanupKeys = new Set<string>()
|
||||
|
||||
const tempKey = this.createTemporaryStorageKey(file.name)
|
||||
@@ -138,20 +144,37 @@ export class SuperAdminBuilderDebugController {
|
||||
},
|
||||
})
|
||||
|
||||
let processed: Awaited<ReturnType<typeof this.photoBuilderService.processPhotoFromStorageObject>> | null = null
|
||||
let processed: Awaited<ReturnType<BuilderWorkerHost['processPhoto']>> | null = null
|
||||
let filesDeleted = false
|
||||
try {
|
||||
processed = await runWithBuilderLogRelay(logEmitter, () =>
|
||||
this.photoBuilderService.processPhotoFromStorageObject(normalizedObject, {
|
||||
builder,
|
||||
builderConfig,
|
||||
processorOptions: {
|
||||
isForceMode: true,
|
||||
isForceManifest: true,
|
||||
isForceThumbnails: true,
|
||||
},
|
||||
}),
|
||||
)
|
||||
const providerSnapshot = this.createDebugProviderSnapshot(storageConfig.provider, storageProvider)
|
||||
processed = await this.builderWorkerHost.processPhoto({
|
||||
builderConfig,
|
||||
storageConfig,
|
||||
storageObject: normalizedObject,
|
||||
processorOptions: {
|
||||
isForceMode: true,
|
||||
isForceManifest: true,
|
||||
isForceThumbnails: true,
|
||||
},
|
||||
logHandler: (event) => {
|
||||
void logEmitter({
|
||||
type: 'log',
|
||||
payload: {
|
||||
level: event.level,
|
||||
message: event.message,
|
||||
stage: null,
|
||||
storageKey: normalizedObject.key,
|
||||
details: {
|
||||
...(event.details ?? {}),
|
||||
tag: event.tag ?? undefined,
|
||||
},
|
||||
timestamp: event.timestamp,
|
||||
},
|
||||
})
|
||||
},
|
||||
providerSnapshots: [providerSnapshot],
|
||||
})
|
||||
|
||||
const thumbnailKey = this.resolveThumbnailStorageKey(processed?.pluginData)
|
||||
if (thumbnailKey) {
|
||||
@@ -233,6 +256,22 @@ export class SuperAdminBuilderDebugController {
|
||||
return safeSegments.join('/')
|
||||
}
|
||||
|
||||
private createDebugProviderSnapshot(
|
||||
provider: string,
|
||||
storageProvider: InMemoryDebugStorageProvider,
|
||||
): BuilderWorkerProviderSnapshotInput {
|
||||
const files = storageProvider.exportSnapshot()
|
||||
return {
|
||||
type: 'in-memory-debug',
|
||||
provider,
|
||||
files: files.map((file) => ({
|
||||
key: file.key,
|
||||
metadata: file.metadata,
|
||||
buffer: file.buffer,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
private async cleanupDebugArtifacts(storageManager: StorageManager, keys: Set<string>): Promise<boolean> {
|
||||
let success = true
|
||||
for (const key of keys) {
|
||||
|
||||
@@ -4,6 +4,7 @@ import { PhotoBuilderService } from 'core/modules/content/photo/builder/photo-bu
|
||||
import { BillingModule } from 'core/modules/platform/billing/billing.module'
|
||||
import { TenantModule } from 'core/modules/platform/tenant/tenant.module'
|
||||
|
||||
import { BuilderWorkerHost } from 'core/workers/builder/builder-worker.host'
|
||||
import { SuperAdminBuilderDebugController } from './super-admin-builder.controller'
|
||||
import { SuperAdminSettingController } from './super-admin-settings.controller'
|
||||
import { SuperAdminTenantController } from './super-admin-tenants.controller'
|
||||
@@ -11,6 +12,6 @@ import { SuperAdminTenantController } from './super-admin-tenants.controller'
|
||||
@Module({
|
||||
imports: [SystemSettingModule, BillingModule, TenantModule],
|
||||
controllers: [SuperAdminSettingController, SuperAdminBuilderDebugController, SuperAdminTenantController],
|
||||
providers: [PhotoBuilderService],
|
||||
providers: [PhotoBuilderService, BuilderWorkerHost],
|
||||
})
|
||||
export class SuperAdminModule {}
|
||||
|
||||
37
be/apps/core/src/workers/builder/builder-log.helpers.ts
Normal file
37
be/apps/core/src/workers/builder/builder-log.helpers.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
import { format as utilFormat } from 'node:util'
|
||||
|
||||
import type { LogMessage } from '@afilmory/builder/logger/index.js'
|
||||
|
||||
export type BuilderWorkerLogLevel = 'info' | 'success' | 'warn' | 'error'
|
||||
|
||||
const LEVEL_MAP: Record<string, BuilderWorkerLogLevel> = {
|
||||
log: 'info',
|
||||
info: 'info',
|
||||
start: 'info',
|
||||
success: 'success',
|
||||
warn: 'warn',
|
||||
error: 'error',
|
||||
fatal: 'error',
|
||||
debug: 'info',
|
||||
trace: 'info',
|
||||
}
|
||||
|
||||
export function mapBuilderLogLevel(level: string): BuilderWorkerLogLevel {
|
||||
return LEVEL_MAP[level] ?? 'info'
|
||||
}
|
||||
|
||||
export function formatBuilderLogMessage(message: LogMessage): string | null {
|
||||
const prefix = message.tag ? `[${message.tag}] ` : ''
|
||||
|
||||
if (!message.args?.length) {
|
||||
return prefix.trim() || null
|
||||
}
|
||||
|
||||
try {
|
||||
return `${prefix}${utilFormat(...message.args)}`.trim()
|
||||
} catch {
|
||||
const fallback = message.args[0] ? String(message.args[0]) : ''
|
||||
const formatted = `${prefix}${fallback}`.trim()
|
||||
return formatted || null
|
||||
}
|
||||
}
|
||||
257
be/apps/core/src/workers/builder/builder-worker.host.ts
Normal file
257
be/apps/core/src/workers/builder/builder-worker.host.ts
Normal file
@@ -0,0 +1,257 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { existsSync, mkdirSync, writeFileSync } from 'node:fs'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url'
|
||||
import type { TransferListItem } from 'node:worker_threads'
|
||||
import { MessageChannel, Worker } from 'node:worker_threads'
|
||||
|
||||
import type {
|
||||
BuilderConfig,
|
||||
PhotoManifestItem,
|
||||
PhotoProcessorOptions,
|
||||
ProcessPhotoResult,
|
||||
StorageConfig,
|
||||
StorageObject,
|
||||
} from '@afilmory/builder'
|
||||
import { createLogger } from '@afilmory/framework'
|
||||
import { injectable } from 'tsyringe'
|
||||
|
||||
import type {
|
||||
BuilderWorkerLogEvent,
|
||||
BuilderWorkerProviderSnapshot,
|
||||
BuilderWorkerRequest,
|
||||
BuilderWorkerResponse,
|
||||
} from './builder-worker.types'
|
||||
import { serializeLivePhotoMap, serializePrefetchedBuffers, serializeStorageObject } from './builder-worker.types'
|
||||
|
||||
type PendingRequest = {
|
||||
resolve: (value: ProcessPhotoResult | null) => void
|
||||
reject: (error: Error) => void
|
||||
cleanup?: () => void
|
||||
}
|
||||
|
||||
export type BuilderWorkerLogHandler = (event: BuilderWorkerLogEvent) => void
|
||||
|
||||
export type BuilderWorkerProviderSnapshotInput = {
|
||||
type: 'in-memory-debug'
|
||||
provider: string
|
||||
files: Array<{
|
||||
key: string
|
||||
metadata: StorageObject
|
||||
buffer: Buffer
|
||||
}>
|
||||
}
|
||||
|
||||
type ProcessPhotoArgs = {
|
||||
builderConfig: BuilderConfig
|
||||
storageConfig?: StorageConfig
|
||||
storageObject: StorageObject
|
||||
existingItem?: PhotoManifestItem
|
||||
livePhotoMap?: Map<string, StorageObject>
|
||||
processorOptions?: Partial<PhotoProcessorOptions>
|
||||
logHandler?: BuilderWorkerLogHandler
|
||||
providerSnapshots?: BuilderWorkerProviderSnapshotInput[]
|
||||
prefetchedBuffers?: Map<string, Buffer>
|
||||
}
|
||||
|
||||
@injectable()
|
||||
export class BuilderWorkerHost {
|
||||
private worker: Worker | null = null
|
||||
private readonly pendingRequests = new Map<string, PendingRequest>()
|
||||
private readonly logger = createLogger('BuilderWorkerHost')
|
||||
private devWorkerUrl: URL | null = null
|
||||
|
||||
async processPhoto(args: ProcessPhotoArgs): Promise<ProcessPhotoResult | null> {
|
||||
const transferList: TransferListItem[] = []
|
||||
const providerSnapshots = this.serializeProviderSnapshots(args.providerSnapshots, transferList)
|
||||
const prefetchedBuffers = serializePrefetchedBuffers(args.prefetchedBuffers, transferList)
|
||||
const messageId = randomUUID()
|
||||
const message: BuilderWorkerRequest = {
|
||||
id: messageId,
|
||||
type: 'process-photo',
|
||||
payload: {
|
||||
builderConfig: args.builderConfig,
|
||||
storageConfig: args.storageConfig,
|
||||
storageObject: serializeStorageObject(args.storageObject),
|
||||
existingItem: args.existingItem,
|
||||
livePhotoMap: serializeLivePhotoMap(args.livePhotoMap),
|
||||
processorOptions: args.processorOptions,
|
||||
providerSnapshots,
|
||||
prefetchedBuffers,
|
||||
},
|
||||
}
|
||||
|
||||
let cleanup: (() => void) | undefined
|
||||
if (args.logHandler) {
|
||||
const { port1, port2 } = new MessageChannel()
|
||||
port1.on('message', (event: BuilderWorkerLogEvent) => {
|
||||
try {
|
||||
args.logHandler?.(event)
|
||||
} catch (error) {
|
||||
this.logger.error('BuilderWorkerHost log handler failed', error)
|
||||
}
|
||||
})
|
||||
transferList.push(port2)
|
||||
message.logPort = port2
|
||||
cleanup = () => {
|
||||
port1.removeAllListeners()
|
||||
port1.close()
|
||||
}
|
||||
}
|
||||
|
||||
return await this.dispatch(message, transferList, cleanup)
|
||||
}
|
||||
|
||||
private async dispatch(
|
||||
message: BuilderWorkerRequest,
|
||||
transferList: TransferListItem[],
|
||||
cleanup?: () => void,
|
||||
): Promise<ProcessPhotoResult | null> {
|
||||
const worker = this.ensureWorker()
|
||||
return await new Promise<ProcessPhotoResult | null>((resolve, reject) => {
|
||||
this.pendingRequests.set(message.id, {
|
||||
resolve,
|
||||
reject,
|
||||
cleanup,
|
||||
})
|
||||
|
||||
try {
|
||||
worker.postMessage(message, transferList)
|
||||
} catch (error) {
|
||||
this.pendingRequests.delete(message.id)
|
||||
cleanup?.()
|
||||
reject(
|
||||
error instanceof Error ? error : new Error('Failed to post message to builder worker thread, unknown error.'),
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
private serializeProviderSnapshots(
|
||||
inputs: BuilderWorkerProviderSnapshotInput[] | undefined,
|
||||
transferList: TransferListItem[],
|
||||
): BuilderWorkerProviderSnapshot[] | undefined {
|
||||
if (!inputs?.length) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
return inputs.map((input) => {
|
||||
switch (input.type) {
|
||||
case 'in-memory-debug': {
|
||||
return {
|
||||
type: 'in-memory-debug',
|
||||
provider: input.provider,
|
||||
files: input.files.map((file) => {
|
||||
const buffer = file.buffer.buffer.slice(
|
||||
file.buffer.byteOffset,
|
||||
file.buffer.byteOffset + file.buffer.byteLength,
|
||||
)
|
||||
transferList.push(buffer)
|
||||
return {
|
||||
key: file.key,
|
||||
metadata: serializeStorageObject(file.metadata),
|
||||
buffer,
|
||||
}
|
||||
}),
|
||||
}
|
||||
}
|
||||
default: {
|
||||
throw new Error(`Unsupported provider snapshot type: ${input satisfies never}`)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
private ensureWorker(): Worker {
|
||||
if (this.worker) {
|
||||
return this.worker
|
||||
}
|
||||
|
||||
const workerUrl = this.resolveWorkerUrl()
|
||||
const worker = new Worker(workerUrl, {
|
||||
env: process.env,
|
||||
execArgv: this.resolveExecArgv(),
|
||||
type: 'module',
|
||||
})
|
||||
|
||||
worker.on('message', (response: BuilderWorkerResponse) => this.handleWorkerResponse(response))
|
||||
worker.on('error', (error) => this.handleWorkerError(error))
|
||||
worker.on('exit', (code) => this.handleWorkerExit(code))
|
||||
|
||||
this.worker = worker
|
||||
return worker
|
||||
}
|
||||
|
||||
private resolveExecArgv(): string[] {
|
||||
return [...(process.execArgv ?? [])]
|
||||
}
|
||||
|
||||
private resolveWorkerUrl(): URL {
|
||||
const jsUrl = new URL('./builder.worker.js', import.meta.url)
|
||||
if (this.isFileAvailable(jsUrl)) {
|
||||
return jsUrl
|
||||
}
|
||||
|
||||
if (this.devWorkerUrl) {
|
||||
return this.devWorkerUrl
|
||||
}
|
||||
|
||||
const tsUrl = new URL('./builder.worker.ts', import.meta.url)
|
||||
this.devWorkerUrl = this.createDevWorkerStub(tsUrl)
|
||||
return this.devWorkerUrl
|
||||
}
|
||||
|
||||
private isFileAvailable(url: URL): boolean {
|
||||
try {
|
||||
return existsSync(fileURLToPath(url))
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
private createDevWorkerStub(tsUrl: URL): URL {
|
||||
const cacheDir = join(process.cwd(), 'node_modules', '.cache', 'afilmory-builder-worker')
|
||||
mkdirSync(cacheDir, { recursive: true })
|
||||
const outFile = join(cacheDir, 'builder-worker-dev.mjs')
|
||||
const content = `import 'tsx/esm'\nimport ${JSON.stringify(tsUrl.href)}\n`
|
||||
writeFileSync(outFile, content)
|
||||
return pathToFileURL(outFile)
|
||||
}
|
||||
|
||||
private handleWorkerResponse(response: BuilderWorkerResponse): void {
|
||||
const pending = this.pendingRequests.get(response.id)
|
||||
if (!pending) {
|
||||
return
|
||||
}
|
||||
|
||||
this.pendingRequests.delete(response.id)
|
||||
pending.cleanup?.()
|
||||
|
||||
if (response.success) {
|
||||
pending.resolve(response.result)
|
||||
return
|
||||
}
|
||||
|
||||
const error = new Error(response.error.message)
|
||||
error.name = response.error.name ?? 'BuilderWorkerError'
|
||||
if (response.error.stack) {
|
||||
error.stack = response.error.stack
|
||||
}
|
||||
pending.reject(error)
|
||||
}
|
||||
|
||||
private handleWorkerError(error: unknown): void {
|
||||
this.logger.error('Builder worker crashed', error)
|
||||
}
|
||||
|
||||
private handleWorkerExit(code: number): void {
|
||||
this.logger.error(`Builder worker exited with code ${code}`)
|
||||
this.worker = null
|
||||
|
||||
for (const [id, pending] of this.pendingRequests.entries()) {
|
||||
pending.cleanup?.()
|
||||
pending.reject(new Error('Builder worker stopped unexpectedly.'))
|
||||
this.pendingRequests.delete(id)
|
||||
}
|
||||
}
|
||||
}
|
||||
140
be/apps/core/src/workers/builder/builder-worker.types.ts
Normal file
140
be/apps/core/src/workers/builder/builder-worker.types.ts
Normal file
@@ -0,0 +1,140 @@
|
||||
import type {
|
||||
BuilderConfig,
|
||||
PhotoManifestItem,
|
||||
PhotoProcessorOptions,
|
||||
ProcessPhotoResult,
|
||||
StorageConfig,
|
||||
StorageObject,
|
||||
} from '@afilmory/builder'
|
||||
import type { MessagePort, TransferListItem } from 'node:worker_threads'
|
||||
|
||||
import type { BuilderWorkerLogLevel } from './builder-log.helpers'
|
||||
|
||||
export type SerializedStorageObject = Omit<StorageObject, 'lastModified'> & {
|
||||
lastModified?: string | null
|
||||
}
|
||||
|
||||
export function serializeStorageObject(object: StorageObject): SerializedStorageObject {
|
||||
return {
|
||||
...object,
|
||||
lastModified: object.lastModified instanceof Date ? object.lastModified.toISOString() : object.lastModified ?? null,
|
||||
}
|
||||
}
|
||||
|
||||
export function deserializeStorageObject(object: SerializedStorageObject): StorageObject {
|
||||
return {
|
||||
...object,
|
||||
lastModified: object.lastModified ? new Date(object.lastModified) : undefined,
|
||||
}
|
||||
}
|
||||
|
||||
export type SerializedLivePhotoMap = Array<[string, SerializedStorageObject]>
|
||||
|
||||
export function serializeLivePhotoMap(map?: Map<string, StorageObject>): SerializedLivePhotoMap | undefined {
|
||||
if (!map?.size) {
|
||||
return undefined
|
||||
}
|
||||
return Array.from(map.entries()).map(([key, value]) => [key, serializeStorageObject(value)])
|
||||
}
|
||||
|
||||
export function deserializeLivePhotoMap(entries?: SerializedLivePhotoMap): Map<string, StorageObject> | undefined {
|
||||
if (!entries?.length) {
|
||||
return undefined
|
||||
}
|
||||
return new Map(entries.map(([key, value]) => [key, deserializeStorageObject(value)]))
|
||||
}
|
||||
|
||||
export type SerializedPrefetchedBufferEntry = {
|
||||
key: string
|
||||
buffer: ArrayBuffer
|
||||
}
|
||||
|
||||
export function serializePrefetchedBuffers(
|
||||
map: Map<string, Buffer> | undefined,
|
||||
transferList: TransferListItem[],
|
||||
): SerializedPrefetchedBufferEntry[] | undefined {
|
||||
if (!map?.size) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const entries: SerializedPrefetchedBufferEntry[] = []
|
||||
for (const [key, buffer] of map.entries()) {
|
||||
const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength)
|
||||
transferList.push(arrayBuffer)
|
||||
entries.push({ key, buffer: arrayBuffer })
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
export function deserializePrefetchedBuffers(
|
||||
entries: SerializedPrefetchedBufferEntry[] | undefined,
|
||||
): Map<string, Buffer> | undefined {
|
||||
if (!entries?.length) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
return new Map(entries.map((entry) => [entry.key, Buffer.from(entry.buffer)]))
|
||||
}
|
||||
|
||||
export type BuilderWorkerProviderSnapshot =
|
||||
| {
|
||||
type: 'in-memory-debug'
|
||||
provider: string
|
||||
files: Array<{
|
||||
key: string
|
||||
metadata: SerializedStorageObject
|
||||
buffer: ArrayBuffer
|
||||
}>
|
||||
}
|
||||
|
||||
export type BuilderWorkerRequest =
|
||||
| {
|
||||
id: string
|
||||
type: 'process-photo'
|
||||
payload: ProcessPhotoRequestPayload
|
||||
logPort?: MessagePort
|
||||
}
|
||||
|
||||
export type BuilderWorkerResponse =
|
||||
| {
|
||||
id: string
|
||||
type: 'process-photo'
|
||||
success: true
|
||||
result: ProcessPhotoResult | null
|
||||
}
|
||||
| {
|
||||
id: string
|
||||
type: 'process-photo'
|
||||
success: false
|
||||
error: SerializedWorkerError
|
||||
}
|
||||
|
||||
export type SerializedWorkerError = {
|
||||
message: string
|
||||
stack?: string
|
||||
name?: string
|
||||
}
|
||||
|
||||
export type BuilderWorkerLogEvent = {
|
||||
level: BuilderWorkerLogLevel
|
||||
message: string
|
||||
timestamp: string
|
||||
tag?: string | null
|
||||
details?: Record<string, unknown> | null
|
||||
}
|
||||
|
||||
export type ProcessPhotoRequestPayload = {
|
||||
builderConfig: BuilderConfig
|
||||
storageConfig?: StorageConfig
|
||||
storageObject: SerializedStorageObject
|
||||
existingItem?: PhotoManifestItem
|
||||
livePhotoMap?: SerializedLivePhotoMap
|
||||
processorOptions?: Partial<PhotoProcessorOptions>
|
||||
providerSnapshots?: BuilderWorkerProviderSnapshot[]
|
||||
prefetchedBuffers?: SerializedPrefetchedBufferEntry[]
|
||||
}
|
||||
|
||||
export type ProviderSnapshotSerializationResult = {
|
||||
snapshots?: BuilderWorkerProviderSnapshot[]
|
||||
transferList: TransferListItem[]
|
||||
}
|
||||
180
be/apps/core/src/workers/builder/builder.worker.ts
Normal file
180
be/apps/core/src/workers/builder/builder.worker.ts
Normal file
@@ -0,0 +1,180 @@
|
||||
import type { MessagePort } from 'node:worker_threads'
|
||||
import { parentPort } from 'node:worker_threads'
|
||||
|
||||
import type { LogMessage } from '@afilmory/builder/logger/index.js'
|
||||
import { setLogListener } from '@afilmory/builder/logger/index.js'
|
||||
import { PhotoBuilderService } from 'core/modules/content/photo/builder/photo-builder.service'
|
||||
import { InMemoryDebugStorageProvider } from 'core/modules/platform/super-admin/InMemoryDebugStorageProvider'
|
||||
import { formatBuilderLogMessage, mapBuilderLogLevel } from 'core/workers/builder/builder-log.helpers'
|
||||
import type {
|
||||
BuilderWorkerLogEvent,
|
||||
BuilderWorkerProviderSnapshot,
|
||||
BuilderWorkerRequest,
|
||||
BuilderWorkerResponse,
|
||||
} from 'core/workers/builder/builder-worker.types'
|
||||
import {
|
||||
deserializeLivePhotoMap,
|
||||
deserializePrefetchedBuffers,
|
||||
deserializeStorageObject,
|
||||
} from 'core/workers/builder/builder-worker.types'
|
||||
|
||||
const photoBuilderService = new PhotoBuilderService()
|
||||
|
||||
if (!parentPort) {
|
||||
throw new Error('Builder worker must be run inside a Node.js worker thread.')
|
||||
}
|
||||
|
||||
parentPort.on('message', (message: BuilderWorkerRequest) => {
|
||||
void handleMessage(message)
|
||||
})
|
||||
|
||||
async function handleMessage(message: BuilderWorkerRequest): Promise<void> {
|
||||
switch (message.type) {
|
||||
case 'process-photo': {
|
||||
await handleProcessPhoto(message)
|
||||
break
|
||||
}
|
||||
default: {
|
||||
parentPort?.postMessage({
|
||||
id: message.id,
|
||||
type: message.type,
|
||||
success: false,
|
||||
error: {
|
||||
message: `Unsupported builder worker message type: ${message.type satisfies never}`,
|
||||
},
|
||||
} satisfies BuilderWorkerResponse)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function handleProcessPhoto(message: Extract<BuilderWorkerRequest, { type: 'process-photo' }>): Promise<void> {
|
||||
const { logPort } = message
|
||||
const detachLog = logPort ? attachBuilderLogPort(logPort) : null
|
||||
|
||||
try {
|
||||
const result = await processPhoto(message.payload)
|
||||
parentPort?.postMessage({
|
||||
id: message.id,
|
||||
type: 'process-photo',
|
||||
success: true,
|
||||
result: result ?? null,
|
||||
} satisfies BuilderWorkerResponse)
|
||||
} catch (error) {
|
||||
parentPort?.postMessage({
|
||||
id: message.id,
|
||||
type: 'process-photo',
|
||||
success: false,
|
||||
error: serializeError(error),
|
||||
} satisfies BuilderWorkerResponse)
|
||||
} finally {
|
||||
detachLog?.()
|
||||
logPort?.close()
|
||||
}
|
||||
}
|
||||
|
||||
async function processPhoto(payload: {
|
||||
builderConfig: Extract<BuilderWorkerRequest, { type: 'process-photo' }>['payload']['builderConfig']
|
||||
storageConfig?: Extract<BuilderWorkerRequest, { type: 'process-photo' }>['payload']['storageConfig']
|
||||
storageObject: Extract<BuilderWorkerRequest, { type: 'process-photo' }>['payload']['storageObject']
|
||||
existingItem?: Extract<BuilderWorkerRequest, { type: 'process-photo' }>['payload']['existingItem']
|
||||
livePhotoMap?: Extract<BuilderWorkerRequest, { type: 'process-photo' }>['payload']['livePhotoMap']
|
||||
processorOptions?: Extract<BuilderWorkerRequest, { type: 'process-photo' }>['payload']['processorOptions']
|
||||
providerSnapshots?: BuilderWorkerProviderSnapshot[]
|
||||
}) {
|
||||
const builder = photoBuilderService.createBuilder(payload.builderConfig)
|
||||
|
||||
if (payload.providerSnapshots?.length) {
|
||||
registerProviderSnapshots(builder, payload.providerSnapshots)
|
||||
}
|
||||
|
||||
if (payload.storageConfig) {
|
||||
photoBuilderService.applyStorageConfig(builder, payload.storageConfig)
|
||||
}
|
||||
|
||||
const storageObject = deserializeStorageObject(payload.storageObject)
|
||||
const livePhotoMap = deserializeLivePhotoMap(payload.livePhotoMap)
|
||||
const prefetchedBuffers = deserializePrefetchedBuffers(payload.prefetchedBuffers)
|
||||
|
||||
return await photoBuilderService.processPhotoFromStorageObject(storageObject, {
|
||||
existingItem: payload.existingItem,
|
||||
livePhotoMap,
|
||||
processorOptions: payload.processorOptions,
|
||||
builder,
|
||||
prefetchedBuffers,
|
||||
})
|
||||
}
|
||||
|
||||
function registerProviderSnapshots(
|
||||
builder: ReturnType<PhotoBuilderService['createBuilder']>,
|
||||
snapshots: BuilderWorkerProviderSnapshot[],
|
||||
): void {
|
||||
for (const snapshot of snapshots) {
|
||||
switch (snapshot.type) {
|
||||
case 'in-memory-debug': {
|
||||
const provider = new InMemoryDebugStorageProvider(
|
||||
snapshot.files.map((file) => ({
|
||||
key: file.key,
|
||||
metadata: deserializeStorageObject(file.metadata),
|
||||
buffer: Buffer.from(file.buffer),
|
||||
})),
|
||||
)
|
||||
builder.registerStorageProvider(snapshot.provider, () => provider, {
|
||||
category: 'local',
|
||||
})
|
||||
break
|
||||
}
|
||||
default: {
|
||||
throw new Error(`Unsupported provider snapshot type: ${snapshot}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function attachBuilderLogPort(port: MessagePort): () => void {
|
||||
const listener = (log: LogMessage): void => {
|
||||
forwardBuilderLog(port, log)
|
||||
}
|
||||
|
||||
setLogListener(listener, { forwardToConsole: false })
|
||||
return () => {
|
||||
setLogListener(null, { forwardToConsole: false })
|
||||
}
|
||||
}
|
||||
|
||||
function forwardBuilderLog(port: MessagePort, message: LogMessage): void {
|
||||
const formatted = formatBuilderLogMessage(message)
|
||||
if (!formatted) {
|
||||
return
|
||||
}
|
||||
|
||||
const payload: BuilderWorkerLogEvent = {
|
||||
level: mapBuilderLogLevel(message.level),
|
||||
message: formatted,
|
||||
timestamp: message.timestamp?.toISOString() ?? new Date().toISOString(),
|
||||
tag: message.tag ?? null,
|
||||
details: {
|
||||
source: 'builder',
|
||||
tag: message.tag,
|
||||
},
|
||||
}
|
||||
|
||||
try {
|
||||
port.postMessage(payload)
|
||||
} catch {
|
||||
// Ignore log forwarding failures to avoid breaking worker execution.
|
||||
}
|
||||
}
|
||||
|
||||
function serializeError(error: unknown): BuilderWorkerResponse['error'] {
|
||||
if (error instanceof Error) {
|
||||
return {
|
||||
message: error.message,
|
||||
stack: error.stack,
|
||||
name: error.name,
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
message: typeof error === 'string' ? error : 'Unknown builder worker error',
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import { promisify } from 'node:util'
|
||||
import swc from 'unplugin-swc'
|
||||
import { defineConfig } from 'vite'
|
||||
import { analyzer } from 'vite-bundle-analyzer'
|
||||
import workerPlugin from 'vite-plugin-node-worker'
|
||||
import tsconfigPaths from 'vite-tsconfig-paths'
|
||||
|
||||
const NODE_BUILT_IN_MODULES = builtinModules.filter((m) => !m.startsWith('_'))
|
||||
@@ -127,7 +128,11 @@ export default defineConfig({
|
||||
swc.vite(),
|
||||
analyzer({ enabled: process.env.ANALYZER === 'true' }),
|
||||
generateExternalsPackageJson(external),
|
||||
workerPlugin(),
|
||||
],
|
||||
worker: {
|
||||
plugins: () => [workerPlugin()],
|
||||
},
|
||||
esbuild: false,
|
||||
resolve: {
|
||||
alias: {
|
||||
|
||||
@@ -5,31 +5,33 @@ import { defineBuilderConfig, githubRepoSyncPlugin } from '@afilmory/builder'
|
||||
import { env } from './env.js'
|
||||
|
||||
export default defineBuilderConfig(() => ({
|
||||
storage: {
|
||||
// "provider": "local",
|
||||
// "basePath": "./apps/web/public/photos",
|
||||
// "baseUrl": "/photos"
|
||||
user: {
|
||||
storage: {
|
||||
// "provider": "local",
|
||||
// "basePath": "./apps/web/public/photos",
|
||||
// "baseUrl": "/photos"
|
||||
|
||||
provider: 's3',
|
||||
bucket: env.S3_BUCKET_NAME,
|
||||
region: env.S3_REGION,
|
||||
endpoint: env.S3_ENDPOINT,
|
||||
accessKeyId: env.S3_ACCESS_KEY_ID,
|
||||
secretAccessKey: env.S3_SECRET_ACCESS_KEY,
|
||||
prefix: env.S3_PREFIX,
|
||||
customDomain: env.S3_CUSTOM_DOMAIN,
|
||||
excludeRegex: env.S3_EXCLUDE_REGEX,
|
||||
maxFileLimit: 1000,
|
||||
keepAlive: true,
|
||||
maxSockets: 64,
|
||||
connectionTimeoutMs: 5_000,
|
||||
socketTimeoutMs: 30_000,
|
||||
requestTimeoutMs: 20_000,
|
||||
idleTimeoutMs: 10_000,
|
||||
totalTimeoutMs: 60_000,
|
||||
retryMode: 'standard',
|
||||
maxAttempts: 3,
|
||||
downloadConcurrency: 16,
|
||||
provider: 's3',
|
||||
bucket: env.S3_BUCKET_NAME,
|
||||
region: env.S3_REGION,
|
||||
endpoint: env.S3_ENDPOINT,
|
||||
accessKeyId: env.S3_ACCESS_KEY_ID,
|
||||
secretAccessKey: env.S3_SECRET_ACCESS_KEY,
|
||||
prefix: env.S3_PREFIX,
|
||||
customDomain: env.S3_CUSTOM_DOMAIN,
|
||||
excludeRegex: env.S3_EXCLUDE_REGEX,
|
||||
maxFileLimit: 1000,
|
||||
keepAlive: true,
|
||||
maxSockets: 64,
|
||||
connectionTimeoutMs: 5_000,
|
||||
socketTimeoutMs: 30_000,
|
||||
requestTimeoutMs: 20_000,
|
||||
idleTimeoutMs: 10_000,
|
||||
totalTimeoutMs: 60_000,
|
||||
retryMode: 'standard',
|
||||
maxAttempts: 3,
|
||||
downloadConcurrency: 16,
|
||||
},
|
||||
},
|
||||
system: {
|
||||
processing: {
|
||||
@@ -59,7 +61,6 @@ export default defineBuilderConfig(() => ({
|
||||
plugins: [
|
||||
githubRepoSyncPlugin({
|
||||
repo: {
|
||||
enable: false,
|
||||
url: process.env.BUILDER_REPO_URL ?? '',
|
||||
token: env.GIT_TOKEN,
|
||||
branch: process.env.BUILDER_REPO_BRANCH ?? 'main',
|
||||
|
||||
84
pnpm-lock.yaml
generated
84
pnpm-lock.yaml
generated
@@ -725,6 +725,9 @@ importers:
|
||||
react-freeze:
|
||||
specifier: 1.0.4
|
||||
version: 1.0.4(react@19.2.0)
|
||||
react-grab:
|
||||
specifier: 0.0.39
|
||||
version: 0.0.39(@types/react@19.2.3)(react@19.2.0)
|
||||
react-i18next:
|
||||
specifier: 16.3.1
|
||||
version: 16.3.1(i18next@25.6.2(typescript@5.9.3))(react-dom@19.2.0(react@19.2.0))(react@19.2.0)(typescript@5.9.3)
|
||||
@@ -1010,6 +1013,9 @@ importers:
|
||||
vite-node:
|
||||
specifier: 5.0.0
|
||||
version: 5.0.0(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.1)(tsx@4.20.6)(yaml@2.8.1)
|
||||
vite-plugin-node-worker:
|
||||
specifier: 1.0.5
|
||||
version: 1.0.5
|
||||
vite-tsconfig-paths:
|
||||
specifier: 5.1.4
|
||||
version: 5.1.4(typescript@5.9.3)(vite@7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.1)(tsx@4.20.6)(yaml@2.8.1))
|
||||
@@ -3697,6 +3703,9 @@ packages:
|
||||
peerDependencies:
|
||||
rollup: '>=2'
|
||||
|
||||
'@medv/finder@4.0.2':
|
||||
resolution: {integrity: sha512-RraNY9SCcx4KZV0Dh6BEW6XEW2swkqYca74pkFFRw6hHItSHiy+O/xMnpbofjYbzXj0tSpBGthUF1hHTsr3vIQ==}
|
||||
|
||||
'@mermaid-js/parser@0.5.0':
|
||||
resolution: {integrity: sha512-AiaN7+VjXC+3BYE+GwNezkpjIcCI2qIMB/K4S2/vMWe0q/XJCBbx5+K7iteuz7VyltX9iAK4FmVTvGc9kjOV4w==}
|
||||
|
||||
@@ -6695,6 +6704,11 @@ packages:
|
||||
peerDependencies:
|
||||
react: '>=17.0.1'
|
||||
|
||||
bippy@0.5.14:
|
||||
resolution: {integrity: sha512-QrY2UHRFiIA8+2nkZfTLucGXhjshbkLnwGCCShQTO1FecJwDLHKOUBzOd1DArUsJnnomFZTggFRmj1Zxt4uamg==}
|
||||
peerDependencies:
|
||||
react: '>=17.0.1'
|
||||
|
||||
birecord@0.1.1:
|
||||
resolution: {integrity: sha512-VUpsf/qykW0heRlC8LooCq28Kxn3mAqKohhDG/49rrsQ1dT1CXyj/pgXS+5BSRzFTR/3DyIBOqQOrGyZOh71Aw==}
|
||||
|
||||
@@ -9505,6 +9519,9 @@ packages:
|
||||
mlly@1.8.0:
|
||||
resolution: {integrity: sha512-l8D9ODSRWLe2KHJSifWGwBqpTZXIXTeo8mlKjY+E2HAakaTeNpqAyBZ8GSqLzHgw4XmHmC8whvpjJNMbFZN7/g==}
|
||||
|
||||
modern-screenshot@4.6.7:
|
||||
resolution: {integrity: sha512-0GhgI6i6le4AhKzCvLYjwEmsP47kTsX45iT5yuAzsLTi/7i3Rjxe8fbH2VjGJLuyOThwsa0CdQAPd4auoEtsZg==}
|
||||
|
||||
motion-dom@12.23.23:
|
||||
resolution: {integrity: sha512-n5yolOs0TQQBRUFImrRfs/+6X4p3Q4n1dUEqt/H58Vx7OW6RF+foWEgmTVDhIWJIMXOuNNL0apKH2S16en9eiA==}
|
||||
|
||||
@@ -10501,6 +10518,9 @@ packages:
|
||||
peerDependencies:
|
||||
react: '>=17.0.0'
|
||||
|
||||
react-grab@0.0.39:
|
||||
resolution: {integrity: sha512-Q1R4mECia8giYHn4C85GzEY8EQISLB6uC8in4bUvtfLd50poIN88Umm037QtiK0ITROiYiMw9gOjU9Dhm6DxXQ==}
|
||||
|
||||
react-hotkeys-hook@5.1.0:
|
||||
resolution: {integrity: sha512-GCNGXjBzV9buOS3REoQFmSmE4WTvBhYQ0YrAeeMZI83bhXg3dRWsLHXDutcVDdEjwJqJCxk5iewWYX5LtFUd7g==}
|
||||
peerDependencies:
|
||||
@@ -11004,6 +11024,16 @@ packages:
|
||||
resolution: {integrity: sha512-owllqNuDDEimQat7EPG0tH7JjO090xKNzUtYz6X+Sk2BXDnOCilDdNLwjWeFywG9xkJul1ULvtUQa9O4pUaY0w==}
|
||||
engines: {node: '>=4.0.0'}
|
||||
|
||||
seroval-plugins@1.3.3:
|
||||
resolution: {integrity: sha512-16OL3NnUBw8JG1jBLUoZJsLnQq0n5Ua6aHalhJK4fMQkz1lqR7Osz1sA30trBtd9VUDc2NgkuRCn8+/pBwqZ+w==}
|
||||
engines: {node: '>=10'}
|
||||
peerDependencies:
|
||||
seroval: ^1.0
|
||||
|
||||
seroval@1.3.2:
|
||||
resolution: {integrity: sha512-RbcPH1n5cfwKrru7v7+zrZvjLurgHhGyso3HTyGtRivGWgYjbOmGuivCQaORNELjNONoK35nj28EoWul9sb1zQ==}
|
||||
engines: {node: '>=10'}
|
||||
|
||||
server-only@0.0.1:
|
||||
resolution: {integrity: sha512-qepMx2JxAa5jjfzxG79yPPq+8BuFToHd1hm7kI+Z4zAq1ftQiP7HcxMhDDItrbtwVeLg/cY2JnKnrcFkmiswNA==}
|
||||
|
||||
@@ -11098,6 +11128,14 @@ packages:
|
||||
smob@1.5.0:
|
||||
resolution: {integrity: sha512-g6T+p7QO8npa+/hNx9ohv1E5pVCmWrVCUzUXJyLdMmftX6ER0oiWY/w9knEonLpnOp6b6FenKnMfR8gqwWdwig==}
|
||||
|
||||
solid-js@1.9.10:
|
||||
resolution: {integrity: sha512-Coz956cos/EPDlhs6+jsdTxKuJDPT7B5SVIWgABwROyxjY7Xbr8wkzD68Et+NxnV7DLJ3nJdAC2r9InuV/4Jew==}
|
||||
|
||||
solid-sonner@0.2.8:
|
||||
resolution: {integrity: sha512-EQ2EIznvHHpAmkYh2CTu0AdCgmPJRJWLGFRWygE8j+vMEfvIV2wotHU5qgWzqzVTG1SODGsay2Lwq6ENWx/rPA==}
|
||||
peerDependencies:
|
||||
solid-js: ^1.6.0
|
||||
|
||||
sonner@2.0.7:
|
||||
resolution: {integrity: sha512-W6ZN4p58k8aDKA4XPcx2hpIQXBRAgyiWVkYhT7CvK6D3iAu7xjvVyhQHg2/iaKJZ1XVJ4r7XuwGL+WGEK37i9w==}
|
||||
peerDependencies:
|
||||
@@ -11993,6 +12031,10 @@ packages:
|
||||
peerDependencies:
|
||||
vite: '>=2.0.0'
|
||||
|
||||
vite-plugin-node-worker@1.0.5:
|
||||
resolution: {integrity: sha512-xYUs1PXWt2HpCjRCoRKCGLrAD63Ul/4Cns91ky+TUPR5lcyfv2lnRakfVdEW2WtxLxpIbRFYp3l0cfwF78f9gw==}
|
||||
engines: {node: '>=21'}
|
||||
|
||||
vite-plugin-pwa@1.1.0:
|
||||
resolution: {integrity: sha512-VsSpdubPzXhHWVINcSx6uHRMpOHVHQcHsef1QgkOlEoaIDAlssFEW88LBq1a59BuokAhsh2kUDJbaX1bZv4Bjw==}
|
||||
engines: {node: '>=16.0.0'}
|
||||
@@ -15013,6 +15055,8 @@ snapshots:
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
'@medv/finder@4.0.2': {}
|
||||
|
||||
'@mermaid-js/parser@0.5.0':
|
||||
dependencies:
|
||||
langium: 3.3.1
|
||||
@@ -18411,6 +18455,13 @@ snapshots:
|
||||
transitivePeerDependencies:
|
||||
- '@types/react'
|
||||
|
||||
bippy@0.5.14(@types/react@19.2.3)(react@19.2.0):
|
||||
dependencies:
|
||||
'@types/react-reconciler': 0.28.9(@types/react@19.2.3)
|
||||
react: 19.2.0
|
||||
transitivePeerDependencies:
|
||||
- '@types/react'
|
||||
|
||||
birecord@0.1.1: {}
|
||||
|
||||
birpc@2.8.0: {}
|
||||
@@ -21938,6 +21989,8 @@ snapshots:
|
||||
pkg-types: 1.3.1
|
||||
ufo: 1.6.1
|
||||
|
||||
modern-screenshot@4.6.7: {}
|
||||
|
||||
motion-dom@12.23.23:
|
||||
dependencies:
|
||||
motion-utils: 12.23.6
|
||||
@@ -23017,6 +23070,17 @@ snapshots:
|
||||
dependencies:
|
||||
react: 19.2.0
|
||||
|
||||
react-grab@0.0.39(@types/react@19.2.3)(react@19.2.0):
|
||||
dependencies:
|
||||
'@medv/finder': 4.0.2
|
||||
bippy: 0.5.14(@types/react@19.2.3)(react@19.2.0)
|
||||
modern-screenshot: 4.6.7
|
||||
solid-js: 1.9.10
|
||||
solid-sonner: 0.2.8(solid-js@1.9.10)
|
||||
transitivePeerDependencies:
|
||||
- '@types/react'
|
||||
- react
|
||||
|
||||
react-hotkeys-hook@5.1.0(react-dom@19.2.0(react@19.2.0))(react@19.2.0):
|
||||
dependencies:
|
||||
react: 19.2.0
|
||||
@@ -23712,6 +23776,12 @@ snapshots:
|
||||
serialize-to-js@3.1.2:
|
||||
optional: true
|
||||
|
||||
seroval-plugins@1.3.3(seroval@1.3.2):
|
||||
dependencies:
|
||||
seroval: 1.3.2
|
||||
|
||||
seroval@1.3.2: {}
|
||||
|
||||
server-only@0.0.1: {}
|
||||
|
||||
set-cookie-parser@2.7.2: {}
|
||||
@@ -23854,6 +23924,16 @@ snapshots:
|
||||
|
||||
smob@1.5.0: {}
|
||||
|
||||
solid-js@1.9.10:
|
||||
dependencies:
|
||||
csstype: 3.1.3
|
||||
seroval: 1.3.2
|
||||
seroval-plugins: 1.3.3(seroval@1.3.2)
|
||||
|
||||
solid-sonner@0.2.8(solid-js@1.9.10):
|
||||
dependencies:
|
||||
solid-js: 1.9.10
|
||||
|
||||
sonner@2.0.7(react-dom@19.2.0(react@19.2.0))(react@19.2.0):
|
||||
dependencies:
|
||||
react: 19.2.0
|
||||
@@ -24758,6 +24838,10 @@ snapshots:
|
||||
pathe: 0.2.0
|
||||
vite: 7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.1)(tsx@4.20.6)(yaml@2.8.1)
|
||||
|
||||
vite-plugin-node-worker@1.0.5:
|
||||
dependencies:
|
||||
magic-string: 0.30.21
|
||||
|
||||
vite-plugin-pwa@1.1.0(vite@7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.1)(tsx@4.20.6)(yaml@2.8.1))(workbox-build@7.3.0(@types/babel__core@7.20.5))(workbox-window@7.3.0):
|
||||
dependencies:
|
||||
debug: 4.4.3(supports-color@5.5.0)
|
||||
|
||||
Reference in New Issue
Block a user