fix: images are dropped when more than 1000 files on S3 (#208)

packages/builder/src/storage/providers/s3-client.ts

@@ -52,7 +52,11 @@ export class S3ProviderClient {
     })
   }
 
-  async listObjects(params?: { prefix?: string | null; maxKeys?: number }): Promise<Response> {
+  async listObjects(params?: {
+    prefix?: string | null
+    maxKeys?: number
+    continuationToken?: string
+  }): Promise<Response> {
     const url = new URL(this.buildObjectUrl())
     url.searchParams.set('list-type', '2')
     if (params?.prefix) {
@@ -61,6 +65,9 @@ export class S3ProviderClient {
     if (params?.maxKeys) {
       url.searchParams.set('max-keys', String(params.maxKeys))
     }
+    if (params?.continuationToken) {
+      url.searchParams.set('continuation-token', params.continuationToken)
+    }
     return await this.client.fetch(url.toString(), { method: 'GET' })
   }
 

packages/builder/src/storage/providers/s3-provider.ts

@@ -303,29 +303,70 @@ export class S3StorageProvider implements StorageProvider {
   }
 
   private async listObjects(prefix?: string): Promise<StorageObject[]> {
+    const maxTotal = this.config.maxFileLimit
+    const shouldPaginate = maxTotal === undefined || maxTotal > 1000
+    const all: StorageObject[] = []
+    let continuationToken: string | null = null
+
+    while (true) {
+      const { objects, nextContinuationToken, isTruncated } = await this.listPagedObjects(prefix, continuationToken)
+      all.push(...objects)
+
+      if (maxTotal && all.length >= maxTotal) {
+        break
+      }
+
+      if (!shouldPaginate || !isTruncated || !nextContinuationToken) {
+        break
+      }
+
+      continuationToken = nextContinuationToken
+    }
+
+    return maxTotal ? all.slice(0, maxTotal) : all
+  }
+
+  private async listPagedObjects(
+    prefix?: string,
+    continuationToken?: string | null,
+  ): Promise<{ objects: StorageObject[]; nextContinuationToken: string | null; isTruncated: boolean }> {
+    const maxKeysPerRequest = this.config.maxFileLimit ? Math.min(this.config.maxFileLimit, 1000) : 1000
     const response = await this.client.listObjects({
       prefix: prefix ?? this.config.prefix,
-      maxKeys: this.config.maxFileLimit,
+      maxKeys: maxKeysPerRequest,
+      continuationToken: continuationToken ?? undefined,
     })
     const text = await response.text()
     if (!response.ok) {
       throw new Error(`列出 S3 对象失败 (status ${response.status}): ${formatS3ErrorBody(text)}`)
     }
     const parsed = xmlParser.parse(text)
-    const contents = parsed?.ListBucketResult?.Contents ?? []
+    const result = parsed?.ListBucketResult ?? {}
+    const contents = result?.Contents ?? []
     const items = Array.isArray(contents) ? contents : contents ? [contents] : []
+    const nextContinuationToken =
+      typeof result?.NextContinuationToken === 'string' && result.NextContinuationToken.trim().length > 0
+        ? result.NextContinuationToken
+        : null
+    const isTruncatedRaw = result?.IsTruncated
+    const isTruncated =
+      typeof isTruncatedRaw === 'string' ? isTruncatedRaw.toLowerCase() === 'true' : Boolean(isTruncatedRaw)
 
-    return items
+    return {
-      .map((item) => {
+      objects: items
-        const key = item?.Key ?? ''
+        .map((item) => {
-        return {
+          const key = item?.Key ?? ''
-          key,
+          return {
-          size: item?.Size !== undefined ? Number(item.Size) : undefined,
+            key,
-          lastModified: item?.LastModified ? new Date(item.LastModified) : undefined,
+            size: item?.Size !== undefined ? Number(item.Size) : undefined,
-          etag: sanitizeS3Etag(typeof item?.ETag === 'string' ? item.ETag : undefined),
+            lastModified: item?.LastModified ? new Date(item.LastModified) : undefined,
-        } satisfies StorageObject
+            etag: sanitizeS3Etag(typeof item?.ETag === 'string' ? item.ETag : undefined),
-      })
+          } satisfies StorageObject
-      .filter((item) => Boolean(item.key))
+        })
+        .filter((item) => Boolean(item.key)),
+      nextContinuationToken,
+      isTruncated,
+    }
   }
 
   async deleteFile(key: string): Promise<void> {