feat: add support for Aliyun OSS and Tencent COS storage providers

- Updated documentation to include Aliyun OSS and Tencent COS as storage options.
- Introduced configuration examples for both providers in the storage providers documentation.
- Enhanced the storage provider registration to accommodate new providers.
- Updated the storage configuration interfaces to support OSS and COS.
- Modified the S3 client and provider implementations to handle requests for OSS and COS.
- Added environment variable configurations for OSS and COS.
- Implemented necessary changes in the UI schema and routes to reflect the new providers.
- Updated localization files for new storage provider types.

Signed-off-by: Innei <tukon479@gmail.com>
This commit is contained in:
Innei
2025-11-24 22:26:47 +08:00
parent 6b48fe7333
commit d9a5be56e6
29 changed files with 554 additions and 156 deletions

View File

@@ -522,7 +522,9 @@ export class AfilmoryBuilder {
private logBuildStart(): void {
const storage = this.getStorageConfig()
switch (storage.provider) {
case 's3': {
case 's3':
case 'oss':
case 'cos': {
const endpoint = storage.endpoint || '默认 AWS S3'
const customDomain = storage.customDomain || '未设置'
const { bucket } = storage

View File

@@ -76,7 +76,9 @@ async function main() {
logger.main.info(` 存储提供商:${storage.provider}`)
switch (storage.provider) {
case 's3': {
case 's3':
case 'oss':
case 'cos': {
logger.main.info(` 存储桶:${storage.bucket}`)
logger.main.info(` 区域:${storage.region || '未设置'}`)
logger.main.info(` 端点:${storage.endpoint || '默认'}`)

View File

@@ -2,7 +2,7 @@ import { AsyncLocalStorage } from 'node:async_hooks'
import type { AfilmoryBuilder } from '../builder/builder.js'
import type { StorageManager } from '../storage/index.js'
import type { B2Config, GitHubConfig, S3Config, StorageConfig } from '../storage/interfaces.js'
import type { B2Config, GitHubConfig, S3CompatibleConfig, StorageConfig } from '../storage/interfaces.js'
import type { PhotoProcessingLoggers } from './logger-adapter.js'
export interface PhotoExecutionContext {
@@ -44,8 +44,10 @@ export function createStorageKeyNormalizer(storageConfig: StorageConfig): (key:
let basePrefix = ''
switch (storageConfig.provider) {
case 's3': {
basePrefix = sanitizeStoragePath((storageConfig as S3Config).prefix)
case 's3':
case 'oss':
case 'cos': {
basePrefix = sanitizeStoragePath((storageConfig as S3CompatibleConfig).prefix)
break
}
case 'b2': {

View File

@@ -1,4 +1,4 @@
import type { S3Config } from '../../storage/interfaces.js'
import type { S3CompatibleConfig } from '../../storage/interfaces.js'
import { S3StorageProvider } from '../../storage/providers/s3-provider.js'
import type { BuilderPlugin } from '../types.js'
@@ -16,7 +16,7 @@ export default function s3StoragePlugin(options: S3StoragePluginOptions = {}): B
registerStorageProvider(
providerName,
(config) => {
return new S3StorageProvider(config as S3Config)
return new S3StorageProvider(config as S3CompatibleConfig)
},
{ category: 'remote' },
)

View File

@@ -1,5 +1,5 @@
import { StorageManager } from '../../storage/index.js'
import type { S3Config, StorageConfig } from '../../storage/interfaces.js'
import type { S3CompatibleConfig, StorageConfig } from '../../storage/interfaces.js'
import type { BuilderPlugin } from '../types.js'
import type { ThumbnailPluginData } from './shared.js'
import {
@@ -55,8 +55,10 @@ function joinSegments(...segments: Array<string | null | undefined>): string {
function resolveRemotePrefix(config: UploadableStorageConfig, directory: string): string {
switch (config.provider) {
case 's3': {
const base = trimSlashes((config as S3Config).prefix)
case 's3':
case 'oss':
case 'cos': {
const base = trimSlashes((config as S3CompatibleConfig).prefix)
return joinSegments(base, directory)
}
case 'github': {

View File

@@ -1,6 +1,6 @@
import crypto from 'node:crypto'
import type { S3Config } from '../storage/interfaces'
import type { S3CompatibleConfig } from '../storage/interfaces'
export interface SimpleS3Client {
fetch: (input: string | URL, init?: RequestInit) => Promise<Response>
@@ -9,9 +9,13 @@ export interface SimpleS3Client {
readonly region: string
}
export function createS3Client(config: S3Config): SimpleS3Client {
if (config.provider !== 's3') {
throw new Error('Storage provider is not s3')
type S3CompatibleProviderName = S3CompatibleConfig['provider']
const S3_COMPATIBLE_PROVIDERS = new Set(['s3', 'oss', 'cos'])
export function createS3Client(config: S3CompatibleConfig): SimpleS3Client {
if (!S3_COMPATIBLE_PROVIDERS.has(config.provider)) {
throw new Error('Storage provider is not S3-compatible')
}
const { accessKeyId, secretAccessKey, endpoint, bucket } = config
@@ -25,14 +29,15 @@ export function createS3Client(config: S3Config): SimpleS3Client {
throw new Error('accessKeyId and secretAccessKey are required')
}
const baseUrl = buildBaseUrl({ bucket, region, endpoint })
const baseUrl = buildBaseUrl({ bucket, region, endpoint, provider: config.provider })
const sigV4Service = config.sigV4Service ?? inferSigV4Service(config.provider)
const signer = new SigV4Signer({
accessKeyId,
secretAccessKey,
sessionToken: config.sessionToken,
region,
service: 's3',
service: sigV4Service,
})
return {
@@ -56,10 +61,25 @@ export function createS3Client(config: S3Config): SimpleS3Client {
}
}
function buildBaseUrl(params: { bucket: string; region: string; endpoint?: string }): string {
const { bucket, region, endpoint } = params
function buildBaseUrl(params: {
bucket: string
region: string
endpoint?: string
provider: S3CompatibleProviderName
}): string {
const { bucket, region, endpoint, provider } = params
if (!endpoint) {
return `https://${bucket}.s3.${region}.amazonaws.com/`
switch (provider) {
case 'oss': {
return `https://${bucket}.${region}.aliyuncs.com/`
}
case 'cos': {
return `https://${bucket}.cos.${region}.myqcloud.com/`
}
default: {
return `https://${bucket}.s3.${region}.amazonaws.com/`
}
}
}
const trimmed = endpoint.replace(/\/$/, '')
@@ -71,6 +91,14 @@ function buildBaseUrl(params: { bucket: string; region: string; endpoint?: strin
return trimmed.endsWith('/') ? trimmed : `${trimmed}/`
}
if (provider === 'oss' || /aliyuncs\.com/.test(trimmed)) {
return ensureTrailingSlash(injectBucketAsSubdomain(trimmed, bucket))
}
if (provider === 'cos' || /myqcloud\.com/.test(trimmed)) {
return ensureTrailingSlash(injectBucketAsSubdomain(trimmed, bucket))
}
return `${trimmed}/${bucket}/`
}
@@ -115,6 +143,37 @@ function encodeRfc3986(value: string): string {
const EMPTY_HASH = crypto.createHash('sha256').update('').digest('hex')
function inferSigV4Service(provider: S3CompatibleProviderName): string {
switch (provider) {
case 'oss': {
return 'oss'
}
default: {
return 's3'
}
}
}
function ensureTrailingSlash(value: string): string {
return value.endsWith('/') ? value : `${value}/`
}
function injectBucketAsSubdomain(endpoint: string, bucket: string): string {
if (!/^https?:\/\//i.test(endpoint)) {
return `${endpoint.replace(/\/$/, '')}/${bucket}`
}
try {
const url = new URL(endpoint)
if (!url.hostname.startsWith(`${bucket}.`)) {
url.hostname = `${bucket}.${url.hostname}`
}
return url.toString()
} catch {
return `${endpoint.replace(/\/$/, '')}/${bucket}`
}
}
class SigV4Signer {
constructor(
private readonly options: {

View File

@@ -79,8 +79,7 @@ export interface StorageProvider {
moveFile: (sourceKey: string, targetKey: string, options?: StorageUploadOptions) => Promise<StorageObject>
}
export type S3Config = {
provider: 's3'
type BaseS3LikeConfig = {
bucket?: string
region?: string
endpoint?: string
@@ -103,8 +102,29 @@ export type S3Config = {
maxAttempts?: number
// Download concurrency limiter within a single process/worker
downloadConcurrency?: number
/**
* Optional override for the SigV4 service name. Defaults to:
* - `s3` for AWS or generic S3-compatible endpoints
* - `oss` for Aliyun OSS
* - `s3` for Tencent COS
*/
sigV4Service?: string
}
export type S3Config = BaseS3LikeConfig & {
provider: 's3'
}
export type OSSConfig = BaseS3LikeConfig & {
provider: 'oss'
}
export type COSConfig = BaseS3LikeConfig & {
provider: 'cos'
}
export type S3CompatibleConfig = S3Config | OSSConfig | COSConfig
export type B2Config = {
provider: 'b2'
applicationKeyId: string
@@ -215,13 +235,13 @@ export interface CustomStorageConfig {
[key: string]: unknown
}
export type RemoteStorageProviderName = 's3' | 'b2' | 'github'
export type RemoteStorageProviderName = 's3' | 'oss' | 'cos' | 'b2' | 'github'
export type LocalStorageProviderName = 'eagle' | 'local'
export const REMOTE_STORAGE_PROVIDERS: readonly RemoteStorageProviderName[] = ['s3', 'b2', 'github']
export const REMOTE_STORAGE_PROVIDERS: readonly RemoteStorageProviderName[] = ['s3', 'oss', 'cos', 'b2', 'github']
export const LOCAL_STORAGE_PROVIDERS: readonly LocalStorageProviderName[] = ['eagle', 'local']
export type RemoteStorageConfig = S3Config | B2Config | GitHubConfig
export type RemoteStorageConfig = S3CompatibleConfig | B2Config | GitHubConfig
export type LocalStorageConfig = EagleConfig | LocalConfig
export type ManagedStorageConfig = {

View File

@@ -4,7 +4,9 @@
## S3 存储提供商
支持 AWS S3 和兼容 S3 API 的存储服务(如 MinIO、阿里云 OSS 等)。
支持 AWS S3 和大多数 S3 API 兼容的对象存储(如 MinIO、Cloudflare R2、DigitalOcean Spaces 等)。
> 如果你使用 **阿里云 OSS** 或 **腾讯云 COS**,现在可以直接使用 `provider: 'oss'` 或 `provider: 'cos'`,它们基于同一套实现,但提供了默认的 Endpoint 和 SigV4 Service 配置,避免重复参数。详见下文。
### 配置示例
@@ -21,6 +23,54 @@ const s3Config: StorageConfig = {
}
```
## 阿里云 OSS 提供商
Aliyun OSS 已提供 SigV4 兼容 API。通过 `provider: 'oss'`Afilmory 会默认使用 `oss` 作为 SigV4 service 并生成标准的 `bucket.oss-xx.aliyuncs.com` Endpoint。
### 配置示例
```typescript
const ossConfig: StorageConfig = {
provider: 'oss',
bucket: 'gallery-assets',
region: 'oss-cn-hangzhou',
accessKeyId: process.env.ALIYUN_AK!,
secretAccessKey: process.env.ALIYUN_SK!,
endpoint: 'https://oss-cn-hangzhou.aliyuncs.com', // 可省略,默认会根据 region 生成
customDomain: 'https://img.example.cn',
}
```
### 提示
- region 需包含 `oss-` 前缀,例如 `oss-cn-shanghai`
- 若使用内网或加速域名,可通过 `endpoint` 覆盖,或设置 `customDomain` 生成公开 URL。
- 仍可通过 `sigV4Service` 手动覆盖签名服务名(默认 `oss`)。
## 腾讯云 COS 提供商
COS 与 AWS S3 API 高度兼容。`provider: 'cos'` 会默认生成 `bucket.cos.<region>.myqcloud.com` 的 Endpoint 并使用 `s3` SigV4 service。
### 配置示例
```typescript
const cosConfig: StorageConfig = {
provider: 'cos',
bucket: 'gallery-1250000000', // 记得包含 APPID
region: 'ap-shanghai',
accessKeyId: process.env.COS_SECRET_ID!,
secretAccessKey: process.env.COS_SECRET_KEY!,
endpoint: 'https://cos.ap-shanghai.myqcloud.com', // 可选
prefix: 'photos/',
}
```
### 提示
- bucket 需包含 COS 要求的 `-APPID` 后缀。
- 如果使用自定义加速域名,仅需设置 `customDomain`
- 默认 SigV4 service 为 `s3`,如需兼容自建网关可自定义 `sigV4Service`
## GitHub 存储提供商
将照片存储在 GitHub 仓库中,利用 GitHub 的免费存储空间和全球 CDN。

View File

@@ -2,9 +2,11 @@ import type { StorageProviderFactory } from '../factory.js'
import { StorageFactory } from '../factory.js'
import type {
B2Config,
COSConfig,
EagleConfig,
GitHubConfig,
LocalConfig,
OSSConfig,
S3Config,
StorageProviderCategory,
} from '../interfaces.js'
@@ -26,6 +28,16 @@ const BUILTIN_PROVIDER_REGISTRATIONS: BuiltinProviderRegistration[] = [
category: 'remote',
factory: (config) => new S3StorageProvider(config as S3Config),
},
{
name: 'oss',
category: 'remote',
factory: (config) => new S3StorageProvider(config as OSSConfig),
},
{
name: 'cos',
category: 'remote',
factory: (config) => new S3StorageProvider(config as COSConfig),
},
{
name: 'b2',
category: 'remote',

View File

@@ -1,12 +1,12 @@
import type { SimpleS3Client } from '../../s3/client.js'
import { createS3Client, encodeS3Key } from '../../s3/client.js'
import type { S3Config } from '../interfaces.js'
import type { S3CompatibleConfig } from '../interfaces.js'
import { sanitizeS3Etag } from './s3-utils.js'
export class S3ProviderClient {
private readonly client: SimpleS3Client
constructor(config: S3Config) {
constructor(config: S3CompatibleConfig) {
this.client = createS3Client(config)
}

View File

@@ -6,7 +6,13 @@ import { backoffDelay, sleep } from '../../../../utils/src/backoff.js'
import { Semaphore } from '../../../../utils/src/semaphore.js'
import { SUPPORTED_FORMATS } from '../../constants/index.js'
import { logger } from '../../logger/index.js'
import type { ProgressCallback, S3Config, StorageObject, StorageProvider, StorageUploadOptions } from '../interfaces'
import type {
ProgressCallback,
S3CompatibleConfig,
StorageObject,
StorageProvider,
StorageUploadOptions,
} from '../interfaces'
import { S3ProviderClient } from './s3-client.js'
import { sanitizeS3Etag } from './s3-utils.js'
@@ -109,11 +115,11 @@ function formatS3ErrorBody(body?: string | null): string {
}
export class S3StorageProvider implements StorageProvider {
private config: S3Config
private config: S3CompatibleConfig
private client: S3ProviderClient
private limiter: Semaphore
constructor(config: S3Config) {
constructor(config: S3CompatibleConfig) {
this.config = config
this.client = new S3ProviderClient(config)
this.limiter = new Semaphore(this.config.downloadConcurrency ?? 16)
@@ -245,36 +251,7 @@ export class S3StorageProvider implements StorageProvider {
const customDomain = this.config.customDomain.replace(/\/$/, '') // 移除末尾的斜杠
return `${customDomain}/${key}`
}
// 如果使用自定义端点,构建相应的 URL
const { endpoint } = this.config
const region = this.config.region ?? 'us-east-1'
if (!endpoint) {
// 默认 AWS S3 端点
return `https://${this.config.bucket}.s3.${region}.amazonaws.com/${key}`
}
// 检查是否是标准 AWS S3 端点
if (endpoint.includes('amazonaws.com')) {
return `https://${this.config.bucket}.s3.${region}.amazonaws.com/${key}`
}
const baseUrl = endpoint.replace(/\/$/, '') // 移除末尾的斜杠
if (endpoint.includes('aliyuncs.com')) {
const protocolEndIndex = baseUrl.indexOf('//')
if (protocolEndIndex === -1) {
throw new Error('Invalid base URL format')
}
// 将 bucket 插入到 'https://` 之后region 之前
const prefix = baseUrl.slice(0, protocolEndIndex + 2) // 包括 'https://'
const suffix = baseUrl.slice(protocolEndIndex + 2) // 剩余部分
return `${prefix}${this.config.bucket}.${suffix}/${key}`
}
// 对于自定义端点(如 MinIO 等)
return `${baseUrl}/${this.config.bucket}/${key}`
return this.client.buildObjectUrl(key)
}
detectLivePhotos(allObjects: StorageObject[]): Map<string, StorageObject> {