mirror of
https://github.com/Afilmory/afilmory
synced 2026-02-01 22:48:17 +00:00
feat: add support for Aliyun OSS and Tencent COS storage providers
- Updated documentation to include Aliyun OSS and Tencent COS as storage options. - Introduced configuration examples for both providers in the storage providers documentation. - Enhanced the storage provider registration to accommodate new providers. - Updated the storage configuration interfaces to support OSS and COS. - Modified the S3 client and provider implementations to handle requests for OSS and COS. - Added environment variable configurations for OSS and COS. - Implemented necessary changes in the UI schema and routes to reflect the new providers. - Updated localization files for new storage provider types. Signed-off-by: Innei <tukon479@gmail.com>
This commit is contained in:
@@ -522,7 +522,9 @@ export class AfilmoryBuilder {
|
||||
private logBuildStart(): void {
|
||||
const storage = this.getStorageConfig()
|
||||
switch (storage.provider) {
|
||||
case 's3': {
|
||||
case 's3':
|
||||
case 'oss':
|
||||
case 'cos': {
|
||||
const endpoint = storage.endpoint || '默认 AWS S3'
|
||||
const customDomain = storage.customDomain || '未设置'
|
||||
const { bucket } = storage
|
||||
|
||||
@@ -76,7 +76,9 @@ async function main() {
|
||||
logger.main.info(` 存储提供商:${storage.provider}`)
|
||||
|
||||
switch (storage.provider) {
|
||||
case 's3': {
|
||||
case 's3':
|
||||
case 'oss':
|
||||
case 'cos': {
|
||||
logger.main.info(` 存储桶:${storage.bucket}`)
|
||||
logger.main.info(` 区域:${storage.region || '未设置'}`)
|
||||
logger.main.info(` 端点:${storage.endpoint || '默认'}`)
|
||||
|
||||
@@ -2,7 +2,7 @@ import { AsyncLocalStorage } from 'node:async_hooks'
|
||||
|
||||
import type { AfilmoryBuilder } from '../builder/builder.js'
|
||||
import type { StorageManager } from '../storage/index.js'
|
||||
import type { B2Config, GitHubConfig, S3Config, StorageConfig } from '../storage/interfaces.js'
|
||||
import type { B2Config, GitHubConfig, S3CompatibleConfig, StorageConfig } from '../storage/interfaces.js'
|
||||
import type { PhotoProcessingLoggers } from './logger-adapter.js'
|
||||
|
||||
export interface PhotoExecutionContext {
|
||||
@@ -44,8 +44,10 @@ export function createStorageKeyNormalizer(storageConfig: StorageConfig): (key:
|
||||
let basePrefix = ''
|
||||
|
||||
switch (storageConfig.provider) {
|
||||
case 's3': {
|
||||
basePrefix = sanitizeStoragePath((storageConfig as S3Config).prefix)
|
||||
case 's3':
|
||||
case 'oss':
|
||||
case 'cos': {
|
||||
basePrefix = sanitizeStoragePath((storageConfig as S3CompatibleConfig).prefix)
|
||||
break
|
||||
}
|
||||
case 'b2': {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { S3Config } from '../../storage/interfaces.js'
|
||||
import type { S3CompatibleConfig } from '../../storage/interfaces.js'
|
||||
import { S3StorageProvider } from '../../storage/providers/s3-provider.js'
|
||||
import type { BuilderPlugin } from '../types.js'
|
||||
|
||||
@@ -16,7 +16,7 @@ export default function s3StoragePlugin(options: S3StoragePluginOptions = {}): B
|
||||
registerStorageProvider(
|
||||
providerName,
|
||||
(config) => {
|
||||
return new S3StorageProvider(config as S3Config)
|
||||
return new S3StorageProvider(config as S3CompatibleConfig)
|
||||
},
|
||||
{ category: 'remote' },
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { StorageManager } from '../../storage/index.js'
|
||||
import type { S3Config, StorageConfig } from '../../storage/interfaces.js'
|
||||
import type { S3CompatibleConfig, StorageConfig } from '../../storage/interfaces.js'
|
||||
import type { BuilderPlugin } from '../types.js'
|
||||
import type { ThumbnailPluginData } from './shared.js'
|
||||
import {
|
||||
@@ -55,8 +55,10 @@ function joinSegments(...segments: Array<string | null | undefined>): string {
|
||||
|
||||
function resolveRemotePrefix(config: UploadableStorageConfig, directory: string): string {
|
||||
switch (config.provider) {
|
||||
case 's3': {
|
||||
const base = trimSlashes((config as S3Config).prefix)
|
||||
case 's3':
|
||||
case 'oss':
|
||||
case 'cos': {
|
||||
const base = trimSlashes((config as S3CompatibleConfig).prefix)
|
||||
return joinSegments(base, directory)
|
||||
}
|
||||
case 'github': {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import crypto from 'node:crypto'
|
||||
|
||||
import type { S3Config } from '../storage/interfaces'
|
||||
import type { S3CompatibleConfig } from '../storage/interfaces'
|
||||
|
||||
export interface SimpleS3Client {
|
||||
fetch: (input: string | URL, init?: RequestInit) => Promise<Response>
|
||||
@@ -9,9 +9,13 @@ export interface SimpleS3Client {
|
||||
readonly region: string
|
||||
}
|
||||
|
||||
export function createS3Client(config: S3Config): SimpleS3Client {
|
||||
if (config.provider !== 's3') {
|
||||
throw new Error('Storage provider is not s3')
|
||||
type S3CompatibleProviderName = S3CompatibleConfig['provider']
|
||||
|
||||
const S3_COMPATIBLE_PROVIDERS = new Set(['s3', 'oss', 'cos'])
|
||||
|
||||
export function createS3Client(config: S3CompatibleConfig): SimpleS3Client {
|
||||
if (!S3_COMPATIBLE_PROVIDERS.has(config.provider)) {
|
||||
throw new Error('Storage provider is not S3-compatible')
|
||||
}
|
||||
|
||||
const { accessKeyId, secretAccessKey, endpoint, bucket } = config
|
||||
@@ -25,14 +29,15 @@ export function createS3Client(config: S3Config): SimpleS3Client {
|
||||
throw new Error('accessKeyId and secretAccessKey are required')
|
||||
}
|
||||
|
||||
const baseUrl = buildBaseUrl({ bucket, region, endpoint })
|
||||
const baseUrl = buildBaseUrl({ bucket, region, endpoint, provider: config.provider })
|
||||
const sigV4Service = config.sigV4Service ?? inferSigV4Service(config.provider)
|
||||
|
||||
const signer = new SigV4Signer({
|
||||
accessKeyId,
|
||||
secretAccessKey,
|
||||
sessionToken: config.sessionToken,
|
||||
region,
|
||||
service: 's3',
|
||||
service: sigV4Service,
|
||||
})
|
||||
|
||||
return {
|
||||
@@ -56,10 +61,25 @@ export function createS3Client(config: S3Config): SimpleS3Client {
|
||||
}
|
||||
}
|
||||
|
||||
function buildBaseUrl(params: { bucket: string; region: string; endpoint?: string }): string {
|
||||
const { bucket, region, endpoint } = params
|
||||
function buildBaseUrl(params: {
|
||||
bucket: string
|
||||
region: string
|
||||
endpoint?: string
|
||||
provider: S3CompatibleProviderName
|
||||
}): string {
|
||||
const { bucket, region, endpoint, provider } = params
|
||||
if (!endpoint) {
|
||||
return `https://${bucket}.s3.${region}.amazonaws.com/`
|
||||
switch (provider) {
|
||||
case 'oss': {
|
||||
return `https://${bucket}.${region}.aliyuncs.com/`
|
||||
}
|
||||
case 'cos': {
|
||||
return `https://${bucket}.cos.${region}.myqcloud.com/`
|
||||
}
|
||||
default: {
|
||||
return `https://${bucket}.s3.${region}.amazonaws.com/`
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const trimmed = endpoint.replace(/\/$/, '')
|
||||
@@ -71,6 +91,14 @@ function buildBaseUrl(params: { bucket: string; region: string; endpoint?: strin
|
||||
return trimmed.endsWith('/') ? trimmed : `${trimmed}/`
|
||||
}
|
||||
|
||||
if (provider === 'oss' || /aliyuncs\.com/.test(trimmed)) {
|
||||
return ensureTrailingSlash(injectBucketAsSubdomain(trimmed, bucket))
|
||||
}
|
||||
|
||||
if (provider === 'cos' || /myqcloud\.com/.test(trimmed)) {
|
||||
return ensureTrailingSlash(injectBucketAsSubdomain(trimmed, bucket))
|
||||
}
|
||||
|
||||
return `${trimmed}/${bucket}/`
|
||||
}
|
||||
|
||||
@@ -115,6 +143,37 @@ function encodeRfc3986(value: string): string {
|
||||
|
||||
const EMPTY_HASH = crypto.createHash('sha256').update('').digest('hex')
|
||||
|
||||
function inferSigV4Service(provider: S3CompatibleProviderName): string {
|
||||
switch (provider) {
|
||||
case 'oss': {
|
||||
return 'oss'
|
||||
}
|
||||
default: {
|
||||
return 's3'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function ensureTrailingSlash(value: string): string {
|
||||
return value.endsWith('/') ? value : `${value}/`
|
||||
}
|
||||
|
||||
function injectBucketAsSubdomain(endpoint: string, bucket: string): string {
|
||||
if (!/^https?:\/\//i.test(endpoint)) {
|
||||
return `${endpoint.replace(/\/$/, '')}/${bucket}`
|
||||
}
|
||||
|
||||
try {
|
||||
const url = new URL(endpoint)
|
||||
if (!url.hostname.startsWith(`${bucket}.`)) {
|
||||
url.hostname = `${bucket}.${url.hostname}`
|
||||
}
|
||||
return url.toString()
|
||||
} catch {
|
||||
return `${endpoint.replace(/\/$/, '')}/${bucket}`
|
||||
}
|
||||
}
|
||||
|
||||
class SigV4Signer {
|
||||
constructor(
|
||||
private readonly options: {
|
||||
|
||||
@@ -79,8 +79,7 @@ export interface StorageProvider {
|
||||
moveFile: (sourceKey: string, targetKey: string, options?: StorageUploadOptions) => Promise<StorageObject>
|
||||
}
|
||||
|
||||
export type S3Config = {
|
||||
provider: 's3'
|
||||
type BaseS3LikeConfig = {
|
||||
bucket?: string
|
||||
region?: string
|
||||
endpoint?: string
|
||||
@@ -103,8 +102,29 @@ export type S3Config = {
|
||||
maxAttempts?: number
|
||||
// Download concurrency limiter within a single process/worker
|
||||
downloadConcurrency?: number
|
||||
/**
|
||||
* Optional override for the SigV4 service name. Defaults to:
|
||||
* - `s3` for AWS or generic S3-compatible endpoints
|
||||
* - `oss` for Aliyun OSS
|
||||
* - `s3` for Tencent COS
|
||||
*/
|
||||
sigV4Service?: string
|
||||
}
|
||||
|
||||
export type S3Config = BaseS3LikeConfig & {
|
||||
provider: 's3'
|
||||
}
|
||||
|
||||
export type OSSConfig = BaseS3LikeConfig & {
|
||||
provider: 'oss'
|
||||
}
|
||||
|
||||
export type COSConfig = BaseS3LikeConfig & {
|
||||
provider: 'cos'
|
||||
}
|
||||
|
||||
export type S3CompatibleConfig = S3Config | OSSConfig | COSConfig
|
||||
|
||||
export type B2Config = {
|
||||
provider: 'b2'
|
||||
applicationKeyId: string
|
||||
@@ -215,13 +235,13 @@ export interface CustomStorageConfig {
|
||||
[key: string]: unknown
|
||||
}
|
||||
|
||||
export type RemoteStorageProviderName = 's3' | 'b2' | 'github'
|
||||
export type RemoteStorageProviderName = 's3' | 'oss' | 'cos' | 'b2' | 'github'
|
||||
export type LocalStorageProviderName = 'eagle' | 'local'
|
||||
|
||||
export const REMOTE_STORAGE_PROVIDERS: readonly RemoteStorageProviderName[] = ['s3', 'b2', 'github']
|
||||
export const REMOTE_STORAGE_PROVIDERS: readonly RemoteStorageProviderName[] = ['s3', 'oss', 'cos', 'b2', 'github']
|
||||
export const LOCAL_STORAGE_PROVIDERS: readonly LocalStorageProviderName[] = ['eagle', 'local']
|
||||
|
||||
export type RemoteStorageConfig = S3Config | B2Config | GitHubConfig
|
||||
export type RemoteStorageConfig = S3CompatibleConfig | B2Config | GitHubConfig
|
||||
export type LocalStorageConfig = EagleConfig | LocalConfig
|
||||
|
||||
export type ManagedStorageConfig = {
|
||||
|
||||
@@ -4,7 +4,9 @@
|
||||
|
||||
## S3 存储提供商
|
||||
|
||||
支持 AWS S3 和兼容 S3 API 的存储服务(如 MinIO、阿里云 OSS 等)。
|
||||
支持 AWS S3 和大多数 S3 API 兼容的对象存储(如 MinIO、Cloudflare R2、DigitalOcean Spaces 等)。
|
||||
|
||||
> 如果你使用 **阿里云 OSS** 或 **腾讯云 COS**,现在可以直接使用 `provider: 'oss'` 或 `provider: 'cos'`,它们基于同一套实现,但提供了默认的 Endpoint 和 SigV4 Service 配置,避免重复参数。详见下文。
|
||||
|
||||
### 配置示例
|
||||
|
||||
@@ -21,6 +23,54 @@ const s3Config: StorageConfig = {
|
||||
}
|
||||
```
|
||||
|
||||
## 阿里云 OSS 提供商
|
||||
|
||||
Aliyun OSS 已提供 SigV4 兼容 API。通过 `provider: 'oss'`,Afilmory 会默认使用 `oss` 作为 SigV4 service 并生成标准的 `bucket.oss-xx.aliyuncs.com` Endpoint。
|
||||
|
||||
### 配置示例
|
||||
|
||||
```typescript
|
||||
const ossConfig: StorageConfig = {
|
||||
provider: 'oss',
|
||||
bucket: 'gallery-assets',
|
||||
region: 'oss-cn-hangzhou',
|
||||
accessKeyId: process.env.ALIYUN_AK!,
|
||||
secretAccessKey: process.env.ALIYUN_SK!,
|
||||
endpoint: 'https://oss-cn-hangzhou.aliyuncs.com', // 可省略,默认会根据 region 生成
|
||||
customDomain: 'https://img.example.cn',
|
||||
}
|
||||
```
|
||||
|
||||
### 提示
|
||||
|
||||
- region 需包含 `oss-` 前缀,例如 `oss-cn-shanghai`。
|
||||
- 若使用内网或加速域名,可通过 `endpoint` 覆盖,或设置 `customDomain` 生成公开 URL。
|
||||
- 仍可通过 `sigV4Service` 手动覆盖签名服务名(默认 `oss`)。
|
||||
|
||||
## 腾讯云 COS 提供商
|
||||
|
||||
COS 与 AWS S3 API 高度兼容。`provider: 'cos'` 会默认生成 `bucket.cos.<region>.myqcloud.com` 的 Endpoint 并使用 `s3` SigV4 service。
|
||||
|
||||
### 配置示例
|
||||
|
||||
```typescript
|
||||
const cosConfig: StorageConfig = {
|
||||
provider: 'cos',
|
||||
bucket: 'gallery-1250000000', // 记得包含 APPID
|
||||
region: 'ap-shanghai',
|
||||
accessKeyId: process.env.COS_SECRET_ID!,
|
||||
secretAccessKey: process.env.COS_SECRET_KEY!,
|
||||
endpoint: 'https://cos.ap-shanghai.myqcloud.com', // 可选
|
||||
prefix: 'photos/',
|
||||
}
|
||||
```
|
||||
|
||||
### 提示
|
||||
|
||||
- bucket 需包含 COS 要求的 `-APPID` 后缀。
|
||||
- 如果使用自定义加速域名,仅需设置 `customDomain`。
|
||||
- 默认 SigV4 service 为 `s3`,如需兼容自建网关可自定义 `sigV4Service`。
|
||||
|
||||
## GitHub 存储提供商
|
||||
|
||||
将照片存储在 GitHub 仓库中,利用 GitHub 的免费存储空间和全球 CDN。
|
||||
|
||||
@@ -2,9 +2,11 @@ import type { StorageProviderFactory } from '../factory.js'
|
||||
import { StorageFactory } from '../factory.js'
|
||||
import type {
|
||||
B2Config,
|
||||
COSConfig,
|
||||
EagleConfig,
|
||||
GitHubConfig,
|
||||
LocalConfig,
|
||||
OSSConfig,
|
||||
S3Config,
|
||||
StorageProviderCategory,
|
||||
} from '../interfaces.js'
|
||||
@@ -26,6 +28,16 @@ const BUILTIN_PROVIDER_REGISTRATIONS: BuiltinProviderRegistration[] = [
|
||||
category: 'remote',
|
||||
factory: (config) => new S3StorageProvider(config as S3Config),
|
||||
},
|
||||
{
|
||||
name: 'oss',
|
||||
category: 'remote',
|
||||
factory: (config) => new S3StorageProvider(config as OSSConfig),
|
||||
},
|
||||
{
|
||||
name: 'cos',
|
||||
category: 'remote',
|
||||
factory: (config) => new S3StorageProvider(config as COSConfig),
|
||||
},
|
||||
{
|
||||
name: 'b2',
|
||||
category: 'remote',
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
import type { SimpleS3Client } from '../../s3/client.js'
|
||||
import { createS3Client, encodeS3Key } from '../../s3/client.js'
|
||||
import type { S3Config } from '../interfaces.js'
|
||||
import type { S3CompatibleConfig } from '../interfaces.js'
|
||||
import { sanitizeS3Etag } from './s3-utils.js'
|
||||
|
||||
export class S3ProviderClient {
|
||||
private readonly client: SimpleS3Client
|
||||
|
||||
constructor(config: S3Config) {
|
||||
constructor(config: S3CompatibleConfig) {
|
||||
this.client = createS3Client(config)
|
||||
}
|
||||
|
||||
|
||||
@@ -6,7 +6,13 @@ import { backoffDelay, sleep } from '../../../../utils/src/backoff.js'
|
||||
import { Semaphore } from '../../../../utils/src/semaphore.js'
|
||||
import { SUPPORTED_FORMATS } from '../../constants/index.js'
|
||||
import { logger } from '../../logger/index.js'
|
||||
import type { ProgressCallback, S3Config, StorageObject, StorageProvider, StorageUploadOptions } from '../interfaces'
|
||||
import type {
|
||||
ProgressCallback,
|
||||
S3CompatibleConfig,
|
||||
StorageObject,
|
||||
StorageProvider,
|
||||
StorageUploadOptions,
|
||||
} from '../interfaces'
|
||||
import { S3ProviderClient } from './s3-client.js'
|
||||
import { sanitizeS3Etag } from './s3-utils.js'
|
||||
|
||||
@@ -109,11 +115,11 @@ function formatS3ErrorBody(body?: string | null): string {
|
||||
}
|
||||
|
||||
export class S3StorageProvider implements StorageProvider {
|
||||
private config: S3Config
|
||||
private config: S3CompatibleConfig
|
||||
private client: S3ProviderClient
|
||||
private limiter: Semaphore
|
||||
|
||||
constructor(config: S3Config) {
|
||||
constructor(config: S3CompatibleConfig) {
|
||||
this.config = config
|
||||
this.client = new S3ProviderClient(config)
|
||||
this.limiter = new Semaphore(this.config.downloadConcurrency ?? 16)
|
||||
@@ -245,36 +251,7 @@ export class S3StorageProvider implements StorageProvider {
|
||||
const customDomain = this.config.customDomain.replace(/\/$/, '') // 移除末尾的斜杠
|
||||
return `${customDomain}/${key}`
|
||||
}
|
||||
|
||||
// 如果使用自定义端点,构建相应的 URL
|
||||
const { endpoint } = this.config
|
||||
|
||||
const region = this.config.region ?? 'us-east-1'
|
||||
|
||||
if (!endpoint) {
|
||||
// 默认 AWS S3 端点
|
||||
return `https://${this.config.bucket}.s3.${region}.amazonaws.com/${key}`
|
||||
}
|
||||
|
||||
// 检查是否是标准 AWS S3 端点
|
||||
if (endpoint.includes('amazonaws.com')) {
|
||||
return `https://${this.config.bucket}.s3.${region}.amazonaws.com/${key}`
|
||||
}
|
||||
|
||||
const baseUrl = endpoint.replace(/\/$/, '') // 移除末尾的斜杠
|
||||
|
||||
if (endpoint.includes('aliyuncs.com')) {
|
||||
const protocolEndIndex = baseUrl.indexOf('//')
|
||||
if (protocolEndIndex === -1) {
|
||||
throw new Error('Invalid base URL format')
|
||||
}
|
||||
// 将 bucket 插入到 'https://` 之后,region 之前
|
||||
const prefix = baseUrl.slice(0, protocolEndIndex + 2) // 包括 'https://'
|
||||
const suffix = baseUrl.slice(protocolEndIndex + 2) // 剩余部分
|
||||
return `${prefix}${this.config.bucket}.${suffix}/${key}`
|
||||
}
|
||||
// 对于自定义端点(如 MinIO 等)
|
||||
return `${baseUrl}/${this.config.bucket}/${key}`
|
||||
return this.client.buildObjectUrl(key)
|
||||
}
|
||||
|
||||
detectLivePhotos(allObjects: StorageObject[]): Map<string, StorageObject> {
|
||||
|
||||
Reference in New Issue
Block a user