feat: add local storage provider and photos static plugin (#40)

Co-authored-by: MaxtuneLee <60775796+MaxtuneLee@users.noreply.github.com>
This commit is contained in:
EmccK
2025-07-09 13:23:32 +08:00
committed by GitHub
parent 3f6f12aba3
commit 256becf604
13 changed files with 643 additions and 24 deletions

5
.gitignore vendored
View File

@@ -18,4 +18,7 @@ apps/web/assets-git
apps/web/public/thumbnails
apps/web/src/data/photos-manifest.json
.vercel
apps/ssr/.next
photos
*/*/.next
.claude

View File

@@ -10,7 +10,10 @@ export const handler = async (req: NextRequest) => {
return new NextResponse(null, { status: 404 })
}
if (req.nextUrl.pathname.startsWith('/thumbnails')) {
if (
req.nextUrl.pathname.startsWith('/thumbnails') ||
req.nextUrl.pathname.startsWith('/photos')
) {
return proxyAssets(req)
}

View File

@@ -7,9 +7,7 @@ export const GET = async (req: NextRequest) => {
if (process.env.NODE_ENV === 'development') {
return import('./[...all]/dev').then((m) => m.handler(req))
}
const indexHtml = await import('../index.html').then((m) => m.default)
const document = new DOMParser().parseFromString(indexHtml, 'text/html')
injectConfigToDocument(document)
return new Response(document.documentElement.outerHTML, {

View File

@@ -0,0 +1,151 @@
import fs from 'node:fs'
import path from 'node:path'
import { fileURLToPath } from 'node:url'
import type { Plugin } from 'vite'
const __dirname = path.dirname(fileURLToPath(import.meta.url))
const projectRoot = path.resolve(__dirname, '../../../..')
/**
* Vite 插件:为本地照片提供静态文件服务
* 在开发模式下,将 /photos/* 请求映射到本地照片目录
*/
export function photosStaticPlugin(): Plugin {
// URL 路径验证正则:只允许字母、数字、点、下划线、连字符、斜杠和空格
const pathValidationRegex = /^[\w\u4e00-\u9fa5\s\-./[\]()]+$/
// 危险路径模式
const dangerousPatterns = [
/\.\.\//, // 路径遍历
/\.\.\\/,
/%2e%2e/i, // URL 编码的 ..
/%252e%252e/i, // 双重编码
/\0/, // null 字节
]
// ETag 生成函数
const generateETag = (stats: fs.Stats): string => {
return `"${stats.mtime.getTime()}-${stats.size}"`
}
return {
name: 'photos-static',
configureServer(server) {
server.middlewares.use('/photos', (req, res, next) => {
if (!req.url) {
next()
return
}
// 解码 URL 以处理特殊字符
let decodedUrl: string
try {
decodedUrl = decodeURIComponent(req.url)
} catch {
// URL 解码失败,可能是恶意请求
console.error('[photos-static] URL 解码失败:', req.url)
res.statusCode = 400
res.end('Bad Request')
return
}
// 移除查询参数
const cleanPath = decodedUrl.split('?')[0]
// 检查危险路径模式
for (const pattern of dangerousPatterns) {
if (pattern.test(cleanPath)) {
console.error('[photos-static] 检测到危险路径模式:', cleanPath)
res.statusCode = 403
res.end('Forbidden')
return
}
}
// 验证路径字符
if (!pathValidationRegex.test(cleanPath)) {
console.error('[photos-static] 路径包含不允许的字符:', cleanPath)
res.statusCode = 403
res.end('Forbidden')
return
}
// 构建本地文件路径
const localPhotoPath = path.join(projectRoot, 'photos', cleanPath)
// 安全检查:确保文件路径在 photos 目录内
const resolvedPath = path.resolve(localPhotoPath)
const resolvedPhotosDir = path.resolve(projectRoot, 'photos')
if (!resolvedPath.startsWith(resolvedPhotosDir)) {
res.statusCode = 403
res.end('Forbidden')
return
}
// 检查文件是否存在
if (!fs.existsSync(localPhotoPath)) {
res.statusCode = 404
res.end('Not Found')
return
}
// 检查是否是文件(不是目录)
const stats = fs.statSync(localPhotoPath)
if (!stats.isFile()) {
res.statusCode = 404
res.end('Not Found')
return
}
// 设置正确的 Content-Type
const ext = path.extname(localPhotoPath).toLowerCase()
const mimeTypes: Record<string, string> = {
'.jpg': 'image/jpeg',
'.jpeg': 'image/jpeg',
'.png': 'image/png',
'.webp': 'image/webp',
'.gif': 'image/gif',
'.bmp': 'image/bmp',
'.tiff': 'image/tiff',
'.tif': 'image/tiff',
'.heic': 'image/heic',
'.heif': 'image/heif',
'.hif': 'image/heif',
'.avif': 'image/avif',
'.svg': 'image/svg+xml',
}
const contentType = mimeTypes[ext] || 'application/octet-stream'
res.setHeader('Content-Type', contentType)
// 设置缓存头
res.setHeader('Cache-Control', 'public, max-age=31536000') // 1 year
const etag = generateETag(stats)
res.setHeader('ETag', etag)
// 检查 If-None-Match 头ETag 缓存)
const ifNoneMatch = req.headers['if-none-match']
if (ifNoneMatch === etag) {
res.statusCode = 304
res.end()
return
}
// 流式传输文件
const stream = fs.createReadStream(localPhotoPath)
stream.on('error', (error) => {
console.error('[photos-static] Error streaming photo file:', error)
if (!res.headersSent) {
res.statusCode = 500
res.end('Internal Server Error')
}
})
stream.pipe(res)
})
},
}
}

View File

@@ -23,6 +23,7 @@ import { createFeedSitemapPlugin } from './plugins/vite/feed-sitemap'
import { localesJsonPlugin } from './plugins/vite/locales-json'
import { manifestInjectPlugin } from './plugins/vite/manifest-inject'
import { ogImagePlugin } from './plugins/vite/og-image-plugin'
import { photosStaticPlugin } from './plugins/vite/photos-static'
const devPrint = (): PluginOption => ({
name: 'dev-print',
@@ -78,6 +79,7 @@ export default defineConfig({
]),
localesJsonPlugin(),
manifestInjectPlugin(),
photosStaticPlugin(),
tailwindcss(),
VitePWA({
registerType: 'autoUpdate',

View File

@@ -1,16 +1,15 @@
import type { S3ClientConfig } from '@aws-sdk/client-s3'
import { S3Client } from '@aws-sdk/client-s3'
import { builderConfig } from '@builder'
import type { S3Config } from '../storage/interfaces'
// 创建 S3 客户端
function createS3Client(): S3Client {
const storageConfig = builderConfig.storage
if (storageConfig.provider !== 's3') {
export function createS3Client(config: S3Config): S3Client {
if (config.provider !== 's3') {
throw new Error('Storage provider is not s3')
}
const { accessKeyId, secretAccessKey, endpoint, region } = storageConfig
const { accessKeyId, secretAccessKey, endpoint, region } = config
if (!accessKeyId || !secretAccessKey) {
throw new Error('accessKeyId and secretAccessKey are required')
}
@@ -30,5 +29,3 @@ function createS3Client(): S3Client {
return new S3Client(s3ClientConfig)
}
export const s3Client = createS3Client()

View File

@@ -1,5 +1,6 @@
import type { StorageConfig, StorageProvider } from './interfaces'
import { GitHubStorageProvider } from './providers/github-provider.js'
import { LocalStorageProvider } from './providers/local-provider.js'
import { S3StorageProvider } from './providers/s3-provider.js'
export class StorageFactory {
@@ -16,6 +17,9 @@ export class StorageFactory {
case 'github': {
return new GitHubStorageProvider(config)
}
case 'local': {
return new LocalStorageProvider(config)
}
}
}
}

View File

@@ -1,5 +1,7 @@
// 导出接口
export type {
ProgressCallback,
ScanProgress,
StorageConfig,
StorageObject,
StorageProvider,
@@ -13,4 +15,5 @@ export { StorageManager } from './manager.js'
// 导出具体提供商(如果需要直接使用)
export { GitHubStorageProvider } from './providers/github-provider.js'
export { LocalStorageProvider } from './providers/local-provider.js'
export { S3StorageProvider } from './providers/s3-provider.js'

View File

@@ -1,5 +1,15 @@
import type { Logger } from '../logger/index.js'
// 扫描进度接口
export interface ScanProgress {
currentPath: string
filesScanned: number
totalFiles?: number
}
// 进度回调类型
export type ProgressCallback = (progress: ScanProgress) => void
// 存储对象的通用接口
export interface StorageObject {
key: string
@@ -26,9 +36,12 @@ export interface StorageProvider {
/**
* 列出存储中的所有文件
* @param progressCallback 可选的进度回调函数
* @returns 所有文件对象数组
*/
listAllFiles: () => Promise<StorageObject[]>
listAllFiles: (
progressCallback?: ProgressCallback,
) => Promise<StorageObject[]>
/**
* 生成文件的公共访问 URL
@@ -67,4 +80,13 @@ export type GitHubConfig = {
path?: string
useRawUrl?: boolean
}
export type StorageConfig = S3Config | GitHubConfig
export type LocalConfig = {
provider: 'local'
basePath: string
baseUrl?: string
excludeRegex?: string
maxFileLimit?: number
}
export type StorageConfig = S3Config | GitHubConfig | LocalConfig

View File

@@ -144,6 +144,122 @@ GitHub 存储提供商会处理以下错误:
- **422 Unprocessable Entity**: 请求格式错误
- **500+ Server Error**: GitHub 服务器错误
## 本地存储提供商
将照片存储在本地文件系统中,适合开发环境或私有部署。
### 特点
- ✅ 无需外部依赖
- ✅ 快速访问速度
- ✅ 完全私有控制
- ✅ 支持递归目录扫描
- ✅ 支持 Live Photos 检测
- ⚠️ 需要确保文件系统权限
- ⚠️ 不适合分布式部署
### 配置示例
```typescript
const localConfig: StorageConfig = {
provider: 'local',
basePath: './photos', // 本地照片存储路径(相对或绝对路径)
baseUrl: 'http://localhost:3000/photos', // 可选:用于生成公共 URL
excludeRegex: '\\.(tmp|cache)$', // 可选:排除文件的正则表达式
maxFileLimit: 1000, // 可选:最大文件数量限制
}
```
### 路径配置
- **相对路径**: 相对于项目根目录,如 `./photos`、`../images`
- **绝对路径**: 完整的文件系统路径,如 `/home/user/photos`、`C:\\Photos`
### 使用示例
```typescript
import { LocalStorageProvider } from '@/core/storage'
const localProvider = new LocalStorageProvider({
provider: 'local',
basePath: './photos',
baseUrl: 'http://localhost:3000/photos',
})
// 获取文件
const buffer = await localProvider.getFile('sunset.jpg')
// 列出所有图片
const images = await localProvider.listImages()
// 生成公共 URL
const url = localProvider.generatePublicUrl('sunset.jpg')
// 结果http://localhost:3000/photos/sunset.jpg
// 检查存储路径
const exists = await localProvider.checkBasePath()
if (!exists) {
await localProvider.ensureBasePath()
}
```
### 目录结构示例
```
photos/
├── 2024/
│ ├── 01-january/
│ │ ├── IMG_001.jpg
│ │ ├── IMG_001.mov # Live Photo 视频
│ │ └── IMG_002.heic
│ └── 02-february/
│ └── sunset.jpg
├── 2023/
│ └── vacation/
│ ├── beach.jpg
│ └── mountain.png
└── misc/
└── screenshot.png
```
### 最佳实践
1. **权限管理**: 确保应用有读取照片目录的权限
2. **路径安全**: 避免使用包含特殊字符的路径
3. **性能优化**: 对于大量文件,考虑使用 `maxFileLimit` 限制
4. **备份策略**: 定期备份重要照片文件
5. **监控空间**: 监控磁盘空间使用情况
### 开发环境配置
对于开发环境,推荐使用相对路径:
```json
{
"storage": {
"provider": "local",
"basePath": "./dev-photos",
"baseUrl": "http://localhost:1924/photos"
}
}
```
### 生产环境配置
对于生产环境,推荐使用绝对路径:
```json
{
"storage": {
"provider": "local",
"basePath": "/var/www/photos",
"baseUrl": "https://yourdomain.com/photos",
"excludeRegex": "\\.(tmp|cache|DS_Store)$",
"maxFileLimit": 5000
}
}
```
### 与其他提供商的对比
| 特性 | S3 | GitHub |

View File

@@ -4,6 +4,7 @@ import { SUPPORTED_FORMATS } from '../../constants/index.js'
import type { Logger } from '../../logger/index.js'
import type {
GitHubConfig,
ProgressCallback,
StorageObject,
StorageProvider,
} from '../interfaces.js'
@@ -155,11 +156,13 @@ export class GitHubStorageProvider implements StorageProvider {
})
}
async listAllFiles(): Promise<StorageObject[]> {
async listAllFiles(
progressCallback?: ProgressCallback,
): Promise<StorageObject[]> {
const files: StorageObject[] = []
const basePath = this.githubConfig.path || ''
await this.listFilesRecursive(basePath, files)
await this.listFilesRecursive(basePath, files, progressCallback)
return files
}
@@ -167,6 +170,7 @@ export class GitHubStorageProvider implements StorageProvider {
private async listFilesRecursive(
dirPath: string,
files: StorageObject[],
progressCallback?: ProgressCallback,
): Promise<void> {
try {
const url = `${this.baseApiUrl}/contents/${dirPath}?ref=${this.githubConfig.branch}`
@@ -207,7 +211,7 @@ export class GitHubStorageProvider implements StorageProvider {
})
} else if (item.type === 'dir') {
// 递归处理子目录
await this.listFilesRecursive(item.path, files)
await this.listFilesRecursive(item.path, files, progressCallback)
}
}
} catch (error) {

View File

@@ -0,0 +1,307 @@
import fs from 'node:fs/promises'
import path from 'node:path'
import { fileURLToPath } from 'node:url'
import { SUPPORTED_FORMATS } from '../../constants/index.js'
import { logger } from '../../logger/index.js'
import type { StorageObject, StorageProvider } from '../interfaces'
export interface LocalConfig {
provider: 'local'
basePath: string // 本地照片存储的基础路径
baseUrl?: string // 用于生成公共 URL 的基础 URL可选
excludeRegex?: string // 排除文件的正则表达式
maxFileLimit?: number // 最大文件数量限制
}
export interface ScanProgress {
currentPath: string
filesScanned: number
totalFiles?: number
}
export type ProgressCallback = (progress: ScanProgress) => void
export class LocalStorageProvider implements StorageProvider {
private config: LocalConfig
private basePath: string
private scanProgress: ScanProgress = {
currentPath: '',
filesScanned: 0,
}
constructor(config: LocalConfig) {
// 参数验证
if (!config.basePath || config.basePath.trim() === '') {
throw new Error('LocalStorageProvider: basePath 不能为空')
}
if (config.maxFileLimit && config.maxFileLimit <= 0) {
throw new Error('LocalStorageProvider: maxFileLimit 必须大于 0')
}
if (config.excludeRegex) {
try {
new RegExp(config.excludeRegex)
} catch (error) {
throw new Error(
`LocalStorageProvider: excludeRegex 不是有效的正则表达式: ${error}`,
)
}
}
this.config = config
// 处理相对路径和绝对路径
if (path.isAbsolute(config.basePath)) {
this.basePath = config.basePath
} else {
// 相对于项目根目录
const __dirname = path.dirname(fileURLToPath(import.meta.url))
const projectRoot = path.resolve(__dirname, '../../../../../')
this.basePath = path.resolve(projectRoot, config.basePath)
}
}
async getFile(key: string, logger?: any): Promise<Buffer | null> {
try {
logger?.info(`读取本地文件:${key}`)
const startTime = Date.now()
const filePath = path.join(this.basePath, key)
// 安全检查:确保文件路径在基础路径内
const resolvedPath = path.resolve(filePath)
const resolvedBasePath = path.resolve(this.basePath)
if (!resolvedPath.startsWith(resolvedBasePath)) {
logger?.error(`文件路径不安全:${key}`)
return null
}
// 检查文件是否存在
try {
await fs.access(filePath)
} catch {
logger?.warn(`文件不存在:${key}`)
return null
}
const buffer = await fs.readFile(filePath)
const duration = Date.now() - startTime
const sizeKB = Math.round(buffer.length / 1024)
logger?.success(`读取完成:${key} (${sizeKB}KB, ${duration}ms)`)
return buffer
} catch (error) {
const errorType = error instanceof Error ? error.name : 'UnknownError'
const errorMessage =
error instanceof Error ? error.message : String(error)
logger?.error(`[${errorType}] 读取文件失败:${key} - ${errorMessage}`)
return null
}
}
async listImages(): Promise<StorageObject[]> {
const allFiles = await this.listAllFiles()
// 过滤出图片文件
return allFiles.filter((file) => {
const ext = path.extname(file.key).toLowerCase()
return SUPPORTED_FORMATS.has(ext)
})
}
async listAllFiles(
progressCallback?: ProgressCallback,
): Promise<StorageObject[]> {
const files: StorageObject[] = []
const excludeRegex = this.config.excludeRegex
? new RegExp(this.config.excludeRegex)
: null
// 重置进度
this.scanProgress = {
currentPath: '',
filesScanned: 0,
}
await this.scanDirectory(
this.basePath,
'',
files,
excludeRegex,
progressCallback,
)
// 应用文件数量限制
if (this.config.maxFileLimit && files.length > this.config.maxFileLimit) {
logger.main.info(
`文件数量超过限制 ${this.config.maxFileLimit},截取前 ${this.config.maxFileLimit} 个文件`,
)
return files.slice(0, this.config.maxFileLimit)
}
return files
}
private async scanDirectory(
dirPath: string,
relativePath: string,
files: StorageObject[],
excludeRegex?: RegExp | null,
progressCallback?: ProgressCallback,
): Promise<void> {
try {
// 更新进度
this.scanProgress.currentPath = relativePath || '/'
progressCallback?.(this.scanProgress)
const entries = await fs.readdir(dirPath, { withFileTypes: true })
for (const entry of entries) {
const fullPath = path.join(dirPath, entry.name)
const relativeFilePath = relativePath
? path.join(relativePath, entry.name).replaceAll('\\', '/')
: entry.name
// 应用排除规则
if (excludeRegex && excludeRegex.test(relativeFilePath)) {
continue
}
if (entry.isDirectory()) {
// 递归扫描子目录
await this.scanDirectory(
fullPath,
relativeFilePath,
files,
excludeRegex,
progressCallback,
)
} else if (entry.isFile()) {
try {
const stats = await fs.stat(fullPath)
files.push({
key: relativeFilePath,
size: stats.size,
lastModified: stats.mtime,
etag: this.generateETag(stats),
})
// 更新已扫描文件数
this.scanProgress.filesScanned++
if (this.scanProgress.filesScanned % 100 === 0) {
// 每 100 个文件报告一次进度
progressCallback?.(this.scanProgress)
}
} catch (error) {
const errorType =
error instanceof Error ? error.name : 'UnknownError'
const errorMessage =
error instanceof Error ? error.message : String(error)
logger.main.warn(
`[${errorType}] 获取文件信息失败:${relativeFilePath} - ${errorMessage}`,
)
}
}
}
} catch (error) {
const errorType = error instanceof Error ? error.name : 'UnknownError'
const errorMessage =
error instanceof Error ? error.message : String(error)
logger.main.error(
`[${errorType}] 扫描目录失败:${dirPath} - ${errorMessage}`,
)
}
}
generatePublicUrl(key: string): string {
if (this.config.baseUrl) {
// 如果配置了基础 URL生成完整的 HTTP URL
return `${this.config.baseUrl.replace(/\/$/, '')}/${key}`
} else {
// 否则返回文件系统路径(用于开发环境)
return `file://${path.join(this.basePath, key)}`
}
}
detectLivePhotos(allObjects: StorageObject[]): Map<string, StorageObject> {
const livePhotos = new Map<string, StorageObject>()
// 创建一个映射来快速查找文件
const fileMap = new Map<string, StorageObject>()
allObjects.forEach((obj) => {
fileMap.set(obj.key.toLowerCase(), obj)
})
// 查找 Live Photos 配对
allObjects.forEach((obj) => {
const ext = path.extname(obj.key).toLowerCase()
// 如果是图片文件,查找对应的视频文件
if (SUPPORTED_FORMATS.has(ext)) {
const baseName = path.basename(obj.key, ext)
const dirName = path.dirname(obj.key)
// 查找对应的 .mov 文件
const videoKey = path
.join(dirName, `${baseName}.mov`)
.replaceAll('\\', '/')
const videoObj = fileMap.get(videoKey.toLowerCase())
if (videoObj) {
livePhotos.set(obj.key, videoObj)
}
}
})
return livePhotos
}
/**
* 生成文件的 ETag
*/
private generateETag(stats: fs.Stats): string {
return `${stats.mtime.getTime()}-${stats.size}`
}
/**
* 获取本地存储的基础路径
*/
getBasePath(): string {
return this.basePath
}
/**
* 检查基础路径是否存在
*/
async checkBasePath(): Promise<boolean> {
try {
const stats = await fs.stat(this.basePath)
return stats.isDirectory()
} catch {
return false
}
}
/**
* 创建基础路径目录(如果不存在)
*/
async ensureBasePath(): Promise<void> {
try {
await fs.mkdir(this.basePath, { recursive: true })
logger.main.info(`创建本地存储目录:${this.basePath}`)
} catch (error) {
const errorType = error instanceof Error ? error.name : 'UnknownError'
const errorMessage =
error instanceof Error ? error.message : String(error)
logger.main.error(
`[${errorType}] 创建本地存储目录失败:${this.basePath} - ${errorMessage}`,
)
throw error
}
}
}

View File

@@ -1,12 +1,17 @@
import path from 'node:path'
import type { _Object } from '@aws-sdk/client-s3'
import type { _Object, S3Client } from '@aws-sdk/client-s3'
import { GetObjectCommand, ListObjectsV2Command } from '@aws-sdk/client-s3'
import { SUPPORTED_FORMATS } from '../../constants/index.js'
import { logger } from '../../logger/index.js'
import { s3Client } from '../../s3/client.js'
import type { S3Config, StorageObject, StorageProvider } from '../interfaces'
import { createS3Client } from '../../s3/client.js'
import type {
ProgressCallback,
S3Config,
StorageObject,
StorageProvider,
} from '../interfaces'
// 将 AWS S3 对象转换为通用存储对象
function convertS3ObjectToStorageObject(s3Object: _Object): StorageObject {
@@ -20,9 +25,11 @@ function convertS3ObjectToStorageObject(s3Object: _Object): StorageObject {
export class S3StorageProvider implements StorageProvider {
private config: S3Config
private s3Client: S3Client
constructor(config: S3Config) {
this.config = config
this.s3Client = createS3Client(config)
}
async getFile(key: string): Promise<Buffer | null> {
@@ -35,7 +42,7 @@ export class S3StorageProvider implements StorageProvider {
Key: key,
})
const response = await s3Client.send(command)
const response = await this.s3Client.send(command)
if (!response.Body) {
logger.s3.error(`S3 响应中没有 Body: ${key}`)
@@ -85,7 +92,7 @@ export class S3StorageProvider implements StorageProvider {
MaxKeys: this.config.maxFileLimit, // 最多获取 1000 张照片
})
const listResponse = await s3Client.send(listCommand)
const listResponse = await this.s3Client.send(listCommand)
const objects = listResponse.Contents || []
const excludeRegex = this.config.excludeRegex
? new RegExp(this.config.excludeRegex)
@@ -105,14 +112,16 @@ export class S3StorageProvider implements StorageProvider {
return imageObjects
}
async listAllFiles(): Promise<StorageObject[]> {
async listAllFiles(
_progressCallback?: ProgressCallback,
): Promise<StorageObject[]> {
const listCommand = new ListObjectsV2Command({
Bucket: this.config.bucket,
Prefix: this.config.prefix,
MaxKeys: this.config.maxFileLimit,
})
const listResponse = await s3Client.send(listCommand)
const listResponse = await this.s3Client.send(listCommand)
const objects = listResponse.Contents || []
return objects.map((obj) => convertS3ObjectToStorageObject(obj))