perf: 优化容量统计,从索引元数据读取,减少 KV 写入操作

This commit is contained in:
axibayuit
2025-12-29 23:20:04 +08:00
parent 13fb3ead77
commit 155aacab63
18 changed files with 138 additions and 100 deletions

View File

@@ -785,6 +785,46 @@ export async function getIndexInfo(context) {
}
}
/**
* 获取索引元数据(轻量级,只读取 meta不读取整个索引
* 用于容量检查等场景,避免读取整个索引
* @param {Object} context - 上下文对象
* @returns {Object} 索引元数据,包含 totalCount, totalSizeMB, channelStats 等
*/
export async function getIndexMeta(context) {
const { env } = context;
const db = getDatabase(env);
try {
const metadataStr = await db.get(INDEX_META_KEY);
if (!metadataStr) {
return {
success: false,
totalCount: 0,
totalSizeMB: 0,
channelStats: {}
};
}
const metadata = JSON.parse(metadataStr);
return {
success: true,
totalCount: metadata.totalCount || 0,
totalSizeMB: metadata.totalSizeMB || 0,
channelStats: metadata.channelStats || {},
lastUpdated: metadata.lastUpdated
};
} catch (error) {
console.error('Error getting index meta:', error);
return {
success: false,
totalCount: 0,
totalSizeMB: 0,
channelStats: {}
};
}
}
/* ============= 原子操作相关函数 ============= */
/**
@@ -1300,10 +1340,31 @@ async function saveChunkedIndex(context, index) {
chunks.push(chunk);
}
// 保存索引元数据
// 计算各渠道容量统计
const channelStats = {};
let totalSizeMB = 0;
for (const file of files) {
const channelName = file.metadata?.ChannelName;
const fileSize = parseFloat(file.metadata?.FileSize) || 0;
totalSizeMB += fileSize;
if (channelName) {
if (!channelStats[channelName]) {
channelStats[channelName] = { usedMB: 0, fileCount: 0 };
}
channelStats[channelName].usedMB += fileSize;
channelStats[channelName].fileCount += 1;
}
}
// 保存索引元数据(包含容量统计)
const metadata = {
lastUpdated: index.lastUpdated,
totalCount: index.totalCount,
totalSizeMB: Math.round(totalSizeMB * 100) / 100,
channelStats,
lastOperationId: index.lastOperationId,
chunkCount: chunks.length,
chunkSize: INDEX_CHUNK_SIZE
@@ -1319,7 +1380,7 @@ async function saveChunkedIndex(context, index) {
await Promise.all(savePromises);
console.log(`Saved chunked index: ${chunks.length} chunks, ${files.length} total files`);
console.log(`Saved chunked index: ${chunks.length} chunks, ${files.length} total files, ${totalSizeMB.toFixed(2)} MB`);
return true;
} catch (error) {

View File

@@ -3,14 +3,19 @@ import { getSecurityConfig } from '../api/manage/sysConfig/security';
import { getPageConfig } from '../api/manage/sysConfig/page';
import { getOthersConfig } from '../api/manage/sysConfig/others';
import { getDatabase } from './databaseAdapter.js';
import { getIndexMeta } from './indexManager.js';
/**
* 根据容量限制过滤渠道
* @param {Object} db - 数据库实例
* @param {Object} context - 上下文对象(包含 env
* @param {Array} channels - 渠道列表
* @returns {Array} 过滤后的渠道列表
*/
async function filterChannelsByQuota(db, channels) {
async function filterChannelsByQuota(context, channels) {
// 获取索引元数据(只需 1 次读取)
const indexMeta = await getIndexMeta(context);
const channelStats = indexMeta.channelStats || {};
const result = [];
for (const channel of channels) {
// 未启用容量限制,直接通过
@@ -20,11 +25,10 @@ async function filterChannelsByQuota(db, channels) {
}
try {
const quotaKey = `manage@quota@${channel.name}`;
const quotaData = await db.get(quotaKey);
const quota = quotaData ? JSON.parse(quotaData) : { usedMB: 0, fileCount: 0 };
// 从索引元数据中获取该渠道的容量统计
const stats = channelStats[channel.name] || { usedMB: 0, fileCount: 0 };
const usedGB = quota.usedMB / 1024;
const usedGB = stats.usedMB / 1024;
const limitGB = channel.quota.limitGB;
const threshold = channel.quota.threshold || 95;
@@ -43,7 +47,7 @@ async function filterChannelsByQuota(db, channels) {
return result;
}
export async function fetchUploadConfig(env) {
export async function fetchUploadConfig(env, context = null) {
try {
const db = getDatabase(env);
const settings = await getUploadConfig(db, env);
@@ -53,8 +57,11 @@ export async function fetchUploadConfig(env) {
settings.s3.channels = settings.s3.channels.filter((channel) => channel.enabled);
// 根据容量限制过滤渠道(仅 R2 和 S3
settings.cfr2.channels = await filterChannelsByQuota(db, settings.cfr2.channels);
settings.s3.channels = await filterChannelsByQuota(db, settings.s3.channels);
// 需要 context 来调用 getIndexMeta
if (context) {
settings.cfr2.channels = await filterChannelsByQuota(context, settings.cfr2.channels);
settings.s3.channels = await filterChannelsByQuota(context, settings.s3.channels);
}
return settings;
} catch (error) {