|
|
|
|
@@ -6,10 +6,6 @@ import { Database } from "../../storage/db"
|
|
|
|
|
import { SessionTable } from "../../session/session.sql"
|
|
|
|
|
import { Project } from "../../project/project"
|
|
|
|
|
import { Instance } from "../../project/instance"
|
|
|
|
|
import { inArray } from "drizzle-orm"
|
|
|
|
|
import { MessageTable, PartTable } from "../../session/session.sql"
|
|
|
|
|
import type { MessageV2 } from "../../session/message-v2"
|
|
|
|
|
import { and, eq, gte } from "drizzle-orm"
|
|
|
|
|
|
|
|
|
|
interface SessionStats {
|
|
|
|
|
totalSessions: number
|
|
|
|
|
@@ -24,7 +20,7 @@ interface SessionStats {
|
|
|
|
|
write: number
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
toolUsage: Record<string, { calls: number; errors: number }>
|
|
|
|
|
toolUsage: Record<string, number>
|
|
|
|
|
modelUsage: Record<
|
|
|
|
|
string,
|
|
|
|
|
{
|
|
|
|
|
@@ -38,7 +34,6 @@ interface SessionStats {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
cost: number
|
|
|
|
|
toolUsage: Record<string, { calls: number; errors: number }>
|
|
|
|
|
}
|
|
|
|
|
>
|
|
|
|
|
dateRange: {
|
|
|
|
|
@@ -67,11 +62,6 @@ export const StatsCommand = cmd({
|
|
|
|
|
.option("models", {
|
|
|
|
|
describe: "show model statistics (default: hidden). Pass a number to show top N, otherwise shows all",
|
|
|
|
|
})
|
|
|
|
|
.option("model", {
|
|
|
|
|
describe: "filter models to show (can be used multiple times)",
|
|
|
|
|
type: "array",
|
|
|
|
|
string: true,
|
|
|
|
|
})
|
|
|
|
|
.option("project", {
|
|
|
|
|
describe: "filter by project (default: all projects, empty string: current project)",
|
|
|
|
|
type: "string",
|
|
|
|
|
@@ -82,20 +72,13 @@ export const StatsCommand = cmd({
|
|
|
|
|
const stats = await aggregateSessionStats(args.days, args.project)
|
|
|
|
|
|
|
|
|
|
let modelLimit: number | undefined
|
|
|
|
|
let modelFilter: string[] | undefined
|
|
|
|
|
|
|
|
|
|
if (args.models === true) {
|
|
|
|
|
modelLimit = Infinity
|
|
|
|
|
} else if (typeof args.models === "number") {
|
|
|
|
|
modelLimit = args.models
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (args.model && args.model.length > 0) {
|
|
|
|
|
modelFilter = args.model as string[]
|
|
|
|
|
modelLimit = modelLimit ?? Infinity
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
displayStats(stats, args.tools, modelLimit, modelFilter)
|
|
|
|
|
displayStats(stats, args.tools, modelLimit)
|
|
|
|
|
})
|
|
|
|
|
},
|
|
|
|
|
})
|
|
|
|
|
@@ -103,7 +86,14 @@ export const StatsCommand = cmd({
|
|
|
|
|
async function getCurrentProject(): Promise<Project.Info> {
|
|
|
|
|
return Instance.project
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function getAllSessions(): Promise<Session.Info[]> {
|
|
|
|
|
const rows = Database.use((db) => db.select().from(SessionTable).all())
|
|
|
|
|
return rows.map((row) => Session.fromRow(row))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export async function aggregateSessionStats(days?: number, projectFilter?: string): Promise<SessionStats> {
|
|
|
|
|
const sessions = await getAllSessions()
|
|
|
|
|
const MS_IN_DAY = 24 * 60 * 60 * 1000
|
|
|
|
|
|
|
|
|
|
const cutoffTime = (() => {
|
|
|
|
|
@@ -122,34 +112,17 @@ export async function aggregateSessionStats(days?: number, projectFilter?: strin
|
|
|
|
|
return days
|
|
|
|
|
})()
|
|
|
|
|
|
|
|
|
|
let projectID: string | undefined
|
|
|
|
|
let filteredSessions = cutoffTime > 0 ? sessions.filter((session) => session.time.updated >= cutoffTime) : sessions
|
|
|
|
|
|
|
|
|
|
if (projectFilter !== undefined) {
|
|
|
|
|
if (projectFilter === "") {
|
|
|
|
|
const currentProject = await getCurrentProject()
|
|
|
|
|
projectID = currentProject.id
|
|
|
|
|
filteredSessions = filteredSessions.filter((session) => session.projectID === currentProject.id)
|
|
|
|
|
} else {
|
|
|
|
|
projectID = projectFilter
|
|
|
|
|
filteredSessions = filteredSessions.filter((session) => session.projectID === projectFilter)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const rows = Database.use((db) => {
|
|
|
|
|
const conditions = []
|
|
|
|
|
if (cutoffTime > 0) {
|
|
|
|
|
conditions.push(gte(SessionTable.time_updated, cutoffTime))
|
|
|
|
|
}
|
|
|
|
|
if (projectID !== undefined) {
|
|
|
|
|
conditions.push(eq(SessionTable.project_id, projectID))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const baseQuery = db.select().from(SessionTable)
|
|
|
|
|
if (conditions.length > 0) {
|
|
|
|
|
return baseQuery.where(and(...conditions)).all()
|
|
|
|
|
}
|
|
|
|
|
return baseQuery.all()
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
const filteredSessions = rows.map((row) => Session.fromRow(row))
|
|
|
|
|
|
|
|
|
|
const stats: SessionStats = {
|
|
|
|
|
totalSessions: filteredSessions.length,
|
|
|
|
|
totalMessages: 0,
|
|
|
|
|
@@ -189,58 +162,16 @@ export async function aggregateSessionStats(days?: number, projectFilter?: strin
|
|
|
|
|
|
|
|
|
|
const sessionTotalTokens: number[] = []
|
|
|
|
|
|
|
|
|
|
const BATCH_SIZE = 100
|
|
|
|
|
const BATCH_SIZE = 20
|
|
|
|
|
for (let i = 0; i < filteredSessions.length; i += BATCH_SIZE) {
|
|
|
|
|
const batch = filteredSessions.slice(i, i + BATCH_SIZE)
|
|
|
|
|
const sessionIds = batch.map((s) => s.id)
|
|
|
|
|
|
|
|
|
|
// Bulk fetch messages for this batch of sessions
|
|
|
|
|
const messageRows = Database.use((db) =>
|
|
|
|
|
db.select().from(MessageTable).where(inArray(MessageTable.session_id, sessionIds)).all(),
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
// Group messages by session_id
|
|
|
|
|
const messagesBySession = new Map<string, typeof messageRows>()
|
|
|
|
|
const messageIds = messageRows.map((r) => r.id)
|
|
|
|
|
|
|
|
|
|
for (const row of messageRows) {
|
|
|
|
|
const msgs = messagesBySession.get(row.session_id) || []
|
|
|
|
|
msgs.push(row)
|
|
|
|
|
messagesBySession.set(row.session_id, msgs)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Bulk fetch parts for all these messages
|
|
|
|
|
let partRows: (typeof PartTable.$inferSelect)[] = []
|
|
|
|
|
if (messageIds.length > 0) {
|
|
|
|
|
// Chunk message IDs if there are too many for a single IN clause (SQLite has limits)
|
|
|
|
|
const PART_BATCH_SIZE = 500
|
|
|
|
|
for (let j = 0; j < messageIds.length; j += PART_BATCH_SIZE) {
|
|
|
|
|
const idBatch = messageIds.slice(j, j + PART_BATCH_SIZE)
|
|
|
|
|
const parts = Database.use((db) =>
|
|
|
|
|
db.select().from(PartTable).where(inArray(PartTable.message_id, idBatch)).all(),
|
|
|
|
|
)
|
|
|
|
|
partRows.push(...parts)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Group parts by message_id
|
|
|
|
|
const partsByMessage = new Map<string, MessageV2.Part[]>()
|
|
|
|
|
for (const row of partRows) {
|
|
|
|
|
const parts = partsByMessage.get(row.message_id) || []
|
|
|
|
|
parts.push({ ...row.data, id: row.id, sessionID: row.session_id, messageID: row.message_id } as MessageV2.Part)
|
|
|
|
|
partsByMessage.set(row.message_id, parts)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const batchResults = batch.map((session) => {
|
|
|
|
|
const rawMessages = messagesBySession.get(session.id) || []
|
|
|
|
|
const messages = rawMessages.map((row) => ({
|
|
|
|
|
info: { ...row.data, id: row.id, sessionID: row.session_id } as MessageV2.Info,
|
|
|
|
|
parts: partsByMessage.get(row.id) || [],
|
|
|
|
|
}))
|
|
|
|
|
const batchPromises = batch.map(async (session) => {
|
|
|
|
|
const messages = await Session.messages({ sessionID: session.id })
|
|
|
|
|
|
|
|
|
|
let sessionCost = 0
|
|
|
|
|
let sessionTokens = { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }
|
|
|
|
|
let sessionToolUsage: Record<string, { calls: number; errors: number }> = {}
|
|
|
|
|
let sessionToolUsage: Record<string, number> = {}
|
|
|
|
|
let sessionModelUsage: Record<
|
|
|
|
|
string,
|
|
|
|
|
{
|
|
|
|
|
@@ -254,7 +185,6 @@ export async function aggregateSessionStats(days?: number, projectFilter?: strin
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
cost: number
|
|
|
|
|
toolUsage: Record<string, { calls: number; errors: number }>
|
|
|
|
|
}
|
|
|
|
|
> = {}
|
|
|
|
|
|
|
|
|
|
@@ -268,7 +198,6 @@ export async function aggregateSessionStats(days?: number, projectFilter?: strin
|
|
|
|
|
messages: 0,
|
|
|
|
|
tokens: { input: 0, output: 0, cache: { read: 0, write: 0 } },
|
|
|
|
|
cost: 0,
|
|
|
|
|
toolUsage: {},
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
sessionModelUsage[modelKey].messages++
|
|
|
|
|
@@ -287,22 +216,11 @@ export async function aggregateSessionStats(days?: number, projectFilter?: strin
|
|
|
|
|
sessionModelUsage[modelKey].tokens.cache.read += message.info.tokens.cache?.read || 0
|
|
|
|
|
sessionModelUsage[modelKey].tokens.cache.write += message.info.tokens.cache?.write || 0
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (const part of message.parts) {
|
|
|
|
|
if (part.type === "tool" && part.tool) {
|
|
|
|
|
const isError =
|
|
|
|
|
part.state && part.state.status === "error" && part.state.error !== "Tool execution aborted"
|
|
|
|
|
|
|
|
|
|
if (!sessionToolUsage[part.tool]) sessionToolUsage[part.tool] = { calls: 0, errors: 0 }
|
|
|
|
|
sessionToolUsage[part.tool].calls++
|
|
|
|
|
if (isError) sessionToolUsage[part.tool].errors++
|
|
|
|
|
|
|
|
|
|
if (!sessionModelUsage[modelKey].toolUsage[part.tool]) {
|
|
|
|
|
sessionModelUsage[modelKey].toolUsage[part.tool] = { calls: 0, errors: 0 }
|
|
|
|
|
}
|
|
|
|
|
sessionModelUsage[modelKey].toolUsage[part.tool].calls++
|
|
|
|
|
if (isError) sessionModelUsage[modelKey].toolUsage[part.tool].errors++
|
|
|
|
|
}
|
|
|
|
|
for (const part of message.parts) {
|
|
|
|
|
if (part.type === "tool" && part.tool) {
|
|
|
|
|
sessionToolUsage[part.tool] = (sessionToolUsage[part.tool] || 0) + 1
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
@@ -324,6 +242,8 @@ export async function aggregateSessionStats(days?: number, projectFilter?: strin
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
const batchResults = await Promise.all(batchPromises)
|
|
|
|
|
|
|
|
|
|
for (const result of batchResults) {
|
|
|
|
|
earliestTime = Math.min(earliestTime, result.earliestTime)
|
|
|
|
|
latestTime = Math.max(latestTime, result.latestTime)
|
|
|
|
|
@@ -338,9 +258,7 @@ export async function aggregateSessionStats(days?: number, projectFilter?: strin
|
|
|
|
|
stats.totalTokens.cache.write += result.sessionTokens.cache.write
|
|
|
|
|
|
|
|
|
|
for (const [tool, count] of Object.entries(result.sessionToolUsage)) {
|
|
|
|
|
if (!stats.toolUsage[tool]) stats.toolUsage[tool] = { calls: 0, errors: 0 }
|
|
|
|
|
stats.toolUsage[tool].calls += count.calls
|
|
|
|
|
stats.toolUsage[tool].errors += count.errors
|
|
|
|
|
stats.toolUsage[tool] = (stats.toolUsage[tool] || 0) + count
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (const [model, usage] of Object.entries(result.sessionModelUsage)) {
|
|
|
|
|
@@ -349,7 +267,6 @@ export async function aggregateSessionStats(days?: number, projectFilter?: strin
|
|
|
|
|
messages: 0,
|
|
|
|
|
tokens: { input: 0, output: 0, cache: { read: 0, write: 0 } },
|
|
|
|
|
cost: 0,
|
|
|
|
|
toolUsage: {},
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
stats.modelUsage[model].messages += usage.messages
|
|
|
|
|
@@ -358,14 +275,6 @@ export async function aggregateSessionStats(days?: number, projectFilter?: strin
|
|
|
|
|
stats.modelUsage[model].tokens.cache.read += usage.tokens.cache.read
|
|
|
|
|
stats.modelUsage[model].tokens.cache.write += usage.tokens.cache.write
|
|
|
|
|
stats.modelUsage[model].cost += usage.cost
|
|
|
|
|
|
|
|
|
|
for (const [tool, toolUsage] of Object.entries(usage.toolUsage)) {
|
|
|
|
|
if (!stats.modelUsage[model].toolUsage[tool]) {
|
|
|
|
|
stats.modelUsage[model].toolUsage[tool] = { calls: 0, errors: 0 }
|
|
|
|
|
}
|
|
|
|
|
stats.modelUsage[model].toolUsage[tool].calls += toolUsage.calls
|
|
|
|
|
stats.modelUsage[model].toolUsage[tool].errors += toolUsage.errors
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
@@ -397,7 +306,7 @@ export async function aggregateSessionStats(days?: number, projectFilter?: strin
|
|
|
|
|
return stats
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function displayStats(stats: SessionStats, toolLimit?: number, modelLimit?: number, modelFilter?: string[]) {
|
|
|
|
|
export function displayStats(stats: SessionStats, toolLimit?: number, modelLimit?: number) {
|
|
|
|
|
const width = 56
|
|
|
|
|
|
|
|
|
|
function renderRow(label: string, value: string): string {
|
|
|
|
|
@@ -437,73 +346,43 @@ export function displayStats(stats: SessionStats, toolLimit?: number, modelLimit
|
|
|
|
|
console.log()
|
|
|
|
|
|
|
|
|
|
// Model Usage section
|
|
|
|
|
if ((modelLimit !== undefined || modelFilter !== undefined) && Object.keys(stats.modelUsage).length > 0) {
|
|
|
|
|
let sortedModels = Object.entries(stats.modelUsage).sort(([, a], [, b]) => b.messages - a.messages)
|
|
|
|
|
if (modelLimit !== undefined && Object.keys(stats.modelUsage).length > 0) {
|
|
|
|
|
const sortedModels = Object.entries(stats.modelUsage).sort(([, a], [, b]) => b.messages - a.messages)
|
|
|
|
|
const modelsToDisplay = modelLimit === Infinity ? sortedModels : sortedModels.slice(0, modelLimit)
|
|
|
|
|
|
|
|
|
|
if (modelFilter && modelFilter.length > 0) {
|
|
|
|
|
sortedModels = sortedModels.filter(([model]) => modelFilter.some((filter) => model.includes(filter)))
|
|
|
|
|
}
|
|
|
|
|
console.log("┌────────────────────────────────────────────────────────┐")
|
|
|
|
|
console.log("│ MODEL USAGE │")
|
|
|
|
|
console.log("├────────────────────────────────────────────────────────┤")
|
|
|
|
|
|
|
|
|
|
const modelsToDisplay =
|
|
|
|
|
modelLimit === Infinity || modelLimit === undefined ? sortedModels : sortedModels.slice(0, modelLimit)
|
|
|
|
|
|
|
|
|
|
if (modelsToDisplay.length > 0) {
|
|
|
|
|
console.log("┌────────────────────────────────────────────────────────┐")
|
|
|
|
|
console.log("│ MODEL USAGE │")
|
|
|
|
|
for (const [model, usage] of modelsToDisplay) {
|
|
|
|
|
console.log(`│ ${model.padEnd(54)} │`)
|
|
|
|
|
console.log(renderRow(" Messages", usage.messages.toLocaleString()))
|
|
|
|
|
console.log(renderRow(" Input Tokens", formatNumber(usage.tokens.input)))
|
|
|
|
|
console.log(renderRow(" Output Tokens", formatNumber(usage.tokens.output)))
|
|
|
|
|
console.log(renderRow(" Cache Read", formatNumber(usage.tokens.cache.read)))
|
|
|
|
|
console.log(renderRow(" Cache Write", formatNumber(usage.tokens.cache.write)))
|
|
|
|
|
console.log(renderRow(" Cost", `$${usage.cost.toFixed(4)}`))
|
|
|
|
|
console.log("├────────────────────────────────────────────────────────┤")
|
|
|
|
|
|
|
|
|
|
for (const [model, usage] of modelsToDisplay) {
|
|
|
|
|
console.log(`│ ${model.padEnd(54)} │`)
|
|
|
|
|
console.log(renderRow(" Messages", usage.messages.toLocaleString()))
|
|
|
|
|
console.log(renderRow(" Input Tokens", formatNumber(usage.tokens.input)))
|
|
|
|
|
console.log(renderRow(" Output Tokens", formatNumber(usage.tokens.output)))
|
|
|
|
|
console.log(renderRow(" Cache Read", formatNumber(usage.tokens.cache.read)))
|
|
|
|
|
console.log(renderRow(" Cache Write", formatNumber(usage.tokens.cache.write)))
|
|
|
|
|
console.log(renderRow(" Cost", `$${usage.cost.toFixed(4)}`))
|
|
|
|
|
|
|
|
|
|
if (Object.keys(usage.toolUsage).length > 0) {
|
|
|
|
|
console.log(`│ │`)
|
|
|
|
|
console.log(`│ Tool Call Rate Error Rate │`)
|
|
|
|
|
|
|
|
|
|
const totalModelTools = Object.values(usage.toolUsage).reduce((sum, t) => sum + t.calls, 0)
|
|
|
|
|
const sortedTools = Object.entries(usage.toolUsage).sort((a, b) => b[1].calls - a[1].calls)
|
|
|
|
|
|
|
|
|
|
for (const [tool, toolStats] of sortedTools) {
|
|
|
|
|
const callRate = ((toolStats.calls / totalModelTools) * 100).toFixed(1) + "%"
|
|
|
|
|
const errorRate = toolStats.calls > 0 ? ((toolStats.errors / toolStats.calls) * 100).toFixed(1) + "%" : "0%"
|
|
|
|
|
|
|
|
|
|
const toolName = tool.length > 22 ? tool.substring(0, 20) + ".." : tool
|
|
|
|
|
const paddedTool = toolName.padEnd(24)
|
|
|
|
|
const callStr = callRate.padStart(13)
|
|
|
|
|
const errStr = errorRate.padStart(15)
|
|
|
|
|
|
|
|
|
|
console.log(`│ ${paddedTool}${callStr}${errStr} │`)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
console.log("├────────────────────────────────────────────────────────┤")
|
|
|
|
|
}
|
|
|
|
|
// Remove last separator and add bottom border
|
|
|
|
|
process.stdout.write("\x1B[1A") // Move up one line
|
|
|
|
|
console.log("└────────────────────────────────────────────────────────┘")
|
|
|
|
|
}
|
|
|
|
|
// Remove last separator and add bottom border
|
|
|
|
|
process.stdout.write("\x1B[1A") // Move up one line
|
|
|
|
|
console.log("└────────────────────────────────────────────────────────┘")
|
|
|
|
|
}
|
|
|
|
|
console.log()
|
|
|
|
|
|
|
|
|
|
// Tool Usage section
|
|
|
|
|
if (Object.keys(stats.toolUsage).length > 0) {
|
|
|
|
|
const sortedTools = Object.entries(stats.toolUsage).sort(([, a], [, b]) => b.calls - a.calls)
|
|
|
|
|
const sortedTools = Object.entries(stats.toolUsage).sort(([, a], [, b]) => b - a)
|
|
|
|
|
const toolsToDisplay = toolLimit ? sortedTools.slice(0, toolLimit) : sortedTools
|
|
|
|
|
|
|
|
|
|
console.log("┌────────────────────────────────────────────────────────┐")
|
|
|
|
|
console.log("│ TOOL USAGE │")
|
|
|
|
|
console.log("├────────────────────────────────────────────────────────┤")
|
|
|
|
|
|
|
|
|
|
const maxCount = Math.max(...toolsToDisplay.map(([, toolStats]) => toolStats.calls))
|
|
|
|
|
const totalToolUsage = Object.values(stats.toolUsage).reduce((a, b) => a + b.calls, 0)
|
|
|
|
|
const maxCount = Math.max(...toolsToDisplay.map(([, count]) => count))
|
|
|
|
|
const totalToolUsage = Object.values(stats.toolUsage).reduce((a, b) => a + b, 0)
|
|
|
|
|
|
|
|
|
|
for (const [tool, toolStats] of toolsToDisplay) {
|
|
|
|
|
const count = toolStats.calls
|
|
|
|
|
for (const [tool, count] of toolsToDisplay) {
|
|
|
|
|
const barLength = Math.max(1, Math.floor((count / maxCount) * 20))
|
|
|
|
|
const bar = "█".repeat(barLength)
|
|
|
|
|
const percentage = ((count / totalToolUsage) * 100).toFixed(1)
|
|
|
|
|
|