From b8ebdd575cf1dd34efa07373eccd764bff22cc65 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Mon, 4 May 2026 07:59:15 +0800 Subject: [PATCH] add server usage stats script (#12557) * enhance(db-sync): add usage stats script * fix(db-sync): use --local for local D1 scripts * fix(db-sync): record activity only on successful sync requests * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- deps/db-sync/README.md | 13 ++ deps/db-sync/package.json | 1 + deps/db-sync/src/logseq/db_sync/index.cljs | 104 +++++++++- .../src/logseq/db_sync/worker/dispatch.cljs | 55 ++++- .../logseq/db_sync/worker/handler/index.cljs | 54 ++++- .../migrations/0005_add_user_created_at.sql | 2 + .../0006_add_daily_active_entities.sql | 11 + deps/db-sync/worker/scripts/graph_user_lib.js | 21 +- .../worker/scripts/show_usage_stats.js | 188 ++++++++++++++++++ 9 files changed, 421 insertions(+), 28 deletions(-) create mode 100644 deps/db-sync/worker/migrations/0005_add_user_created_at.sql create mode 100644 deps/db-sync/worker/migrations/0006_add_daily_active_entities.sql create mode 100644 deps/db-sync/worker/scripts/show_usage_stats.js diff --git a/deps/db-sync/README.md b/deps/db-sync/README.md index fe58610957..4ebb59bcde 100644 --- a/deps/db-sync/README.md +++ b/deps/db-sync/README.md @@ -46,6 +46,19 @@ pnpm show-graphs-for-user --user-id us-east-1:example-user-id The script uses `worker/wrangler.toml`, runs against the remote D1 binding `DB`, defaults to `--env prod`, and prints JSON when `--json` is added. +Show usage totals (total users, total graphs, users created today, graphs created today): + +```bash +cd deps/db-sync +pnpm show-usage-stats +pnpm show-usage-stats --days 7 +pnpm show-usage-stats --json +``` + +`created today` uses UTC day boundaries from D1 (`date('now')`). +`active_*_last_n_days` uses deduplicated UTC-day activity rows from +`daily_active_entities` with the provided `--days` window. + Download a graph snapshot into a local sqlite debug file matching local graph DB schema (`kvs` table only): ```bash diff --git a/deps/db-sync/package.json b/deps/db-sync/package.json index 73fd874f59..1b83a638e3 100644 --- a/deps/db-sync/package.json +++ b/deps/db-sync/package.json @@ -11,6 +11,7 @@ "delete-user-totally": "node worker/scripts/delete_user_totally.js", "download-graph-db": "node worker/scripts/download_graph_db.js", "show-graphs-for-user": "node worker/scripts/show_graphs_for_user.js", + "show-usage-stats": "node worker/scripts/show_usage_stats.js", "build:node-adapter": "clojure -M:cljs release db-sync-node", "dev:node-adapter": "clojure -M:cljs watch db-sync-node", "start:node-adapter": "node worker/dist/node-adapter.js", diff --git a/deps/db-sync/src/logseq/db_sync/index.cljs b/deps/db-sync/src/logseq/db_sync/index.cljs index 21d342e3b7..06d1965726 100644 --- a/deps/db-sync/src/logseq/db_sync/index.cljs +++ b/deps/db-sync/src/logseq/db_sync/index.cljs @@ -6,6 +6,8 @@ (def ^:private user-upsert-cache-ttl-ms (* 60 60 1000)) (def ^:private user-upsert-cache-max 1024) (defonce ^:private *user-upsert-cache (atom {})) +(def ^:private activity-touch-cache-max 8192) +(defonce ^:private *activity-touch-cache (atom {})) (defn- prune-user-upsert-cache! [now-ms] (swap! *user-upsert-cache @@ -56,6 +58,19 @@ "alter table graphs add column graph_e2ee INTEGER DEFAULT 1") (def ^:private graph-ready-for-use-migration-sql "alter table graphs add column graph_ready_for_use integer default 1") +(def ^:private user-created-at-migration-sql + "alter table users add column created_at integer") +(def ^:private daily-active-entities-create-table-sql + (str "create table if not exists daily_active_entities (" + "day_utc TEXT," + "entity_type TEXT," + "entity_id TEXT," + "first_seen_at INTEGER," + "primary key (day_utc, entity_type, entity_id)," + "check (entity_type in ('user', 'graph'))" + ");")) +(def ^:private daily-active-entities-create-index-sql + "create index if not exists idx_daily_active_entities_type_day on daily_active_entities (entity_type, day_utc)") (defn- duplicate-column-error? [error column-name] @@ -96,6 +111,22 @@ (p/catch (fn [_] ( (common/ (p/let [result (common/ (.toISOString (js/Date. timestamp-ms)) + (subs 0 10))) + +(defn- prune-activity-touch-cache! + [cache] + (if (<= (count cache) activity-touch-cache-max) + cache + (let [drop-count (- (count cache) activity-touch-cache-max)] + (->> cache + (sort-by val) + (drop drop-count) + (into {}))))) + +(defn- cache-activity-touch! + [cache-key now-ms] + (swap! *activity-touch-cache + (fn [cache] + (-> cache + (assoc cache-key now-ms) + (prune-activity-touch-cache!))))) + +(defn- activity-touch-cached? + [cache-key] + (contains? @*activity-touch-cache cache-key)) + +(defn- claims (aget "sub"))] + (when (string? user-id) + user-id))) + +(defn- (p/let [_ (index/ (p/do @@ -75,14 +111,17 @@ (subs rest-path slash-idx)) new-url (js/URL. (str (.-origin url) tail (.-search url)))] (if (seq graph-id) - (if (= method "OPTIONS") - (common/options-response) - (if (admin-token-valid? request env) - (forward-sync-request request env graph-id new-url) - (p/let [access-resp (index-handler/graph-access-response request env graph-id)] - (if (.-ok access-resp) - (forward-sync-request request env graph-id new-url) - access-resp)))) + (if (= method "OPTIONS") + (common/options-response) + (if (admin-token-valid? request env) + (forward-sync-request request env graph-id new-url) + (p/let [access-resp (index-handler/graph-access-response request env graph-id)] + (if (.-ok access-resp) + (p/let [response (forward-sync-request request env graph-id new-url) + _ (when (< (.-status response) 400) + ( (index/ (index/ route :path-params :graph-id) + _ (when (and (string? user-id) + (string? graph-id) + (< (.-status response) 400)) + ( Wrangler environment to use. Defaults to "prod"; use "local" to query the local D1 database. + --days Active window size in days. Defaults to 1. + --database D1 binding or database name. Defaults to "DB". + --config Wrangler config path. Defaults to worker/wrangler.toml. + --json Print JSON instead of a table. + --help Show this message. + +Output: + days + active_since_utc + active_users_last_n_days + active_graphs_last_n_days + total_users + total_graphs + users_created_today + graphs_created_today + today_utc +`); +} + +function parseDays(value) { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 1) { + fail("--days must be an integer >= 1."); + } + return parsed; +} + +function parseCliArgs(argv) { + const { values } = parseArgs({ + args: argv, + options: { + env: { type: "string", default: "prod" }, + days: { type: "string", default: "1" }, + database: { type: "string", default: "DB" }, + config: { type: "string", default: defaultConfigPath }, + json: { type: "boolean", default: false }, + help: { type: "boolean", default: false }, + }, + strict: true, + allowPositionals: false, + }); + + if (values.help) { + printHelp(); + process.exit(0); + } + + return { + env: values.env, + days: parseDays(values.days), + database: values.database, + config: path.resolve(values.config), + json: values.json, + }; +} + +function buildUsageStatsSql(days) { + const sinceDays = days - 1; + return `with bounds as ( + select + cast(strftime('%s', 'now', 'start of day') as integer) * 1000 as today_start_ms, + cast(strftime('%s', 'now', 'start of day', '+1 day') as integer) * 1000 as tomorrow_start_ms, + date('now') as today_utc, + date('now', '-${sinceDays} days') as active_since_utc +) +select + ${days} as days, + (select active_since_utc from bounds) as active_since_utc, + (select count(distinct entity_id) + from daily_active_entities a + join bounds b on 1 = 1 + where a.entity_type = 'user' + and a.day_utc >= b.active_since_utc + and a.day_utc <= b.today_utc) as active_users_last_n_days, + (select count(distinct entity_id) + from daily_active_entities a + join bounds b on 1 = 1 + where a.entity_type = 'graph' + and a.day_utc >= b.active_since_utc + and a.day_utc <= b.today_utc) as active_graphs_last_n_days, + (select count(1) from users) as total_users, + (select count(1) from graphs) as total_graphs, + (select count(1) + from users u + join bounds b on 1 = 1 + where u.created_at is not null + and u.created_at >= b.today_start_ms + and u.created_at < b.tomorrow_start_ms) as users_created_today, + (select count(1) + from graphs g + join bounds b on 1 = 1 + where g.created_at is not null + and g.created_at >= b.today_start_ms + and g.created_at < b.tomorrow_start_ms) as graphs_created_today, + (select today_utc from bounds) as today_utc;`; +} + +function sqlCountToNumber(value) { + const numeric = Number(value); + return Number.isFinite(numeric) ? numeric : 0; +} + +function formatUsageStats(rows) { + if (!rows.length) { + return null; + } + + const [row] = rows; + return { + days: sqlCountToNumber(row.days), + active_since_utc: typeof row.active_since_utc === "string" ? row.active_since_utc : null, + active_users_last_n_days: sqlCountToNumber(row.active_users_last_n_days), + active_graphs_last_n_days: sqlCountToNumber(row.active_graphs_last_n_days), + total_users: sqlCountToNumber(row.total_users), + total_graphs: sqlCountToNumber(row.total_graphs), + users_created_today: sqlCountToNumber(row.users_created_today), + graphs_created_today: sqlCountToNumber(row.graphs_created_today), + today_utc: typeof row.today_utc === "string" ? row.today_utc : null, + }; +} + +function printUsageStatsTable(stats) { + console.table([stats]); +} + +function main() { + const options = parseCliArgs(process.argv.slice(2)); + const sql = buildUsageStatsSql(options.days); + const wranglerArgs = buildWranglerArgs({ + database: options.database, + config: options.config, + env: options.env, + sql, + }); + const rows = parseWranglerResults(runWranglerQuery(wranglerArgs)); + const stats = formatUsageStats(rows); + + if (!stats) { + fail("No stats returned from D1."); + } + + if (options.json) { + console.log(JSON.stringify(stats, null, 2)); + } else { + printUsageStatsTable(stats); + } +} + +if (require.main === module) { + try { + main(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (/no such table: daily_active_entities/i.test(message)) { + fail("Missing daily_active_entities. Apply worker migration 0006_add_daily_active_entities.sql."); + } + if (/no such column: created_at/i.test(message)) { + fail("Missing users.created_at. Apply worker migration 0005_add_user_created_at.sql."); + } + fail(message); + } +} + +module.exports = { + buildUsageStatsSql, + formatUsageStats, + parseCliArgs, +};