From 7b2ef7fce20709fbe485089781d2c1db08f4a08e Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Thu, 5 Mar 2026 08:40:49 -0500 Subject: [PATCH] milestone 1+2 --- .../db-worker-browser-api-inventory.md | 27 + .../task--db-worker-nodejs-compatible.md | 36 +- scripts/src/logseq/tasks/dev.clj | 5 +- src/main/frontend/worker/db_core.cljs | 894 +++++++++++++++ src/main/frontend/worker/db_worker.cljs | 1013 +---------------- src/main/frontend/worker/platform.cljs | 74 ++ .../frontend/worker/platform/browser.cljs | 111 ++ 7 files changed, 1147 insertions(+), 1013 deletions(-) create mode 100644 docs/agent-guide/db-worker-browser-api-inventory.md create mode 100644 src/main/frontend/worker/db_core.cljs create mode 100644 src/main/frontend/worker/platform.cljs create mode 100644 src/main/frontend/worker/platform/browser.cljs diff --git a/docs/agent-guide/db-worker-browser-api-inventory.md b/docs/agent-guide/db-worker-browser-api-inventory.md new file mode 100644 index 0000000000..2a92e844a8 --- /dev/null +++ b/docs/agent-guide/db-worker-browser-api-inventory.md @@ -0,0 +1,27 @@ +# db-worker browser-only API inventory + +## Worker entry +- frontend.worker.db-worker + - importScripts + - js/self postMessage + - js/location.href + - navigator.storage (OPFS) + - Comlink expose/wrap/transfer + - setInterval + +## Shared service +- frontend.worker.shared-service + - navigator.locks.request/query + - BroadcastChannel + - js/self postMessage + +## RTC and crypto +- frontend.worker.rtc.ws + - WebSocket +- frontend.worker.rtc.crypt + - OPFS file access via frontend.common.file.opfs + - js/self location + +## Worker util +- frontend.worker-common.util + - wfu/post-message (worker postMessage bridge) diff --git a/docs/agent-guide/task--db-worker-nodejs-compatible.md b/docs/agent-guide/task--db-worker-nodejs-compatible.md index e746fcfe91..ebfb427e44 100644 --- a/docs/agent-guide/task--db-worker-nodejs-compatible.md +++ b/docs/agent-guide/task--db-worker-nodejs-compatible.md @@ -48,15 +48,25 @@ Make `frontend.worker.db-worker` and its dependencies run in both browser and No ## Refactor Steps (Milestones + Status) ### Milestone 1: Architecture & Abstractions -- TODO 1. Inventory db-worker dependencies and classify browser-only APIs. -- TODO 2. Define a platform adapter interface (storage, kv, broadcast, websocket, crypto, timers, env flags). -- TODO 3. Extract db-worker core logic into a platform-agnostic module (e.g. `frontend.worker.db-core`). +- DONE 1. Inventory db-worker dependencies and classify browser-only APIs. +- DONE 2. Define a platform adapter interface (storage, kv, broadcast, websocket, crypto, timers, env flags). +- DONE 3. Extract db-worker core logic into a platform-agnostic module (e.g. `frontend.worker.db-core`). +#### Acceptance Criteria +- Core worker module has zero direct references to `js/self`, `js/location`, `js/navigator`, `importScripts`, `BroadcastChannel`, or `navigator.locks`. +- `frontend.worker.platform` exists with required sections and validation; platform adapter passes validation at init time. +- Browser worker entry initializes via `init!`/`init-core!` with a platform adapter. +- `bb dev:lint-and-test` passes. ### Milestone 2: Browser Path Parity -- TODO 4. Implement `frontend.worker.platform.browser`. -- TODO 5. Update db-worker entry to inject the platform adapter and call core init. -- TODO 6. Route OPFS/IDB usage through the platform adapter in worker submodules. -- TODO 7. Replace direct `js/WebSocket` usage with platform websocket factory. +- DONE 4. Implement `frontend.worker.platform.browser`. +- DONE 5. Update db-worker entry to inject the platform adapter and call core init. +- DONE 6. Route OPFS/IDB usage through the platform adapter in worker submodules. +- DONE 7. Replace direct `js/WebSocket` usage with platform websocket factory. +#### Acceptance Criteria +- Browser platform adapter encapsulates OPFS/IDB storage, kv-store, and WebSocket factory; worker submodules no longer import OPFS/IDB directly. +- RTC WebSocket creation uses the platform adapter. +- Browser db-worker entry injects the platform adapter and serves Comlink RPC. +- `bb dev:lint-and-test` passes. ### Milestone 3: Node Path & Daemon - TODO 8. Implement `frontend.worker.platform.node` in single-client mode (no locks or BroadcastChannel). @@ -64,10 +74,22 @@ Make `frontend.worker.db-worker` and its dependencies run in both browser and No - TODO 10. Add Node build target in `shadow-cljs.edn` for db-worker. - TODO 11. Implement Node daemon entrypoint and HTTP server. - TODO 12. Add a Node client in frontend to call the daemon (HTTP + SSE/WS events). +#### Acceptance Criteria +- Node platform adapter provides storage/kv/broadcast/websocket/crypto/timers and validates via `frontend.worker.platform`. +- Node build target compiles db-worker core without browser-only APIs. +- Node daemon starts via CLI and reports readiness; `GET /healthz` and `GET /readyz` return `200 OK`. +- `POST /v1/invoke` handles `list-db`, `create-or-open-db`, `q`, `transact` in a smoke test. +- Node client can invoke at least one RPC and receive one event (SSE or WS). +- `bb dev:lint-and-test` passes. ### Milestone 4: Validation - TODO 13. Add tests: adapter unit tests + daemon integration smoke test. - TODO 14. Verify browser worker path still works with Comlink. +#### Acceptance Criteria +- Adapter unit tests cover browser and node implementations for storage/kv/broadcast/websocket factories. +- Daemon integration smoke test starts the node process and exercises `/v1/invoke` with at least one method. +- Browser worker path verified with Comlink RPCs (smoke test). +- `bb dev:lint-and-test` passes. ## Node.js Daemon Requirements The db-worker should be runnable as a standalone process for Node.js environments. diff --git a/scripts/src/logseq/tasks/dev.clj b/scripts/src/logseq/tasks/dev.clj index 0e3f70a913..331dae4f34 100644 --- a/scripts/src/logseq/tasks/dev.clj +++ b/scripts/src/logseq/tasks/dev.clj @@ -18,7 +18,8 @@ "Run tests. Pass args through to cmd 'yarn cljs:run-test'" [& args] (shell "yarn cljs:test") - (apply shell "yarn cljs:run-test" args)) + (let [args* (or (seq args) ["-e" "long" "-e" "fix-me"])] + (apply shell "yarn cljs:run-test" args*))) (defn test-no-worker "Run tests without compiling worker namespaces. Pass args through to cmd 'yarn cljs:run-test-no-worker'" @@ -31,7 +32,7 @@ pass args through to cmd 'yarn cljs:run-test'" [] (dev-lint/dev) - (test "-e" "long" "-e" "fix-me")) + (test)) (defn e2e-basic-test "Run e2e basic tests. HTTP server should be available at localhost:3001" diff --git a/src/main/frontend/worker/db_core.cljs b/src/main/frontend/worker/db_core.cljs new file mode 100644 index 0000000000..16b169eaab --- /dev/null +++ b/src/main/frontend/worker/db_core.cljs @@ -0,0 +1,894 @@ +(ns frontend.worker.db-core + "Core db-worker logic without host-specific bootstrap." + (:require ["@sqlite.org/sqlite-wasm" :default sqlite3InitModule] + [cljs-bean.core :as bean] + [cljs.cache :as cache] + [clojure.edn :as edn] + [clojure.set] + [clojure.string :as string] + [datascript.core :as d] + [datascript.storage :refer [IStorage] :as storage] + [frontend.common.cache :as common.cache] + [frontend.common.graph-view :as graph-view] + [frontend.common.missionary :as c.m] + [frontend.common.thread-api :as thread-api :refer [def-thread-api]] + [frontend.worker.platform :as platform] + [frontend.worker-common.util :as worker-util] + [frontend.worker.db-listener :as db-listener] + [frontend.worker.db-metadata :as worker-db-metadata] + [frontend.worker.db.fix :as db-fix] + [frontend.worker.db.migrate :as db-migrate] + [frontend.worker.db.validate :as worker-db-validate] + [frontend.worker.embedding :as embedding] + [frontend.worker.export :as worker-export] + [frontend.worker.handler.page :as worker-page] + [frontend.worker.pipeline :as worker-pipeline] + [frontend.worker.publish] + [frontend.worker.rtc.asset-db-listener] + [frontend.worker.rtc.client-op :as client-op] + [frontend.worker.rtc.core :as rtc.core] + [frontend.worker.rtc.db-listener] + [frontend.worker.rtc.debug-log :as rtc-debug-log] + [frontend.worker.rtc.migrate :as rtc-migrate] + [frontend.worker.search :as search] + [frontend.worker.shared-service :as shared-service] + [frontend.worker.state :as worker-state] + [frontend.worker.thread-atom] + [lambdaisland.glogi :as log] + [logseq.cli.common.mcp.tools :as cli-common-mcp-tools] + [logseq.common.util :as common-util] + [logseq.db :as ldb] + [logseq.db.common.entity-plus :as entity-plus] + [logseq.db.common.entity-util :as common-entity-util] + [logseq.db.common.initial-data :as common-initial-data] + [logseq.db.common.order :as db-order] + [logseq.db.common.reference :as db-reference] + [logseq.db.common.sqlite :as common-sqlite] + [logseq.db.common.view :as db-view] + [logseq.db.frontend.class :as db-class] + [logseq.db.frontend.property :as db-property] + [logseq.db.sqlite.create-graph :as sqlite-create-graph] + [logseq.db.sqlite.export :as sqlite-export] + [logseq.db.sqlite.gc :as sqlite-gc] + [logseq.db.sqlite.util :as sqlite-util] + [logseq.outliner.core :as outliner-core] + [logseq.outliner.op :as outliner-op] + [me.tonsky.persistent-sorted-set :as set :refer [BTSet]] + [missionary.core :as m] + [promesa.core :as p])) + +(defonce *sqlite worker-state/*sqlite) +(defonce *sqlite-conns worker-state/*sqlite-conns) +(defonce *datascript-conns worker-state/*datascript-conns) +(defonce *client-ops-conns worker-state/*client-ops-conns) +(defonce *opfs-pools worker-state/*opfs-pools) +(defonce *publishing? (atom false)) + +(defn- js {:print #(log/info :init-sqlite-module! %) + :printErr #(log/error :init-sqlite-module! %)}))] + (reset! *publishing? publishing?) + (reset! *sqlite sqlite) + nil))) + +(def repo-path "/db.sqlite") +(def debug-log-path "/debug-log/db.sqlite") + +(defn- (.exec db #js {:sql "select content, addresses from kvs where addr = ?" + :bind #js [addr] + :rowMode "array"}) + first)] + (let [[content addresses] (bean/->clj result) + addresses (when addresses + (js/JSON.parse addresses)) + data (sqlite-util/transit-read content)] + (if (and addresses (map? data)) + (assoc data :addresses addresses) + data)))) + +(defn new-sqlite-storage + "Update sqlite-cli/new-sqlite-storage when making changes" + [^Object db] + (reify IStorage + (-store [_ addr+data-seq _delete-addrs] + (let [data (map + (fn [[addr data]] + (let [data' (if (map? data) (dissoc data :addresses) data) + addresses (when (map? data) + (when-let [addresses (:addresses data)] + (js/JSON.stringify (bean/->js addresses))))] + #js {:$addr addr + :$content (sqlite-util/transit-write data') + :$addresses addresses})) + addr+data-seq)] + (upsert-addr-content! db data))) + + (-restore [_ addr] + (restore-data-from-addr db addr)))) + +(defn- close-db-aux! + [repo ^Object db ^Object search ^Object client-ops ^Object debug-log] + (swap! *sqlite-conns dissoc repo) + (swap! *datascript-conns dissoc repo) + (swap! *client-ops-conns dissoc repo) + (when db (.close db)) + (when search (.close search)) + (when client-ops (.close client-ops)) + (when debug-log (.close debug-log)) + (when-let [^js pool (worker-state/get-opfs-pool repo)] + (.pauseVfs pool)) + (swap! *opfs-pools dissoc repo)) + +(defn- close-other-dbs! + [repo] + (doseq [[r {:keys [db search client-ops debug-log]}] @*sqlite-conns] + (when-not (= repo r) + (close-db-aux! r db search client-ops debug-log)))) + +(defn close-db! + [repo] + (let [{:keys [db search client-ops debug-log]} (get @*sqlite-conns repo)] + (close-db-aux! repo db search client-ops debug-log))) + +(defn reset-db! + [repo db-transit-str] + (when-let [conn (get @*datascript-conns repo)] + (let [new-db (ldb/read-transit-str db-transit-str) + new-db' (update new-db :eavt (fn [^BTSet s] + (set! (.-storage s) (.-storage (:eavt @conn))) + s))] + (d/reset-conn! conn new-db' {:reset-conn! true}) + (d/reset-schema! conn (:schema new-db))))) + +(defn- get-dbs + [repo] + (if @*publishing? + (p/let [^object DB (.-DB ^object (.-oo1 ^object @*sqlite)) + db (new DB "/db.sqlite" "c") + search-db (new DB "/search-db.sqlite" "c")] + [db search-db]) + (p/let [^js pool ( (- (common-util/time-ms) last-gc-at) (* 3 24 3600 1000))) ; 3 days ago + (log/info :gc-sqlite-dbs "gc current graph") + (doseq [db (if @*publishing? [sqlite-db] [sqlite-db client-ops-db])] + (sqlite-gc/gc-kvs-table! db {:full-gc? full-gc?}) + (.exec db "VACUUM")) + (rtc-debug-log/gc! debug-log-db) + (ldb/transact! datascript-conn [{:db/ident :logseq.kv/graph-last-gc-at + :kv/value (common-util/time-ms)}])))) + +(defn- datoms (group-by :e datoms) + {properties true non-properties false} (group-by + (fn [[_eid datoms]] + (boolean + (some (fn [datom] (and (= (:a datom) :db/ident) + (db-property/property? (:v datom)))) + datoms))) + eid->datoms) + datoms (concat (mapcat second properties) + (mapcat second non-properties)) + data (map (fn [datom] + [:db/add (:e datom) (:a datom) (:v datom)]) + datoms)] + (d/transact! conn data {:initial-db? true}))) + client-ops-conn (when-not @*publishing? (common-sqlite/get-storage-conn + client-ops-storage + client-op/schema-in-db)) + initial-data-exists? (when (nil? datoms) + (and (d/entity @conn :logseq.class/Root) + (= "db" (:kv/value (d/entity @conn :logseq.kv/db-type)))))] + (swap! *datascript-conns assoc repo conn) + (swap! *client-ops-conns assoc repo client-ops-conn) + (when (and (not @*publishing?) (not= client-op/schema-in-db (d/schema @client-ops-conn))) + (d/reset-schema! client-ops-conn client-op/schema-in-db)) + (when (and db-based? (not initial-data-exists?) (not datoms)) + (let [config (or config "") + initial-data (sqlite-create-graph/build-db-initial-data + config (select-keys opts [:import-type :graph-git-sha]))] + (ldb/transact! conn initial-data {:initial-db? true}))) + + (gc-sqlite-dbs! db client-ops-db debug-log-db conn {}) + + (let [migration-result (db-migrate/migrate conn)] + (when (client-op/rtc-db-graph? repo) + (let [client-ops (rtc-migrate/migration-results=>client-ops migration-result)] + (client-op/add-ops! repo client-ops)))) + + (db-listener/listen-db-changes! repo (get @*datascript-conns repo)))))) + +(defn- db-id {})) + (start-db! repo opts)) + +(def-thread-api :thread-api/q + [repo inputs] + (when-let [conn (worker-state/get-datascript-conn repo)] + (worker-util/profile + (str "Datalog query: " inputs) + (apply d/q (first inputs) @conn (rest inputs))))) + +(def-thread-api :thread-api/datoms + [repo & args] + (when-let [conn (worker-state/get-datascript-conn repo)] + (let [result (apply d/datoms @conn args)] + (map (fn [d] [(:e d) (:a d) (:v d) (:tx d) (:added d)]) result)))) + +(def-thread-api :thread-api/pull + [repo selector id] + (when-let [conn (worker-state/get-datascript-conn repo)] + (let [eid (if (and (vector? id) (= :block/name (first id))) + (:db/id (ldb/get-page @conn (second id))) + id)] + (some->> eid + (d/pull @conn selector) + (common-initial-data/with-parent @conn))))) + +(def ^:private *get-blocks-cache (volatile! (cache/lru-cache-factory {} :threshold 1000))) +(def ^:private get-blocks-with-cache + (common.cache/cache-fn + *get-blocks-cache + (fn [repo requests] + (let [db (some-> (worker-state/get-datascript-conn repo) deref)] + [[repo (:max-tx db) requests] + [db requests]])) + (fn [db requests] + (when db + (->> requests + (mapv (fn [{:keys [id opts]}] + (let [id' (if (and (string? id) (common-util/uuid-string? id)) (uuid id) id)] + (-> (common-initial-data/get-block-and-children db id' opts) + (assoc :id id))))) + ldb/write-transit-str))))) + +(def-thread-api :thread-api/get-blocks + [repo requests] + (let [requests (ldb/read-transit-str requests)] + (get-blocks-with-cache repo requests))) + +(def-thread-api :thread-api/get-block-refs + [repo id] + (when-let [conn (worker-state/get-datascript-conn repo)] + (->> (db-reference/get-linked-references @conn id) + :ref-blocks + (map (fn [b] (assoc (into {} b) :db/id (:db/id b))))))) + +(def-thread-api :thread-api/get-block-refs-count + [repo id] + (when-let [conn (worker-state/get-datascript-conn repo)] + (ldb/get-block-refs-count @conn id))) + +(def-thread-api :thread-api/get-block-source + [repo id] + (when-let [conn (worker-state/get-datascript-conn repo)] + (:db/id (first (:block/_alias (d/entity @conn id)))))) + +(defn- search-blocks + [repo q option] + (let [search-db (get-search-db repo) + conn (worker-state/get-datascript-conn repo)] + (search/search-blocks repo conn search-db q option))) + +(def-thread-api :thread-api/block-refs-check + [repo id {:keys [unlinked?]}] + (m/sp + (when-let [conn (worker-state/get-datascript-conn repo)] + (let [db @conn + block (d/entity db id)] + (if unlinked? + (let [title (string/lower-case (:block/title block)) + result (m/? (search-blocks repo title {:limit 100}))] + (boolean (some (fn [b] + (let [block (d/entity db (:db/id b))] + (and (not= id (:db/id block)) + (not ((set (map :db/id (:block/refs block))) id)) + (string/includes? (string/lower-case (:block/title block)) title)))) result))) + (some? (first (common-initial-data/get-block-refs db (:db/id block))))))))) + +(def-thread-api :thread-api/get-block-parents + [repo id depth] + (when-let [conn (worker-state/get-datascript-conn repo)] + (let [block-id (:block/uuid (d/entity @conn id))] + (->> (ldb/get-block-parents @conn block-id {:depth (or depth 3)}) + (map (fn [b] (d/pull @conn '[*] (:db/id b)))))))) + +(def-thread-api :thread-api/set-context + [context] + (when context (worker-state/update-context! context)) + nil) + +(def-thread-api :thread-api/transact + [repo tx-data tx-meta context] + (assert (some? repo)) + (worker-state/set-db-latest-tx-time! repo) + (let [conn (worker-state/get-datascript-conn repo)] + (assert (some? conn) {:repo repo}) + (try + (let [tx-data' (if (contains? #{:insert-blocks} (:outliner-op tx-meta)) + (map (fn [m] + (if (and (map? m) (nil? (:block/order m))) + (assoc m :block/order (db-order/gen-key nil)) + m)) tx-data) + tx-data) + _ (when context (worker-state/set-context! context)) + tx-meta' (cond-> tx-meta + (and (not (:whiteboard/transact? tx-meta)) + (not (:rtc-download-graph? tx-meta))) ; delay writes to the disk + (assoc :skip-store? true) + + true + (dissoc :insert-blocks?))] + (when-not (and (:create-today-journal? tx-meta) + (:today-journal-name tx-meta) + (seq tx-data') + (ldb/get-page @conn (:today-journal-name tx-meta))) ; today journal created already + + ;; (prn :debug :transact :tx-data tx-data' :tx-meta tx-meta') + + (worker-util/profile "Worker db transact" + (ldb/transact! conn tx-data' tx-meta'))) + nil) + (catch :default e + (prn :debug :worker-transact-failed :tx-meta tx-meta :tx-data tx-data) + (log/error ::worker-transact-failed e) + (throw e))))) + +(def-thread-api :thread-api/get-initial-data + [repo opts] + (when-let [conn (worker-state/get-datascript-conn repo)] + (if (:file-graph-import? opts) + {:schema (:schema @conn) + :initial-data (vec (d/datoms @conn :eavt))} + (common-initial-data/get-initial-data @conn)))) + +(def-thread-api :thread-api/reset-db + [repo db-transit] + (reset-db! repo db-transit) + nil) + +(def-thread-api :thread-api/unsafe-unlink-db + [repo] + (p/let [pool ( (p/let [data (js blocks)) + nil)) + +(def-thread-api :thread-api/search-delete-blocks + [repo ids] + (p/let [db (get-search-db repo)] + (search/delete-blocks! db ids) + nil)) + +(def-thread-api :thread-api/search-truncate-tables + [repo] + (p/let [db (get-search-db repo)] + (search/truncate-table! db) + nil)) + +(def-thread-api :thread-api/search-build-blocks-indice + [repo] + (when-let [conn (worker-state/get-datascript-conn repo)] + (search/build-blocks-indice repo @conn))) + +(def-thread-api :thread-api/search-build-pages-indice + [_repo] + nil) + +(def-thread-api :thread-api/apply-outliner-ops + [repo ops opts] + (when-let [conn (worker-state/get-datascript-conn repo)] + (try + (worker-util/profile + "apply outliner ops" + (outliner-op/apply-ops! conn ops opts)) + (catch :default e + (let [data (ex-data e) + {:keys [type payload]} (when (map? data) data)] + (case type + :notification + (do + (log/error ::apply-outliner-ops-failed e) + (shared-service/broadcast-to-clients! :notification [(:message payload) (:type payload) (:clear? payload) (:uid payload) (:timeout payload)])) + (throw e))))))) + +(def-thread-api :thread-api/sync-app-state + [new-state] + (when (and (contains? new-state :git/current-repo) + (nil? (:git/current-repo new-state))) + (log/error :thread-api/sync-app-state new-state)) + (worker-state/set-new-state! new-state) + nil) + +(def-thread-api :thread-api/export-get-debug-datoms + [repo] + (when-let [conn (worker-state/get-datascript-conn repo)] + (worker-export/get-debug-datoms conn))) + +(def-thread-api :thread-api/export-get-all-pages + [repo] + (when-let [conn (worker-state/get-datascript-conn repo)] + (worker-export/get-all-pages repo @conn))) + +(def-thread-api :thread-api/export-get-all-page->content + [repo options] + (when-let [conn (worker-state/get-datascript-conn repo)] + (worker-export/get-all-page->content repo @conn options))) + +(def-thread-api :thread-api/validate-db + [repo] + (when-let [conn (worker-state/get-datascript-conn repo)] + (worker-db-validate/validate-db conn))) + +;; Returns an export-edn map for given repo. When there's an unexpected error, a map +;; with key :export-edn-error is returned +(def-thread-api :thread-api/export-edn + [repo options] + (let [conn (worker-state/get-datascript-conn repo)] + (try + (sqlite-export/build-export @conn options) + (catch :default e + (js/console.error "export-edn error: " e) + (js/console.error "Stack:\n" (.-stack e)) + (platform/post-message! (platform/current) + :notification + ["An unexpected error occurred during export. See the javascript console for details." + :error]) + {:export-edn-error (.-message e)})))) + +(def-thread-api :thread-api/get-view-data + [repo view-id option] + (let [db @(worker-state/get-datascript-conn repo)] + (db-view/get-view-data db view-id option))) + +(def-thread-api :thread-api/get-class-objects + [repo class-id] + (let [db @(worker-state/get-datascript-conn repo)] + (->> (db-class/get-class-objects db class-id) + (map common-entity-util/entity->map)))) + +(def-thread-api :thread-api/get-property-values + [repo {:keys [property-ident] :as option}] + (let [conn (worker-state/get-datascript-conn repo)] + (db-view/get-property-values @conn property-ident option))) + +(def-thread-api :thread-api/build-graph + [repo option] + (let [conn (worker-state/get-datascript-conn repo)] + (graph-view/build-graph @conn option))) + +(def ^:private *get-all-page-titles-cache (volatile! (cache/lru-cache-factory {}))) +(defn- get-all-page-titles + [db] + (let [pages (ldb/get-all-pages db)] + (sort (map :block/title pages)))) + +(def ^:private get-all-page-titles-with-cache + (common.cache/cache-fn + *get-all-page-titles-cache + (fn [repo] + (let [db @(worker-state/get-datascript-conn repo)] + [[repo (:max-tx db)] ;cache-key + [db] ;f-args + ])) + get-all-page-titles)) + +(def-thread-api :thread-api/get-all-page-titles + [repo] + (get-all-page-titles-with-cache repo)) + +(def-thread-api :thread-api/gc-graph + [repo] + (let [{:keys [db client-ops debug-log]} (get @*sqlite-conns repo) + conn (get @*datascript-conns repo)] + (when (and db conn) + (gc-sqlite-dbs! db client-ops debug-log conn {:full-gc? true}) + nil))) + +(def-thread-api :thread-api/vec-search-embedding-model-info + [repo] + (embedding/task--embedding-model-info repo)) + +(def-thread-api :thread-api/vec-search-init-embedding-model + [repo] + (js/Promise. (embedding/task--init-embedding-model repo))) + +(def-thread-api :thread-api/vec-search-load-model + [repo model-name] + (js/Promise. (embedding/task--load-model repo model-name))) + +(def-thread-api :thread-api/vec-search-embedding-graph + [repo opts] + (embedding/embedding-graph! repo opts)) + +(def-thread-api :thread-api/vec-search-search + [repo query-string nums-neighbors] + (embedding/task--search repo query-string nums-neighbors)) + +(def-thread-api :thread-api/vec-search-cancel-indexing + [repo] + (embedding/cancel-indexing repo)) + +(def-thread-api :thread-api/vec-search-update-index-info + [repo] + (js/Promise. (embedding/task--update-index-info! repo))) + +(def-thread-api :thread-api/mobile-logs + [] + @worker-state/*log) + +(def-thread-api :thread-api/get-rtc-graph-uuid + [repo] + (when-let [conn (worker-state/get-datascript-conn repo)] + (ldb/get-graph-rtc-uuid @conn))) + +(def-thread-api :thread-api/api-get-page-data + [repo page-title] + (let [conn (worker-state/get-datascript-conn repo)] + (cli-common-mcp-tools/get-page-data @conn page-title))) + +(def-thread-api :thread-api/api-list-properties + [repo options] + (let [conn (worker-state/get-datascript-conn repo)] + (cli-common-mcp-tools/list-properties @conn options))) + +(def-thread-api :thread-api/api-list-tags + [repo options] + (let [conn (worker-state/get-datascript-conn repo)] + (cli-common-mcp-tools/list-tags @conn options))) + +(def-thread-api :thread-api/api-list-pages + [repo options] + (let [conn (worker-state/get-datascript-conn repo)] + (cli-common-mcp-tools/list-pages @conn options))) + +(def-thread-api :thread-api/api-build-upsert-nodes-edn + [repo ops] + (let [conn (worker-state/get-datascript-conn repo)] + (cli-common-mcp-tools/build-upsert-nodes-edn @conn ops))) + +(comment + (def-thread-api :general/dangerousRemoveAllDbs + [] + (p/let [r (string + [:sync-db-changes + :notification + :log + :add-repo + :rtc-log + :rtc-sync-state]))) + +(defn- prev-graph close-db!) + (when graph + (if (= graph prev-graph) + service + (p/let [service (shared-service/js fns) + #(on-become-master graph start-opts) + broadcast-data-types + {:import? (:import-type? start-opts)})] + (assert (p/promise? (get-in service [:status :ready]))) + (reset! *service [graph service]) + service))))) + +(defn- notify-invalid-data + [{:keys [tx-meta]} errors] + ;; don't notify on production when undo/redo failed + (when-not (and (or (:undo? tx-meta) (:redo? tx-meta)) + (not worker-util/dev?)) + (shared-service/broadcast-to-clients! :notification + [["Invalid data writing to db!"] :error]) + (platform/post-message! (platform/current) + :capture-error + {:error (ex-info "Invalid data writing to db" tx-meta) + :payload {} + :extra {:errors (str errors) + :tx-meta tx-meta}}))) + +(defn- build-proxy-object + [] + (->> + fns + (map + (fn [[k f]] + [k + (fn [& args] + (let [[_graph service] @*service + method-k (keyword (first args))] + (cond + (= :thread-api/create-or-open-db method-k) + ;; because shared-service operates at the graph level, + ;; creating a new database or switching to another one requires re-initializing the service. + (let [[graph opts] (ldb/read-transit-str (last args))] + (p/let [service (js)) + +(defn init-core! + [platform'] + (platform/set-platform! platform') + (ldb/register-transact-invalid-callback-fn! notify-invalid-data) + (outliner-register-op-handlers!) + (build-proxy-object)) + +(comment + (defn js {:print #(log/info :init-sqlite-module! %) - :printErr #(log/error :init-sqlite-module! %)}))] - (reset! *publishing? publishing?) - (reset! *sqlite sqlite) - nil))) - -(def repo-path "/db.sqlite") - -(defn- sqlite-binds - [rows] - (mapv (fn [[addr content addresses]] - #js {:$addr addr - :$content content - :$addresses addresses}) - rows)) - -(defn- enable-sqlite-wal-mode! - [^Object db] - (.exec db "PRAGMA locking_mode=exclusive") - (.exec db "PRAGMA journal_mode=WAL")) - -(defn- ensure-db-sync-import-db! - [repo reset?] - (if-let [sqlite @*sqlite] - (let [^js DB (.-DB ^js (.-oo1 sqlite)) - ^js db (new DB ":memory:" "c")] - (common-sqlite/create-kvs-table! db) - (when reset? - (.exec db "delete from kvs")) - db) - (db-sync/fail-fast :db-sync/missing-field {:repo repo :field :sqlite}))) - -(defn restore-data-from-addr - "Update sqlite-cli/restore-data-from-addr when making changes" - [db addr] - (assert (some? db) "sqlite db not exists") - (when-let [result (-> (.exec db #js {:sql "select content, addresses from kvs where addr = ?" - :bind #js [addr] - :rowMode "array"}) - first)] - (let [[content addresses] (bean/->clj result) - addresses (when addresses - (js/JSON.parse addresses)) - data (sqlite-util/transit-read content)] - (if (and addresses (map? data)) - (assoc data :addresses addresses) - data)))) - -(defn new-sqlite-storage - "Update sqlite-cli/new-sqlite-storage when making changes" - [^Object db] - (reify IStorage - (-store [_ addr+data-seq _delete-addrs] - (let [data (map - (fn [[addr data]] - (let [data' (if (map? data) (dissoc data :addresses) data) - addresses (when (map? data) - (when-let [addresses (:addresses data)] - (js/JSON.stringify (bean/->js addresses))))] - #js {:$addr addr - :$content (sqlite-util/transit-write data') - :$addresses addresses})) - addr+data-seq)] - (upsert-addr-content! db data))) - - (-restore [_ addr] - (restore-data-from-addr db addr)))) - -(defn- close-db-aux! - [repo ^Object db ^Object search ^Object client-ops] - (swap! *sqlite-conns dissoc repo) - (swap! *datascript-conns dissoc repo) - (swap! *client-ops-conns dissoc repo) - (when db (.close db)) - (when search (.close search)) - (when client-ops (.close client-ops)) - (when-let [^js pool (worker-state/get-opfs-pool repo)] - (.pauseVfs pool)) - (swap! *opfs-pools dissoc repo)) - -(defn- close-other-dbs! - [repo] - (doseq [[r {:keys [db search client-ops]}] @*sqlite-conns] - (when-not (= repo r) - (close-db-aux! r db search client-ops)))) - -(defn close-db! - [repo] - (let [{:keys [db search client-ops]} (get @*sqlite-conns repo)] - (close-db-aux! repo db search client-ops))) - -(defn reset-db! - [repo db-transit-str] - (when-let [conn (get @*datascript-conns repo)] - (let [new-db (ldb/read-transit-str db-transit-str) - new-db' (update new-db :eavt (fn [^BTSet s] - (set! (.-storage s) (.-storage (:eavt @conn))) - s))] - (d/reset-conn! conn new-db' {:reset-conn! true}) - (d/reset-schema! conn (:schema new-db))))) - -(defn- get-dbs - [repo] - (if @*publishing? - (p/let [^object DB (.-DB ^object (.-oo1 ^object @*sqlite)) - db (new DB "/db.sqlite" "c") - search-db (new DB "/search-db.sqlite" "c")] - [db search-db]) - (p/let [^js pool ( (- (common-util/time-ms) last-gc-at) (* 3 24 3600 1000))) ; 3 days ago - (log/info :gc-sqlite-dbs "gc current graph") - (doseq [db (if @*publishing? [sqlite-db] [sqlite-db client-ops-db])] - (sqlite-gc/gc-kvs-table! db {:full-gc? full-gc?}) - (.exec db "VACUUM")) - (ldb/transact! datascript-conn [{:db/ident :logseq.kv/graph-last-gc-at - :kv/value (common-util/time-ms)}] - {:skip-validate-db? true})))) - -(defn- datoms (group-by :e datoms) - {properties true non-properties false} (group-by - (fn [[_eid datoms]] - (boolean - (some (fn [datom] (and (= (:a datom) :db/ident) - (db-property/property? (:v datom)))) - datoms))) - eid->datoms) - datoms (concat (mapcat second properties) - (mapcat second non-properties)) - data (map (fn [datom] - [:db/add (:e datom) (:a datom) (:v datom)]) - datoms)] - (d/transact! conn data {:initial-db? true}))) - client-ops-conn (when-not @*publishing? (common-sqlite/get-storage-conn - client-ops-storage - client-op/schema-in-db)) - initial-data-exists? (when (nil? datoms) - (and (d/entity @conn :logseq.class/Root) - (= "db" (:kv/value (d/entity @conn :logseq.kv/db-type)))))] - (swap! *datascript-conns assoc repo conn) - (swap! *client-ops-conns assoc repo client-ops-conn) - (when (and (not @*publishing?) (not= client-op/schema-in-db (d/schema @client-ops-conn))) - (d/reset-schema! client-ops-conn client-op/schema-in-db)) - (let [initial-tx-report (when (and (not initial-data-exists?) (not datoms)) - (let [config (or config "") - initial-data (sqlite-create-graph/build-db-initial-data - config (select-keys opts [:import-type :graph-git-sha :remote-graph?]))] - (ldb/transact! conn initial-data - {:initial-db? true})))] - (db-migrate/migrate conn) - - (gc-sqlite-dbs! db client-ops-db conn {}) - - (when initial-tx-report - (db-sync/handle-local-tx! repo initial-tx-report)) - - (db-listener/listen-db-changes! repo (get @*datascript-conns repo))))))) - -(defn- iter->vec [iter'] - (when iter' - (p/loop [acc []] - (p/let [elem (.next iter')] - (if (.-done elem) - acc - (p/recur (conj acc (.-value elem)))))))) - -(comment - (defn- vec values-iter)) - current-dir-dirs (filter dir? values) - result (concat result values) - dirs (concat - current-dir-dirs - (rest dirs))] - (p/recur result dirs)))))))) - -(defn- vec values-iter)) - current-dir-dirs (filter dir? values) - db-dirs (filter (fn [file] - (string/starts-with? (.-name file) db-dir-prefix)) - current-dir-dirs)] - (p/all (map (fn [dir] - (p/let [graph-name (-> (.-name dir) - (string/replace-first ".logseq-pool-" "") - ;; TODO: DRY - (string/replace "+3A+" ":") - (string/replace "++" "/"))] - {:name graph-name})) db-dirs))))) - -(def-thread-api :thread-api/list-db - [] - ( - (p/let [^js root (.getDirectory js/navigator.storage) - _dir-handle (.getDirectoryHandle root (str "." (worker-util/get-pool-name graph)))] - true) - (p/catch - (fn [_e] ; not found - false)))) - -(defn- remove-vfs! - [^js pool] - (when pool - (.removeVfs ^js pool))) - -(defn- get-search-db - [repo] - (worker-state/get-sqlite-conn repo :search)) - -(comment - (def-thread-api :thread-api/get-version - [] - (when-let [sqlite @*sqlite] - (.-version sqlite)))) - -(def-thread-api :thread-api/init - [] - (init-sqlite-module!)) - -(def-thread-api :thread-api/set-db-sync-config - [config] - (reset! worker-state/*db-sync-config config) - nil) - -(def-thread-api :thread-api/db-sync-start - [repo] - (db-sync/start! repo)) - -(def-thread-api :thread-api/db-sync-stop - [] - (db-sync/stop!)) - -(def-thread-api :thread-api/db-sync-update-presence - [editing-block-uuid] - (db-sync/update-presence! editing-block-uuid)) - -(def-thread-api :thread-api/db-sync-request-asset-download - [repo asset-uuid] - (db-sync/request-asset-download! repo asset-uuid)) - -(def-thread-api :thread-api/db-sync-grant-graph-access - [repo graph-id target-email] - (sync-crypt/db-id {})) - (start-db! repo opts)) - -(def-thread-api :thread-api/q - [repo inputs] - (when-let [conn (worker-state/get-datascript-conn repo)] - (worker-util/profile - (str "Datalog query: " inputs) - (apply d/q (first inputs) @conn (rest inputs))))) - -(def-thread-api :thread-api/datoms - [repo & args] - (when-let [conn (worker-state/get-datascript-conn repo)] - (let [result (apply d/datoms @conn args)] - (map (fn [d] [(:e d) (:a d) (:v d) (:tx d) (:added d)]) result)))) - -(def-thread-api :thread-api/pull - [repo selector id] - (when-let [conn (worker-state/get-datascript-conn repo)] - (let [eid (if (and (vector? id) (= :block/name (first id))) - (:db/id (ldb/get-page @conn (second id))) - id)] - (some->> eid - (d/pull @conn selector) - (common-initial-data/with-parent @conn))))) - -(def ^:private *get-blocks-cache (volatile! (cache/lru-cache-factory {} :threshold 1000))) -(def ^:private get-blocks-with-cache - (common.cache/cache-fn - *get-blocks-cache - (fn [repo requests] - (let [db (some-> (worker-state/get-datascript-conn repo) deref)] - [[repo (:max-tx db) requests] - [db requests]])) - (fn [db requests] - (when db - (->> requests - (mapv (fn [{:keys [id opts]}] - (let [id' (if (and (string? id) (common-util/uuid-string? id)) (uuid id) id)] - (-> (common-initial-data/get-block-and-children db id' opts) - (assoc :id id))))) - ldb/write-transit-str))))) - -(def-thread-api :thread-api/get-blocks - [repo requests] - (let [requests (ldb/read-transit-str requests)] - (get-blocks-with-cache repo requests))) - -(def-thread-api :thread-api/get-block-refs - [repo id] - (when-let [conn (worker-state/get-datascript-conn repo)] - (->> (db-reference/get-linked-references @conn id) - :ref-blocks - (map (fn [b] (assoc (into {} b) :db/id (:db/id b))))))) - -(def-thread-api :thread-api/get-block-refs-count - [repo id] - (when-let [conn (worker-state/get-datascript-conn repo)] - (ldb/get-block-refs-count @conn id))) - -(def-thread-api :thread-api/get-block-source - [repo id] - (when-let [conn (worker-state/get-datascript-conn repo)] - (:db/id (first (:block/_alias (d/entity @conn id)))))) - -(defn- search-blocks - [repo q option] - (let [search-db (get-search-db repo) - conn (worker-state/get-datascript-conn repo)] - (search/search-blocks repo conn search-db q option))) - -(def-thread-api :thread-api/block-refs-check - [repo id {:keys [unlinked?]}] - (m/sp - (when-let [conn (worker-state/get-datascript-conn repo)] - (let [db @conn - block (d/entity db id)] - (if unlinked? - (let [title (string/lower-case (:block/title block)) - result (m/? (search-blocks repo title {:limit 100}))] - (boolean (some (fn [b] - (let [block (d/entity db (:db/id b))] - (and (not= id (:db/id block)) - (not ((set (map :db/id (:block/refs block))) id)) - (string/includes? (string/lower-case (:block/title block)) title)))) result))) - (some? (first (common-initial-data/get-block-refs db (:db/id block))))))))) - -(def-thread-api :thread-api/get-block-parents - [repo id depth] - (when-let [conn (worker-state/get-datascript-conn repo)] - (let [block-id (:block/uuid (d/entity @conn id))] - (->> (ldb/get-block-parents @conn block-id {:depth (or depth 3)}) - (map (fn [b] (d/pull @conn '[*] (:db/id b)))))))) - -(def-thread-api :thread-api/set-context - [context] - (when context (worker-state/update-context! context)) - nil) - -(def-thread-api :thread-api/transact - [repo tx-data tx-meta context] - (assert (some? repo)) - (worker-state/set-db-latest-tx-time! repo) - (let [conn (worker-state/get-datascript-conn repo)] - (assert (some? conn) {:repo repo}) - (try - (let [tx-data' (if (contains? #{:insert-blocks} (:outliner-op tx-meta)) - (map (fn [m] - (if (and (map? m) (nil? (:block/order m))) - (assoc m :block/order (db-order/gen-key nil)) - m)) tx-data) - tx-data) - _ (when context (worker-state/set-context! context)) - tx-meta' (cond-> tx-meta - true - (dissoc :insert-blocks?))] - (when-not (and (:create-today-journal? tx-meta) - (:today-journal-name tx-meta) - (seq tx-data') - (ldb/get-page @conn (:today-journal-name tx-meta))) ; today journal created already - - ;; (prn :debug :transact :tx-data tx-data' :tx-meta tx-meta') - - (when (and (or (:undo? tx-meta) (:redo? tx-meta)) - (not (undo-validate/valid-undo-redo-tx? conn tx-data'))) - (throw (ex-info "undo/redo tx invalid" - {:repo repo - :undo? (:undo? tx-meta) - :redo? (:redo? tx-meta)}))) - (worker-util/profile "Worker db transact" - (ldb/transact! conn tx-data' tx-meta'))) - nil) - (catch :default e - (prn :debug :worker-transact-failed :tx-meta tx-meta :tx-data tx-data) - (log/error ::worker-transact-failed e) - (throw e))))) - -(def-thread-api :thread-api/get-initial-data - [repo opts] - (when-let [conn (worker-state/get-datascript-conn repo)] - (if (:file-graph-import? opts) - {:schema (:schema @conn) - :initial-data (vec (d/datoms @conn :eavt))} - (common-initial-data/get-initial-data @conn)))) - -(def-thread-api :thread-api/reset-db - [repo db-transit] - (reset-db! repo db-transit) - nil) - -(defn- (p/do! - (rtc-log-and-state/rtc-log :rtc.log/download - {:sub-type :download-progress - :graph-uuid graph-id - :message "Saving data to DB"}) - ((@thread-api/*thread-apis :thread-api/create-or-open-db) repo {:close-other-db? true - :datoms datoms}) - (db-sync/rehydrate-large-titles-from-db! repo graph-id) - (rtc-log-and-state/rtc-log :rtc.log/download - {:sub-type :download-completed - :graph-uuid graph-id - :message "Graph is ready!"}) - ((@thread-api/*thread-apis :thread-api/export-db) repo) - (client-op/update-local-tx repo remote-tx) - (shared-service/broadcast-to-clients! :add-repo {:repo repo})) - (p/catch (fn [error] - (js/console.error error))))) - -(def-thread-api :thread-api/db-sync-import-kvs-rows - [repo rows reset? graph-id remote-tx graph-e2ee?] - (let [graph-e2ee? (if (nil? graph-e2ee?) true (true? graph-e2ee?))] - (p/let [_ (when reset? (close-db! repo)) - aes-key (when graph-e2ee? - (sync-crypt/sqlite-binds rows-batch)) - (rtc-log-and-state/rtc-log :rtc.log/download - {:sub-type :download-progress - :graph-uuid graph-id - :message (str (if graph-e2ee? - "Decrypting data" - "Importing data") - " " - (inc i) - "/" - (count batches))}))) - (let [storage (new-sqlite-storage db) - conn (common-sqlite/get-storage-conn storage db-schema/schema) - datoms (vec (d/datoms @conn :eavt))] - (.close db) - (import-datoms-to-db! repo graph-id remote-tx datoms))))) - -(def-thread-api :thread-api/release-access-handles - [repo] - (when-let [^js pool (worker-state/get-opfs-pool repo)] - (.pauseVfs pool) - nil)) - -(def-thread-api :thread-api/db-exists - [repo] - (js blocks)) - nil)) - -(def-thread-api :thread-api/search-delete-blocks - [repo ids] - (p/let [db (get-search-db repo)] - (search/delete-blocks! db ids) - nil)) - -(def-thread-api :thread-api/search-truncate-tables - [repo] - (p/let [db (get-search-db repo)] - (search/truncate-table! db) - nil)) - -(def-thread-api :thread-api/search-build-blocks-indice - [repo] - (when-let [conn (worker-state/get-datascript-conn repo)] - (search/build-blocks-indice repo @conn))) - -(def-thread-api :thread-api/search-build-pages-indice - [_repo] - nil) - -(def-thread-api :thread-api/apply-outliner-ops - [repo ops opts] - (when-let [conn (worker-state/get-datascript-conn repo)] - (try - (worker-util/profile - "apply outliner ops" - (outliner-op/apply-ops! conn ops opts)) - (catch :default e - (let [data (ex-data e) - {:keys [type payload]} (when (map? data) data)] - (case type - :notification - (do - (log/error ::apply-outliner-ops-failed e) - (shared-service/broadcast-to-clients! :notification [(:message payload) (:type payload) (:clear? payload) (:uid payload) (:timeout payload)])) - (throw e))))))) - -(def-thread-api :thread-api/sync-app-state - [new-state] - (when (and (contains? new-state :git/current-repo) - (nil? (:git/current-repo new-state))) - (log/error :thread-api/sync-app-state new-state)) - (worker-state/set-new-state! new-state) - nil) - -(def-thread-api :thread-api/export-get-debug-datoms - [repo] - (when-let [conn (worker-state/get-datascript-conn repo)] - (worker-export/get-debug-datoms conn))) - -(def-thread-api :thread-api/export-get-all-page->content - [repo options] - (when-let [conn (worker-state/get-datascript-conn repo)] - (worker-export/get-all-page->content @conn options))) - -(def-thread-api :thread-api/validate-db - [repo] - (when-let [conn (worker-state/get-datascript-conn repo)] - (worker-db-validate/validate-db conn))) - -;; Returns an export-edn map for given repo. When there's an unexpected error, a map -;; with key :export-edn-error is returned -(def-thread-api :thread-api/export-edn - [repo options] - (let [conn (worker-state/get-datascript-conn repo)] - (try - (sqlite-export/build-export @conn options) - (catch :default e - (js/console.error "export-edn error: " e) - (js/console.error "Stack:\n" (.-stack e)) - (worker-util/post-message :notification - ["An unexpected error occurred during export. See the javascript console for details." - :error]) - {:export-edn-error (.-message e)})))) - -(def-thread-api :thread-api/get-view-data - [repo view-id option] - (let [db @(worker-state/get-datascript-conn repo)] - (db-view/get-view-data db view-id option))) - -(def-thread-api :thread-api/get-class-objects - [repo class-id] - (let [db @(worker-state/get-datascript-conn repo)] - (->> (db-class/get-class-objects db class-id) - (map entity-util/entity->map)))) - -(def-thread-api :thread-api/get-property-values - [repo {:keys [property-ident] :as option}] - (let [conn (worker-state/get-datascript-conn repo)] - (db-view/get-property-values @conn property-ident option))) - -(def-thread-api :thread-api/get-bidirectional-properties - [repo {:keys [target-id]}] - (let [conn (worker-state/get-datascript-conn repo)] - (worker-util/profile "get-bidirectional-properties" - (ldb/get-bidirectional-properties @conn target-id)))) - -(def-thread-api :thread-api/build-graph - [repo option] - (let [conn (worker-state/get-datascript-conn repo)] - (graph-view/build-graph @conn option))) - -(def ^:private *get-all-page-titles-cache (volatile! (cache/lru-cache-factory {}))) -(defn- get-all-page-titles - [db] - (let [pages (ldb/get-all-pages db)] - (sort (map :block/title pages)))) - -(def ^:private get-all-page-titles-with-cache - (common.cache/cache-fn - *get-all-page-titles-cache - (fn [repo] - (let [db @(worker-state/get-datascript-conn repo)] - [[repo (:max-tx db)] ;cache-key - [db] ;f-args - ])) - get-all-page-titles)) - -(def-thread-api :thread-api/get-all-page-titles - [repo] - (get-all-page-titles-with-cache repo)) - -(def-thread-api :thread-api/gc-graph - [repo] - (let [{:keys [db client-ops]} (get @*sqlite-conns repo) - conn (get @*datascript-conns repo)] - (when (and db conn) - (gc-sqlite-dbs! db client-ops conn {:full-gc? true}) - nil))) - -(def-thread-api :thread-api/vec-search-embedding-model-info - [repo] - (embedding/task--embedding-model-info repo)) - -(def-thread-api :thread-api/vec-search-init-embedding-model - [repo] - (js/Promise. (embedding/task--init-embedding-model repo))) - -(def-thread-api :thread-api/vec-search-load-model - [repo model-name] - (js/Promise. (embedding/task--load-model repo model-name))) - -(def-thread-api :thread-api/vec-search-embedding-graph - [repo opts] - (embedding/embedding-graph! repo opts)) - -(def-thread-api :thread-api/vec-search-search - [repo query-string nums-neighbors] - (embedding/task--search repo query-string nums-neighbors)) - -(def-thread-api :thread-api/vec-search-cancel-indexing - [repo] - (embedding/cancel-indexing repo)) - -(def-thread-api :thread-api/vec-search-update-index-info - [repo] - (js/Promise. (embedding/task--update-index-info! repo))) - -(def-thread-api :thread-api/mobile-logs - [] - @worker-state/*log) - -(def-thread-api :thread-api/get-rtc-graph-uuid - [repo] - (when-let [conn (worker-state/get-datascript-conn repo)] - (ldb/get-graph-rtc-uuid @conn))) - -(def-thread-api :thread-api/api-get-page-data - [repo page-title] - (let [conn (worker-state/get-datascript-conn repo)] - (cli-common-mcp-tools/get-page-data @conn page-title))) - -(def-thread-api :thread-api/api-list-properties - [repo options] - (let [conn (worker-state/get-datascript-conn repo)] - (cli-common-mcp-tools/list-properties @conn options))) - -(def-thread-api :thread-api/api-list-tags - [repo options] - (let [conn (worker-state/get-datascript-conn repo)] - (cli-common-mcp-tools/list-tags @conn options))) - -(def-thread-api :thread-api/api-list-pages - [repo options] - (let [conn (worker-state/get-datascript-conn repo)] - (cli-common-mcp-tools/list-pages @conn options))) - -(def-thread-api :thread-api/api-build-upsert-nodes-edn - [repo ops] - (let [conn (worker-state/get-datascript-conn repo)] - (cli-common-mcp-tools/build-upsert-nodes-edn @conn ops))) - -(comment - (def-thread-api :general/dangerousRemoveAllDbs - [] - (p/let [r (string - [:sync-db-changes - :notification - :log - :add-repo - :rtc-log - :rtc-sync-state]))) - -(defn- prev-graph close-db!) - (when graph - (if (= graph prev-graph) - service - (p/let [service (shared-service/js fns) - #(on-become-master graph start-opts) - broadcast-data-types - {:import? (:import-type? start-opts)})] - (assert (p/promise? (get-in service [:status :ready]))) - (reset! *service [graph service]) - service))))) - -(defn- notify-invalid-data - [{:keys [tx-meta]} errors] - ;; don't notify on production when undo/redo failed - (when-not (and (or (:undo? tx-meta) (:redo? tx-meta)) - (not worker-util/dev?)) - (shared-service/broadcast-to-clients! :notification - [["Invalid data writing to db!"] :error]) - (worker-util/post-message :capture-error - {:error (ex-info "Invalid data writing to db" tx-meta) - :payload {} - :extra {:errors (str errors) - :tx-meta tx-meta}}))) - (defn init "web worker entry" [] - (ldb/register-transact-invalid-callback-fn! notify-invalid-data) - - (let [proxy-object (->> - fns - (map - (fn [[k f]] - [k - (fn [& args] - (let [[_graph service] @*service - method-k (keyword (first args))] - (cond - (= :thread-api/create-or-open-db method-k) - ;; because shared-service operates at the graph level, - ;; creating a new database or switching to another one requires re-initializing the service. - (let [[graph opts] (ldb/read-transit-str (last args))] - (p/let [service (js)] + (let [platform (platform-browser/browser-platform) + proxy-object (db-core/init-core! platform)] (glogi-console/install!) (log/set-levels {:glogi/root :info}) (log/add-handler worker-state/log-append!) (check-worker-scope!) - (outliner-register-op-handlers!) - (js/setInterval #(.postMessage js/self "keepAliveResponse") (* 1000 25)) + ((get-in platform [:timers :set-interval!]) + #(.postMessage js/self "keepAliveResponse") + (* 1000 25)) (Comlink/expose proxy-object) (let [^js wrapped-main-thread* (Comlink/wrap js/self) wrapped-main-thread (fn [qkw direct-pass? & args] @@ -1028,13 +43,3 @@ result (ldb/read-transit-str result))))] (reset! worker-state/*main-thread wrapped-main-thread)))) - -(comment - (defn vec + [iter'] + (when iter' + (p/loop [acc []] + (p/let [elem (.next iter')] + (if (.-done elem) + acc + (p/recur (conj acc (.-value elem)))))))) + +(defn- list-graphs + [] + (let [dir? #(= (.-kind %) "directory") + db-dir-prefix ".logseq-pool-"] + (p/let [^js root (.getDirectory js/navigator.storage) + values-iter (when (dir? root) (.values root)) + values (when values-iter (iter->vec values-iter)) + current-dir-dirs (filter dir? values) + db-dirs (filter (fn [file] + (string/starts-with? (.-name file) db-dir-prefix)) + current-dir-dirs) + graph-names (map (fn [dir] + (-> (.-name dir) + (string/replace-first ".logseq-pool-" "") + ;; TODO: DRY + (string/replace "+3A+" ":") + (string/replace "++" "/"))) + db-dirs)] + (log/info :db-dirs (map #(.-name %) db-dirs) :all-dirs (map #(.-name %) current-dir-dirs)) + (vec graph-names)))) + +(defn- db-exists? + [graph] + (-> + (p/let [^js root (.getDirectory js/navigator.storage) + _dir-handle (.getDirectoryHandle root (str "." (worker-util/get-pool-name graph)))] + true) + (p/catch + (fn [_e] ; not found + false)))) + +(defonce ^:private kv-store + (delay (idb-keyval/newStore "localforage" "keyvaluepairs" 2))) + +(defn- kv-get + [k] + (idb-keyval/get k @kv-store)) + +(defn- kv-set! + [k value] + (idb-keyval/set k value @kv-store)) + +(defn- install-opfs-pool + [sqlite pool-name] + (.installOpfsSAHPoolVfs ^js sqlite #js {:name pool-name + :initialCapacity 20})) + +(defn- export-file + [pool path] + (.exportFile ^js pool path)) + +(defn- import-db + [pool path data] + (.importDb ^js pool path data)) + +(defn- remove-vfs! + [pool] + (when pool + (.removeVfs ^js pool))) + +(defn- read-text! + [path] + (opfs/