mirror of
https://github.com/logseq/logseq.git
synced 2026-05-29 15:09:41 +00:00
milestone 1+2
This commit is contained in:
Gabriel Horner
27
docs/agent-guide/db-worker-browser-api-inventory.md
Normal file
27
docs/agent-guide/db-worker-browser-api-inventory.md
Normal file
@@ -0,0 +1,27 @@
|
||||
# db-worker browser-only API inventory
|
||||
|
||||
## Worker entry
|
||||
- frontend.worker.db-worker
|
||||
- importScripts
|
||||
- js/self postMessage
|
||||
- js/location.href
|
||||
- navigator.storage (OPFS)
|
||||
- Comlink expose/wrap/transfer
|
||||
- setInterval
|
||||
|
||||
## Shared service
|
||||
- frontend.worker.shared-service
|
||||
- navigator.locks.request/query
|
||||
- BroadcastChannel
|
||||
- js/self postMessage
|
||||
|
||||
## RTC and crypto
|
||||
- frontend.worker.rtc.ws
|
||||
- WebSocket
|
||||
- frontend.worker.rtc.crypt
|
||||
- OPFS file access via frontend.common.file.opfs
|
||||
- js/self location
|
||||
|
||||
## Worker util
|
||||
- frontend.worker-common.util
|
||||
- wfu/post-message (worker postMessage bridge)
|
||||
@@ -48,15 +48,25 @@ Make `frontend.worker.db-worker` and its dependencies run in both browser and No
|
||||
## Refactor Steps (Milestones + Status)
|
||||
|
||||
### Milestone 1: Architecture & Abstractions
|
||||
- TODO 1. Inventory db-worker dependencies and classify browser-only APIs.
|
||||
- TODO 2. Define a platform adapter interface (storage, kv, broadcast, websocket, crypto, timers, env flags).
|
||||
- TODO 3. Extract db-worker core logic into a platform-agnostic module (e.g. `frontend.worker.db-core`).
|
||||
- DONE 1. Inventory db-worker dependencies and classify browser-only APIs.
|
||||
- DONE 2. Define a platform adapter interface (storage, kv, broadcast, websocket, crypto, timers, env flags).
|
||||
- DONE 3. Extract db-worker core logic into a platform-agnostic module (e.g. `frontend.worker.db-core`).
|
||||
#### Acceptance Criteria
|
||||
- Core worker module has zero direct references to `js/self`, `js/location`, `js/navigator`, `importScripts`, `BroadcastChannel`, or `navigator.locks`.
|
||||
- `frontend.worker.platform` exists with required sections and validation; platform adapter passes validation at init time.
|
||||
- Browser worker entry initializes via `init!`/`init-core!` with a platform adapter.
|
||||
- `bb dev:lint-and-test` passes.
|
||||
|
||||
### Milestone 2: Browser Path Parity
|
||||
- TODO 4. Implement `frontend.worker.platform.browser`.
|
||||
- TODO 5. Update db-worker entry to inject the platform adapter and call core init.
|
||||
- TODO 6. Route OPFS/IDB usage through the platform adapter in worker submodules.
|
||||
- TODO 7. Replace direct `js/WebSocket` usage with platform websocket factory.
|
||||
- DONE 4. Implement `frontend.worker.platform.browser`.
|
||||
- DONE 5. Update db-worker entry to inject the platform adapter and call core init.
|
||||
- DONE 6. Route OPFS/IDB usage through the platform adapter in worker submodules.
|
||||
- DONE 7. Replace direct `js/WebSocket` usage with platform websocket factory.
|
||||
#### Acceptance Criteria
|
||||
- Browser platform adapter encapsulates OPFS/IDB storage, kv-store, and WebSocket factory; worker submodules no longer import OPFS/IDB directly.
|
||||
- RTC WebSocket creation uses the platform adapter.
|
||||
- Browser db-worker entry injects the platform adapter and serves Comlink RPC.
|
||||
- `bb dev:lint-and-test` passes.
|
||||
|
||||
### Milestone 3: Node Path & Daemon
|
||||
- TODO 8. Implement `frontend.worker.platform.node` in single-client mode (no locks or BroadcastChannel).
|
||||
@@ -64,10 +74,22 @@ Make `frontend.worker.db-worker` and its dependencies run in both browser and No
|
||||
- TODO 10. Add Node build target in `shadow-cljs.edn` for db-worker.
|
||||
- TODO 11. Implement Node daemon entrypoint and HTTP server.
|
||||
- TODO 12. Add a Node client in frontend to call the daemon (HTTP + SSE/WS events).
|
||||
#### Acceptance Criteria
|
||||
- Node platform adapter provides storage/kv/broadcast/websocket/crypto/timers and validates via `frontend.worker.platform`.
|
||||
- Node build target compiles db-worker core without browser-only APIs.
|
||||
- Node daemon starts via CLI and reports readiness; `GET /healthz` and `GET /readyz` return `200 OK`.
|
||||
- `POST /v1/invoke` handles `list-db`, `create-or-open-db`, `q`, `transact` in a smoke test.
|
||||
- Node client can invoke at least one RPC and receive one event (SSE or WS).
|
||||
- `bb dev:lint-and-test` passes.
|
||||
|
||||
### Milestone 4: Validation
|
||||
- TODO 13. Add tests: adapter unit tests + daemon integration smoke test.
|
||||
- TODO 14. Verify browser worker path still works with Comlink.
|
||||
#### Acceptance Criteria
|
||||
- Adapter unit tests cover browser and node implementations for storage/kv/broadcast/websocket factories.
|
||||
- Daemon integration smoke test starts the node process and exercises `/v1/invoke` with at least one method.
|
||||
- Browser worker path verified with Comlink RPCs (smoke test).
|
||||
- `bb dev:lint-and-test` passes.
|
||||
|
||||
## Node.js Daemon Requirements
|
||||
The db-worker should be runnable as a standalone process for Node.js environments.
|
||||
|
||||
@@ -18,7 +18,8 @@
|
||||
"Run tests. Pass args through to cmd 'yarn cljs:run-test'"
|
||||
[& args]
|
||||
(shell "yarn cljs:test")
|
||||
(apply shell "yarn cljs:run-test" args))
|
||||
(let [args* (or (seq args) ["-e" "long" "-e" "fix-me"])]
|
||||
(apply shell "yarn cljs:run-test" args*)))
|
||||
|
||||
(defn test-no-worker
|
||||
"Run tests without compiling worker namespaces. Pass args through to cmd 'yarn cljs:run-test-no-worker'"
|
||||
@@ -31,7 +32,7 @@
|
||||
pass args through to cmd 'yarn cljs:run-test'"
|
||||
[]
|
||||
(dev-lint/dev)
|
||||
(test "-e" "long" "-e" "fix-me"))
|
||||
(test))
|
||||
|
||||
(defn e2e-basic-test
|
||||
"Run e2e basic tests. HTTP server should be available at localhost:3001"
|
||||
|
||||
894
src/main/frontend/worker/db_core.cljs
Normal file
894
src/main/frontend/worker/db_core.cljs
Normal file
@@ -0,0 +1,894 @@
|
||||
(ns frontend.worker.db-core
|
||||
"Core db-worker logic without host-specific bootstrap."
|
||||
(:require ["@sqlite.org/sqlite-wasm" :default sqlite3InitModule]
|
||||
[cljs-bean.core :as bean]
|
||||
[cljs.cache :as cache]
|
||||
[clojure.edn :as edn]
|
||||
[clojure.set]
|
||||
[clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[datascript.storage :refer [IStorage] :as storage]
|
||||
[frontend.common.cache :as common.cache]
|
||||
[frontend.common.graph-view :as graph-view]
|
||||
[frontend.common.missionary :as c.m]
|
||||
[frontend.common.thread-api :as thread-api :refer [def-thread-api]]
|
||||
[frontend.worker.platform :as platform]
|
||||
[frontend.worker-common.util :as worker-util]
|
||||
[frontend.worker.db-listener :as db-listener]
|
||||
[frontend.worker.db-metadata :as worker-db-metadata]
|
||||
[frontend.worker.db.fix :as db-fix]
|
||||
[frontend.worker.db.migrate :as db-migrate]
|
||||
[frontend.worker.db.validate :as worker-db-validate]
|
||||
[frontend.worker.embedding :as embedding]
|
||||
[frontend.worker.export :as worker-export]
|
||||
[frontend.worker.handler.page :as worker-page]
|
||||
[frontend.worker.pipeline :as worker-pipeline]
|
||||
[frontend.worker.publish]
|
||||
[frontend.worker.rtc.asset-db-listener]
|
||||
[frontend.worker.rtc.client-op :as client-op]
|
||||
[frontend.worker.rtc.core :as rtc.core]
|
||||
[frontend.worker.rtc.db-listener]
|
||||
[frontend.worker.rtc.debug-log :as rtc-debug-log]
|
||||
[frontend.worker.rtc.migrate :as rtc-migrate]
|
||||
[frontend.worker.search :as search]
|
||||
[frontend.worker.shared-service :as shared-service]
|
||||
[frontend.worker.state :as worker-state]
|
||||
[frontend.worker.thread-atom]
|
||||
[lambdaisland.glogi :as log]
|
||||
[logseq.cli.common.mcp.tools :as cli-common-mcp-tools]
|
||||
[logseq.common.util :as common-util]
|
||||
[logseq.db :as ldb]
|
||||
[logseq.db.common.entity-plus :as entity-plus]
|
||||
[logseq.db.common.entity-util :as common-entity-util]
|
||||
[logseq.db.common.initial-data :as common-initial-data]
|
||||
[logseq.db.common.order :as db-order]
|
||||
[logseq.db.common.reference :as db-reference]
|
||||
[logseq.db.common.sqlite :as common-sqlite]
|
||||
[logseq.db.common.view :as db-view]
|
||||
[logseq.db.frontend.class :as db-class]
|
||||
[logseq.db.frontend.property :as db-property]
|
||||
[logseq.db.sqlite.create-graph :as sqlite-create-graph]
|
||||
[logseq.db.sqlite.export :as sqlite-export]
|
||||
[logseq.db.sqlite.gc :as sqlite-gc]
|
||||
[logseq.db.sqlite.util :as sqlite-util]
|
||||
[logseq.outliner.core :as outliner-core]
|
||||
[logseq.outliner.op :as outliner-op]
|
||||
[me.tonsky.persistent-sorted-set :as set :refer [BTSet]]
|
||||
[missionary.core :as m]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(defonce *sqlite worker-state/*sqlite)
|
||||
(defonce *sqlite-conns worker-state/*sqlite-conns)
|
||||
(defonce *datascript-conns worker-state/*datascript-conns)
|
||||
(defonce *client-ops-conns worker-state/*client-ops-conns)
|
||||
(defonce *opfs-pools worker-state/*opfs-pools)
|
||||
(defonce *publishing? (atom false))
|
||||
|
||||
(defn- <get-opfs-pool
|
||||
[graph]
|
||||
(when-not @*publishing?
|
||||
(or (worker-state/get-opfs-pool graph)
|
||||
(p/let [storage (platform/storage (platform/current))
|
||||
^js pool ((:install-opfs-pool storage) @*sqlite (worker-util/get-pool-name graph))]
|
||||
(swap! *opfs-pools assoc graph pool)
|
||||
pool))))
|
||||
|
||||
(defn- init-sqlite-module!
|
||||
[]
|
||||
(when-not @*sqlite
|
||||
(p/let [publishing? (platform/env-flag (platform/current) :publishing?)
|
||||
sqlite (sqlite3InitModule (clj->js {:print #(log/info :init-sqlite-module! %)
|
||||
:printErr #(log/error :init-sqlite-module! %)}))]
|
||||
(reset! *publishing? publishing?)
|
||||
(reset! *sqlite sqlite)
|
||||
nil)))
|
||||
|
||||
(def repo-path "/db.sqlite")
|
||||
(def debug-log-path "/debug-log/db.sqlite")
|
||||
|
||||
(defn- <export-db-file
|
||||
([repo]
|
||||
(<export-db-file repo repo-path))
|
||||
([repo path]
|
||||
(p/let [^js pool (<get-opfs-pool repo)]
|
||||
(when pool
|
||||
(let [storage (platform/storage (platform/current))]
|
||||
((:export-file storage) pool path))))))
|
||||
|
||||
(defn- <import-db
|
||||
[^js pool data]
|
||||
(let [storage (platform/storage (platform/current))]
|
||||
((:import-db storage) pool repo-path data)))
|
||||
|
||||
(defn upsert-addr-content!
|
||||
"Upsert addr+data-seq. Update sqlite-cli/upsert-addr-content! when making changes"
|
||||
[db data]
|
||||
(assert (some? db) "sqlite db not exists")
|
||||
(.transaction
|
||||
db
|
||||
(fn [tx]
|
||||
(doseq [item data]
|
||||
(.exec tx #js {:sql "INSERT INTO kvs (addr, content, addresses) values ($addr, $content, $addresses) on conflict(addr) do update set content = $content, addresses = $addresses"
|
||||
:bind item})))))
|
||||
|
||||
(defn restore-data-from-addr
|
||||
"Update sqlite-cli/restore-data-from-addr when making changes"
|
||||
[db addr]
|
||||
(assert (some? db) "sqlite db not exists")
|
||||
(when-let [result (-> (.exec db #js {:sql "select content, addresses from kvs where addr = ?"
|
||||
:bind #js [addr]
|
||||
:rowMode "array"})
|
||||
first)]
|
||||
(let [[content addresses] (bean/->clj result)
|
||||
addresses (when addresses
|
||||
(js/JSON.parse addresses))
|
||||
data (sqlite-util/transit-read content)]
|
||||
(if (and addresses (map? data))
|
||||
(assoc data :addresses addresses)
|
||||
data))))
|
||||
|
||||
(defn new-sqlite-storage
|
||||
"Update sqlite-cli/new-sqlite-storage when making changes"
|
||||
[^Object db]
|
||||
(reify IStorage
|
||||
(-store [_ addr+data-seq _delete-addrs]
|
||||
(let [data (map
|
||||
(fn [[addr data]]
|
||||
(let [data' (if (map? data) (dissoc data :addresses) data)
|
||||
addresses (when (map? data)
|
||||
(when-let [addresses (:addresses data)]
|
||||
(js/JSON.stringify (bean/->js addresses))))]
|
||||
#js {:$addr addr
|
||||
:$content (sqlite-util/transit-write data')
|
||||
:$addresses addresses}))
|
||||
addr+data-seq)]
|
||||
(upsert-addr-content! db data)))
|
||||
|
||||
(-restore [_ addr]
|
||||
(restore-data-from-addr db addr))))
|
||||
|
||||
(defn- close-db-aux!
|
||||
[repo ^Object db ^Object search ^Object client-ops ^Object debug-log]
|
||||
(swap! *sqlite-conns dissoc repo)
|
||||
(swap! *datascript-conns dissoc repo)
|
||||
(swap! *client-ops-conns dissoc repo)
|
||||
(when db (.close db))
|
||||
(when search (.close search))
|
||||
(when client-ops (.close client-ops))
|
||||
(when debug-log (.close debug-log))
|
||||
(when-let [^js pool (worker-state/get-opfs-pool repo)]
|
||||
(.pauseVfs pool))
|
||||
(swap! *opfs-pools dissoc repo))
|
||||
|
||||
(defn- close-other-dbs!
|
||||
[repo]
|
||||
(doseq [[r {:keys [db search client-ops debug-log]}] @*sqlite-conns]
|
||||
(when-not (= repo r)
|
||||
(close-db-aux! r db search client-ops debug-log))))
|
||||
|
||||
(defn close-db!
|
||||
[repo]
|
||||
(let [{:keys [db search client-ops debug-log]} (get @*sqlite-conns repo)]
|
||||
(close-db-aux! repo db search client-ops debug-log)))
|
||||
|
||||
(defn reset-db!
|
||||
[repo db-transit-str]
|
||||
(when-let [conn (get @*datascript-conns repo)]
|
||||
(let [new-db (ldb/read-transit-str db-transit-str)
|
||||
new-db' (update new-db :eavt (fn [^BTSet s]
|
||||
(set! (.-storage s) (.-storage (:eavt @conn)))
|
||||
s))]
|
||||
(d/reset-conn! conn new-db' {:reset-conn! true})
|
||||
(d/reset-schema! conn (:schema new-db)))))
|
||||
|
||||
(defn- get-dbs
|
||||
[repo]
|
||||
(if @*publishing?
|
||||
(p/let [^object DB (.-DB ^object (.-oo1 ^object @*sqlite))
|
||||
db (new DB "/db.sqlite" "c")
|
||||
search-db (new DB "/search-db.sqlite" "c")]
|
||||
[db search-db])
|
||||
(p/let [^js pool (<get-opfs-pool repo)
|
||||
capacity (.getCapacity pool)
|
||||
_ (when (zero? capacity) ; file handle already releases since pool will be initialized only once
|
||||
(.unpauseVfs pool))
|
||||
db (new (.-OpfsSAHPoolDb pool) repo-path)
|
||||
search-db (new (.-OpfsSAHPoolDb pool) (str "search" repo-path))
|
||||
client-ops-db (new (.-OpfsSAHPoolDb pool) (str "client-ops-" repo-path))
|
||||
debug-log-db (new (.-OpfsSAHPoolDb pool) (str "debug-log" repo-path))]
|
||||
[db search-db client-ops-db debug-log-db])))
|
||||
|
||||
(defn- enable-sqlite-wal-mode!
|
||||
[^Object db]
|
||||
(.exec db "PRAGMA locking_mode=exclusive")
|
||||
(.exec db "PRAGMA journal_mode=WAL"))
|
||||
|
||||
(defn- gc-sqlite-dbs!
|
||||
"Gc main db weekly and rtc ops db each time when opening it"
|
||||
[sqlite-db client-ops-db debug-log-db datascript-conn {:keys [full-gc?]}]
|
||||
(let [last-gc-at (:kv/value (d/entity @datascript-conn :logseq.kv/graph-last-gc-at))]
|
||||
(when (or full-gc?
|
||||
(nil? last-gc-at)
|
||||
(not (number? last-gc-at))
|
||||
(> (- (common-util/time-ms) last-gc-at) (* 3 24 3600 1000))) ; 3 days ago
|
||||
(log/info :gc-sqlite-dbs "gc current graph")
|
||||
(doseq [db (if @*publishing? [sqlite-db] [sqlite-db client-ops-db])]
|
||||
(sqlite-gc/gc-kvs-table! db {:full-gc? full-gc?})
|
||||
(.exec db "VACUUM"))
|
||||
(rtc-debug-log/gc! debug-log-db)
|
||||
(ldb/transact! datascript-conn [{:db/ident :logseq.kv/graph-last-gc-at
|
||||
:kv/value (common-util/time-ms)}]))))
|
||||
|
||||
(defn- <create-or-open-db!
|
||||
[repo {:keys [config datoms] :as opts}]
|
||||
(when-not (worker-state/get-sqlite-conn repo)
|
||||
(p/let [[db search-db client-ops-db debug-log-db :as dbs] (get-dbs repo)
|
||||
storage (new-sqlite-storage db)
|
||||
client-ops-storage (when-not @*publishing?
|
||||
(new-sqlite-storage client-ops-db))
|
||||
db-based? true]
|
||||
(swap! *sqlite-conns assoc repo {:db db
|
||||
:search search-db
|
||||
:client-ops client-ops-db
|
||||
:debug-log debug-log-db})
|
||||
(doseq [db' dbs]
|
||||
(enable-sqlite-wal-mode! db'))
|
||||
(common-sqlite/create-kvs-table! db)
|
||||
(when-not @*publishing? (common-sqlite/create-kvs-table! client-ops-db))
|
||||
(rtc-debug-log/create-tables! debug-log-db)
|
||||
(search/create-tables-and-triggers! search-db)
|
||||
(ldb/register-transact-pipeline-fn!
|
||||
(fn [tx-report]
|
||||
(worker-pipeline/transact-pipeline repo tx-report)))
|
||||
(let [schema (ldb/get-schema repo)
|
||||
conn (common-sqlite/get-storage-conn storage schema)
|
||||
_ (db-fix/check-and-fix-schema! repo conn)
|
||||
_ (when datoms
|
||||
(let [eid->datoms (group-by :e datoms)
|
||||
{properties true non-properties false} (group-by
|
||||
(fn [[_eid datoms]]
|
||||
(boolean
|
||||
(some (fn [datom] (and (= (:a datom) :db/ident)
|
||||
(db-property/property? (:v datom))))
|
||||
datoms)))
|
||||
eid->datoms)
|
||||
datoms (concat (mapcat second properties)
|
||||
(mapcat second non-properties))
|
||||
data (map (fn [datom]
|
||||
[:db/add (:e datom) (:a datom) (:v datom)])
|
||||
datoms)]
|
||||
(d/transact! conn data {:initial-db? true})))
|
||||
client-ops-conn (when-not @*publishing? (common-sqlite/get-storage-conn
|
||||
client-ops-storage
|
||||
client-op/schema-in-db))
|
||||
initial-data-exists? (when (nil? datoms)
|
||||
(and (d/entity @conn :logseq.class/Root)
|
||||
(= "db" (:kv/value (d/entity @conn :logseq.kv/db-type)))))]
|
||||
(swap! *datascript-conns assoc repo conn)
|
||||
(swap! *client-ops-conns assoc repo client-ops-conn)
|
||||
(when (and (not @*publishing?) (not= client-op/schema-in-db (d/schema @client-ops-conn)))
|
||||
(d/reset-schema! client-ops-conn client-op/schema-in-db))
|
||||
(when (and db-based? (not initial-data-exists?) (not datoms))
|
||||
(let [config (or config "")
|
||||
initial-data (sqlite-create-graph/build-db-initial-data
|
||||
config (select-keys opts [:import-type :graph-git-sha]))]
|
||||
(ldb/transact! conn initial-data {:initial-db? true})))
|
||||
|
||||
(gc-sqlite-dbs! db client-ops-db debug-log-db conn {})
|
||||
|
||||
(let [migration-result (db-migrate/migrate conn)]
|
||||
(when (client-op/rtc-db-graph? repo)
|
||||
(let [client-ops (rtc-migrate/migration-results=>client-ops migration-result)]
|
||||
(client-op/add-ops! repo client-ops))))
|
||||
|
||||
(db-listener/listen-db-changes! repo (get @*datascript-conns repo))))))
|
||||
|
||||
(defn- <list-all-dbs
|
||||
[]
|
||||
(p/let [storage (platform/storage (platform/current))
|
||||
graph-names ((:list-graphs storage))]
|
||||
(p/all (map (fn [graph-name]
|
||||
(p/let [repo (str sqlite-util/db-version-prefix graph-name)
|
||||
metadata (worker-db-metadata/<get repo)]
|
||||
{:name graph-name
|
||||
:metadata (edn/read-string metadata)}))
|
||||
graph-names))))
|
||||
|
||||
(def-thread-api :thread-api/list-db
|
||||
[]
|
||||
(<list-all-dbs))
|
||||
|
||||
(defn- <db-exists?
|
||||
[graph]
|
||||
(let [storage (platform/storage (platform/current))]
|
||||
((:db-exists? storage) graph)))
|
||||
|
||||
(defn- remove-vfs!
|
||||
[^js pool]
|
||||
(when pool
|
||||
(let [storage (platform/storage (platform/current))]
|
||||
((:remove-vfs! storage) pool))))
|
||||
|
||||
(defn- get-search-db
|
||||
[repo]
|
||||
(worker-state/get-sqlite-conn repo :search))
|
||||
|
||||
(comment
|
||||
(def-thread-api :thread-api/get-version
|
||||
[]
|
||||
(when-let [sqlite @*sqlite]
|
||||
(.-version sqlite))))
|
||||
|
||||
(def-thread-api :thread-api/init
|
||||
[rtc-ws-url]
|
||||
(reset! worker-state/*rtc-ws-url rtc-ws-url)
|
||||
(init-sqlite-module!))
|
||||
|
||||
(def-thread-api :thread-api/set-infer-worker-proxy
|
||||
[infer-worker-proxy]
|
||||
(reset! worker-state/*infer-worker infer-worker-proxy)
|
||||
nil)
|
||||
|
||||
;; [graph service]
|
||||
(defonce *service (atom []))
|
||||
|
||||
(defonce fns {"remoteInvoke" thread-api/remote-function})
|
||||
|
||||
(defn- start-db!
|
||||
[repo {:keys [close-other-db?]
|
||||
:or {close-other-db? true}
|
||||
:as opts}]
|
||||
(p/do!
|
||||
(when close-other-db?
|
||||
(close-other-dbs! repo))
|
||||
(when @shared-service/*master-client?
|
||||
(<create-or-open-db! repo (dissoc opts :close-other-db?)))
|
||||
nil))
|
||||
|
||||
(def-thread-api :thread-api/create-or-open-db
|
||||
[repo opts]
|
||||
(when-not (= repo (worker-state/get-current-repo)) ; graph switched
|
||||
(reset! worker-state/*deleted-block-uuid->db-id {}))
|
||||
(start-db! repo opts))
|
||||
|
||||
(def-thread-api :thread-api/q
|
||||
[repo inputs]
|
||||
(when-let [conn (worker-state/get-datascript-conn repo)]
|
||||
(worker-util/profile
|
||||
(str "Datalog query: " inputs)
|
||||
(apply d/q (first inputs) @conn (rest inputs)))))
|
||||
|
||||
(def-thread-api :thread-api/datoms
|
||||
[repo & args]
|
||||
(when-let [conn (worker-state/get-datascript-conn repo)]
|
||||
(let [result (apply d/datoms @conn args)]
|
||||
(map (fn [d] [(:e d) (:a d) (:v d) (:tx d) (:added d)]) result))))
|
||||
|
||||
(def-thread-api :thread-api/pull
|
||||
[repo selector id]
|
||||
(when-let [conn (worker-state/get-datascript-conn repo)]
|
||||
(let [eid (if (and (vector? id) (= :block/name (first id)))
|
||||
(:db/id (ldb/get-page @conn (second id)))
|
||||
id)]
|
||||
(some->> eid
|
||||
(d/pull @conn selector)
|
||||
(common-initial-data/with-parent @conn)))))
|
||||
|
||||
(def ^:private *get-blocks-cache (volatile! (cache/lru-cache-factory {} :threshold 1000)))
|
||||
(def ^:private get-blocks-with-cache
|
||||
(common.cache/cache-fn
|
||||
*get-blocks-cache
|
||||
(fn [repo requests]
|
||||
(let [db (some-> (worker-state/get-datascript-conn repo) deref)]
|
||||
[[repo (:max-tx db) requests]
|
||||
[db requests]]))
|
||||
(fn [db requests]
|
||||
(when db
|
||||
(->> requests
|
||||
(mapv (fn [{:keys [id opts]}]
|
||||
(let [id' (if (and (string? id) (common-util/uuid-string? id)) (uuid id) id)]
|
||||
(-> (common-initial-data/get-block-and-children db id' opts)
|
||||
(assoc :id id)))))
|
||||
ldb/write-transit-str)))))
|
||||
|
||||
(def-thread-api :thread-api/get-blocks
|
||||
[repo requests]
|
||||
(let [requests (ldb/read-transit-str requests)]
|
||||
(get-blocks-with-cache repo requests)))
|
||||
|
||||
(def-thread-api :thread-api/get-block-refs
|
||||
[repo id]
|
||||
(when-let [conn (worker-state/get-datascript-conn repo)]
|
||||
(->> (db-reference/get-linked-references @conn id)
|
||||
:ref-blocks
|
||||
(map (fn [b] (assoc (into {} b) :db/id (:db/id b)))))))
|
||||
|
||||
(def-thread-api :thread-api/get-block-refs-count
|
||||
[repo id]
|
||||
(when-let [conn (worker-state/get-datascript-conn repo)]
|
||||
(ldb/get-block-refs-count @conn id)))
|
||||
|
||||
(def-thread-api :thread-api/get-block-source
|
||||
[repo id]
|
||||
(when-let [conn (worker-state/get-datascript-conn repo)]
|
||||
(:db/id (first (:block/_alias (d/entity @conn id))))))
|
||||
|
||||
(defn- search-blocks
|
||||
[repo q option]
|
||||
(let [search-db (get-search-db repo)
|
||||
conn (worker-state/get-datascript-conn repo)]
|
||||
(search/search-blocks repo conn search-db q option)))
|
||||
|
||||
(def-thread-api :thread-api/block-refs-check
|
||||
[repo id {:keys [unlinked?]}]
|
||||
(m/sp
|
||||
(when-let [conn (worker-state/get-datascript-conn repo)]
|
||||
(let [db @conn
|
||||
block (d/entity db id)]
|
||||
(if unlinked?
|
||||
(let [title (string/lower-case (:block/title block))
|
||||
result (m/? (search-blocks repo title {:limit 100}))]
|
||||
(boolean (some (fn [b]
|
||||
(let [block (d/entity db (:db/id b))]
|
||||
(and (not= id (:db/id block))
|
||||
(not ((set (map :db/id (:block/refs block))) id))
|
||||
(string/includes? (string/lower-case (:block/title block)) title)))) result)))
|
||||
(some? (first (common-initial-data/get-block-refs db (:db/id block)))))))))
|
||||
|
||||
(def-thread-api :thread-api/get-block-parents
|
||||
[repo id depth]
|
||||
(when-let [conn (worker-state/get-datascript-conn repo)]
|
||||
(let [block-id (:block/uuid (d/entity @conn id))]
|
||||
(->> (ldb/get-block-parents @conn block-id {:depth (or depth 3)})
|
||||
(map (fn [b] (d/pull @conn '[*] (:db/id b))))))))
|
||||
|
||||
(def-thread-api :thread-api/set-context
|
||||
[context]
|
||||
(when context (worker-state/update-context! context))
|
||||
nil)
|
||||
|
||||
(def-thread-api :thread-api/transact
|
||||
[repo tx-data tx-meta context]
|
||||
(assert (some? repo))
|
||||
(worker-state/set-db-latest-tx-time! repo)
|
||||
(let [conn (worker-state/get-datascript-conn repo)]
|
||||
(assert (some? conn) {:repo repo})
|
||||
(try
|
||||
(let [tx-data' (if (contains? #{:insert-blocks} (:outliner-op tx-meta))
|
||||
(map (fn [m]
|
||||
(if (and (map? m) (nil? (:block/order m)))
|
||||
(assoc m :block/order (db-order/gen-key nil))
|
||||
m)) tx-data)
|
||||
tx-data)
|
||||
_ (when context (worker-state/set-context! context))
|
||||
tx-meta' (cond-> tx-meta
|
||||
(and (not (:whiteboard/transact? tx-meta))
|
||||
(not (:rtc-download-graph? tx-meta))) ; delay writes to the disk
|
||||
(assoc :skip-store? true)
|
||||
|
||||
true
|
||||
(dissoc :insert-blocks?))]
|
||||
(when-not (and (:create-today-journal? tx-meta)
|
||||
(:today-journal-name tx-meta)
|
||||
(seq tx-data')
|
||||
(ldb/get-page @conn (:today-journal-name tx-meta))) ; today journal created already
|
||||
|
||||
;; (prn :debug :transact :tx-data tx-data' :tx-meta tx-meta')
|
||||
|
||||
(worker-util/profile "Worker db transact"
|
||||
(ldb/transact! conn tx-data' tx-meta')))
|
||||
nil)
|
||||
(catch :default e
|
||||
(prn :debug :worker-transact-failed :tx-meta tx-meta :tx-data tx-data)
|
||||
(log/error ::worker-transact-failed e)
|
||||
(throw e)))))
|
||||
|
||||
(def-thread-api :thread-api/get-initial-data
|
||||
[repo opts]
|
||||
(when-let [conn (worker-state/get-datascript-conn repo)]
|
||||
(if (:file-graph-import? opts)
|
||||
{:schema (:schema @conn)
|
||||
:initial-data (vec (d/datoms @conn :eavt))}
|
||||
(common-initial-data/get-initial-data @conn))))
|
||||
|
||||
(def-thread-api :thread-api/reset-db
|
||||
[repo db-transit]
|
||||
(reset-db! repo db-transit)
|
||||
nil)
|
||||
|
||||
(def-thread-api :thread-api/unsafe-unlink-db
|
||||
[repo]
|
||||
(p/let [pool (<get-opfs-pool repo)
|
||||
_ (close-db! repo)
|
||||
_result (remove-vfs! pool)]
|
||||
nil))
|
||||
|
||||
(def-thread-api :thread-api/release-access-handles
|
||||
[repo]
|
||||
(when-let [^js pool (worker-state/get-opfs-pool repo)]
|
||||
(.pauseVfs pool)
|
||||
nil))
|
||||
|
||||
(def-thread-api :thread-api/db-exists
|
||||
[repo]
|
||||
(<db-exists? repo))
|
||||
|
||||
(def-thread-api :thread-api/export-db
|
||||
[repo]
|
||||
(when-let [^js db (worker-state/get-sqlite-conn repo :db)]
|
||||
(.exec db "PRAGMA wal_checkpoint(2)"))
|
||||
(p/let [data (<export-db-file repo)]
|
||||
(platform/transfer (platform/current) data #js [(.-buffer data)])))
|
||||
|
||||
(def-thread-api :thread-api/export-debug-log-db
|
||||
[repo]
|
||||
(when-let [^js db (worker-state/get-sqlite-conn repo :debug-log)]
|
||||
(.exec db "PRAGMA wal_checkpoint(2)"))
|
||||
(-> (p/let [data (<export-db-file
|
||||
repo
|
||||
debug-log-path)]
|
||||
(when data
|
||||
(platform/transfer (platform/current) data #js [(.-buffer data)])))
|
||||
(p/catch (fn [error]
|
||||
(throw error)))))
|
||||
|
||||
(def-thread-api :thread-api/reset-debug-log-db
|
||||
[repo]
|
||||
(when-let [^js db (worker-state/get-sqlite-conn repo :debug-log)]
|
||||
(rtc-debug-log/reset-tables! db)))
|
||||
|
||||
(def-thread-api :thread-api/import-db
|
||||
[repo data]
|
||||
(when-not (string/blank? repo)
|
||||
(p/let [pool (<get-opfs-pool repo)]
|
||||
(<import-db pool data)
|
||||
nil)))
|
||||
|
||||
(def-thread-api :thread-api/search-blocks
|
||||
[repo q option]
|
||||
(search-blocks repo q option))
|
||||
|
||||
(def-thread-api :thread-api/search-upsert-blocks
|
||||
[repo blocks]
|
||||
(p/let [db (get-search-db repo)]
|
||||
(search/upsert-blocks! db (bean/->js blocks))
|
||||
nil))
|
||||
|
||||
(def-thread-api :thread-api/search-delete-blocks
|
||||
[repo ids]
|
||||
(p/let [db (get-search-db repo)]
|
||||
(search/delete-blocks! db ids)
|
||||
nil))
|
||||
|
||||
(def-thread-api :thread-api/search-truncate-tables
|
||||
[repo]
|
||||
(p/let [db (get-search-db repo)]
|
||||
(search/truncate-table! db)
|
||||
nil))
|
||||
|
||||
(def-thread-api :thread-api/search-build-blocks-indice
|
||||
[repo]
|
||||
(when-let [conn (worker-state/get-datascript-conn repo)]
|
||||
(search/build-blocks-indice repo @conn)))
|
||||
|
||||
(def-thread-api :thread-api/search-build-pages-indice
|
||||
[_repo]
|
||||
nil)
|
||||
|
||||
(def-thread-api :thread-api/apply-outliner-ops
|
||||
[repo ops opts]
|
||||
(when-let [conn (worker-state/get-datascript-conn repo)]
|
||||
(try
|
||||
(worker-util/profile
|
||||
"apply outliner ops"
|
||||
(outliner-op/apply-ops! conn ops opts))
|
||||
(catch :default e
|
||||
(let [data (ex-data e)
|
||||
{:keys [type payload]} (when (map? data) data)]
|
||||
(case type
|
||||
:notification
|
||||
(do
|
||||
(log/error ::apply-outliner-ops-failed e)
|
||||
(shared-service/broadcast-to-clients! :notification [(:message payload) (:type payload) (:clear? payload) (:uid payload) (:timeout payload)]))
|
||||
(throw e)))))))
|
||||
|
||||
(def-thread-api :thread-api/sync-app-state
|
||||
[new-state]
|
||||
(when (and (contains? new-state :git/current-repo)
|
||||
(nil? (:git/current-repo new-state)))
|
||||
(log/error :thread-api/sync-app-state new-state))
|
||||
(worker-state/set-new-state! new-state)
|
||||
nil)
|
||||
|
||||
(def-thread-api :thread-api/export-get-debug-datoms
|
||||
[repo]
|
||||
(when-let [conn (worker-state/get-datascript-conn repo)]
|
||||
(worker-export/get-debug-datoms conn)))
|
||||
|
||||
(def-thread-api :thread-api/export-get-all-pages
|
||||
[repo]
|
||||
(when-let [conn (worker-state/get-datascript-conn repo)]
|
||||
(worker-export/get-all-pages repo @conn)))
|
||||
|
||||
(def-thread-api :thread-api/export-get-all-page->content
|
||||
[repo options]
|
||||
(when-let [conn (worker-state/get-datascript-conn repo)]
|
||||
(worker-export/get-all-page->content repo @conn options)))
|
||||
|
||||
(def-thread-api :thread-api/validate-db
|
||||
[repo]
|
||||
(when-let [conn (worker-state/get-datascript-conn repo)]
|
||||
(worker-db-validate/validate-db conn)))
|
||||
|
||||
;; Returns an export-edn map for given repo. When there's an unexpected error, a map
|
||||
;; with key :export-edn-error is returned
|
||||
(def-thread-api :thread-api/export-edn
|
||||
[repo options]
|
||||
(let [conn (worker-state/get-datascript-conn repo)]
|
||||
(try
|
||||
(sqlite-export/build-export @conn options)
|
||||
(catch :default e
|
||||
(js/console.error "export-edn error: " e)
|
||||
(js/console.error "Stack:\n" (.-stack e))
|
||||
(platform/post-message! (platform/current)
|
||||
:notification
|
||||
["An unexpected error occurred during export. See the javascript console for details."
|
||||
:error])
|
||||
{:export-edn-error (.-message e)}))))
|
||||
|
||||
(def-thread-api :thread-api/get-view-data
|
||||
[repo view-id option]
|
||||
(let [db @(worker-state/get-datascript-conn repo)]
|
||||
(db-view/get-view-data db view-id option)))
|
||||
|
||||
(def-thread-api :thread-api/get-class-objects
|
||||
[repo class-id]
|
||||
(let [db @(worker-state/get-datascript-conn repo)]
|
||||
(->> (db-class/get-class-objects db class-id)
|
||||
(map common-entity-util/entity->map))))
|
||||
|
||||
(def-thread-api :thread-api/get-property-values
|
||||
[repo {:keys [property-ident] :as option}]
|
||||
(let [conn (worker-state/get-datascript-conn repo)]
|
||||
(db-view/get-property-values @conn property-ident option)))
|
||||
|
||||
(def-thread-api :thread-api/build-graph
|
||||
[repo option]
|
||||
(let [conn (worker-state/get-datascript-conn repo)]
|
||||
(graph-view/build-graph @conn option)))
|
||||
|
||||
(def ^:private *get-all-page-titles-cache (volatile! (cache/lru-cache-factory {})))
|
||||
(defn- get-all-page-titles
|
||||
[db]
|
||||
(let [pages (ldb/get-all-pages db)]
|
||||
(sort (map :block/title pages))))
|
||||
|
||||
(def ^:private get-all-page-titles-with-cache
|
||||
(common.cache/cache-fn
|
||||
*get-all-page-titles-cache
|
||||
(fn [repo]
|
||||
(let [db @(worker-state/get-datascript-conn repo)]
|
||||
[[repo (:max-tx db)] ;cache-key
|
||||
[db] ;f-args
|
||||
]))
|
||||
get-all-page-titles))
|
||||
|
||||
(def-thread-api :thread-api/get-all-page-titles
|
||||
[repo]
|
||||
(get-all-page-titles-with-cache repo))
|
||||
|
||||
(def-thread-api :thread-api/gc-graph
|
||||
[repo]
|
||||
(let [{:keys [db client-ops debug-log]} (get @*sqlite-conns repo)
|
||||
conn (get @*datascript-conns repo)]
|
||||
(when (and db conn)
|
||||
(gc-sqlite-dbs! db client-ops debug-log conn {:full-gc? true})
|
||||
nil)))
|
||||
|
||||
(def-thread-api :thread-api/vec-search-embedding-model-info
|
||||
[repo]
|
||||
(embedding/task--embedding-model-info repo))
|
||||
|
||||
(def-thread-api :thread-api/vec-search-init-embedding-model
|
||||
[repo]
|
||||
(js/Promise. (embedding/task--init-embedding-model repo)))
|
||||
|
||||
(def-thread-api :thread-api/vec-search-load-model
|
||||
[repo model-name]
|
||||
(js/Promise. (embedding/task--load-model repo model-name)))
|
||||
|
||||
(def-thread-api :thread-api/vec-search-embedding-graph
|
||||
[repo opts]
|
||||
(embedding/embedding-graph! repo opts))
|
||||
|
||||
(def-thread-api :thread-api/vec-search-search
|
||||
[repo query-string nums-neighbors]
|
||||
(embedding/task--search repo query-string nums-neighbors))
|
||||
|
||||
(def-thread-api :thread-api/vec-search-cancel-indexing
|
||||
[repo]
|
||||
(embedding/cancel-indexing repo))
|
||||
|
||||
(def-thread-api :thread-api/vec-search-update-index-info
|
||||
[repo]
|
||||
(js/Promise. (embedding/task--update-index-info! repo)))
|
||||
|
||||
(def-thread-api :thread-api/mobile-logs
|
||||
[]
|
||||
@worker-state/*log)
|
||||
|
||||
(def-thread-api :thread-api/get-rtc-graph-uuid
|
||||
[repo]
|
||||
(when-let [conn (worker-state/get-datascript-conn repo)]
|
||||
(ldb/get-graph-rtc-uuid @conn)))
|
||||
|
||||
(def-thread-api :thread-api/api-get-page-data
|
||||
[repo page-title]
|
||||
(let [conn (worker-state/get-datascript-conn repo)]
|
||||
(cli-common-mcp-tools/get-page-data @conn page-title)))
|
||||
|
||||
(def-thread-api :thread-api/api-list-properties
|
||||
[repo options]
|
||||
(let [conn (worker-state/get-datascript-conn repo)]
|
||||
(cli-common-mcp-tools/list-properties @conn options)))
|
||||
|
||||
(def-thread-api :thread-api/api-list-tags
|
||||
[repo options]
|
||||
(let [conn (worker-state/get-datascript-conn repo)]
|
||||
(cli-common-mcp-tools/list-tags @conn options)))
|
||||
|
||||
(def-thread-api :thread-api/api-list-pages
|
||||
[repo options]
|
||||
(let [conn (worker-state/get-datascript-conn repo)]
|
||||
(cli-common-mcp-tools/list-pages @conn options)))
|
||||
|
||||
(def-thread-api :thread-api/api-build-upsert-nodes-edn
|
||||
[repo ops]
|
||||
(let [conn (worker-state/get-datascript-conn repo)]
|
||||
(cli-common-mcp-tools/build-upsert-nodes-edn @conn ops)))
|
||||
|
||||
(comment
|
||||
(def-thread-api :general/dangerousRemoveAllDbs
|
||||
[]
|
||||
(p/let [r (<list-all-dbs)
|
||||
dbs (ldb/read-transit-str r)]
|
||||
(p/all (map #(.unsafeUnlinkDB this (:name %)) dbs)))))
|
||||
|
||||
(defn- delete-page!
|
||||
[conn page-uuid]
|
||||
(let [error-handler (fn [{:keys [msg]}]
|
||||
(platform/post-message! (platform/current)
|
||||
:notification
|
||||
[[:div [:p msg]] :error]))]
|
||||
(worker-page/delete! conn page-uuid {:error-handler error-handler})))
|
||||
|
||||
(defn- create-page!
|
||||
[conn title options]
|
||||
(try
|
||||
(worker-page/create! conn title options)
|
||||
(catch :default e
|
||||
(js/console.error e)
|
||||
(throw e))))
|
||||
|
||||
(defn- outliner-register-op-handlers!
|
||||
[]
|
||||
(outliner-op/register-op-handlers!
|
||||
{:create-page (fn [conn [title options]]
|
||||
(create-page! conn title options))
|
||||
:rename-page (fn [conn [page-uuid new-title]]
|
||||
(if (string/blank? new-title)
|
||||
(throw (ex-info "Page name shouldn't be blank" {:block/uuid page-uuid
|
||||
:block/title new-title}))
|
||||
(outliner-core/save-block! conn
|
||||
{:block/uuid page-uuid
|
||||
:block/title new-title})))
|
||||
:delete-page (fn [conn [page-uuid]]
|
||||
(delete-page! conn page-uuid))}))
|
||||
|
||||
(defn- on-become-master
|
||||
[repo start-opts]
|
||||
(js/Promise.
|
||||
(m/sp
|
||||
(c.m/<? (init-sqlite-module!))
|
||||
(when-not (:import-type start-opts)
|
||||
(c.m/<? (start-db! repo start-opts))
|
||||
(assert (some? (worker-state/get-datascript-conn repo))))
|
||||
;; Don't wait for rtc started because the app will be slow to be ready
|
||||
;; for users.
|
||||
(when @worker-state/*rtc-ws-url
|
||||
(rtc.core/new-task--rtc-start true)))))
|
||||
|
||||
(def broadcast-data-types
|
||||
(set (map
|
||||
common-util/keyword->string
|
||||
[:sync-db-changes
|
||||
:notification
|
||||
:log
|
||||
:add-repo
|
||||
:rtc-log
|
||||
:rtc-sync-state])))
|
||||
|
||||
(defn- <init-service!
|
||||
[graph start-opts]
|
||||
(let [[prev-graph service] @*service]
|
||||
(some-> prev-graph close-db!)
|
||||
(when graph
|
||||
(if (= graph prev-graph)
|
||||
service
|
||||
(p/let [service (shared-service/<create-service graph
|
||||
(bean/->js fns)
|
||||
#(on-become-master graph start-opts)
|
||||
broadcast-data-types
|
||||
{:import? (:import-type? start-opts)})]
|
||||
(assert (p/promise? (get-in service [:status :ready])))
|
||||
(reset! *service [graph service])
|
||||
service)))))
|
||||
|
||||
(defn- notify-invalid-data
|
||||
[{:keys [tx-meta]} errors]
|
||||
;; don't notify on production when undo/redo failed
|
||||
(when-not (and (or (:undo? tx-meta) (:redo? tx-meta))
|
||||
(not worker-util/dev?))
|
||||
(shared-service/broadcast-to-clients! :notification
|
||||
[["Invalid data writing to db!"] :error])
|
||||
(platform/post-message! (platform/current)
|
||||
:capture-error
|
||||
{:error (ex-info "Invalid data writing to db" tx-meta)
|
||||
:payload {}
|
||||
:extra {:errors (str errors)
|
||||
:tx-meta tx-meta}})))
|
||||
|
||||
(defn- build-proxy-object
|
||||
[]
|
||||
(->>
|
||||
fns
|
||||
(map
|
||||
(fn [[k f]]
|
||||
[k
|
||||
(fn [& args]
|
||||
(let [[_graph service] @*service
|
||||
method-k (keyword (first args))]
|
||||
(cond
|
||||
(= :thread-api/create-or-open-db method-k)
|
||||
;; because shared-service operates at the graph level,
|
||||
;; creating a new database or switching to another one requires re-initializing the service.
|
||||
(let [[graph opts] (ldb/read-transit-str (last args))]
|
||||
(p/let [service (<init-service! graph opts)
|
||||
client-id (:client-id service)]
|
||||
(when client-id
|
||||
(platform/post-message! (platform/current)
|
||||
:record-worker-client-id
|
||||
{:client-id client-id}))
|
||||
(get-in service [:status :ready])
|
||||
;; wait for service ready
|
||||
(js-invoke (:proxy service) k args)))
|
||||
|
||||
(or
|
||||
(contains? #{:thread-api/set-infer-worker-proxy :thread-api/sync-app-state} method-k)
|
||||
(nil? service))
|
||||
;; only proceed down this branch before shared-service is initialized
|
||||
(apply f args)
|
||||
|
||||
:else
|
||||
;; ensure service is ready
|
||||
(p/let [_ready-value (get-in service [:status :ready])]
|
||||
(js-invoke (:proxy service) k args)))))]))
|
||||
(into {})
|
||||
bean/->js))
|
||||
|
||||
(defn init-core!
|
||||
[platform']
|
||||
(platform/set-platform! platform')
|
||||
(ldb/register-transact-invalid-callback-fn! notify-invalid-data)
|
||||
(outliner-register-op-handlers!)
|
||||
(build-proxy-object))
|
||||
|
||||
(comment
|
||||
(defn <remove-all-files!
|
||||
"!! Dangerous: use it only for development."
|
||||
[]
|
||||
(p/let [all-files (<list-all-files)
|
||||
files (filter #(= (.-kind %) "file") all-files)
|
||||
dirs (filter #(= (.-kind %) "directory") all-files)
|
||||
_ (p/all (map (fn [file] (.remove file)) files))]
|
||||
(p/all (map (fn [dir] (.remove dir)) dirs)))))
|
||||
File diff suppressed because it is too large
Load Diff
74
src/main/frontend/worker/platform.cljs
Normal file
74
src/main/frontend/worker/platform.cljs
Normal file
@@ -0,0 +1,74 @@
|
||||
(ns frontend.worker.platform
|
||||
"Platform adapter contract for db-worker runtimes.")
|
||||
|
||||
(def ^:private required-sections
|
||||
[:env :storage :kv :broadcast :websocket :crypto :timers])
|
||||
|
||||
(defonce ^:private *platform (atom nil))
|
||||
|
||||
(defn validate-platform!
|
||||
[platform]
|
||||
(doseq [section required-sections]
|
||||
(when-not (contains? platform section)
|
||||
(throw (ex-info (str "platform adapter missing section: " section)
|
||||
{:section section
|
||||
:platform-keys (keys platform)}))))
|
||||
platform)
|
||||
|
||||
(defn set-platform!
|
||||
[platform]
|
||||
(reset! *platform (validate-platform! platform)))
|
||||
|
||||
(defn current
|
||||
[]
|
||||
(or @*platform
|
||||
(throw (ex-info "platform adapter not initialized" {}))))
|
||||
|
||||
(defn env-flag
|
||||
[platform flag]
|
||||
(get-in platform [:env flag]))
|
||||
|
||||
(defn storage
|
||||
[platform]
|
||||
(:storage platform))
|
||||
|
||||
(defn kv-get
|
||||
[platform k]
|
||||
(if-let [f (get-in platform [:kv :get])]
|
||||
(f k)
|
||||
(throw (ex-info "platform kv/get missing" {:key k}))))
|
||||
|
||||
(defn kv-set!
|
||||
[platform k value]
|
||||
(if-let [f (get-in platform [:kv :set!])]
|
||||
(f k value)
|
||||
(throw (ex-info "platform kv/set! missing" {:key k}))))
|
||||
|
||||
(defn read-text!
|
||||
[platform path]
|
||||
(if-let [f (get-in platform [:storage :read-text!])]
|
||||
(f path)
|
||||
(throw (ex-info "platform storage/read-text! missing" {:path path}))))
|
||||
|
||||
(defn write-text!
|
||||
[platform path text]
|
||||
(if-let [f (get-in platform [:storage :write-text!])]
|
||||
(f path text)
|
||||
(throw (ex-info "platform storage/write-text! missing" {:path path}))))
|
||||
|
||||
(defn websocket-connect
|
||||
[platform url]
|
||||
(if-let [f (get-in platform [:websocket :connect])]
|
||||
(f url)
|
||||
(throw (ex-info "platform websocket/connect missing" {:url url}))))
|
||||
|
||||
(defn post-message!
|
||||
[platform type payload]
|
||||
(when-let [f (get-in platform [:broadcast :post-message!])]
|
||||
(f type payload)))
|
||||
|
||||
(defn transfer
|
||||
[platform data transferables]
|
||||
(if-let [f (get-in platform [:storage :transfer])]
|
||||
(f data transferables)
|
||||
data))
|
||||
111
src/main/frontend/worker/platform/browser.cljs
Normal file
111
src/main/frontend/worker/platform/browser.cljs
Normal file
@@ -0,0 +1,111 @@
|
||||
(ns frontend.worker.platform.browser
|
||||
"Browser platform adapter for db-worker."
|
||||
(:require ["/frontend/idbkv" :as idb-keyval]
|
||||
["comlink" :as Comlink]
|
||||
[clojure.string :as string]
|
||||
[frontend.common.file.opfs :as opfs]
|
||||
[frontend.worker-common.util :as worker-util]
|
||||
[lambdaisland.glogi :as log]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(defn- iter->vec
|
||||
[iter']
|
||||
(when iter'
|
||||
(p/loop [acc []]
|
||||
(p/let [elem (.next iter')]
|
||||
(if (.-done elem)
|
||||
acc
|
||||
(p/recur (conj acc (.-value elem))))))))
|
||||
|
||||
(defn- list-graphs
|
||||
[]
|
||||
(let [dir? #(= (.-kind %) "directory")
|
||||
db-dir-prefix ".logseq-pool-"]
|
||||
(p/let [^js root (.getDirectory js/navigator.storage)
|
||||
values-iter (when (dir? root) (.values root))
|
||||
values (when values-iter (iter->vec values-iter))
|
||||
current-dir-dirs (filter dir? values)
|
||||
db-dirs (filter (fn [file]
|
||||
(string/starts-with? (.-name file) db-dir-prefix))
|
||||
current-dir-dirs)
|
||||
graph-names (map (fn [dir]
|
||||
(-> (.-name dir)
|
||||
(string/replace-first ".logseq-pool-" "")
|
||||
;; TODO: DRY
|
||||
(string/replace "+3A+" ":")
|
||||
(string/replace "++" "/")))
|
||||
db-dirs)]
|
||||
(log/info :db-dirs (map #(.-name %) db-dirs) :all-dirs (map #(.-name %) current-dir-dirs))
|
||||
(vec graph-names))))
|
||||
|
||||
(defn- db-exists?
|
||||
[graph]
|
||||
(->
|
||||
(p/let [^js root (.getDirectory js/navigator.storage)
|
||||
_dir-handle (.getDirectoryHandle root (str "." (worker-util/get-pool-name graph)))]
|
||||
true)
|
||||
(p/catch
|
||||
(fn [_e] ; not found
|
||||
false))))
|
||||
|
||||
(defonce ^:private kv-store
|
||||
(delay (idb-keyval/newStore "localforage" "keyvaluepairs" 2)))
|
||||
|
||||
(defn- kv-get
|
||||
[k]
|
||||
(idb-keyval/get k @kv-store))
|
||||
|
||||
(defn- kv-set!
|
||||
[k value]
|
||||
(idb-keyval/set k value @kv-store))
|
||||
|
||||
(defn- install-opfs-pool
|
||||
[sqlite pool-name]
|
||||
(.installOpfsSAHPoolVfs ^js sqlite #js {:name pool-name
|
||||
:initialCapacity 20}))
|
||||
|
||||
(defn- export-file
|
||||
[pool path]
|
||||
(.exportFile ^js pool path))
|
||||
|
||||
(defn- import-db
|
||||
[pool path data]
|
||||
(.importDb ^js pool path data))
|
||||
|
||||
(defn- remove-vfs!
|
||||
[pool]
|
||||
(when pool
|
||||
(.removeVfs ^js pool)))
|
||||
|
||||
(defn- read-text!
|
||||
[path]
|
||||
(opfs/<read-text! path))
|
||||
|
||||
(defn- write-text!
|
||||
[path text]
|
||||
(opfs/<write-text! path text))
|
||||
|
||||
(defn- websocket-connect
|
||||
[url]
|
||||
(js/WebSocket. url))
|
||||
|
||||
(defn browser-platform
|
||||
[]
|
||||
{:env {:publishing? (string/includes? (.. js/location -href) "publishing=true")
|
||||
:runtime :browser}
|
||||
:storage {:install-opfs-pool install-opfs-pool
|
||||
:list-graphs list-graphs
|
||||
:db-exists? db-exists?
|
||||
:export-file export-file
|
||||
:import-db import-db
|
||||
:remove-vfs! remove-vfs!
|
||||
:read-text! read-text!
|
||||
:write-text! write-text!
|
||||
:transfer (fn [data transferables]
|
||||
(Comlink/transfer data transferables))}
|
||||
:kv {:get kv-get
|
||||
:set! kv-set!}
|
||||
:broadcast {:post-message! worker-util/post-message}
|
||||
:websocket {:connect websocket-connect}
|
||||
:crypto {}
|
||||
:timers {:set-interval! (fn [f ms] (js/setInterval f ms))}})
|
||||
Reference in New Issue
Block a user