clone/logseq
1
0
Fork 0
mirror of https://github.com/logseq/logseq.git synced 2026-05-21 11:22:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/feat/db' into feat/hnswlib+transformer-js

Browse Source
This commit is contained in:
rcmerci
2025-03-21 15:42:50 +08:00
parent 1f31f08bf6 1565e7541d
commit ca558450c1
19 changed files with 129 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bb.edn
View File

@@ -75,7 +75,9 @@
dev:db-transact
{:doc "Transact against a DB graph's datascript db"
:task (apply shell {:dir "deps/outliner"} "yarn -s nbb-logseq script/transact.cljs" *command-line-args*)}
:requires ([babashka.fs :as fs])
:task (apply shell {:dir "deps/outliner" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
"yarn -s nbb-logseq script/transact.cljs" *command-line-args*)}
dev:db-create
{:doc "Create a DB graph given a sqlite.build EDN file"

15
deps/common/src/logseq/common/uuid.cljs vendored
View File

@@ -12,7 +12,7 @@ the remaining chars for data of this type"
(let [journal-day-str (str journal-day)
part1 (subs journal-day-str 0 4)
part2 (subs journal-day-str 4 8)]
(uuid (str "00000001-" part1 "-" part2 "-0000-000000000000"))))
(uuid (str "00000001" "-" part1 "-" part2 "-0000-000000000000"))))
(defn- fill-with-0
[s length]
@@ -49,3 +49,16 @@ the remaining chars for data of this type"
:db-ident-block-uuid (gen-db-ident-block-uuid v)
:migrate-new-block-uuid (gen-block-uuid v "00000003")
:builtin-block-uuid (gen-block-uuid v "00000004"))))
(defn gen-journal-template-block
"Persistent uuid for journal template block"
[journal-uuid template-block-uuid]
(assert (uuid? journal-uuid) (str journal-uuid))
(assert (uuid? template-block-uuid) (str template-block-uuid))
(uuid
(str "00000005"
"-"
;; journal day
(subs (str journal-uuid) 9 23)
;; template block uuid
(subs (str template-block-uuid) 23))))

15
deps/db/script/create_graph.cljs vendored
View File

@@ -20,6 +20,17 @@
path
(node-path/join (or js/process.env.ORIGINAL_PWD ".") path)))
(defn- get-dir-and-db-name
"Gets dir and db name for use with open-db! Works for relative and absolute paths and
defaults to ~/logseq/graphs/ when no '/' present in name"
[graph-dir]
(if (string/includes? graph-dir "/")
(let [resolve-path' #(if (node-path/isAbsolute %) %
;; $ORIGINAL_PWD used by bb tasks to correct current dir
(node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir)))
[(node-path/join (os/homedir) "logseq" "graphs") graph-dir]))
(def spec
"Options spec"
{:help {:alias :h
@@ -34,9 +45,7 @@
(println (str "Usage: $0 GRAPH-NAME EDN-PATH [OPTIONS]\nOptions:\n"
(cli/format-opts {:spec spec})))
(js/process.exit 1))
[dir db-name] (if (string/includes? graph-dir "/")
((juxt node-path/dirname node-path/basename) graph-dir)
[(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
[dir db-name] (get-dir-and-db-name graph-dir)
sqlite-build-edn (merge {:auto-create-ontology? true}
(-> (resolve-path edn-path) fs/readFileSync str edn/read-string))
conn (outliner-cli/init-conn dir db-name {:classpath (cp/get-classpath) :import-type :cli/create-graph})

10
deps/db/script/query.cljs vendored
View File

@@ -23,12 +23,14 @@
(clj->js (merge {:stdio "inherit"} opts))))
(defn- get-dir-and-db-name
"Gets dir and db name for use with open-db!"
"Gets dir and db name for use with open-db! Works for relative and absolute paths and
defaults to ~/logseq/graphs/ when no '/' present in name"
[graph-dir]
(if (string/includes? graph-dir "/")
(let [graph-dir'
(node-path/join (or js/process.env.ORIGINAL_PWD ".") graph-dir)]
((juxt node-path/dirname node-path/basename) graph-dir'))
(let [resolve-path' #(if (node-path/isAbsolute %) %
;; $ORIGINAL_PWD used by bb tasks to correct current dir
(node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir)))
[(node-path/join (os/homedir) "logseq" "graphs") graph-dir]))
(def spec

17
deps/db/script/validate_db.cljs vendored
View File

@@ -50,6 +50,17 @@
(js/process.exit 1))
(println "Valid!"))))
(defn- get-dir-and-db-name
"Gets dir and db name for use with open-db! Works for relative and absolute paths and
defaults to ~/logseq/graphs/ when no '/' present in name"
[graph-dir]
(if (string/includes? graph-dir "/")
(let [resolve-path' #(if (node-path/isAbsolute %) %
;; $ORIGINAL_PWD used by bb tasks to correct current dir
(node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir)))
[(node-path/join (os/homedir) "logseq" "graphs") graph-dir]))
(def spec
"Options spec"
{:help {:alias :h
@@ -75,11 +86,7 @@
(validate-db* db ent-maps options)))
(defn- validate-graph [graph-dir options]
(let [[dir db-name] (if (string/includes? graph-dir "/")
(let [graph-dir'
(node-path/join (or js/process.env.ORIGINAL_PWD ".") graph-dir)]
((juxt node-path/dirname node-path/basename) graph-dir'))
[(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
(let [[dir db-name] (get-dir-and-db-name graph-dir)
conn (try (sqlite-cli/open-db! dir db-name)
(catch :default e
(println "Error: For graph" (str (pr-str graph-dir) ":") (str e))

16
deps/graph-parser/script/db_import.cljs vendored
View File

@@ -125,6 +125,17 @@
(p/let [_ (gp-exporter/export-doc-files conn files' <read-file doc-options)]
{:import-state (:import-state doc-options)}))))
(defn- get-dir-and-db-name
"Gets dir and db name for use with open-db! Works for relative and absolute paths and
defaults to ~/logseq/graphs/ when no '/' present in name"
[graph-dir]
(if (string/includes? graph-dir "/")
(let [resolve-path' #(if (node-path/isAbsolute %) %
;; $ORIGINAL_PWD used by bb tasks to correct current dir
(node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir)))
[(node-path/join (os/homedir) "logseq" "graphs") graph-dir]))
(def spec
"Options spec"
{:help {:alias :h
@@ -160,10 +171,7 @@
(println (str "Usage: $0 FILE-GRAPH DB-GRAPH [OPTIONS]\nOptions:\n"
(cli/format-opts {:spec spec})))
(js/process.exit 1))
[dir db-name] (if (string/includes? db-graph-dir "/")
(let [graph-dir' (resolve-path db-graph-dir)]
((juxt node-path/dirname node-path/basename) graph-dir'))
[(node-path/join (os/homedir) "logseq" "graphs") db-graph-dir])
[dir db-name] (get-dir-and-db-name db-graph-dir)
file-graph' (resolve-path file-graph)
conn (outliner-cli/init-conn dir db-name {:classpath (cp/get-classpath)})
directory? (.isDirectory (fs/statSync file-graph'))

15
deps/outliner/script/transact.cljs vendored
View File

@@ -10,15 +10,24 @@
[logseq.outliner.db-pipeline :as db-pipeline]
[nbb.core :as nbb]))
(defn- get-dir-and-db-name
"Gets dir and db name for use with open-db! Works for relative and absolute paths and
defaults to ~/logseq/graphs/ when no '/' present in name"
[graph-dir]
(if (string/includes? graph-dir "/")
(let [resolve-path' #(if (node-path/isAbsolute %) %
;; $ORIGINAL_PWD used by bb tasks to correct current dir
(node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir)))
[(node-path/join (os/homedir) "logseq" "graphs") graph-dir]))
(defn -main [args]
(when (< (count args) 3)
(println "Usage: $0 GRAPH-DIR QUERY TRANSACT-FN")
(js/process.exit 1))
(let [[graph-dir query* transact-fn*] args
dry-run? (contains? (set args) "-n")
[dir db-name] (if (string/includes? graph-dir "/")
((juxt node-path/dirname node-path/basename) graph-dir)
[(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
[dir db-name] (get-dir-and-db-name graph-dir)
conn (sqlite-cli/open-db! dir db-name)
;; find blocks to update
query (into (edn/read-string query*) [:in '$ '%]) ;; assumes no :in are in queries

2
e2e-tests/logseq-api.spec.ts
View File

@@ -215,7 +215,7 @@ test('(DB graph): block related apis',
prop1 = await callAPI('get_property', 'map1')
const b1p = await callAPI('get_block_property', b1.uuid, 'map1')
expect(prop1.schema.type).toBe('map')
expect(prop1.type).toBe('map')
expect(b1p).toEqual({a: 1})
// await page.pause()

3
libs/src/LSPlugin.caller.ts
View File

@@ -50,6 +50,7 @@ class LSPluginCaller extends EventEmitter {
}
}
// run in host
async connectToChild() {
if (this._connected) return
@@ -303,7 +304,7 @@ class LSPluginCaller extends EventEmitter {
this._call = async (...args: any) => {
// parent all will get message before handshake
await refChild.call(LSPMSGFn(pl.id), {
refChild.call(LSPMSGFn(pl.id), {
type: args[0],
payload: Object.assign(args[1] || {}, {
$$pid: pl.id,

4
libs/src/LSPlugin.core.ts
View File

@@ -695,7 +695,9 @@ class PluginLocal extends EventEmitter<
options.url = this._resolveResourceFullUrl(options.url, this._localRoot)
// file:// for native
if (!this.isWebPlugin && !options.url.startsWith('file:')) {
if (!this.isWebPlugin &&
!options.url.startsWith('file:') &&
!options.url.startsWith('lsp:')) {
options.url = 'assets://' + options.url
}
}

2
libs/src/LSPlugin.ts
View File

@@ -315,7 +315,7 @@ export type ExternalCommandType =
| 'logseq.ui/toggle-theme'
| 'logseq.ui/toggle-wide-mode'
export type UserProxyTags = 'app' | 'editor' | 'db' | 'git' | 'ui' | 'assets' | 'utils'
export type UserProxyNSTags = 'app' | 'editor' | 'db' | 'git' | 'ui' | 'assets' | 'utils'
export type SearchIndiceInitStatus = boolean
export type SearchBlockItem = {

16
libs/src/LSPlugin.user.ts
View File

@@ -30,7 +30,7 @@ import {
IUserOffHook,
IGitProxy,
IUIProxy,
UserProxyTags,
UserProxyNSTags,
BlockUUID,
BlockEntity,
IDatom,
@@ -734,7 +734,7 @@ export class LSPluginUser
/**
* @internal
*/
_makeUserProxy(target: any, tag?: UserProxyTags) {
_makeUserProxy(target: any, nstag?: UserProxyNSTags) {
const that = this
const caller = this.caller
@@ -744,13 +744,13 @@ export class LSPluginUser
return function (this: any, ...args: any) {
if (origMethod) {
if (args?.length !== 0) args.concat(tag)
if (args?.length !== 0) args.concat(nstag)
const ret = origMethod.apply(that, args)
if (ret !== PROXY_CONTINUE) return ret
}
// Handle hook
if (tag) {
if (nstag) {
const hookMatcher = propKey.toString().match(/^(once|off|on)/i)
if (hookMatcher != null) {
@@ -771,7 +771,7 @@ export class LSPluginUser
opts = args[2]
}
type = `hook:${tag}:${safeSnakeCase(type)}`
type = `hook:${nstag}:${safeSnakeCase(type)}`
caller[f](type, handler)
@@ -796,13 +796,13 @@ export class LSPluginUser
let method = propKey as string
// TODO: refactor api call with the explicit tag
if ((['git', 'ui', 'assets', 'utils'] as UserProxyTags[]).includes(tag)) {
method = tag + '_' + method
if ((['git', 'ui', 'assets', 'utils'] as UserProxyNSTags[]).includes(nstag)) {
method = nstag + '_' + method
}
// Call host
return caller.callAsync(`api:call`, {
tag,
tag: nstag,
method,
args: args,
})

2
src/electron/electron/core.cljs
View File

@@ -72,7 +72,7 @@
(re-find #"(?i)^/[a-zA-Z]:" path))
(callback #js {:path path})
;; assume winwdows unc path
;; assume windows unc path
utils/win32?
(do (logger/debug :resolve-assets-url url)
(callback #js {:path (str "//" path)}))

18
src/main/frontend/common/file/core.cljs
View File

@@ -31,11 +31,16 @@
(defn- transform-content
[repo db {:block/keys [collapsed? format pre-block? title page properties] :as b} level {:keys [heading-to-list?]} context]
(let [block-ref-not-saved? (and (seq (:block/_refs (d/entity db (:db/id b))))
(let [db-based? (sqlite-util/db-based-graph? repo)
block-ref-not-saved? (and (seq (:block/_refs (d/entity db (:db/id b))))
(not (string/includes? title (str (:block/uuid b))))
(not (sqlite-util/db-based-graph? repo)))
(not db-based?))
heading (:heading properties)
markdown? (= :markdown format)
title (if db-based?
;; replace [[uuid]] with block's content
(:block/title (assoc (d/entity db (:db/id b)) :block.temp/search? true))
title)
content (or title "")
page-first-child? (= (:db/id b) (ldb/get-first-child db (:db/id page)))
pre-block? (or pre-block?
@@ -82,11 +87,10 @@
(string/blank? new-content))
""
" ")]
(str prefix sep new-content)))
content (if block-ref-not-saved?
(gp-property/insert-property repo format content :id (str (:block/uuid b)))
content)]
content))
(str prefix sep new-content)))]
(if block-ref-not-saved?
(gp-property/insert-property repo format content :id (str (:block/uuid b)))
content)))
(defn- tree->file-content-aux
[repo db tree {:keys [init-level] :as opts} context]

8
src/main/frontend/handler/dnd.cljs
View File

@@ -1,6 +1,7 @@
(ns frontend.handler.dnd
"Provides fns for drag and drop"
(:require [frontend.db :as db]
(:require [frontend.config :as config]
[frontend.db :as db]
[frontend.handler.block :as block-handler]
[frontend.handler.editor :as editor-handler]
[frontend.handler.property :as property-handler]
@@ -8,6 +9,7 @@
[frontend.modules.outliner.ui :as ui-outliner-tx]
[frontend.state :as state]
[logseq.common.util.block-ref :as block-ref]
[logseq.common.util.page-ref :as page-ref]
[logseq.db :as ldb]))
(defn move-blocks
@@ -25,10 +27,10 @@
(cond
;; alt pressed, make a block-ref
(and alt-key? (= (count blocks) 1))
(do
(let [->ref (if (config/db-based-graph?) page-ref/->page-ref block-ref/->block-ref)]
(property-handler/file-persist-block-id! (state/get-current-repo) (:block/uuid first-block))
(editor-handler/api-insert-new-block!
(block-ref/->block-ref (:block/uuid first-block))
(->ref (:block/uuid first-block))
{:block-uuid (:block/uuid target-block)
:sibling? (not nested?)
:before? top?}))

3
src/main/frontend/handler/editor.cljs
View File

@@ -1941,7 +1941,8 @@
size)
(let [new-meta (merge metadata size)
image-part (first (string/split full_text #"\{"))
new-full-text (str image-part (pr-str new-meta))
md-link? (string/starts-with? image-part "![")
new-full-text (str (if md-link? image-part (str "![image](" image-part ")")) (pr-str new-meta))
block (db/entity [:block/uuid block-id])
value (:block/title block)
new-value (string/replace value full_text new-full-text)]

19
src/main/frontend/worker/pipeline.cljs
View File

@@ -7,6 +7,7 @@
[frontend.worker.react :as worker-react]
[frontend.worker.state :as worker-state]
[logseq.common.defkeywords :refer [defkeywords]]
[logseq.common.uuid :as common-uuid]
[logseq.db :as ldb]
[logseq.db.frontend.validate :as db-validate]
[logseq.db.sqlite.export :as sqlite-export]
@@ -49,6 +50,8 @@
(defn- insert-tag-templates
[repo conn tx-report]
(let [db (:db-after tx-report)
journal-id (:db/id (d/entity db :logseq.class/Journal))
journal-template? (some (fn [d] (and (:added d) (= (:a d) :block/tags) (= (:v d) journal-id))) (:tx-data tx-report))
tx-data (some->> (:tx-data tx-report)
(filter (fn [d] (and (= (:a d) :block/tags) (:added d))))
(group-by :e)
@@ -56,9 +59,12 @@
(let [object (d/entity db e)
template-blocks (->> (mapcat (fn [id]
(let [tag (d/entity db id)
journal? (= journal-id id)
parents (ldb/get-page-parents tag {:node-class? true})
templates (mapcat :logseq.property/_template-applied-to (conj parents tag))]
templates))
(cond->> templates
journal?
(map (fn [t] (assoc t :journal tag))))))
(set (map :v datoms)))
distinct
(sort-by :block/created-at)
@@ -68,10 +74,17 @@
blocks (->>
(cons (assoc (first template-blocks) :logseq.property/used-template (:db/id template))
(rest template-blocks))
(map (fn [e] (assoc (into {} e) :db/id (:db/id e)))))]
(map (fn [e]
(cond->
(assoc (into {} e) :db/id (:db/id e))
(:journal template)
(assoc :block/uuid
(common-uuid/gen-journal-template-block (:block/uuid (:journal template))
(:block/uuid e)))))))]
blocks))))]
(when (seq template-blocks)
(let [result (outliner-core/insert-blocks repo conn template-blocks object {:sibling? false})]
(let [result (outliner-core/insert-blocks repo conn template-blocks object {:sibling? false
:keep-uuid? journal-template?})]
(:tx-data result)))))))]
tx-data))

4
src/main/frontend/worker/search.cljs
View File

@@ -334,7 +334,7 @@ DROP TRIGGER IF EXISTS blocks_au;
(drop-tables-and-triggers! db)
(create-tables-and-triggers! db))
(defn get-all-block-contents
(defn get-all-blocks
[db]
(when db
(->> (d/datoms db :avet :block/uuid)
@@ -347,7 +347,7 @@ DROP TRIGGER IF EXISTS blocks_au;
(defn build-blocks-indice
[repo db]
(build-fuzzy-search-indice repo db)
(->> (get-all-block-contents db)
(->> (get-all-blocks db)
(keep block->index)
(bean/->js)))

8
src/main/logseq/api.cljs
View File

@@ -903,7 +903,10 @@
[k]
(this-as this
(p/let [prop (-get-property this k)]
(bean/->js (sdk-utils/normalize-keyword-for-json prop)))))
(-> prop
(assoc :type (:logseq.property/type prop))
(sdk-utils/normalize-keyword-for-json)
(bean/->js)))))
(defn ^:export upsert_property
"schema:
@@ -929,7 +932,8 @@
(string? (:cardinality schema))
(update :cardinality keyword)
(string? (:type schema))
(update :type keyword))
(-> (assoc :logseq.property/type (keyword (:type schema)))
(dissoc :type)))
p (db-property-handler/upsert-property! k schema
(cond-> opts
name