clone/logseq
1
0
Fork 0
mirror of https://github.com/logseq/logseq.git synced 2026-05-25 13:14:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:logseq/logseq into master

Browse Source
This commit is contained in:
Tienson Qin
2020-12-14 19:19:17 +08:00
parent cfed1530d3 b8fe3bbad7
commit 1d5b7e024f
14 changed files with 231 additions and 254 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/main/frontend/components/search.cljs
View File

@@ -85,7 +85,8 @@
:path-params {:path data}})
:block
(let [page (:page/name (:block/page data))
(let [block-uuid (uuid (:block/uuid data))
page (:page/name (:block/page (db/entity [:block/uuid block-uuid])))
path (str "/page/" (util/encode-str page) "#ls-block-" (:block/uuid data))]
(route/redirect-with-fragment! path))
nil))
@@ -100,7 +101,8 @@
{:page page}))
:block
(let [block (db/entity [:block/uuid (:block/uuid data)])]
(let [block-uuid (uuid (:block/uuid data))
block (db/entity [:block/uuid block-uuid])]
(state/sidebar-add-block!
(state/get-current-repo)
(:db/id block)

31
src/main/frontend/db.cljs
View File

@@ -1,6 +1,7 @@
(ns frontend.db
(:require [datascript.core :as d]
[frontend.date :as date]
[frontend.search.db :as search-db]
[medley.core :as medley]
[datascript.transit :as dt]
[frontend.format :as format]
@@ -21,7 +22,8 @@
[frontend.db-schema :as db-schema]
[clojure.core.async :as async]
[lambdaisland.glogi :as log]
[frontend.idb :as idb]))
[frontend.idb :as idb]
[cljs-bean.core :as bean]))
;; Query atom of map of Key ([repo q inputs]) -> atom
;; TODO: replace with LRUCache, only keep the latest 20 or 50 items?
@@ -1930,6 +1932,9 @@
(swap! persistent-jobs assoc repo job)))
;; only save when user's idle
;; TODO: pass as a parameter
(defonce *sync-search-indice-f (atom nil))
(defn- repo-listen-to-tx!
[repo conn files-db?]
(d/listen! conn :persistence
@@ -1937,7 +1942,17 @@
(let [tx-id (get-tx-id tx-report)]
(state/set-last-transact-time! repo (util/time-ms))
;; (state/persist-transaction! repo files-db? tx-id (:tx-data tx-report))
(persist-if-idle! repo)))))
(persist-if-idle! repo))
;; rebuild search indices
(when-not files-db?
(let [data (:tx-data tx-report)
datoms (filter
(fn [datom]
(contains? #{:page/name :block/content} (:a datom)))
data)]
(when-let [f @*sync-search-indice-f]
(f datoms)))))))
(defn- listen-and-persist!
[repo]
@@ -2415,6 +2430,18 @@
(reset! blocks-count-cache n)
n))))
;; block/uuid and block/content
(defn get-all-block-contents
[]
(->> (d/datoms (get-conn) :avet :block/uuid)
(map :v)
(map (fn [id]
(let [e (entity [:block/uuid id])]
{:db/id (:db/id e)
:block/uuid id
:block/content (:block/content e)
:block/format (:block/format e)})))))
(defn get-all-templates
[]
(let [pred (fn [db properties]

4
src/main/frontend/extensions/code.cljs
View File

@@ -36,9 +36,7 @@
["codemirror/mode/smalltalk/smalltalk"]
["codemirror/mode/sql/sql"]
["codemirror/mode/swift/swift"]
["codemirror/mode/xml/xml"]
;; ["parinfer-codemirror" :as par-cm]
))
["codemirror/mode/xml/xml"]))
;; codemirror

16
src/main/frontend/handler.cljs
View File

@@ -9,6 +9,8 @@
[promesa.core :as p]
[cljs-bean.core :as bean]
[frontend.date :as date]
[frontend.search :as search]
[frontend.search.db :as search-db]
[frontend.handler.notification :as notification]
[frontend.handler.page :as page-handler]
[frontend.handler.repo :as repo-handler]
@@ -25,9 +27,16 @@
(defn- watch-for-date!
[]
(js/setInterval (fn []
(when-not (state/nfs-refreshing?)
(repo-handler/create-today-journal!))) 1000))
(let [f (fn []
(when-not (state/nfs-refreshing?)
(repo-handler/create-today-journal!))
(when-let [repo (state/get-current-repo)]
(when (and (search-db/empty? repo)
(not (state/file-in-writing!))
(state/input-idle? repo))
(search/rebuild-indices!))))]
(f)
(js/setInterval f 5000)))
(defn store-schema!
[]
@@ -157,6 +166,7 @@
:example? true}])]
(state/set-repos! repos)
(restore-and-setup! me repos logged?)))
(reset! db/*sync-search-indice-f search/sync-search-indice!)
(db/run-batch-txs!)
(file-handler/run-writes-chan!)
(editor-handler/periodically-save!)))

18
src/main/frontend/handler/editor.cljs
View File

@@ -1547,17 +1547,11 @@
editing-page (and block
(when-let [page-id (:db/id (:block/page block))]
(:page/name (db/entity page-id))))]
(let [pages (db/get-pages (state/get-current-repo))
pages (if editing-page
;; To prevent self references
(remove (fn [p] (= (string/lower-case p) editing-page)) pages)
pages)]
(filter
(fn [page]
(string/index-of
(string/lower-case page)
(string/lower-case q)))
pages))))
(let [pages (search/page-search q 20)]
(if editing-page
;; To prevent self references
(remove (fn [p] (= (string/lower-case p) editing-page)) pages)
pages))))
(defn get-matched-blocks
[q]
@@ -1567,7 +1561,7 @@
(fn [h]
(= (:block/uuid current-block)
(:block/uuid h)))
(search/search q 21))))
(search/search q 10))))
(defn get-matched-templates
[q]

13
src/main/frontend/handler/search.cljs
View File

@@ -2,14 +2,15 @@
(:require [goog.object :as gobj]
[frontend.state :as state]
[goog.dom :as gdom]
[frontend.search :as search]))
[frontend.search :as search]
[frontend.handler.notification :as notification-handler]))
(defn search
[q]
(swap! state/state assoc :search/result
{:pages (search/page-search q)
:files (search/file-search q)
:blocks (search/search q)}))
:blocks (search/search q 10)}))
(defn clear-search!
[]
@@ -18,3 +19,11 @@
:search/q "")
(when-let [input (gdom/getElement "search_field")]
(gobj/set input "value" "")))
(defn rebuild-indices!
[]
(println "Starting to rebuild search indices!")
(search/rebuild-indices!)
(notification-handler/show!
"Search indices rebuilt successfully!"
:success))

8
src/main/frontend/keyboards.cljs
View File

@@ -3,7 +3,9 @@
[frontend.handler.history :as history-handler]
[frontend.handler.ui :as ui-handler]
[frontend.handler.route :as route-handler]
[frontend.handler.search :as search-handler]
[frontend.state :as state]
[frontend.search :as search]
[frontend.util :as util]
[medley.core :as medley]
["mousetrap" :as mousetrap]
@@ -70,7 +72,8 @@
"ctrl+h" editor-handler/highlight-format!
"ctrl+shift+a" editor-handler/select-all-blocks!
"alt+shift+up" (fn [state e] (editor-handler/move-up-down e true))
"alt+shift+down" (fn [state e] (editor-handler/move-up-down e false))}
"alt+shift+down" (fn [state e] (editor-handler/move-up-down e false))
"ctrl+c ctrl+s" (fn [state e] (search-handler/rebuild-indices!))}
(medley/map-keys util/->system-modifier)))
(defonce chords
@@ -79,7 +82,8 @@
"t t" state/toggle-theme!
"t r" ui-handler/toggle-right-sidebar!
"t e" state/toggle-new-block-shortcut!
"s" route-handler/toggle-between-page-and-file!})
"s" route-handler/toggle-between-page-and-file!
})
(defonce bind! (gobj/get mousetrap "bind"))

161
src/main/frontend/search.cljs
View File

@@ -1,12 +1,67 @@
(ns frontend.search
(:require [frontend.db :as db]
[frontend.search.db :as search-db :refer [indices]]
[frontend.config :as config]
[frontend.state :as state]
[frontend.util :as util]
[frontend.util :as util :refer-macros [profile]]
[cljs-bean.core :as bean]
[clojure.string :as string]
[clojure.set :as set]
[frontend.regex :as regex]
[frontend.text :as text]))
[frontend.text :as text]
[cljs-bean.core :as bean]
[goog.object :as gobj]
["fuzzysort" :as fuzzy]))
(def fuzzy-go (gobj/get fuzzy "go"))
(defonce prepare (gobj/get fuzzy "prepare"))
(defonce highlight (gobj/get fuzzy "highlight"))
(defn go
[q indice opts]
(fuzzy-go q indice opts))
(defn block->index
[{:block/keys [uuid content format] :as block}]
(when (<= (count content) 1000) ; performance
(when-let [result (->> (text/remove-level-spaces content format)
(text/remove-properties!)
(prepare))]
(gobj/set result "id" (:db/id block))
(gobj/set result "uuid" (str uuid))
result)))
(defn make-blocks-indice!
[]
(when-let [repo (state/get-current-repo)]
(let [blocks (->> (db/get-all-block-contents)
(map block->index)
(remove nil?)
(bean/->js))]
(swap! indices assoc-in [repo :blocks] blocks)
blocks)))
(defn make-pages-indice!
[]
(when-let [repo (state/get-current-repo)]
(let [pages (->> (db/get-pages (state/get-current-repo))
(remove string/blank?)
(map (fn [p] {:name p}))
(bean/->js))]
(swap! indices assoc-in [repo :pages] pages)
pages)))
;; TODO: persist indices to indexeddb, it'll be better if the future db
;; can has the direct fuzzy search support.
(defn rebuild-indices!
([]
(rebuild-indices! (state/get-current-repo)))
([repo]
(when repo
(let [result {:pages (make-pages-indice!)
:blocks (make-blocks-indice!)}]
(swap! indices assoc repo result)
result))))
;; Copied from https://gist.github.com/vaughnd/5099299
(defn str-len-distance
@@ -78,32 +133,48 @@
(defn search
"Block search"
([q]
(search q 20))
(search q 10))
([q limit]
(when-not (string/blank? q)
(let [q (escape-str q)
q-pattern (re-pattern (str "(?i)" q))]
(when-not (string/blank? q)
(let [blocks (db/get-matched-blocks
(fn [content]
(re-find q-pattern content))
;; (fn [content]
;; (> (score q (.toLowerCase content)) 0))
limit)]
(map (fn [{:block/keys [content format _properties] :as block}]
(assoc block :block/content
(->> (text/remove-level-spaces content format)
(text/remove-properties!)))) blocks)))))))
(when-let [repo (state/get-current-repo)]
(when-not (string/blank? q)
(let [q (string/lower-case q)
q (escape-str q)]
(when-not (string/blank? q)
(let [indice (or (get-in @indices [repo :blocks])
(make-blocks-indice!))
result (->
(go q indice (clj->js {:limit limit
:allowTypo false
:threshold -10000}))
(bean/->clj))]
(->>
(map
(fn [{:keys [target uuid]}]
{:block/uuid uuid
:block/content target})
result)
(remove nil?)))))))))
(defn page-search
([q]
(page-search q 3))
([q limit]
(let [q (clean-str q)]
(when-not (string/blank? q)
(let [pages (db/get-pages (state/get-current-repo))]
(when (seq pages)
(fuzzy-search pages q :limit limit)))))))
(when-let [repo (state/get-current-repo)]
(let [q (string/lower-case q)
q (clean-str q)]
(when-not (string/blank? q)
(let [indice (or (get-in @indices [repo :pages])
(make-pages-indice!))
result (->> (go q indice (clj->js {:limit limit
:key "name"
:allowTypo false
:threshold -10000}))
(bean/->clj))]
(->> (map
(fn [{:keys [obj]}]
(:name obj))
result)
(remove nil?))))))))
(defn file-search
([q]
@@ -128,3 +199,49 @@
(when (seq templates)
(let [result (fuzzy-search (keys templates) q :limit limit)]
(vec (select-keys templates result))))))))
(defn sync-search-indice!
[datoms]
(when (seq datoms)
(when-let [repo (state/get-current-repo)]
(let [datoms (group-by :a datoms)
pages (:page/name datoms)
blocks (:block/content datoms)]
(when (seq pages)
(let [pages-result (db/pull-many '[:db/id :page/original-name] (set (map :e pages)))
pages-to-add-set (->> (filter :added pages)
(map :e)
(set))
pages-to-add (->> (filter (fn [page]
(contains? pages-to-add-set (:db/id page))) pages-result)
(map (fn [p] {:name (:page/original-name p)}))
(set))
pages-to-remove-set (->> (remove :added pages)
(map :v)
(set))]
(swap! search-db/indices update-in [repo :pages]
(fn [pages]
(let [pages (or pages (array))
pages (.filter pages (fn [page]
(not (contains? pages-to-remove-set
(string/lower-case (gobj/get page "name"))))))]
(.concat pages (bean/->js pages-to-add)))))))
(when (seq blocks)
(let [blocks-result (db/pull-many '[:db/id :block/uuid :block/format :block/content] (set (map :e blocks)))
blocks-to-add-set (->> (filter :added blocks)
(map :e)
(set))
blocks-to-add (->> (filter (fn [block]
(contains? blocks-to-add-set (:db/id block)))
blocks-result)
(map block->index)
(set))
blocks-to-remove-set (->> (remove :added blocks)
(map :e)
(set))]
(swap! search-db/indices update-in [repo :blocks]
(fn [blocks]
(let [blocks (or blocks (array))
blocks (.filter blocks (fn [block]
(not (contains? blocks-to-remove-set (gobj/get block "id")))))]
(.concat blocks (bean/->js blocks-to-add)))))))))))

8
src/main/frontend/search/db.cljs Normal file
View File

@@ -0,0 +1,8 @@
(ns frontend.search.db
(:refer-clojure :exclude [empty?]))
(defonce indices (atom nil))
(defn empty?
[repo]
(nil? (get @indices repo)))

58
src/main/frontend/sync/dropbox.cljs
View File

@@ -1,58 +0,0 @@
(ns frontend.sync.dropbox)
;; (ns frontend.sync.dropbox
;; (:require ["dropbox" :as dropbox]
;; [goog.object :as gobj]
;; [frontend.sync.protocol :refer [Sync] :as sync]
;; [promesa.core :as p]
;; [cljs-bean.core :as bean]))
;; ;; Note: there's also a `DropboxTeam`
;; (defonce DropboxModule (gobj/get dropbox "Dropbox"))
;; (defonce *dropbox-client (atom nil))
;; (defn upload-file
;; [client path contents]
;; (.filesUpload ^Object client
;; (bean/->js {:path path
;; :contents contents
;; :mode {".tag" "overwrite"}
;; :autorename true})))
;; (defrecord Dropbox [token]
;; Sync
;; (get-client [this]
;; (if-let [client @*dropbox-client]
;; client
;; (let [client (DropboxModule. #js {:accessToken token})]
;; (reset! *dropbox-client client)
;; client)))
;; (signed? [this]
;; true)
;; (get-dir [this path]
;; (p/let [resp (.filesListFolder ^Object (sync/get-client this) (bean/->js {:path path}))]
;; (bean/->clj resp)))
;; (get-more-dir [this cursor]
;; (p/let [resp (.filesListFolderContinue ^Object (sync/get-client this) (bean/->js {:cursor cursor}))]
;; (bean/->clj resp)))
;; (create-file [this path contents]
;; (upload-file ^Object (sync/get-client this) path contents))
;; (update-file [this path contents]
;; (upload-file ^Object (sync/get-client this) path contents))
;; (get-file-contents-and-metadata [this path]
;; (->
;; (p/let [resp (.filesDownload ^Object (sync/get-client this) (bean/->js {:path path}))
;; file-binary (gobj/get resp "fileBinary")
;; contents (.toString file-binary)
;; last-modified-at (gobj/get resp "server_modified")]
;; {:contents contents
;; :last-modified-at last-modified-at})
;; (p/catch
;; (fn [error]
;; ;; TODO:
;; (println "Dropbox get file " path " failed:")
;; (js/console.dir error))
;; )))
;; (delete-file [this path]
;; (.filesDelete ^Object (sync/get-client this) (bean/->js {:path path}))))

11
src/main/frontend/sync/protocol.cljs
View File

@@ -1,11 +0,0 @@
(ns frontend.sync.protocol)
(defprotocol Sync
(get-client [this])
(signed? [this])
(get-dir [this path])
(get-more-dir [this more-state])
(create-file [this path contents])
(update-file [this path contents])
(get-file-contents-and-metadata [this path])
(delete-file [this path]))