clone/logseq
1
0
Fork 0
mirror of https://github.com/logseq/logseq.git synced 2026-04-24 22:25:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: mv common worker/frontend ns to frontend/common

Browse Source 
Move common core.async, date and search utils to frontend/common.
Part of LOG-3171
This commit is contained in:
Gabriel Horner
2024-08-09 11:18:32 -04:00
parent dff466801d
commit 20eac03cea
18 changed files with 138 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.clj-kondo/config.edn
View File

@@ -54,6 +54,10 @@
frontend.components.query.result query-result
frontend.components.class class-component
frontend.components.property property-component
frontend.common.date common-date
frontend.common.missionary-util c.m
frontend.common.schema-register sr
frontend.common.search-fuzzy fuzzy
frontend.config config
frontend.date date
frontend.db db

2
deps/common/.carve/config.edn vendored
View File

@@ -1,7 +1,5 @@
{:paths ["src"]
:api-namespaces [logseq.common.path
logseq.common.fractional-index
logseq.common.missionary-util
logseq.common.uuid
logseq.common.util.page-ref
logseq.common.util.block-ref

90
src/main/frontend/common/async_util.cljc Normal file
View File

@@ -0,0 +1,90 @@
(ns frontend.common.async-util
"Some cljs.core.async relate macros and fns, used in worker and frontend
namespaces. See also: https://gist.github.com/vvvvalvalval/f1250cec76d3719a8343"
#?(:cljs (:require [promesa.core :as p]
[logseq.common.util :as common-util]
[clojure.core.async :as async]
[cljs.core.async.impl.channels :refer [ManyToManyChannel]])))
#?(:cljs
(defn throw-err
[v]
(if (instance? ExceptionInfo v) (throw v) v)))
(defmacro <?
[port]
`(throw-err (cljs.core.async/<! ~port)))
#?(:cljs
(defn c->p
"Converts a Core.async channel to a Promise"
[chan]
(let [d (p/deferred)]
(if chan
(async/go
(let [result (async/<! chan)]
(if (instance? ExceptionInfo result)
(p/reject! d result)
(p/resolve! d result))))
(p/resolve! d nil))
d)))
#?(:cljs
(defn drain-chan
"drop all stuffs in CH, and return all of them"
[ch]
(->> (repeatedly #(async/poll! ch))
(take-while identity))))
#?(:cljs
(defn <ratelimit
"return a channel CH,
ratelimit flush items in in-ch every max-duration(ms),
opts:
- :filter-fn filter item before putting items into returned CH, (filter-fn item)
will poll it when its return value is channel,
- :flush-fn exec flush-fn when time to flush, (flush-fn item-coll)
- :stop-ch stop go-loop when stop-ch closed
- :distinct-key-fn distinct coll when put into CH
- :chan-buffer buffer of return CH, default use (async/chan 1000)
- :flush-now-ch flush the content in the queue immediately
- :refresh-timeout-ch refresh (timeout max-duration)"
[in-ch max-duration & {:keys [filter-fn flush-fn stop-ch distinct-key-fn chan-buffer flush-now-ch refresh-timeout-ch]}]
(let [ch (if chan-buffer (async/chan chan-buffer) (async/chan 1000))
stop-ch* (or stop-ch (async/chan))
flush-now-ch* (or flush-now-ch (async/chan))
refresh-timeout-ch* (or refresh-timeout-ch (async/chan))]
(async/go-loop [timeout-ch (async/timeout max-duration) coll []]
(let [{:keys [refresh-timeout timeout e stop flush-now]}
(async/alt! refresh-timeout-ch* {:refresh-timeout true}
timeout-ch {:timeout true}
in-ch ([e] {:e e})
stop-ch* {:stop true}
flush-now-ch* {:flush-now true})]
(cond
refresh-timeout
(recur (async/timeout max-duration) coll)
(or flush-now timeout)
(do (async/onto-chan! ch coll false)
(flush-fn coll)
(drain-chan flush-now-ch*)
(recur (async/timeout max-duration) []))
(some? e)
(let [filter-v (filter-fn e)
filter-v* (if (instance? ManyToManyChannel filter-v)
(async/<! filter-v)
filter-v)]
(if filter-v*
(recur timeout-ch (cond->> (conj coll e)
distinct-key-fn (common-util/distinct-by distinct-key-fn)
true vec))
(recur timeout-ch coll)))
(or stop
;; got nil from in-ch, means in-ch is closed
;; so we stop the whole go-loop
(nil? e))
(async/close! ch))))
ch)))

5
src/main/frontend/worker/date.cljs → src/main/frontend/common/date.cljs
View File

@@ -1,5 +1,6 @@
(ns frontend.worker.date
"Date related fns that used by worker"
(ns frontend.common.date
"Date related fns shared by worker and frontend namespaces. Eventually some
of this should go to logseq.common.util.date-time"
(:require [cljs-time.format :as tf]
[logseq.common.util :as common-util]))

15
src/main/frontend/common/search_fuzzy.cljs
View File

@@ -2,7 +2,7 @@
"fuzzy search. Used by frontend and worker namespaces"
(:require [clojure.string :as string]
[cljs-bean.core :as bean]
[frontend.worker.util :as worker-util]))
["remove-accents" :as removeAccents]))
(def MAX-STRING-LENGTH 1000.0)
@@ -56,15 +56,24 @@
(dec idx)
(- score 0.1)))))))
(defn search-normalize
"Normalize string for searching (loose)"
[s remove-accents?]
(when s
(let [normalize-str (.normalize (string/lower-case s) "NFKC")]
(if remove-accents?
(removeAccents normalize-str)
normalize-str))))
(defn fuzzy-search
[data query & {:keys [limit extract-fn]
:or {limit 20}}]
(let [query (worker-util/search-normalize query true)]
(let [query (search-normalize query true)]
(->> (take limit
(sort-by :score (comp - compare)
(filter #(< 0 (:score %))
(for [item data]
(let [s (str (if extract-fn (extract-fn item) item))]
{:data item
:score (score query (worker-util/search-normalize s true))})))))
:score (score query (search-normalize s true))})))))
(map :data))))

2
src/main/frontend/components/repo.cljs
View File

@@ -24,7 +24,7 @@
[logseq.shui.ui :as shui]
[frontend.handler.db-based.rtc :as rtc-handler]
[frontend.handler.graph :as graph]
[frontend.worker.async-util :as async-util]))
[frontend.common.async-util :as async-util]))
(rum/defc normalized-graph-label
[{:keys [url remote? GraphName GraphUUID] :as graph} on-click]

12
src/main/frontend/date.cljs
View File

@@ -10,7 +10,7 @@
[logseq.common.util.date-time :as date-time-util]
[goog.object :as gobj]
[lambdaisland.glogi :as log]
[frontend.worker.date :as worker-date]))
[frontend.common.date :as common-date]))
(defn nld-parse
[s]
@@ -21,7 +21,7 @@
(defn journal-title-formatters
[]
(worker-date/journal-title-formatters (state/get-date-formatter)))
(common-date/journal-title-formatters (state/get-date-formatter)))
(defn get-date-time-string
([]
@@ -111,15 +111,15 @@
(defn normalize-date
[s]
(worker-date/normalize-date s (state/get-date-formatter)))
(common-date/normalize-date s (state/get-date-formatter)))
(defn normalize-journal-title
[title]
(worker-date/normalize-journal-title title (state/get-date-formatter)))
(common-date/normalize-journal-title title (state/get-date-formatter)))
(defn valid-journal-title?
[title]
(worker-date/valid-journal-title? title (state/get-date-formatter)))
(common-date/valid-journal-title? title (state/get-date-formatter)))
(defn journal-title->
([journal-title then-fn]
@@ -144,7 +144,7 @@
[journal-title]
(journal-title-> journal-title #(tc/to-long %)))
(def default-journal-filename-formatter worker-date/default-journal-filename-formatter)
(def default-journal-filename-formatter common-date/default-journal-filename-formatter)
(defn journal-title->default
"Journal title to filename format"

2
src/main/frontend/db/transact.cljs
View File

@@ -3,7 +3,7 @@
(:require [clojure.core.async :as async]
[clojure.core.async.interop :refer [p->c]]
[promesa.core :as p]
[frontend.worker.async-util :include-macros true :refer [<?]]))
[frontend.common.async-util :include-macros true :refer [<?]]))
(defonce *request-id (atom 0))
(defonce requests (async/chan 1000))

3
src/main/frontend/fs/sync.cljs
View File

@@ -15,6 +15,7 @@
[clojure.set :as set]
[clojure.string :as string]
[electron.ipc :as ipc]
[frontend.common.async-util :as async-util]
[frontend.config :as config]
[frontend.context.i18n :refer [t]]
[frontend.db :as db]
@@ -2680,7 +2681,7 @@
(<ratelimit [this from-chan]
(let [<fast-filter-e-fn (.filter-file-change-events-fn this)]
(util/<ratelimit
(async-util/<ratelimit
from-chan rate
:filter-fn
(fn [e]

9
src/main/frontend/handler/search.cljs
View File

@@ -2,6 +2,7 @@
"Provides util handler fns for search"
(:require [clojure.string :as string]
[frontend.config :as config]
[frontend.common.search-fuzzy :as fuzzy]
[frontend.db :as db]
[frontend.handler.notification :as notification]
[frontend.search :as search]
@@ -123,8 +124,8 @@
content
(when (and content q)
(let [q-words (string/split q #" ")
lc-content (util/search-normalize content (state/enable-search-remove-accents?))
lc-q (util/search-normalize q (state/enable-search-remove-accents?))]
lc-content (fuzzy/search-normalize content (state/enable-search-remove-accents?))
lc-q (fuzzy/search-normalize q (state/enable-search-remove-accents?))]
(if (and (string/includes? lc-content lc-q)
(not (util/safe-re-find #" " q)))
(let [i (string/index-of lc-content lc-q)
@@ -140,8 +141,8 @@
result []]
(if (and (seq words) content)
(let [word (first words)
lc-word (util/search-normalize word (state/enable-search-remove-accents?))
lc-content (util/search-normalize content (state/enable-search-remove-accents?))]
lc-word (fuzzy/search-normalize word (state/enable-search-remove-accents?))
lc-content (fuzzy/search-normalize content (state/enable-search-remove-accents?))]
(if-let [i (string/index-of lc-content lc-word)]
(recur (rest words)
(subs content (+ i (count word)))

2
src/main/frontend/search.cljs
View File

@@ -25,7 +25,7 @@
(defn block-search
[repo q option]
(when-let [engine (get-engine repo)]
(let [q (util/search-normalize q (state/enable-search-remove-accents?))]
(let [q (fuzzy/search-normalize q (state/enable-search-remove-accents?))]
(when-not (string/blank? q)
(protocol/query engine q option)))))

7
src/main/frontend/util.cljc
View File

@@ -27,7 +27,6 @@
[rum.core :as rum]
[clojure.core.async :as async]
[frontend.pubsub :as pubsub]
[frontend.worker.util :as worker-util]
[datascript.impl.entity :as de]))
#?(:cljs (:import [goog.async Debouncer]))
(:require
@@ -1006,9 +1005,6 @@
[string]
(some-> string str (js/encodeURIComponent) (.replace "+" "%20"))))
#?(:cljs
(def search-normalize worker-util/search-normalize))
#?(:cljs
(def page-name-sanity-lc
"Delegate to common-util to loosely couple app usages to graph-parser"
@@ -1126,9 +1122,6 @@
(->> (repeatedly #(async/poll! ch))
(take-while identity))))
#?(:cljs
(def <ratelimit worker-util/<ratelimit))
#?(:cljs
(defn trace!
[]

28
src/main/frontend/worker/async_util.cljc
View File

@@ -1,28 +0,0 @@
(ns frontend.worker.async-util
"Some cljs.core.async relate macros and fns.
see also: https://gist.github.com/vvvvalvalval/f1250cec76d3719a8343"
#?(:cljs (:require [promesa.core :as p]
[clojure.core.async :as async])))
#?(:cljs
(defn throw-err
[v]
(if (instance? ExceptionInfo v) (throw v) v)))
(defmacro <?
[port]
`(throw-err (cljs.core.async/<! ~port)))
#?(:cljs
(defn c->p
"Converts a Core.async channel to a Promise"
[chan]
(let [d (p/deferred)]
(if chan
(async/go
(let [result (async/<! chan)]
(if (instance? ExceptionInfo result)
(p/reject! d result)
(p/resolve! d result))))
(p/resolve! d nil))
d)))

3
src/main/frontend/worker/file.cljs
View File

@@ -9,6 +9,7 @@
[cljs-time.core :as t]
[cljs-time.coerce :as tc]
[frontend.worker.util :as worker-util]
[frontend.common.async-util :as async-util]
[datascript.core :as d]
[logseq.db :as ldb]
[malli.core :as m]
@@ -116,7 +117,7 @@
(defn <ratelimit-file-writes!
[]
(worker-util/<ratelimit file-writes-chan batch-write-interval
(async-util/<ratelimit file-writes-chan batch-write-interval
:filter-fn (fn [_] true)
:flush-fn
(fn [col]

8
src/main/frontend/worker/file/core.cljs
View File

@@ -6,7 +6,7 @@
[logseq.common.path :as path]
[datascript.core :as d]
[logseq.db :as ldb]
[frontend.worker.date :as worker-date]
[frontend.common.date :as common-date]
[frontend.worker.util :as worker-util]
[logseq.db.sqlite.util :as sqlite-util]))
@@ -136,11 +136,11 @@
title (string/capitalize (:block/name page-block))
whiteboard-page? (ldb/whiteboard? page-block)
format (if whiteboard-page? "edn" format)
journal-page? (worker-date/valid-journal-title? title date-formatter)
journal-title (worker-date/normalize-journal-title title date-formatter)
journal-page? (common-date/valid-journal-title? title date-formatter)
journal-title (common-date/normalize-journal-title title date-formatter)
journal-page? (and journal-page? (not (string/blank? journal-title)))
filename (if journal-page?
(worker-date/date->file-name journal-title (:journal-file-name-format context))
(common-date/date->file-name journal-title (:journal-file-name-format context))
(-> (or (:block/title page-block) (:block/name page-block))
wfu/file-name-sanity))
sub-dir (cond

4
src/main/frontend/worker/handler/page/file_based/page.cljs
View File

@@ -9,7 +9,7 @@
[logseq.graph-parser.text :as text]
[logseq.common.util :as common-util]
[logseq.common.config :as common-config]
[frontend.worker.date :as date]
[frontend.common.date :as common-date]
[logseq.db.frontend.order :as db-order]))
(defn- file-based-properties-block
@@ -74,7 +74,7 @@
:as options}]
(let [date-formatter (common-config/get-date-formatter config)
split-namespace? (not (or (string/starts-with? title "hls__")
(date/valid-journal-title? date-formatter title)))
(common-date/valid-journal-title? date-formatter title)))
[title page-name] (get-title-and-pagename title)]
(when-not (ldb/get-page @conn page-name)
(let [pages (if split-namespace?

9
src/main/frontend/worker/search.cljs
View File

@@ -7,7 +7,6 @@
[goog.object :as gobj]
[datascript.core :as d]
[frontend.common.search-fuzzy :as fuzzy]
[frontend.worker.util :as worker-util]
[logseq.db.sqlite.util :as sqlite-util]
[logseq.common.util :as common-util]
[logseq.db :as ldb]))
@@ -172,8 +171,8 @@ DROP TRIGGER IF EXISTS blocks_au;
(if (seq coll')
(rest coll')
(reduced false))))
(seq (worker-util/search-normalize match true))
(seq (worker-util/search-normalize q true))))))
(seq (fuzzy/search-normalize match true))
(seq (fuzzy/search-normalize q true))))))
(defn- page-or-object?
[entity]
@@ -194,7 +193,7 @@ DROP TRIGGER IF EXISTS blocks_au;
(defn- sanitize
[content]
(some-> content
(worker-util/search-normalize true)))
(fuzzy/search-normalize true)))
(defn block->index
"Convert a block to the index for searching"
@@ -238,7 +237,7 @@ DROP TRIGGER IF EXISTS blocks_au;
[repo db q {:keys [limit]
:or {limit 100}}]
(when repo
(let [q (worker-util/search-normalize q true)
(let [q (fuzzy/search-normalize q true)
q (fuzzy/clean-str q)
q (if (= \# (first q)) (subs q 1) q)]
(when-not (string/blank? q)

73
src/main/frontend/worker/util.cljc
View File

@@ -2,14 +2,10 @@
"Worker utils"
#?(:cljs (:require-macros [frontend.worker.util]))
#?(:cljs (:refer-clojure :exclude [format]))
#?(:cljs (:require ["remove-accents" :as removeAccents]
[cljs.core.async.impl.channels :refer [ManyToManyChannel]]
[clojure.core.async :as async]
[clojure.string :as string]
#?(:cljs (:require [clojure.string :as string]
[goog.crypt :as crypt]
[goog.crypt.Hmac]
[goog.crypt.Sha256]
[logseq.common.util :as common-util]
[logseq.db :as ldb]
[logseq.db.sqlite.common-db :as sqlite-common-db])))
@@ -27,73 +23,6 @@
#?(:cljs
(do
(defn search-normalize
"Normalize string for searching (loose)"
[s remove-accents?]
(when s
(let [normalize-str (.normalize (string/lower-case s) "NFKC")]
(if remove-accents?
(removeAccents normalize-str)
normalize-str))))
(defn drain-chan
"drop all stuffs in CH, and return all of them"
[ch]
(->> (repeatedly #(async/poll! ch))
(take-while identity)))
(defn <ratelimit
"return a channel CH,
ratelimit flush items in in-ch every max-duration(ms),
opts:
- :filter-fn filter item before putting items into returned CH, (filter-fn item)
will poll it when its return value is channel,
- :flush-fn exec flush-fn when time to flush, (flush-fn item-coll)
- :stop-ch stop go-loop when stop-ch closed
- :distinct-key-fn distinct coll when put into CH
- :chan-buffer buffer of return CH, default use (async/chan 1000)
- :flush-now-ch flush the content in the queue immediately
- :refresh-timeout-ch refresh (timeout max-duration)"
[in-ch max-duration & {:keys [filter-fn flush-fn stop-ch distinct-key-fn chan-buffer flush-now-ch refresh-timeout-ch]}]
(let [ch (if chan-buffer (async/chan chan-buffer) (async/chan 1000))
stop-ch* (or stop-ch (async/chan))
flush-now-ch* (or flush-now-ch (async/chan))
refresh-timeout-ch* (or refresh-timeout-ch (async/chan))]
(async/go-loop [timeout-ch (async/timeout max-duration) coll []]
(let [{:keys [refresh-timeout timeout e stop flush-now]}
(async/alt! refresh-timeout-ch* {:refresh-timeout true}
timeout-ch {:timeout true}
in-ch ([e] {:e e})
stop-ch* {:stop true}
flush-now-ch* {:flush-now true})]
(cond
refresh-timeout
(recur (async/timeout max-duration) coll)
(or flush-now timeout)
(do (async/onto-chan! ch coll false)
(flush-fn coll)
(drain-chan flush-now-ch*)
(recur (async/timeout max-duration) []))
(some? e)
(let [filter-v (filter-fn e)
filter-v* (if (instance? ManyToManyChannel filter-v)
(async/<! filter-v)
filter-v)]
(if filter-v*
(recur timeout-ch (cond->> (conj coll e)
distinct-key-fn (common-util/distinct-by distinct-key-fn)
true vec))
(recur timeout-ch coll)))
(or stop
;; got nil from in-ch, means in-ch is closed
;; so we stop the whole go-loop
(nil? e))
(async/close! ch))))
ch))
(defn post-message
[type data]
(when (exists? js/self)