mirror of
https://github.com/logseq/logseq.git
synced 2026-04-24 22:25:01 +00:00
fix: remove file graph :block/file and dependent code
Also remove all file graph references in publishing and add back publishing.db tests with db graph. Also remove file graph behavior for cmd-k and files
This commit is contained in:
Gabriel Horner
2
deps/publishing/package.json
vendored
2
deps/publishing/package.json
vendored
@@ -11,6 +11,6 @@
|
||||
"fs-extra": "9.1.0"
|
||||
},
|
||||
"scripts": {
|
||||
"test": "yarn nbb-logseq -cp test:../graph-parser/src -m nextjournal.test-runner"
|
||||
"test": "yarn nbb-logseq -cp test -m nextjournal.test-runner"
|
||||
}
|
||||
}
|
||||
|
||||
1
deps/publishing/script/publishing.cljs
vendored
1
deps/publishing/script/publishing.cljs
vendored
@@ -22,7 +22,6 @@
|
||||
output-path
|
||||
(merge opts {:repo (str sqlite-util/db-version-prefix db-name)
|
||||
:repo-config repo-config
|
||||
:db-graph? true
|
||||
:ui/theme "dark"
|
||||
:ui/radix-color :cyan}))))
|
||||
|
||||
|
||||
1
deps/publishing/src/logseq/publishing.cljs
vendored
1
deps/publishing/src/logseq/publishing.cljs
vendored
@@ -18,7 +18,6 @@ can be passed:
|
||||
can be icon, name, alias, title, description and url
|
||||
* :repo - Name of repo
|
||||
* :repo-config - A graph's configuration
|
||||
* :db-graph? - Boolean which indicates if graph is db based
|
||||
* :default-notification-fn - Configure how errors are reported when creating the export.
|
||||
Default is to throw an exception when it occurs."
|
||||
[db static-dir graph-dir output-dir {:keys [notification-fn dev?]
|
||||
|
||||
149
deps/publishing/src/logseq/publishing/db.cljs
vendored
149
deps/publishing/src/logseq/publishing/db.cljs
vendored
@@ -1,47 +1,10 @@
|
||||
(ns logseq.publishing.db
|
||||
"Provides db fns and associated util fns for publishing"
|
||||
(:require [clojure.set :as set]
|
||||
[clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[logseq.db.common.entity-plus :as entity-plus]
|
||||
[logseq.db.frontend.malli-schema :as db-malli-schema]
|
||||
[logseq.db.frontend.rules :as rules]))
|
||||
|
||||
(defn ^:api get-area-block-asset-url
|
||||
"Returns asset url for an area block used by pdf assets. This lives in this ns
|
||||
because it is used by this dep and needs to be independent from the frontend app"
|
||||
[db block page]
|
||||
(let [db-based? (entity-plus/db-based-graph? db)]
|
||||
(when-some [uuid' (:block/uuid block)]
|
||||
(if db-based?
|
||||
(when-let [image (:logseq.property.pdf/hl-image block)]
|
||||
(str "./assets/" (:block/uuid image) ".png"))
|
||||
(let [props (and block page (:block/properties block))
|
||||
prop-lookup-fn #(get %1 (keyword (name %2)))]
|
||||
(when-some [stamp (:hl-stamp props)]
|
||||
(let [group-key (string/replace-first (:block/title page) #"^hls__" "")
|
||||
hl-page (prop-lookup-fn props :logseq.property.pdf/hl-page)
|
||||
encoded-chars? (boolean (re-find #"(?i)%[0-9a-f]{2}" group-key))
|
||||
group-key (if encoded-chars? (js/encodeURI group-key) group-key)]
|
||||
(str "./assets/" group-key "/" (str hl-page "_" uuid' "_" stamp ".png")))))))))
|
||||
|
||||
(defn- clean-asset-path-prefix
|
||||
[path]
|
||||
(when (string? path)
|
||||
(string/replace-first path #"^[.\/\\]*(assets)[\/\\]+" "")))
|
||||
|
||||
(defn- get-public-pages
|
||||
[db]
|
||||
(->> (d/q
|
||||
'[:find ?p
|
||||
:where
|
||||
[?p :block/name]
|
||||
[?p :block/properties ?properties]
|
||||
[(get ?properties :public) ?pub]
|
||||
[(= true ?pub)]]
|
||||
db)
|
||||
(map first)))
|
||||
|
||||
(defn- get-db-public-pages
|
||||
"Returns public pages and anything they are directly related to: their tags,
|
||||
their properties and any property values that are pages. Anything on the
|
||||
@@ -78,76 +41,6 @@
|
||||
(map first)
|
||||
set))
|
||||
|
||||
(defn- get-public-false-pages
|
||||
[db]
|
||||
(->> (d/q
|
||||
'[:find ?p
|
||||
:where
|
||||
[?p :block/name]
|
||||
[?p :block/properties ?properties]
|
||||
[(get ?properties :public) ?pub]
|
||||
[(= false ?pub)]]
|
||||
db)
|
||||
(map first)))
|
||||
|
||||
(defn- get-public-false-block-ids
|
||||
[db]
|
||||
(->> (d/q
|
||||
'[:find ?b
|
||||
:where
|
||||
[?p :block/name]
|
||||
[?p :block/properties ?properties]
|
||||
[(get ?properties :public) ?pub]
|
||||
[(= false ?pub)]
|
||||
[?b :block/page ?p]]
|
||||
db)
|
||||
(map first)))
|
||||
|
||||
(defn- hl-type-area-fn
|
||||
[db]
|
||||
(if (entity-plus/db-based-graph? db)
|
||||
(fn [datom]
|
||||
(and (= :logseq.property.pdf/hl-type (:a datom))
|
||||
(= (keyword (:v datom)) :area)))
|
||||
(fn [datom]
|
||||
(and
|
||||
(= :block/properties (:a datom))
|
||||
(= (keyword (get (:v datom) :hl-type)) :area)))))
|
||||
|
||||
(defn- get-file-assets
|
||||
[db datoms]
|
||||
(let [pull (fn [eid db]
|
||||
(d/pull db '[*] eid))
|
||||
get-page-by-eid
|
||||
(memoize #(some->
|
||||
(pull % db)
|
||||
:block/page
|
||||
:db/id
|
||||
(pull db)))
|
||||
hl-type-area? (hl-type-area-fn db)]
|
||||
(->>
|
||||
(keep
|
||||
(fn [datom]
|
||||
(cond-> []
|
||||
(= :block/title (:a datom))
|
||||
(concat (let [matched (re-seq #"\([./]*/assets/([^)]+)\)" (:v datom))]
|
||||
(when (seq matched)
|
||||
(for [[_ path] matched]
|
||||
(when (and (string? path)
|
||||
(not (string/ends-with? path ".js")))
|
||||
path)))))
|
||||
;; area image assets
|
||||
(hl-type-area? datom)
|
||||
(#(let [path (some-> (pull (:e datom) db)
|
||||
(get-area-block-asset-url
|
||||
db
|
||||
(get-page-by-eid (:e datom))))
|
||||
path (clean-asset-path-prefix path)]
|
||||
(conj % path)))))
|
||||
datoms)
|
||||
flatten
|
||||
distinct)))
|
||||
|
||||
(defn- get-aliases-for-page-ids
|
||||
[db page-ids]
|
||||
(->> (d/q '[:find ?e
|
||||
@@ -170,36 +63,22 @@
|
||||
(map #(str (:block/uuid %) "." (:logseq.property.asset/type %)))))
|
||||
|
||||
(defn clean-export!
|
||||
"Prepares a database assuming all pages are public unless a page has a 'public:: false'"
|
||||
[db {:keys [db-graph?]}]
|
||||
(let [remove? #(contains? #{"recent" "file"} %)
|
||||
non-public-datom-ids (if db-graph?
|
||||
(get-db-public-false-pages db)
|
||||
(set (concat (get-public-false-pages db) (get-public-false-block-ids db))))
|
||||
"Prepares a database assuming all pages are public unless a page has a publishing-public? property set to false"
|
||||
[db]
|
||||
(let [remove? #(contains? #{"file"} %)
|
||||
non-public-datom-ids (get-db-public-false-pages db)
|
||||
filtered-db (d/filter db
|
||||
(fn [_db datom]
|
||||
(let [ns' (namespace (:a datom))]
|
||||
(and (not (remove? ns'))
|
||||
(not (contains? #{:block/file} (:a datom)))
|
||||
(not (contains? non-public-datom-ids (:e datom)))))))
|
||||
(not (contains? non-public-datom-ids (:e datom)))
|
||||
(not (and (contains? non-public-datom-ids (:v datom))
|
||||
(= :block/page (:a datom))))))))
|
||||
datoms (d/datoms filtered-db :eavt)
|
||||
assets (if db-graph? (get-db-assets filtered-db) (get-file-assets db datoms))]
|
||||
assets (get-db-assets filtered-db)]
|
||||
;; (prn :public-counts :datoms (count datoms) :assets (count assets))
|
||||
[@(d/conn-from-datoms datoms (:schema db)) assets]))
|
||||
|
||||
(defn- file-filter-only-public
|
||||
[public-pages db datom]
|
||||
(let [ns' (namespace (:a datom))]
|
||||
(and
|
||||
(not (contains? #{:block/file} (:a datom)))
|
||||
(not= ns' "file")
|
||||
(or
|
||||
(not (contains? #{"block" "recent"} ns'))
|
||||
(and (= ns' "block")
|
||||
(or
|
||||
(contains? public-pages (:e datom))
|
||||
(contains? public-pages (:db/id (:block/page (d/entity db (:e datom)))))))))))
|
||||
|
||||
(defn- db-filter-only-public
|
||||
[public-ents _db datom]
|
||||
(contains? public-ents (:e datom)))
|
||||
@@ -263,17 +142,15 @@
|
||||
ents))
|
||||
|
||||
(defn filter-only-public-pages-and-blocks
|
||||
"Prepares a database assuming all pages are private unless a page has a 'public:: true'"
|
||||
[db {:keys [db-graph?]}]
|
||||
"Prepares a database assuming all pages are private unless a page has a publishing-public? property set to true"
|
||||
[db]
|
||||
{:post [(some? %) (sequential? %)]}
|
||||
(let [public-pages* (seq (if db-graph? (get-db-public-pages db) (get-public-pages db)))
|
||||
(let [public-pages* (seq (get-db-public-pages db))
|
||||
public-pages (set/union (set public-pages*)
|
||||
(get-aliases-for-page-ids db public-pages*))
|
||||
filter-fn (if db-graph?
|
||||
(partial db-filter-only-public (get-db-public-ents db public-pages))
|
||||
(partial file-filter-only-public public-pages))
|
||||
filter-fn (partial db-filter-only-public (get-db-public-ents db public-pages))
|
||||
filtered-db (d/filter db filter-fn)
|
||||
datoms (d/datoms filtered-db :eavt)
|
||||
assets (if db-graph? (get-db-assets filtered-db) (get-file-assets db datoms))]
|
||||
assets (get-db-assets filtered-db)]
|
||||
;; (prn :private-counts :internal (count internal-ents) :datoms (count datoms) :assets (count assets))
|
||||
[@(d/conn-from-datoms datoms (:schema db)) assets]))
|
||||
|
||||
@@ -138,13 +138,13 @@ necessary db filtering"
|
||||
(defn build-html
|
||||
"Given the graph's db, filters the db using the given options and returns the
|
||||
generated index.html string and assets used by the html"
|
||||
[db* {:keys [repo app-state repo-config html-options db-graph? dev?]}]
|
||||
[db* {:keys [repo app-state repo-config html-options dev?]}]
|
||||
(let [all-pages-public? (if-let [value (:publishing/all-pages-public? repo-config)]
|
||||
value
|
||||
(:all-pages-public? repo-config))
|
||||
[db asset-filenames'] (if all-pages-public?
|
||||
(db/clean-export! db* {:db-graph? db-graph?})
|
||||
(db/filter-only-public-pages-and-blocks db* {:db-graph? db-graph?}))
|
||||
(db/clean-export! db*)
|
||||
(db/filter-only-public-pages-and-blocks db*))
|
||||
_ (when dev?
|
||||
(println "Exporting" (count (d/datoms db :eavt)) "of" (count (d/datoms db* :eavt)) "datoms and"
|
||||
(count asset-filenames') "asset(s)..."))
|
||||
|
||||
85
deps/publishing/test/logseq/publishing/db_test.cljs
vendored
Normal file
85
deps/publishing/test/logseq/publishing/db_test.cljs
vendored
Normal file
@@ -0,0 +1,85 @@
|
||||
(ns logseq.publishing.db-test
|
||||
(:require [cljs.test :refer [deftest is]]
|
||||
[clojure.set :as set]
|
||||
[datascript.core :as d]
|
||||
[logseq.db :as ldb]
|
||||
[logseq.publishing.db :as publish-db]
|
||||
[logseq.db.test.helper :as db-test]))
|
||||
|
||||
(deftest clean-export!
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
[{:page {:block/title "page1"
|
||||
:build/properties {:logseq.property/publishing-public? false}}
|
||||
:blocks [{:block/title "b11"}
|
||||
{:block/title "b12"}
|
||||
{:block/title "![awesome.png](../assets/awesome_1648822509908_0.png"}]}
|
||||
{:page {:block/title "page2"}
|
||||
:blocks [{:block/title "b21"}
|
||||
{:block/title ""}]}
|
||||
{:page {:block/title "page3"}
|
||||
:blocks [{:block/title "b31"}]}])
|
||||
[filtered-db _assets] (publish-db/clean-export! @conn)
|
||||
exported-pages (->> (d/q '[:find (pull ?b [*])
|
||||
:where [?b :block/name]]
|
||||
filtered-db)
|
||||
(map (comp :block/name first))
|
||||
set)
|
||||
exported-blocks (->> (d/q '[:find (pull ?p [*])
|
||||
:where
|
||||
[?b :block/title]
|
||||
[?b :block/page ?p]
|
||||
[(missing? $ ?p :logseq.property/built-in?)]]
|
||||
filtered-db)
|
||||
(map (comp :block/name first))
|
||||
set)]
|
||||
(is (set/subset? #{"page2" "page3"} exported-pages)
|
||||
"Contains all pages that haven't been marked private")
|
||||
(is (not (contains? exported-pages "page1"))
|
||||
"Doesn't contain private page")
|
||||
(is (= #{"page2" "page3"} exported-blocks)
|
||||
"Only exports blocks from public pages")
|
||||
;; TODO: Create assets to test this
|
||||
#_(is (= ["thumb-on-fire_1648822523866_0.PNG"] assets)
|
||||
"Only exports assets from public pages")))
|
||||
|
||||
(deftest filter-only-public-pages-and-blocks
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
[{:page {:block/title "page1"
|
||||
:build/properties {:logseq.property/publishing-public? false}}
|
||||
:blocks [{:block/title "b11"}
|
||||
{:block/title "b12"}
|
||||
{:block/title "![awesome.png](../assets/awesome_1648822509908_0.png"}]}
|
||||
{:page {:block/title "page2"
|
||||
:build/properties {:logseq.property/publishing-public? true
|
||||
:block/alias #{[:build/page {:block/title "page2-alias"}]}}}
|
||||
:blocks [{:block/title "b21"}
|
||||
{:block/title ""}]}
|
||||
{:page {:block/title "page3"
|
||||
:build/properties {:logseq.property/publishing-public? true}}
|
||||
:blocks [{:block/title "b31"}]}])
|
||||
[filtered-db _assets] (publish-db/filter-only-public-pages-and-blocks @conn)
|
||||
exported-pages (->> (d/q '[:find (pull ?b [*])
|
||||
:where [?b :block/name]]
|
||||
filtered-db)
|
||||
(map (comp :block/name first))
|
||||
set)
|
||||
exported-block-pages (->> (d/q '[:find (pull ?p [*])
|
||||
:where
|
||||
[?b :block/title]
|
||||
[?b :block/page ?p]
|
||||
[(missing? $ ?p :logseq.property/built-in?)]]
|
||||
filtered-db)
|
||||
(map (comp :block/name first))
|
||||
set)]
|
||||
|
||||
(is (set/subset? #{"page2" "page3"} exported-pages)
|
||||
"Contains all pages that have been marked public")
|
||||
(is (not (contains? exported-pages "page1"))
|
||||
"Doesn't contain private page")
|
||||
(is (seq (ldb/get-page filtered-db "page2-alias"))
|
||||
"Alias of public page is exported")
|
||||
(is (= #{"page2" "page3"} exported-block-pages)
|
||||
"Only exports blocks from public pages")
|
||||
;; TODO: Create assets to test this
|
||||
#_(is (= ["thumb-on-fire_1648822523866_0.PNG"] assets)
|
||||
"Only exports assets from public pages")))
|
||||
Reference in New Issue
Block a user