mirror of
https://github.com/logseq/logseq.git
synced 2026-05-24 12:44:22 +00:00
Merge branch 'master' into feat/hnswlib+transformer-js
This commit is contained in:
Tienson Qin
@@ -4,9 +4,9 @@
|
||||
NOTE: This script is also used in CI to confirm graph creation works"
|
||||
(:require ["fs" :as fs]
|
||||
["fs-extra$default" :as fse]
|
||||
["os" :as os]
|
||||
["path" :as node-path]
|
||||
[babashka.cli :as cli]
|
||||
[cljs.pprint :as pprint]
|
||||
[clojure.edn :as edn]
|
||||
[clojure.set :as set]
|
||||
[clojure.string :as string]
|
||||
@@ -14,6 +14,7 @@
|
||||
[logseq.common.util :as common-util]
|
||||
[logseq.common.util.date-time :as date-time-util]
|
||||
[logseq.common.util.page-ref :as page-ref]
|
||||
[logseq.db.common.sqlite-cli :as sqlite-cli]
|
||||
[logseq.db.frontend.property.type :as db-property-type]
|
||||
[logseq.outliner.cli :as outliner-cli]
|
||||
[nbb.classpath :as cp]
|
||||
@@ -89,7 +90,9 @@
|
||||
{:page {:build/journal (date-time-util/date->int two-days-ago)}}
|
||||
|
||||
;; Block property blocks and queries
|
||||
{:page {:block/title "Block Properties"}
|
||||
{:page {:block/title "Block Properties"
|
||||
:build/properties
|
||||
{:logseq.property/description "This page demonstrates all the combinations of property types and single/multiple values that are possible."}}
|
||||
:blocks
|
||||
[{:block/title "default property block" :build/properties {:default "haha"}}
|
||||
{:block/title "default property block" :build/properties {:default-many #{"yee" "haw" "sir"}}}
|
||||
@@ -108,7 +111,9 @@
|
||||
{:block/title "date-many property block" :build/properties {:date-many #{[:build/page {:build/journal today-int}]
|
||||
[:build/page {:build/journal yesterday-int}]}}}
|
||||
{:block/title "datetime property block" :build/properties {:datetime timestamp}}]}
|
||||
{:page {:block/title "Property Queries"}
|
||||
{:page {:block/title "Property Queries"
|
||||
:build/properties
|
||||
{:logseq.property/description "This page demonstrates all property type combinations being queried for a specific value. There should be 2 results for each query, one block and one page."}}
|
||||
:blocks
|
||||
[(query "(property default \"haha\")")
|
||||
(query "(property default-many \"haw\")")
|
||||
@@ -146,7 +151,9 @@
|
||||
[:build/page {:build/journal yesterday-int}]}}}}
|
||||
{:page {:block/title "datetime page" :build/properties {:datetime timestamp}}}
|
||||
|
||||
{:page {:block/title "Has Property Queries"}
|
||||
{:page {:block/title "Has Property Queries"
|
||||
:build/properties
|
||||
{:logseq.property/description "This page demonstrates all property type combinations being queried for having a specific property. There should be 2 results for each query, one block and one page."}}
|
||||
:blocks
|
||||
[(query "(property default)")
|
||||
(query "(property default-many)")
|
||||
@@ -190,6 +197,8 @@
|
||||
"Options spec"
|
||||
{:help {:alias :h
|
||||
:desc "Print help"}
|
||||
:file {:alias :f
|
||||
:desc "File to save generated sqlite.build EDN"}
|
||||
:config {:alias :c
|
||||
:coerce edn/read-string
|
||||
:desc "EDN map to add to config.edn"}})
|
||||
@@ -201,21 +210,23 @@
|
||||
(println (str "Usage: $0 GRAPH-NAME [OPTIONS]\nOptions:\n"
|
||||
(cli/format-opts {:spec spec})))
|
||||
(js/process.exit 1))
|
||||
[dir db-name] (if (string/includes? graph-dir "/")
|
||||
((juxt node-path/dirname node-path/basename) graph-dir)
|
||||
[(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
|
||||
db-path (node-path/join dir db-name "db.sqlite")
|
||||
_ (when (fs/existsSync db-path)
|
||||
init-conn-args (sqlite-cli/->open-db-args graph-dir)
|
||||
db-name (if (= 1 (count init-conn-args)) (first init-conn-args) (second init-conn-args))
|
||||
db-path (apply node-path/join init-conn-args)
|
||||
;; Only remove the directory if the directory is being overwritten
|
||||
_ (when (and (= 2 (count init-conn-args)) (fs/existsSync db-path))
|
||||
(fse/removeSync db-path))
|
||||
conn (outliner-cli/init-conn dir db-name {:additional-config (:config options)
|
||||
:classpath (cp/get-classpath)})
|
||||
{:keys [init-tx block-props-tx]} (outliner-cli/build-blocks-tx (create-init-data))
|
||||
conn (apply outliner-cli/init-conn
|
||||
(conj init-conn-args {:additional-config (:config options)
|
||||
:classpath (cp/get-classpath)}))
|
||||
init-data (create-init-data)
|
||||
_ (when (:file options) (fs/writeFileSync (:file options) (with-out-str (pprint/pprint init-data))))
|
||||
{:keys [init-tx block-props-tx]} (outliner-cli/build-blocks-tx init-data)
|
||||
existing-names (set (map :v (d/datoms @conn :avet :block/title)))
|
||||
conflicting-names (set/intersection existing-names (set (keep :block/title init-tx)))]
|
||||
(when (seq conflicting-names)
|
||||
(println "Error: Following names conflict -" (string/join "," conflicting-names))
|
||||
(js/process.exit 1))
|
||||
(println "DB dir: " (node-path/join dir db-name))
|
||||
(println "Generating" (count (filter :block/name init-tx)) "pages and"
|
||||
(count (filter :block/title init-tx)) "blocks ...")
|
||||
(d/transact! conn init-tx)
|
||||
|
||||
@@ -11,16 +11,15 @@
|
||||
type logseq doesnt' support yet
|
||||
* schema.org assumes no cardinality. For now, only :node properties are given a :cardinality :many"
|
||||
(:require ["fs" :as fs]
|
||||
["os" :as os]
|
||||
["path" :as node-path]
|
||||
[babashka.cli :as cli]
|
||||
[clojure.edn :as edn]
|
||||
[clojure.pprint :as pprint]
|
||||
[clojure.set :as set]
|
||||
[clojure.string :as string]
|
||||
[clojure.walk :as w]
|
||||
[datascript.core :as d]
|
||||
[logseq.db.frontend.malli-schema :as db-malli-schema]
|
||||
[logseq.db.frontend.property :as db-property]
|
||||
[logseq.db.common.sqlite-cli :as sqlite-cli]
|
||||
[logseq.db.sqlite.export :as sqlite-export]
|
||||
[logseq.outliner.cli :as outliner-cli]
|
||||
[nbb.classpath :as cp]
|
||||
[nbb.core :as nbb]))
|
||||
@@ -39,25 +38,23 @@
|
||||
(defn- strip-schema-prefix [s]
|
||||
(string/replace-first s "schema:" ""))
|
||||
|
||||
(defn- ->class-page [class-m class-properties {:keys [verbose renamed-classes renamed-pages]}]
|
||||
(let [parent-class* (class-m "rdfs:subClassOf")
|
||||
parent-class (cond
|
||||
(map? parent-class*)
|
||||
(parent-class* "@id")
|
||||
(vector? parent-class*)
|
||||
(do (when verbose
|
||||
(println "Picked first class for multi-parent class" (pr-str (class-m "@id"))))
|
||||
(get (first parent-class*) "@id"))
|
||||
;; DataTypes are weird in that they are subclassed from
|
||||
;; rdfs:class but that info is omitted. It seems schema
|
||||
;; does this on purpose to display it as a separate tree but
|
||||
;; we want all classes under one tree
|
||||
(contains? (set (as-> (class-m "@type") type'
|
||||
(if (string? type') [type'] type')))
|
||||
"schema:DataType")
|
||||
"schema:DataType")
|
||||
(defn- ->class-page [class-m class-properties {:keys [renamed-classes renamed-pages]}]
|
||||
(let [parent-classes* (class-m "rdfs:subClassOf")
|
||||
parent-classes (cond
|
||||
(map? parent-classes*)
|
||||
[(parent-classes* "@id")]
|
||||
(vector? parent-classes*)
|
||||
(mapv #(get % "@id") parent-classes*)
|
||||
;; DataTypes are weird in that they are subclassed from
|
||||
;; rdfs:class but that info is omitted. It seems schema
|
||||
;; does this on purpose to display it as a separate tree but
|
||||
;; we want all classes under one tree
|
||||
(contains? (set (as-> (class-m "@type") type'
|
||||
(if (string? type') [type'] type')))
|
||||
"schema:DataType")
|
||||
["schema:DataType"])
|
||||
;; Map of owl:equivalentClass exceptions
|
||||
parent-class' (get {"rdfs:Class" "Class"} parent-class parent-class)
|
||||
parent-classes' (mapv #(get {"rdfs:Class" "Class"} % %) parent-classes)
|
||||
properties (class-properties (class-m "@id"))
|
||||
inverted-renamed-classes (set/map-invert renamed-classes)
|
||||
class-name (strip-schema-prefix (class-m "@id"))
|
||||
@@ -66,8 +63,8 @@
|
||||
:build/properties (cond-> {:url url}
|
||||
(class-m "rdfs:comment")
|
||||
(assoc :logseq.property/description (get-comment-string (class-m "rdfs:comment") renamed-pages)))}
|
||||
parent-class'
|
||||
(assoc :build/class-parent (keyword (strip-schema-prefix parent-class')))
|
||||
(seq parent-classes')
|
||||
(assoc :build/class-extends (mapv #(keyword (strip-schema-prefix %)) parent-classes'))
|
||||
(seq properties)
|
||||
(assoc :build/class-properties (mapv (comp keyword strip-schema-prefix) properties)))))
|
||||
|
||||
@@ -79,11 +76,12 @@
|
||||
"schema:Text" :default
|
||||
"schema:URL" :url
|
||||
"schema:Boolean" :checkbox
|
||||
"schema:Date" :date})
|
||||
"schema:Date" :date
|
||||
"schema:DateTime" :datetime})
|
||||
|
||||
(def unsupported-data-types
|
||||
"Schema datatypes, https://schema.org/DataType, that don't have Logseq equivalents"
|
||||
#{"schema:Time" "schema:DateTime"})
|
||||
#{"schema:Time"})
|
||||
|
||||
(defn- get-range-includes [property-m]
|
||||
(let [range-includes (as-> (property-m "schema:rangeIncludes") range-includes*
|
||||
@@ -97,11 +95,12 @@
|
||||
(when (class-map %) :node))
|
||||
range-includes))
|
||||
|
||||
(defn- ->property-page [property-m class-map {:keys [verbose renamed-pages renamed-properties]}]
|
||||
(defn- ->property-page [property-m class-map {:keys [verbose renamed-pages]}]
|
||||
(let [range-includes (get-range-includes property-m)
|
||||
schema-type (get-schema-type range-includes class-map)
|
||||
;; Pick first range to determine type as only one range is supported currently
|
||||
_ (when (and verbose (> (count range-includes) 1))
|
||||
;; Notify when a property is only using one of the available property types (rangeIncludes)
|
||||
;; Doesn't apply to :node since they do include all rangeIncludes
|
||||
_ (when (and verbose (> (count range-includes) 1) (not= schema-type :node))
|
||||
(println "Picked property type:"
|
||||
{:property (property-m "@id") :type schema-type :range-includes (vec range-includes)}))
|
||||
_ (assert schema-type (str "No schema found for property " (property-m "@id")))
|
||||
@@ -111,9 +110,7 @@
|
||||
(throw (ex-info (str "property " (pr-str (property-m "@id"))
|
||||
" with type :node has DataType class values which aren't supported: " datatype-classes) {}))))
|
||||
|
||||
inverted-renamed-properties (set/map-invert renamed-properties)
|
||||
class-name (strip-schema-prefix (property-m "@id"))
|
||||
url (str "https://schema.org/" (get inverted-renamed-properties class-name class-name))]
|
||||
url (str "https://schema.org/" (strip-schema-prefix (property-m "@id")))]
|
||||
{(keyword (strip-schema-prefix (property-m "@id")))
|
||||
(cond-> {:logseq.property/type schema-type
|
||||
:build/properties (cond-> {:url url}
|
||||
@@ -149,10 +146,10 @@
|
||||
(let [range-includes
|
||||
(as-> (prop "schema:rangeIncludes") range-includes*
|
||||
(set (map (fn [m] (m "@id")) (if (map? range-includes*) [range-includes*] range-includes*))))
|
||||
unsupported-data-types
|
||||
unsupported-data-types'
|
||||
(set/intersection range-includes unsupported-data-types)]
|
||||
(and (seq range-includes)
|
||||
(every? (fn [x] (contains? unsupported-data-types x)) range-includes))))
|
||||
(every? (fn [x] (contains? unsupported-data-types' x)) range-includes))))
|
||||
|
||||
(defn- get-vector-conflicts
|
||||
"Given a seq of tuples returns a seq of tuples that conflict i.e. their first element
|
||||
@@ -166,32 +163,14 @@
|
||||
|
||||
(defn- detect-final-conflicts
|
||||
"Does one final detection for conflicts after everything has been renamed"
|
||||
[all-properties all-classes page-tuples]
|
||||
(let [property-ids (map #(vector (% "@id") :property) all-properties)
|
||||
class-ids (map #(vector (% "@id") :class) all-classes)
|
||||
existing-conflicts (get-vector-conflicts (concat property-ids class-ids page-tuples))]
|
||||
[all-classes page-tuples]
|
||||
(let [class-ids (map #(vector (% "@id") :class) all-classes)
|
||||
existing-conflicts (get-vector-conflicts (concat class-ids page-tuples))]
|
||||
(when (seq existing-conflicts) (prn :CONFLICTS existing-conflicts))
|
||||
(assert (empty? existing-conflicts)
|
||||
"There are no conflicts between existing pages, schema classes and properties")))
|
||||
|
||||
(defn- detect-property-conflicts-and-get-renamed-properties
|
||||
"Detects conflicts between properties and existing pages and returns renamed properties"
|
||||
[property-ids existing-pages {:keys [verbose]}]
|
||||
(let [conflicts (get-vector-conflicts (concat property-ids existing-pages))
|
||||
_ (assert (every? #(= 2 (count %)) conflicts) "All conflicts must only be between two elements")
|
||||
renamed-properties (->> conflicts
|
||||
(map #(-> % second first))
|
||||
;; Renaming properties '_property' suffix guarantees uniqueness
|
||||
;; b/c schema.org doesn't use '_' in their names
|
||||
(map #(vector % (str % "_property")))
|
||||
(into {}))]
|
||||
(if verbose
|
||||
(println "Renaming the following properties because they have names that conflict with Logseq's built in pages"
|
||||
(keys renamed-properties) "\n")
|
||||
(when (pos? (count renamed-properties))
|
||||
(println "Renaming" (count renamed-properties) "properties due to page name conflicts")))
|
||||
renamed-properties))
|
||||
|
||||
;; NOTE: There are currently no class conflicts but if some come up this fn can be updated to exclude built-in properties
|
||||
(defn- detect-id-conflicts-and-get-renamed-classes
|
||||
"Detects conflicts between classes AND properties and existing
|
||||
pages. Renames any detected conflicts. Properties and class names conflict in
|
||||
@@ -215,11 +194,12 @@
|
||||
;; b/c schema.org doesn't use '_' in their names
|
||||
(map #(vector % (str % "_Class")))
|
||||
(into {}))]
|
||||
(if verbose
|
||||
(println "Renaming the following classes because they have property names that conflict with Logseq's case insensitive :block/name:"
|
||||
(keys renamed-classes) "\n")
|
||||
(when (pos? (count renamed-classes))
|
||||
(println "Renaming" (count renamed-classes) "classes due to page name conflicts")))
|
||||
(when (seq renamed-classes)
|
||||
(if verbose
|
||||
(println "Renaming the following classes because they have names that conflict with Logseq's case insensitive :block/name:"
|
||||
(keys renamed-classes) "\n")
|
||||
(when (pos? (count renamed-classes))
|
||||
(println "Renaming" (count renamed-classes) "classes due to page name conflicts"))))
|
||||
renamed-classes))
|
||||
|
||||
(defn- get-all-properties [schema-data {:keys [verbose]}]
|
||||
@@ -240,7 +220,7 @@
|
||||
(map #(vector (keyword (strip-schema-prefix (get % "@id")))
|
||||
(->class-page % class-to-properties options)))
|
||||
(into {}))]
|
||||
(assert (= ["Thing"] (keep #(when-not (:build/class-parent %)
|
||||
(assert (= ["Thing"] (keep #(when-not (:build/class-extends %)
|
||||
(:block/title %))
|
||||
(vals classes)))
|
||||
"Thing is the only class that doesn't have a schema.org parent class")
|
||||
@@ -273,31 +253,24 @@
|
||||
"rdfs:Class")
|
||||
schema-data)
|
||||
;; Use built-in description
|
||||
all-properties* (remove #(= "schema:description" (% "@id")) (get-all-properties schema-data options))
|
||||
property-tuples (map #(vector (% "@id") :property) all-properties*)
|
||||
all-properties (remove #(= "schema:description" (% "@id")) (get-all-properties schema-data options))
|
||||
property-tuples (map #(vector (% "@id") :property) all-properties)
|
||||
class-tuples (map #(vector (% "@id") :class) all-classes*)
|
||||
page-tuples (map #(vector (str "schema:" %) :node) existing-pages)
|
||||
renamed-classes (detect-id-conflicts-and-get-renamed-classes
|
||||
property-tuples class-tuples page-tuples options)
|
||||
renamed-properties (detect-property-conflicts-and-get-renamed-properties
|
||||
property-tuples page-tuples options)
|
||||
renamed-pages (merge renamed-classes renamed-properties)
|
||||
;; Note: schema:description refs don't get renamed but they aren't used
|
||||
;; Updates keys like @id, @subClassOf
|
||||
rename-page-ids (fn [m]
|
||||
(w/postwalk (fn [x]
|
||||
(if-let [new-page (and (map? x) (renamed-pages (x "@id")))]
|
||||
(merge x {"@id" new-page})
|
||||
x)) m))
|
||||
rename-class-ids (fn [m]
|
||||
(w/postwalk (fn [x]
|
||||
(if-let [new-page (and (map? x) (renamed-classes (x "@id")))]
|
||||
(merge x {"@id" new-page})
|
||||
x)) m))
|
||||
;; Updates keys like @id, @rangeIncludes, @domainIncludes
|
||||
all-classes (map rename-page-ids all-classes*)
|
||||
all-properties (map rename-page-ids all-properties*)]
|
||||
(detect-final-conflicts all-properties all-classes page-tuples)
|
||||
all-classes (map rename-class-ids all-classes*)]
|
||||
(detect-final-conflicts all-classes page-tuples)
|
||||
{:all-classes all-classes
|
||||
:all-properties all-properties
|
||||
:renamed-properties (->> renamed-properties
|
||||
(map (fn [[k v]] [(strip-schema-prefix k) (strip-schema-prefix v)]))
|
||||
(into {}))
|
||||
:renamed-classes (->> renamed-classes
|
||||
(map (fn [[k v]] [(strip-schema-prefix k) (strip-schema-prefix v)]))
|
||||
(into {}))}))
|
||||
@@ -307,7 +280,7 @@
|
||||
js/JSON.parse
|
||||
(js->clj)
|
||||
(get "@graph"))
|
||||
{:keys [all-classes all-properties renamed-classes renamed-properties]}
|
||||
{:keys [all-classes all-properties renamed-classes]}
|
||||
(get-all-classes-and-properties schema-data existing-pages options)
|
||||
;; Generate data shared across pages and properties
|
||||
class-map (->> all-classes
|
||||
@@ -316,14 +289,14 @@
|
||||
select-class-ids
|
||||
(if (:subset options)
|
||||
["schema:Person" "schema:CreativeWorkSeries" "schema:Organization"
|
||||
"schema:Intangible" "schema:Series" "schema:TVSeries" ; Test class multiple inheritance
|
||||
"schema:Movie" "schema:CreativeWork" "schema:Thing" "schema:Comment"]
|
||||
(keys class-map))
|
||||
class-to-properties (get-class-to-properties select-class-ids all-properties)
|
||||
select-properties (set (mapcat val class-to-properties))
|
||||
options' (assoc options
|
||||
:renamed-classes renamed-classes
|
||||
:renamed-properties renamed-properties
|
||||
:renamed-pages (merge renamed-properties renamed-classes))
|
||||
:renamed-pages renamed-classes)
|
||||
;; Generate pages and properties
|
||||
properties (generate-properties
|
||||
(filter #(contains? select-properties (% "@id")) all-properties)
|
||||
@@ -353,51 +326,32 @@
|
||||
:config {:alias :c
|
||||
:coerce edn/read-string
|
||||
:desc "EDN map to add to config.edn"}
|
||||
:debug {:alias :d
|
||||
:desc "Prints additional debug info and a schema.edn for debugging"}
|
||||
:export {:alias :e
|
||||
:desc "Exports graph to schema.edn"}
|
||||
:subset {:alias :s
|
||||
:desc "Only generate a subset of data for testing purposes"}
|
||||
:verbose {:alias :v
|
||||
:desc "Verbose mode"}})
|
||||
|
||||
(defn- write-debug-file [db]
|
||||
(let [ents (remove #(db-malli-schema/internal-ident? (:db/ident %))
|
||||
(d/q '[:find [(pull ?b [*
|
||||
{:logseq.property.class/properties [:block/title]}
|
||||
{:logseq.property/classes [:block/title]}
|
||||
{:logseq.property/parent [:block/title]}
|
||||
{:block/tags [:block/title]}
|
||||
{:block/refs [:block/title]}]) ...]
|
||||
:in $
|
||||
:where [?b :db/ident ?ident]]
|
||||
db))
|
||||
top-level-properties [:logseq.property/type :logseq.property.class/properties :logseq.property/classes
|
||||
:logseq.property/parent :block/tags]
|
||||
debug-attributes (into [:block/name :block/title :db/cardinality :db/ident :block/refs]
|
||||
top-level-properties)]
|
||||
(fs/writeFileSync "schema-org.edn"
|
||||
(pr-str
|
||||
(->> ents
|
||||
(map (fn [m]
|
||||
(let [props (apply dissoc (db-property/properties m) top-level-properties)]
|
||||
(cond-> (select-keys m debug-attributes)
|
||||
(seq props)
|
||||
(assoc :block/properties (-> (update-keys props name)
|
||||
(update-vals (fn [v]
|
||||
(if (:db/id v)
|
||||
(db-property/property-value-content (d/entity db (:db/id v)))
|
||||
v)))))
|
||||
(seq (:logseq.property.class/properties m))
|
||||
(update :logseq.property.class/properties #(set (map :block/title %)))
|
||||
(some? (:logseq.property/parent m))
|
||||
(update :logseq.property/parent :block/title)
|
||||
(seq (:logseq.property/classes m))
|
||||
(update :logseq.property/classes #(set (map :block/title %)))
|
||||
(seq (:block/tags m))
|
||||
(update :block/tags #(set (map :block/title %)))
|
||||
(seq (:block/refs m))
|
||||
(update :block/refs #(set (map :block/title %)))))))
|
||||
set)))))
|
||||
(defn- write-export-file [db]
|
||||
(let [export-map* (sqlite-export/build-export db {:export-type :graph-ontology})
|
||||
;; Modify export to provide stable diff like diff-graphs.
|
||||
;; Remove this when export has stable sort order for these keys
|
||||
export-map (-> export-map*
|
||||
(update :classes update-vals
|
||||
(fn [m]
|
||||
(cond-> m
|
||||
(:build/class-extends m)
|
||||
(update :build/class-extends (comp vec sort))
|
||||
(:build/class-properties m)
|
||||
(update :build/class-properties (comp vec sort)))))
|
||||
(update :properties update-vals
|
||||
(fn [m]
|
||||
(cond-> m
|
||||
(:build/property-classes m)
|
||||
(update :build/property-classes (comp vec sort))))))]
|
||||
(fs/writeFileSync "schema.edn"
|
||||
(with-out-str (pprint/pprint export-map)))))
|
||||
|
||||
(defn -main [args]
|
||||
(let [[graph-dir] args
|
||||
@@ -406,11 +360,11 @@
|
||||
(println (str "Usage: $0 GRAPH-NAME [OPTIONS]\nOptions:\n"
|
||||
(cli/format-opts {:spec spec})))
|
||||
(js/process.exit 1))
|
||||
[dir db-name] (if (string/includes? graph-dir "/")
|
||||
((juxt node-path/dirname node-path/basename) graph-dir)
|
||||
[(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
|
||||
conn (outliner-cli/init-conn dir db-name {:additional-config (:config options)
|
||||
:classpath (cp/get-classpath)})
|
||||
init-conn-args (sqlite-cli/->open-db-args graph-dir)
|
||||
db-name (if (= 1 (count init-conn-args)) (first init-conn-args) (second init-conn-args))
|
||||
conn (apply outliner-cli/init-conn
|
||||
(conj init-conn-args {:additional-config (:config options)
|
||||
:classpath (cp/get-classpath)}))
|
||||
init-data (create-init-data (d/q '[:find [?name ...] :where [?b :block/name ?name]] @conn)
|
||||
options)
|
||||
{:keys [init-tx block-props-tx]} (outliner-cli/build-blocks-tx init-data)]
|
||||
@@ -420,7 +374,7 @@
|
||||
(d/transact! conn init-tx)
|
||||
(d/transact! conn block-props-tx)
|
||||
(when (:verbose options) (println "Transacted" (count (d/datoms @conn :eavt)) "datoms"))
|
||||
(when (:debug options) (write-debug-file @conn))
|
||||
(when (:export options) (write-export-file @conn))
|
||||
(println "Created graph" (str db-name "!"))))
|
||||
|
||||
(when (= nbb/*file* (nbb/invoked-file))
|
||||
|
||||
@@ -18,8 +18,10 @@
|
||||
"electron.db"
|
||||
"frontend.handler.db-based."
|
||||
"frontend.worker.handler.page.db-based"
|
||||
"frontend.components.property" "frontend.components.class"
|
||||
"frontend.components.db-based" "frontend.components.objects" "frontend.components.query.view"]))
|
||||
"frontend.components.property" "frontend.components.class" "frontend.components.quick-add"
|
||||
"frontend.components.db-based" "frontend.components.objects" "frontend.components.query.view"
|
||||
"mobile.core" "mobile.events" "mobile.externals" "mobile.init" "mobile.ionic" "mobile.state"
|
||||
"mobile.components"]))
|
||||
|
||||
(def file-graph-ns
|
||||
"Namespaces or parent namespaces _only_ for file graphs"
|
||||
@@ -55,13 +57,16 @@
|
||||
"src/main/frontend/components/property.cljs"
|
||||
"src/main/frontend/components/property"
|
||||
"src/main/frontend/components/objects.cljs"
|
||||
"src/main/frontend/components/quick_add.cljs"
|
||||
"src/main/frontend/components/db_based"
|
||||
"src/main/frontend/components/query/view.cljs"
|
||||
"src/electron/electron/db.cljs"]))
|
||||
"src/electron/electron/db.cljs"
|
||||
"src/main/mobile"]))
|
||||
|
||||
(def file-graph-paths
|
||||
"Paths _only_ for file graphs"
|
||||
["deps/graph-parser/src/logseq/graph_parser/db.cljs"
|
||||
["deps/db/src/logseq/db/file_based"
|
||||
"deps/graph-parser/src/logseq/graph_parser/db.cljs"
|
||||
"deps/graph-parser/src/logseq/graph_parser/extract.cljc"
|
||||
"deps/graph-parser/src/logseq/graph_parser/property.cljs"
|
||||
"deps/graph-parser/src/logseq/graph_parser.cljs"
|
||||
@@ -98,8 +103,10 @@
|
||||
|
||||
(defn- validate-multi-graph-fns-not-in-file-or-db
|
||||
[]
|
||||
;; TODO: Lint `(db-based-graph?` when db.frontend.entity-plus is split into separate graph contexts
|
||||
(let [multi-graph-fns ["/db-based-graph\\?"
|
||||
(let [multi-graph-fns ["/db-based-graph\\?" "\\(db-based-graph\\?"
|
||||
;; from frontend.handler.property.util
|
||||
"/get-pid"
|
||||
"logseq.db.common.property-util"
|
||||
;; Use file-entity-util and entity-util when in a single graph context
|
||||
"ldb/whiteboard\\?" "ldb/journal\\?" "ldb/page\\?"]
|
||||
res (grep-many multi-graph-fns (into file-graph-paths db-graph-paths))]
|
||||
@@ -118,18 +125,22 @@
|
||||
(map str)
|
||||
(into [;; e.g. block/properties :title
|
||||
"block/properties :"
|
||||
;; anything org mode except for org.babashka
|
||||
"org[^\\.]"
|
||||
;; anything org mode except for org.babashka or urls like schema.org
|
||||
"[^\\.]org[^\\.]"
|
||||
"file-based"
|
||||
"#+BEGIN_"
|
||||
"#+END_"
|
||||
"pre-block"]))
|
||||
;; For now use the whole code line. If this is too brittle can make this smaller
|
||||
allowed-exceptions #{":block/pre-block? :block/scheduled :block/deadline :block/type :block/name :block/marker"
|
||||
"(dissoc :block/format))]"
|
||||
;; The next 3 are from components.property.value
|
||||
"{:block/name page-title})"
|
||||
"(when-not (db/get-page journal)"
|
||||
"(let [value (if datetime? (tc/to-long d) (db/get-page journal))]"}
|
||||
;; TODO: Mv these 2 file-based ns out of db files
|
||||
"(:require [logseq.db.file-based.rules :as file-rules]))"
|
||||
"[logseq.db.file-based.schema :as file-schema]))"
|
||||
;; :block/name ones from src/main/mobile
|
||||
"(if-let [journal (db/get-page page-name)]"
|
||||
"(p/then #(mobile-state/open-block-modal! (db/get-page page-name)))))))]"}
|
||||
res (grep-many file-concepts db-graph-paths)
|
||||
invalid-lines (when (= 0 (:exit res))
|
||||
(remove #(some->> (string/split % #":\s+") second string/trim (contains? allowed-exceptions))
|
||||
@@ -159,7 +170,7 @@
|
||||
(let [db-concepts
|
||||
;; from logseq.db.frontend.schema
|
||||
["closed-value" "class/properties" "classes" "property/parent"
|
||||
"logseq.property" "logseq.class"]
|
||||
"logseq.property" "logseq.class" "db-based" "library" "quick-add"]
|
||||
res (grep-many db-concepts file-graph-paths)]
|
||||
(when-not (and (= 1 (:exit res)) (= "" (:out res)))
|
||||
(println "The following files should not have contained db specific concepts:")
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
(ns logseq.tasks.dev.lint
|
||||
(:require [clojure.string :as string]
|
||||
[babashka.process :refer [shell]]))
|
||||
(:require [babashka.process :refer [shell]]
|
||||
[clojure.string :as string]))
|
||||
|
||||
(defn dev
|
||||
"Run all lint tasks
|
||||
@@ -34,14 +34,14 @@
|
||||
;; remove files that aren't in a kondo dir
|
||||
((fn [x] (dissoc x nil))))]
|
||||
(if (seq dir-to-files)
|
||||
(doseq [[dir* files*] dir-to-files]
|
||||
(let [dir (if (= dir* "src") "." dir*)
|
||||
files (mapv #(string/replace-first % (str dir "/") "") files*)
|
||||
cmd (str "cd " dir " && clj-kondo --lint " (string/join " " files))
|
||||
_ (println cmd)
|
||||
res (apply shell {:dir dir :continue :true} "clj-kondo --lint" files)]
|
||||
(when (pos? (:exit res)) (System/exit (:exit res)))))
|
||||
(println "No clj* files have changed to lint."))))
|
||||
(doseq [[dir* files*] dir-to-files]
|
||||
(let [dir (if (= dir* "src") "." dir*)
|
||||
files (mapv #(string/replace-first % (str dir "/") "") files*)
|
||||
cmd (str "cd " dir " && clj-kondo --lint " (string/join " " files))
|
||||
_ (println cmd)
|
||||
res (apply shell {:dir dir :continue :true} "clj-kondo --lint" files)]
|
||||
(when (pos? (:exit res)) (System/exit (:exit res)))))
|
||||
(println "No clj* files have changed to lint."))))
|
||||
|
||||
(defn- validate-frontend-not-in-worker
|
||||
[]
|
||||
@@ -62,12 +62,16 @@
|
||||
[]
|
||||
(let [res (shell {:out :string :continue true}
|
||||
"grep -r --exclude-dir=worker" "\\[frontend.worker.*:" "src/main/frontend")
|
||||
req-lines (->> (:out res) string/split-lines)]
|
||||
(if-not (and (= 1 (:exit res)) (= "" (:out res)))
|
||||
(do
|
||||
(println "The following worker requires should not be in frontend namespaces:")
|
||||
(println (:out res))
|
||||
(System/exit 1))
|
||||
;; allow reset-file b/c it's only affects tests
|
||||
allowed-exceptions #{"src/main/frontend/handler/file_based/file.cljs: [frontend.worker.file.reset :as file-reset]"}
|
||||
invalid-lines (when (= 0 (:exit res))
|
||||
(remove #(some->> % (contains? allowed-exceptions))
|
||||
(string/split-lines (:out res))))
|
||||
_ (when (> (:exit res) 1) (System/exit 1))]
|
||||
(if (and (= 0 (:exit res)) (seq invalid-lines))
|
||||
(do (println "The following worker requires should not be in frontend namespaces:")
|
||||
(println (string/join "\n" invalid-lines))
|
||||
(System/exit 1))
|
||||
(println "Valid frontend namespaces!"))))
|
||||
|
||||
(defn worker-and-frontend-separate
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
(ns logseq.tasks.dev.mobile
|
||||
"Tasks for mobile development"
|
||||
(:require [babashka.tasks :refer [shell]]
|
||||
[babashka.fs :as fs]
|
||||
(:require [babashka.fs :as fs]
|
||||
[babashka.tasks :refer [shell]]
|
||||
[clojure.string :as string]
|
||||
[logseq.tasks.util :as task-util]))
|
||||
|
||||
@@ -15,16 +15,17 @@
|
||||
(shell cmd)
|
||||
(println "Waiting for app to build..."))
|
||||
(Thread/sleep 1000)
|
||||
(when-not (or (and (fs/exists? "ios/App/App/public/static/js/main.js")
|
||||
(task-util/file-modified-later-than? "ios/App/App/public/static/js/main.js" start-time))
|
||||
(and (fs/exists? "android/App/src/main/assets/public/static/js/main.js")
|
||||
(task-util/file-modified-later-than? "android/App/src/main/assets/public/static/js/main.js" start-time)))
|
||||
(when-not (or (and (fs/exists? "ios/App/App/public/js/main.js")
|
||||
(task-util/file-modified-later-than? "ios/App/App/public/js/main.js" start-time))
|
||||
(and (fs/exists? "android/App/src/main/assets/public/js/main.js")
|
||||
(task-util/file-modified-later-than? "android/App/src/main/assets/public/js/main.js" start-time)))
|
||||
(recur (dec n))))))
|
||||
|
||||
(defn- set-system-env
|
||||
"Updates capacitor.config.ts serve url with IP from ifconfig"
|
||||
[]
|
||||
(let [ip (string/trim (:out (shell {:out :string} "ipconfig getifaddr en0")))
|
||||
(let [ip (string/trim (:out (or (shell {:out :string :continue true} "ipconfig getifaddr en0")
|
||||
(shell {:out :string} "ipconfig getifaddr en1"))))
|
||||
logseq-app-server-url (format "%s://%s:%s" "http" ip "3001")]
|
||||
(println "Server URL:" logseq-app-server-url)
|
||||
(shell "git checkout capacitor.config.ts")
|
||||
@@ -37,7 +38,6 @@
|
||||
(string/replace "// }" " }"))]
|
||||
(spit "capacitor.config.ts" new-body))))
|
||||
|
||||
|
||||
(defn app-watch
|
||||
"Watches environment to reload cljs, css and other assets for mobile"
|
||||
[]
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
(ns logseq.tasks.lang
|
||||
"Tasks related to language translations"
|
||||
(:require [clojure.set :as set]
|
||||
(:require [babashka.cli :as cli]
|
||||
[babashka.fs :as fs]
|
||||
[babashka.process :refer [shell]]
|
||||
[borkdude.rewrite-edn :as rewrite]
|
||||
[clojure.set :as set]
|
||||
[clojure.string :as string]
|
||||
[frontend.dicts :as dicts]
|
||||
[logseq.tasks.util :as task-util]
|
||||
[babashka.cli :as cli]
|
||||
[babashka.process :refer [shell]]
|
||||
[babashka.fs :as fs]
|
||||
[borkdude.rewrite-edn :as r]))
|
||||
[logseq.tasks.util :as task-util]))
|
||||
|
||||
(defn- get-dicts
|
||||
[]
|
||||
@@ -78,10 +78,10 @@
|
||||
[invalid-keys-by-lang]
|
||||
(doseq [[lang invalid-keys] invalid-keys-by-lang]
|
||||
(let [path (fs/path "src/resources/dicts" (str (name lang) ".edn"))
|
||||
result (r/parse-string (String. (fs/read-all-bytes path)))
|
||||
result (rewrite/parse-string (String. (fs/read-all-bytes path)))
|
||||
new-content (str (reduce
|
||||
(fn [result k]
|
||||
(r/dissoc result k))
|
||||
(rewrite/dissoc result k))
|
||||
result invalid-keys))]
|
||||
(spit (fs/file path) new-content))))
|
||||
|
||||
@@ -148,10 +148,10 @@
|
||||
[invalid-keys]
|
||||
(let [paths (fs/list-dir "src/resources/dicts")]
|
||||
(doseq [path paths]
|
||||
(let [result (r/parse-string (String. (fs/read-all-bytes path)))
|
||||
(let [result (rewrite/parse-string (String. (fs/read-all-bytes path)))
|
||||
new-content (str (reduce
|
||||
(fn [result k]
|
||||
(r/dissoc result k))
|
||||
(rewrite/dissoc result k))
|
||||
result invalid-keys))]
|
||||
(spit (fs/file path) new-content)))))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user