Merge branch 'feat/db' into refactor/block-schema

deps/db/src/logseq/db/frontend/content.cljs

@@ -130,8 +130,8 @@
     item))
 
 (defn replace-tags-with-id-refs
-  "Replace tags in content with page-ref ids. Ignore case because tags in
-  content can have any case and still have a valid ref"
+  "Replace tag names in content with page-ref ids e.g. #TAG -> [[UUID]].
+   Ignore case because tags in content can have any case and still have a valid ref"
   [content tags]
   (->>
    (reduce
@@ -147,3 +147,21 @@
     content
     (sort-refs tags))
    (string/trim)))
+
+(defn replace-tag-refs-with-page-refs
+  "Replace tag refs in content with page refs e.g. #[[UUID]] -> [[UUID]]"
+  [content tags]
+  (->>
+   (reduce
+    (fn [content tag]
+      (let [id-ref (page-ref/->page-ref (:block/uuid tag))]
+        (-> content
+            ;; #[[favorite book]]
+            (common-util/replace-ignore-case
+             (str "#" id-ref)
+             id-ref)
+            ;; #book
+            (common-util/replace-ignore-case (str "#" id-ref) id-ref))))
+    content
+    (sort-refs tags))
+   (string/trim)))

deps/graph-parser/script/db_import.cljs

@@ -51,7 +51,9 @@
 (defn- notify-user [{:keys [continue debug]} m]
   (println (:msg m))
   (when (:ex-data m)
-    (println "Ex-data:" (pr-str (dissoc (:ex-data m) :error)))
+    (println "Ex-data:" (pr-str (merge (dissoc (:ex-data m) :error)
+                                       (when-let [err (get-in m [:ex-data :error])]
+                                         {:original-error (ex-data (.-cause err))}))))
     (println "Stacktrace:")
     (if-let [stack (some-> (get-in m [:ex-data :error]) ex-data :sci.impl/callstack deref)]
       (println (string/join
@@ -172,6 +174,8 @@
 
       (when-let [ignored-props (seq @(:ignored-properties import-state))]
         (println "Ignored properties:" (pr-str ignored-props)))
+      (when-let [ignored-files (seq @(:ignored-files import-state))]
+        (println (count ignored-files) "ignored file(s):" (pr-str (vec ignored-files))))
       (when (:verbose options') (println "Transacted" (count (d/datoms @conn :eavt)) "datoms"))
       (println "Created graph" (str db-name "!")))))
 

deps/graph-parser/src/logseq/graph_parser/exporter.cljs

@@ -1112,6 +1112,8 @@
    ;; Properties are ignored to keep graph valid and notify users of ignored properties.
    ;; Properties with :schema are ignored due to property schema changes
    :ignored-properties (atom [])
+   ;; Vec of maps with keys :path and :reason
+   :ignored-files (atom [])
    ;; Map of property names (keyword) and their current schemas (map).
    ;; Used for adding schemas to properties and detecting changes across a property's usage
    :property-schemas (atom {})
@@ -1232,8 +1234,10 @@
 
 (defn- extract-pages-and-blocks
   "Main fn which calls graph-parser to convert markdown into data"
-  [db file content {:keys [extract-options notify-user]}]
+  [db file content {:keys [extract-options import-state]}]
   (let [format (common-util/get-format file)
+        ;; TODO: Remove once pdf highlights are supported
+        ignored-highlight-file? (string/starts-with? (str (path/basename file)) "hls__")
         extract-options' (merge {:block-pattern (common-config/get-block-pattern format)
                                  :date-formatter "MMM do, yyyy"
                                  :uri-encoded? false
@@ -1241,7 +1245,7 @@
                                  :filename-format :legacy}
                                 extract-options
                                 {:db db})]
-    (cond (contains? common-config/mldoc-support-formats format)
+    (cond (and (contains? common-config/mldoc-support-formats format) (not ignored-highlight-file?))
           (-> (extract/extract file content extract-options')
               (update :pages (fn [pages]
                                (map #(dissoc % :block.temp/original-page-name) pages)))
@@ -1259,7 +1263,11 @@
               (update :blocks update-whiteboard-blocks format))
 
           :else
-          (notify-user {:msg (str "Skipped file since its format is not supported: " file)}))))
+          (if ignored-highlight-file?
+            (swap! (:ignored-files import-state) conj
+                   {:path file :reason :pdf-highlight})
+            (swap! (:ignored-files import-state) conj
+                   {:path file :reason :unsupported-file-format})))))
 
 (defn- build-journal-created-ats
   "Calculate created-at timestamps for journals"

deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs

@@ -66,7 +66,9 @@
 (defn- notify-user [m]
   (println (:msg m))
   (when (:ex-data m)
-    (println "Ex-data:" (pr-str (dissoc (:ex-data m) :error)))
+    (println "Ex-data:" (pr-str (merge (dissoc (:ex-data m) :error)
+                                       (when-let [err (get-in m [:ex-data :error])]
+                                         {:original-error (ex-data (.-cause err))}))))
     (println "Stacktrace:")
     (if-let [stack (some-> (get-in m [:ex-data :error]) ex-data :sci.impl/callstack deref)]
       (println (string/join
@@ -201,7 +203,8 @@
                      count))
           "Correct number of user classes")
       (is (= 4 (count (d/datoms @conn :avet :block/tags :logseq.class/Whiteboard))))
-      (is (= 0 (count @(:ignored-properties import-state))) ":filters should be the only ignored property")
+      (is (= 0 (count @(:ignored-properties import-state))) "No ignored properties")
+      (is (= 1 (count @(:ignored-files import-state))) "Ignore .edn for now")
       (is (= 1 (count @assets))))
 
     (testing "logseq files"

e2e-tests/logseq-api.spec.ts

@@ -4,7 +4,7 @@ import { callPageAPI } from './utils'
 import { Page } from 'playwright'
 
 async function createDBGraph(page: Page) {
-  await page.locator(`a.cp__repos-select-trigger`).click()
+  await page.locator(`#left-sidebar .cp__graphs-selector > a`).click()
   await page.click('text="Create db graph"')
   await page.waitForSelector('.new-graph')
   const name = `e2e-db-${Date.now()}`
@@ -86,6 +86,8 @@ test('(File graph): block related apis',
     expect(mb.uuid).toBe(b.uuid)
 
     // properties
+    // FIXME: redundant api call
+    await callAPI('upsert_block_property', b1.uuid, 'a')
     await callAPI('upsert_block_property', b1.uuid, 'a', 1)
     let prop1 = await callAPI('get_block_property', b1.uuid, 'a')
 
@@ -201,30 +203,12 @@ test('(DB graph): block related apis',
 
     expect(prop1).toEqual({ ':plugin.property/a': 'a', ':plugin.property/b': 'b' })
 
+    // properties schema
+    await callAPI('upsert_property', 'p1')
+    prop1 = await callAPI('get_property', 'p1')
+
+    expect(prop1.title).toBe('p1')
+    expect(prop1.ident).toBe(':plugin.property/p1')
+
     // await page.pause()
   })
-
-/**
- * load local tests plugin
- */
-export async function loadLocalE2eTestsPlugin(page) {
-  const pid = 'a-logseq-plugin-for-e2e-tests'
-  const hasLoaded = await page.evaluate(async ([pid]) => {
-    // @ts-ignore
-    const p = window.LSPluginCore.registeredPlugins.get(pid)
-    // @ts-ignore
-    await window.LSPluginCore.enable(pid)
-    return p != null
-  }, [pid])
-
-  if (hasLoaded) return true
-
-  await callPageAPI(page, 'set_state_from_store',
-    'ui/developer-mode?', true)
-  await page.keyboard.press('t+p')
-  await page.locator('text=Load unpacked plugin')
-  await callPageAPI(page, 'set_state_from_store',
-    'plugin/selected-unpacked-pkg', `${__dirname}/plugin`)
-  await page.keyboard.press('Escape')
-  await page.keyboard.press('Escape')
-}

e2e-tests/plugins.spec.ts

@@ -1,8 +1,32 @@
 import { expect } from '@playwright/test'
 import { test } from './fixtures'
-import {loadLocalE2eTestsPlugin } from './logseq-api.spec'
 import { callPageAPI } from './utils'
 
+/**
+ * load local tests plugin
+ */
+export async function loadLocalE2eTestsPlugin(page) {
+  const pid = 'a-logseq-plugin-for-e2e-tests'
+  const hasLoaded = await page.evaluate(async ([pid]) => {
+    // @ts-ignore
+    const p = window.LSPluginCore.registeredPlugins.get(pid)
+    // @ts-ignore
+    await window.LSPluginCore.enable(pid)
+    return p != null
+  }, [pid])
+
+  if (hasLoaded) return true
+
+  await callPageAPI(page, 'set_state_from_store',
+    'ui/developer-mode?', true)
+  await page.keyboard.press('t+p')
+  await page.locator('text=Load unpacked plugin')
+  await callPageAPI(page, 'set_state_from_store',
+    'plugin/selected-unpacked-pkg', `${__dirname}/plugin`)
+  await page.keyboard.press('Escape')
+  await page.keyboard.press('Escape')
+}
+
 test.skip('enabled plugin system default', async ({ page }) => {
   const callAPI = callPageAPI.bind(null, page)
 

e2e-tests/utils.ts

@@ -140,12 +140,10 @@ export async function loadLocalGraph(page: Page, path: string): Promise<void> {
       await expect(sidebar).toHaveClass(/is-open/)
     }
 
-    await page.click('#left-sidebar .repo-switch');
+    await page.click('#left-sidebar .cp__graphs-selector > a');
     await page.waitForSelector('.cp__repos-quick-actions >> text="Add new graph"',
       { state: 'visible', timeout: 5000 })
     await page.click('text=Add new graph')
-
-    expect(page.locator('.repo-name')).toHaveText(pathlib.basename(path))
   }
 
   setMockedOpenDirPath(page, ''); // reset it

src/main/frontend/components/imports.cljs

@@ -284,6 +284,13 @@
     (log/info :org-files (mapv :path org-files))
     (notification/show! (str "Imported " (count org-files) " org file(s) as markdown. Support for org files will be added later.")
                         :info false))
+  (when-let [ignored-files (seq @(:ignored-files import-state))]
+    (notification/show! (str "Import ignored " (count ignored-files) " "
+                             (if (= 1 (count ignored-files)) "file" "files")
+                             ". See the javascript console for more details.")
+                        :info false)
+    (log/error :import-ignored-files {:msg (str "Import ignored " (count ignored-files) " file(s)")})
+    (pprint/pprint ignored-files))
   (when-let [ignored-props (seq @(:ignored-properties import-state))]
     (notification/show!
      [:.mb-2

src/main/frontend/components/page_menu.cljs

@@ -160,6 +160,11 @@
              :options {:on-click (fn []
                                    (db-page-handler/convert-to-tag! page))}})
 
+          (when (and db-based? (ldb/class? page))
+            {:title (t :page/convert-tag-to-page)
+             :options {:on-click (fn []
+                                   (db-page-handler/convert-tag-to-page! page))}})
+
           (when developer-mode?
             {:title   (t :dev/show-page-data)
              :options {:on-click (fn []

src/main/frontend/components/property.css

@@ -187,6 +187,10 @@
   .select-item {
     @apply flex items-center shrink;
   }
+
+  &[data-type="datetime"] {
+    @apply whitespace-nowrap;
+  }
 }
 
 .block-main-container .ls-properties-area {

src/main/frontend/components/repo.cljs

@@ -411,7 +411,8 @@
                           "Select a Graph")]
     [:div.cp__graphs-selector.flex.items-center.justify-between
      [:a.item.flex.items-center.gap-1.select-none
-      {:on-click (fn [^js e]
+      {:title current-repo
+       :on-click (fn [^js e]
                    (shui/popup-show! (.closest (.-target e) "a")
                                      (fn [{:keys [id]}] (repos-dropdown-content {:contentid id}))
                                      {:as-dropdown? true

src/main/frontend/handler/db_based/page.cljs

@@ -1,19 +1,22 @@
 (ns frontend.handler.db-based.page
   "DB graph only page util fns"
   (:require [clojure.string :as string]
+            [datascript.impl.entity :as de]
             [frontend.db :as db]
-            [frontend.handler.editor :as editor-handler]
             [frontend.handler.common.page :as page-common-handler]
             [frontend.handler.db-based.property :as db-property-handler]
+            [frontend.handler.editor :as editor-handler]
             [frontend.handler.notification :as notification]
             [frontend.state :as state]
-            [logseq.outliner.validate :as outliner-validate]
-            [logseq.db.frontend.class :as db-class]
             [logseq.common.util :as common-util]
             [logseq.common.util.page-ref :as page-ref]
-            [datascript.impl.entity :as de]
+            [logseq.db]
+            [logseq.db.frontend.class :as db-class]
+            [logseq.outliner.validate :as outliner-validate]
             [promesa.core :as p]
-            [logseq.db]))
+            [frontend.db.async :as db-async]
+            [logseq.db.frontend.content :as db-content]
+            [logseq.shui.ui :as shui]))
 
 (defn- valid-tag?
   "Returns a boolean indicating whether the new tag passes all valid checks.
@@ -40,6 +43,7 @@
      (db-property-handler/set-block-property! block-id :block/tags (:db/id tag-entity)))))
 
 (defn convert-to-tag!
+  "Converts a Page to a Tag"
   [page-entity]
   (if (db/page-exists? (:block/title page-entity) #{:logseq.class/Tag})
     (notification/show! (str "A tag with the name \"" (:block/title page-entity) "\" already exists.") :warning false)
@@ -51,6 +55,30 @@
 
       (db/transact! (state/get-current-repo) txs {:outliner-op :save-block}))))
 
+(defn convert-tag-to-page!
+  [page-entity]
+  (if (db/page-exists? (:block/title page-entity) #{:logseq.class/Page})
+    (notification/show! (str "A page with the name \"" (:block/title page-entity) "\" already exists.") :warning false)
+    (p/let [objects (db-async/<get-tag-objects (state/get-current-repo) (:db/id page-entity))]
+      (let [convert-fn
+            (fn convert-fn []
+              (let [page-txs [[:db/retract (:db/id page-entity) :db/ident]
+                              [:db/retract (:db/id page-entity) :block/tags :logseq.class/Tag]
+                              [:db/add (:db/id page-entity) :block/tags :logseq.class/Page]]
+                    obj-txs (mapcat (fn [obj]
+                                      (let [tags (map #(db/entity (state/get-current-repo) (:db/id %)) (:block/tags obj))]
+                                        [{:db/id (:db/id obj)
+                                          :block/title (db-content/replace-tag-refs-with-page-refs (:block/title obj) tags)}
+                                         [:db/retract (:db/id obj) :block/tags (:db/id page-entity)]]))
+                                    objects)
+                    txs (concat page-txs obj-txs)]
+                (db/transact! (state/get-current-repo) txs {:outliner-op :save-block})))]
+        (-> (shui/dialog-confirm!
+             "Converting a tag to page also removes tags from any nodes that have that tag. Are you ok with that?"
+             {:id :convert-tag-to-page
+              :data-reminder :ok})
+            (p/then convert-fn))))))
+
 (defn <create-class!
   "Creates a class page and provides class-specific error handling"
   [title options]

src/main/frontend/handler/profiler.clj

@@ -0,0 +1,9 @@
+(ns ^:no-doc frontend.handler.profiler)
+
+(defmacro arity-n-fn
+  [n f-sym]
+  (let [arg-seq (mapv #(symbol (str "x" %)) (range n))]
+    (vec
+     (for [i (range n)]
+       (let [arg-seq* (vec (take i arg-seq))]
+         `(~'fn ~arg-seq* (apply ~f-sym ~arg-seq*)))))))

src/main/frontend/handler/profiler.cljs

@@ -1,44 +1,54 @@
 (ns frontend.handler.profiler
   "Provides fns for profiling.
   TODO: support both main thread and worker thread."
-  (:require [clojure.string :as string]
-            [goog.object :as g]))
+  (:require-macros [frontend.handler.profiler :refer [arity-n-fn]])
+  (:require [goog.object :as g]))
 
 (def ^:private *fn-symbol->key->call-count (volatile! {}))
 (def ^:private *fn-symbol->key->time-sum (volatile! {}))
 
 (def *fn-symbol->origin-fn (atom {}))
 
+(def ^:private arity-pattern #"cljs\$core\$IFn\$_invoke\$arity\$([0-9]+)")
+
 (defn- get-profile-fn
-  [fn-sym original-fn]
-  (fn profile-fn-inner [& args]
-    (let [start (system-time)
-          r (apply original-fn args)
-          elapsed-time (- (system-time) start)]
-      (vswap! *fn-symbol->key->call-count update-in [fn-sym :total] inc)
-      (vswap! *fn-symbol->key->time-sum update-in [fn-sym :total] #(+ % elapsed-time))
-      r)))
+  [fn-sym original-fn custom-key-fn]
+  (let [arity-ns (keep #(some-> (re-find arity-pattern %) second parse-long) (g/getKeys original-fn))
+        f (fn profile-fn-inner [& args]
+            (let [start (system-time)
+                  r (apply original-fn args)
+                  elapsed-time (- (system-time) start)
+                  k (when custom-key-fn (custom-key-fn args r))]
+              (vswap! *fn-symbol->key->call-count update-in [fn-sym :total] inc)
+              (vswap! *fn-symbol->key->time-sum update-in [fn-sym :total] #(+ % elapsed-time))
+              (when k
+                (vswap! *fn-symbol->key->call-count update-in [fn-sym k] inc)
+                (vswap! *fn-symbol->key->time-sum update-in [fn-sym k] #(+ % elapsed-time)))
+              r))
+        arity-n-fns (arity-n-fn 20 f)]
+    (doseq [n arity-ns]
+      (g/set f (str "cljs$core$IFn$_invoke$arity$" n) (nth arity-n-fns n)))
+    f))
 
 (defn- replace-fn-helper!
-  [ns munged-name fn-sym original-fn-obj]
+  [ns munged-name fn-sym original-fn-obj custom-key-fn]
   (let [ns-obj (find-ns-obj ns)
-        obj-cljs-keys (filter #(string/starts-with? % "cljs$") (js-keys original-fn-obj))]
-    (g/set ns-obj munged-name (get-profile-fn fn-sym original-fn-obj))
-    (let [new-obj (find-ns-obj (str ns "." munged-name))]
-      (doseq [k obj-cljs-keys]
-        (g/set new-obj k (g/get original-fn-obj k))))))
+        profile-fn (get-profile-fn fn-sym original-fn-obj custom-key-fn)]
+    (g/set ns-obj munged-name profile-fn)))
 
 (defn register-fn!
-  [fn-sym & {:as _opts}]
+  "(custom-key-fn args-seq result) return non-nil key"
+  [fn-sym & {:keys [custom-key-fn] :as _opts}]
   (assert (qualified-symbol? fn-sym))
   (let [ns (namespace fn-sym)
         s (munge (name fn-sym))]
     (if-let [original-fn (find-ns-obj (str ns "." s))]
-      (do (replace-fn-helper! ns s fn-sym original-fn)
+      (do (replace-fn-helper! ns s fn-sym original-fn custom-key-fn)
           (swap! *fn-symbol->origin-fn assoc fn-sym original-fn))
       (throw (ex-info (str "fn-sym not found: " fn-sym) {})))))
 
 (defn unregister-fn!
+  "TODO: not working on multi-arity fns"
   [fn-sym]
   (let [ns (namespace fn-sym)
         s (munge (name fn-sym))]
@@ -69,6 +79,5 @@
 (comment
   ;; test multi-arity, variadic fn
   (defn test-fn-to-profile
-    ([] 1)
-    ([_a] 2)
-    ([_a & _args] 3)))
+    ([a b] 1)
+    ([b c d] 2)))

src/main/logseq/api.cljs

@@ -887,10 +887,14 @@
 ;; properties (db only)
 (defn ^:export get_property
   [k]
-  (when-let [k' (and (string? k) (some-> k (sanitize-user-property-name) (keyword)))]
-    (p/let [k (if (qualified-keyword? k') k' (api-block/get-db-ident-for-user-property-name k))
-            p (db-utils/pull k)]
-      (bean/->js (sdk-utils/normalize-keyword-for-json p)))))
+  (this-as this
+    (when-let [k' (and (string? k) (some-> k (sanitize-user-property-name) (keyword)))]
+      (let [prefix (when (some-> js/window.LSPlugin (.-PluginLocal) (instance? this))
+                     (str (.-id this) "."))]
+        (p/let [k (if (qualified-keyword? k') k'
+                    (api-block/get-db-ident-for-user-property-name (str prefix k)))
+                p (db-utils/pull k)]
+          (bean/->js (sdk-utils/normalize-keyword-for-json p)))))))
 
 (defn ^:export upsert_property
   "schema:
@@ -901,24 +905,27 @@
      :public? false}
   "
   [k ^js schema ^js opts]
-  (when-let [k' (and (string? k) (keyword k))]
-    (p/let [opts (or (some-> opts (bean/->clj)) {})
-            name (or (:name opts) (some-> (str k) (string/trim)))
-            k (if (qualified-keyword? k') k'
-                  (api-block/get-db-ident-for-user-property-name k))
-            schema (or (some-> schema (bean/->clj)
-                               (update-keys #(if (contains? #{:hide :public} %)
-                                               (keyword (str (name %) "?")) %))) {})
-            schema (cond-> schema
-                     (string? (:cardinality schema))
-                     (update :cardinality keyword)
-                     (string? (:type schema))
-                     (update :type keyword))
-            p (db-property-handler/upsert-property! k schema
-                                                    (cond-> opts
-                                                      name
-                                                      (assoc :property-name name)))]
-      (bean/->js (sdk-utils/normalize-keyword-for-json p)))))
+  (this-as this
+    (when-let [k' (and (string? k) (keyword k))]
+      (let [prefix (when (some-> js/window.LSPlugin (.-PluginLocal) (instance? this))
+                     (str (.-id this) "."))]
+        (p/let [opts (or (some-> opts (bean/->clj)) {})
+                name (or (:name opts) (some-> (str k) (string/trim)))
+                k (if (qualified-keyword? k') k'
+                    (api-block/get-db-ident-for-user-property-name (str prefix k)))
+                schema (or (some-> schema (bean/->clj)
+                             (update-keys #(if (contains? #{:hide :public} %)
+                                             (keyword (str (name %) "?")) %))) {})
+                schema (cond-> schema
+                         (string? (:cardinality schema))
+                         (update :cardinality keyword)
+                         (string? (:type schema))
+                         (update :type keyword))
+                p (db-property-handler/upsert-property! k schema
+                    (cond-> opts
+                      name
+                      (assoc :property-name name)))]
+          (bean/->js (sdk-utils/normalize-keyword-for-json p)))))))
 
 ;; block properties
 (def ^:export upsert_block_property

src/resources/dicts/en.edn

@@ -156,6 +156,7 @@
  :page/page-already-exists "Page “{1}” already exists!"
  :page/whiteboard-to-journal-error "Whiteboard pages cannot be renamed to journal titles!"
  :page/convert-to-tag "Convert to Tag"
+ :page/convert-tag-to-page "Convert Tag to Page"
  :file/name "File name"
  :file/last-modified-at "Last modified at"
  :file/no-data "No data"