refactor: block properties

- Added :block/created-at and :block/updated-at
- Update block will update page's timestamp too
- Store properties under `:block/properties` in metadata.edn

src/main/frontend/components/block.cljs

@@ -51,7 +51,8 @@
             [shadow.loader :as loader]
             [frontend.search :as search]
             [frontend.debug :as debug]
-            [frontend.modules.outliner.tree :as tree]))
+            [frontend.modules.outliner.tree :as tree]
+            [clojure.walk :as walk]))
 
 ;; TODO: remove rum/with-context because it'll make reactive queries not working
 
@@ -1025,8 +1026,7 @@
                         (seq body)))
         collapsed? (rum/react *collapsed?)
         control-show? (util/react *control-show?)
-        dark? (= "dark" (state/sub :ui/theme))
-        heading? (= (get (:block/properties block) "heading") "true")]
+        dark? (= "dark" (state/sub :ui/theme))]
     [:div.mr-2.flex.flex-row.items-center
      {:style {:height 24
               :margin-top 0
@@ -1065,8 +1065,7 @@
                     (when (and (:document/mode? config)
                                (not collapsed?))
                       "hide-inner-bullet"))}
-       [:span.bullet {:blockid (str uuid)
-                      :class (if heading? "bullet-heading" "")}]]]]))
+       [:span.bullet {:blockid (str uuid)}]]]]))
 
 (defn- build-id
   [config]
@@ -1213,8 +1212,7 @@
         priority (priority-cp t)
         tags (block-tags-cp t)
         contents? (= (:id config) "contents")
-        heading? (= (get properties "heading") "true")
-        bg-color (get properties "background_color")]
+        bg-color (:background-color properties)]
     (->elem
      :div
      (merge
@@ -1281,9 +1279,13 @@
 (rum/defc property-cp
   [config block k v]
   [:div.my-1
-   [:b k]
+   [:b (name k)]
    [:span.mr-1 ":"]
-   (if (coll? v)
+   (cond
+     (int? v)
+     v
+
+     (coll? v)
      (let [v (->> (remove string/blank? v)
                   (filter string?))
            vals (for [v-item v]
@@ -1291,6 +1293,8 @@
            elems (interpose (span-comma) vals)]
        (for [elem elems]
          (rum/with-key elem (str (random-uuid)))))
+
+     :else
      (let [page-name (string/lower-case (str v))]
        (if (db/entity [:block/name page-name])
          (page-cp config {:block/name page-name})
@@ -1298,7 +1302,8 @@
 
 (rum/defc properties-cp
   [config block]
-  (let [properties (apply dissoc (:block/properties block) text/hidden-properties)]
+  (let [properties (walk/keywordize-keys (:block/properties block))
+        properties (apply dissoc properties text/hidden-properties)]
     (when (seq properties)
       [:div.blocks-properties.text-sm.opacity-80.my-1.p-2
        (for [[k v] properties]

src/main/frontend/components/content.cljs

@@ -128,9 +128,7 @@
   [target block-id]
   (rum/with-context [[t] i18n/*tongue-context*]
     (when-let [block (db/entity [:block/uuid block-id])]
-      (let [properties (:block/properties block)
-            heading (get properties "heading")
-            heading? (= heading "true")]
+      (let [properties (:block/properties block)]
         [:div#custom-context-menu
          [:div.py-1.rounded-md.bg-base-3.shadow-xs
           [:div.flex-row.flex.justify-between.py-4.pl-2
@@ -138,25 +136,15 @@
             (for [color block-background-colors]
               [:a.m-2.shadow-sm
                {:on-click (fn [_e]
-                            (editor-handler/set-block-property! block-id "background_color" color))}
+                            (editor-handler/set-block-property! block-id :background-color color))}
                [:div.heading-bg {:style {:background-color color}}]])]
            [:a.text-sm
             {:title (t :remove-background)
              :style {:margin-right 14
                      :margin-top 4}
              :on-click (fn [_e]
-                         (editor-handler/remove-block-property! block-id "background_color"))}
+                         (editor-handler/remove-block-property! block-id :background-color))}
             "Clear"]]
-          (ui/menu-link
-           {:key "Convert heading"
-            :on-click (fn [_e]
-                        (if heading?
-                          (editor-handler/remove-block-property! block-id "heading")
-                          (editor-handler/set-block-as-a-heading! block-id true)))}
-           (if heading?
-             "Convert back to a block"
-             "Convert to a heading"))
-
           (let [empty-properties? (not (text/contains-properties? (:block/content block)))
                 all-hidden? (text/properties-hidden? (:block/properties block))]
             (when (or empty-properties? all-hidden?)

src/main/frontend/format/block.cljs

@@ -157,14 +157,8 @@
    (= "List" (first block))
    (:name (first (second block)))))
 
-(defn- ->schema-properties
-  [properties]
-  (-> properties
-      (update "created_at" util/safe-parse-int)
-      (update "last_modified_at" util/safe-parse-int)))
-
 (defonce non-parsing-properties
-  (atom #{"background_color"}))
+  (atom #{:background-color}))
 
 (defn extract-properties
   [[_ properties] _start-pos _end-pos]
@@ -184,7 +178,8 @@
         properties (->> properties
                         (medley/map-kv (fn [k v]
                                          (let [v (string/trim v)
-                                               k (string/replace k " " "_")]
+                                               k (string/replace k " " "-")
+                                               k (string/replace k "_" "-")]
                                            (cond
                                              (and (= "\"" (first v) (last v))) ; wrapped in ""
                                              [(string/lower-case k) (string/trim (subs v 1 (dec (count v))))]
@@ -202,8 +197,7 @@
                                                                (util/safe-parse-int v'))
                                                         (util/safe-parse-int v')
                                                         (text/split-page-refs-without-brackets v' comma?))]
-                                               [k' v'])))))
-                        (->schema-properties))]
+                                               [k' v']))))))]
     {:properties properties
      :page-refs page-refs}))
 

src/main/frontend/handler/config.cljs

@@ -1,27 +1,14 @@
 (ns frontend.handler.config
   (:require [frontend.state :as state]
             [frontend.handler.file :as file-handler]
-            [borkdude.rewrite-edn :as rewrite]
             [frontend.config :as config]
             [frontend.db :as db]
             [clojure.string :as string]))
 
 (defn set-config!
   [k v]
-  (when-let [repo (state/get-current-repo)]
-    (let [path (config/get-config-path)]
-      (when-let [config (db/get-file-no-sub path)]
-        (let [config (try
-                       (rewrite/parse-string config)
-                       (catch js/Error e
-                         (println "Parsing config file failed: ")
-                         (js/console.dir e)
-                         {}))
-              ks (if (vector? k) k [k])
-              new-config (rewrite/assoc-in config ks v)]
-          (state/set-config! repo new-config)
-          (let [new-content (str new-config)]
-            (file-handler/set-file-content! repo path new-content)))))))
+  (let [path (config/get-config-path)]
+    (file-handler/edn-file-set-key-value path k v state/set-config!)))
 
 (defn toggle-ui-show-brackets! []
   (let [show-brackets? (state/show-brackets?)]

src/main/frontend/handler/editor.cljs

@@ -455,7 +455,7 @@
         new-m {:block/uuid (db/new-block-id)
                :block/content snd-block-text}
         next-block (-> (merge block new-m)
-                     (dissoc :db/id :block/collapsed?)
+                       (dissoc :db/id :block/collapsed? :block/properties :block/pre-block? :block/meta)
                      (wrap-parse-block))]
     (do
       (profile
@@ -728,33 +728,37 @@
                     :data blocks}]
           (db/refresh repo opts))))))
 
+(defn- block-property-aux!
+  [block-id key value]
+  (let [block-id (if (string? block-id) (uuid block-id) block-id)
+        key (keyword (string/lower-case (name key)))
+        repo (state/get-current-repo)]
+    (when repo
+      (when-let [block (db/entity [:block/uuid block-id])]
+        (let [properties (:block/properties block)
+              properties (if value        ; add
+                           (assoc properties key value)
+                           (dissoc properties key))
+              block (outliner-core/block {:block/uuid block-id
+                                          :block/properties properties})]
+          (outliner-core/save-node block)
+          (let [opts {:key :block/change
+                      :data [block]}]
+            (db/refresh repo opts)))))))
+
 (defn remove-block-property!
   [block-id key]
-  (let [block-id (if (string? block-id) (uuid block-id) block-id)
-        key (string/lower-case (str key))]
-    (when-let [block (db/pull [:block/uuid block-id])]
-      (let [{:block/keys [content properties]} block]
-        (when (get properties key)
-          (save-block-if-changed! block content
-                                  {:remove-properties [key]}))))))
+  (block-property-aux! block-id key nil)
+  (db/refresh (state/get-current-repo)
+              {:key :block/change
+               :data [(db/pull [:block/uuid block-id])]}))
 
 (defn set-block-property!
   [block-id key value]
-  (let [block-id (if (string? block-id) (uuid block-id) block-id)
-        key (string/lower-case (str key))
-        value (str value)]
-    (when-let [block (db/pull [:block/uuid block-id])]
-      (when-not (:block/pre-block? block)
-        (let [{:block/keys [content properties]} block]
-          (cond
-            (and (string? (get properties key))
-                 (= (string/trim (get properties key)) value))
-            nil
-
-            :else
-            (save-block-if-changed! block content
-                                    {:custom-properties {key value}
-                                     :rebuild-content? false})))))))
+  (block-property-aux! block-id key value)
+  (db/refresh (state/get-current-repo)
+              {:key :block/change
+               :data [(db/pull [:block/uuid block-id])]}))
 
 (defn set-block-timestamp!
   [block-id key value]
@@ -1561,10 +1565,6 @@
       (.focus input)
       (util/move-cursor-to input saved-cursor))))
 
-(defn set-block-as-a-heading!
-  [block-id value]
-  (set-block-property! block-id "heading" value))
-
 (defn open-block!
   [first?]
   (fn [e]

src/main/frontend/handler/extract.cljs

@@ -143,7 +143,7 @@
        file content utf8-content journal?))))
 
 (defn extract-all-blocks-pages
-  [repo-url files]
+  [repo-url files metadata]
   (when (seq files)
     (let [result (->> files
                       (map
@@ -155,6 +155,11 @@
                       (remove empty?))]
       (when (seq result)
         (let [[pages block-ids blocks] (apply map concat result)
+              blocks (map (fn [block]
+                            (let [id (:block/uuid block)
+                                  properties (get-in metadata [:block/properties id])]
+                              (update block :block/properties merge properties)))
+                       blocks)
               pages (util/distinct-by :block/name pages)
               block-ids-set (set (map (fn [{:block/keys [uuid]}] [:block/uuid uuid]) block-ids))
               blocks (map (fn [b]

src/main/frontend/handler/file.cljs

@@ -24,6 +24,7 @@
             [cljs-time.core :as t]
             [cljs-time.coerce :as tc]
             [frontend.utf8 :as utf8]
+            [borkdude.rewrite-edn :as rewrite]
             ["/frontend/utils" :as utils]))
 
 ;; TODO: extract all git ops using a channel
@@ -326,3 +327,20 @@
             file-exists? (fs/create-if-not-exists repo-url repo-dir file-path default-content)]
       (when-not file-exists?
         (reset-file! repo-url path default-content)))))
+
+(defn edn-file-set-key-value
+  [path k v ok-handler]
+  (when-let [repo (state/get-current-repo)]
+    (when-let [content (db/get-file-no-sub path)]
+      (let [result (try
+                     (rewrite/parse-string content)
+                     (catch js/Error e
+                       (println "Parsing config file failed: ")
+                       (js/console.dir e)
+                       {}))
+            ks (if (vector? k) k [k])
+            new-result (rewrite/assoc-in result ks v)]
+        (when ok-handler (ok-handler repo new-result))
+        (state/set-config! repo new-result)
+        (let [new-content (str new-result)]
+          (set-file-content! repo path new-content))))))

src/main/frontend/handler/metadata.cljs

@@ -4,8 +4,10 @@
             [cljs.reader :as reader]
             [frontend.config :as config]
             [frontend.db :as db]
+            [datascript.db :as ddb]
             [clojure.string :as string]
-            [promesa.core :as p]))
+            [promesa.core :as p]
+            [medley.core :as medley]))
 
 (def default-metadata-str "{}")
 
@@ -23,11 +25,16 @@
                            (println "Parsing metadata.edn failed: ")
                            (js/console.dir e)
                            {}))
-              ks (if (vector? k) k [k])
-              new-metadata (assoc-in metadata ks v)
+              new-metadata (cond
+                             (= k :block/properties)
+                             (update metadata :block/properties v) ; v should be a function
+                             :else
+                             (let [ks (if (vector? k) k [k])]
+                               (assoc-in metadata ks v)))
               new-metadata (if encrypted?
                              (assoc new-metadata :db/encrypted? true)
                              new-metadata)
+              _ (prn "New metadata:\n" new-metadata)
               new-content (pr-str new-metadata)]
           (file-handler/set-file-content! repo path new-content))))))
 
@@ -35,3 +42,26 @@
   [encrypted-secret]
   (when-not (string/blank? encrypted-secret)
     (set-metadata! :db/encrypted-secret encrypted-secret)))
+
+(defn- handler-properties!
+  [all-properties properties-tx]
+  (reduce
+   (fn [acc datom]
+     (let [v (:v datom)
+           id (or (get v :id)
+                  (get v :title))]
+       (if id
+         (let [added? (ddb/datom-added datom)
+               remove-all-properties? (and (not added?)
+                                           ;; only id
+                                           (= 1 (count v)))]
+           (if remove-all-properties?
+             (dissoc acc id)
+             (assoc acc id v)))
+         acc)))
+   all-properties
+   properties-tx))
+
+(defn update-properties!
+  [properties-tx]
+  (set-metadata! :block/properties #(handler-properties! % properties-tx)))

src/main/frontend/handler/repo.cljs

@@ -207,14 +207,14 @@
     (db/transact! repo-url all-data)))
 
 (defn- parse-files-and-create-default-files-inner!
-  [repo-url files delete-files delete-blocks file-paths first-clone? db-encrypted? re-render? re-render-opts]
+  [repo-url files delete-files delete-blocks file-paths first-clone? db-encrypted? re-render? re-render-opts metadata]
   (let [parsed-files (filter
                       (fn [file]
                         (let [format (format/get-format (:file/path file))]
                           (contains? config/mldoc-support-formats format)))
                       files)
         blocks-pages (if (seq parsed-files)
-                       (extract-handler/extract-all-blocks-pages repo-url parsed-files)
+                       (extract-handler/extract-all-blocks-pages repo-url parsed-files metadata)
                        [])]
     (let [config-file (config/get-config-path)]
       (when (contains? (set file-paths) config-file)
@@ -234,15 +234,15 @@
     (state/set-importing-to-db! false)))
 
 (defn- parse-files-and-create-default-files!
-  [repo-url files delete-files delete-blocks file-paths first-clone? db-encrypted? re-render? re-render-opts]
+  [repo-url files delete-files delete-blocks file-paths first-clone? db-encrypted? re-render? re-render-opts metadata]
   (if db-encrypted?
     (p/let [files (p/all
                    (map (fn [file]
                           (p/let [content (encrypt/decrypt (:file/content file))]
                             (assoc file :file/content content)))
                         files))]
-      (parse-files-and-create-default-files-inner! repo-url files delete-files delete-blocks file-paths first-clone? db-encrypted? re-render? re-render-opts))
-    (parse-files-and-create-default-files-inner! repo-url files delete-files delete-blocks file-paths first-clone? db-encrypted? re-render? re-render-opts)))
+      (parse-files-and-create-default-files-inner! repo-url files delete-files delete-blocks file-paths first-clone? db-encrypted? re-render? re-render-opts metadata))
+    (parse-files-and-create-default-files-inner! repo-url files delete-files delete-blocks file-paths first-clone? db-encrypted? re-render? re-render-opts metadata)))
 
 (defn parse-files-and-load-to-db!
   [repo-url files {:keys [first-clone? delete-files delete-blocks re-render? re-render-opts] :as opts
@@ -262,8 +262,8 @@
          (encryption/encryption-input-secret-dialog
           repo-url
           db-encrypted-secret
-          #(parse-files-and-create-default-files! repo-url files delete-files delete-blocks file-paths first-clone? db-encrypted? re-render? re-render-opts)))
-        (parse-files-and-create-default-files! repo-url files delete-files delete-blocks file-paths first-clone? db-encrypted? re-render? re-render-opts)))))
+          #(parse-files-and-create-default-files! repo-url files delete-files delete-blocks file-paths first-clone? db-encrypted? re-render? re-render-opts metadata)))
+        (parse-files-and-create-default-files! repo-url files delete-files delete-blocks file-paths first-clone? db-encrypted? re-render? re-render-opts metadata)))))
 
 (defn load-repo-to-db!
   [repo-url {:keys [first-clone? diffs nfs-files]

src/main/frontend/modules/datascript_report/core.cljs

@@ -1,6 +1,7 @@
 (ns frontend.modules.datascript-report.core
   (:require [lambdaisland.glogi :as log]
-            [datascript.core :as d]))
+            [datascript.core :as d]
+            [datascript.db :as db]))
 
 (def keys-of-deleted-entity 1)
 
@@ -16,23 +17,26 @@
 
 (defn get-blocks-and-pages
   [{:keys [db-before db-after tx-data] :as _tx-report}]
-  (let [updated-db-ids (-> (mapv first tx-data) (set))]
-   (reduce
-     (fn [acc x]
-       (let [block-entity
-             (get-entity-from-db-after-or-before db-before db-after x)
-             page-entity
-             (when-let [page-id (-> block-entity :block/page :db/id)]
-               (get-entity-from-db-after-or-before db-before db-after page-id))]
-         (cond-> acc
-           (some? block-entity)
-           (update :blocks conj block-entity)
+  (let [properties (filter (fn [datom]
+                             (= :block/properties (:a datom))) tx-data)
+        updated-db-ids (-> (mapv first tx-data) (set))
+        result (reduce
+                (fn [acc x]
+                  (let [block-entity
+                        (get-entity-from-db-after-or-before db-before db-after x)
+                        page-entity
+                        (when-let [page-id (-> block-entity :block/page :db/id)]
+                          (get-entity-from-db-after-or-before db-before db-after page-id))]
+                    (cond-> acc
+                      (some? block-entity)
+                      (update :blocks conj block-entity)
 
-           (some? page-entity)
-           (update :pages conj page-entity))))
-     {:blocks #{}
-      :pages #{}}
-     updated-db-ids)))
+                      (some? page-entity)
+                      (update :pages conj page-entity))))
+                {:blocks #{}
+                 :pages #{}}
+                updated-db-ids)]
+    (assoc result :properties properties)))
 
 (defn get-blocks
   [{:keys [db-before db-after tx-data] :as _tx-report}]
@@ -45,4 +49,4 @@
             (some? block-entity)
             (conj block-entity))))
       []
-      updated-db-ids)))
+      updated-db-ids)))

src/main/frontend/modules/outliner/core.cljs

@@ -62,6 +62,34 @@
      (outliner-state/get-by-parent-id repo [:block/uuid id])
      (mapv block))))
 
+;; TODO: we might need to store created-at and updated-at as datom attributes
+;; instead of being attributes of properties.
+;; which might improve the db performance, we can improve it later
+(defn- with-timestamp
+  [data]
+  (prn {:data data})
+  (let [updated-at (util/time-ms)
+        m (-> data
+              (dissoc :block/children :block/dummy? :block/level :block/meta)
+              (util/remove-nils))
+        properties (assoc (:block/properties m)
+                          :id (:block/uuid data)
+                          :updated-at updated-at)
+        properties (if-let [created-at (get properties :created-at)]
+                     properties
+                     (assoc properties :created-at updated-at))
+        m (assoc m :block/properties properties)
+        page-id (or (get-in data [:block/page :db/id])
+                    (:db/id (:block/page (db/entity (:db/id data)))))
+        page (db/entity page-id)
+        page-properties (:block/properties page)
+        page-tx {:db/id page-id
+                 :block/properties (assoc page-properties
+                                          :id (:block/uuid page)
+                                          :updated-at updated-at
+                                          :created-at (get page-properties :created-at updated-at))}]
+    [m page-tx]))
+
 ;; -get-id, -get-parent-id, -get-left-id return block-id
 ;; the :block/parent, :block/left should be datascript lookup ref
 
@@ -107,10 +135,8 @@
   (-save [this txs-state]
     (assert (ds/outliner-txs-state? txs-state)
             "db should be satisfied outliner-tx-state?")
-    (let [m (-> (:data this)
-                (dissoc :block/children :block/dummy? :block/level :block/meta)
-                (util/remove-nils))]
-      (swap! txs-state conj m)
+    (let [[m page-tx] (with-timestamp (:data this))]
+      (swap! txs-state conj m page-tx)
       m))
 
   (-del [this txs-state]

src/main/frontend/modules/outliner/pipeline.cljs

@@ -3,7 +3,8 @@
             [lambdaisland.glogi :as log]
             [frontend.modules.outliner.file :as file]
             [frontend.modules.editor.undo-redo :as undo-redo]
-            [frontend.modules.datascript-report.core :as ds-report]))
+            [frontend.modules.datascript-report.core :as ds-report]
+            [frontend.handler.metadata :as metadata-handler]))
 
 (defn updated-block-hook
   [block])
@@ -12,11 +13,16 @@
   [page]
   (file/sync-to-file page))
 
+(defn updated-properties-hook
+  [properties]
+  (metadata-handler/update-properties! properties))
+
 (defn invoke-hooks
   [tx-report]
-  (let [{:keys [blocks pages]} (ds-report/get-blocks-and-pages tx-report)]
+  (let [{:keys [blocks pages properties]} (ds-report/get-blocks-and-pages tx-report)]
     (doseq [p (seq pages)] (updated-page-hook p))
-    (doseq [b (seq blocks)] (updated-block-hook b))))
+    (doseq [b (seq blocks)] (updated-block-hook b))
+    (when (seq properties) (updated-properties-hook properties))))
 
 (defn after-transact-pipelines
   [{:keys [_db-before _db-after _tx-data _tempids _tx-meta] :as tx-report}]

src/main/frontend/state.cljs

@@ -191,6 +191,10 @@
   (true? (:feature/enable-grammarly?
           (get (sub-config) (get-current-repo)))))
 
+(defn store-all-ids-in-text?
+  []
+  (true? (:text/store-all-ids (get-config))))
+
 (defn scheduled-deadlines-disabled?
   []
   (true? (:feature/disable-scheduled-and-deadline-query?

src/main/frontend/text.cljs

@@ -122,14 +122,14 @@
 
 (def hidden-properties
   (set/union
-   #{"id" "custom_id" "heading" "background_color"
-     "created_at" "last_modified_at"}
+   #{:id :custom-id :background-color
+     :created-at :updated-at}
    config/markers))
 
 (defn properties-hidden?
   [properties]
   (and (seq properties)
-       (let [ks (map string/lower-case (keys properties))]
+       (let [ks (map (comp keyword string/lower-case name) (keys properties))]
          (every? hidden-properties ks))))
 
 (defn remove-properties!