diff --git a/deps/db/src/logseq/db/frontend/asset.cljs b/deps/db/src/logseq/db/frontend/asset.cljs new file mode 100644 index 0000000000..e56ff9cb2f --- /dev/null +++ b/deps/db/src/logseq/db/frontend/asset.cljs @@ -0,0 +1,15 @@ +(ns logseq.db.frontend.asset) + +(defn- decode-digest + [^js/Uint8Array digest] + (.. (js/Array.from digest) + (map (fn [s] (.. s (toString 16) (padStart 2 "0")))) + (join ""))) + +;; TODO: Reuse with frontend +(defn (js/crypto.subtle.digest "SHA-256" file-array-buffer) + (.then (fn [dig] (js/Uint8Array. dig))) + (.then decode-digest))) \ No newline at end of file diff --git a/deps/graph-parser/script/db_import.cljs b/deps/graph-parser/script/db_import.cljs index 8504b3fa7b..4a3cf944ee 100644 --- a/deps/graph-parser/script/db_import.cljs +++ b/deps/graph-parser/script/db_import.cljs @@ -11,13 +11,14 @@ [clojure.string :as string] [datascript.core :as d] [logseq.common.graph :as common-graph] + [logseq.db.common.sqlite-cli :as sqlite-cli] + [logseq.db.frontend.asset :as db-asset] [logseq.graph-parser.exporter :as gp-exporter] [logseq.outliner.cli :as outliner-cli] [logseq.outliner.pipeline :as outliner-pipeline] [nbb.classpath :as cp] [nbb.core :as nbb] - [promesa.core :as p] - [logseq.db.common.sqlite-cli :as sqlite-cli])) + [promesa.core :as p])) (def tx-queue (atom cljs.core/PersistentQueue.EMPTY)) (def original-transact! d/transact!) @@ -47,11 +48,22 @@ (p/let [s (fsp/readFile (:path file))] (str s))) +(defn- > block-ast + (mapcat (fn [n] + (some->> n + second + :title + (filter #(and (= "Link" (first %)) + (common-config/local-asset? (second (:url (second %)))))))))) + asset-name (some-> asset-links first second :url second path/basename)] + (when (seq asset-links) + (prn :asset-added! asset-name (get @assets asset-name))) + ;; TODO: Handle asset metadata + ;; TODO: Handle: multiple asset links + (if-let [asset-data (and asset-name (get @assets asset-name))] + (do + (swap! assets assoc-in [asset-name :block/uuid] (:block/uuid block)) + (merge block + {:block/tags [:logseq.class/Asset] + :logseq.property.asset/type (:type asset-data) + :logseq.property.asset/checksum (:checksum asset-data) + :logseq.property.asset/size (:size asset-data) + ;; TODO: Ensure this matches DB version + :block/title (path/file-stem asset-name)})) + block))) + (defn- build-block-tx - [db block* pre-blocks {:keys [page-names-to-uuids] :as per-file-state} {:keys [import-state journal-created-ats] :as options}] + [db block* pre-blocks {:keys [page-names-to-uuids block-to-ast] :as per-file-state} {:keys [import-state journal-created-ats] :as options}] ;; (prn ::block-in block*) (let [;; needs to come before update-block-refs to detect new property schemas {:keys [block properties-tx]} @@ -898,11 +925,13 @@ prepared-block (cond-> block-after-built-in-props journal-page-created-at (assoc :block/created-at journal-page-created-at)) + block-ast (get block-to-ast (:block/uuid block*)) block' (-> prepared-block (fix-pre-block-references pre-blocks page-names-to-uuids) (fix-block-name-lookup-ref page-names-to-uuids) (update-block-refs page-names-to-uuids options) (update-block-tags db (:user-options options) per-file-state (:all-idents import-state)) + (handle-assets-in-block block-ast (:assets import-state)) (update-block-marker options) (update-block-priority options) add-missing-timestamps @@ -1154,7 +1183,9 @@ :classes-from-property-parents (atom #{}) ;; Map of block uuids to their :block/properties-text-values value. ;; Used if a property value changes to :default - :block-properties-text-values (atom {})}) + :block-properties-text-values (atom {}) + ;; Track asset data for use across asset and doc import steps + :assets (atom {})}) (defn- build-tx-options [{:keys [user-options] :as options}] (merge @@ -1262,6 +1293,25 @@ (update :block/refs fix-block-uuids {:ref? true :properties (:block/properties b)}))) blocks))) +(defn- extract-page [file content extract-options] + (let [{:keys [blocks ast] :as parsed-page} + (-> (extract/extract file content extract-options) + (update :pages (fn [pages] + (map #(dissoc % :block.temp/original-page-name) pages))) + (update :blocks fix-extracted-block-tags-and-refs)) + page-blocks (remove :block/pre-block? blocks) + ;; Like in gp-block/extract-blocks, only treat heading blocks as blocks + block-asts (filter (comp gp-block/heading-block? first) ast) + ;; TODO: Add warning if page-blocks and block-asts don't match + block-to-ast (when (= (count page-blocks) (count block-asts)) + (into {} + (map vector + (map :block/uuid (remove :block/pre-block? blocks)) + (filter (comp gp-block/heading-block? first) ast))))] + (cond-> parsed-page + (some? block-to-ast) + (assoc :block-to-ast block-to-ast)))) + (defn- extract-pages-and-blocks "Main fn which calls graph-parser to convert markdown into data" [db file content {:keys [extract-options import-state]}] @@ -1277,10 +1327,7 @@ extract-options {:db db})] (cond (and (contains? common-config/mldoc-support-formats format) (not ignored-highlight-file?)) - (-> (extract/extract file content extract-options') - (update :pages (fn [pages] - (map #(dissoc % :block.temp/original-page-name) pages))) - (update :blocks fix-extracted-block-tags-and-refs)) + (extract-page file content extract-options') (common-config/whiteboard? file) (-> (extract/extract-whiteboard-edn file content extract-options') @@ -1363,7 +1410,7 @@ log-fn prn} :as *options}] (let [options (assoc *options :notify-user notify-user :log-fn log-fn) - {:keys [pages blocks]} (extract-pages-and-blocks @conn file content options) + {:keys [pages blocks block-to-ast]} (extract-pages-and-blocks @conn file content options) tx-options (merge (build-tx-options options) {:journal-created-ats (build-journal-created-ats pages)}) old-properties (keys @(get-in options [:import-state :property-schemas])) @@ -1378,7 +1425,7 @@ pre-blocks (->> blocks (keep #(when (:block/pre-block? %) (:block/uuid %))) set) blocks-tx (->> blocks (remove :block/pre-block?) - (mapcat #(build-block-tx @conn % pre-blocks per-file-state + (mapcat #(build-block-tx @conn % pre-blocks (assoc per-file-state :block-to-ast block-to-ast) (assoc tx-options :whiteboard? (some? (seq whiteboard-pages))))) vec) {:keys [property-pages-tx property-page-properties-tx] pages-tx' :pages-tx} @@ -1542,32 +1589,62 @@ class-to-prop-uuids)] (ldb/transact! repo-or-conn tx))) -(defn- export-asset-files - "Exports files under assets/" +(defn- (p/loop [_ (async-fn (get args-to-loop 0)) + i 0] + (when-not (>= i (dec (count args-to-loop))) + (p/recur (async-fn (get args-to-loop (inc i))) + (inc i)))) + (p/catch (fn [e] + (notify-user {:msg (str "Import has an unexpected error:\n" (.-message e)) + :level :error + :ex-data {:error e}}))))) + +(defn- read-asset-files + "Reads files under assets/" + [*asset-files ( (p/loop [_ (copy-asset (get asset-files 0)) - i 0] - (when-not (>= i (dec (count asset-files))) - (p/recur (copy-asset (get asset-files (inc i))) - (inc i)))) - (p/catch (fn [e] - (notify-user {:msg (str "Import has an unexpected error:\n" (.-message e)) - :level :error - :ex-data {:error e}}))))))) + (set-ui-state [:graph/importing-state :current-page] "Copy asset files") + ( {:extract-options {:date-formatter (common-config/get-date-formatter config) ;; Remove config keys that break importing @@ -1628,9 +1705,10 @@ * : (select-keys options [:notify-user : (.arrayBuffer (:file-object file)) + #_(p/then (fn [buffer] + (p/let [checksum (db-asset/