feat: native filesystem api integration WIP

src/main/frontend/handler/common.cljs

@@ -18,8 +18,6 @@
     ;; TODO: what if the remote is not named "origin", check the api from isomorphic-git
     (git/resolve-ref repo-url (str "refs/remotes/origin/" branch))))
 
-
-;; Should include un-pushed committed files too
 (defn check-changed-files-status
   ([]
    (check-changed-files-status (state/get-current-repo)))

src/main/frontend/handler/git.cljs

@@ -10,6 +10,7 @@
             [frontend.handler.notification :as notification]
             [frontend.handler.route :as route-handler]
             [frontend.handler.common :as common-handler]
+            [frontend.config :as config]
             [cljs-time.local :as tl]))
 
 (defn- set-git-status!
@@ -30,12 +31,13 @@
   ([repo-url file]
    (git-add repo-url file true))
   ([repo-url file update-status?]
-   (-> (p/let [result (git/add repo-url file)]
-         (when update-status?
-           (common-handler/check-changed-files-status)))
-       (p/catch (fn [error]
-                  (println "git add '" file "' failed: " error)
-                  (js/console.error error))))))
+   (when-not (config/local-db? repo-url)
+     (-> (p/let [result (git/add repo-url file)]
+           (when update-status?
+             (common-handler/check-changed-files-status)))
+         (p/catch (fn [error]
+                    (println "git add '" file "' failed: " error)
+                    (js/console.error error)))))))
 
 (defn commit-and-force-push!
   [commit-message pushing?]

src/main/frontend/handler/image.cljs

@@ -33,8 +33,9 @@
                      (subs path 1)
                      path)]
           (util/p-handle
-           (fs/read-file-2 (util/get-repo-dir (state/get-current-repo))
-                           path)
+           (fs/read-file (util/get-repo-dir (state/get-current-repo))
+                           path
+                           {})
            (fn [blob]
              (let [blob (js/Blob. (array blob) (clj->js {:type "image"}))
                    img-url (image/create-object-url blob)]

src/main/frontend/handler/repo.cljs

@@ -95,32 +95,22 @@
         (p/let [file-exists? (fs/create-if-not-exists repo-dir (str app-dir "/" config/config-file) default-content)]
           (let [path (str app-dir "/" config/config-file)
                 old-content (when file-exists?
-                              (db/get-file repo-url path))
-                content (or
-                         (and old-content
-                              (string/replace old-content "heading" "block"))
-                         default-content)]
-            (db/reset-file! repo-url path content)
-            (db/reset-config! repo-url content)
-            (when-not (= content old-content)
-              (git-handler/git-add repo-url path))))
-        ;; (p/let [file-exists? (fs/create-if-not-exists repo-dir (str app-dir "/" config/metadata-file) default-content)]
-        ;;   (let [path (str app-dir "/" config/metadata-file)]
-        ;;     (when-not file-exists?
-        ;;       (db/reset-file! repo-url path "{:tx-data []}")
-        ;;       (git-handler/git-add repo-url path))))
-))))
+                              (db/get-file repo-url path))]
+            (db/reset-file! repo-url path default-content)
+            (db/reset-config! repo-url default-content)
+            (when (not= default-content old-content)
+              (git-handler/git-add repo-url path))))))))
 
 (defn create-contents-file
   [repo-url]
   (let [repo-dir (util/get-repo-dir repo-url)
         format (state/get-preferred-format)
-        path (str "pages/contents." (if (= (name format) "markdown")
-                                      "md"
-                                      (name format)))
+        path (str (state/get-pages-directory)
+                  "/contents."
+                  (if (= (name format) "markdown") "md" (name format)))
         file-path (str "/" path)
         default-content (util/default-content-with-title format "contents")]
-    (p/let [_ (-> (fs/mkdir (str repo-dir "/pages"))
+    (p/let [_ (-> (fs/mkdir (str repo-dir "/" (state/get-pages-directory)))
                   (p/catch (fn [_e])))
             file-exists? (fs/create-if-not-exists repo-dir file-path default-content)]
       (when-not file-exists?
@@ -189,48 +179,58 @@
 
 (defn create-default-files!
   [repo-url]
-  (when-let [name (get-in @state/state [:me :name])]
-    (create-config-file-if-not-exists repo-url)
-    (create-today-journal-if-not-exists repo-url)
-    (create-contents-file repo-url)
-    (create-custom-theme repo-url)))
+  (create-config-file-if-not-exists repo-url)
+  (create-today-journal-if-not-exists repo-url)
+  (create-contents-file repo-url)
+  (create-custom-theme repo-url))
+
+(defn- parse-files-and-load-to-db!
+  [repo-url files contents {:keys [first-clone? delete-files delete-blocks re-render? additional-files-info]}]
+  (state/set-state! :repo/loading-files? false)
+  (state/set-state! :repo/importing-to-db? true)
+  (let [parsed-files (filter
+                      (fn [[file _]]
+                        (let [format (format/get-format file)]
+                          (contains? config/mldoc-support-formats format)))
+                      contents)
+        blocks-pages (if (seq parsed-files)
+                       (db/extract-all-blocks-pages repo-url parsed-files)
+                       [])]
+    (db/reset-contents-and-blocks! repo-url contents blocks-pages delete-files delete-blocks)
+    (let [config-file (str config/app-name "/" config/config-file)]
+      (if (contains? (set files) config-file)
+        (when-let [content (get contents config-file)]
+          (file-handler/restore-config! repo-url content true))))
+    (when first-clone? (create-default-files! repo-url))
+    (state/set-state! :repo/importing-to-db? false)
+    (when re-render?
+      (ui-handler/re-render-root!))))
 
 (defn load-repo-to-db!
-  [repo-url diffs first-clone?]
-  (let [load-contents (fn [files delete-files delete-blocks re-render?]
+  [repo-url {:keys [first-clone? diffs nfs-files nfs-contents additional-files-info]}]
+  (let [load-contents (fn [files option]
                         (file-handler/load-files-contents!
                          repo-url
                          files
-                         (fn [contents]
-                           (state/set-state! :repo/loading-files? false)
-                           (state/set-state! :repo/importing-to-db? true)
-                           (let [parsed-files (filter
-                                               (fn [[file _]]
-                                                 (let [format (format/get-format file)]
-                                                   (contains? config/mldoc-support-formats format)))
-                                               contents)
-                                 blocks-pages (if (seq parsed-files)
-                                                (db/extract-all-blocks-pages repo-url parsed-files)
-                                                [])]
-                             (db/reset-contents-and-blocks! repo-url contents blocks-pages delete-files delete-blocks)
-                             (let [config-file (str config/app-name "/" config/config-file)]
-                               (if (contains? (set files) config-file)
-                                 (when-let [content (get contents config-file)]
-                                   (file-handler/restore-config! repo-url content true))))
-                             (when first-clone? (create-default-files! repo-url))
-                             (state/set-state! :repo/importing-to-db? false)
-                             (when re-render?
-                               (ui-handler/re-render-root!))))))]
-    (if first-clone?
+                         (fn [contents] (parse-files-and-load-to-db! repo-url files contents option))))]
+    (cond
+      (seq nfs-files)
+      (parse-files-and-load-to-db! repo-url nfs-files nfs-contents
+                                   {:first-clone? true
+                                    :additional-files-info additional-files-info})
+
+      first-clone?
       (->
        (p/let [files (file-handler/load-files repo-url)]
-         (load-contents files nil nil false))
+         (load-contents files {:first-clone? first-clone?}))
        (p/catch (fn [error]
                   (println "loading files failed: ")
                   (js/console.dir error)
                   ;; Empty repo
                   (create-default-files! repo-url)
                   (state/set-state! :repo/loading-files? false))))
+
+      :else
       (when (seq diffs)
         (let [filter-diffs (fn [type] (->> (filter (fn [f] (= type (:type f))) diffs)
                                            (map :path)))
@@ -244,7 +244,11 @@
                              (db/delete-pages-by-files remove-files)
                              [])
               add-or-modify-files (util/remove-nils (concat add-files modify-files))]
-          (load-contents add-or-modify-files (concat delete-files delete-pages) delete-blocks true))))))
+          (load-contents add-or-modify-files
+                         {:first-clone? first-clone?
+                          :delete-files (concat delete-files delete-pages)
+                          :delete-blocks delete-blocks
+                          :re-render? true}))))))
 
 (defn persist-repo!
   [repo]
@@ -256,7 +260,8 @@
 (defn load-db-and-journals!
   [repo-url diffs first-clone?]
   (when (or diffs first-clone?)
-    (load-repo-to-db! repo-url diffs first-clone?)))
+    (load-repo-to-db! repo-url {:first-clone? first-clone?
+                                :diffs diffs})))
 
 (defn transact-react-and-alter-file!
   [repo tx transact-option files]
@@ -520,6 +525,11 @@
                (fn [error]
                  (prn "Delete repo failed, error: " error))))
 
+(defn start-repo-db-if-not-exists!
+  [repo option]
+  (state/set-current-repo! repo)
+  (db/start-db-conn! nil repo option))
+
 (defn setup-local-repo-if-not-exists!
   []
   (if js/window.pfs

src/main/frontend/handler/user.cljs

@@ -2,6 +2,7 @@
   (:require [frontend.util :as util :refer-macros [profile]]
             [frontend.state :as state]
             [frontend.db :as db]
+            [frontend.idb :as idb]
             [frontend.config :as config]
             [frontend.storage :as storage]
             [promesa.core :as p]
@@ -58,19 +59,12 @@
                    (notification/show! "Workflow set successfully!" :success))
                  (fn [_e])))))
 
-(defn- clear-store!
-  []
-  (p/let [_ (.clear db/localforage-instance)
-          dbs (js/window.indexedDB.databases)]
-    (doseq [db dbs]
-      (js/window.indexedDB.deleteDatabase (gobj/get db "name")))))
-
 (defn sign-out!
   [e]
   (->
    (do
      (storage/clear)
-     (clear-store!))
+     (idb/clear-store!))
    (p/catch (fn [e]
               (println "sign out error: ")
               (js/console.dir e)))

src/main/frontend/handler/web/nfs.cljs

@@ -0,0 +1,93 @@
+(ns frontend.handler.web.nfs
+  "The File System Access API, https://web.dev/file-system-access/."
+  (:require [cljs-bean.core :as bean]
+            [promesa.core :as p]
+            [goog.object :as gobj]
+            [goog.dom :as gdom]
+            [frontend.util :as util]
+            ["/frontend/utils" :as utils]
+            [frontend.handler.repo :as repo-handler]
+            [frontend.idb :as idb]
+            [frontend.state :as state]
+            [clojure.string :as string]
+            [frontend.ui :as ui]
+            [frontend.config :as config]))
+
+(defn ls-dir-files
+  []
+  (->
+   (p/let [result (utils/openDirectory #js {:recursive true})
+           root-handle (nth result 0)
+           dir-name (gobj/get root-handle "name")
+           repo (str config/local-db-prefix dir-name)
+           _ (idb/set-item! (str "handle-" repo) root-handle)
+           result (nth result 1)
+           result (flatten (bean/->clj result))
+           files (doall
+                  (map (fn [file]
+                         (let [handle (gobj/get file "handle")
+                               get-attr #(gobj/get file %)]
+                           {:file/path (get-attr "webkitRelativePath")
+                            :file/last-modified-at (get-attr "lastModified")
+                            :file/size (get-attr "size")
+                            :file/type (get-attr "type")
+                            :file/file file
+                            :file/handle handle})) result))
+           text-files (filter (fn [file] (contains? #{"org" "md" "markdown"} (util/get-file-ext (:file/path file)))) files)]
+     (doseq [file text-files]
+       (idb/set-item! (str "handle-" repo "/" (:file/path file))
+                      (:file/handle file)))
+     (-> (p/all (map (fn [file]
+                       (p/let [content (.text (:file/file file))]
+                         (assoc file :file/content content))) text-files))
+         (p/then (fn [result]
+                   (let [files (map #(dissoc % :file/file) result)]
+                     (repo-handler/start-repo-db-if-not-exists! repo {:db-type :local-native-fs})
+                     (repo-handler/load-repo-to-db! repo
+                                                    {:first-clone? true
+                                                     :nfs-files (map :file/path files)
+                                                     :nfs-contents (mapv (fn [f] [(:file/path f) (:file/content f)]) files)
+                                                     :additional-files-info files})
+                     ;; create default directories and files
+                     )))
+         (p/catch (fn [error]
+                    (println "Load files content error: ")
+                    (js/console.dir error)))))
+   (p/catch (fn [error]
+              (println "Open directory error: ")
+              (js/console.dir error)))))
+
+(defn open-file-picker
+  "Shows a file picker that lets a user select a single existing file, returning a handle for the selected file. "
+  ([]
+   (open-file-picker {}))
+  ([option]
+   (js/window.showOpenFilePicker (bean/->js option))))
+
+(defn get-local-repo
+  []
+  (when-let [repo (state/get-current-repo)]
+    (when (config/local-db? repo)
+      repo)))
+
+(defn check-directory-permission!
+  [repo]
+  (p/let [handle (idb/get-item (str "handle-" repo))]
+    (utils/verifyPermission handle true)))
+
+(defn ask-permission
+  [repo]
+  (fn [close-fn]
+    [:div
+     [:p.text-gray-700
+      "Grant native filesystem permission for directory: "
+      [:b (config/get-local-dir repo)]]
+     (ui/button
+       "Grant"
+       :on-click (fn []
+                   (check-directory-permission! repo)
+                   (close-fn)))]))
+
+(defn trigger-check! []
+  (when-let [repo (get-local-repo)]
+    (state/set-modal! (ask-permission repo))))