Add :hidden config support to graph-parser.cli

Also fix lints and tests from previous commits

.clj-kondo/config.edn

@@ -82,6 +82,7 @@
              frontend.mixins mixins
              frontend.mobile.util mobile-util
              frontend.page page
+             frontend.schema.handler.common-config common-config-schema
              frontend.search search
              frontend.state state
              frontend.template template
@@ -97,6 +98,8 @@
              frontend.util.thingatpt thingatpt
              lambdaisland.glogi log
              logseq.common.path path
+             logseq.common.graph common-graph
+             logseq.common.config common-config
              logseq.graph-parser graph-parser
              logseq.graph-parser.text text
              logseq.graph-parser.block gp-block

deps/common/.carve/config.edn

@@ -0,0 +1,3 @@
+{:paths ["src"]
+ :api-namespaces [logseq.common.path]
+ :report {:format :ignore}}

deps/common/.carve/ignore

@@ -0,0 +1,4 @@
+;; API fn
+logseq.common.config/remove-hidden-files
+;; API fn
+logseq.common.graph/get-files

deps/common/src/logseq/common/config.cljs

@@ -0,0 +1,25 @@
+(ns logseq.common.config
+  "This ns provides common fns related to user config"
+  (:require [clojure.string :as string]))
+
+(defn- hidden?
+  [path patterns]
+  (let [path (if (and (string? path)
+                      (= \/ (first path)))
+               (subs path 1)
+               path)]
+    (some (fn [pattern]
+            (let [pattern (if (and (string? pattern)
+                                   (not= \/ (first pattern)))
+                            (str "/" pattern)
+                            pattern)]
+              (string/starts-with? (str "/" path) pattern))) patterns)))
+
+(defn remove-hidden-files
+  "Removes files that match a pattern specified by :hidden config"
+  [files config get-path-fn]
+  (if-let [patterns (seq (:hidden config))]
+    (remove (fn [file]
+              (let [path (get-path-fn file)]
+                (hidden? path patterns))) files)
+    files))

deps/common/src/logseq/common/graph.cljs

@@ -1,4 +1,4 @@
-(ns logseq.common.graph
+(ns ^:node-only logseq.common.graph
   "This ns provides common fns for a graph directory and only runs in a node environment"
   (:require ["fs" :as fs]
             ["path" :as node-path]
@@ -38,15 +38,6 @@
        (map fix-win-path!)
        (vec)))
 
-(def ^:private allowed-formats
-  #{:org :markdown :md :edn :json :js :css :excalidraw :tldr})
-
-(defn- get-ext
-  [p]
-  (-> (node-path/extname p)
-      (subs 1)
-      keyword))
-
 (defn ignored-path?
   "Given a graph directory and path, returns truthy value on whether the path is
   ignored. Useful for contexts like reading a graph's directory and file watcher
@@ -65,6 +56,15 @@
        (or (re-find #"/\.[^.]+" relpath)
            (re-find #"^\.[^.]+" relpath))))))
 
+(def ^:private allowed-formats
+  #{:org :markdown :md :edn :json :js :css :excalidraw :tldr})
+
+(defn- get-ext
+  [p]
+  (-> (node-path/extname p)
+      (subs 1)
+      keyword))
+
 (defn get-files
   "Given a graph's root dir, returns a list of all files that it recognizes"
   [graph-dir]

deps/common/test/logseq/common/config_test.cljs

@@ -0,0 +1,21 @@
+(ns logseq.common.config-test
+  (:require [logseq.common.config :as common-config]
+            [cljs.test :refer [deftest is]]))
+
+(deftest remove-hidden-files
+  (let [files ["pages/foo.md" "pages/bar.md"
+               "script/README.md" "script/config.edn"
+               "dev/README.md" "dev/config.edn"]]
+    (is (= ["pages/foo.md" "pages/bar.md"]
+           (common-config/remove-hidden-files
+            files
+            {:hidden ["script" "/dev"]}
+            identity))
+        "Removes hidden relative files")
+
+    (is (= ["/pages/foo.md" "/pages/bar.md"]
+           (common-config/remove-hidden-files
+            (map #(str "/" %) files)
+            {:hidden ["script" "/dev"]}
+            identity))
+        "Removes hidden files if they start with '/'")))

deps/graph-parser/src/logseq/graph_parser/cli.cljs

@@ -1,9 +1,10 @@
-(ns logseq.graph-parser.cli
+(ns ^:node-only logseq.graph-parser.cli
   "Primary ns to parse graphs with node.js based CLIs"
   (:require ["fs" :as fs]
-            ["child_process" :as child-process]
+            ["path" :as path]
             [clojure.edn :as edn]
             [logseq.common.graph :as common-graph]
+            [logseq.common.config :as common-config]
             [logseq.graph-parser :as graph-parser]
             [logseq.graph-parser.config :as gp-config]
             [logseq.graph-parser.util :as gp-util]
@@ -14,13 +15,22 @@
   [file]
   (str (fs/readFileSync file)))
 
+(defn- remove-hidden-files [dir config files]
+  (if (seq (:hidden config))
+    (->> files
+         (map #(assoc % ::rel-path (path/relative dir (:file/path %))))
+         ((fn [files] (common-config/remove-hidden-files files config ::rel-path)))
+         (map #(dissoc % ::rel-path)))
+    files))
+
 (defn- build-graph-files
   "Given a graph directory, return allowed file paths and their contents in preparation
    for parsing"
-  [dir]
+  [dir config]
   (->> (common-graph/get-files dir)
        (map #(hash-map :file/path %))
        graph-parser/filter-files
+       (remove-hidden-files dir config)
        (mapv #(assoc % :file/content (slurp (:file/path %))))))
 
 (defn- read-config
@@ -28,8 +38,8 @@
   [dir]
   (let [config-file (str dir "/" gp-config/app-name "/config.edn")]
     (if (fs/existsSync config-file)
-     (-> config-file fs/readFileSync str edn/read-string)
-     {})))
+      (-> config-file fs/readFileSync str edn/read-string)
+      {})))
 
 (defn- parse-files
   [conn files {:keys [config] :as options}]
@@ -62,10 +72,10 @@
   ([dir]
    (parse-graph dir {}))
   ([dir options]
-   (let [files (or (:files options) (build-graph-files dir))
+   (let [config (read-config dir)
+         files (or (:files options) (build-graph-files dir config))
          conn (or (:conn options) (ldb/start-conn))
-         config (read-config dir)
-        _ (when-not (:files options) (println "Parsing" (count files) "files..."))
+         _ (when-not (:files options) (println "Parsing" (count files) "files..."))
          asts (parse-files conn files (merge options {:config config}))]
      {:conn conn
       :files (map :file/path files)

deps/graph-parser/src/logseq/graph_parser/test/docs_graph_helper.cljs

@@ -152,8 +152,8 @@
   ;; Counts assertions help check for no major regressions. These counts should
   ;; only increase over time as the docs graph rarely has deletions
   (testing "Counts"
-    (is (= 304 (count files)) "Correct file count")
-    (is (= 69502 (count (d/datoms db :eavt))) "Correct datoms count")
+    (is (= 303 (count files)) "Correct file count")
+    (is (= 69499 (count (d/datoms db :eavt))) "Correct datoms count")
 
     (is (= 5866
            (ffirst

deps/graph-parser/test/logseq/graph_parser/mldoc_test.cljs

@@ -122,7 +122,7 @@ body"
 (deftest ^:integration test->edn
   (let [graph-dir "test/docs-0.9.2"
         _ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir "v0.9.2")
-        files (gp-cli/build-graph-files graph-dir)
+        files (#'gp-cli/build-graph-files graph-dir {})
         asts-by-file (->> files
                           (map (fn [{:file/keys [path content]}]
                                  (let [format (if (string/ends-with? path ".org")

deps/publishing/src/logseq/publishing.cljs

@@ -1,4 +1,4 @@
-(ns logseq.publishing
+(ns ^:node-only logseq.publishing
   "This node only ns provides api fns for exporting a publishing app"
   (:require [logseq.publishing.html :as publish-html]
             [logseq.publishing.export :as publish-export]))

deps/publishing/src/logseq/publishing/export.cljs

@@ -1,4 +1,4 @@
-(ns logseq.publishing.export
+(ns ^:node-only logseq.publishing.export
   "This electron only ns (for the main process) exports files from multiple
   locations to provide a complete publishing app"
   (:require ["fs-extra" :as fse]

src/main/frontend/handler/common.cljs

@@ -2,7 +2,6 @@
   "Common fns for handlers"
   (:require [cljs-bean.core :as bean]
             [cljs.reader :as reader]
-            [clojure.string :as string]
             [frontend.date :as date]
             [frontend.state :as state]
             [frontend.util :as util]
@@ -28,27 +27,6 @@
       (.filter (bean/->js paths))
       (bean/->clj)))
 
-(defn- hidden?
-  [path patterns]
-  (let [path (if (and (string? path)
-                      (= \/ (first path)))
-               (subs path 1)
-               path)]
-    (some (fn [pattern]
-            (let [pattern (if (and (string? pattern)
-                                   (not= \/ (first pattern)))
-                            (str "/" pattern)
-                            pattern)]
-              (string/starts-with? (str "/" path) pattern))) patterns)))
-
-(defn remove-hidden-files
-  [files config get-path-fn]
-  (if-let [patterns (seq (:hidden config))]
-    (remove (fn [file]
-              (let [path (get-path-fn file)]
-                (hidden? path patterns))) files)
-    files))
-
 (defn safe-read-string
   [content error-message-or-handler]
   (try

src/main/frontend/handler/repo.cljs

@@ -8,7 +8,6 @@
             [frontend.db :as db]
             [frontend.fs :as fs]
             [frontend.fs.nfs :as nfs]
-            [frontend.handler.common :as common-handler]
             [frontend.handler.file :as file-handler]
             [frontend.handler.repo-config :as repo-config-handler]
             [frontend.handler.common.file :as file-common-handler]
@@ -31,7 +30,8 @@
             [clojure.core.async :as async]
             [frontend.mobile.util :as mobile-util]
             [medley.core :as medley]
-            [logseq.common.path :as path]))
+            [logseq.common.path :as path]
+            [logseq.common.config :as common-config]))
 
 ;; Project settings should be checked in two situations:
 ;; 1. User changes the config.edn directly in logseq.com (fn: alter-file)
@@ -273,7 +273,7 @@
         ;; config should be loaded to state first
         _ (state/set-config! repo-url config)
         ;; remove :hidden files from file-objs, :hidden
-        file-objs (common-handler/remove-hidden-files file-objs config :file/path)]
+        file-objs (common-config/remove-hidden-files file-objs config :file/path)]
 
     ;; Load to db even it's empty, (will create default files)
     (parse-files-and-load-to-db! repo-url file-objs {:new-graph? new-graph?
@@ -293,8 +293,8 @@
                    (state/get-config repo-url))
         ;; NOTE: Use config while parsing. Make sure it's the current journal title format
         _ (state/set-config! repo-url config)
-        nfs-files (common-handler/remove-hidden-files file-objs config :file/path)
-        diffs (common-handler/remove-hidden-files diffs config :path)
+        nfs-files (common-config/remove-hidden-files file-objs config :file/node-node-path)
+        diffs (common-config/remove-hidden-files diffs config :path)
         load-contents (fn [files option]
                         (file-handler/load-files-contents!
                          repo-url

src/main/frontend/schema/handler/global_config.cljc

@@ -1,9 +1,9 @@
 (ns frontend.schema.handler.global-config
   "Malli schemas for global-config"
-  (:require [frontend.schema.handler.common-config :as common-config]))
+  (:require [frontend.schema.handler.common-config :as common-config-schema]))
 
 ;; For now this just references a common schema but repo-config and
 ;; global-config could diverge
 (def Config-edn
   "Schema for global config.edn"
-  common-config/Config-edn)
+  common-config-schema/Config-edn)

src/main/frontend/schema/handler/repo_config.cljc

@@ -1,9 +1,9 @@
 (ns frontend.schema.handler.repo-config
   "Malli schemas for repo-config"
-  (:require [frontend.schema.handler.common-config :as common-config]))
+  (:require [frontend.schema.handler.common-config :as common-config-schema]))
 
 ;; For now this just references a common schema but repo-config and
 ;; global-config could diverge
 (def Config-edn
   "Schema for repo config.edn"
-  common-config/Config-edn)
+  common-config-schema/Config-edn)

src/test/frontend/handler/repo_conversion_test.cljs

@@ -97,8 +97,8 @@
   ;; Counts assertions help check for no major regressions. These counts should
   ;; only increase over time as the docs graph rarely has deletions
   (testing "Counts"
-    (is (= 212 (count files)) "Correct file count")
-    (is (= 42315 (count (d/datoms db :eavt))) "Correct datoms count")
+    (is (= 211 (count files)) "Correct file count")
+    (is (= 42304 (count (d/datoms db :eavt))) "Correct datoms count")
 
     (is (= 3600
            (ffirst
@@ -137,7 +137,7 @@
 (deftest ^:integration convert-v067-filenames-parse-and-load-files-to-db
   (let [graph-dir "src/test/docs"
         _ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir "v0.6.7")
-        files (gp-cli/build-graph-files graph-dir)
+        files (#'gp-cli/build-graph-files graph-dir {})
         ;; Converting the v0.6.7 ver docs graph under the old namespace naming rule to the new one (:repo/dir-version 0->3)
         files (convert-graph-files-path files convert-to-triple-lowbar)
         _ (repo-handler/parse-files-and-load-to-db! test-helper/test-db files {:re-render? false :verbose false})

src/test/frontend/handler/repo_test.cljs

@@ -16,8 +16,8 @@
 (deftest ^:integration parse-and-load-files-to-db
   (let [graph-dir "src/test/docs-0.9.2"
         _ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir "v0.9.2")
-        files (gp-cli/build-graph-files graph-dir)
         repo-config (edn/read-string (str (fs/readFileSync (node-path/join graph-dir "logseq/config.edn"))))
+        files (#'gp-cli/build-graph-files graph-dir repo-config)
         _ (test-helper/with-config repo-config
             (repo-handler/parse-files-and-load-to-db! test-helper/test-db files {:re-render? false :verbose false}))
         db (conn/get-db test-helper/test-db)]