Move logseq.graph-parser.db* namespaces to their own dep

Also add its own CI file

.github/workflows/db.yml

@@ -0,0 +1,85 @@
+name: logseq/db CI
+
+on:
+  # Path filters ensure jobs only kick off if a change is made to db
+  push:
+    branches: [master]
+    paths:
+      - 'deps/db/**'
+      - '!deps/db/**.md'
+  pull_request:
+    branches: [master]
+    paths:
+      - 'deps/db/**'
+      - '!deps/db/**.md'
+
+env:
+  CLOJURE_VERSION: '1.10.1.727'
+  # setup-java@v2 dropped support for legacy Java version syntax.
+  # This is the same as 1.8.
+  JAVA_VERSION: '8'
+  # This is the latest node version we can run.
+  NODE_VERSION: '16'
+  BABASHKA_VERSION: '0.8.2'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          cache: 'yarn'
+          cache-dependency-path: deps/db/yarn.lock
+
+      - name: Set up Java
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: ${{ env.JAVA_VERSION }}
+
+      - name: Set up Clojure
+        uses: DeLaGuardo/setup-clojure@master
+        with:
+          cli: ${{ env.CLOJURE_VERSION }}
+          bb: ${{ env.BABASHKA_VERSION }}
+
+      - name: Fetch yarn deps
+        run: cd deps/db && yarn install --frozen-lockfile
+
+      # In this job because it depends on an npm package
+      - name: Load namespaces into nbb-logseq
+        run: bb test:load-all-namespaces-with-nbb deps/db
+
+  lint:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up Java
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: ${{ env.JAVA_VERSION }}
+
+      - name: Set up Clojure
+        uses: DeLaGuardo/setup-clojure@master
+        with:
+          cli: ${{ env.CLOJURE_VERSION }}
+          bb: ${{ env.BABASHKA_VERSION }}
+
+      - name: Run clj-kondo lint
+        run: cd deps/db && clojure -M:clj-kondo --parallel --lint src
+
+      - name: Carve lint for unused vars
+        run: cd deps/db && ../../scripts/carve.clj
+
+      - name: Lint for vars that are too large
+        run: scripts/large_vars.clj deps/db/src '{:max-lines-count 30}'

.github/workflows/graph-parser.yml

@@ -1,16 +1,20 @@
-name: logseq graph-parser CI
+name: logseq/graph-parser CI
 
 on:
-  # Path filters ensure jobs only kick off if a change is made to graph-parser
+  # Path filters ensure jobs only kick off if a change is made to graph-parser or
+  # its local dependencies
   push:
     branches: [master]
     paths:
       - 'deps/graph-parser/**'
+      # db is a local dep that could break functionality in this lib and should trigger this
+      - 'deps/db/**'
       - '!deps/graph-parser/**.md'
   pull_request:
     branches: [master]
     paths:
       - 'deps/graph-parser/**'
+      - 'deps/db/**'
       - '!deps/graph-parser/**.md'
 
 env:
@@ -74,11 +78,11 @@ jobs:
         run: cd deps/graph-parser && clojure -M:test
 
       - name: Run nbb-logseq tests
-        run: cd deps/graph-parser && yarn nbb-logseq -cp src:test -m logseq.graph-parser.nbb-test-runner/run-tests
+        run: cd deps/graph-parser && yarn nbb-logseq -cp src:test:../db/src -m logseq.graph-parser.nbb-test-runner/run-tests
 
       # In this job because it depends on an npm package
       - name: Load namespaces into nbb-logseq
-        run: bb test:load-all-namespaces-with-nbb deps/graph-parser src
+        run: bb test:load-all-namespaces-with-nbb deps/graph-parser
 
   lint:
     runs-on: ubuntu-latest

deps/db/.carve/config.edn

@@ -0,0 +1,6 @@
+{:paths ["src"]
+ :api-namespaces [
+                  ;; Some fns are used by frontend but not worth moving over yet
+                  logseq.db.schema
+                  ]
+ :report {:format :ignore}}

deps/db/.carve/ignore

@@ -0,0 +1,2 @@
+;; API
+logseq.db/start-conn

deps/db/deps.edn

@@ -0,0 +1,7 @@
+{:deps
+ ;; External deps should be kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn
+ {datascript/datascript {:mvn/version "1.3.8"}}
+ :aliases
+ {:clj-kondo
+  {:replace-deps {clj-kondo/clj-kondo {:mvn/version "2022.05.31"}}
+   :main-opts  ["-m" "clj-kondo.main"]}}}

deps/db/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "@logseq/db",
+  "version": "1.0.0",
+  "private": true,
+  "devDependencies": {
+    "@logseq/nbb-logseq": "^0.5.103"
+  }
+}

deps/db/src/logseq/db.cljs

@@ -1,6 +1,6 @@
-(ns logseq.graph-parser.db
-  (:require [logseq.graph-parser.db.default :as default-db]
-            [logseq.graph-parser.db.schema :as db-schema]
+(ns logseq.db
+  (:require [logseq.db.default :as default-db]
+            [logseq.db.schema :as db-schema]
             [datascript.core :as d]))
 
 (defn start-conn

deps/db/src/logseq/db/default.cljs

@@ -1,4 +1,4 @@
-(ns logseq.graph-parser.db.default
+(ns logseq.db.default
   (:require [clojure.string :as string]))
 
 (defonce built-in-pages-names

deps/db/src/logseq/db/schema.cljs

@@ -1,4 +1,4 @@
-(ns logseq.graph-parser.db.schema)
+(ns logseq.db.schema)
 
 (defonce version 1)
 (defonce ast-version 1)

deps/db/yarn.lock

@@ -0,0 +1,43 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"@logseq/nbb-logseq@^0.5.103":
+  version "0.5.103"
+  resolved "https://registry.yarnpkg.com/@logseq/nbb-logseq/-/nbb-logseq-0.5.103.tgz#1084380cd54c92ca8cc94a8934cc777206e45cc0"
+  integrity sha512-V9UW0XrCaaadHUc6/Hp9wfGpQqkzqzoqnDGeSVZkWR6l3QwyqGi9mkhnhVcfTwAvxIfOgrfz93GcaeepV4pYNA==
+  dependencies:
+    import-meta-resolve "^1.1.1"
+
+builtins@^4.0.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/builtins/-/builtins-4.1.0.tgz#1edd016dd91ce771a1ed6fc3b2b71fb918953250"
+  integrity sha512-1bPRZQtmKaO6h7qV1YHXNtr6nCK28k0Zo95KM4dXfILcZZwoHJBN1m3lfLv9LPkcOZlrSr+J1bzMaZFO98Yq0w==
+  dependencies:
+    semver "^7.0.0"
+
+import-meta-resolve@^1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/import-meta-resolve/-/import-meta-resolve-1.1.1.tgz#244fd542fd1fae73550d4f8b3cde3bba1d7b2b18"
+  integrity sha512-JiTuIvVyPaUg11eTrNDx5bgQ/yMKMZffc7YSjvQeSMXy58DO2SQ8BtAf3xteZvmzvjYh14wnqNjL8XVeDy2o9A==
+  dependencies:
+    builtins "^4.0.0"
+
+lru-cache@^6.0.0:
+  version "6.0.0"
+  resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-6.0.0.tgz#6d6fe6570ebd96aaf90fcad1dafa3b2566db3a94"
+  integrity sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==
+  dependencies:
+    yallist "^4.0.0"
+
+semver@^7.0.0:
+  version "7.3.7"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.7.tgz#12c5b649afdbf9049707796e22a4028814ce523f"
+  integrity sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==
+  dependencies:
+    lru-cache "^6.0.0"
+
+yallist@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72"
+  integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==

deps/graph-parser/.carve/config.edn

@@ -2,8 +2,6 @@
  :api-namespaces [
                   ;; carve doesn't detect nbb only usage
                   logseq.graph-parser.log
-                  ;; Used by logseq but not worth splitting up
-                  logseq.graph-parser.db.schema
                   ;; Used in tests
                   logseq.graph-parser.test.docs-graph-helper]
  :report {:format :ignore}}

deps/graph-parser/deps.edn

@@ -1,10 +1,11 @@
 {:paths ["src"]
  :deps
- ;; Deps should be kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn
- {datascript/datascript {:mvn/version "1.3.8"}
-  frankiesardo/linked {:mvn/version "1.3.0"}
+ ;; External deps should be kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn
+ {frankiesardo/linked {:mvn/version "1.3.0"}
   com.andrewmcveigh/cljs-time {:git/url "https://github.com/logseq/cljs-time" ;; fork
                                :sha     "5704fbf48d3478eedcf24d458c8964b3c2fd59a9"}
+  ;; local dep
+  logseq/db {:local/root "../db"}
   ;; stubbed in nbb
   com.lambdaisland/glogi {:mvn/version "1.1.144"}
   ;; built in to nbb

deps/graph-parser/package.json

@@ -6,6 +6,6 @@
     "@logseq/nbb-logseq": "^0.5.103"
   },
   "dependencies": {
-    "mldoc": "^1.3.3"
+    "mldoc": "^1.3.9"
   }
 }

deps/graph-parser/src/logseq/graph_parser/cli.cljs

@@ -6,7 +6,7 @@
             [clojure.string :as string]
             [logseq.graph-parser :as graph-parser]
             [logseq.graph-parser.config :as gp-config]
-            [logseq.graph-parser.db :as gp-db]))
+            [logseq.db :as ldb]))
 
 (defn slurp
   "Return file contents like clojure.core/slurp"
@@ -60,7 +60,7 @@ TODO: Fail fast when process exits 1"
    (parse-graph dir {}))
   ([dir options]
    (let [files (or (:files options) (build-graph-files dir))
-         conn (gp-db/start-conn)
+         conn (ldb/start-conn)
          config (read-config dir)]
      (when-not (:files options) (println "Parsing" (count files) "files..."))
      (parse-files conn files (merge options {:config config}))

deps/graph-parser/test/logseq/graph_parser_test.cljs

@@ -1,13 +1,13 @@
 (ns logseq.graph-parser-test
   (:require [cljs.test :refer [deftest testing is]]
             [logseq.graph-parser :as graph-parser]
-            [logseq.graph-parser.db :as gp-db]
+            [logseq.db :as ldb]
             [logseq.graph-parser.block :as gp-block]
             [datascript.core :as d]))
 
 (deftest parse-file
   (testing "id properties"
-    (let [conn (gp-db/start-conn)]
+    (let [conn (ldb/start-conn)]
       (graph-parser/parse-file conn "foo.md" "- id:: 628953c1-8d75-49fe-a648-f4c612109098" {})
       (is (= [{:id "628953c1-8d75-49fe-a648-f4c612109098"}]
              (->> (d/q '[:find (pull ?b [*])
@@ -18,7 +18,7 @@
                   (map :block/properties)))
           "id as text has correct :block/properties"))
 
-    (let [conn (gp-db/start-conn)]
+    (let [conn (ldb/start-conn)]
       (graph-parser/parse-file conn "foo.md" "- id:: [[628953c1-8d75-49fe-a648-f4c612109098]]" {})
       (is (= [{:id #{"628953c1-8d75-49fe-a648-f4c612109098"}}]
              (->> (d/q '[:find (pull ?b [*])
@@ -30,7 +30,7 @@
           "id as linked ref has correct :block/properties")))
 
   (testing "unexpected failure during block extraction"
-    (let [conn (gp-db/start-conn)
+    (let [conn (ldb/start-conn)
           deleted-page (atom nil)]
       (with-redefs [gp-block/with-pre-block-if-exists (fn stub-failure [& _args]
                                               (throw (js/Error "Testing unexpected failure")))]

deps/graph-parser/yarn.lock

@@ -177,10 +177,10 @@ mimic-fn@^2.0.0:
   resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b"
   integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==
 
-mldoc@^1.3.3:
-  version "1.3.3"
-  resolved "https://registry.yarnpkg.com/mldoc/-/mldoc-1.3.3.tgz#b7f39b48eb0ef3558619d3e3522265977bd78fe3"
-  integrity sha512-TzW06GBltdKxwWAxOvflPmIVedu6bzl9T4YoYqnDUyZ3kELFMllEgiYCh65PPW3xsRMA/5OcRQqqGZGiKEJEug==
+mldoc@^1.3.9:
+  version "1.3.9"
+  resolved "https://registry.yarnpkg.com/mldoc/-/mldoc-1.3.9.tgz#9e45a25ba79596f3b0b0eace65f651a4c5a0c30a"
+  integrity sha512-UfqNuBphOj7paSBvozTdin5BLB5+W2tr2SGKTfp5eae6VQPx23oICx6RPQprft7KGFtn8T3rpM1YMUN8FaJLhg==
   dependencies:
     yargs "^12.0.2"
 

scripts/src/logseq/tasks/nbb.clj

@@ -24,6 +24,7 @@
   ;; distinct b/c sometimes namespaces are duplicated with .cljc analysis
   (doseq [n (distinct namespaces)]
     (println "Requiring" n "...")
+    ;; Run from current dir so that yarn command runs correctly
     (shell {:dir dir} "yarn nbb-logseq -cp" classpath "-e" (format "(require '[%s])" n)))
   (println "Success!"))
 
@@ -36,12 +37,12 @@
     (validate-namespaces namespaces "src/main" ".")))
 
 (defn load-all-namespaces
-  "Check all namespaces in source path(s) can be required by nbb-logseq"
-  [dir & paths]
+  "Check all namespaces in a directory can be required by nbb-logseq"
+  [dir]
   (let [{{:keys [namespace-definitions]} :analysis}
         (clj-kondo/run!
-         {:lint (map #(str dir "/" %) paths)
+         {:lint (map #(str dir "/" %) ["src"])
           :config {:output {:analysis {:namespace-definitions {:lang :cljs}}}}})]
     (validate-namespaces (map :name namespace-definitions)
-                         (str/join ":" paths)
+                         (str/trim (:out (shell {:dir dir :out :string} "clojure -Spath")))
                          dir)))

src/main/frontend/db.cljs

@@ -1,9 +1,9 @@
 (ns frontend.db
   (:require [clojure.core.async :as async]
             [datascript.core :as d]
-            [logseq.graph-parser.db.schema :as db-schema]
+            [logseq.db.schema :as db-schema]
             [frontend.db.conn :as conn]
-            [logseq.graph-parser.db.default :as default-db]
+            [logseq.db.default :as default-db]
             [frontend.db.model]
             [frontend.db.query-custom]
             [frontend.db.query-react]
@@ -66,7 +66,7 @@
  [frontend.db.query-react
   react-query custom-query-result-transform]
 
- [logseq.graph-parser.db.default built-in-pages-names built-in-pages])
+ [logseq.db.default built-in-pages-names built-in-pages])
 
 (defn get-schema-version [db]
   (d/q

src/main/frontend/db/conn.cljs

@@ -7,7 +7,7 @@
             [frontend.config :as config]
             [frontend.util.text :as text-util]
             [logseq.graph-parser.text :as text]
-            [logseq.graph-parser.db :as gp-db]))
+            [logseq.db :as ldb]))
 
 (defonce conns (atom {}))
 
@@ -71,7 +71,7 @@
    (start! repo {}))
   ([repo {:keys [listen-handler]}]
    (let [db-name (datascript-db repo)
-         db-conn (gp-db/start-conn)]
+         db-conn (ldb/start-conn)]
      (swap! conns assoc db-name db-conn)
      (when listen-handler
        (listen-handler repo)))))

src/main/frontend/db/model.cljs

@@ -8,7 +8,7 @@
             [datascript.core :as d]
             [frontend.config :as config]
             [frontend.date :as date]
-            [logseq.graph-parser.db.schema :as db-schema]
+            [logseq.db.schema :as db-schema]
             [frontend.db.conn :as conn]
             [frontend.db.react :as react]
             [frontend.db.utils :as db-utils]
@@ -16,7 +16,7 @@
             [frontend.util :as util :refer [react]]
             [logseq.graph-parser.util :as gp-util]
             [frontend.db.rules :refer [rules]]
-            [logseq.graph-parser.db.default :as default-db]
+            [logseq.db.default :as default-db]
             [frontend.util.drawer :as drawer]))
 
 ;; lazy loading

src/main/frontend/handler.cljs

@@ -6,7 +6,7 @@
             [frontend.config :as config]
             [frontend.context.i18n :as i18n]
             [frontend.db :as db]
-            [logseq.graph-parser.db.schema :as db-schema]
+            [logseq.db.schema :as db-schema]
             [frontend.db.conn :as conn]
             [frontend.db.react :as react]
             [frontend.error :as error]

src/main/frontend/handler/editor.cljs

@@ -13,7 +13,7 @@
             [frontend.config :as config]
             [frontend.date :as date]
             [frontend.db :as db]
-            [logseq.graph-parser.db.schema :as db-schema]
+            [logseq.db.schema :as db-schema]
             [frontend.db.model :as db-model]
             [frontend.db.utils :as db-utils]
             [frontend.diff :as diff]

src/main/frontend/handler/events.cljs

@@ -13,7 +13,7 @@
             [frontend.config :as config]
             [frontend.context.i18n :refer [t]]
             [frontend.db :as db]
-            [logseq.graph-parser.db.schema :as db-schema]
+            [logseq.db.schema :as db-schema]
             [frontend.encrypt :as encrypt]
             [frontend.extensions.srs :as srs]
             [frontend.fs :as fs]

src/main/frontend/handler/graph.cljs

@@ -2,7 +2,7 @@
   (:require [clojure.set :as set]
             [clojure.string :as string]
             [frontend.db :as db]
-            [logseq.graph-parser.db.default :as default-db]
+            [logseq.db.default :as default-db]
             [frontend.state :as state]
             [frontend.util :as util]))
 

src/main/frontend/handler/page.cljs

@@ -7,7 +7,7 @@
             [frontend.config :as config]
             [frontend.date :as date]
             [frontend.db :as db]
-            [logseq.graph-parser.db.schema :as db-schema]
+            [logseq.db.schema :as db-schema]
             [frontend.db.model :as model]
             [frontend.db.utils :as db-utils]
             [frontend.db.conn :as conn]

src/main/frontend/modules/outliner/core.cljs

@@ -4,7 +4,7 @@
             [datascript.impl.entity :as de]
             [frontend.db :as db]
             [frontend.db.model :as db-model]
-            [logseq.graph-parser.db.schema :as db-schema]
+            [logseq.db.schema :as db-schema]
             [frontend.db.conn :as conn]
             [frontend.db.outliner :as db-outliner]
             [frontend.modules.outliner.datascript :as ds]

src/main/frontend/publishing.cljs

@@ -2,7 +2,7 @@
   (:require [frontend.state :as state]
             [datascript.core :as d]
             [frontend.db :as db]
-            [logseq.graph-parser.db.schema :as db-schema]
+            [logseq.db.schema :as db-schema]
             [rum.core :as rum]
             [frontend.handler.route :as route]
             [frontend.page :as page]

src/test/frontend/test/fixtures.cljs

@@ -1,7 +1,7 @@
 (ns frontend.test.fixtures
   (:require [datascript.core :as d]
             [frontend.config :as config]
-            [logseq.graph-parser.db.schema :as db-schema]
+            [logseq.db.schema :as db-schema]
             [frontend.db.conn :as conn]
             [frontend.db.react :as react]
             [frontend.state :as state]