From 60297778a8466496e1ec983dd57dadc6ca6c5922 Mon Sep 17 00:00:00 2001 From: megayu Date: Fri, 30 Jan 2026 17:36:12 +0800 Subject: [PATCH 1/3] Fix issues related to importing file to db (#12353) * fix multiline block containing attributes will be truncated and lose data after importing * fix the issues with block attribute name contains "/" after importing * standardize eol, all files except .bat files use lf * compatible with windows path * fix multiple line importing issues --- .gitattributes | 3 ++ .prettierrc.js | 30 ++++++------- .../src/logseq/graph_parser/block.cljs | 45 ++++++++++--------- .../src/logseq/graph_parser/exporter.cljs | 13 +++--- .../src/logseq/graph_parser/property.cljs | 45 ++++++++++++++++++- .../logseq/graph_parser/exporter_test.cljs | 36 +++++++++++++-- .../journals/2024_08_07.md | 20 ++++++++- 7 files changed, 143 insertions(+), 49 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..7e3c52331a --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +* text=auto eol=lf + +*.bat text eol=crlf diff --git a/.prettierrc.js b/.prettierrc.js index ac1adffabf..76b1f5349a 100644 --- a/.prettierrc.js +++ b/.prettierrc.js @@ -1,15 +1,15 @@ -/** - * Copyright (c) Facebook, Inc. and its affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - * - */ - -'use strict' - -module.exports = { - singleQuote: true, - trailingComma: 'es5', - semi: false, -} +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + */ + +'use strict' + +module.exports = { + singleQuote: true, + trailingComma: 'es5', + semi: false, +} diff --git a/deps/graph-parser/src/logseq/graph_parser/block.cljs b/deps/graph-parser/src/logseq/graph_parser/block.cljs index f58e14187f..067159d9b2 100644 --- a/deps/graph-parser/src/logseq/graph_parser/block.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/block.cljs @@ -226,6 +226,7 @@ (subs (str k) 1) k) k (-> (string/lower-case k) + (string/replace "/" "-") (string/replace " " "-") (string/replace "_" "-"))] (if (gp-property/valid-property-name? (str ":" k)) @@ -644,7 +645,7 @@ properties)) (defn- construct-block - [block properties* timestamps body encoded-content format pos-meta {:keys [block-pattern db date-formatter remove-properties? db-graph-mode? export-to-db-graph?]}] + [block properties* timestamps body encoded-content format pos-meta {:keys [block-pattern db date-formatter remove-properties? remove-logbook? remove-deadline-scheduled? db-graph-mode? export-to-db-graph?]}] (let [id (get-custom-id-or-new-id properties*) block-tags (and export-to-db-graph? (get-in properties* [:properties :tags])) ;; For export, remove tags from properties as they are being converted to classes @@ -686,7 +687,11 @@ block) title (cond->> (get-block-content encoded-content block format pos-meta block-pattern) remove-properties? - (gp-property/remove-properties (get block :format :markdown))) + (gp-property/remove-properties (get block :format :markdown)) + remove-logbook? + (gp-property/remove-logbook) + remove-deadline-scheduled? + (gp-property/remove-deadline-scheduled)) block (assoc block :block/title title) block (if (seq timestamps) (merge block (timestamps->scheduled-and-deadline timestamps)) @@ -756,28 +761,23 @@ block-idx 0 timestamps {} properties {} - body []] + body [] + prev-block-num 0] (if (seq ast-blocks) (let [[ast-block pos-meta] (first ast-blocks)] (cond (paragraph-timestamp-block? ast-block) - (let [timestamps (extract-timestamps ast-block) - timestamps' (merge timestamps timestamps)] - (recur headings (rest ast-blocks) (inc block-idx) timestamps' properties body)) + (let [ts (extract-timestamps ast-block) + timestamps' (merge timestamps ts)] + (recur headings (rest ast-blocks) (inc block-idx) timestamps' properties body (inc prev-block-num))) (gp-property/properties-ast? ast-block) (let [properties (extract-properties (second ast-block) (assoc user-config :format format))] - (recur headings (rest ast-blocks) (inc block-idx) timestamps properties body)) + (recur headings (rest ast-blocks) (inc block-idx) timestamps properties body (inc prev-block-num))) (heading-block? ast-block) - ;; for db-graphs cut multi-line when there is property, deadline/scheduled or logbook text in :block/title - (let [cut-multiline? (and export-to-db-graph? - (when-let [prev-block (first (get all-blocks (dec block-idx)))] - (or (and (gp-property/properties-ast? prev-block) - (not= "Custom" (ffirst (get all-blocks (- block-idx 2))))) - (= ["Drawer" "logbook"] (take 2 prev-block)) - (and (= "Paragraph" (first prev-block)) - (seq (set/intersection (set (flatten prev-block)) #{"Deadline" "Scheduled"})))))) + (let [cut-multiline? (and export-to-db-graph? (= prev-block-num 0)) + prev-blocks (map first (subvec all-blocks (max 0 (- block-idx prev-block-num)) block-idx)) pos-meta' (if cut-multiline? pos-meta ;; fix start_pos @@ -785,12 +785,14 @@ (if (seq headings) (get-in (last headings) [:meta :start_pos]) nil))) - ;; Remove properties text from custom queries in db graphs + ;; Remove properties, deadline/scheduled and logbook text from title in db graphs options' (assoc options :remove-properties? - (and export-to-db-graph? - (and (gp-property/properties-ast? (first (get all-blocks (dec block-idx)))) - (= "Custom" (ffirst (get all-blocks (- block-idx 2))))))) + (and export-to-db-graph? (some gp-property/properties-ast? prev-blocks)) + :remove-logbook? + (and export-to-db-graph? (some #(= ["Drawer" "logbook"] (take 2 %)) prev-blocks)) + :remove-deadline-scheduled? + (and export-to-db-graph? (some #(seq (set/intersection (set (flatten %)) #{"Deadline" "Scheduled"})) prev-blocks))) block' (construct-block ast-block properties timestamps body encoded-content format pos-meta' options') block'' (cond db-graph-mode? @@ -799,11 +801,10 @@ (assoc block' :block.temp/ast-blocks (cons ast-block body)) :else (assoc block' :macros (extract-macros-from-ast (cons ast-block body))))] - - (recur (conj headings block'') (rest ast-blocks) (inc block-idx) {} {} [])) + (recur (conj headings block'') (rest ast-blocks) (inc block-idx) {} {} [] 0)) :else - (recur headings (rest ast-blocks) (inc block-idx) timestamps properties (conj body ast-block)))) + (recur headings (rest ast-blocks) (inc block-idx) timestamps properties (conj body ast-block) (inc prev-block-num)))) [(-> (reverse headings) sanity-blocks-data) body diff --git a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs index e50e0d2ee8..a2b57c9bb0 100644 --- a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs @@ -1229,9 +1229,10 @@ (defn- build-pdf-annotations-tx "Builds tx for pdf annotations when a pdf has an annotations EDN file under assets/" [parent-asset-path assets parent-asset pdf-annotation-pages opts] - (let [asset-edn-path (node-path/join common-config/local-assets-dir - (safe-sanitize-file-name - (node-path/basename (string/replace-first parent-asset-path #"(?i)\.pdf$" ".edn")))) + (let [asset-edn-path (path/path-normalize + (node-path/join common-config/local-assets-dir + (safe-sanitize-file-name + (node-path/basename (string/replace-first parent-asset-path #"(?i)\.pdf$" ".edn"))))) asset-md-name (str "hls__" (safe-sanitize-file-name (node-path/basename (string/replace-first parent-asset-path #"(?i)\.pdf$" ".md"))))] (when-let [asset-edn-map (get @assets asset-edn-path)] @@ -2168,8 +2169,10 @@ (-> (select-keys options [:notify-user :default-config : (get f rpath-key) path/path-normalize)) + logseq-file? #(string/starts-with? (normalized-rpath %) "logseq/") + asset-file? #(string/starts-with? (normalized-rpath %) "assets/") doc-files (->> files (remove #(or (logseq-file? %) (asset-file? %))) (filter #(contains? #{"md" "org" "markdown" "edn"} (path/file-ext (:path %))))) diff --git a/deps/graph-parser/src/logseq/graph_parser/property.cljs b/deps/graph-parser/src/logseq/graph_parser/property.cljs index 311b186eb2..ff3aaa527a 100644 --- a/deps/graph-parser/src/logseq/graph_parser/property.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/property.cljs @@ -175,4 +175,47 @@ (string/join "\n" lines)) :else - content)) \ No newline at end of file + content)) + +(defn remove-logbook + [content] + (when (string? content) + (let [lines (string/split-lines content) + [result _in-logbook?] + (reduce (fn [[acc in-logbook?] line] + (let [trimmed (string/trim line) + upper (string/upper-case trimmed)] + (cond + (string/starts-with? upper ":LOGBOOK:") + [acc true] + + (and in-logbook? (string/starts-with? upper ":END:")) + [acc false] + + in-logbook? + [acc true] + + :else + [(conj acc line) in-logbook?]))) + [[] false] + lines)] + (string/join "\n" result)))) + +(defn remove-deadline-scheduled + [content] + (when (string? content) + (let [lines (string/split-lines content)] + (if (= 1 (count lines)) + content + (let [first-line (first lines) + rest-lines (rest lines) + rest-lines (keep (fn [line] + (let [upper (string/upper-case (string/triml line))] + (if (or (string/starts-with? upper "DEADLINE: ") + (string/starts-with? upper "SCHEDULED: ")) + (let [cleaned (-> line (string/replace #"(?i)(?:^|\s)(DEADLINE|SCHEDULED):\s+<[^>]*>" "") string/trim)] + (when-not (string/blank? cleaned) + cleaned)) + line))) + rest-lines)] + (string/join "\n" (cons first-line rest-lines))))))) diff --git a/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs b/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs index d46b286fd8..87f71d8be8 100644 --- a/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs +++ b/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs @@ -213,7 +213,7 @@ (is (= 32 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Journal]] @conn)))) (is (= 5 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Asset]] @conn)))) - (is (= 4 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Task]] @conn)))) + (is (= 5 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Task]] @conn)))) (is (= 4 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Query]] @conn)))) (is (= 2 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Card]] @conn)))) (is (= 5 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Quote-block]] @conn)))) @@ -589,8 +589,36 @@ (testing "multiline blocks" (is (= "|markdown| table|\n|some|thing|" (:block/title (db-test/find-block-by-content @conn #"markdown.*table")))) - (is (= "multiline block\na 2nd\nand a 3rd" (:block/title (db-test/find-block-by-content @conn #"multiline block")))) - (is (= "logbook block" (:block/title (db-test/find-block-by-content @conn #"logbook block"))))) + (is (= "normal multiline block\na 2nd\nand a 3rd" (:block/title (db-test/find-block-by-content @conn #"normal multiline block")))) + (is (= "colored multiline block\nlast line" (:block/title (db-test/find-block-by-content @conn #"colored multiline block")))) + + (let [block (db-test/find-block-by-content @conn #"multiline block with prop and deadline")] + (is (= "multiline block with prop and deadline\nlast line" (:block/title block))) + (is (= 20221126 + (-> (db-test/readable-properties block) + :logseq.property/deadline + date-time-util/ms->journal-day)) + "multiline block has correct journal as property value") + (is (= "red" + (-> (db-test/readable-properties block) + :logseq.property/background-color)) + "multiline block has correct background color as property value")) + + (let [block (db-test/find-block-by-content @conn #"multiline block with deadline and scheduled in 1 line and sth else")] + (is (= "multiline block with deadline and scheduled in 1 line and sth else\nsomething else\nlast line" (:block/title block))) + (is (= 20221126 + (-> (db-test/readable-properties block) + :logseq.property/deadline + date-time-util/ms->journal-day)) + "multiline block with deadline and scheduled has correct deadline journal as property value") + (is (= 20221126 + (-> (db-test/readable-properties block) + :logseq.property/scheduled + date-time-util/ms->journal-day)) + "multiline block with deadline and scheduled has correct scheduled journal as property value")) + + (is (= "logbook block" (:block/title (db-test/find-block-by-content @conn #"^logbook block")))) + (is (= "multiline logbook block\nlast line" (:block/title (db-test/find-block-by-content @conn #"multiline logbook block"))))) (testing ":block/refs" (let [page (db-test/find-page-by-title @conn "chat-gpt")] @@ -623,7 +651,7 @@ count)) "Correct number of user classes") - (is (= 4 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Task]] @conn)))) + (is (= 5 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Task]] @conn)))) (is (= 4 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Query]] @conn)))) (is (= 2 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Card]] @conn)))) diff --git a/deps/graph-parser/test/resources/exporter-test-graph/journals/2024_08_07.md b/deps/graph-parser/test/resources/exporter-test-graph/journals/2024_08_07.md index eae6224ac7..9c221d6526 100644 --- a/deps/graph-parser/test/resources/exporter-test-graph/journals/2024_08_07.md +++ b/deps/graph-parser/test/resources/exporter-test-graph/journals/2024_08_07.md @@ -5,9 +5,25 @@ |some|thing| - block with props prop-num:: 10 -- multiline block +- normal multiline block a 2nd and a 3rd +- colored multiline block + background-color:: red + last line +- multiline block with prop and deadline + background-color:: red + DEADLINE: <2022-11-26 Sat> + last line +- multiline block with deadline and scheduled in 1 line and sth else + DEADLINE: <2022-11-26 Sat> SCHEDULED: <2022-11-26 Sat> something else + last line +- DONE multiline logbook block + :LOGBOOK: + CLOCK: [2024-08-07 Wed 11:47:50] + CLOCK: [2024-08-07 Wed 11:47:53] + :END: + last line - DOING logbook block :LOGBOOK: CLOCK: [2024-08-07 Wed 11:47:50] @@ -56,4 +72,4 @@ #+BEGIN_EXAMPLE {{query (and [[tag2]] (not [[tag1]]))}} - #+END_EXAMPLE \ No newline at end of file + #+END_EXAMPLE From e028b223a2d108e0e38861c4a9a97569a78bf894 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Fri, 30 Jan 2026 21:19:40 +0800 Subject: [PATCH 2/3] add sync deploy workflow --- .github/workflows/deploy-sync-test.yml | 78 ++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 .github/workflows/deploy-sync-test.yml diff --git a/.github/workflows/deploy-sync-test.yml b/.github/workflows/deploy-sync-test.yml new file mode 100644 index 0000000000..cb0e604b64 --- /dev/null +++ b/.github/workflows/deploy-sync-test.yml @@ -0,0 +1,78 @@ +name: Deploy new sync test + +on: + workflow_dispatch: + inputs: + branch: + description: "Git branch to build & deploy" + required: true + default: "feat/worker-sync" + +env: + CLOJURE_VERSION: "1.11.1.1413" + NODE_VERSION: "22" + JAVA_VERSION: "11" + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + + steps: + - name: Checkout selected branch + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + fetch-depth: 0 + + - name: Setup Java JDK + uses: actions/setup-java@v4 + with: + distribution: "zulu" + java-version: ${{ env.JAVA_VERSION }} + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Setup Clojure + uses: DeLaGuardo/setup-clojure@11.0 + with: + cli: ${{ env.CLOJURE_VERSION }} + + - name: Install Yarn deps + run: yarn install --frozen-lockfile + + - name: Set build environment flags + run: | + echo "ENABLE_FILE_SYNC_PRODUCTION=true" >> $GITHUB_ENV + echo "ENABLE_RTC_SYNC_PRODUCTION=true" >> $GITHUB_ENV + + - name: Build web + workers + run: | + yarn gulp:build + clojure -M:cljs release app db-worker inference-worker \ + --config-merge '{:compiler-options {:source-map true :source-map-include-sources-content true :source-map-detail-level :symbols}}' + yarn webpack-app-build + + rsync -avz \ + --exclude node_modules \ + --exclude android \ + --exclude ios \ + --exclude mobile \ + ./static/ ./public/ + + rm ./public/js/*.map + + env: + LOGSEQ_POSTHOG_TOKEN: ${{ secrets.LOGSEQ_POSTHOG_TOKEN }} + + - name: Publish to Cloudflare Pages + uses: cloudflare/pages-action@1 + with: + apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} + accountId: 2553ea8236c11ea0f88de28fce1cbfee + projectName: "sync" + directory: "public" + gitHubToken: ${{ secrets.GITHUB_TOKEN }} + branch: ${{ inputs.branch }} From 4cad271f6a42a71f644548704dbebe6f1bf2c8a0 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Fri, 30 Jan 2026 21:33:37 +0800 Subject: [PATCH 3/3] update project name --- .github/workflows/deploy-sync-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-sync-test.yml b/.github/workflows/deploy-sync-test.yml index cb0e604b64..6451f4c5da 100644 --- a/.github/workflows/deploy-sync-test.yml +++ b/.github/workflows/deploy-sync-test.yml @@ -72,7 +72,7 @@ jobs: with: apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} accountId: 2553ea8236c11ea0f88de28fce1cbfee - projectName: "sync" + projectName: "test-sync" directory: "public" gitHubToken: ${{ secrets.GITHUB_TOKEN }} branch: ${{ inputs.branch }}