clone/logseq
1
0
Fork 0
mirror of https://github.com/logseq/logseq.git synced 2026-04-24 22:25:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into enhance/ios-native-navigation

Browse Source
This commit is contained in:
Tienson Qin
2025-11-24 22:36:50 +08:00
parent 9af504e568 4684f8d74c
commit 73d4ee7caa
102 changed files with 2860 additions and 1586 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
.github/workflows/build-ios.yml vendored
View File

@@ -1,84 +0,0 @@
# This workflow tries to build iOS app if any changes detected on the iOS source tree,
# ensuring at least it builds.
name: CI-iOS
on:
push:
branches: [master]
paths:
- 'ios/App'
- package.json
pull_request:
branches: [master]
paths:
- 'ios/App'
- package.json
env:
CLOJURE_VERSION: '1.11.1.1413'
NODE_VERSION: '22'
JAVA_VERSION: '11'
jobs:
build-app:
runs-on: macos-14
steps:
- name: Check out Git repository
uses: actions/checkout@v4
- name: Install Node.js, NPM and Yarn
uses: actions/setup-node@v4
with:
node-version: ${{ env.NODE_VERSION }}
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
- name: Cache yarn cache directory
uses: actions/cache@v4
id: yarn-cache
with:
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Setup Java JDK
uses: actions/setup-java@v4
with:
distribution: 'zulu'
java-version: ${{ env.JAVA_VERSION }}
- name: Cache clojure deps
uses: actions/cache@v4
with:
path: |
~/.m2/repository
~/.gitlibs
key: ${{ runner.os }}-clojure-lib-${{ hashFiles('**/deps.edn') }}
- name: Setup clojure
uses: DeLaGuardo/setup-clojure@10.1
with:
cli: ${{ env.CLOJURE_VERSION }}
- name: Set Build Environment Variables
run: |
echo "ENABLE_FILE_SYNC_PRODUCTION=true" >> $GITHUB_ENV
- name: Compile CLJS
run: yarn install && yarn release-mobile
- name: Prepare iOS build
run: npx cap sync ios
- name: List iOS build targets
run: xcodebuild -list -workspace App.xcworkspace
working-directory: ./ios/App
- name: Build iOS App
run: |
xcodebuild -workspace App.xcworkspace -scheme Logseq -destination generic/platform=iOS build CODE_SIGN_IDENTITY="" CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO
working-directory: ./ios/App

1
android/app/capacitor.build.gradle
View File

@@ -9,6 +9,7 @@ android {
apply from: "../capacitor-cordova-android-plugins/cordova.variables.gradle"
dependencies {
implementation project(':aparajita-capacitor-secure-storage')
implementation project(':capacitor-community-safe-area')
implementation project(':capacitor-action-sheet')
implementation project(':capacitor-app')

4
android/app/src/main/assets/capacitor.plugins.json
View File

@@ -1,4 +1,8 @@
[
{
"pkg": "@aparajita/capacitor-secure-storage",
"classpath": "com.aparajita.capacitor.securestorage.SecureStorage"
},
{
"pkg": "@capacitor-community/safe-area",
"classpath": "com.getcapacitor.community.safearea.SafeAreaPlugin"

3
android/capacitor.settings.gradle
View File

@@ -2,6 +2,9 @@
include ':capacitor-android'
project(':capacitor-android').projectDir = new File('../node_modules/@capacitor/android/capacitor')
include ':aparajita-capacitor-secure-storage'
project(':aparajita-capacitor-secure-storage').projectDir = new File('../node_modules/@aparajita/capacitor-secure-storage/android')
include ':capacitor-community-safe-area'
project(':capacitor-community-safe-area').projectDir = new File('../node_modules/@capacitor-community/safe-area/android')

26
clj-e2e/src/logseq/e2e/graph.clj
View File

@@ -16,21 +16,42 @@
[]
(util/search-and-click "Go to all graphs"))
(defn new-graph
(defn- input-e2ee-password
[]
(w/click "input[type=\"password\"]")
(util/input "e2etest")
(w/click "button:text(\"Submit\")"))
(defn- new-graph-helper
[graph-name enable-sync?]
(util/search-and-click "Add a DB graph")
(w/wait-for "h2:text(\"Create a new graph\")")
(w/click "input[placeholder=\"your graph name\"]")
(util/input graph-name)
(when enable-sync?
(w/wait-for "button#rtc-sync" {:timeout 3000})
(w/click "button#rtc-sync"))
(w/click "button:text(\"Submit\")")
(w/click "button:not([disabled]):text(\"Submit\")")
(when enable-sync?
(input-e2ee-password)
(w/wait-for "button.cloud.on.idle" {:timeout 20000}))
;; new graph can blocks the ui because the db need to be created and restored,
;; I have no idea why `search-and-click` failed to auto-wait sometimes.
(util/wait-timeout 1000))
(defn new-graph
[graph-name enable-sync?]
(try
(new-graph-helper graph-name enable-sync?)
(catch com.microsoft.playwright.TimeoutError e
;; sometimes, 'Use Logseq Sync?' option not showing
;; because of user-group not recv from server yet
;; workaround: try again
(if enable-sync?
(do (w/click "button.ui__dialog-close")
(new-graph-helper graph-name enable-sync?))
(throw e)))))
(defn wait-for-remote-graph
[graph-name]
(goto-all-graphs)
@@ -52,6 +73,7 @@
(goto-all-graphs)
(w/click (.last (w/-query (format "div[data-testid='logseq_db_%1$s'] span:has-text('%1$s')" to-graph-name))))
(when wait-sync?
(input-e2ee-password)
(w/wait-for "button.cloud.on.idle" {:timeout 20000}))
(assert/assert-graph-loaded?))

1
deps/cli/.carve/ignore vendored
View File

@@ -8,3 +8,4 @@ logseq.cli.commands.export/export
logseq.cli.commands.append/append
logseq.cli.commands.mcp-server/start
logseq.cli.commands.import-edn/import-edn
logseq.cli.commands.validate/validate

11
deps/cli/CHANGELOG.md vendored
View File

@@ -1,3 +1,14 @@
## 0.4.0
* BREAKING CHANGE: Commands that call local graphs are invoked with `-g` instead of as an argument e.g. `logseq search foo -g db-name` instead of `logseq search db-name foo`
* Add `import-edn` command for local and in-app graphs
* Add `validate` command for local graphs
* Add `export-edn` command for API mode
* Fix most commands with API mode not respecting `$LOGSEQ_API_SERVER_TOKEN`
* Fix API `mcp-server` command failing lazily
* Fix commands failing confusingly when given a file graph
* Fix `query` command with multiple local graphs not switching graphs
* Fix API `search` command
## 0.3.0
* Add mcp-server command to run a MCP server
* All commands that have graph args and options now support local paths e.g. `logseq search $HOME/Downloads/logseq_db_yep_1751032977.sqlite foo`

62
deps/cli/README.md vendored
View File

@@ -1,6 +1,6 @@
## Description
This library provides a `logseq` CLI for DB graphs. The CLI currently only applies to desktop DB graphs and requires the [database-version](/README.md#-database-version) desktop app to be installed. The CLI works offline by default which means it can also be used on CI/CD platforms like Github Actions. Some CLI commands can also interact with the current DB graph if the [HTTP Server](https://docs.logseq.com/#/page/local%20http%20server) is turned on in the Desktop app.
This library provides a `logseq` CLI for DB graphs created using the [database-version](/README.md#-database-version). By default, the CLI works offline with local graphs. This allows for running commands automatically on CI/CD platforms like Github Actions. Most CLI commands also connect to the current DB graph in a desktop app (a.k.a. in-app graph) if the [HTTP API Server](https://docs.logseq.com/#/page/local%20http%20server) is turned on.
## Install
@@ -12,11 +12,11 @@ This section assumes you have installed the CLI from npm or via the [dev
setup](#setup). If you haven't, substitute `node cli.mjs` for `logseq` e.g.
`node.cli.mjs -h`.
All commands except for `append` can be used offline or on CI. The `search` command and any command that has an api-server-token option require the [HTTP API Server](https://docs.logseq.com/#/page/local%20http%20server) to be turned on.
All commands work with both local graphs and the current in-app graph except for `append` (in-app graph only), `validate` (local graph only) and `export` (local graph only). For a command to work with an in-app graph, the [HTTP API Server](https://docs.logseq.com/#/page/local%20http%20server) must be turned on.
Now let's use it!
Now let's use the CLI!
```
```sh
$ logseq -h
Usage: logseq [command] [options]
@@ -24,15 +24,16 @@ Options:
-v, --version Print version
Commands:
list List graphs
list List local graphs
show Show DB graph(s) info
search [options] Search DB graph
query [options] Query DB graph(s)
export [options] Export DB graph as Markdown
export-edn [options] Export DB graph as EDN
import-edn [options] Import into DB graph with EDN
append [options] Appends text to current page
mcp-server [options] Run a MCP server
import-edn [options] Import into DB graph with EDN
validate [options] Validate DB graph
help Print a command's help
$ logseq list
@@ -56,7 +57,11 @@ $ logseq show db-test
| Graph initial schema version | {:major 65, :minor 7} |
| Graph created by commit | https://github.com/logseq/logseq/commit/3c93fd2637 |
| Graph imported by | :cli/create-graph |
```
To run a command against the current desktop graph, set `$LOGSEQ_API_SERVER_TOKEN` once or set `-a` each time with a valid token for the desktop's HTTP API server:
```sh
# Search your current graph and print highlighted results one per line like grep
$ logseq search woot -a my-token
Search found 100 results:
@@ -64,11 +69,15 @@ dev:db-export woot woot.edn && dev:db-create woot2 woot.edn
dev:db-diff woot woot2
...
# Can also authenticate api with $LOGSEQ_API_SERVER_TOKEN
$ LOGSEQ_API_SERVER_TOKEN=my-token logseq search woot
$ export LOGSEQ_API_SERVER_TOKEN=my-token
$ logseq search woot
...
```
Here are more examples of all the available commands:
```sh
# Search a local graph
$ logseq search woot page
$ logseq search page -g woot
Search found 23 results:
Node page
Annotation page
@@ -76,7 +85,7 @@ Annotation page
# Query a graph locally using `d/entity` id(s) like an integer or a :db/ident
# Can also specify a uuid string to fetch an entity
$ logseq query woot 10 :logseq.class/Tag
$ logseq query 10 :logseq.class/Tag -g woot
({:db/id 10,
:db/ident :logseq.kv/graph-git-sha,
:kv/value "f736895b1b-dirty"}
@@ -92,7 +101,7 @@ $ logseq query woot 10 :logseq.class/Tag
:block/name "tag"})
# Query a graph using a datalog query
$ logseq query woot '[:find (pull ?b [*]) :where [?b :kv/value]]'
$ logseq query '[:find (pull ?b [*]) :where [?b :kv/value]]' -g woot
[{:db/id 5, :db/ident :logseq.kv/db-type, :kv/value "db"}
{:db/id 6,
:db/ident :logseq.kv/schema-version,
@@ -116,18 +125,26 @@ $ logseq query '(task DOING)' -a my-token
:uuid "68795144-e5f6-48e8-849d-79cd6473b952"}
...
# Export DB graph as markdown
$ logseq export yep
# Export local graph as markdown
$ logseq export -g yep
Exported 41 pages to yep_markdown_1756128259.zip
# Export DB graph as EDN
$ logseq export-edn woot -f woot.edn
# Export current graph as EDN
$ logseq export-edn -a my-token
Exported 16 properties, 3 classes and 16 pages to yep_1763407592.edn
# Export local graph as EDN to specified file
$ logseq export-edn -g woot -f woot.edn
Exported 16 properties, 1 classes and 36 pages to woot.edn
# Import into current graph with EDN
$ logseq import-edn -f woot-ontology.edn
Imported 16 properties, 1 classes and 0 pages!
# Validate a local graph. Useful to run in CI
$ logseq validate -g woot
Read graph woot with counts: {:entities 317, :pages 159, :blocks 147, :classes 17, :properties 112, :objects 64, :property-pairs 669, :datoms 3964}
Valid!
# Append text to current page
$ logseq append add this text -a my-token
Success!
@@ -159,7 +176,7 @@ First install the following dependencies:
* Run `yarn install` to install npm dependencies.
* Install [babashka](https://github.com/babashka/babashka).
To install the CLI locally, `yarn link`.
To install the CLI locally so that local changes are immediately reflected in `logseq`, `yarn link`.
### Testing
@@ -167,7 +184,7 @@ Testing is done with nbb-logseq and
[nbb-test-runner](https://github.com/nextjournal/nbb-test-runner). Some basic
usage:
```
```sh
# Run all tests
$ yarn test
# List available options
@@ -178,4 +195,15 @@ $ yarn test -i focus
### Managing dependencies
See [standard nbb/cljs library advice in graph-parser](/deps/graph-parser/README.md#managing-dependencies).
See [standard nbb/cljs library advice in graph-parser](/deps/graph-parser/README.md#managing-dependencies).
### Build
To build and install a local version of the CLI:
```sh
$ bb build:vendor-nbb-deps && npm pack && npm install -g ./logseq-cli-*.tgz
# Run this to bring local code back to a clean state. Not running this will cause local dev issues
$ git checkout nbb.edn && rm -rf vendor logseq-cli*.tgz
```
The above is useful for testing the build process and ensuring the released tarball has no issues.

2
deps/cli/package.json vendored
View File

@@ -1,6 +1,6 @@
{
"name": "@logseq/cli",
"version": "0.3.0",
"version": "0.4.0",
"description": "Logseq CLI",
"bin": {
"logseq": "cli.mjs"

33
deps/cli/src/logseq/cli.cljs vendored
View File

@@ -68,7 +68,7 @@
(js/process.exit 1))))))
(def ^:private table
[{:cmds ["list"] :desc "List graphs"
[{:cmds ["list"] :desc "List local graphs"
:fn (lazy-load-fn 'logseq.cli.commands.graph/list-graphs)}
{:cmds ["show"] :desc "Show DB graph(s) info"
:description "For each graph, prints information related to a graph's creation and anything that is helpful for debugging."
@@ -78,35 +78,38 @@
:fn (lazy-load-fn 'logseq.cli.commands.search/search)
:desc "Search DB graph"
:description "Search a local graph or the current in-app graph if --api-server-token is given. For a local graph it only searches the :block/title of blocks."
:args->opts [:graph :search-terms] :coerce {:search-terms []} :require [:graph]
:args->opts [:search-terms] :coerce {:search-terms []}
:spec cli-spec/search}
{:cmds ["query"] :desc "Query DB graph(s)"
:description "Query a local graph or the current in-app graph if --api-server-token is given. For a local graph, queries are a datalog query or an entity query. A datalog query can use built-in rules. An entity query consists of one or more integers, uuids or :db/ident keywords. For an in-app query, queries can be an advanced or simple query."
:fn (lazy-load-fn 'logseq.cli.commands.query/query)
:args->opts [:graph :args] :coerce {:args []} :no-keyword-opts true :require [:graph]
:args->opts [:args] :coerce {:args []} :no-keyword-opts true
:spec cli-spec/query}
{:cmds ["export"] :desc "Export DB graph as Markdown"
:description "Export a graph to Markdown like the in-app graph export."
:description "Export a local graph to Markdown like the in-app graph export."
:fn (lazy-load-fn 'logseq.cli.commands.export/export)
:args->opts [:graph] :require [:graph]
:spec cli-spec/export}
{:cmds ["export-edn"] :desc "Export DB graph as EDN"
:description "Export a graph to EDN like the in-app graph EDN export. See https://github.com/logseq/docs/blob/master/db-version.md#edn-data-export for more about this export type."
:description "Export a local graph to EDN or the current in-app graph if --api-server-token is given. See https://github.com/logseq/docs/blob/master/db-version.md#edn-data-export for more about this export type."
:fn (lazy-load-fn 'logseq.cli.commands.export-edn/export)
:args->opts [:graph] :require [:graph]
:spec cli-spec/export-edn}
{:cmds ["append"] :desc "Appends text to current page"
:fn (lazy-load-fn 'logseq.cli.commands.append/append)
:args->opts [:args] :require [:args] :coerce {:args []}
:spec cli-spec/append}
{:cmds ["mcp-server"] :desc "Run a MCP server"
:description "Run a MCP server against a local graph if --graph is given or against the current in-app graph. By default the MCP server runs as a HTTP Streamable server. Use --stdio to run it as a stdio server."
:fn (lazy-load-fn 'logseq.cli.commands.mcp-server/start)
:spec cli-spec/mcp-server}
{:cmds ["import-edn"] :desc "Import into DB graph with EDN"
:description "Import with EDN into a local graph or the current in-app graph if --api-server-token is given. See https://github.com/logseq/docs/blob/master/db-version.md#edn-data-export for more about this import type."
:fn (lazy-load-fn 'logseq.cli.commands.import-edn/import-edn)
:spec cli-spec/import-edn}
{:cmds ["append"] :desc "Append text to current page"
:description "Append text to current page of current in-app graph."
:fn (lazy-load-fn 'logseq.cli.commands.append/append)
:args->opts [:args] :require [:args] :coerce {:args []}
:spec cli-spec/append}
{:cmds ["mcp-server"] :desc "Run a MCP server"
:description "Run a MCP server against a local graph if --graph is given or against the current in-app graph. For the in-app graph, the API server must be on in the app. By default the MCP server runs as a HTTP Streamable server. Use --stdio to run it as a stdio server."
:fn (lazy-load-fn 'logseq.cli.commands.mcp-server/start)
:spec cli-spec/mcp-server}
{:cmds ["validate"] :desc "Validate DB graph"
:description "Validate a local DB graph. Exit 1 if there are validation errors"
:fn (lazy-load-fn 'logseq.cli.commands.validate/validate)
:spec cli-spec/validate}
{:cmds ["help"] :fn help-command :desc "Print a command's help"
:args->opts [:command] :require [:command]}
{:cmds []

3
deps/cli/src/logseq/cli/commands/export.cljs vendored
View File

@@ -74,7 +74,10 @@
(println "Exported" (count exported-files) "pages to" file-name)))))
(defn export [{{:keys [graph] :as opts} :opts}]
(when-not graph
(cli-util/error "Command missing required option 'graph'"))
(if (fs/existsSync (cli-util/get-graph-path graph))
(let [conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph))]
(cli-util/ensure-db-graph-for-command @conn)
(export-repo-as-markdown! (str common-config/db-version-prefix graph) @conn opts))
(cli-util/error "Graph" (pr-str graph) "does not exist")))

49
deps/cli/src/logseq/cli/commands/export_edn.cljs vendored
View File

@@ -2,20 +2,45 @@
"Export edn command"
(:require ["fs" :as fs]
[clojure.pprint :as pprint]
[logseq.cli.util :as cli-util]
[logseq.common.util :as common-util]
[logseq.db.common.sqlite-cli :as sqlite-cli]
[logseq.db.sqlite.export :as sqlite-export]
[logseq.common.util :as common-util]
[logseq.cli.util :as cli-util]))
[logseq.db.sqlite.util :as sqlite-util]
[promesa.core :as p]))
(defn export [{{:keys [graph] :as options} :opts}]
(defn- write-export-edn-map [export-map {:keys [graph file]}]
(let [file' (or file (str graph "_" (quot (common-util/time-ms) 1000) ".edn"))]
(println (str "Exported " (cli-util/summarize-build-edn export-map) " to " file'))
(fs/writeFileSync file' (with-out-str (pprint/pprint export-map)))))
(defn- build-export-options [options]
(cond-> {:export-type (:export-type options)}
(= :graph (:export-type options))
(assoc :graph-options (dissoc options :file :export-type :graph))))
(defn- local-export [{{:keys [graph] :as options} :opts}]
(when-not graph
(cli-util/error "Command missing required option 'graph'"))
(if (fs/existsSync (cli-util/get-graph-path graph))
(let [conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph))
export-map (sqlite-export/build-export @conn
(cond-> {:export-type (:export-type options)}
(= :graph (:export-type options))
(assoc :graph-options (dissoc options :file :export-type :graph))))
file (or (:file options) (str graph "_" (quot (common-util/time-ms) 1000) ".edn"))]
(println (str "Exported " (cli-util/summarize-build-edn export-map) " to " file))
(fs/writeFileSync file
(with-out-str (pprint/pprint export-map))))
(cli-util/error "Graph" (pr-str graph) "does not exist")))
_ (cli-util/ensure-db-graph-for-command @conn)
export-map (sqlite-export/build-export @conn (build-export-options options))]
(write-export-edn-map export-map options))
(cli-util/error "Graph" (pr-str graph) "does not exist")))
(defn- api-export
[{{:keys [api-server-token] :as options} :opts}]
(let [opts (build-export-options options)]
(-> (p/let [resp (cli-util/api-fetch api-server-token "logseq.cli.export_edn" [(clj->js opts)])]
(if (= 200 (.-status resp))
(p/let [body (.json resp)
export-map (sqlite-util/transit-read (aget body "export-body"))]
(write-export-edn-map export-map (assoc options :graph (.-graph body))))
(cli-util/api-handle-error-response resp)))
(p/catch cli-util/command-catch-handler))))
(defn export [{opts :opts :as m}]
(if (cli-util/api-command? opts)
(api-export m)
(local-export m)))

1
deps/cli/src/logseq/cli/commands/graph.cljs vendored
View File

@@ -22,6 +22,7 @@
(let [graph-dir (cli-util/get-graph-path graph)]
(if (fs/existsSync graph-dir)
(let [conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph))
_ (cli-util/ensure-db-graph-for-command @conn)
kv-value #(:kv/value (d/entity @conn %))]
(pprint/print-table
(map #(array-map "Name" (first %) "Value" (second %))

13
deps/cli/src/logseq/cli/commands/import_edn.cljs vendored
View File

@@ -12,7 +12,7 @@
(defn- print-success [import-map]
(println (str "Imported " (cli-util/summarize-build-edn import-map) "!")))
(defn- api-import [api-server-token import-map]
(defn- api-import [{:keys [api-server-token]} import-map]
(-> (p/let [resp (cli-util/api-fetch api-server-token "logseq.cli.import_edn" [(sqlite-util/transit-write import-map)])]
(if (= 200 (.-status resp))
(print-success import-map)
@@ -20,8 +20,11 @@
(p/catch cli-util/command-catch-handler)))
(defn- local-import [{:keys [graph]} import-map]
(if (and graph (fs/existsSync (cli-util/get-graph-path graph)))
(when-not graph
(cli-util/error "Command missing required option 'graph'"))
(if (fs/existsSync (cli-util/get-graph-path graph))
(let [conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph))
_ (cli-util/ensure-db-graph-for-command @conn)
{:keys [init-tx block-props-tx misc-tx]}
(sqlite-export/build-import import-map @conn {})
txs (vec (concat init-tx block-props-tx misc-tx))]
@@ -29,8 +32,8 @@
(print-success import-map))
(cli-util/error "Graph" (pr-str graph) "does not exist")))
(defn import-edn [{{:keys [api-server-token file] :as opts} :opts}]
(defn import-edn [{{:keys [file] :as opts} :opts}]
(let [edn (edn/read-string (str (fs/readFileSync file)))]
(if api-server-token
(api-import api-server-token edn)
(if (cli-util/api-command? opts)
(api-import opts edn)
(local-import opts edn))))

13
deps/cli/src/logseq/cli/commands/mcp_server.cljs vendored
View File

@@ -76,14 +76,17 @@
#js {:error (str "Server status " (.-status resp)
"\nAPI Response: " (pr-str body))}))))
(defn- create-mcp-server [{{:keys [api-server-token]} :opts} graph]
(if graph
(defn- create-mcp-server [{{:keys [api-server-token] :as opts} :opts} graph]
(if (cli-util/api-command? opts)
;; Make an initial /api call to ensure the API server is on
(-> (p/let [_resp (call-api api-server-token "logseq.app.search" ["foo"])]
(cli-common-mcp-server/create-mcp-api-server (partial call-api api-server-token)))
(p/catch cli-util/command-catch-handler))
(let [mcp-server (cli-common-mcp-server/create-mcp-server)
conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph))]
(doseq [[k v] local-tools]
(.registerTool mcp-server (name k) (:config v) (partial (:fn v) conn)))
mcp-server)
(cli-common-mcp-server/create-mcp-api-server (partial call-api api-server-token))))
mcp-server)))
(defn start [{{:keys [debug-tool graph stdio api-server-token] :as opts} :opts :as m}]
(when (and graph (not (fs/existsSync (cli-util/get-graph-path graph))))
@@ -101,7 +104,7 @@
(clj->js (dissoc opts :debug-tool)))]
(js/console.log resp))
(cli-util/error "Tool" (pr-str debug-tool) "not found")))
(let [mcp-server (create-mcp-server m graph)]
(p/let [mcp-server (create-mcp-server m graph)]
(if stdio
(nbb/await (.connect mcp-server (StdioServerTransport.)))
(start-http-server mcp-server (select-keys opts [:port :host]))))))

61
deps/cli/src/logseq/cli/commands/query.cljs vendored
View File

@@ -71,36 +71,37 @@
(if (= 1 (count (first res))) (mapv first res) res)))
(defn- local-query
[{{:keys [graph args graphs properties-readable title-query]} :opts}]
(let [graphs' (into [graph] graphs)]
(doseq [graph' graphs']
(if (fs/existsSync (cli-util/get-graph-path graph'))
(let [conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph))
query* (when (string? (first args)) (common-util/safe-read-string {:log-error? false} (first args)))
results (cond
;; Run datalog query if detected
(and (vector? query*) (= :find (first query*)))
(local-datalog-query @conn query*)
;; Runs predefined title query. Predefined queries could better off in a separate command
;; since they could be more powerful and have different args than query command
title-query
(let [query '[:find (pull ?b [*])
:in $ % ?search-term
:where (block-content ?b ?search-term)]
res (d/q query @conn (rules/extract-rules rules/db-query-dsl-rules)
(string/join " " args))]
;; Remove nesting for most queries which just have one :find binding
(if (= 1 (count (first res))) (mapv first res) res))
:else
(local-entities-query @conn properties-readable args))]
(when (> (count graphs') 1)
(println "Results for graph" (pr-str graph')))
(pprint/pprint results))
(cli-util/error "Graph" (pr-str graph') "does not exist")))))
[{{:keys [args graphs properties-readable title-query]} :opts}]
(when-not graphs
(cli-util/error "Command missing required option 'graphs'"))
(doseq [graph graphs]
(if (fs/existsSync (cli-util/get-graph-path graph))
(let [conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph))
_ (cli-util/ensure-db-graph-for-command @conn)
query* (when (string? (first args)) (common-util/safe-read-string {:log-error? false} (first args)))
results (cond
;; Run datalog query if detected
(and (vector? query*) (= :find (first query*)))
(local-datalog-query @conn query*)
;; Runs predefined title query. Predefined queries could better off in a separate command
;; since they could be more powerful and have different args than query command
title-query
(let [query '[:find (pull ?b [*])
:in $ % ?search-term
:where (block-content ?b ?search-term)]
res (d/q query @conn (rules/extract-rules rules/db-query-dsl-rules)
(string/join " " args))]
;; Remove nesting for most queries which just have one :find binding
(if (= 1 (count (first res))) (mapv first res) res))
:else
(local-entities-query @conn properties-readable args))]
(when (> (count graphs) 1)
(println "Results for graph" (pr-str graph)))
(pprint/pprint results))
(cli-util/error "Graph" (pr-str graph) "does not exist"))))
(defn query
[{{:keys [graph args api-server-token]} :opts :as m}]
(if api-server-token
;; graph can be query since it's not used for api-query
(api-query (or graph (first args)) api-server-token)
[{{:keys [args api-server-token] :as opts} :opts :as m}]
(if (cli-util/api-command? opts)
(api-query (first args) api-server-token)
(local-query m)))

17
deps/cli/src/logseq/cli/commands/search.cljs vendored
View File

@@ -36,9 +36,9 @@
highlight-content-query
#(string/replace % search-term (highlight search-term)))]
(println (string/join "\n"
(->> results
(map #(string/replace % "\n" "\\\\n"))
(map highlight-fn)))))))
(->> results
(map #(string/replace % "\n" "\\\\n"))
(map highlight-fn)))))))
(defn- api-search
[search-term {{:keys [api-server-token raw limit]} :opts}]
@@ -46,13 +46,16 @@
(if (= 200 (.-status resp))
(p/let [body (.json resp)]
(let [{:keys [blocks]} (js->clj body :keywordize-keys true)]
(format-results (map :block/title blocks) search-term {:raw raw :api? true})))
(format-results (map :title blocks) search-term {:raw raw :api? true})))
(cli-util/api-handle-error-response resp)))
(p/catch cli-util/command-catch-handler)))
(defn- local-search [search-term {{:keys [graph raw limit]} :opts}]
(when-not graph
(cli-util/error "Command missing required option 'graph'"))
(if (fs/existsSync (cli-util/get-graph-path graph))
(let [conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph))
_ (cli-util/ensure-db-graph-for-command @conn)
nodes (->> (d/datoms @conn :aevt :block/title)
(filter (fn [datom]
(string/includes? (:v datom) search-term)))
@@ -61,7 +64,7 @@
(format-results nodes search-term {:raw raw}))
(cli-util/error "Graph" (pr-str graph) "does not exist")))
(defn search [{{:keys [graph search-terms api-server-token]} :opts :as m}]
(if api-server-token
(api-search (string/join " " (into [graph] search-terms)) m)
(defn search [{{:keys [search-terms] :as opts} :opts :as m}]
(if (cli-util/api-command? opts)
(api-search (string/join " " search-terms) m)
(local-search (string/join " " search-terms) m)))

53
deps/cli/src/logseq/cli/commands/validate.cljs vendored Normal file
View File

@@ -0,0 +1,53 @@
(ns logseq.cli.commands.validate
"Validate graph command"
(:require ["fs" :as fs]
[cljs.pprint :as pprint]
[datascript.core :as d]
[logseq.cli.util :as cli-util]
[logseq.db.common.sqlite-cli :as sqlite-cli]
[logseq.db.frontend.malli-schema :as db-malli-schema]
[logseq.db.frontend.validate :as db-validate]
[malli.error :as me]))
(defn- validate-db*
"Validate datascript db as a vec of entity maps"
[db ent-maps* {:keys [closed]}]
(let [ent-maps (db-malli-schema/update-properties-in-ents db ent-maps*)
explainer (db-validate/get-schema-explainer closed)]
(if-let [explanation (binding [db-malli-schema/*db-for-validate-fns* db
db-malli-schema/*closed-values-validate?* true]
(->> (map (fn [e] (dissoc e :db/id)) ent-maps) explainer not-empty))]
(let [ent-errors
(->> (db-validate/group-errors-by-entity db ent-maps (:errors explanation))
(map #(update % :errors
(fn [errs]
;; errs looks like: {178 {:logseq.property/hide? ["disallowed key"]}}
;; map is indexed by :in which is unused since all errors are for the same map
(->> (me/humanize {:errors errs})
vals
(apply merge-with into))))))]
(println "Found" (count ent-errors)
(if (= 1 (count ent-errors)) "entity" "entities")
"with errors:")
(pprint/pprint ent-errors)
(js/process.exit 1))
(println "Valid!"))))
(defn- validate-db [db db-name options]
(let [datoms (d/datoms db :eavt)
ent-maps (db-malli-schema/datoms->entities datoms)]
(println "Read graph" (str db-name " with counts: "
(pr-str (assoc (db-validate/graph-counts db ent-maps)
:datoms (count datoms)))))
(validate-db* db ent-maps options)))
(defn- validate-graph [graph options]
(if (fs/existsSync (cli-util/get-graph-path graph))
(let [conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph))
_ (cli-util/ensure-db-graph-for-command @conn)]
(validate-db @conn graph options))
(cli-util/error "Graph" (pr-str graph) "does not exist")))
(defn validate [{{:keys [graphs] :as opts} :opts}]
(doseq [graph graphs]
(validate-graph graph opts)))

35
deps/cli/src/logseq/cli/spec.cljs vendored
View File

@@ -3,11 +3,17 @@
commands but are separate because command namespaces are lazy loaded")
(def export
{:file {:alias :f
{:graph {:alias :g
:desc "Local graph to export"}
:file {:alias :f
:desc "File to save export"}})
(def export-edn
{:include-timestamps? {:alias :T
{:api-server-token {:alias :a
:desc "API server token to export current graph"}
:graph {:alias :g
:desc "Local graph to export"}
:include-timestamps? {:alias :T
:desc "Include timestamps in export"}
:file {:alias :f
:desc "File to save export"}
@@ -27,7 +33,7 @@
(def import-edn
{:api-server-token {:alias :a
:desc "API server token to query current graph"}
:desc "API server token to import into current graph"}
:graph {:alias :g
:desc "Local graph to import into"}
:file {:alias :f
@@ -35,20 +41,22 @@
:desc "EDN File to import"}})
(def query
{:graphs {:alias :g
{:api-server-token {:alias :a
:desc "API server token to query current graph"}
:graphs {:alias :g
:coerce []
:desc "Additional graphs to local query"}
:desc "Local graph(s) to query"}
:properties-readable {:alias :p
:coerce :boolean
:desc "Make properties on local, entity queries show property values instead of ids"}
:title-query {:alias :t
:desc "Invoke local query on :block/title"}
:api-server-token {:alias :a
:desc "API server token to query current graph"}})
:desc "Invoke local query on :block/title"}})
(def search
{:api-server-token {:alias :a
:desc "API server token to search current graph"}
:graph {:alias :g
:desc "Local graph to search"}
:raw {:alias :r
:desc "Print raw response"}
:limit {:alias :l
@@ -74,4 +82,13 @@
:desc "Host for streamable HTTP server"}
:debug-tool {:alias :t
:coerce :keyword
:desc "Debug mcp tool with direct invocation"}})
:desc "Debug mcp tool with direct invocation"}})
(def validate
{:graphs {:alias :g
:coerce []
:require true
:desc "Local graph(s) to validate"}
:closed {:alias :c
:default true
:desc "Validate entities have no extra keys"}})

16
deps/cli/src/logseq/cli/util.cljs vendored
View File

@@ -4,6 +4,7 @@
["path" :as node-path]
[clojure.string :as string]
[logseq.cli.common.graph :as cli-common-graph]
[logseq.db.common.entity-plus :as entity-plus]
[logseq.db.common.sqlite :as common-sqlite]
[nbb.error]
[promesa.core :as p]))
@@ -79,4 +80,17 @@
"class" "classes"} word (str word "s"))))]
(str (count (:properties edn-map)) " " (pluralize "property" (count (:properties edn-map))) ", "
(count (:classes edn-map)) " " (pluralize "class" (count (:classes edn-map))) " and "
(count (:pages-and-blocks edn-map)) " " (pluralize "page" (count (:pages-and-blocks edn-map))))))
(count (:pages-and-blocks edn-map)) " " (pluralize "page" (count (:pages-and-blocks edn-map))))))
(defn ensure-db-graph-for-command
[db]
(when-not (entity-plus/db-based-graph? db)
(error "This command must be called on a DB graph")))
(defn api-command?
"Given user options and $LOGSEQ_API_SERVER_TOKEN, determines if
given command is an api (true) or local (false) command"
[{:keys [graph graphs api-server-token]}]
(or api-server-token
;; graph(s) check overrides env since it is more explicit
(and js/process.env.LOGSEQ_API_SERVER_TOKEN (not graph) (not graphs))))

10
deps/common/src/logseq/common/config.cljs vendored
View File

@@ -41,7 +41,7 @@
(defonce library-page-name "Library")
(defonce quick-add-page-name "Quick add")
(defn local-asset?
(defn local-relative-asset?
[s]
(and (string? s)
(re-find (re-pattern (str "^[./]*" local-assets-dir)) s)))
@@ -51,6 +51,14 @@
(when (string? s)
(string/starts-with? s asset-protocol)))
(defn protocol-path?
[s]
(try
(let [url (js/URL. s)]
(some? (.-protocol url)))
(catch :default _
false)))
(defn remove-asset-protocol
[s]
(if (local-protocol-asset? s)

6
deps/db/script/validate_db.cljs vendored
View File

@@ -26,8 +26,10 @@
verbose
(pprint/pprint ent-errors)
humanize
(pprint/pprint (map #(-> (dissoc % :errors-by-type)
(update :errors (fn [errs] (me/humanize {:errors errs}))))
(pprint/pprint (map #(update % :errors (fn [errs]
(->> (me/humanize {:errors errs})
vals
(apply merge-with into))))
ent-errors))
:else
(pprint/pprint (map :entity ent-errors))))

9
deps/db/src/logseq/db.cljs vendored
View File

@@ -159,7 +159,9 @@
(remove-temp-block-data)
(remove (fn [m] (and (map? m) (= (:db/ident m) :block/path-refs))))
(common-util/fast-remove-nils)
(remove empty?))
(remove (fn [m] (or ;; db/id
(integer? m)
(empty? m)))))
delete-blocks-tx (when-not (string? repo-or-conn)
(delete-blocks/update-refs-history-and-macros @repo-or-conn tx-data tx-meta))
tx-data (distinct (concat tx-data delete-blocks-tx))]
@@ -552,6 +554,7 @@
(defn get-key-value
[db key-ident]
(assert (= "logseq.kv" (namespace key-ident)) key-ident)
(:kv/value (d/entity db key-ident)))
(def kv sqlite-util/kv)
@@ -570,6 +573,10 @@
[db]
(when db (get-key-value db :logseq.kv/remote-schema-version)))
(defn get-graph-rtc-e2ee?
[db]
(when db (get-key-value db :logseq.kv/graph-rtc-e2ee?)))
(def get-all-properties db-db/get-all-properties)
(def get-class-extends db-class/get-class-extends)
(def get-classes-parents db-db/get-classes-parents)

1
deps/db/src/logseq/db/common/initial_data.cljs vendored
View File

@@ -339,6 +339,7 @@
[:logseq.kv/db-type
:logseq.kv/schema-version
:logseq.kv/graph-uuid
:logseq.kv/graph-rtc-e2ee?
:logseq.kv/latest-code-lang
:logseq.kv/graph-backup-folder
:logseq.kv/graph-text-embedding-model-name

3
deps/db/src/logseq/db/frontend/kv_entity.cljs vendored
View File

@@ -36,4 +36,5 @@ RTC won't start when major-schema-versions don't match"
:logseq.kv/graph-text-embedding-model-name {:doc "Graph's text-embedding model name"
:rtc {:rtc/ignore-entity-when-init-upload true
:rtc/ignore-entity-when-init-download true}})))
:rtc/ignore-entity-when-init-download true}}
:logseq.kv/graph-rtc-e2ee? {:doc "true if it's a rtc graph with E2EE enabled"})))

18
deps/db/src/logseq/db/frontend/malli_schema.cljs vendored
View File

@@ -89,15 +89,16 @@
expected to be a coll if the property has a :many cardinality. validate-fn is
a fn that is called directly on each value to return a truthy value.
validate-fn varies by property type"
[db validate-fn [property property-val] & {:keys [new-closed-value? _skip-strict-url-validate?]
:as validate-option}]
[db validate-fn [property property-val] & {:keys [new-closed-value? :closed-values-validate? _skip-strict-url-validate?]
:as validate-options}]
;; For debugging
;; (when (not (internal-ident? (:db/ident property))) (prn :validate-val (dissoc property :property/closed-values) property-val))
(let [validate-fn' (if (db-property-type/property-types-with-db (:logseq.property/type property))
(fn [value]
(validate-fn db value validate-option))
(validate-fn db value validate-options))
validate-fn)
validate-fn'' (if (and (db-property-type/closed-value-property-types (:logseq.property/type property))
validate-fn'' (if (and closed-values-validate?
(db-property-type/closed-value-property-types (:logseq.property/type property))
;; new closed values aren't associated with the property yet
(not new-closed-value?)
(seq (:property/closed-values property)))
@@ -226,6 +227,12 @@
"`true` allows updating a block's other property when it has invalid URL value"
false)
(def ^:dynamic *closed-values-validate?*
"By default this is false because we can't ensure this when merging updates from server.
`true` allows for non RTC graphs to have higher data quality and avoid
possible UX bugs related to closed values."
false)
(def property-tuple
"A tuple of a property map and a property value"
(into
@@ -241,7 +248,8 @@
{:error/message error-message})
(fn [tuple]
(validate-property-value *db-for-validate-fns* schema-fn tuple
{:skip-strict-url-validate? *skip-strict-url-validate?*}))])])
{:skip-strict-url-validate? *skip-strict-url-validate?*
:closed-values-validate? *closed-values-validate?*}))])])
db-property-type/built-in-validation-schemas)))
(def block-properties

20
deps/db/src/logseq/db/frontend/validate.cljs vendored
View File

@@ -77,22 +77,7 @@
(fn [id] (select-keys (d/entity db id)
[:block/name :block/tags :db/id :block/created-at]))))
:dispatch-key (->> (dissoc ent :db/id) (db-malli-schema/entity-dispatch-key db))
:errors errors'
;; Group by type to reduce verbosity
;; TODO: Move/remove this to another fn if unused
:errors-by-type
(->> (group-by :type errors')
(map (fn [[type' type-errors]]
[type'
{:in-value-distinct (->> type-errors
(map #(select-keys % [:in :value]))
distinct
vec)
:schema-distinct (->> (map :schema type-errors)
(map m/form)
distinct
vec)}]))
(into {}))})))))
:errors errors'})))))
(defn validate-db!
"Validates all the entities of the given db using :eavt datoms. Returns a map
@@ -112,8 +97,7 @@
(cond-> {:datom-count (count datoms)
:entities ent-maps*}
(some? errors)
(assoc :errors (map #(-> (dissoc % :errors-by-type)
(update :errors (fn [errs] (me/humanize {:errors errs}))))
(assoc :errors (map #(update % :errors (fn [errs] (me/humanize {:errors errs})))
(group-errors-by-entity db ent-maps errors))))))
(defn graph-counts

3
deps/db/src/logseq/db/sqlite/export.cljs vendored
View File

@@ -878,7 +878,8 @@
:graph-ontology
(build-graph-ontology-export db {})
:graph
(build-graph-export db (:graph-options options)))
(build-graph-export db (:graph-options options))
(throw (ex-info (str (pr-str export-type) " is an invalid export-type") {})))
export-map (patch-invalid-keywords export-map*)]
(if (get-in options [:graph-options :catch-validation-errors?])
(try

11
deps/db/src/logseq/db/sqlite/util.cljs vendored
View File

@@ -137,9 +137,10 @@
;; Timestamp is useful as this can occur much later than :logseq.kv/graph-created-at
(kv :logseq.kv/imported-at (common-util/time-ms))]
(mapv
;; Don't import some RTC related entities
(fn [db-ident] [:db/retractEntity db-ident])
[:logseq.kv/graph-uuid
:logseq.kv/graph-local-tx
:logseq.kv/remote-schema-version
:logseq.kv/graph-text-embedding-model-name])))
[:logseq.kv/graph-uuid ;rtc related
:logseq.kv/graph-local-tx ;rtc related
:logseq.kv/remote-schema-version ;rtc related
:logseq.kv/graph-rtc-e2ee? ;rtc related
:logseq.kv/graph-text-embedding-model-name ;embedding
])))

77
deps/graph-parser/src/logseq/graph_parser/block.cljs vendored
View File

@@ -52,7 +52,7 @@
(and
(= url-type "Page_ref")
(and (string? value)
(not (or (common-config/local-asset? value)
(not (or (common-config/local-relative-asset? value)
(common-config/draw? value))))
value)
@@ -63,7 +63,7 @@
(and (= url-type "Search")
(= format :org)
(not (common-config/local-asset? value))
(not (common-config/local-relative-asset? value))
value)
(and
@@ -316,7 +316,9 @@
(text/namespace-page? original-page-name'))
page-entity (when (and db (not skip-existing-page-check?))
(if class?
(ldb/get-case-page db original-page-name')
(some->> (ldb/page-exists? db original-page-name' #{:logseq.class/Tag})
first
(d/entity db))
(ldb/get-page db original-page-name')))
original-page-name' (or from-page (:block/title page-entity) original-page-name')
page (merge
@@ -410,26 +412,30 @@
(not (common-date/valid-journal-title-with-slash? page))))
(defn- ref->map
[db *col {:keys [date-formatter db-based? *name->id tag?]}]
(let [col (remove string/blank? @*col)
children-pages (when-not db-based?
(->> (mapcat (fn [p]
(let [p (if (map? p)
(:block/title p)
p)]
(when (string? p)
(let [p (or (text/get-nested-page-name p) p)]
(when (text/namespace-page? p)
(common-util/split-namespace-pages p))))))
col)
(remove string/blank?)
(distinct)))
[db *col {:keys [date-formatter *name->id tag? db-based? structured-tags]}]
(let [col (distinct (remove string/blank? @*col))
children-pages (->> (mapcat (fn [p]
(let [p (if (map? p)
(:block/title p)
p)]
(when (string? p)
(let [p (or (text/get-nested-page-name p) p)]
(if (and (text/namespace-page? p) (not tag?))
(common-util/split-namespace-pages p)
[p])))))
col)
(remove string/blank?)
(distinct))
col (->> (distinct (concat col children-pages))
(remove nil?))]
(remove nil?))
export-to-db-graph? @*export-to-db-graph?]
(map
(fn [item]
(let [macro? (and (map? item)
(= "macro" (:type item)))]
(= "macro" (:type item)))
tag? (if export-to-db-graph?
tag?
(or (contains? structured-tags item) tag?))]
(when-not macro?
(let [m (page-name->map item db true date-formatter {:class? tag?})
result (cond->> m
@@ -441,13 +447,16 @@
(swap! *name->id assoc page-name (:block/uuid result)))
;; Changing a :block/uuid should be done cautiously here as it can break
;; the identity of built-in concepts in db graphs
(if id
(if (and id
(or (when-let [ident (:db/ident result)]
(nil? (d/entity db ident)))
export-to-db-graph?))
(assoc result :block/uuid id)
result))))) col)))
(defn- with-page-refs-and-tags
[{:keys [title body tags refs marker priority] :as block} db date-formatter parse-block]
(let [db-based? (and (ldb/db-based-graph? db) (not *export-to-db-graph?))
[{:keys [title body tags refs marker priority] :as block} db date-formatter]
(let [db-based? (and (ldb/db-based-graph? db) (not @*export-to-db-graph?))
refs (->> (concat tags refs (when-not db-based? [marker priority]))
(remove string/blank?)
(distinct))
@@ -476,18 +485,14 @@
(let [*name->id (atom {})
ref->map-options {:db-based? db-based?
:date-formatter date-formatter
:*name->id *name->id}
:*name->id *name->id
:structured-tags (set @*structured-tags)}
refs (->> (ref->map db *refs ref->map-options)
(remove nil?)
(map (fn [ref]
(let [ref' (if-let [entity (ldb/get-case-page db (:block/title ref))]
(if (= (:db/id parse-block) (:db/id entity))
ref
(select-keys entity [:block/uuid :block/title :block/name]))
ref)]
(cond-> ref'
(:block.temp/original-page-name ref)
(assoc :block.temp/original-page-name (:block.temp/original-page-name ref)))))))
(cond-> ref
(:block.temp/original-page-name ref)
(assoc :block.temp/original-page-name (:block.temp/original-page-name ref))))))
tags (ref->map db *structured-tags (assoc ref->map-options :tag? true))]
(assoc block
:refs refs
@@ -551,9 +556,9 @@
(map (fn [page] (page-name->map page db true date-formatter)) page-refs)))
(defn- with-page-block-refs
[block db date-formatter & {:keys [parse-block]}]
[block db date-formatter]
(some-> block
(with-page-refs-and-tags db date-formatter parse-block)
(with-page-refs-and-tags db date-formatter)
with-block-refs
(update :refs (fn [col] (remove nil? col)))))
@@ -625,7 +630,7 @@
properties))
(defn- construct-block
[block properties timestamps body encoded-content format pos-meta {:keys [block-pattern db date-formatter parse-block remove-properties? db-graph-mode? export-to-db-graph?]}]
[block properties timestamps body encoded-content format pos-meta {:keys [block-pattern db date-formatter remove-properties? db-graph-mode? export-to-db-graph?]}]
(let [id (get-custom-id-or-new-id properties)
ref-pages-in-properties (->> (:page-refs properties)
(remove string/blank?))
@@ -666,7 +671,7 @@
db-based? (or db-graph-mode? export-to-db-graph?)
block (-> block
(assoc :body body)
(with-page-block-refs db date-formatter {:parse-block parse-block}))
(with-page-block-refs db date-formatter))
block (if db-based? block
(-> block
(update :tags (fn [tags] (map #(assoc % :block/format format) tags)))
@@ -714,7 +719,7 @@
* `ast`: mldoc ast.
* `content`: markdown or org-mode text.
* `format`: content's format, it could be either :markdown or :org-mode.
* `options`: Options are :user-config, :block-pattern, :parse-block, :date-formatter, :db and
* `options`: Options are :user-config, :block-pattern, :date-formatter, :db and
* :db-graph-mode? : Set when a db graph in the frontend
* :export-to-db-graph? : Set when exporting to a db graph"
[ast content format {:keys [user-config db-graph-mode? export-to-db-graph?] :as options}]

2
deps/graph-parser/src/logseq/graph_parser/exporter.cljs vendored
View File

@@ -900,7 +900,7 @@
(cond
(and (vector? x)
(= "Link" (first x))
(common-config/local-asset? (second (:url (second x)))))
(common-config/local-relative-asset? (second (:url (second x)))))
(swap! results update :asset-links conj x)
(and (vector? x)
(= "Macro" (first x))

38
deps/graph-parser/src/logseq/graph_parser/mldoc.cljc vendored
View File

@@ -8,15 +8,15 @@
:default ["mldoc" :refer [Mldoc]])
#?(:org.babashka/nbb [logseq.common.log :as log]
:default [lambdaisland.glogi :as log])
[goog.object :as gobj]
[cljs-bean.core :as bean]
[logseq.graph-parser.utf8 :as utf8]
[clojure.string :as string]
[logseq.common.util :as common-util]
[logseq.common.config :as common-config]
#_:clj-kondo/ignore
[cljs-bean.core :as bean]
[clojure.string :as string]
[goog.object :as gobj]
[logseq.common.config :as common-config]
[logseq.common.util :as common-util]
[logseq.db.sqlite.util :as sqlite-util]
[logseq.graph-parser.schema.mldoc :as mldoc-schema]
[logseq.db.sqlite.util :as sqlite-util]))
[logseq.graph-parser.utf8 :as utf8]))
(defonce parseJson (gobj/get Mldoc "parseJson"))
(defonce parseInlineJson (gobj/get Mldoc "parseInlineJson"))
@@ -103,7 +103,7 @@
(common-util/safe-subs line level)
;; Otherwise, trim these invalid spaces
(string/triml line)))
(if remove-first-line? lines r))
(if remove-first-line? lines r))
content (if remove-first-line? body (cons f body))]
(string/join "\n" content)))
@@ -111,16 +111,16 @@
[ast content]
(let [content (utf8/encode content)]
(map (fn [[block pos-meta]]
(if (and (vector? block)
(= "Src" (first block)))
(let [{:keys [start_pos end_pos]} pos-meta
content (utf8/substring content start_pos end_pos)
spaces (re-find #"^[\t ]+" (first (string/split-lines content)))
content (if spaces (remove-indentation-spaces content (count spaces) true)
content)
block ["Src" (assoc (second block) :full_content content)]]
[block pos-meta])
[block pos-meta])) ast)))
(if (and (vector? block)
(= "Src" (first block)))
(let [{:keys [start_pos end_pos]} pos-meta
content (utf8/substring content start_pos end_pos)
spaces (re-find #"^[\t ]+" (first (string/split-lines content)))
content (if spaces (remove-indentation-spaces content (count spaces) true)
content)
block ["Src" (assoc (second block) :full_content content)]]
[block pos-meta])
[block pos-meta])) ast)))
(defn collect-page-properties
[ast config]
@@ -196,7 +196,7 @@
(common-config/draw? ref-value)
;; 3. local asset link
(boolean (common-config/local-asset? ref-value))))))))
(boolean (common-config/local-relative-asset? ref-value))))))))
(defn link?
[format link]

2
deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs vendored
View File

@@ -231,7 +231,7 @@
#_(map #(select-keys % [:block/title :block/tags]))
count))
"Correct number of pages with block content")
(is (= 13 (->> @conn
(is (= 12 (->> @conn
(d/q '[:find [?ident ...]
:where [?b :block/tags :logseq.class/Tag] [?b :db/ident ?ident] (not [?b :logseq.property/built-in?])])
count))

4
deps/outliner/src/logseq/outliner/core.cljs vendored
View File

@@ -207,7 +207,9 @@
db-graph?
;; Remove tags changing case with `Escape`
((fn [tags']
(let [ref-titles (set (map :block/title (:block/refs m)))
(let [ref-titles (->> (map :block/title (:block/refs m))
(remove nil?)
set)
lc-ref-titles (set (map string/lower-case ref-titles))]
(remove (fn [tag]
(when-let [title (:block/title tag)]

4
deps/outliner/src/logseq/outliner/property.cljs vendored
View File

@@ -163,7 +163,9 @@
(let [new-type (:logseq.property/type schema)
cardinality (:db/cardinality schema)
ident (:db/ident property)
cardinality (if (= cardinality :many) :db.cardinality/many :db.cardinality/one)
cardinality (if (#{:many :db.cardinality/many} cardinality)
:db.cardinality/many
:db.cardinality/one)
old-type (:logseq.property/type property)
old-ref-type? (db-property-type/user-ref-property-types old-type)
ref-type? (db-property-type/user-ref-property-types new-type)]

8
deps/shui/src/logseq/shui/dialog/core.cljs vendored
View File

@@ -53,14 +53,16 @@
(filter #(= id (:id (second %)))) (first))))
(defn update-modal!
[id ks val]
[id ks val & {:keys [closing?]}]
(when-let [[index config] (get-modal id)]
(let [ks (if (coll? ks) ks [ks])
config (if (nil? val)
(medley/dissoc-in config ks)
(assoc-in config ks val))]
(swap! *modals assoc index config)
(when (and (false? (:open? config)) (fn? (:on-close config)))
(when (and (false? (:open? config))
(fn? (:on-close config))
(not closing?))
((:on-close config) id)))))
(defn upsert-modal!
@@ -115,7 +117,7 @@
(defn close!
([] (close! (get-last-modal-id)))
([id] (update-modal! id :open? false)))
([id] (update-modal! id :open? false {:closing? true})))
(defn close-all! []
(doseq [{:keys [id]} @*modals]

24
deps/shui/src/logseq/shui/form/password.cljs vendored Normal file
View File

@@ -0,0 +1,24 @@
(ns logseq.shui.form.password
(:require [clojure.string :as string]
[logseq.shui.base.core :as base-core]
[logseq.shui.form.core :as form-core]
[logseq.shui.hooks :as hooks]
[logseq.shui.icon.v2 :as icon-v2]
[rum.core :as rum]))
(rum/defc toggle-password
[option]
(let [[visible? set-visible!] (hooks/use-state false)]
[:div.ls-toggle-password-input.relative
(form-core/input
(merge
option
{:type (if visible? "text" "password")}))
(when-not (string/blank? (:value option))
(base-core/button
{:variant :ghost
:class "absolute right-1"
:style {:top 6}
:size :sm
:on-click #(set-visible! (not visible?))}
(icon-v2/root (if visible? "eye-off" "eye"))))]))

25
deps/shui/src/logseq/shui/hooks.cljs vendored
View File

@@ -132,3 +132,28 @@
:onMouseUp #(clear % true)
:onMouseLeave #(clear % false)
:onTouchEnd #(clear % true)}))
(defn- use-atom-fn
[a getter-fn setter-fn]
(let [[val set-val] (use-state (getter-fn @a))]
(use-effect!
(fn []
(let [id (str (random-uuid))]
(add-watch a id (fn [_ _ prev-state next-state]
(let [prev-value (getter-fn prev-state)
next-value (getter-fn next-state)]
(when-not (= prev-value next-value)
(set-val next-value)))))
#(remove-watch a id)))
[])
[val #(swap! a setter-fn %)]))
(defn use-atom
"(use-atom my-atom)"
[a]
(use-atom-fn a identity (fn [_ v] v)))
(defn use-atom-in
[a ks]
(let [ks (if (keyword? ks) [ks] ks)]
(use-atom-fn a #(get-in % ks) (fn [a' v] (assoc-in a' ks v)))))

3
deps/shui/src/logseq/shui/ui.cljs vendored
View File

@@ -2,6 +2,7 @@
(:require [logseq.shui.base.core :as base-core]
[logseq.shui.dialog.core :as dialog-core]
[logseq.shui.form.core :as form-core]
[logseq.shui.form.password :as form-password]
[logseq.shui.icon.v2 :as icon-v2]
[logseq.shui.popup.core :as popup-core]
[logseq.shui.select.core :as select-core]
@@ -153,3 +154,5 @@
(def table-cell table-core/table-cell)
(def table-actions table-core/table-actions)
(def table-get-selection-rows table-core/get-selection-rows)
(def toggle-password form-password/toggle-password)

1
ios/App/Podfile
View File

@@ -11,6 +11,7 @@ install! 'cocoapods', :disable_input_output_paths => true
def capacitor_pods
pod 'Capacitor', :path => '../../node_modules/@capacitor/ios'
pod 'CapacitorCordova', :path => '../../node_modules/@capacitor/ios'
pod 'AparajitaCapacitorSecureStorage', :path => '../../node_modules/@aparajita/capacitor-secure-storage'
pod 'CapacitorCommunitySafeArea', :path => '../../node_modules/@capacitor-community/safe-area'
pod 'CapacitorActionSheet', :path => '../../node_modules/@capacitor/action-sheet'
pod 'CapacitorApp', :path => '../../node_modules/@capacitor/app'

15
ios/App/Podfile.lock
View File

@@ -1,4 +1,7 @@
PODS:
- AparajitaCapacitorSecureStorage (7.1.6):
- Capacitor
- KeychainSwift (~> 21.0)
- Capacitor (7.2.0):
- CapacitorCordova
- CapacitorActionSheet (7.0.1):
@@ -32,10 +35,12 @@ PODS:
- Capacitor
- JcesarmobileSslSkip (0.4.0):
- Capacitor
- KeychainSwift (21.0.0)
- SendIntent (7.0.0):
- Capacitor
DEPENDENCIES:
- "AparajitaCapacitorSecureStorage (from `../../node_modules/@aparajita/capacitor-secure-storage`)"
- "Capacitor (from `../../node_modules/@capacitor/ios`)"
- "CapacitorActionSheet (from `../../node_modules/@capacitor/action-sheet`)"
- "CapacitorApp (from `../../node_modules/@capacitor/app`)"
@@ -55,7 +60,13 @@ DEPENDENCIES:
- "JcesarmobileSslSkip (from `../../node_modules/@jcesarmobile/ssl-skip`)"
- SendIntent (from `../../node_modules/send-intent`)
SPEC REPOS:
trunk:
- KeychainSwift
EXTERNAL SOURCES:
AparajitaCapacitorSecureStorage:
:path: "../../node_modules/@aparajita/capacitor-secure-storage"
Capacitor:
:path: "../../node_modules/@capacitor/ios"
CapacitorActionSheet:
@@ -94,6 +105,7 @@ EXTERNAL SOURCES:
:path: "../../node_modules/send-intent"
SPEC CHECKSUMS:
AparajitaCapacitorSecureStorage: 502bff73187cf9d0164459458ccf47ec65d5895a
Capacitor: 03bc7cbdde6a629a8b910a9d7d78c3cc7ed09ea7
CapacitorActionSheet: 4213427449132ae4135674d93010cb011725647e
CapacitorApp: febecbb9582cb353aed037e18ec765141f880fe9
@@ -111,8 +123,9 @@ SPEC CHECKSUMS:
CapacitorStatusBar: 6e7af040d8fc4dd655999819625cae9c2d74c36f
CapgoCapacitorNavigationBar: 067b1c1d1ede5ce96200a730ce7fd498e9641509
JcesarmobileSslSkip: 5fa98636a64c36faa50f32ab4daf34e38f4d45b9
KeychainSwift: 4a71a45c802fd9e73906457c2dcbdbdc06c9419d
SendIntent: 8a6f646a4489f788d253ffbd1082a98ea388d870
PODFILE CHECKSUM: 00fbb7ba3788966b68cb8a5a6f2abc380d7b7b9a
PODFILE CHECKSUM: bf3859ae3f2ef96dbee7c801e4be9d91c6e68077
COCOAPODS: 1.16.2

1
package.json
View File

@@ -108,6 +108,7 @@
"postinstall": "yarn tldraw:build && yarn ui:build"
},
"dependencies": {
"@aparajita/capacitor-secure-storage": "^7.1.6",
"@capacitor-community/safe-area": "7.0.0-alpha.1",
"@capacitor/action-sheet": "7.0.1",
"@capacitor/android": "7.2.0",

6
prompts/review.md
View File

@@ -23,3 +23,9 @@ You're Clojure(script) expert, you're responsible to check those common errors:
- e.g. `["65.9" {:properties [:logseq.property.embedding/hnsw-label-updated-at]}]`
- If common keywords are added or modified, make corresponding changes in their definitions.
- common keywords are defined by `logseq.common.defkeywords/defkeywords`
- A function that returns a promise, and its function name starts with "<".
- Prohibit converting js/Uint8Array to vector. e.g. `(vec uint8-array)`
- This operation is very slow when the Uint8Array is large (e.g. an asset).

3
resources/package.json
View File

@@ -46,7 +46,8 @@
"semver": "7.5.2",
"socks-proxy-agent": "8.0.2",
"update-electron-app": "2.0.1",
"zod": "^4.1.5"
"zod": "^4.1.5",
"keytar": "^7.9.0"
},
"devDependencies": {
"@electron-forge/cli": "^7.8.3",

13
src/electron/electron/handler.cljs
View File

@@ -22,6 +22,7 @@
[electron.fs-watcher :as watcher]
[electron.git :as git]
[electron.handler-interface :refer [handle]]
[electron.keychain :as keychain]
[electron.logger :as logger]
[electron.plugin :as plugin]
[electron.server :as server]
@@ -98,6 +99,9 @@
(defmethod handle :readFile [_window [_ path]]
(utils/read-file path))
(defmethod handle :readFileRaw [_window [_ path]]
(utils/read-file-raw path))
(defn writable?
[path]
(assert (string? path))
@@ -614,6 +618,15 @@
(defmethod handle :cancel-all-requests [_ args]
(apply rsapi/cancel-all-requests (rest args)))
(defmethod handle :keychain/save-e2ee-password [_window [_ key encrypted-text]]
(keychain/<set-password! key encrypted-text))
(defmethod handle :keychain/get-e2ee-password [_window [_ key]]
(keychain/<get-password key))
(defmethod handle :keychain/delete-e2ee-password [_window [_ key]]
(keychain/<delete-password! key))
(defmethod handle :default [args]
(logger/error "Error: no ipc handler for:" args))

55
src/electron/electron/keychain.cljs Normal file
View File

@@ -0,0 +1,55 @@
(ns electron.keychain
"Helper functions for storing E2EE secrets inside the OS keychain."
(:require ["electron" :refer [app]]
["keytar" :as keytar]
[clojure.string :as string]
[electron.logger :as logger]
[promesa.core :as p]))
(defonce ^:private service-name
(delay
(let [app-name (try (.getName app)
(catch :default _ nil))]
(if (string/blank? app-name)
"Logseq"
app-name))))
(defn- keychain-service
[]
(str (force service-name) " E2EE"))
(defn supported?
[]
(boolean keytar))
(defn <set-password!
"Persist `encrypted-text` for the `refresh-token` entry."
[key encrypted-text]
(if-let [account (and (supported?) key)]
(-> (p/let [_ (.setPassword keytar (keychain-service) account encrypted-text)]
true)
(p/catch (fn [e]
(logger/error ::set-password {:error e})
(throw e))))
(p/resolved false)))
(defn <get-password
"Fetch encrypted text stored for `refresh-token`."
[key]
(if-let [account (and (supported?) key)]
(-> (p/let [password (.getPassword keytar (keychain-service) account)]
password)
(p/catch (fn [e]
(logger/error ::get-password {:error e})
(throw e))))
(p/resolved nil)))
(defn <delete-password!
[key]
(if-let [account (and (supported?) key)]
(-> (p/let [_ (.deletePassword keytar (keychain-service) account)]
true)
(p/catch (fn [e]
(logger/error ::delete-password {:error e})
(throw e))))
(p/resolved false)))

6
src/electron/electron/utils.cljs
View File

@@ -7,8 +7,8 @@
[clojure.string :as string]
[electron.configs :as cfgs]
[electron.logger :as logger]
[logseq.db.sqlite.util :as sqlite-util]
[logseq.cli.common.graph :as cli-common-graph]
[logseq.db.sqlite.util :as sqlite-util]
[promesa.core :as p]))
(defonce *win (atom nil)) ;; The main window
@@ -214,6 +214,10 @@
(let [ext (string/lower-case (node-path/extname path))]
(contains? #{".md" ".markdown" ".org" ".js" ".edn" ".css"} ext)))
(defn read-file-raw
[path]
(fs/readFileSync path))
(defn read-file
[path]
(try

346
src/main/frontend/common/crypt.cljs Normal file
View File

@@ -0,0 +1,346 @@
(ns frontend.common.crypt
"crypto utils"
(:require [lambdaisland.glogi :as log]
[logseq.db :as ldb]
[promesa.core :as p]))
(defonce subtle (.. js/crypto -subtle))
(defn <export-aes-key
[aes-key]
(assert (instance? js/CryptoKey aes-key))
(p/let [exported (.exportKey subtle "raw" aes-key)]
(js/Uint8Array. exported)))
(defn <import-aes-key
[exported-aes-key]
(assert (instance? js/Uint8Array exported-aes-key))
(.importKey subtle
"raw"
exported-aes-key
"AES-GCM"
true
#js ["encrypt" "decrypt"]))
(defn <export-public-key
[public-key]
(assert (instance? js/CryptoKey public-key))
(p/let [exported (.exportKey subtle "spki" public-key)]
(js/Uint8Array. exported)))
(defn <import-public-key
[exported-public-key]
(assert (instance? js/Uint8Array exported-public-key))
(.importKey subtle "spki" exported-public-key
#js {:name "RSA-OAEP" :hash "SHA-256"}
true
#js ["encrypt"]))
(defn <export-private-key
[private-key]
(assert (instance? js/CryptoKey private-key))
(p/let [exported (.exportKey subtle "pkcs8" private-key)]
(js/Uint8Array. exported)))
(defn <import-private-key
[exported-private-key]
(assert (instance? js/Uint8Array exported-private-key))
(.importKey subtle "pkcs8" exported-private-key
#js {:name "RSA-OAEP" :hash "SHA-256"}
true
#js ["decrypt"]))
(comment
(->
(p/let [kp (<generate-rsa-key-pair)
public-key (:publicKey kp)
exported-public-key (<export-public-key public-key)
public-key* (<import-public-key exported-public-key)
exported-public-key2 (<export-public-key public-key*)]
(prn (= (vec exported-public-key) (vec exported-public-key2))))
(p/catch (fn [e] (prn :e e)))))
(defn <generate-rsa-key-pair
"Generates a new RSA public/private key pair.
Return
{:publicKey #object [CryptoKey [object CryptoKey]],
:privateKey #object [CryptoKey [object CryptoKey]]}"
[]
(p/let [r (.generateKey subtle
#js {:name "RSA-OAEP"
:modulusLength 4096
:publicExponent (js/Uint8Array. [1 0 1])
:hash "SHA-256"}
true
#js ["encrypt" "decrypt"])]
{:publicKey (.-publicKey r)
:privateKey (.-privateKey r)}))
(defn <generate-aes-key
"Generates a new AES-GCM-256 key."
[]
(.generateKey subtle
#js {:name "AES-GCM"
:length 256}
true
#js ["encrypt" "decrypt"]))
(defn <encrypt-private-key
"Encrypts a private key with a password."
[password private-key]
(assert (and (string? password) (instance? js/CryptoKey private-key)))
(p/let [salt (js/crypto.getRandomValues (js/Uint8Array. 16))
iv (js/crypto.getRandomValues (js/Uint8Array. 12))
password-key (.importKey subtle "raw"
(.encode (js/TextEncoder.) password)
"PBKDF2"
false
#js ["deriveKey"])
derived-key (.deriveKey subtle
#js {:name "PBKDF2"
:salt salt
:iterations 100000
:hash "SHA-256"}
password-key
#js {:name "AES-GCM" :length 256}
true
#js ["encrypt" "decrypt"])
exported-private-key (.exportKey subtle "pkcs8" private-key)
encrypted-private-key (.encrypt subtle
#js {:name "AES-GCM" :iv iv}
derived-key
exported-private-key)]
[salt iv (js/Uint8Array. encrypted-private-key)]))
(defn <decrypt-private-key
"Decrypts a private key with a password."
[password encrypted-key-data]
(assert (and (vector? encrypted-key-data) (= 3 (count encrypted-key-data))))
(->
(p/let [[salt-data iv-data encrypted-private-key-data] encrypted-key-data
salt (js/Uint8Array. salt-data)
iv (js/Uint8Array. iv-data)
encrypted-private-key (js/Uint8Array. encrypted-private-key-data)
password-key (.importKey subtle "raw"
(.encode (js/TextEncoder.) password)
"PBKDF2"
false
#js ["deriveKey"])
derived-key (.deriveKey subtle
#js {:name "PBKDF2"
:salt salt
:iterations 100000
:hash "SHA-256"}
password-key
#js {:name "AES-GCM" :length 256}
true
#js ["encrypt" "decrypt"])
decrypted-private-key-data (.decrypt subtle
#js {:name "AES-GCM" :iv iv}
derived-key
encrypted-private-key)
private-key (.importKey subtle "pkcs8"
decrypted-private-key-data
#js {:name "RSA-OAEP" :hash "SHA-256"}
true
#js ["decrypt"])]
private-key)
(p/catch (fn [e]
(log/error "decrypt-private-key" e)
(ex-info "decrypt-private-key" {} e)))))
(defn <encrypt-aes-key
"Encrypts an AES key with a public key."
[public-key aes-key]
(assert (and (instance? js/CryptoKey public-key)
(instance? js/CryptoKey aes-key)))
(p/let [exported-aes-key (<export-aes-key aes-key)
encrypted-aes-key (.encrypt subtle
#js {:name "RSA-OAEP"}
public-key
exported-aes-key)]
(js/Uint8Array. encrypted-aes-key)))
(defn <decrypt-aes-key
"Decrypts an AES key with a private key."
[private-key encrypted-aes-key-data]
(assert (and (instance? js/CryptoKey private-key)
(instance? js/Uint8Array encrypted-aes-key-data)))
(->
(p/let [encrypted-aes-key (js/Uint8Array. encrypted-aes-key-data)
decrypted-key-data (.decrypt subtle
#js {:name "RSA-OAEP"}
private-key
encrypted-aes-key)]
(.importKey subtle
"raw"
decrypted-key-data
"AES-GCM"
true
#js ["encrypt" "decrypt"]))
(p/catch (fn [e]
(log/error "decrypt-aes-key" e)
(ex-info "decrypt-aes-key" {} e)))))
(defn <encrypt-uint8array
[aes-key arr]
(assert (and (instance? js/CryptoKey aes-key) (instance? js/Uint8Array arr)))
(p/let [iv (js/crypto.getRandomValues (js/Uint8Array. 12))
encrypted-data (.encrypt subtle
#js {:name "AES-GCM" :iv iv}
aes-key
arr)]
[iv (js/Uint8Array. encrypted-data)]))
(defn <decrypt-uint8array
[aes-key encrypted-data-vector]
(->
(p/let [[iv-data encrypted-data] encrypted-data-vector
_ (assert (instance? js/Uint8Array encrypted-data))
iv (js/Uint8Array. iv-data)
decrypted-data (.decrypt subtle
#js {:name "AES-GCM" :iv iv}
aes-key
encrypted-data)]
(js/Uint8Array. decrypted-data))
(p/catch
(fn [e]
(log/error "decrypt-uint8array" e)
(ex-info "decrypt-uint8array" {} e)))))
(defn <encrypt-text
"Encrypts text with an AES key."
[aes-key text]
(assert (and (string? text) (instance? js/CryptoKey aes-key)))
(p/let [iv (js/crypto.getRandomValues (js/Uint8Array. 12))
encoded-text (.encode (js/TextEncoder.) text)
encrypted-data (.encrypt subtle
#js {:name "AES-GCM" :iv iv}
aes-key
encoded-text)]
[iv (js/Uint8Array. encrypted-data)]))
(defn <decrypt-text
"Decrypts text with an AES key."
[aes-key encrypted-text-data-vector]
(-> (p/let [[iv-data encrypted-data] encrypted-text-data-vector
iv (js/Uint8Array. iv-data)
encrypted-data (js/Uint8Array. encrypted-data)
decrypted-data (.decrypt subtle
#js {:name "AES-GCM" :iv iv}
aes-key
encrypted-data)
decoded-text (.decode (js/TextDecoder.) decrypted-data)]
decoded-text)
(p/catch
(fn [e]
(log/error "decrypt-text" e)
(ex-info "decrypt-text" {} e)))))
(defn <decrypt-text-if-encrypted
"return nil if not a encrypted-package"
[aes-key maybe-encrypted-package]
(when (and (vector? maybe-encrypted-package)
(<= 2 (count maybe-encrypted-package)))
(<decrypt-text aes-key maybe-encrypted-package)))
(defn <encrypt-map
[aes-key encrypt-attr-set m]
(assert (map? m))
(reduce
(fn [map-p encrypt-attr]
(p/let [m map-p]
(if-let [v (get m encrypt-attr)]
(p/let [v' (p/chain (<encrypt-text aes-key v) ldb/write-transit-str)]
(assoc m encrypt-attr v'))
m)))
(p/promise m) encrypt-attr-set))
(defn <encrypt-av-coll
"see also `rtc-schema/av-schema`"
[aes-key encrypt-attr-set av-coll]
(p/all
(mapv
(fn [[a v & others]]
(p/let [v' (if (and (contains? encrypt-attr-set a)
(string? v))
(p/chain (<encrypt-text aes-key v) ldb/write-transit-str)
v)]
(apply conj [a v'] others)))
av-coll)))
(defn <decrypt-map
[aes-key encrypt-attr-set m]
(assert (map? m))
(reduce
(fn [map-p encrypt-attr]
(p/let [m map-p]
(if-let [v (get m encrypt-attr)]
(if (string? v)
(->
(p/let [v' (<decrypt-text-if-encrypted aes-key (ldb/read-transit-str v))]
(if v'
(assoc m encrypt-attr v')
m))
(p/catch (fn [e] (ex-info "decrypt map" {:m m :decrypt-attr encrypt-attr} e))))
m)
m)))
(p/promise m) encrypt-attr-set))
(defn <encrypt-text-by-text-password
[text-password text]
(assert (and (string? text-password) (string? text)))
(p/let [salt (js/crypto.getRandomValues (js/Uint8Array. 16))
iv (js/crypto.getRandomValues (js/Uint8Array. 12))
password-key (.importKey subtle "raw"
(.encode (js/TextEncoder.) text-password)
"PBKDF2"
false
#js ["deriveKey"])
derived-key (.deriveKey subtle
#js {:name "PBKDF2"
:salt salt
:iterations 100000
:hash "SHA-256"}
password-key
#js {:name "AES-GCM" :length 256}
true
#js ["encrypt" "decrypt"])
encoded-text (.encode (js/TextEncoder.) text)
encrypted-text (.encrypt subtle
#js {:name "AES-GCM" :iv iv}
derived-key
encoded-text)]
[salt iv (js/Uint8Array. encrypted-text)]))
(defn <decrypt-text-by-text-password
[text-password encrypted-data-vector]
(assert (and (string? text-password) (vector? encrypted-data-vector)))
(->
(p/let [[salt-data iv-data encrypted-data] encrypted-data-vector
salt (js/Uint8Array. salt-data)
iv (js/Uint8Array. iv-data)
encrypted-data (js/Uint8Array. encrypted-data)
password-key (.importKey subtle "raw"
(.encode (js/TextEncoder.) text-password)
"PBKDF2"
false
#js ["deriveKey"])
derived-key (.deriveKey subtle
#js {:name "PBKDF2"
:salt salt
:iterations 100000
:hash "SHA-256"}
password-key
#js {:name "AES-GCM" :length 256}
true
#js ["encrypt" "decrypt"])
decrypted-data (.decrypt subtle
#js {:name "AES-GCM" :iv iv}
derived-key
encrypted-data)]
(.decode (js/TextDecoder.) decrypted-data))
(p/catch
(fn [e]
(log/error "decrypt-text-by-text-password" e)
(ex-info "decrypt-text-by-text-password" {} e)))))

44
src/main/frontend/common/file/opfs.cljs Normal file
View File

@@ -0,0 +1,44 @@
(ns frontend.common.file.opfs
"OPFS fs api"
(:require [promesa.core :as p]))
(defn <write-text!
"Write `text` to `filename` in Origin Private File System.
Returns a promise."
[filename text]
(p/let [;; OPFS root dir
root (.. js/navigator -storage (getDirectory))
;; get (or create) a file handle
file-handle (.getFileHandle root filename #js {:create true})
;; open a writable stream
writable (.createWritable file-handle)]
;; write string directly
(.write writable text)
;; always close!
(.close writable)))
(defn <read-text!
"Read text content from `filename` in Origin Private File System (OPFS).
Returns a promise that resolves to the file content string."
[filename]
(p/let [root (.. js/navigator -storage (getDirectory))
file-handle (.getFileHandle root filename)
file (.getFile file-handle)]
(.text file)))
(comment
(defn <delete-file!
"Delete `filename` from Origin Private File System.
Options:
- :ignore-not-found? (default true) → don't treat missing file as error.
Returns a promise that resolves to nil."
[filename & {:keys [ignore-not-found?]
:or {ignore-not-found? true}}]
(-> (p/let [root (.. js/navigator -storage (getDirectory))]
(.removeEntry root filename))
(p/catch (fn [err]
(if (and ignore-not-found?
(= (.-name err) "NotFoundError"))
nil
(throw err)))))))

272
src/main/frontend/components/block.cljs
View File

@@ -147,7 +147,7 @@
(match url
["File" s]
(-> (string/replace s "file://" "")
;; "file:/Users/ll/Downloads/test.pdf" is a normal org file link
;; "file:/Users/ll/Downloads/test.pdf" is a normal org file link
(string/replace "file:" ""))
["Complex" m]
@@ -196,22 +196,22 @@
< rum/reactive
(rum/local nil ::exist?)
(rum/local false ::loading?)
{:will-mount (fn [state]
(let [src (first (:rum/args state))]
(if (and (common-config/local-protocol-asset? src)
(file-sync/current-graph-sync-on?))
(let [*exist? (::exist? state)
;; special handling for asset:// protocol
;; Capacitor uses a special URL for assets loading
asset-path (common-config/remove-asset-protocol src)
asset-path (fs/asset-path-normalize asset-path)]
(if (string/blank? asset-path)
(reset! *exist? false)
;; FIXME(andelf): possible bug here
(p/let [exist? (fs/asset-href-exists? asset-path)]
(reset! *exist? (boolean exist?))))
(assoc state ::asset-path asset-path ::asset-file? true))
state)))
{:will-mount (fn [state]
(let [src (first (:rum/args state))]
(if (and (common-config/local-protocol-asset? src)
(file-sync/current-graph-sync-on?))
(let [*exist? (::exist? state)
;; special handling for asset:// protocol
;; Capacitor uses a special URL for assets loading
asset-path (common-config/remove-asset-protocol src)
asset-path (fs/asset-path-normalize asset-path)]
(if (string/blank? asset-path)
(reset! *exist? false)
;; FIXME(andelf): possible bug here
(p/let [exist? (fs/asset-href-exists? asset-path)]
(reset! *exist? (boolean exist?))))
(assoc state ::asset-path asset-path ::asset-file? true))
state)))
:will-update (fn [state]
(let [src (first (:rum/args state))
asset-file? (boolean (::asset-file? state))
@@ -516,12 +516,15 @@
(rum/local nil ::src)
[state config title href metadata full_text]
(let [src (::src state)
^js js-url (:link-js-url config)
repo (state/get-current-repo)
href (config/get-local-asset-absolute-path href)
href (cond-> href
(nil? js-url)
(config/get-local-asset-absolute-path))
db-based? (config/db-based-graph? repo)]
(when (nil? @src)
(p/then (assets-handler/<make-asset-url href)
(p/then (assets-handler/<make-asset-url href js-url)
#(reset! src (common-util/safe-decode-uri-component %))))
(:image-placeholder config)
(if-not @src
@@ -616,7 +619,7 @@
repo (state/get-current-repo)]
(ui/catch-error
[:span.warning full_text]
(if (and (common-config/local-asset? href)
(if (and (common-config/local-relative-asset? href)
(or (config/local-file-based-graph? repo)
(config/db-based-graph? repo)))
(asset-link config title href metadata full_text)
@@ -805,7 +808,7 @@
(cond
(and label
(string? label)
(not (string/blank? label))) ; alias
(not (string/blank? label))) ; alias
label
(coll? label)
@@ -842,7 +845,7 @@
(some-> (hooks/deref *el-popup) (.focus))))}
:as-dropdown? false}))
;; teardown
;; teardown
(fn []
(when visible?
(shui/popup-hide!))))
@@ -875,8 +878,8 @@
(rum/defc page-preview-trigger
[{:keys [children sidebar? open? manual?] :as config} page-entity]
(let [*timer (hooks/use-ref nil) ;; show
*timer1 (hooks/use-ref nil) ;; hide
(let [*timer (hooks/use-ref nil) ;; show
*timer1 (hooks/use-ref nil) ;; hide
*el-popup (hooks/use-ref nil)
*el-wrap (hooks/use-ref nil)
[in-popup? set-in-popup!] (rum/use-state nil)
@@ -980,8 +983,8 @@
(assoc state :*entity *result)))}
"Component for a page. `page` argument contains :block/name which can be (un)sanitized page name.
Keys for `config`:
- `:preview?`: Is this component under preview mode? (If true, `page-preview-trigger` won't be registered to this `page-cp`)"
Keys for `config`:
- `:preview?`: Is this component under preview mode? (If true, `page-preview-trigger` won't be registered to this `page-cp`)"
[state {:keys [label children preview? disable-preview? show-non-exists-page? tag? _skip-async-load?] :as config} page]
(let [entity' (rum/react (:*entity state))
entity (or (db/sub-block (:db/id entity')) entity')
@@ -1078,7 +1081,7 @@
asset-type (:logseq.property.asset/type block)
path (path/path-join common-config/local-assets-dir (str (:block/uuid block) "." asset-type))]
(p/let [result (if config/publishing?
;; publishing doesn't have window.pfs defined
;; publishing doesn't have window.pfs defined
true
(fs/file-exists? (config/get-repo-dir (state/get-current-repo)) path))]
(reset! (::file-exists? state) result))
@@ -1291,8 +1294,8 @@
(rum/defc block-reference-preview
[children {:keys [repo config id]}]
(let [*timer (hooks/use-ref nil) ;; show
*timer1 (hooks/use-ref nil) ;; hide
(let [*timer (hooks/use-ref nil) ;; show
*timer1 (hooks/use-ref nil) ;; hide
[visible? set-visible!] (rum/use-state nil)
_ #_:clj-kondo/ignore (rum/defc render []
[:div.tippy-wrapper.as-block
@@ -1344,7 +1347,7 @@
:else
title)]
[:div.block-ref-wrap.inline
{:data-type (name (or block-type :default))
{:data-type (name (or block-type :default))
:data-hl-type hl-type
:on-pointer-down
(fn [^js/MouseEvent e]
@@ -1371,13 +1374,13 @@
:else
(match [block-type (util/electron?)]
;; pdf annotation
;; pdf annotation
[:annotation true] (pdf-assets/open-block-ref! block)
[:whiteboard-shape true] (route-handler/redirect-to-page!
(get-in block [:block/page :block/uuid]) {:block-id block-id})
;; default open block page
;; default open block page
:else (route-handler/redirect-to-page! block-id))))))}
(if (and (not (util/mobile?))
@@ -1466,7 +1469,7 @@
(and
(nil? metadata-show)
(or
(common-config/local-asset? s)
(common-config/local-relative-asset? s)
(text-util/media-link? media-formats s)))
(true? (boolean metadata-show))))
@@ -1490,7 +1493,7 @@
(rum/defc audio-link
[config url href _label metadata full_text]
(if (and (common-config/local-asset? href)
(if (and (common-config/local-relative-asset? href)
(or (config/local-file-based-graph? (state/get-current-repo))
(config/db-based-graph? (state/get-current-repo))))
(asset-link config nil href metadata full_text)
@@ -1616,13 +1619,17 @@
:else
(let [href (string-of-url url)
[protocol path] (or (and (= "Complex" (first url)) [(:protocol (second url)) (:link (second url))])
(and (= "File" (first url)) ["file" (second url)]))]
(and (= "File" (first url)) ["file" (second url)]))
config (cond-> config
(not (string/blank? protocol))
(assoc :link-js-url (try (js/URL. href)
(catch :default _ nil))))]
(cond
(and (= (get-in config [:block :block/format] :markdown) :org)
(= "Complex" protocol)
(= (string/lower-case (:protocol path)) "id")
(string? (:link path))
(util/uuid-string? (:link path))) ; org mode id
(util/uuid-string? (:link path))) ; org mode id
(let [id (uuid (:link path))
block (db/entity [:block/uuid id])]
(if (:block/pre-block? block)
@@ -1645,9 +1652,9 @@
(util/stop e)
(js/window.apis.openPath path))
:data-href href*}
{:href (path/path-join "file://" href*)
{:href (path/path-join "file://" href*)
:data-href href*
:target "_blank"})
:target "_blank"})
title (assoc :title title))
(map-inline config label))]))
@@ -1703,7 +1710,7 @@
{:keys [link-depth]} config
link-depth (or link-depth 0)]
(cond
(nil? a) ; empty embed
(nil? a) ; empty embed
nil
(> link-depth max-depth-of-links)
@@ -1719,7 +1726,7 @@
(when-let [id (some-> s parse-uuid)]
(block-embed (assoc config :link-depth (inc link-depth)) id)))
:else ;TODO: maybe collections?
:else ;TODO: maybe collections?
nil)))
(defn- macro-vimeo-cp
@@ -2033,7 +2040,7 @@
[:span {:dangerouslySetInnerHTML
{:__html (security/sanitize-html (:html e))}}]
["Latex_Fragment" [display s]] ;display can be "Displayed" or "Inline"
["Latex_Fragment" [display s]] ;display can be "Displayed" or "Inline"
(if html-export?
(latex/html-export s false true)
(latex/latex s false (not= display "Inline")))
@@ -2063,7 +2070,7 @@
[:span {:dangerouslySetInnerHTML
{:__html (security/sanitize-html s)}}])
["Inline_Hiccup" s] ;; String to hiccup
["Inline_Hiccup" s] ;; String to hiccup
(ui/catch-error
[:div.warning {:title "Invalid hiccup"} s]
[:span {:dangerouslySetInnerHTML
@@ -2163,17 +2170,17 @@
(declare block-list)
(rum/defc block-children < rum/reactive
[config block children collapsed?]
(let [ref? (:ref? config)
query? (:custom-query? config)
library? (:library? config)
children (when (coll? children)
(let [ref-matched-children-ids (:ref-matched-children-ids config)]
(cond->> (remove nil? children)
ref-matched-children-ids
;; Block children will not be rendered if the filters do not match them
(filter (fn [b] (ref-matched-children-ids (:db/id b))))
library?
(filter (fn [b] (and (ldb/page? b) (not (or (ldb/class? b) (ldb/property? b)))))))))]
(let [ref? (:ref? config)
query? (:custom-query? config)
library? (:library? config)
children (when (coll? children)
(let [ref-matched-children-ids (:ref-matched-children-ids config)]
(cond->> (remove nil? children)
ref-matched-children-ids
;; Block children will not be rendered if the filters do not match them
(filter (fn [b] (ref-matched-children-ids (:db/id b))))
library?
(filter (fn [b] (and (ldb/page? b) (not (or (ldb/class? b) (ldb/property? b)))))))))]
(when (and (coll? children)
(seq children)
(not collapsed?))
@@ -2198,51 +2205,50 @@
(rum/defcs ^:large-vars/cleanup-todo block-control < rum/reactive
(rum/local false ::dragging?)
[state config block {:keys [uuid block-id collapsed? *control-show? edit? selected? top? bottom?]}]
(let [*bullet-dragging? (::dragging? state)
doc-mode? (state/sub :document/mode?)
control-show? (util/react *control-show?)
ref? (:ref? config)
empty-content? (block-content-empty? block)
(let [*bullet-dragging? (::dragging? state)
doc-mode? (state/sub :document/mode?)
control-show? (util/react *control-show?)
ref? (:ref? config)
empty-content? (block-content-empty? block)
fold-button-right? (state/enable-fold-button-right?)
own-number-list? (:own-order-number-list? config)
order-list? (boolean own-number-list?)
order-list-idx (:own-order-list-index config)
page-title? (:page-title? config)
collapsable? (editor-handler/collapsable? uuid {:semantic? true
:ignore-children? page-title?})
link? (boolean (:original-block config))
icon-size (if collapsed? 12 14)
icon (icon-component/get-node-icon-cp block {:size icon-size :color? true :link? link?})
with-icon? (and (some? icon)
(or (and (db/page? block)
(not (:library? config)))
(:logseq.property/icon block)
link?
(some :logseq.property/icon (:block/tags block))
(contains? #{"pdf"} (:logseq.property.asset/type block))))]
own-number-list? (:own-order-number-list? config)
order-list? (boolean own-number-list?)
order-list-idx (:own-order-list-index config)
page-title? (:page-title? config)
collapsable? (editor-handler/collapsable? uuid {:semantic? true
:ignore-children? page-title?})
link? (boolean (:original-block config))
icon-size (if collapsed? 12 14)
icon (icon-component/get-node-icon-cp block {:size icon-size :color? true :link? link?})
with-icon? (and (some? icon)
(or (and (db/page? block)
(not (:library? config)))
(:logseq.property/icon block)
link?
(some :logseq.property/icon (:block/tags block))
(contains? #{"pdf"} (:logseq.property.asset/type block))))]
[:div.block-control-wrap.flex.flex-row.items-center.h-6
{:class (util/classnames [{:is-order-list order-list?
:is-with-icon with-icon?
:is-with-icon with-icon?
:bullet-closed collapsed?
:bullet-hidden (:hide-bullet? config)}])}
(when (and (or (not fold-button-right?) collapsable? collapsed?)
(not (:table? config)))
[:a.block-control
{:id (str "control-" uuid)
:on-pointer-down
(fn [event]
(util/stop event)
(state/clear-edit!)
(p/do!
(if ref?
(state/toggle-collapsed-block! uuid)
(if collapsed?
(editor-handler/expand-block! uuid)
(editor-handler/collapse-block! uuid)))
(haptics/haptics))
;; debug config context
(when (and (state/developer-mode?) (.-metaKey event))
(js/console.debug "[block config]==" config)))}
:on-click (fn [event]
(util/stop event)
(state/clear-edit!)
(p/do!
(if ref?
(state/toggle-collapsed-block! uuid)
(if collapsed?
(editor-handler/expand-block! uuid)
(editor-handler/collapse-block! uuid)))
(haptics/haptics))
;; debug config context
(when (and (state/developer-mode?) (.-metaKey event))
(js/console.debug "[block config]==" config)))}
[:span {:class (if (or (and control-show? (or collapsed? collapsable?))
(and collapsed? (or page-title? order-list? config/publishing? (util/mobile?))))
"control-show cursor-pointer"
@@ -2295,7 +2301,7 @@
(not top?)
(not bottom?)
(not (util/react *control-show?))
(not (:logseq.property/created-from-property block)))
(not (:logseq.property/created-from-property block)))
(and doc-mode?
(not collapsed?)
(not (util/react *control-show?))))
@@ -2380,14 +2386,14 @@
{:data-marker (str (string/lower-case marker))}))
;; children
(let [area? (= :area (keyword (pu/lookup block :logseq.property.pdf/hl-type)))
(let [area? (= :area (keyword (pu/lookup block :logseq.property.pdf/hl-type)))
hl-ref #(when (not (#{:default :whiteboard-shape} block-type))
[:div.prefix-link
{:on-pointer-down
(fn [^js e]
(let [^js target (.-target e)]
(case block-type
;; pdf annotation
;; pdf annotation
:annotation
(if (and area? (.contains (.-classList target) "blank"))
:actions
@@ -2405,9 +2411,9 @@
(when (and area?
(or
;; db graphs
;; db graphs
(:logseq.property.pdf/hl-image block)
;; file graphs
;; file graphs
(get-in block [:block/properties :hl-stamp])))
(pdf-assets/area-display block))])]
(remove-nils
@@ -2990,9 +2996,9 @@
:default)
mouse-down-key (if (util/mobile?)
:on-click
:on-pointer-down) ; TODO: it seems that Safari doesn't work well with on-pointer-down
:on-pointer-down) ; TODO: it seems that Safari doesn't work well with on-pointer-down
attrs (cond->
{:blockid (str uuid)
{:blockid (str uuid)
:class (util/classnames [{:jtrigger (:property-block? config)
:!cursor-pointer (or (:property? config) (:page-title? config))}])
:containerid (:container-id config)
@@ -3088,7 +3094,7 @@
:class (str "px-1 py-0 w-5 h-5 opacity-70 hover:opacity-100" (when (and (util/mobile?)
(seq (:block/_parent block)))
" !pr-4"))
:size :sm
:size :sm
:on-click (fn [e]
(if (gobj/get e "shiftKey")
(state/sidebar-add-block!
@@ -3441,10 +3447,10 @@
(let [text (.getData data-transfer "text/plain")]
(editor-handler/api-insert-new-block!
text
{:block-uuid uuid
{:block-uuid uuid
:edit-block? false
:sibling? (= @*move-to' :sibling)
:before? (= @*move-to' :top)}))
:sibling? (= @*move-to' :sibling)
:before? (= @*move-to' :top)}))
(contains? transfer-types "Files")
(let [files (.-files data-transfer)
@@ -3468,11 +3474,11 @@
image?)]
(editor-handler/api-insert-new-block!
link-content
{:block-uuid uuid
{:block-uuid uuid
:edit-block? false
:replace-empty-target? true
:sibling? true
:before? false}))
:sibling? true
:before? false}))
(recur (rest res))))))))
:else
@@ -3540,10 +3546,10 @@
true
(assoc :block block)
;; Each block might have multiple queries, but we store only the first query's result.
;; This :query-result atom is used by the query function feature to share results between
;; the parent's query block and the children blocks. This works because config is shared
;; between parent and children blocks
;; Each block might have multiple queries, but we store only the first query's result.
;; This :query-result atom is used by the query function feature to share results between
;; the parent's query block and the children blocks. This works because config is shared
;; between parent and children blocks
(nil? (:query-result config))
(assoc :query-result (atom nil))
@@ -3599,8 +3605,8 @@
(mixins/event-mixin
(fn [state]
(let [*ref (::ref state)]
;; React doesn't let us directly control passive via onTouchMove
;; So here we listen `touchmove` on the block node
;; React doesn't let us directly control passive via onTouchMove
;; So here we listen `touchmove` on the block node
(mixins/listen state @*ref "touchmove" block-handler/on-touch-move))))
[state container-state repo config* block {:keys [navigating-block navigated? editing? selected?] :as opts}]
(let [*ref (::ref state)
@@ -3891,13 +3897,13 @@
(defn- config-block-should-update?
[old-state new-state]
(let [config-compare-keys [:show-cloze? :hide-children? :own-order-list-type :own-order-list-index :original-block :edit? :hide-bullet? :ref-matched-children-ids]
b1 (second (:rum/args old-state))
b2 (second (:rum/args new-state))
result (or
(block-changed? b1 b2)
;; config changed
(not= (select-keys (first (:rum/args old-state)) config-compare-keys)
(select-keys (first (:rum/args new-state)) config-compare-keys)))]
b1 (second (:rum/args old-state))
b2 (second (:rum/args new-state))
result (or
(block-changed? b1 b2)
;; config changed
(not= (select-keys (first (:rum/args old-state)) config-compare-keys)
(select-keys (first (:rum/args new-state)) config-compare-keys)))]
(boolean result)))
(defn- set-collapsed-block!
@@ -3937,7 +3943,7 @@
(or linked-block? (nil? (:container-id config)))
(assoc ::container-id (state/get-next-container-id)))))
:will-unmount (fn [state]
;; restore root block's collapsed state
;; restore root block's collapsed state
(let [[config block] (:rum/args state)
block-id (:block/uuid block)]
(when (root-block? config block)
@@ -3986,11 +3992,11 @@
(defn divide-lists
[[f & l]]
(loop [l l
(loop [l l
ordered? (:ordered f)
result [[f]]]
result [[f]]]
(if (seq l)
(let [cur (first l)
(let [cur (first l)
cur-ordered? (:ordered cur)]
(if (= ordered? cur-ordered?)
(recur
@@ -4111,13 +4117,13 @@
[log]
(let [clocks (filter #(string/starts-with? % "CLOCK:") log)
clocks (reverse (sort-by str clocks))]
;; TODO: display states change log
; states (filter #(not (string/starts-with? % "CLOCK:")) log)
;; TODO: display states change log
; states (filter #(not (string/starts-with? % "CLOCK:")) log)
(when (seq clocks)
(let [tr (fn [elm cols] (->elem :tr
(mapv (fn [col] (->elem elm col)) cols)))
head [:thead.overflow-x-scroll (tr :th.py-0 ["Type" "Start" "End" "Span"])]
head [:thead.overflow-x-scroll (tr :th.py-0 ["Type" "Start" "End" "Span"])]
clock-tbody (->elem
:tbody.overflow-scroll.sm:overflow-auto
(mapv (fn [clock]
@@ -4252,11 +4258,11 @@
(and
(= name "logbook")
(state/enable-timetracking?)
(or (get-in (state/get-config) [:logbook/settings :enabled-in-all-blocks])
(when (get-in (state/get-config)
[:logbook/settings :enabled-in-timestamped-blocks] true)
(or (:block/scheduled (:block config))
(:block/deadline (:block config)))))))
(or (get-in (state/get-config) [:logbook/settings :enabled-in-all-blocks])
(when (get-in (state/get-config)
[:logbook/settings :enabled-in-timestamped-blocks] true)
(or (:block/scheduled (:block config))
(:block/deadline (:block config)))))))
[:div
[:div.text-sm
[:div.drawer {:data-drawer-name name}
@@ -4271,8 +4277,8 @@
{:default-collapsed? true
:title-trigger? true})]]])
;; for file-level property in orgmode: #+key: value
;; only display caption. https://orgmode.org/manual/Captions.html.
;; for file-level property in orgmode: #+key: value
;; only display caption. https://orgmode.org/manual/Captions.html.
["Directive" key value]
[:div.file-level-property
(when (contains? #{"caption"} (string/lower-case key))
@@ -4281,7 +4287,7 @@
(str ": " value)])]
["Paragraph" l]
;; TODO: speedup
;; TODO: speedup
(if (util/safe-re-find #"\"Export_Snippet\" \"embed\"" (str l))
(->elem :div (map-inline config l))
(->elem :div.is-paragraph (map-inline config l)))
@@ -4516,7 +4522,7 @@
(fn []
(when-let [h (and (hooks/deref *wrap-ref)
(.-height (.-style target)))]
;(prn "==>> debug: " h)
;(prn "==>> debug: " h)
(set-wrap-h! h))))]
(.observe ob target)
(vreset! *ob ob))))))
@@ -4555,7 +4561,7 @@
{:init (fn [state]
(let [first-block (ffirst (:rum/args state))]
(assoc state
::initial-block first-block
::initial-block first-block
::navigating-block (atom (:block/uuid first-block)))))}
[state blocks config]
(let [*navigating-block (::navigating-block state)

100
src/main/frontend/components/e2ee.cljs Normal file
View File

@@ -0,0 +1,100 @@
(ns frontend.components.e2ee
(:require [clojure.string :as string]
[frontend.common.crypt :as crypt]
[frontend.state :as state]
[frontend.ui :as ui]
[frontend.util :as util]
[logseq.shui.hooks :as hooks]
[logseq.shui.ui :as shui]
[promesa.core :as p]
[rum.core :as rum]))
(rum/defc e2ee-request-new-password
[password-promise]
(let [[password set-password!] (hooks/use-state "")
[password-confirm set-password-confirm!] (hooks/use-state "")
[matched? set-matched!] (hooks/use-state nil)
on-submit (fn []
(p/resolve! password-promise password)
(shui/dialog-close!))]
[:div.e2ee-password-modal-overlay
[:div.encryption-password.max-w-2xl.e2ee-password-modal-content.flex.flex-col.gap-8.p-4
[:div.text-2xl.font-medium "Set password for remote graphs"]
[:div.init-remote-pw-tips.space-x-4.hidden.sm:flex
[:div.flex-1.flex.items-center
[:span.px-3.flex (ui/icon "key")]
[:p
[:span "Please make sure you "]
"remember the password you have set, as we are unable to reset or retrieve it in case you forget it, "
[:span "and we recommend you "]
"keep a secure backup "
[:span "of the password."]]]
[:div.flex-1.flex.items-center
[:span.px-3.flex (ui/icon "lock")]
[:p
"If you lose your password, all of your data in the cloud can’t be decrypted. "
[:span "You will still be able to access the local version of your graph."]]]]
[:div.flex.flex-col.gap-4
(shui/toggle-password
{:placeholder "Enter password"
:value password
:on-change (fn [e] (set-password! (-> e .-target .-value)))
:on-blur (fn []
(when-not (string/blank? password-confirm)
(set-matched! (= password-confirm password))))})
[:div.flex.flex-col.gap-2
(shui/input
{:type "password-confirm"
:placeholder "Enter password again"
:value password-confirm
:on-change (fn [e] (set-password-confirm! (-> e .-target .-value)))
:on-blur (fn [] (set-matched! (= password-confirm password)))})
(when (false? matched?)
[:div.text-warning.text-sm
"Password not matched"])]
(shui/button
{:on-click on-submit
:disabled (or (string/blank? password)
(false? matched?))}
"Submit")]]]))
(rum/defc e2ee-password-to-decrypt-private-key
[encrypted-private-key private-key-promise refresh-token]
(let [[password set-password!] (hooks/use-state "")
[decrypt-fail? set-decrypt-fail!] (hooks/use-state false)
on-submit (fn []
(->
(p/let [private-key (crypt/<decrypt-private-key password encrypted-private-key)]
(state/<invoke-db-worker :thread-api/save-e2ee-password refresh-token password)
(p/resolve! private-key-promise private-key)
(shui/dialog-close!))
(p/catch (fn [e]
(when (= "decrypt-private-key" (ex-message e))
(set-decrypt-fail! true))))))]
[:div.e2ee-password-modal-overlay
[:div.e2ee-password-modal-content.flex.flex-col.gap-8.p-4
[:div.text-2xl.font-medium "Enter password for remote graphs"]
[:div.flex.flex-col.gap-4
[:div.flex.flex-col.gap-1
(shui/toggle-password
{:value password
:on-key-press (fn [e]
(when (= "Enter" (util/ekey e))
(on-submit)))
:on-change (fn [e]
(set-decrypt-fail! false)
(set-password! (-> e .-target .-value)))})
(when decrypt-fail? [:p.text-warning.text-sm "Wrong password"])]
(shui/button
{:on-click on-submit
:disabled (string/blank? password)
:on-key-press (fn [e]
(when (= "Enter" (util/ekey e))
(on-submit)))}
"Submit")]]]))

2
src/main/frontend/components/export.cljs
View File

@@ -283,7 +283,7 @@
:on-click #(do (reset! *export-block-type :edn)
(p/let [result (<export-edn-helper top-level-uuids export-type)
pull-data (with-out-str (pprint/pprint result))]
(when-not (= :export-edn-error result)
(when-not (:export-edn-error result)
(reset! *content pull-data))))))])
(if (= :png tp)
[:div.flex.items-center.justify-center.relative

2
src/main/frontend/components/page.cljs
View File

@@ -698,7 +698,7 @@
:container-id (:container-id state)
:whiteboard? whiteboard?}))])])
(when (and (not preview?) (not show-tabs?))
(when-not preview?
[:div.ml-1.flex.flex-col.gap-8
(when today?
(today-queries repo today? sidebar?))

9
src/main/frontend/components/plugins.cljs
View File

@@ -1598,15 +1598,16 @@
(cond-> routes
config/lsp-enabled?
(concat (some->> (plugin-handler/get-route-renderers)
(mapv #(when-let [{:keys [name path render]} %]
(when (not (string/blank? path))
[path {:name name :view (fn [r] (render r %))}])))
(mapv (fn [custom-route]
(when-let [{:keys [name path render]} custom-route]
(when (not (string/blank? path))
[path {:name name :view (fn [r] (render r custom-route))}]))))
(remove nil?)))))
(defn hook-daemon-renderers
[]
(when-let [rs (seq (plugin-handler/get-daemon-renderers))]
[:div.lsp-daemon-container.fixed.z-10
[:div.lsp-daemon-container
(for [{:keys [key _pid render]} rs]
(when (fn? render)
[:div.lsp-daemon-container-card {:data-key key} (render)]))]))

4
src/main/frontend/components/plugins.css
View File

@@ -769,6 +769,10 @@
}
}
.lsp-daemon-container {
@apply fixed top-0 left-0 z-10;
}
.lsp-ui-float-container {
top: 40%;
left: 30%;

7
src/main/frontend/components/property/value.cljs
View File

@@ -1096,7 +1096,12 @@
[:span.number (str value')]
:else
(inline-text {} :markdown (str value'))))))
[:span.inline-flex.w-full
(let [value' (str value')
value' (if (string/blank? value')
"Empty"
value')]
(inline-text {} :markdown value'))]))))
(rum/defc select-item
[property type value {:keys [page-cp inline-text other-position? property-position table-view? _icon?] :as opts}]

191
src/main/frontend/components/repo.cljs
View File

@@ -22,6 +22,7 @@
[frontend.util.text :as text-util]
[goog.object :as gobj]
[lambdaisland.glogi :as log]
[logseq.shui.hooks :as hooks]
[logseq.shui.ui :as shui]
[medley.core :as medley]
[promesa.core :as p]
@@ -36,11 +37,11 @@
db-based?)
(let [local-dir (config/get-local-dir url)
graph-name (text-util/get-graph-name-from-path url)]
[:a.flex.items-center {:title local-dir
[:a.flex.items-center {:title local-dir
:on-click #(on-click graph)}
[:span graph-name (when (and GraphName (not db-based?)) [:strong.pl-1 "(" GraphName ")"])]
(when remote? [:strong.px-1.flex.items-center (ui/icon "cloud")])])
[:a.flex.items-center {:title GraphUUID
[:a.flex.items-center {:title GraphUUID
:on-click #(on-click graph)}
(db/get-repo-path (or url GraphName))
(when remote? [:strong.pl-1.flex.items-center (ui/icon "cloud")])])])))
@@ -262,32 +263,32 @@
GraphName)
downloading? (and downloading-graph-id (= GraphUUID downloading-graph-id))]
(when short-repo-name
{:title [:span.flex.items-center.title-wrap short-repo-name
(when remote? [:span.pl-1.flex.items-center
{:title (str "<" GraphName "> #" GraphUUID)}
(ui/icon "cloud" {:size 18})
(when downloading?
[:span.opacity.text-sm.pl-1 "downloading"])])]
{:title [:span.flex.items-center.title-wrap short-repo-name
(when remote? [:span.pl-1.flex.items-center
{:title (str "<" GraphName "> #" GraphUUID)}
(ui/icon "cloud" {:size 18})
(when downloading?
[:span.opacity.text-sm.pl-1 "downloading"])])]
:hover-detail repo-url ;; show full path on hover
:options {:on-click
(fn [e]
(when-not downloading?
(when-let [on-click (:on-click opts)]
(on-click e))
(if (and (gobj/get e "shiftKey")
(not (and rtc-graph? remote?)))
(state/pub-event! [:graph/open-new-window url])
(cond
:options {:on-click
(fn [e]
(when-not downloading?
(when-let [on-click (:on-click opts)]
(on-click e))
(if (and (gobj/get e "shiftKey")
(not (and rtc-graph? remote?)))
(state/pub-event! [:graph/open-new-window url])
(cond
;; exists locally?
(or (:root graph) (not rtc-graph?))
(state/pub-event! [:graph/switch url])
(or (:root graph) (not rtc-graph?))
(state/pub-event! [:graph/switch url])
(and rtc-graph? remote?)
(state/pub-event!
[:rtc/download-remote-graph GraphName GraphUUID GraphSchemaVersion])
(and rtc-graph? remote?)
(state/pub-event!
[:rtc/download-remote-graph GraphName GraphUUID GraphSchemaVersion])
:else
(state/pub-event! [:graph/pull-down-remote-graph graph])))))}})))
:else
(state/pub-event! [:graph/pull-down-remote-graph graph])))))}})))
switch-repos)]
(->> repo-links (remove nil?))))
@@ -354,7 +355,7 @@
(if (and (or (seq remotes) (seq rtc-graphs)) login?)
(repo-handler/combine-local-&-remote-graphs repos (concat remotes rtc-graphs)) repos))
items-fn #(repos-dropdown-links repos current-repo downloading-graph-id opts)
header-fn #(when (> (count repos) 1) ; show switch to if there are multiple repos
header-fn #(when (> (count repos) 1) ; show switch to if there are multiple repos
[:div.font-medium.md:text-sm.md:opacity-50.px-1.py-1.flex.flex-row.justify-between.items-center
[:h4.pb-1 (t :left-side-bar/switch)]
@@ -450,67 +451,79 @@
(string/includes? graph-name "+")
(string/includes? graph-name "/")))
(rum/defcs new-db-graph < rum/reactive
(rum/local "" ::graph-name)
(rum/local false ::cloud?)
(rum/local false ::creating-db?)
(rum/local (rum/create-ref) ::input-ref)
{:did-mount (fn [s]
(when-let [^js input (some-> @(::input-ref s)
(rum/deref))]
(js/setTimeout #(.focus input) 32))
s)}
[state]
(let [*creating-db? (::creating-db? state)
*graph-name (::graph-name state)
*cloud? (::cloud? state)
input-ref @(::input-ref state)
new-db-f (fn []
(when-not (or (string/blank? @*graph-name)
@*creating-db?)
(if (invalid-graph-name? @*graph-name)
(invalid-graph-name-warning)
(do
(reset! *creating-db? true)
(p/let [repo (repo-handler/new-db! @*graph-name)]
(when @*cloud?
(->
(p/do
(state/set-state! :rtc/uploading? true)
(rtc-handler/<rtc-create-graph! repo)
(rtc-flows/trigger-rtc-start repo)
(rtc-handler/<get-remote-graphs))
(p/catch (fn [error]
(log/error :create-db-failed error)))
(p/finally (fn []
(state/set-state! :rtc/uploading? false)
(reset! *creating-db? false)))))
(shui/dialog-close!))))))
submit! (fn [^js e click?]
(when-let [value (and (or click? (= (gobj/get e "key") "Enter"))
(util/trim-safe (.-value (rum/deref input-ref))))]
(reset! *graph-name value)
(new-db-f)))]
[:div.new-graph.flex.flex-col.gap-4.p-1.pt-2
(shui/input
{:default-value @*graph-name
:disabled @*creating-db?
:ref input-ref
:placeholder "your graph name"
:on-key-down submit!})
(when (user-handler/rtc-group?)
[:div.flex.flex-row.items-center.gap-1
(shui/checkbox
{:id "rtc-sync"
:value @*cloud?
:on-checked-change #(swap! *cloud? not)})
[:label.opacity-70.text-sm
{:for "rtc-sync"}
"Use Logseq Sync?"]])
(shui/button
{:on-click #(submit! % true)
:on-key-down submit!}
(if @*creating-db?
(ui/loading "Creating graph")
"Submit"))]))
(rum/defc new-db-graph
[]
(let [[creating-db? set-creating-db?] (hooks/use-state false)
[cloud? set-cloud?] (hooks/use-state false)
[e2ee-rsa-key-ensured? set-e2ee-rsa-key-ensured?] (hooks/use-state nil)
input-ref (hooks/create-ref)]
(hooks/use-effect!
(fn []
(when-let [^js input (hooks/deref input-ref)]
(js/setTimeout #(.focus input) 32)))
[])
(letfn [(new-db-f [graph-name]
(when-not (or (string/blank? graph-name)
creating-db?)
(if (invalid-graph-name? graph-name)
(invalid-graph-name-warning)
(do
(set-creating-db? true)
(p/let [repo (repo-handler/new-db! graph-name)]
(when cloud?
(->
(p/do
(state/set-state! :rtc/uploading? true)
(rtc-handler/<rtc-create-graph! repo)
(rtc-flows/trigger-rtc-start repo)
(rtc-handler/<get-remote-graphs))
(p/catch (fn [error]
(log/error :create-db-failed error)))
(p/finally (fn []
(state/set-state! :rtc/uploading? false)
(set-creating-db? false)))))
(shui/dialog-close!))))))
(submit! [^js e click?]
(when-let [value (and (or click? (= (gobj/get e "key") "Enter"))
(util/trim-safe (.-value (rum/deref input-ref))))]
(new-db-f value)))]
[:div.new-graph.flex.flex-col.gap-4.p-1.pt-2
(shui/input
{:disabled creating-db?
:ref input-ref
:placeholder "your graph name"
:on-key-down submit!
:autoComplete "off"})
(when (user-handler/rtc-group?)
[:div.flex.flex-col
[:div.flex.flex-row.items-center.gap-1
(shui/checkbox
{:id "rtc-sync"
:value cloud?
:on-checked-change
(fn []
(let [v (boolean (not cloud?))
token (state/get-auth-id-token)
user-uuid (user-handler/user-uuid)]
(set-cloud? v)
(when (and (true? v) (not e2ee-rsa-key-ensured?))
(when (and token user-uuid)
(-> (p/let [rsa-key-pair (state/<invoke-db-worker :thread-api/get-user-rsa-key-pair token user-uuid)]
(set-e2ee-rsa-key-ensured? (some? rsa-key-pair)))
(p/catch (fn [e]
(log/error :get-user-rsa-key-pair e)
e)))))))})
[:label.opacity-70.text-sm
{:for "rtc-sync"}
"Use Logseq Sync?"]]
(when (false? e2ee-rsa-key-ensured?)
[:label.opacity-70.text-sm
{:for "rtc-sync"}
"Need to init E2EE settings first, Settings > Encryption"])])
(shui/button
{:disabled (and cloud? (not e2ee-rsa-key-ensured?))
:on-click #(submit! % true)
:on-key-down submit!}
(if creating-db?
(ui/loading "Creating graph")
"Submit"))])))

211
src/main/frontend/components/settings.cljs
View File

@@ -36,6 +36,7 @@
[frontend.version :as fv]
[goog.object :as gobj]
[goog.string :as gstring]
[lambdaisland.glogi :as log]
[logseq.db :as ldb]
[logseq.shui.hooks :as hooks]
[logseq.shui.ui :as shui]
@@ -1161,38 +1162,182 @@
[:<>])
(rum/defcs settings-collaboration < rum/reactive
(rum/local "" ::invite-email)
{:will-mount (fn [state]
(rtc-handler/<rtc-get-users-info)
state)}
[state]
(let [*invite-email (::invite-email state)
(rum/defc settings-rtc-members
[]
(let [[invite-email set-invite-email!] (hooks/use-state "")
current-repo (state/get-current-repo)
users (get (state/sub :rtc/users-info) current-repo)]
[:div.panel-wrap.is-collaboration.mb-8
[:div.flex.flex-col.gap-2.mt-4
[:h2.opacity-50.font-medium "Members:"]
[:div.users.flex.flex-col.gap-1
(for [{user-name :user/name
user-email :user/email
graph<->user-user-type :graph<->user/user-type} users]
[:div.flex.flex-row.items-center.gap-2 {:key (str "user-" user-name)}
[:div user-name]
(when user-email [:div.opacity-50.text-sm user-email])
(when graph<->user-user-type [:div.opacity-50.text-sm (name graph<->user-user-type)])])]
[:div.flex.flex-col.gap-4.mt-4
(shui/input
{:placeholder "Email address"
:on-change #(reset! *invite-email (util/evalue %))})
[users-info] (hooks/use-atom (:rtc/users-info @state/state))
users (get users-info current-repo)]
(hooks/use-effect!
#(c.m/run-task* (m/sp (c.m/<? (rtc-handler/<rtc-get-users-info))))
[])
[:div.flex.flex-col.gap-2.mt-4
[:h2.opacity-50.font-medium "Members:"]
[:div.users.flex.flex-col.gap-1
(for [{user-name :user/name
user-email :user/email
graph<->user-user-type :graph<->user/user-type} users]
[:div.flex.flex-row.items-center.gap-2 {:key (str "user-" user-name)}
[:div user-name]
(when user-email [:div.opacity-50.text-sm user-email])
(when graph<->user-user-type [:div.opacity-50.text-sm (name graph<->user-user-type)])])]
[:div.flex.flex-col.gap-4.mt-4
(shui/input
{:placeholder "Email address"
:on-change #(set-invite-email! (util/evalue %))})
(shui/button
{:on-click (fn []
(let [user-email invite-email
graph-uuid (ldb/get-graph-rtc-uuid (db/get-db))]
(when-not (string/blank? user-email)
(when graph-uuid
(rtc-handler/<rtc-invite-email graph-uuid user-email)))))}
"Invite")]]))
(rum/defc settings-collaboration
[]
[:div.panel-wrap.is-collaboration.mb-8
(settings-rtc-members)])
(rum/defc forgot-password
[token refresh-token user-uuid]
(let [[new-password set-new-password!] (hooks/use-state "")
[force-reset-status set-force-reset-status!] (hooks/use-state nil)
<force-reset-password-fn
(fn []
(-> (p/do!
(set-force-reset-status! "Force resetting password ...")
(state/<invoke-db-worker :thread-api/reset-user-rsa-key-pair
token refresh-token user-uuid new-password)
(set-force-reset-status! "Force reset password successfully!"))
(p/catch (fn [e]
(log/error :forgot-password e)
(set-force-reset-status! "Failed to force resetting password.")))))]
[:div.flex.flex-col.gap-4
[:p
"If you forget your password, you can force a reset of your encryption password. However, this will make all currently encrypted graph data stored on the server permanently unreadable. After resetting, you’ll need to re-upload your graphs from the client."]
[:label.opacity-70 {:for "new-password"} "Set new Password"]
(shui/toggle-password
{:id "new-password"
:value new-password
:on-change #(set-new-password! (util/evalue %))})
(when force-reset-status [:p force-reset-status])
(shui/button
{:on-click <force-reset-password-fn
:disabled (string/blank? new-password)}
"Force reset password")]))
(rum/defc reset-encryption-password
[current-password new-password {:keys [set-new-password!
set-current-password!
reset-password-status
on-click forgot? set-forgot!
token refresh-token user-uuid]}]
(let [[reset? set-reset!] (hooks/use-state false)]
(cond
forgot?
(forgot-password token refresh-token user-uuid)
reset?
[:div.flex.flex-col.gap-4
[:label.opacity-70 {:for "current-password"} "Current password"]
(shui/toggle-password
{:id "current-password"
:value current-password
:on-change #(set-current-password! (util/evalue %))})
[:label.opacity-70 {:for "new-password"} "Set new Password"]
(shui/toggle-password
{:id "new-password"
:value new-password
:on-change #(set-new-password! (util/evalue %))})
(when reset-password-status [:p reset-password-status])
(shui/button
{:on-click (fn []
(let [user-email @*invite-email
graph-uuid (ldb/get-graph-rtc-uuid (db/get-db))]
(when-not (string/blank? user-email)
(when graph-uuid
(rtc-handler/<rtc-invite-email graph-uuid user-email)))))}
"Invite")]]]))
{:on-click on-click
:disabled (string/blank? new-password)}
"Reset password")
[:a.opacity-70.hover:opacity-100 {:on-click #(set-forgot! true)}
"Forgot password?"]]
:else
[:a.opacity-70.hover:opacity-100 {:on-click #(set-reset! true)}
"Reset password"])))
(rum/defc encryption
[]
(let [user-uuid (user-handler/user-uuid)
token (state/get-auth-id-token)
refresh-token (state/get-auth-refresh-token)
[rsa-key-pair set-rsa-key-pair!] (hooks/use-state :not-inited)
[init-key-err set-init-key-err!] (hooks/use-state nil)
[get-key-err set-get-key-err!] (hooks/use-state nil)
[current-password set-current-password!] (hooks/use-state nil)
[new-password set-new-password!] (hooks/use-state nil)
[reset-password-status set-reset-password-status!] (hooks/use-state nil)
[forgot? set-forgot!] (hooks/use-state false)]
[:div.panel-wrap.is-encryption.mb-8
(hooks/use-effect!
(fn []
(when (and user-uuid token)
(-> (p/let [r (state/<invoke-db-worker :thread-api/get-user-rsa-key-pair token user-uuid)]
(set-rsa-key-pair! r))
(p/catch set-get-key-err!))
(-> (p/let [{:keys [password]} (state/<invoke-db-worker :thread-api/get-e2ee-password refresh-token)]
(set-current-password! password))
(p/catch (fn [_] (set-current-password! ""))))))
[user-uuid token])
[:div.flex.flex-col.gap-2.mt-4
(when (and user-uuid token)
(cond
get-key-err
[:p (str "Fetching user rsa-key-pair err: " get-key-err)]
(= rsa-key-pair :not-inited)
[:p "Fetching user rsa-key-pair..."]
(nil? rsa-key-pair)
[:div.flex.flex-col.gap-2
(when init-key-err [:p (str "Init key-pair err:" init-key-err)])
(shui/button
{:on-click (fn []
(-> (p/do!
(state/<invoke-db-worker :thread-api/init-user-rsa-key-pair
token
refresh-token
user-uuid)
(p/let [r (state/<invoke-db-worker :thread-api/get-user-rsa-key-pair token user-uuid)]
(set-rsa-key-pair! r)))
(p/catch set-init-key-err!)))}
"Init E2EE encrypt-key-pair")]
rsa-key-pair
(let [on-submit (fn []
(-> (p/do!
(set-reset-password-status! "Updating password ...")
(state/<invoke-db-worker :thread-api/reset-e2ee-password
token refresh-token user-uuid current-password new-password)
(set-reset-password-status! "Password updated successfully!"))
(p/catch (fn [e]
(log/error :reset-password-failed e)
(set-reset-password-status! "Failed to update password.")))))]
[:div.flex.flex-col.gap-4
;; [:p "E2EE key-pair already generated!"]
(when-not forgot?
[:div.flex.flex-col
[:p
[:span "Please make sure you "]
"remember the password you have set, as we are unable to reset or retrieve it in case you forget it, "
[:span "and we recommend you "]
"keep a secure backup "
[:span "of the password."]]
[:p
"If you lose your password, all of your data in the cloud can’t be decrypted. "
[:span "You will still be able to access the local version of your graph."]]])
(reset-encryption-password current-password new-password
{:reset-password-status reset-password-status
:set-new-password! set-new-password!
:set-current-password! set-current-password!
:on-click on-submit
:token token
:forgot? forgot?
:set-forgot! set-forgot!
:refresh-token refresh-token
:user-uuid user-uuid})])))]]))
(rum/defc mcp-server-row
[t]
@@ -1366,6 +1511,9 @@
(when logged-in?
[:collaboration "collaboration" (t :settings-page/tab-collaboration) (ui/icon "users")])
(when logged-in?
[:encryption "encryption" (t :settings-page/tab-encryption) (ui/icon "lock")])
(when plugins-of-settings
[:plugins-setting "plugins" (t :settings-of-plugins) (ui/icon "puzzle")])]]
@@ -1420,6 +1568,9 @@
:collaboration
(settings-collaboration)
:encryption
(encryption)
:ai
(settings-ai)

194
src/main/frontend/db/rtc/debug_ui.cljs
View File

@@ -7,7 +7,7 @@
[frontend.handler.user :as user]
[frontend.state :as state]
[frontend.ui :as ui]
[frontend.util :as util]
[lambdaisland.glogi :as log]
[logseq.db.frontend.schema :as db-schema]
[logseq.shui.ui :as shui]
[missionary.core :as m]
@@ -136,128 +136,7 @@
:on-click (fn [] (stop))}
(shui/tabler-icon "player-stop") "stop")]])
(when (some? debug-state*)
[:hr]
[:div.flex.flex-row.items-center.gap-2
(ui/button "grant graph access to"
{:icon "award"
:on-click (fn []
(let [token (state/get-auth-id-token)
user-uuid (some-> (:grant-access-to-user debug-state*) parse-uuid)
user-email (when-not user-uuid (:grant-access-to-user debug-state*))]
(when-let [graph-uuid (:graph-uuid debug-state*)]
(state/<invoke-db-worker :thread-api/rtc-grant-graph-access
token graph-uuid
(some-> user-uuid vector)
(some-> user-email vector)))))})
[:b "➡️"]
[:input.form-input.my-2.py-1
{:on-change (fn [e] (swap! debug-state assoc :grant-access-to-user (util/evalue e)))
:on-focus (fn [e] (let [v (.-value (.-target e))]
(when (= v "input email or user-uuid here")
(set! (.-value (.-target e)) ""))))
:placeholder "input email or user-uuid here"}]])
[:hr.my-2]
[:div.flex.flex-row.items-center.gap-2
(ui/button (str "download graph to")
{:icon "download"
:class "mr-2"
:on-click (fn []
(when-let [graph-name (:download-graph-to-repo debug-state*)]
(when-let [{:keys [graph-uuid graph-schema-version]}
(:graph-uuid-to-download debug-state*)]
(prn :download-graph graph-uuid graph-schema-version :to graph-name)
(p/let [token (state/get-auth-id-token)
download-info-uuid (state/<invoke-db-worker
:thread-api/rtc-request-download-graph
token graph-uuid graph-schema-version)
{:keys [_download-info-uuid
download-info-s3-url
_download-info-tx-instant
_download-info-t
_download-info-created-at]
:as result}
(state/<invoke-db-worker :thread-api/rtc-wait-download-graph-info-ready
token download-info-uuid graph-uuid graph-schema-version 60000)]
(when (not= result :timeout)
(assert (some? download-info-s3-url) result)
(state/<invoke-db-worker :thread-api/rtc-download-graph-from-s3
graph-uuid graph-name download-info-s3-url))))))})
[:b "➡"]
[:div.flex.flex-row.items-center.gap-2
(shui/select
{:on-value-change (fn [[graph-uuid graph-schema-version]]
(when (and (parse-uuid graph-uuid) graph-schema-version)
(swap! debug-state assoc
:graph-uuid-to-download
{:graph-uuid graph-uuid
:graph-schema-version graph-schema-version})))}
(shui/select-trigger
{:class "!px-2 !py-0 !h-8 border-gray-04"}
(shui/select-value
{:placeholder "Select a graph-uuid"}))
(shui/select-content
(shui/select-group
(for [{:keys [graph-uuid graph-schema-version graph-status]} (sort-by :graph-uuid (:remote-graphs debug-state*))]
(shui/select-item {:value [graph-uuid graph-schema-version] :disabled (some? graph-status)} graph-uuid)))))
[:b "＋"]
[:input.form-input.my-2.py-1
{:on-change (fn [e] (swap! debug-state assoc :download-graph-to-repo (util/evalue e)))
:on-focus (fn [e] (let [v (.-value (.-target e))]
(when (= v "repo name here")
(set! (.-value (.-target e)) ""))))
:placeholder "repo name here"}]]]
[:div.flex.my-2.items-center.gap-2
(ui/button (str "upload current repo")
{:icon "upload"
:on-click (fn []
(let [repo (state/get-current-repo)
token (state/get-auth-id-token)
remote-graph-name (:upload-as-graph-name debug-state*)]
(state/<invoke-db-worker :thread-api/rtc-async-upload-graph
repo token remote-graph-name)))})
[:b "➡️"]
[:input.form-input.my-2.py-1.w-32
{:on-change (fn [e] (swap! debug-state assoc :upload-as-graph-name (util/evalue e)))
:on-focus (fn [e] (let [v (.-value (.-target e))]
(when (= v "remote graph name here")
(set! (.-value (.-target e)) ""))))
:placeholder "remote graph name here"}]]
[:div.pb-2.flex.flex-row.items-center.gap-2
(ui/button (str "delete graph")
{:icon "trash"
:on-click (fn []
(when-let [{:keys [graph-uuid graph-schema-version]} (:graph-uuid-to-delete debug-state*)]
(let [token (state/get-auth-id-token)]
(prn ::delete-graph graph-uuid graph-schema-version)
(state/<invoke-db-worker :thread-api/rtc-delete-graph
token graph-uuid graph-schema-version))))})
(shui/select
{:on-value-change (fn [[graph-uuid graph-schema-version]]
(when (and (parse-uuid graph-uuid) graph-schema-version)
(swap! debug-state assoc
:graph-uuid-to-delete
{:graph-uuid graph-uuid
:graph-schema-version graph-schema-version})))}
(shui/select-trigger
{:class "!px-2 !py-0 !h-8"}
(shui/select-value
{:placeholder "Select a graph-uuid"}))
(shui/select-content
(shui/select-group
(for [{:keys [graph-uuid graph-schema-version graph-status]} (:remote-graphs debug-state*)]
(shui/select-item {:value [graph-uuid graph-schema-version] :disabled (some? graph-status)} graph-uuid)))))]
[:hr.my-2]
(let [*keys-state (get state ::keys-state)
keys-state @*keys-state]
[:div
@@ -265,61 +144,22 @@
(shui/button
{:size :sm
:on-click (fn [_]
(p/let [graph-keys (state/<invoke-db-worker :thread-api/rtc-get-graph-keys (state/get-current-repo))
devices (some->> (state/get-auth-id-token)
(state/<invoke-db-worker :thread-api/list-devices))]
(swap! (get state ::keys-state) #(merge % graph-keys {:devices devices}))))}
(shui/tabler-icon "refresh") "keys-state")]
(when-let [user-uuid (user/user-uuid)]
(p/let [user-rsa-key-pair (state/<invoke-db-worker
:thread-api/get-user-rsa-key-pair
(state/get-auth-id-token) user-uuid)]
(reset! *keys-state user-rsa-key-pair))))}
(shui/tabler-icon "refresh") "keys-state")
(shui/button
{:size :sm
:on-click (fn [_]
(when-let [token (state/get-auth-id-token)]
(p/let [r (state/<invoke-db-worker :thread-api/init-user-rsa-key-pair token (user/user-uuid))]
(when (instance? ExceptionInfo r)
(log/error :init-user-rsa-key-pair r)))))}
(shui/tabler-icon "upload") "init upload user rsa-key-pair")]
[:div.pb-4
[:pre.select-text
(-> {:devices (:devices keys-state)
:graph-aes-key-jwk (:aes-key-jwk keys-state)}
(-> keys-state
(fipp/pprint {:width 20})
with-out-str)]]
(shui/button
{:size :sm
:on-click (fn [_]
(when-let [device-uuid (not-empty (:remove-device-device-uuid keys-state))]
(when-let [token (state/get-auth-id-token)]
(state/<invoke-db-worker :thread-api/remove-device token device-uuid))))}
"Remove device:")
[:input.form-input.my-2.py-1.w-32
{:on-change (fn [e] (swap! *keys-state assoc :remove-device-device-uuid (util/evalue e)))
:on-focus (fn [e] (let [v (.-value (.-target e))]
(when (= v "device-uuid here")
(set! (.-value (.-target e)) ""))))
:placeholder "device-uuid here"}]
(shui/button
{:size :sm
:on-click (fn [_]
(when-let [device-uuid (not-empty (:remove-public-key-device-uuid keys-state))]
(when-let [key-name (not-empty (:remove-public-key-key-name keys-state))]
(when-let [token (state/get-auth-id-token)]
(state/<invoke-db-worker :thread-api/remove-device-public-key token device-uuid key-name)))))}
"Remove public-key:")
[:input.form-input.my-2.py-1.w-32
{:on-change (fn [e] (swap! *keys-state assoc :remove-public-key-device-uuid (util/evalue e)))
:on-focus (fn [e] (let [v (.-value (.-target e))]
(when (= v "device-uuid here")
(set! (.-value (.-target e)) ""))))
:placeholder "device-uuid here"}]
[:input.form-input.my-2.py-1.w-32
{:on-change (fn [e] (swap! *keys-state assoc :remove-public-key-key-name (util/evalue e)))
:on-focus (fn [e] (let [v (.-value (.-target e))]
(when (= v "key-name here")
(set! (.-value (.-target e)) ""))))
:placeholder "key-name here"}]
(shui/button
{:size :sm
:on-click (fn [_]
(when-let [token (state/get-auth-id-token)]
(when-let [device-uuid (not-empty (:sync-private-key-device-uuid keys-state))]
(state/<invoke-db-worker :thread-api/rtc-sync-current-graph-encrypted-aes-key
token [(parse-uuid device-uuid)]))))}
"Sync CurrentGraph EncryptedAesKey")
[:input.form-input.my-2.py-1.w-32
{:on-change (fn [e] (swap! *keys-state assoc :sync-private-key-device-uuid (util/evalue e)))
:on-focus (fn [e] (let [v (.-value (.-target e))]
(when (= v "device-uuid here")
(set! (.-value (.-target e)) ""))))
:placeholder "device-uuid here"}]])]))
with-out-str)]]])]))

30
src/main/frontend/extensions/pdf/assets.cljs
View File

@@ -35,26 +35,28 @@
(defn get-in-repo-assets-full-filename
[url]
(let [repo-dir (config/get-repo-dir (state/get-current-repo))]
(when (some-> url (string/trim) (string/includes? repo-dir))
(if (some-> url (string/trim) (string/includes? repo-dir))
(some-> (string/split url repo-dir)
(last)
(string/replace-first "/assets/" "")))))
(string/replace-first "/assets/" ""))
url)))
(defn inflate-asset
[original-path & {:keys [href block]}]
(let [web-link? (string/starts-with? original-path "http")
blob-res? (some-> href (string/starts-with? "blob"))
asset-res? (some-> href (string/starts-with? "assets"))
filename (util/node-path.basename original-path)
ext-name "pdf"
url (if blob-res? href
(assets-handler/normalize-asset-resource-url original-path))
filename' (if (or asset-res? web-link? blob-res?) filename
(some-> url (js/decodeURIComponent)
(get-in-repo-assets-full-filename)
(string/replace '"/" "_")))
filekey (gp-exporter/safe-sanitize-file-name
(subs filename' 0 (- (count filename') (inc (count ext-name)))))]
protocol-link? (common-config/protocol-path? href)
filename (util/node-path.basename original-path)
ext-name "pdf"
url (if protocol-link?
href
(assets-handler/normalize-asset-resource-url original-path))
filename' (if protocol-link?
filename
(some-> url (js/decodeURIComponent)
(get-in-repo-assets-full-filename)
(string/replace '"/" "_")))
filekey (gp-exporter/safe-sanitize-file-name
(subs filename' 0 (- (count filename') (inc (count ext-name)))))]
(when-let [key (and (not (string/blank? filekey))
(if web-link?
(str filekey "__" (hash url))

126
src/main/frontend/extensions/pdf/core.cljs
View File

@@ -767,7 +767,7 @@
:add-hl! add-hl!})]))
(rum/defc ^:large-vars/data-var pdf-viewer
[_url ^js pdf-document {:keys [identity filename initial-hls initial-page initial-scale initial-error]} ops]
[_url ^js pdf-document {:keys [identity filename pdf-current initial-hls initial-page initial-scale initial-error]} ops]
(let [*el-ref (rum/create-ref)
[state, set-state!] (rum/use-state {:viewer nil :bus nil :link nil :el nil})
[ano-state, set-ano-state!] (rum/use-state {:loaded-pages []})
@@ -879,7 +879,11 @@
(when (and page-ready? viewer)
[(when-not in-system-window?
(rum/with-key (pdf-resizer viewer) "pdf-resizer"))
(rum/with-key (pdf-toolbar viewer {:on-external-window! #(open-external-win! (state/get-current-pdf))}) "pdf-toolbar")])])))
(rum/with-key
(pdf-toolbar viewer
{:on-external-window! #(open-external-win! (state/get-current-pdf))
:pdf-current pdf-current})
"pdf-toolbar")])])))
(rum/defcs pdf-password-input <
(rum/local "" ::password)
@@ -931,9 +935,10 @@
"auto"))
(rum/defc ^:large-vars/data-var pdf-loader
[{:keys [url hls-file identity filename] :as pdf-current}]
[{:keys [url hls-file identity filename block] :as pdf-current}]
(let [repo (state/get-current-repo)
db-based? (config/db-based-graph?)
file-based? (not db-based?)
*doc-ref (rum/use-ref nil)
[loader-state, set-loader-state!] (rum/use-state {:error nil :pdf-document nil :status nil})
[hls-state, set-hls-state!] (rum/use-state {:initial-hls nil :latest-hls nil :extra nil :loaded false :error nil})
@@ -944,23 +949,20 @@
(set-hls-state! #(merge % {:initial-hls [] :latest-hls latest-hls})))
set-hls-extra! (fn [extra]
(if db-based?
(do
(when block
(debounce-set-last-visit-scale! (:block pdf-current) (:scale extra))
(debounce-set-last-visit-page! (:block pdf-current) (:page extra)))
(set-hls-state! #(merge % {:extra extra}))))]
;; current pdf effects
(when-not db-based?
(hooks/use-effect!
(fn []
(hooks/use-effect!
(fn []
(when file-based?
;; ensure ref page
(when pdf-current
(pdf-assets/file-based-ensure-ref-page! pdf-current)))
[pdf-current]))
;; load highlights
(if db-based?
(hooks/use-effect!
(fn []
(when file-based?
;; load highlights
(when pdf-current
(let [pdf-block (:block pdf-current)]
(p/let [data (db-async/<get-pdf-annotations repo (:db/id pdf-block))
@@ -970,53 +972,53 @@
1))
(set-initial-scale! (get-last-visit-scale pdf-block))
(set-hls-state! {:initial-hls highlights :latest-hls highlights :loaded true})))))
[pdf-current])
(hooks/use-effect!
(fn []
(p/catch
(p/let [data (pdf-assets/file-based-load-hls-data$ pdf-current)
{:keys [highlights extra]} data]
(set-initial-page! (or (when-let [page (:page extra)]
(util/safe-parse-int page)) 1))
(set-initial-scale! (or (:scale extra) "auto"))
(set-hls-state! {:initial-hls highlights :latest-hls highlights :extra extra :loaded true}))
#())
[pdf-current])
;; error
(fn [^js e]
(js/console.error "[load hls error]" e)
(let [msg (str (util/format "Error: failed to load the highlights file: \"%s\". \n"
(:hls-file pdf-current))
e)]
(notification/show! msg :error)
(set-hls-state! {:loaded true :error e}))))
;; cancel
#())
[hls-file]))
(hooks/use-effect!
(fn []
(if file-based?
(-> (p/let [data (pdf-assets/file-based-load-hls-data$ pdf-current)
{:keys [highlights extra]} data]
(set-initial-page! (or (when-let [page (:page extra)]
(util/safe-parse-int page)) 1))
(set-initial-scale! (or (:scale extra) "auto"))
(set-hls-state! {:initial-hls highlights :latest-hls highlights :extra extra :loaded true}))
(p/catch
(fn [^js e]
(js/console.error "[load hls error]" e)
(let [msg (str (util/format "Error: failed to load the highlights file: \"%s\". \n"
(:hls-file pdf-current))
e)]
(notification/show! msg :error)
(set-hls-state! {:loaded true :error e})))))
;; for db-based, just mark loaded
(set-hls-state! {:loaded true}))
#())
[hls-file pdf-current])
;; cache highlights
(when-not db-based?
(let [persist-hls-data!
(hooks/use-callback
(util/debounce
(fn [latest-hls extra]
(pdf-assets/file-based-persist-hls-data$
pdf-current latest-hls extra))
4000) [pdf-current])]
(let [persist-hls-data!
(hooks/use-callback
(util/debounce
(fn [latest-hls extra]
(pdf-assets/file-based-persist-hls-data$
pdf-current latest-hls extra))
4000) [pdf-current])]
(hooks/use-effect!
(fn []
(hooks/use-effect!
(fn []
;; persist highlights
(when file-based?
(when (= :completed (:status loader-state))
(p/catch
(when-not (:error hls-state)
(p/do! (persist-hls-data! (:latest-hls hls-state) (:extra hls-state))))
(-> (when-not (:error hls-state)
(p/do! (persist-hls-data! (:latest-hls hls-state) (:extra hls-state))))
(p/catch
(fn [e]
(js/console.error "[write hls error]" e))))))
#())
;; write hls file error
(fn [e]
(js/console.error "[write hls error]" e)))))
[(:latest-hls hls-state) (:extra hls-state)])))
[(:latest-hls hls-state) (:extra hls-state)]))
;; load document
(hooks/use-effect!
@@ -1033,7 +1035,6 @@
:supportsMouseWheelZoomCtrlKey true
:supportsMouseWheelZoomMetaKey true}]
(set-loader-state! {:status :loading})
(-> (get-doc$ (clj->js opts))
(p/then (fn [doc]
(set-loader-state! {:pdf-document doc :status :completed})))
@@ -1041,6 +1042,7 @@
#()))
[url doc-password])
;; handle load errors
(hooks/use-effect!
(fn []
(when-let [error (:error loader-state)]
@@ -1092,17 +1094,15 @@
initial-error (:error hls-state)]
(if (= status-doc :loading)
[:div.flex.justify-center.items-center.h-screen.text-gray-500.text-lg
svg/loading]
[:div.flex.justify-center.items-center.h-screen.text-gray-500.text-lg svg/loading]
(when-let [pdf-document (and (:loaded hls-state) (:pdf-document loader-state))]
[(rum/with-key (pdf-viewer
url pdf-document
{:identity identity
:filename filename
:initial-hls initial-hls
:initial-page initial-page
{:identity identity
:filename filename
:pdf-current pdf-current
:initial-hls initial-hls
:initial-page initial-page
:initial-scale initial-scale
:initial-error initial-error}
{:set-dirty-hls! set-dirty-hls!

13
src/main/frontend/extensions/pdf/toolbar.cljs
View File

@@ -477,7 +477,7 @@
(pdf-highlights-list viewer))]]]))
(rum/defc ^:large-vars/cleanup-todo pdf-toolbar
[^js viewer {:keys [on-external-window!]}]
[^js viewer {:keys [on-external-window! pdf-current]}]
(let [[area-mode?, set-area-mode!] (use-atom *area-mode?)
[outline-visible?, set-outline-visible!] (rum/use-state false)
[finder-visible?, set-finder-visible!] (rum/use-state false)
@@ -490,6 +490,8 @@
group-id (.-$groupIdentity viewer)
in-system-window? (.-$inSystemWindow viewer)
doc (pdf-windows/resolve-own-document viewer)
;; asset block container for db mode
asset-block (:block pdf-current)
dispatch-extra-state!
(fn []
(js/setTimeout
@@ -594,10 +596,11 @@
(svg/search2 19)]
;; annotations
[:a.button
{:title "Annotations page"
:on-click #(pdf-assets/goto-annotations-page! (:pdf/current @state/state))}
(svg/annotations 16)]
(when asset-block
[:a.button
{:title "Annotations page"
:on-click #(pdf-assets/goto-annotations-page! (:pdf/current @state/state))}
(svg/annotations 16)])
;; system window
[:a.button

5
src/main/frontend/format/block.cljs
View File

@@ -18,12 +18,11 @@
(defn extract-blocks
"Wrapper around logseq.graph-parser.block/extract-blocks that adds in system state
and handles unexpected failure."
[blocks content format {:keys [page-name parse-block]}]
[blocks content format {:keys [page-name]}]
(let [repo (state/get-current-repo)]
(try
(let [blocks (gp-block/extract-blocks blocks content format
{:user-config (state/get-config)
:parse-block parse-block
:block-pattern (config/get-block-pattern format)
:db (db/get-db repo)
:date-formatter (state/get-date-formatter)
@@ -86,7 +85,7 @@ and handles unexpected failure."
(:logseq.property.node/display-type block))
[block]
(let [ast (format/to-edn title format parse-config)]
(extract-blocks ast title format {:parse-block block})))
(extract-blocks ast title format {})))
new-block (first blocks)
block (cond-> (merge block new-block)
(> (count blocks) 1)

6
src/main/frontend/fs.cljs
View File

@@ -109,6 +109,7 @@
;; (js/alert "Current file can't be saved! Please copy its content to your local file system and click the refresh button.")
))))))
;; read-file should return string on all platforms
(defn read-file
([dir path]
(let [fs (get-fs dir)
@@ -119,6 +120,11 @@
([dir path options]
(protocol/read-file (get-fs dir) dir path options)))
(defn read-file-raw
[dir path & {:as options}]
(let [fs (get-fs dir)]
(protocol/read-file-raw fs dir path options)))
(defn rename!
"Rename files, incoming relative path, converted to absolute path"
[repo old-path new-path]

15
src/main/frontend/fs/memory_fs.cljs
View File

@@ -30,7 +30,6 @@
(p/recur result (concat (rest dirs) dir-content)))))]
result))
(defn- <ensure-dir!
"dir is path, without memory:// prefix for simplicity"
[dir]
@@ -72,6 +71,15 @@
(p/do! (js/window.pfs.mkdir (first remains))
(p/recur (rest remains)))))))
(defn- read-file-aux
[dir path {:keys [text?]
:as options}]
(p/let [fpath (path/url-to-path (path/path-join dir path))
result (js/window.pfs.readFile fpath (clj->js options))]
(if text?
(.toString ^js result)
result)))
(defrecord MemoryFs []
protocol/Fs
(mkdir! [_this dir]
@@ -106,8 +114,9 @@
(let [fpath (path/url-to-path dir)]
(js/window.workerThread.rimraf fpath)))
(read-file [_this dir path options]
(let [fpath (path/url-to-path (path/path-join dir path))]
(js/window.pfs.readFile fpath (clj->js options))))
(read-file-aux dir path (assoc options :text? true)))
(read-file-raw [_this dir path options]
(read-file-aux dir path options))
(write-file! [_this _repo dir rpath content _opts]
(p/let [fpath (path/url-to-path (path/path-join dir rpath))
containing-dir (path/parent fpath)

10
src/main/frontend/fs/node.cljs
View File

@@ -9,8 +9,8 @@
[frontend.util :as util]
[goog.object :as gobj]
[lambdaisland.glogi :as log]
[promesa.core :as p]
[logseq.common.path :as path]))
[logseq.common.path :as path]
[promesa.core :as p]))
(defn- <contents-matched?
[disk-content db-content]
@@ -95,6 +95,12 @@
(path/path-join dir path))]
(ipc/ipc "readFile" path)))
(read-file-raw [_this dir path _options]
(let [path (if (nil? dir)
path
(path/path-join dir path))]
(ipc/ipc "readFileRaw" path)))
(write-file! [this repo dir path content opts]
(p/let [fpath (path/path-join dir path)
stat (p/catch

7
src/main/frontend/fs/protocol.cljs
View File

@@ -10,12 +10,13 @@
(readdir [this dir]
"Read directory and return list of files. Won't read file out.
Used by initial watcher, version files of Logseq Sync.
=> [string]")
(unlink! [this repo path opts])
;; FIXME(andelf): remove this API? since the only usage is plugin API
(rmdir! [this dir])
(read-file [this dir path opts])
(read-file-raw [this dir path opts])
(write-file! [this repo dir path content opts])
(rename! [this repo old-path new-path])
(copy! [this repo old-path new-path])
@@ -26,12 +27,12 @@
(open-dir [this dir]
"Open a directory and return the files in it.
Used by open a new graph.
=> {:path string :files [{...}]}")
(get-files [this dir]
"Almost the same as `open-dir`. For returning files.
Used by re-index/refresh.
=> [{:path string :content string}] (absolute path)")
(watch-dir! [this dir options])
(unwatch-dir! [this dir]))

2
src/main/frontend/fs/watcher_handler.cljs
View File

@@ -95,7 +95,7 @@
(and (= "change" type)
(= dir repo-dir)
(not (common-config/local-asset? path)))
(not (common-config/local-relative-asset? path)))
(handle-add-and-change! repo path content db-content ctime mtime (not global-dir)) ;; no backup for global dir
(and (= "unlink" type)

2
src/main/frontend/handler.cljs
View File

@@ -18,7 +18,9 @@
[frontend.error :as error]
[frontend.handler.command-palette :as command-palette]
[frontend.handler.db-based.vector-search-flows :as vector-search-flows]
[frontend.handler.e2ee]
[frontend.handler.events :as events]
[frontend.handler.events.rtc]
[frontend.handler.events.ui]
[frontend.handler.file-based.events]
[frontend.handler.file-based.file :as file-handler]

117
src/main/frontend/handler/assets.cljs
View File

@@ -1,6 +1,7 @@
(ns ^:no-doc frontend.handler.assets
(:require [cljs-http-missionary.client :as http]
[clojure.string :as string]
[frontend.common.crypt :as crypt]
[frontend.common.missionary :as c.m]
[frontend.common.thread-api :as thread-api :refer [def-thread-api]]
[frontend.config :as config]
@@ -10,6 +11,7 @@
[logseq.common.config :as common-config]
[logseq.common.path :as path]
[logseq.common.util :as common-util]
[logseq.db :as ldb]
[logseq.db.frontend.asset :as db-asset]
[medley.core :as medley]
[missionary.core :as m]
@@ -87,8 +89,7 @@
(defn normalize-asset-resource-url
"try to convert resource file to url asset link"
[path]
(let [protocol-link? (->> #{"file://" "http://" "https://" "assets://"}
(some #(string/starts-with? (string/lower-case path) %)))]
(let [protocol-link? (common-config/protocol-path? path)]
(cond
protocol-link?
path
@@ -132,7 +133,7 @@
(defn <make-data-url
[path]
(let [repo-dir (config/get-repo-dir (state/get-current-repo))]
(p/let [binary (fs/read-file repo-dir path {})
(p/let [binary (fs/read-file-raw repo-dir path {})
blob (js/Blob. (array binary) (clj->js {:type "image"}))]
(when blob (js/URL.createObjectURL blob)))))
@@ -152,35 +153,39 @@
(defn <make-asset-url
"Make asset URL for UI element, to fill img.src"
[path] ;; path start with "/assets"(editor) or compatible for "../assets"(whiteboards)
(if config/publishing?
;; Relative path needed since assets are not under '/' if published graph is not under '/'
(string/replace-first path #"^/" "")
(let [repo (state/get-current-repo)
repo-dir (config/get-repo-dir repo)
;; Hack for path calculation
path (string/replace path #"^(\.\.)?/" "./")
full-path (path/path-join repo-dir path)
data-url? (string/starts-with? path "data:")]
(cond
data-url?
path ;; just return the original
([path] (<make-asset-url path (try (js/URL. path) (catch :default _ nil))))
([path ^js js-url]
;; path start with "/assets"(editor) or compatible for "../assets"(whiteboards)
(if config/publishing?
;; Relative path needed since assets are not under '/' if published graph is not under '/'
(string/replace-first path #"^/" "")
(let [repo (state/get-current-repo)
repo-dir (config/get-repo-dir repo)
local-asset? (common-config/local-relative-asset? path)
;; Hack for path calculation
path (string/replace path #"^(\.\.)?/" "./")
js-url? (not (nil? js-url))]
(cond
js-url?
path ;; just return the original
(and (alias-enabled?)
(check-alias-path? path))
(resolve-asset-real-path-url (state/get-current-repo) path)
(and (alias-enabled?)
(check-alias-path? path))
(resolve-asset-real-path-url (state/get-current-repo) path)
(util/electron?)
;; fullpath will be encoded
(path/prepend-protocol "file:" full-path)
(util/electron?)
(let [full-path (if local-asset?
(path/path-join repo-dir path) path)]
;; fullpath will be encoded
(path/prepend-protocol "file:" full-path))
;(mobile-util/native-platform?)
;(mobile-util/convert-file-src full-path)
;(mobile-util/native-platform?)
;(mobile-util/convert-file-src full-path)
(config/db-based-graph? (state/get-current-repo)) ; memory fs
(p/let [binary (fs/read-file repo-dir path {})
blob (js/Blob. (array binary) (clj->js {:type "image"}))]
(when blob (js/URL.createObjectURL blob)))))))
(config/db-based-graph? (state/get-current-repo)) ; memory fs
(p/let [binary (fs/read-file-raw repo-dir path {})
blob (js/Blob. (array binary) (clj->js {:type "image"}))]
(when blob (js/URL.createObjectURL blob))))))))
(defn get-file-checksum
[^js/Blob file]
@@ -193,7 +198,7 @@
(p/let [result (p/catch (fs/readdir path {:path-only? true})
(constantly nil))]
(p/all (map (fn [path]
(p/let [data (fs/read-file path "" {})]
(p/let [data (fs/read-file-raw path "" {})]
(let [path' (util/node-path.join "assets" (util/node-path.basename path))]
[path' data]))) result)))))
@@ -221,7 +226,7 @@
(let [repo-dir (config/get-repo-dir repo)
file-path (path/path-join common-config/local-assets-dir
(str asset-block-id "." asset-type))]
(fs/read-file repo-dir file-path {})))
(fs/read-file-raw repo-dir file-path {})))
(defn <get-asset-file-metadata
[repo asset-block-id asset-type]
@@ -243,6 +248,18 @@
:assets/asset-file-write-finish
(fn [m] (assoc-in m [repo asset-block-id-str] (common-util/time-ms)))))))
(comment
;; en/decrypt assets
(def repo (state/get-current-repo))
(p/let [aes-key (crypt/<generate-aes-key)
asset (<read-asset repo "6903201e-9573-4914-ae88-7d3f1d095d1f" "png")
encrypted-asset (crypt/<encrypt-uint8array aes-key asset)
decrypted-asset (crypt/<decrypt-uint8array aes-key encrypted-asset)]
(def asset asset)
(def xxxx encrypted-asset)
(prn :decrypted (.-length decrypted-asset)
:origin (.-length asset))))
(defn <unlink-asset
[repo asset-block-id asset-type]
(let [file-path (path/path-join (config/get-repo-dir repo)
@@ -251,14 +268,18 @@
(p/catch (fs/unlink! repo file-path {}) (constantly nil))))
(defn new-task--rtc-upload-asset
[repo asset-block-uuid-str asset-type checksum put-url]
[repo aes-key asset-block-uuid-str asset-type checksum put-url]
(assert (and asset-type checksum))
(m/sp
(let [asset-file (c.m/<? (<read-asset repo asset-block-uuid-str asset-type))
asset-file* (if (not aes-key)
asset-file
(ldb/write-transit-str
(c.m/<? (crypt/<encrypt-uint8array aes-key asset-file))))
*progress-flow (atom nil)
http-task (http/put put-url {:headers {"x-amz-meta-checksum" checksum
"x-amz-meta-type" asset-type}
:body asset-file
:body asset-file*
:with-credentials? false
:*progress-flow *progress-flow})]
(c.m/run-task :upload-asset-progress
@@ -273,7 +294,7 @@
{:ex-data {:type :rtc.exception/upload-asset-failed :data (dissoc r :body)}})))))
(defn new-task--rtc-download-asset
[repo asset-block-uuid-str asset-type get-url]
[repo aes-key asset-block-uuid-str asset-type get-url]
(m/sp
(let [*progress-flow (atom nil)
http-task (http/get get-url {:with-credentials? false
@@ -291,8 +312,22 @@
(let [{:keys [status body] :as r} (m/? http-task)]
(if-not (http/unexceptional-status? status)
{:ex-data {:type :rtc.exception/download-asset-failed :data (dissoc r :body)}}
(do (c.m/<? (<write-asset repo asset-block-uuid-str asset-type body))
nil)))
(let [asset-file
(if (not aes-key)
body
(try
(let [asset-file-untransited (ldb/read-transit-str (.decode (js/TextDecoder.) body))]
(c.m/<? (crypt/<decrypt-uint8array aes-key asset-file-untransited)))
(catch js/SyntaxError _
body)
(catch :default e
;; if decrypt failed, write origin-body
(if (= "decrypt-uint8array" (ex-message e))
body
(throw e)))))]
(c.m/<? (<write-asset repo asset-block-uuid-str asset-type asset-file))
nil)))
(catch Cancelled e
(progress-canceler)
(throw e))))))
@@ -310,12 +345,16 @@
(<get-asset-file-metadata repo asset-block-id asset-type))
(def-thread-api :thread-api/rtc-upload-asset
[repo asset-block-uuid-str asset-type checksum put-url]
(new-task--rtc-upload-asset repo asset-block-uuid-str asset-type checksum put-url))
[repo exported-aes-key asset-block-uuid-str asset-type checksum put-url]
(m/sp
(let [aes-key (when exported-aes-key (c.m/<? (crypt/<import-aes-key exported-aes-key)))]
(m/? (new-task--rtc-upload-asset repo aes-key asset-block-uuid-str asset-type checksum put-url)))))
(def-thread-api :thread-api/rtc-download-asset
[repo asset-block-uuid-str asset-type get-url]
(new-task--rtc-download-asset repo asset-block-uuid-str asset-type get-url))
[repo exported-aes-key asset-block-uuid-str asset-type get-url]
(m/sp
(let [aes-key (when exported-aes-key (c.m/<? (crypt/<import-aes-key exported-aes-key)))]
(m/? (new-task--rtc-download-asset repo aes-key asset-block-uuid-str asset-type get-url)))))
(comment
;; read asset

10
src/main/frontend/handler/db_based/export.cljs
View File

@@ -17,7 +17,7 @@
(state/get-current-repo)
{:export-type :block :block-id [:block/uuid block-uuid]})
pull-data (with-out-str (pprint/pprint result))]
(when-not (= :export-edn-error result)
(when-not (:export-edn-error result)
(.writeText js/navigator.clipboard pull-data)
(println pull-data)
(notification/show! "Copied block's data!" :success)))
@@ -30,7 +30,7 @@
:rows rows
:group-by? group-by?})
pull-data (with-out-str (pprint/pprint result))]
(when-not (= :export-edn-error result)
(when-not (:export-edn-error result)
(.writeText js/navigator.clipboard pull-data)
(println pull-data)
(notification/show! "Copied view nodes' data!" :success))))
@@ -41,7 +41,7 @@
(state/get-current-repo)
{:export-type :page :page-id page-id})
pull-data (with-out-str (pprint/pprint result))]
(when-not (= :export-edn-error result)
(when-not (:export-edn-error result)
(.writeText js/navigator.clipboard pull-data)
(println pull-data)
(notification/show! "Copied page's data!" :success)))
@@ -52,7 +52,7 @@
(state/get-current-repo)
{:export-type :graph-ontology})
pull-data (with-out-str (pprint/pprint result))]
(when-not (= :export-edn-error result)
(when-not (:export-edn-error result)
(.writeText js/navigator.clipboard pull-data)
(println pull-data)
(js/console.log (str "Exported " (count (:classes result)) " classes and "
@@ -73,7 +73,7 @@
{:export-type :graph
:graph-options {:include-timestamps? true}})
pull-data (with-out-str (pprint/pprint result))]
(when-not (= :export-edn-error result)
(when-not (:export-edn-error result)
(let [data-str (some->> pull-data
js/encodeURIComponent
(str "data:text/edn;charset=utf-8,"))

25
src/main/frontend/handler/db_based/rtc.cljs
View File

@@ -49,7 +49,10 @@
(->
(when (not= result :timeout)
(assert (some? download-info-s3-url) result)
(state/<invoke-db-worker :thread-api/rtc-download-graph-from-s3 graph-uuid graph-name download-info-s3-url))
(p/let [r (state/<invoke-db-worker :thread-api/rtc-download-graph-from-s3
graph-uuid graph-name download-info-s3-url)]
(when (instance? ExceptionInfo r)
(log/error :rtc-download-graph-from-s3 r))))
(p/finally
#(state/set-state! :rtc/downloading-graph-uuid nil)))))
@@ -160,12 +163,14 @@
(defn <rtc-invite-email
[graph-uuid email]
(let [token (state/get-auth-id-token)]
(->
(p/do!
(state/<invoke-db-worker :thread-api/rtc-grant-graph-access
token (str graph-uuid) [] [email])
(notification/show! "Invitation sent!" :success))
(p/catch (fn [e]
(notification/show! "Something wrong, please try again." :error)
(js/console.error e))))))
(let [token (state/get-auth-id-token)
user-uuid (user-handler/user-uuid)]
(when (and user-uuid token)
(->
(p/do!
(state/<invoke-db-worker :thread-api/rtc-grant-graph-access
token (str graph-uuid) user-uuid email)
(notification/show! "Invitation sent!" :success))
(p/catch (fn [e]
(notification/show! "Something wrong, please try again." :error)
(js/console.error e)))))))

82
src/main/frontend/handler/e2ee.cljs Normal file
View File

@@ -0,0 +1,82 @@
(ns frontend.handler.e2ee
"rtc E2EE related fns"
(:require [electron.ipc :as ipc]
[frontend.common.crypt :as crypt]
[frontend.common.thread-api :refer [def-thread-api]]
[frontend.mobile.secure-storage :as secure-storage]
[frontend.state :as state]
[frontend.util :as util]
[lambdaisland.glogi :as log]
[promesa.core :as p]))
(def ^:private save-op :keychain/save-e2ee-password)
(def ^:private get-op :keychain/get-e2ee-password)
(def ^:private delete-op :keychain/delete-e2ee-password)
(defn- <keychain-save!
[key encrypted-text]
(cond
(util/electron?)
(ipc/ipc save-op key encrypted-text)
(util/capacitor?)
(secure-storage/<set-item! key encrypted-text)
:else
(p/resolved nil)))
(defn- <keychain-get
[key]
(cond
(util/electron?)
(ipc/ipc get-op key)
(util/capacitor?)
(secure-storage/<get-item key)
:else
(p/resolved nil)))
(defn- <keychain-delete!
[key]
(cond
(util/electron?)
(ipc/ipc delete-op key)
(util/capacitor?)
(secure-storage/<remove-item! key)
:else
(p/resolved nil)))
(def-thread-api :thread-api/request-e2ee-password
[]
(p/let [password-promise (state/pub-event! [:rtc/request-e2ee-password])
password password-promise]
{:password password}))
(defn- <decrypt-user-e2ee-private-key
[encrypted-private-key]
(->
(p/let [private-key-promise (state/pub-event! [:rtc/decrypt-user-e2ee-private-key encrypted-private-key])
private-key private-key-promise]
(crypt/<export-private-key private-key))
(p/catch (fn [e]
(log/error :<decrypt-user-e2ee-private-key e)
e))))
(def-thread-api :thread-api/decrypt-user-e2ee-private-key
[encrypted-private-key]
(<decrypt-user-e2ee-private-key encrypted-private-key))
(def-thread-api :thread-api/native-save-e2ee-password
[encrypted-text]
(<keychain-save! "logseq-encrypted-password" encrypted-text))
(def-thread-api :thread-api/native-get-e2ee-password
[]
(<keychain-get "logseq-encrypted-password"))
(def-thread-api :thread-api/native-delete-e2ee-password
[]
(<keychain-delete! "logseq-encrypted-password"))

41
src/main/frontend/handler/events/rtc.cljs Normal file
View File

@@ -0,0 +1,41 @@
(ns frontend.handler.events.rtc
"RTC events"
(:require [frontend.common.crypt :as crypt]
[frontend.components.e2ee :as e2ee]
[frontend.handler.events :as events]
[frontend.state :as state]
[lambdaisland.glogi :as log]
[logseq.shui.ui :as shui]
[promesa.core :as p]))
(defmethod events/handle :rtc/decrypt-user-e2ee-private-key [[_ encrypted-private-key]]
(let [private-key-promise (p/deferred)
refresh-token (state/get-auth-refresh-token)]
(shui/dialog-close-all!)
(->
(p/let [{:keys [password]} (state/<invoke-db-worker :thread-api/get-e2ee-password refresh-token)
private-key (crypt/<decrypt-private-key password encrypted-private-key)]
(p/resolve! private-key-promise private-key))
(p/catch
(fn [error]
(log/error :read-e2ee-password-failed error)
(shui/dialog-open!
#(e2ee/e2ee-password-to-decrypt-private-key encrypted-private-key private-key-promise refresh-token)
{:auto-width? true
:content-props {:onPointerDownOutside #(.preventDefault %)}
:on-close (fn []
(p/reject! private-key-promise (ex-info "input E2EE password cancelled" {}))
(shui/dialog-close!))}))))
private-key-promise))
(defmethod events/handle :rtc/request-e2ee-password [[_]]
(let [password-promise (p/deferred)]
(shui/dialog-close-all!)
(shui/dialog-open!
#(e2ee/e2ee-request-new-password password-promise)
{:auto-width? true
:content-props {:onPointerDownOutside #(.preventDefault %)}
:on-close (fn []
(p/reject! password-promise (ex-info "cancelled" {}))
(shui/dialog-close!))})
password-promise))

48
src/main/frontend/handler/plugin.cljs
View File

@@ -153,7 +153,7 @@
(p/then (fn [manifests]
(let [mft (some #(when (= (:id %) id) %) manifests)
opts (merge (dissoc pkg :logger) mft)]
;;TODO: (throw (js/Error. [:not-found-in-marketplace id]))
;;TODO: (throw (js/Error. [:not-found-in-marketplace id]))
(if (util/electron?)
(ipc/ipc :updateMarketPlugin opts)
(plugin-common-handler/async-install-or-update-for-web! opts)))
@@ -229,7 +229,7 @@
(p/then
(.reload pl)
#(do
;;(if theme (select-a-plugin-theme id))
;;(if theme (select-a-plugin-theme id))
(when (not (util/electron?))
(set! (.-version (.-options pl)) (:version web-pkg))
(set! (.-webPkg (.-options pl)) (bean/->js web-pkg))
@@ -441,13 +441,19 @@
([type *providers] (create-local-renderer-getter type *providers false))
([type *providers many?]
(fn [key]
(when-let [key (and (seq @*providers) key (keyword key))]
(when-let [rs (->> @*providers
(map (fn [pid] (state/get-plugin-resource pid type key)))
(remove nil?)
(flatten)
(seq))]
(if many? rs (first rs)))))))
(when (seq @*providers)
(if key
(when-let [rs (->> @*providers
(map (fn [pid] (state/get-plugin-resource pid type key)))
(remove nil?)
(flatten)
(seq))]
(if many? rs (first rs)))
(->> @*providers
(mapcat (fn [pid]
(some-> (state/get-plugin-resources-with-type pid type)
(vals))))
(seq)))))))
(defonce *fenced-code-providers (atom #{}))
(def register-fenced-code-renderer
@@ -469,11 +475,13 @@
(create-local-renderer-getter
:extensions-enhancers *extensions-enhancer-providers true))
(def *route-renderer-providers (atom #{}))
(defonce *route-renderer-providers (atom #{}))
(def register-route-renderer
;; [pid key payload]
(create-local-renderer-register
:route-renderers *route-renderer-providers))
(def get-route-renderers
;; [key] optional
(create-local-renderer-getter
:route-renderers *route-renderer-providers true))
@@ -496,9 +504,9 @@
(defn update-plugin-settings-state
[id settings]
(state/set-state! [:plugin/installed-plugins id :settings]
;; TODO: force settings related ui reactive
;; Sometimes toggle to `disable` not working
;; But related-option data updated?
;; TODO: force settings related ui reactive
;; Sometimes toggle to `disable` not working
;; But related-option data updated?
(assoc settings :disabled (boolean (:disabled settings)))))
(defn open-settings-file-in-default-app!
@@ -896,8 +904,8 @@
(.on "theme-selected" (fn [^js theme]
(let [theme (bean/->clj theme)
theme (assets-theme-to-file theme)
url (:url theme)
mode (or (:mode theme) (state/sub :ui/theme))]
url (:url theme)
mode (or (:mode theme) (state/sub :ui/theme))]
(when mode
(state/set-custom-theme! mode theme)
(state/set-theme-mode! mode))
@@ -909,7 +917,7 @@
custom-theme (dissoc themes :mode)
mode (:mode themes)]
(state/set-custom-theme! {:light (if (nil? (:light custom-theme)) {:mode "light"} (:light custom-theme))
:dark (if (nil? (:dark custom-theme)) {:mode "dark"} (:dark custom-theme))})
:dark (if (nil? (:dark custom-theme)) {:mode "dark"} (:dark custom-theme))})
(state/set-theme-mode! mode))))
(.on "settings-changed" (fn [id ^js settings]
@@ -926,9 +934,9 @@
(when-let [end (and (some-> v (.-o) (.-disabled) (not))
(.-e v))]
(when (and (number? end)
;; valid end time
;; valid end time
(> end 0)
;; greater than 6s
;; greater than 6s
(> (- end (.-s v)) 6000))
v))))
((fn [perfs]
@@ -947,9 +955,9 @@
(p/then
(fn [plugins-async]
;; true indicate for preboot finished
;; true indicate for preboot finished
(state/set-state! :plugin/indicator-text true)
;; wait for the plugin register async messages
;; wait for the plugin register async messages
(js/setTimeout
(fn []
(some-> (seq plugins-async)

3
src/main/frontend/handler/user.cljs
View File

@@ -119,8 +119,7 @@
(let [refresh-token (js/localStorage.getItem refresh-token-key)]
(when (and refresh-token (not= refresh-token "undefined"))
(state/set-auth-refresh-token refresh-token)
(js/localStorage.setItem "refresh-token" refresh-token)))))
)
(js/localStorage.setItem "refresh-token" refresh-token))))))
(defn- clear-tokens
([]

57
src/main/frontend/mobile/secure_storage.cljs Normal file
View File

@@ -0,0 +1,57 @@
(ns frontend.mobile.secure-storage
"Wrapper around the Capacitor secure storage plugin."
(:require ["@aparajita/capacitor-secure-storage" :refer [SecureStorage]]
[frontend.mobile.util :as mobile-util]
[lambdaisland.glogi :as log]
[promesa.core :as p]))
(defonce ^:private *initialized? (atom false))
(def ^:private key-prefix "logseq.e2ee.")
(defn- <ensure-initialized!
[]
(cond
(not (mobile-util/native-platform?))
(p/resolved false)
@*initialized?
(p/resolved true)
:else
(-> (p/let [_ (.setKeyPrefix SecureStorage key-prefix)]
(reset! *initialized? true))
(p/catch (fn [e]
(log/error ::init {:error e})
(throw e)))))) ;; propagate so callers can fallback if needed
(defn <set-item!
[key value]
(if (mobile-util/native-platform?)
(-> (p/let [_ (<ensure-initialized!)
_ (.setItem SecureStorage key value)]
true)
(p/catch (fn [e]
(log/error ::set-item {:error e})
(throw e))))
(p/resolved false)))
(defn <get-item
[key]
(if (mobile-util/native-platform?)
(-> (p/let [_ (<ensure-initialized!)]
(.getItem SecureStorage key))
(p/catch (fn [e]
(log/error ::get-item {:error e})
(throw e))))
(p/resolved nil)))
(defn <remove-item!
[key]
(if (mobile-util/native-platform?)
(-> (p/let [_ (<ensure-initialized!)
_ (.removeItem SecureStorage key)]
true)
(p/catch (fn [e]
(log/error ::remove-item {:error e})
(throw e))))
(p/resolved false)))

12
src/main/frontend/persist_db/browser.cljs
View File

@@ -119,7 +119,11 @@
(p/do!
(reload-app-if-old-db-worker-exists)
(let [worker-url (if config/publishing? "static/js/db-worker.js" "js/db-worker.js")
worker (js/Worker. (str worker-url "?electron=" (util/electron?) "&publishing=" config/publishing?))
worker (js/Worker.
(str worker-url
"?electron=" (util/electron?)
"&capacitor=" (util/capacitor?)
"&publishing=" config/publishing?))
_ (set-worker-fs worker)
wrapped-worker* (Comlink/wrap worker)
wrapped-worker (fn [qkw direct-pass? & args]
@@ -166,7 +170,11 @@
[]
(when-not util/node-test?
(let [worker-url "js/inference-worker.js"
^js worker (js/SharedWorker. (str worker-url "?electron=" (util/electron?) "&publishing=" config/publishing?))
^js worker (js/SharedWorker.
(str worker-url
"?electron=" (util/electron?)
"&capacitor=" (util/capacitor?)
"&publishing=" config/publishing?))
^js port (.-port worker)
wrapped-worker (Comlink/wrap port)
t1 (util/time-ms)]

28
src/main/frontend/rum.cljs
View File

@@ -6,7 +6,7 @@
[clojure.walk :as w]
[daiquiri.interpreter :as interpreter]
[logseq.shui.hooks :as hooks]
[rum.core :refer [use-state] :as rum]))
[rum.core :as rum]))
;; copy from https://github.com/priornix/antizer/blob/35ba264cf48b84e6597743e28b3570d8aa473e74/src/antizer/core.cljs
@@ -66,30 +66,8 @@
(bean/->js (map-keys->camel-case new-options :html-props true))
new-children)))))
(defn use-atom-fn
[a getter-fn setter-fn]
(let [[val set-val] (use-state (getter-fn @a))]
(hooks/use-effect!
(fn []
(let [id (str (random-uuid))]
(add-watch a id (fn [_ _ prev-state next-state]
(let [prev-value (getter-fn prev-state)
next-value (getter-fn next-state)]
(when-not (= prev-value next-value)
(set-val next-value)))))
#(remove-watch a id)))
[])
[val #(swap! a setter-fn %)]))
(defn use-atom
"(use-atom my-atom)"
[a]
(use-atom-fn a identity (fn [_ v] v)))
(defn use-atom-in
[a ks]
(let [ks (if (keyword? ks) [ks] ks)]
(use-atom-fn a #(get-in % ks) (fn [a' v] (assoc-in a' ks v)))))
(def use-atom hooks/use-atom)
(def use-atom-in hooks/use-atom-in)
(defn use-mounted
[]

4
src/main/frontend/state.cljs
View File

@@ -286,7 +286,7 @@
:reactive/query-dbs {}
;; login, userinfo, token, ...
:auth/refresh-token (storage/get "refresh-token")
:auth/refresh-token (some-> (storage/get "refresh-token") str)
:auth/access-token nil
:auth/id-token nil
@@ -2149,7 +2149,7 @@ Similar to re-frame subscriptions"
(sub :auth/id-token))
(defn get-auth-refresh-token []
(:auth/refresh-token @state))
(str (:auth/refresh-token @state)))
(defn set-file-sync-manager [graph-uuid v]
(when (and graph-uuid v)

130
src/main/frontend/worker/crypt.cljs
View File

@@ -1,130 +0,0 @@
(ns frontend.worker.crypt
"Fns to en/decrypt some block attrs"
(:require [datascript.core :as d]
[frontend.common.thread-api :refer [def-thread-api]]
[frontend.worker.state :as worker-state]
[logseq.db :as ldb]
[promesa.core :as p]))
(defonce ^:private encoder (new js/TextEncoder "utf-8"))
(comment (defonce ^:private decoder (new js/TextDecoder "utf-8")))
(defn string=>arraybuffer
[s]
(.encode encoder s))
(defn <rsa-encrypt
"Return an arraybuffer"
[message public-key]
(assert (string? message))
(let [data (string=>arraybuffer message)]
(js/crypto.subtle.encrypt
#js{:name "RSA-OAEP"}
public-key
data)))
(comment
(defn <decrypt
[cipher-text private-key]
(p/let [result (js/crypto.subtle.decrypt
#js{:name "RSA-OAEP"}
private-key
cipher-text)]
(.decode decoder result))))
(comment
(defn <aes-encrypt
[message aes-key]
(p/let [data (.encode encoder message)
iv (js/crypto.getRandomValues (js/Uint8Array. 12))
ciphertext (js/crypto.subtle.encrypt
#js{:name "AES-GCM" :iv iv}
aes-key
data)]
{:ciphertext ciphertext
:iv iv})))
(comment
(defn <aes-decrypt
[encrypted-data aes-key]
(p/let [{:keys [ciphertext iv]} encrypted-data
decrypted (js/crypto.subtle.decrypt
#js{:name "AES-GCM" :iv iv}
aes-key
ciphertext)]
(.decode decoder decrypted))))
(defonce ^:private key-algorithm
#js{:name "RSA-OAEP"
:modulusLength 4096
:publicExponent (new js/Uint8Array #js[1 0 1])
:hash "SHA-256"})
(defn <gen-key-pair
[]
(p/let [result (js/crypto.subtle.generateKey
key-algorithm
true
#js["encrypt" "decrypt"])]
(js->clj result :keywordize-keys true)))
(defonce ^:private aes-key-algorithm
#js{:name "AES-GCM"
:length 256})
(defn <gen-aes-key
[]
(p/let [result (js/crypto.subtle.generateKey
aes-key-algorithm
true
#js["encrypt" "decrypt"])]
(js->clj result :keywordize-keys true)))
(defn <export-key
[key']
(assert (instance? js/CryptoKey key') key')
(js/crypto.subtle.exportKey "jwk" key'))
(defn <import-public-key
[jwk]
(assert (instance? js/Object jwk) jwk)
(js/crypto.subtle.importKey "jwk" jwk key-algorithm true #js["encrypt"]))
(defn <import-private-key
[jwk]
(assert (instance? js/Object jwk) jwk)
(js/crypto.subtle.importKey "jwk" jwk key-algorithm true #js["decrypt"]))
(comment
(p/let [{:keys [publicKey privateKey]} (<gen-key-pair)]
(p/doseq [msg (map #(str "message" %) (range 1000))]
(p/let [encrypted (<encrypt msg publicKey)
plaintxt (<decrypt encrypted privateKey)]
(prn :encrypted msg)
(prn :plaintxt plaintxt))))
(p/let [k (<gen-aes-key)
kk (<export-key k)
encrypted (<aes-encrypt (apply str (repeat 1000 "x")) k)
plaintxt (<aes-decrypt encrypted k)]
(prn :encrypted encrypted)
(prn :plaintxt plaintxt)))
(defn store-graph-keys-jwk
[repo aes-key-jwk]
(let [conn (worker-state/get-client-ops-conn repo)]
(assert (some? conn) repo)
(let [aes-key-datom (first (d/datoms @conn :avet :aes-key-jwk))]
(assert (nil? aes-key-datom) aes-key-datom)
(ldb/transact! conn [[:db/add "e1" :aes-key-jwk aes-key-jwk]]))))
(defn get-graph-keys-jwk
[repo]
(let [conn (worker-state/get-client-ops-conn repo)]
(assert (some? conn) repo)
(let [aes-key-datom (first (d/datoms @conn :avet :aes-key-jwk))]
{:aes-key-jwk (:v aes-key-datom)})))
(def-thread-api :thread-api/rtc-get-graph-keys
[repo]
(get-graph-keys-jwk repo))

14
src/main/frontend/worker/db_worker.cljs
View File

@@ -260,7 +260,7 @@
(ldb/transact! datascript-conn [{:db/ident :logseq.kv/graph-last-gc-at
:kv/value (common-util/time-ms)}]))))
(defn- create-or-open-db!
(defn- <create-or-open-db!
[repo {:keys [config datoms] :as opts}]
(when-not (worker-state/get-sqlite-conn repo)
(p/let [[db search-db client-ops-db :as dbs] (get-dbs repo)
@@ -414,7 +414,7 @@
(when close-other-db?
(close-other-dbs! repo))
(when @shared-service/*master-client?
(create-or-open-db! repo (dissoc opts :close-other-db?)))
(<create-or-open-db! repo (dissoc opts :close-other-db?)))
nil))
(def-thread-api :thread-api/create-or-open-db
@@ -694,6 +694,8 @@
(when-let [conn (worker-state/get-datascript-conn repo)]
(worker-db-validate/validate-db conn)))
;; Returns an export-edn map for given repo. When there's an unexpected error, a map
;; with key :export-edn-error is returned
(def-thread-api :thread-api/export-edn
[repo options]
(let [conn (worker-state/get-datascript-conn repo)]
@@ -705,7 +707,7 @@
(worker-util/post-message :notification
["An unexpected error occurred during export. See the javascript console for details."
:error])
:export-edn-error))))
{:export-edn-error (.-message e)}))))
(def-thread-api :thread-api/get-view-data
[repo view-id option]
@@ -853,7 +855,11 @@
(defn- create-page!
[repo conn title options]
(let [config (worker-state/get-config repo)]
(worker-page/create! repo conn config title options)))
(try
(worker-page/create! repo conn config title options)
(catch :default e
(js/console.error e)
(throw e)))))
(defn- outliner-register-op-handlers!
[]

224
src/main/frontend/worker/device.cljs
View File

@@ -1,224 +0,0 @@
(ns frontend.worker.device
"Each device is assigned an id, and has some metadata(e.g. public&private-key for each device)"
(:require ["/frontend/idbkv" :as idb-keyval]
[cljs-time.coerce :as tc]
[cljs-time.core :as t]
[clojure.string :as string]
[frontend.common.missionary :as c.m]
[frontend.common.thread-api :refer [def-thread-api]]
[frontend.worker.crypt :as crypt]
[frontend.worker.rtc.client-op :as client-op]
[frontend.worker.rtc.ws-util :as ws-util]
[frontend.worker.state :as worker-state]
[goog.crypt.base64 :as base64]
[logseq.db :as ldb]
[missionary.core :as m]
[promesa.core :as p]))
;;; TODO: move frontend.idb to deps/, then we can use it in both frontend and db-worker
;;; now, I just direct use "/frontend/idbkv" here
(defonce ^:private store (delay (idb-keyval/newStore "localforage" "keyvaluepairs" 2)))
(defn- <get-item
[key']
(when (and key' @store)
(idb-keyval/get key' @store)))
(defn- <set-item!
[key' value]
(when (and key' @store)
(idb-keyval/set key' value @store)))
(defn- <remove-item!
[key']
(idb-keyval/del key' @store))
(def ^:private item-key-device-id "device-id")
(def ^:private item-key-device-name "device-name")
(def ^:private item-key-device-created-at "device-created-at")
(def ^:private item-key-device-updated-at "device-updated-at")
(def ^:private item-key-device-public-key-jwk "device-public-key-jwk")
(def ^:private item-key-device-private-key-jwk "device-private-key-jwk")
(defonce *device-id (atom nil :validator uuid?))
(defonce *device-name (atom nil))
(defonce *device-public-key (atom nil :validator #(instance? js/CryptoKey %)))
(defonce *device-private-key (atom nil :validator #(instance? js/CryptoKey %)))
(defn- new-task--get-user-devices
[get-ws-create-task]
(m/join :devices (ws-util/send&recv get-ws-create-task {:action "get-user-devices"})))
(defn- new-task--add-user-device
[get-ws-create-task device-name]
(m/join :device (ws-util/send&recv get-ws-create-task {:action "add-user-device"
:device-name device-name})))
(defn- new-task--remove-user-device*
[get-ws-create-task device-uuid]
(ws-util/send&recv get-ws-create-task {:action "remove-user-device"
:device-uuid device-uuid}))
(comment
(defn- new-task--update-user-device-name
[get-ws-create-task device-uuid device-name]
(ws-util/send&recv get-ws-create-task {:action "update-user-device-name"
:device-uuid device-uuid
:device-name device-name})))
(defn- new-task--add-device-public-key
[get-ws-create-task device-uuid key-name public-key-jwk]
(ws-util/send&recv get-ws-create-task {:action "add-device-public-key"
:device-uuid device-uuid
:key-name key-name
:public-key (ldb/write-transit-str public-key-jwk)}))
(defn- new-task--remove-device-public-key*
[get-ws-create-task device-uuid key-name]
(ws-util/send&recv get-ws-create-task {:action "remove-device-public-key"
:device-uuid device-uuid
:key-name key-name}))
(defn- new-task--sync-encrypted-aes-key*
[get-ws-create-task device-uuid->encrypted-aes-key graph-uuid]
(ws-util/send&recv get-ws-create-task
{:action "sync-encrypted-aes-key"
:device-uuid->encrypted-aes-key device-uuid->encrypted-aes-key
:graph-uuid graph-uuid}))
(defn- new-get-ws-create-task
[token]
(:get-ws-create-task (ws-util/gen-get-ws-create-map--memoized (ws-util/get-ws-url token))))
(defn new-task--ensure-device-metadata!
"Generate new device items if not exists.
Store in indexeddb.
Import to `*device-id`, `*device-public-key`, `*device-private-key`"
[token]
(m/sp
(let [device-uuid (c.m/<? (<get-item item-key-device-id))]
(when-not device-uuid
(let [get-ws-create-task (new-get-ws-create-task token)
agent-data (js->clj (some-> js/navigator.userAgentData .toJSON) :keywordize-keys true)
generated-device-name (string/join
"-"
[(:platform agent-data)
(when (:mobile agent-data) "mobile")
(:brand (first (:brands agent-data)))
(tc/to-epoch (t/now))])
{:keys [device-id device-name created-at updated-at]}
(m/? (new-task--add-user-device get-ws-create-task generated-device-name))
{:keys [publicKey privateKey]} (c.m/<? (crypt/<gen-key-pair))
public-key-jwk (c.m/<? (crypt/<export-key publicKey))
private-key-jwk (c.m/<? (crypt/<export-key privateKey))]
(c.m/<? (<set-item! item-key-device-id (str device-id)))
(c.m/<? (<set-item! item-key-device-name device-name))
(c.m/<? (<set-item! item-key-device-created-at created-at))
(c.m/<? (<set-item! item-key-device-updated-at updated-at))
(c.m/<? (<set-item! item-key-device-public-key-jwk public-key-jwk))
(c.m/<? (<set-item! item-key-device-private-key-jwk private-key-jwk))
(m/? (new-task--add-device-public-key
get-ws-create-task device-id "default-public-key" public-key-jwk))))
(c.m/<?
(p/let [device-uuid-str (<get-item item-key-device-id)
device-name (<get-item item-key-device-name)
device-public-key-jwk (<get-item item-key-device-public-key-jwk)
device-public-key (crypt/<import-public-key device-public-key-jwk)
device-private-key-jwk (<get-item item-key-device-private-key-jwk)
device-private-key (crypt/<import-private-key device-private-key-jwk)]
(reset! *device-id (uuid device-uuid-str))
(reset! *device-name device-name)
(reset! *device-public-key device-public-key)
(reset! *device-private-key device-private-key))))))
(defn new-task--list-devices
"Return device list.
Also sync local device metadata to remote if not exists in remote side"
[token]
(m/sp
(let [get-ws-create-task (new-get-ws-create-task token)
devices (m/? (new-task--get-user-devices get-ws-create-task))]
(when
;; check current device has been synced to remote
;; if not exists in remote, remove local-metadata and recreate in local and remote
(and @*device-id @*device-name @*device-public-key
(not (some
(fn [device]
(let [{:keys [device-id]} device]
(when (= device-id (str @*device-id))
true)))
devices)))
(c.m/<? (<remove-item! item-key-device-id))
(c.m/<? (<remove-item! item-key-device-name))
(c.m/<? (<remove-item! item-key-device-created-at))
(c.m/<? (<remove-item! item-key-device-updated-at))
(c.m/<? (<remove-item! item-key-device-public-key-jwk))
(c.m/<? (<remove-item! item-key-device-private-key-jwk))
(m/? (new-task--ensure-device-metadata! token)))
devices)))
(defn new-task--remove-device-public-key
[token device-uuid key-name]
(assert (some? key-name))
(m/sp
(when-let [device-uuid* (cond-> device-uuid (string? device-uuid) parse-uuid)]
(let [get-ws-create-task (new-get-ws-create-task token)]
(m/? (new-task--remove-device-public-key* get-ws-create-task device-uuid* key-name))))))
(defn new-task--remove-device
[token device-uuid]
(m/sp
(when-let [device-uuid* (cond-> device-uuid (string? device-uuid) parse-uuid)]
(let [get-ws-create-task (new-get-ws-create-task token)]
(m/? (new-task--remove-user-device* get-ws-create-task device-uuid*))))))
(defn new-task--sync-current-graph-encrypted-aes-key
[token device-uuids]
(let [repo (worker-state/get-current-repo)]
(assert (and (seq device-uuids) (every? uuid? device-uuids)) device-uuids)
(m/sp
(when-let [graph-uuid (client-op/get-graph-uuid repo)]
(when-let [{:keys [aes-key-jwk]} (crypt/get-graph-keys-jwk repo)]
(let [device-uuids (set device-uuids)
get-ws-create-task (new-get-ws-create-task token)
devices (m/? (new-task--get-user-devices get-ws-create-task))]
(when-let [devices* (not-empty
(filter
(fn [device]
(and (contains? device-uuids (uuid (:device-id device)))
(some? (get-in device [:keys :default-public-key]))))
devices))]
(let [device-uuid->encrypted-aes-key
(m/?
(apply m/join (fn [& x] (into {} x))
(map (fn [device]
(m/sp
(let [device-public-key
(c.m/<?
(crypt/<import-public-key
(clj->js
(ldb/read-transit-str
(get-in device [:keys :default-public-key :public-key])))))]
[(uuid (:device-id device))
(base64/encodeByteArray
(js/Uint8Array.
(c.m/<? (crypt/<rsa-encrypt aes-key-jwk device-public-key))))])))
devices*)))]
(m/? (new-task--sync-encrypted-aes-key*
get-ws-create-task device-uuid->encrypted-aes-key graph-uuid))))))))))
(def-thread-api :thread-api/rtc-sync-current-graph-encrypted-aes-key
[token device-uuids]
(new-task--sync-current-graph-encrypted-aes-key token device-uuids))
(def-thread-api :thread-api/list-devices
[token]
(new-task--list-devices token))
(def-thread-api :thread-api/remove-device-public-key
[token device-uuid key-name]
(new-task--remove-device-public-key token device-uuid key-name))
(def-thread-api :thread-api/remove-device
[token device-uuid]
(new-task--remove-device token device-uuid))

98
src/main/frontend/worker/rtc/asset.cljs
View File

@@ -7,12 +7,14 @@
indicates need to upload the asset to server"
(:require [clojure.set :as set]
[datascript.core :as d]
[frontend.common.crypt :as crypt]
[frontend.common.missionary :as c.m]
[frontend.worker.rtc.client-op :as client-op]
[frontend.worker.rtc.exception :as r.ex]
[frontend.worker.rtc.log-and-state :as rtc-log-and-state]
[frontend.worker.rtc.ws-util :as ws-util]
[frontend.worker.state :as worker-state]
[lambdaisland.glogi :as log]
[logseq.common.path :as path]
[logseq.db :as ldb]
[malli.core :as ma]
@@ -118,44 +120,52 @@
(defn- new-task--concurrent-download-assets
"Concurrently download assets with limited max concurrent count"
[repo asset-uuid->url asset-uuid->asset-type]
(->> (fn [[asset-uuid url]]
(m/sp
(let [r (c.m/<?
(worker-state/<invoke-main-thread :thread-api/rtc-download-asset
repo (str asset-uuid)
(get asset-uuid->asset-type asset-uuid) url))]
(when-let [edata (:ex-data r)]
;; if download-url return 404, ignore this asset
(when (not= 404 (:status (:data edata)))
(throw (ex-info "download asset failed" r)))))))
(c.m/concurrent-exec-flow 5 (m/seed asset-uuid->url))
(m/reduce (constantly nil))))
[repo aes-key asset-uuid->url asset-uuid->asset-type]
(m/sp
(let [exported-aes-key (when aes-key (c.m/<? (crypt/<export-aes-key aes-key)))]
(m/?
(->> (fn [[asset-uuid url]]
(m/sp
(let [r (c.m/<?
(worker-state/<invoke-main-thread :thread-api/rtc-download-asset
repo exported-aes-key (str asset-uuid)
(get asset-uuid->asset-type asset-uuid) url))]
(when-let [edata (:ex-data r)]
;; if download-url return 404, ignore this asset
(when (not= 404 (:status (:data edata)))
(throw (ex-info "download asset failed" r)))))))
(c.m/concurrent-exec-flow 5 (m/seed asset-uuid->url))
(m/reduce (constantly nil)))))))
(defn- new-task--concurrent-upload-assets
"Concurrently upload assets with limited max concurrent count"
[repo conn asset-uuid->url asset-uuid->asset-metadata]
(->> (fn [[asset-uuid url]]
(m/sp
(let [[asset-type checksum] (get asset-uuid->asset-metadata asset-uuid)
r (c.m/<?
(worker-state/<invoke-main-thread :thread-api/rtc-upload-asset
repo (str asset-uuid) asset-type checksum url))]
(when (:ex-data r)
(throw (ex-info "upload asset failed" r)))
;; asset might be deleted by the user before uploaded successfully
(when (d/entity @conn [:block/uuid asset-uuid])
(ldb/transact! conn
[{:block/uuid asset-uuid
:logseq.property.asset/remote-metadata {:checksum checksum :type asset-type}}]
;; Don't generate rtc ops again, (block-ops & asset-ops)
{:persist-op? false}))
(client-op/remove-asset-op repo asset-uuid))))
(c.m/concurrent-exec-flow 3 (m/seed asset-uuid->url))
(m/reduce (constantly nil))))
[repo conn aes-key asset-uuid->url asset-uuid->asset-metadata]
(m/sp
(let [exported-aes-key (when aes-key (c.m/<? (crypt/<export-aes-key aes-key)))]
(m/?
(->> (fn [[asset-uuid url]]
(m/sp
(let [[asset-type checksum] (get asset-uuid->asset-metadata asset-uuid)
_ (prn :xxx exported-aes-key)
r (c.m/<?
(worker-state/<invoke-main-thread :thread-api/rtc-upload-asset
repo exported-aes-key (str asset-uuid)
asset-type checksum url))]
(when (:ex-data r)
(throw (ex-info "upload asset failed" r)))
;; asset might be deleted by the user before uploaded successfully
(when (d/entity @conn [:block/uuid asset-uuid])
(ldb/transact! conn
[{:block/uuid asset-uuid
:logseq.property.asset/remote-metadata {:checksum checksum :type asset-type}}]
;; Don't generate rtc ops again, (block-ops & asset-ops)
{:persist-op? false}))
(client-op/remove-asset-op repo asset-uuid))))
(c.m/concurrent-exec-flow 3 (m/seed asset-uuid->url))
(m/reduce (constantly nil)))))))
(defn- new-task--push-local-asset-updates
[repo get-ws-create-task conn graph-uuid major-schema-version add-log-fn]
[repo get-ws-create-task conn graph-uuid major-schema-version aes-key add-log-fn]
(m/sp
(when-let [asset-ops (not-empty (client-op/get-all-asset-ops repo))]
(let [upload-asset-uuids (keep
@@ -197,7 +207,7 @@
:asset-uuid->url))]
(when (seq asset-uuid->url)
(add-log-fn :rtc.asset.log/upload-assets {:asset-uuids (keys asset-uuid->url)}))
(m/? (new-task--concurrent-upload-assets repo conn asset-uuid->url asset-uuid->asset-metadata))
(m/? (new-task--concurrent-upload-assets repo conn aes-key asset-uuid->url asset-uuid->asset-metadata))
(when (seq remove-asset-uuids)
(add-log-fn :rtc.asset.log/remove-assets {:asset-uuids remove-asset-uuids})
(m/? (ws-util/send&recv get-ws-create-task
@@ -212,7 +222,7 @@
(concat (keys asset-uuid->url) remove-asset-uuids))))))
(defn- new-task--pull-remote-asset-updates
[repo get-ws-create-task conn graph-uuid add-log-fn asset-update-ops]
[repo get-ws-create-task conn graph-uuid aes-key add-log-fn asset-update-ops]
(m/sp
(when (seq asset-update-ops)
(let [update-asset-uuids (keep (fn [op]
@@ -251,7 +261,7 @@
repo (str asset-uuid) asset-type)))
(when (seq asset-uuid->url)
(add-log-fn :rtc.asset.log/download-assets {:asset-uuids (keys asset-uuid->url)}))
(m/? (new-task--concurrent-download-assets repo asset-uuid->url asset-uuid->asset-type))))))
(m/? (new-task--concurrent-download-assets repo aes-key asset-uuid->url asset-uuid->asset-type))))))
(defn- get-all-asset-blocks
[db]
@@ -266,7 +276,7 @@
db))
(defn- new-task--initial-download-missing-assets
[repo get-ws-create-task graph-uuid conn add-log-fn]
[repo get-ws-create-task graph-uuid conn aes-key add-log-fn]
(m/sp
(let [local-all-asset-file-paths
(c.m/<? (worker-state/<invoke-main-thread :thread-api/get-all-asset-file-paths repo))
@@ -278,10 +288,10 @@
(set/difference local-all-asset-uuids local-all-asset-file-uuids)))]
(add-log-fn :rtc.asset.log/initial-download-missing-assets {:count (count asset-update-ops)})
(m/? (new-task--pull-remote-asset-updates
repo get-ws-create-task conn graph-uuid add-log-fn asset-update-ops))))))
repo get-ws-create-task conn graph-uuid aes-key add-log-fn asset-update-ops))))))
(defn create-assets-sync-loop
[repo get-ws-create-task graph-uuid major-schema-version conn *auto-push?]
[repo get-ws-create-task graph-uuid major-schema-version conn *auto-push? *aes-key]
(let [started-dfv (m/dfv)
add-log-fn (fn [type message]
(assert (map? message) message)
@@ -293,18 +303,22 @@
started-dfv
(m/sp
(try
(log/info :rtc-asset :loop-starting)
;; check aes-key exists
(when (ldb/get-graph-rtc-e2ee? @conn) (assert @*aes-key))
(started-dfv true)
(m/? (new-task--initial-download-missing-assets repo get-ws-create-task graph-uuid conn add-log-fn))
(m/? (new-task--initial-download-missing-assets
repo get-ws-create-task graph-uuid conn @*aes-key add-log-fn))
(->>
(let [event (m/?> mixed-flow)]
(case (:type event)
:remote-updates
(when-let [asset-update-ops (not-empty (:value event))]
(m/? (new-task--pull-remote-asset-updates
repo get-ws-create-task conn graph-uuid add-log-fn asset-update-ops)))
repo get-ws-create-task conn graph-uuid @*aes-key add-log-fn asset-update-ops)))
:local-update-check
(m/? (new-task--push-local-asset-updates
repo get-ws-create-task conn graph-uuid major-schema-version add-log-fn))))
repo get-ws-create-task conn graph-uuid major-schema-version @*aes-key add-log-fn))))
m/ap
(m/reduce {} nil)
m/?)

124
src/main/frontend/worker/rtc/client.cljs
View File

@@ -2,10 +2,12 @@
"Fns about push local updates"
(:require [clojure.string :as string]
[datascript.core :as d]
[frontend.common.crypt :as crypt]
[frontend.common.missionary :as c.m]
[frontend.worker.flows :as worker-flows]
[frontend.worker.rtc.branch-graph :as r.branch-graph]
[frontend.worker.rtc.client-op :as client-op]
[frontend.worker.rtc.const :as rtc-const]
[frontend.worker.rtc.exception :as r.ex]
[frontend.worker.rtc.log-and-state :as rtc-log-and-state]
[frontend.worker.rtc.malli-schema :as rtc-schema]
@@ -19,21 +21,23 @@
[missionary.core :as m]
[tick.core :as tick]))
(defn- apply-remote-updates-from-apply-ops
[apply-ops-resp graph-uuid repo conn date-formatter add-log-fn]
(if-let [remote-ex (:ex-data apply-ops-resp)]
(do (add-log-fn :rtc.log/pull-remote-data (assoc remote-ex :sub-type :pull-remote-data-exception))
(case (:type remote-ex)
:graph-lock-failed nil
:graph-lock-missing
(throw r.ex/ex-remote-graph-lock-missing)
:rtc.exception/get-s3-object-failed
(throw (ex-info (:ex-message apply-ops-resp) (:ex-data apply-ops-resp)))
;;else
(throw (ex-info "Unavailable3" {:remote-ex remote-ex}))))
(do (assert (pos? (:t apply-ops-resp)) apply-ops-resp)
(r.remote-update/apply-remote-update
graph-uuid repo conn date-formatter {:type :remote-update :value apply-ops-resp} add-log-fn))))
(defn- task--apply-remote-updates-from-apply-ops
[apply-ops-resp graph-uuid repo conn date-formatter aes-key add-log-fn]
(m/sp
(if-let [remote-ex (:ex-data apply-ops-resp)]
(do (add-log-fn :rtc.log/pull-remote-data (assoc remote-ex :sub-type :pull-remote-data-exception))
(case (:type remote-ex)
:graph-lock-failed nil
:graph-lock-missing
(throw r.ex/ex-remote-graph-lock-missing)
:rtc.exception/get-s3-object-failed
(throw (ex-info (:ex-message apply-ops-resp) (:ex-data apply-ops-resp)))
;;else
(throw (ex-info "Unavailable3" {:remote-ex remote-ex}))))
(do (assert (and (pos? (:t apply-ops-resp)) (pos? (:t-query-end apply-ops-resp))) apply-ops-resp)
(m/?
(r.remote-update/task--apply-remote-update
graph-uuid repo conn date-formatter {:type :remote-update :value apply-ops-resp} aes-key add-log-fn))))))
(defn- new-task--init-request
[get-ws-create-task graph-uuid major-schema-version repo conn *last-calibrate-t *server-schema-version add-log-fn]
@@ -42,15 +46,19 @@
get-graph-skeleton? (or (nil? @*last-calibrate-t)
(< 500 (- t-before @*last-calibrate-t)))]
(try
(let [{remote-t :t
(let [{_remote-t :t
remote-t-query-end :t-query-end
server-schema-version :server-schema-version
server-builtin-db-idents :server-builtin-db-idents
:as resp}
(m/? (ws-util/send&recv get-ws-create-task {:action "init-request"
:graph-uuid graph-uuid
:schema-version (str major-schema-version)
:t-before t-before
:get-graph-skeleton get-graph-skeleton?}))]
(m/? (ws-util/send&recv get-ws-create-task
{:action "init-request"
:graph-uuid graph-uuid
:schema-version (str major-schema-version)
:api-version "20251124"
:t-before t-before
:get-graph-skeleton get-graph-skeleton?}
:timeout-ms 30000))]
(if-let [remote-ex (:ex-data resp)]
(do
(add-log-fn :rtc.log/init-request remote-ex)
@@ -62,11 +70,11 @@
(do
(when server-schema-version
(reset! *server-schema-version server-schema-version)
(reset! *last-calibrate-t remote-t))
(when remote-t
(rtc-log-and-state/update-remote-t graph-uuid remote-t)
(reset! *last-calibrate-t remote-t-query-end))
(when remote-t-query-end
(rtc-log-and-state/update-remote-t graph-uuid remote-t-query-end)
(when (not t-before)
(client-op/update-local-tx repo remote-t)))
(client-op/update-local-tx repo remote-t-query-end)))
(when (and server-schema-version server-builtin-db-idents)
(r.skeleton/calibrate-graph-skeleton server-schema-version server-builtin-db-idents @conn))
resp)))
@@ -95,7 +103,7 @@
see also `ws/get-mws-create`.
But ensure `init-request` and `calibrate-graph-skeleton` has been sent"
[get-ws-create-task graph-uuid major-schema-version repo conn date-formatter
*last-calibrate-t *online-users *server-schema-version add-log-fn]
*last-calibrate-t *online-users *server-schema-version *aes-key add-log-fn]
(m/sp
(let [ws (m/? get-ws-create-task)
sent-3rd-value [graph-uuid major-schema-version repo]
@@ -141,7 +149,8 @@
:repo repo
:graph-uuid graph-uuid
:remote-schema-version max-remote-schema-version}))
(apply-remote-updates-from-apply-ops init-request-resp graph-uuid repo conn date-formatter add-log-fn)))
(m/? (task--apply-remote-updates-from-apply-ops
init-request-resp graph-uuid repo conn date-formatter @*aes-key add-log-fn))))
ws)))
(defn- ->pos
@@ -437,9 +446,34 @@
(client-op/add-ops! repo rename-db-ident-ops)
nil))
(defn- task--encrypt-remote-ops
[aes-key remote-ops]
(assert aes-key)
(let [encrypt-attr-set (conj rtc-const/encrypt-attr-set :page-name)]
(m/sp
(loop [[remote-op & rest-remote-ops] remote-ops
result []]
(if-not remote-op
result
(let [[op-type op-value] remote-op]
(case op-type
:update-page
(recur rest-remote-ops
(conj result
[op-type (c.m/<? (crypt/<encrypt-map aes-key encrypt-attr-set op-value))]))
:update
(let [av-coll* (c.m/<?
(crypt/<encrypt-av-coll
aes-key rtc-const/encrypt-attr-set (:av-coll op-value)))]
(recur rest-remote-ops
(conj result [op-type (assoc op-value :av-coll av-coll*)])))
;; else
(recur rest-remote-ops (conj result remote-op)))))))))
(defn new-task--push-local-ops
"Return a task: push local updates"
[repo conn graph-uuid major-schema-version date-formatter get-ws-create-task *remote-profile? add-log-fn]
[repo conn graph-uuid major-schema-version date-formatter get-ws-create-task *remote-profile? aes-key add-log-fn]
(m/sp
(let [block-ops-map-coll (client-op/get&remove-all-block-ops repo)
update-kv-value-ops-map-coll (client-op/get&remove-all-update-kv-value-ops repo)
@@ -453,12 +487,18 @@
other-remote-ops)]
(when-let [ops-for-remote (rtc-schema/to-ws-ops-decoder remote-ops)]
(let [local-tx (client-op/get-local-tx repo)
ops-for-remote* (if aes-key
(m/? (task--encrypt-remote-ops aes-key ops-for-remote))
ops-for-remote)
r (try
(let [message (cond-> {:action "apply-ops"
:graph-uuid graph-uuid :schema-version (str major-schema-version)
:ops ops-for-remote :t-before local-tx}
:graph-uuid graph-uuid
:schema-version (str major-schema-version)
:api-version "20251124"
:ops ops-for-remote*
:t-before local-tx}
(true? @*remote-profile?) (assoc :profile true))
r (m/? (ws-util/send&recv get-ws-create-task message))]
r (m/? (ws-util/send&recv get-ws-create-task message :timeout-ms 30000))]
(r.throttle/add-rtc-api-call-record! message)
r)
(catch :default e
@@ -485,18 +525,22 @@
(do (rollback repo block-ops-map-coll update-kv-value-ops-map-coll rename-db-ident-ops-map-coll)
(throw (ex-info "Unavailable1" {:remote-ex remote-ex})))))
(do (assert (pos? (:t r)) r)
(r.remote-update/apply-remote-update
graph-uuid repo conn date-formatter {:type :remote-update :value r} add-log-fn)
(add-log-fn :rtc.log/push-local-update {:remote-t (:t r)}))))))))
(do (assert (and (pos? (:t r)) (pos? (:t-query-end r))) r)
(m/?
(r.remote-update/task--apply-remote-update
graph-uuid repo conn date-formatter {:type :remote-update :value r} aes-key add-log-fn))
(add-log-fn :rtc.log/push-local-update {:remote-t (:t r) :remote-t-query-end (:t-query-end r)}))))))))
(defn new-task--pull-remote-data
[repo conn graph-uuid major-schema-version date-formatter get-ws-create-task add-log-fn]
[repo conn graph-uuid major-schema-version date-formatter get-ws-create-task aes-key add-log-fn]
(m/sp
(let [local-tx (client-op/get-local-tx repo)
message {:action "apply-ops"
:graph-uuid graph-uuid :schema-version (str major-schema-version)
:ops [] :t-before (or local-tx 1)}
r (m/? (ws-util/send&recv get-ws-create-task message))]
:graph-uuid graph-uuid
:schema-version (str major-schema-version)
:api-version "20251124"
:ops []
:t-before local-tx}
r (m/? (ws-util/send&recv get-ws-create-task message :timeout-ms 30000))]
(r.throttle/add-rtc-api-call-record! message)
(apply-remote-updates-from-apply-ops r graph-uuid repo conn date-formatter add-log-fn))))
(m/? (task--apply-remote-updates-from-apply-ops r graph-uuid repo conn date-formatter aes-key add-log-fn)))))

8
src/main/frontend/worker/rtc/client_op.cljs
View File

@@ -90,13 +90,7 @@
:db-ident {:db/unique :db.unique/identity}
:db-ident-or-block-uuid {:db/unique :db.unique/identity}
:local-tx {:db/index true}
:graph-uuid {:db/index true}
:aes-key-jwk {:db/index true}
;; device
:device/uuid {:db/unique :db.unique/identity}
:device/public-key-jwk {}
:device/private-key-jwk {}})
:graph-uuid {:db/index true}})
(defn update-graph-uuid
[repo graph-uuid]

4
src/main/frontend/worker/rtc/const.cljs
View File

@@ -45,3 +45,7 @@
(into #{}
(keep (fn [[kw config]] (when (get-in config [:rtc :rtc/ignore-entity-when-init-download]) kw)))
kv-entity/kv-entities))
(def encrypt-attr-set
"block attributes that need to be encrypted"
#{:block/title :block/name})

93
src/main/frontend/worker/rtc/core.cljs
View File

@@ -5,11 +5,11 @@
[frontend.common.missionary :as c.m]
[frontend.common.thread-api :refer [def-thread-api]]
[frontend.worker-common.util :as worker-util]
[frontend.worker.device :as worker-device]
[frontend.worker.rtc.asset :as r.asset]
[frontend.worker.rtc.branch-graph :as r.branch-graph]
[frontend.worker.rtc.client :as r.client]
[frontend.worker.rtc.client-op :as client-op]
[frontend.worker.rtc.crypt :as rtc-crypt]
[frontend.worker.rtc.db :as rtc-db]
[frontend.worker.rtc.exception :as r.ex]
[frontend.worker.rtc.full-upload-download-graph :as r.upload-download]
@@ -188,11 +188,22 @@
(swap! *graph-uuid->*online-users assoc graph-uuid *online-users)
*online-users)))
(defn- task--update-*aes-key
[get-ws-create-task db user-uuid graph-uuid *aes-key]
(m/sp
(when (ldb/get-graph-rtc-e2ee? db)
(let [aes-key (m/? (rtc-crypt/task--get-aes-key get-ws-create-task user-uuid graph-uuid))]
(when (nil? aes-key)
(throw (ex-info "not found aes-key" {:type :rtc.exception/not-found-graph-aes-key
:graph-uuid graph-uuid
:user-uuid user-uuid})))
(reset! *aes-key aes-key)))))
(declare new-task--inject-users-info)
(defn- create-rtc-loop
"Return a map with [:rtc-state-flow :rtc-loop-task :*rtc-auto-push? :onstarted-task]
TODO: auto refresh token if needed"
[graph-uuid schema-version repo conn date-formatter token
[graph-uuid schema-version repo conn date-formatter token user-uuid
& {:keys [auto-push? debug-ws-url] :or {auto-push? true}}]
(let [major-schema-version (db-schema/major-version schema-version)
ws-url (or debug-ws-url (ws-util/get-ws-url token))
@@ -202,17 +213,19 @@
*online-users (get-or-create-*online-users graph-uuid)
*assets-sync-loop-canceler (atom nil)
*server-schema-version (atom nil)
*aes-key (atom nil)
started-dfv (m/dfv)
add-log-fn (fn [type message]
(assert (map? message) message)
(rtc-log-and-state/rtc-log type (assoc message :graph-uuid graph-uuid)))
{:keys [*current-ws get-ws-create-task]}
{:keys [*current-ws] get-ws-create-task0 :get-ws-create-task}
(gen-get-ws-create-map--memoized ws-url)
get-ws-create-task (r.client/ensure-register-graph-updates--memoized
get-ws-create-task graph-uuid major-schema-version repo conn date-formatter
*last-calibrate-t *online-users *server-schema-version add-log-fn)
get-ws-create-task0 graph-uuid major-schema-version repo conn date-formatter
*last-calibrate-t *online-users *server-schema-version *aes-key add-log-fn)
{:keys [assets-sync-loop-task]}
(r.asset/create-assets-sync-loop repo get-ws-create-task graph-uuid major-schema-version conn *auto-push?)
(r.asset/create-assets-sync-loop
repo get-ws-create-task graph-uuid major-schema-version conn *auto-push? *aes-key)
mixed-flow (create-mixed-flow repo get-ws-create-task *auto-push? *online-users)]
(assert (some? *current-ws))
{:rtc-state-flow (create-rtc-state-flow (create-ws-state-flow *current-ws))
@@ -227,6 +240,7 @@
(try
(log/info :rtc :loop-starting)
;; init run to open a ws
(m/? (task--update-*aes-key get-ws-create-task0 @conn user-uuid graph-uuid *aes-key))
(m/? get-ws-create-task)
;; NOTE: Set dfv after ws connection is established,
;; ensuring the ws connection is already up when the cloud-icon turns green.
@@ -234,29 +248,41 @@
(update-remote-schema-version! conn @*server-schema-version)
(reset! *assets-sync-loop-canceler
(c.m/run-task :assets-sync-loop-task
assets-sync-loop-task))
assets-sync-loop-task
:fail #(log/info :assets-sync-loop-task-stopped %)))
(->>
(let [event (m/?> mixed-flow)]
(case (:type event)
(:remote-update :remote-asset-block-update)
(try (r.remote-update/apply-remote-update graph-uuid repo conn date-formatter event add-log-fn)
(catch :default e
(if (= ::r.remote-update/need-pull-remote-data (:type (ex-data e)))
(m/? (r.client/new-task--pull-remote-data
repo conn graph-uuid major-schema-version date-formatter get-ws-create-task add-log-fn))
(throw (r.ex/e->ex-info e)))))
(try
(m/? (r.remote-update/task--apply-remote-update
graph-uuid repo conn date-formatter event @*aes-key add-log-fn))
(catch :default e
(if (= :rtc.exception/local-graph-too-old (:type (ex-data e)))
(m/? (r.client/new-task--pull-remote-data
repo conn graph-uuid major-schema-version date-formatter get-ws-create-task @*aes-key
add-log-fn))
(throw e))))
:local-update-check
(m/? (r.client/new-task--push-local-ops
repo conn graph-uuid major-schema-version date-formatter
get-ws-create-task *remote-profile? add-log-fn))
(try
(m/? (r.client/new-task--push-local-ops
repo conn graph-uuid major-schema-version date-formatter
get-ws-create-task *remote-profile? @*aes-key add-log-fn))
(catch :default e
(if (= :rtc.exception/local-graph-too-old (:type (ex-data e)))
(m/? (r.client/new-task--pull-remote-data
repo conn graph-uuid major-schema-version date-formatter get-ws-create-task @*aes-key
add-log-fn))
(throw e))))
:online-users-updated
(reset! *online-users (:online-users (:value event)))
:pull-remote-updates
(m/? (r.client/new-task--pull-remote-data
repo conn graph-uuid major-schema-version date-formatter get-ws-create-task add-log-fn))
repo conn graph-uuid major-schema-version date-formatter get-ws-create-task @*aes-key
add-log-fn))
:inject-users-info
(m/? (new-task--inject-users-info token graph-uuid major-schema-version))))
@@ -335,20 +361,20 @@
(defn- new-task--rtc-start*
[repo token]
(m/sp
;; ensure device metadata existing first
(m/? (worker-device/new-task--ensure-device-metadata! token))
(let [{:keys [conn user-uuid graph-uuid schema-version remote-schema-version date-formatter] :as r}
(validate-rtc-start-conditions repo token)]
(if (instance? ExceptionInfo r)
r
(let [{:keys [rtc-state-flow *rtc-auto-push? *rtc-remote-profile? rtc-loop-task *online-users onstarted-task]}
(create-rtc-loop graph-uuid schema-version repo conn date-formatter token)
(create-rtc-loop graph-uuid schema-version repo conn date-formatter token user-uuid)
*last-stop-exception (atom nil)
canceler (c.m/run-task :rtc-loop-task
rtc-loop-task
:fail (fn [e]
(reset! *last-stop-exception e)
(log/info :rtc-loop-task e)
(when-not (or (instance? Cancelled e) (= "missionary.Cancelled" (ex-message e)))
(println (.-stack e)))
(when (= :rtc.exception/ws-timeout (some-> e ex-data :type))
;; if fail reason is websocket-timeout, try to restart rtc
(worker-state/<invoke-main-thread :thread-api/rtc-start-request repo))))
@@ -460,13 +486,20 @@
:schema-version (str major-schema-version)})))
(defn new-task--grant-access-to-others
[token graph-uuid & {:keys [target-user-uuids target-user-emails]}]
(let [{:keys [get-ws-create-task]} (gen-get-ws-create-map--memoized (ws-util/get-ws-url token))]
(ws-util/send&recv get-ws-create-task
(cond-> {:action "grant-access"
:graph-uuid graph-uuid}
target-user-uuids (assoc :target-user-uuids target-user-uuids)
target-user-emails (assoc :target-user-emails target-user-emails)))))
[token graph-uuid user-uuid target-user-email]
(m/sp
(let [{:keys [get-ws-create-task]} (gen-get-ws-create-map--memoized (ws-util/get-ws-url token))
encrypted-aes-key
(m/? (rtc-crypt/task--encrypt-graph-aes-key-by-other-user-public-key
get-ws-create-task graph-uuid user-uuid target-user-email))
resp (m/? (ws-util/send&recv get-ws-create-task
(cond-> {:action "grant-access"
:graph-uuid graph-uuid
:target-user-email+encrypted-aes-key-coll
[{:user/email target-user-email
:encrypted-aes-key (ldb/write-transit-str encrypted-aes-key)}]})))]
(when (:ex-data resp)
(throw (ex-info (:ex-message resp) (:ex-data resp)))))))
(defn new-task--get-block-content-versions
"Return a task that return map [:ex-data :ex-message :versions]"
@@ -589,10 +622,8 @@
(rtc-toggle-remote-profile))
(def-thread-api :thread-api/rtc-grant-graph-access
[token graph-uuid target-user-uuids target-user-emails]
(new-task--grant-access-to-others token graph-uuid
:target-user-uuids target-user-uuids
:target-user-emails target-user-emails))
[token graph-uuid user-uuid target-user-email]
(new-task--grant-access-to-others token graph-uuid user-uuid target-user-email))
(def-thread-api :thread-api/rtc-get-graphs
[token]

278
src/main/frontend/worker/rtc/crypt.cljs Normal file
View File

@@ -0,0 +1,278 @@
(ns frontend.worker.rtc.crypt
"rtc e2ee related.
Each user has an RSA key pair.
Each graph has an AES key.
Server stores the encrypted AES key, public key, and encrypted private key."
(:require ["/frontend/idbkv" :as idb-keyval]
[clojure.string :as string]
[frontend.common.crypt :as crypt]
[frontend.common.file.opfs :as opfs]
[frontend.common.missionary :as c.m]
[frontend.common.thread-api :refer [def-thread-api]]
[frontend.worker.rtc.ws-util :as ws-util]
[frontend.worker.state :as worker-state]
[lambdaisland.glogi :as log]
[logseq.db :as ldb]
[missionary.core :as m]
[promesa.core :as p])
(:import [missionary Cancelled]))
(defonce ^:private store (delay (idb-keyval/newStore "localforage" "keyvaluepairs" 2)))
(defonce ^:private e2ee-password-file "e2ee-password")
(defonce ^:private native-env?
(let [href (try (.. js/self -location -href)
(catch :default _ nil))]
(boolean (and (string? href)
(or (string/includes? href "electron=true")
(string/includes? href "capacitor=true"))))))
(defn- native-worker?
[]
native-env?)
(defn- <native-save-password-text!
[encrypted-text]
(worker-state/<invoke-main-thread :thread-api/native-save-e2ee-password encrypted-text))
(defn- <native-read-password-text
[]
(worker-state/<invoke-main-thread :thread-api/native-get-e2ee-password))
(defn- <save-e2ee-password
[refresh-token password]
(p/let [result (crypt/<encrypt-text-by-text-password refresh-token password)
text (ldb/write-transit-str result)]
(if (native-worker?)
(-> (p/let [_ (<native-save-password-text! text)]
nil)
(p/catch (fn [e]
(log/error :native-save-e2ee-password {:error e})
(opfs/<write-text! e2ee-password-file text))))
(opfs/<write-text! e2ee-password-file text))))
(defn- <read-e2ee-password
[refresh-token]
(p/let [text (if (native-worker?)
(<native-read-password-text)
(opfs/<read-text! e2ee-password-file))
data (ldb/read-transit-str text)
password (crypt/<decrypt-text-by-text-password refresh-token data)]
password))
(defn- <get-item
[k]
(assert (and k @store))
(p/let [r (idb-keyval/get k @store)]
(js->clj r :keywordize-keys true)))
(defn- <set-item!
[k value]
(assert (and k @store))
(idb-keyval/set k value @store))
(defn- graph-encrypted-aes-key-idb-key
[repo]
(assert (some? repo))
(str "rtc-encrypted-aes-key###" repo))
(defn- <import-public-key-transit-str
"Return js/CryptoKey"
[public-key-transit-str]
(when-let [exported-public-key (ldb/read-transit-str public-key-transit-str)]
(crypt/<import-public-key exported-public-key)))
(defn task--upload-user-rsa-key-pair
"Uploads the user's RSA key pair to the server."
[get-ws-create-task user-uuid public-key encrypted-private-key & {:keys [reset-private-key]
:or {reset-private-key false}}]
(m/sp
(let [exported-public-key-str (ldb/write-transit-str (c.m/<? (crypt/<export-public-key public-key)))
encrypted-private-key-str (ldb/write-transit-str encrypted-private-key)
response (m/? (ws-util/send&recv get-ws-create-task
{:action "upload-user-rsa-key-pair"
:user-uuid user-uuid
:public-key exported-public-key-str
:encrypted-private-key encrypted-private-key-str
:reset-private-key reset-private-key}))]
(when (:ex-data response)
(throw (ex-info (:ex-message response) (:ex-data response)))))))
(defn task--reset-user-rsa-key-pair
"Reset rsa-key-pair in server."
[get-ws-create-task user-uuid public-key encrypted-private-key]
(assert (and public-key encrypted-private-key))
(m/sp
(let [exported-public-key-str (ldb/write-transit-str (c.m/<? (crypt/<export-public-key public-key)))
encrypted-private-key-str (ldb/write-transit-str encrypted-private-key)
resp (m/? (ws-util/send&recv get-ws-create-task
{:action "reset-user-rsa-key-pair"
:user-uuid user-uuid
:public-key exported-public-key-str
:encrypted-private-key encrypted-private-key-str}))]
(when (:ex-data resp)
(throw (ex-info (:ex-message resp) (:ex-data resp)))))))
(defn task--fetch-user-rsa-key-pair
"Fetches the user's RSA key pair from server.
Return {:public-key CryptoKey, :encrypted-private-key [array,array,array]}
Return nil if not exists"
[get-ws-create-task user-uuid]
(m/sp
(let [response (m/? (ws-util/send&recv get-ws-create-task
{:action "fetch-user-rsa-key-pair"
:user-uuid user-uuid}))]
(if (:ex-data response)
(throw (ex-info (:ex-message response)
(assoc (:ex-data response)
:type :rtc.exception/fetch-user-rsa-key-pair-error)))
(let [{:keys [public-key encrypted-private-key]} response]
(when (and public-key encrypted-private-key)
{:public-key (c.m/<? (<import-public-key-transit-str public-key))
:encrypted-private-key (ldb/read-transit-str encrypted-private-key)}))))))
(defn- task--remote-fetch-graph-encrypted-aes-key
"Return nil if not exists."
[get-ws-create-task graph-uuid]
(m/sp
(let [response (m/? (ws-util/send&recv get-ws-create-task
{:action "fetch-graph-encrypted-aes-key"
:graph-uuid graph-uuid}))]
(if (:ex-data response)
(throw (ex-info (:ex-message response) (assoc (:ex-data response)
:type :rtc.exception/fetch-graph-aes-key-error)))
(ldb/read-transit-str (:encrypted-aes-key response))))))
(defn task--fetch-graph-aes-key
"Fetches the AES key for a graph, from indexeddb or server.
Return nil if not exists"
[get-ws-create-task graph-uuid private-key]
(m/sp
(let [encrypted-aes-key (c.m/<? (<get-item (graph-encrypted-aes-key-idb-key graph-uuid)))]
(if encrypted-aes-key
(c.m/<? (crypt/<decrypt-aes-key private-key encrypted-aes-key))
(when-let [encrypted-aes-key (m/? (task--remote-fetch-graph-encrypted-aes-key get-ws-create-task graph-uuid))]
(let [aes-key (c.m/<? (crypt/<decrypt-aes-key private-key encrypted-aes-key))]
(c.m/<? (<set-item! (graph-encrypted-aes-key-idb-key graph-uuid) encrypted-aes-key))
aes-key))))))
(defn task--persist-graph-encrypted-aes-key
[graph-uuid encrypted-aes-key]
(m/sp
(c.m/<? (<set-item! (graph-encrypted-aes-key-idb-key graph-uuid) encrypted-aes-key))))
(defn task--generate-graph-aes-key
[]
(m/sp (c.m/<? (crypt/<generate-aes-key))))
(defn task--get-decrypted-rsa-key-pair
[get-ws-create-task user-uuid]
(m/sp
(let [{:keys [public-key encrypted-private-key]}
(m/? (task--fetch-user-rsa-key-pair get-ws-create-task user-uuid))
exported-private-key (c.m/<? (worker-state/<invoke-main-thread
:thread-api/decrypt-user-e2ee-private-key encrypted-private-key))
private-key (c.m/<? (crypt/<import-private-key exported-private-key))]
{:public-key public-key
:private-key private-key})))
(defn task--get-aes-key
"Return nil if not exists"
[get-ws-create-task user-uuid graph-uuid]
(m/sp
(let [{:keys [_public-key private-key]} (m/? (task--get-decrypted-rsa-key-pair get-ws-create-task user-uuid))]
(m/? (task--fetch-graph-aes-key get-ws-create-task graph-uuid private-key)))))
(defn task--reset-user-rsa-private-key
"Throw if decrypt encrypted-private-key failed."
[get-ws-create-task refresh-token user-uuid old-password new-password]
(m/sp
(let [{:keys [public-key encrypted-private-key]}
(m/? (task--fetch-user-rsa-key-pair get-ws-create-task user-uuid))
private-key (c.m/<? (crypt/<decrypt-private-key old-password encrypted-private-key))
new-encrypted-private-key (c.m/<? (crypt/<encrypt-private-key new-password private-key))]
(m/? (task--upload-user-rsa-key-pair get-ws-create-task user-uuid public-key new-encrypted-private-key
:reset-private-key true))
(c.m/<? (<save-e2ee-password refresh-token new-password)))))
(defn- task--fetch-user-rsa-public-key
"Fetches the user's RSA public-key from server.
Return js/CryptoKey.
Return nil if not exists"
[get-ws-create-task user-email]
(m/sp
(let [{:keys [public-key] :as response}
(m/? (ws-util/send&recv get-ws-create-task
{:action "fetch-user-rsa-public-key"
:user/email user-email}))]
(if (:ex-data response)
(throw (ex-info (:ex-message response)
(assoc (:ex-data response)
:type :rtc.exception/fetch-user-rsa-public-key-error)))
(when public-key
(c.m/<? (<import-public-key-transit-str public-key)))))))
(defn task--encrypt-graph-aes-key-by-other-user-public-key
"Return encrypted-aes-key,
which is decrypted by current user's private-key, then other-user's public-key"
[get-ws-create-task graph-uuid user-uuid other-user-email]
(m/sp
(when-let [graph-aes-key (m/? (task--get-aes-key get-ws-create-task user-uuid graph-uuid))]
(when-let [public-key (m/? (task--fetch-user-rsa-public-key get-ws-create-task other-user-email))]
(c.m/<? (crypt/<encrypt-aes-key public-key graph-aes-key))))))
(def-thread-api :thread-api/get-user-rsa-key-pair
[token user-uuid]
(m/sp
(let [{:keys [get-ws-create-task]} (ws-util/gen-get-ws-create-map--memoized (ws-util/get-ws-url token))
{:keys [public-key encrypted-private-key]}
(m/? (task--fetch-user-rsa-key-pair get-ws-create-task user-uuid))]
(when (and public-key encrypted-private-key)
{:public-key (c.m/<? (crypt/<export-public-key public-key))
:encrypted-private-key encrypted-private-key}))))
(def-thread-api :thread-api/init-user-rsa-key-pair
[token refresh-token user-uuid]
(m/sp
(try
(let [{:keys [get-ws-create-task]} (ws-util/gen-get-ws-create-map--memoized (ws-util/get-ws-url token))]
(when-not (m/? (task--fetch-user-rsa-key-pair get-ws-create-task user-uuid))
(let [{:keys [publicKey privateKey]} (c.m/<? (crypt/<generate-rsa-key-pair))
{:keys [password]} (c.m/<? (worker-state/<invoke-main-thread :thread-api/request-e2ee-password))
encrypted-private-key (c.m/<? (crypt/<encrypt-private-key password privateKey))]
(m/? (task--upload-user-rsa-key-pair get-ws-create-task user-uuid publicKey encrypted-private-key))
(c.m/<? (<save-e2ee-password refresh-token password))
nil)))
(catch Cancelled _)
(catch :default e e))))
(def-thread-api :thread-api/reset-user-rsa-key-pair
[token refresh-token user-uuid new-password]
(m/sp
(try
(let [{:keys [get-ws-create-task]} (ws-util/gen-get-ws-create-map--memoized (ws-util/get-ws-url token))]
(when (some? (m/? (task--fetch-user-rsa-key-pair get-ws-create-task user-uuid)))
(let [{:keys [publicKey privateKey]} (c.m/<? (crypt/<generate-rsa-key-pair))
encrypted-private-key (c.m/<? (crypt/<encrypt-private-key new-password privateKey))]
(m/? (task--reset-user-rsa-key-pair get-ws-create-task user-uuid publicKey encrypted-private-key))
(c.m/<? (<save-e2ee-password refresh-token new-password))
nil)))
(catch Cancelled _)
(catch :default e e))))
(def-thread-api :thread-api/reset-e2ee-password
[token refresh-token user-uuid old-password new-password]
(m/sp
(let [{:keys [get-ws-create-task]} (ws-util/gen-get-ws-create-map--memoized (ws-util/get-ws-url token))]
(m/? (task--reset-user-rsa-private-key get-ws-create-task refresh-token user-uuid old-password new-password)))))
(def-thread-api :thread-api/get-e2ee-password
[refresh-token]
(-> (p/let [password (<read-e2ee-password refresh-token)]
{:password password})
(p/catch (fn [e]
(log/error :read-e2ee-password e)
(ex-info ":thread-api/get-e2ee-password" {})))))
(def-thread-api :thread-api/save-e2ee-password
[refresh-token password]
(<save-e2ee-password refresh-token password))

4
src/main/frontend/worker/rtc/db.cljs
View File

@@ -9,14 +9,14 @@
(when-let [conn (worker-state/get-datascript-conn repo)]
(ldb/transact! conn [[:db/retractEntity :logseq.kv/graph-uuid]
[:db/retractEntity :logseq.kv/graph-local-tx]
[:db/retractEntity :logseq.kv/remote-schema-version]])))
[:db/retractEntity :logseq.kv/remote-schema-version]
[:db/retractEntity :logseq.kv/graph-rtc-e2ee?]])))
(defn reset-client-op-conn
[repo]
(when-let [conn (worker-state/get-client-ops-conn repo)]
(let [tx-data (->> (concat (d/datoms @conn :avet :graph-uuid)
(d/datoms @conn :avet :local-tx)
(d/datoms @conn :avet :aes-key-jwk)
(d/datoms @conn :avet :block/uuid))
(map (fn [datom] [:db/retractEntity (:e datom)])))]
(ldb/transact! conn tx-data))))

24
src/main/frontend/worker/rtc/exception.cljs
View File

@@ -20,21 +20,21 @@ Trying to start rtc loop but there's already one running, need to cancel that on
graph doesn't have :logseq.kv/remote-schema-version value"}
:rtc.exception/major-schema-version-mismatched {:doc "Local exception.
local-schema-version, remote-schema-version, app-schema-version are not equal, cannot start rtc"}
:rtc.exception/local-graph-too-old {:doc "Local exception.
Local graph's tx is too old, need to pull earlier remote-data first"}
:rtc.exception/get-s3-object-failed {:doc "Failed to fetch response from s3.
When response from remote is too huge(> 32KB),
the server will put it to s3 and return its presigned-url to clients."}
:rtc.exception/bad-request-body {:doc "bad request body, rejected by server-schema"}
:rtc.exception/not-allowed {:doc "this api-call is not allowed"}
:rtc.exception/ws-timeout {:doc "websocket timeout"})
:rtc.exception/ws-timeout {:doc "websocket timeout"}
(def ex-ws-already-disconnected
(ex-info "websocket conn is already disconnected" {:type :rtc.exception/ws-already-disconnected}))
(def ex-remote-graph-not-exist
(ex-info "remote graph not exist" {:type :rtc.exception/remote-graph-not-exist}))
(def ex-remote-graph-not-ready
(ex-info "remote graph still creating" {:type :rtc.exception/remote-graph-not-ready}))
:rtc.exception/fetch-user-rsa-key-pair-error {:doc "Failed to fetch user RSA key pair from server"}
:rtc.exception/fetch-user-rsa-public-key-error {:doc "Failed to fetch user RSA public-key from server"}
:rtc.exception/fetch-graph-aes-key-error {:doc "Failed to fetch graph AES key from server"}
:rtc.exception/not-found-user-rsa-key-pair {:doc "user rsa-key-pair not found"}
:rtc.exception/not-found-graph-aes-key {:doc "graph aes-key not found"})
(def ex-remote-graph-lock-missing
(ex-info "remote graph lock missing(server internal error)"
@@ -43,12 +43,6 @@ the server will put it to s3 and return its presigned-url to clients."}
(def ex-local-not-rtc-graph
(ex-info "RTC is not supported for this local-graph" {:type :rtc.exception/not-rtc-graph}))
(def ex-bad-request-body
(ex-info "bad request body" {:type :rtc.exception/bad-request-body}))
(def ex-not-allowed
(ex-info "not allowed" {:type :rtc.exception/not-allowed}))
(def ex-unknown-server-error
(ex-info "Unknown server error" {:type :rtc.exception/unknown-server-error}))

153
src/main/frontend/worker/rtc/full_upload_download_graph.cljs
View File

@@ -4,13 +4,14 @@
(:require [cljs-http-missionary.client :as http]
[clojure.set :as set]
[datascript.core :as d]
[frontend.common.crypt :as crypt]
[frontend.common.missionary :as c.m]
[frontend.common.thread-api :as thread-api]
[frontend.worker-common.util :as worker-util]
[frontend.worker.crypt :as crypt]
[frontend.worker.db-metadata :as worker-db-metadata]
[frontend.worker.rtc.client-op :as client-op]
[frontend.worker.rtc.const :as rtc-const]
[frontend.worker.rtc.crypt :as rtc-crypt]
[frontend.worker.rtc.db :as rtc-db]
[frontend.worker.rtc.log-and-state :as rtc-log-and-state]
[frontend.worker.rtc.ws-util :as ws-util]
@@ -122,50 +123,84 @@
(:db/ident block) (update :db/ident ldb/read-transit-str)
(:block/order block) (update :block/order ldb/read-transit-str)))))))
(defn- task--encrypt-blocks
[encrypt-key encrypt-attr-set blocks]
(m/sp
(loop [[block & rest-blocks] blocks
result []]
(if-not block
result
(let [block' (c.m/<? (crypt/<encrypt-map encrypt-key encrypt-attr-set block))]
(recur rest-blocks (conj result block')))))))
(comment
(def db @(frontend.worker.state/get-datascript-conn (frontend.worker.state/get-current-repo)))
(def blocks (export-as-blocks db))
(def salt (rtc-encrypt/gen-salt))
(def canceler ((m/sp
(let [k (c.m/<? (rtc-encrypt/<salt+password->key salt "password"))]
(m/? (task--encrypt-blocks k #{:block/title :block/name} blocks))))
#(def encrypted-blocks %) prn)))
(defn new-task--upload-graph
[get-ws-create-task repo conn remote-graph-name major-schema-version]
(m/sp
(rtc-log-and-state/rtc-log :rtc.log/upload {:sub-type :fetching-presigned-put-url
:message "fetching presigned put-url"})
(let [[{:keys [url key]} all-blocks-str]
(m/?
(m/join
vector
(ws-util/send&recv get-ws-create-task {:action "presign-put-temp-s3-obj"})
(m/sp
(let [all-blocks (export-as-blocks
@conn
:ignore-attr-set rtc-const/ignore-attrs-when-init-upload
:ignore-entity-set rtc-const/ignore-entities-when-init-upload)]
(ldb/write-transit-str all-blocks)))))]
(rtc-log-and-state/rtc-log :rtc.log/upload {:sub-type :upload-data
:message "uploading data"})
(m/? (http/put url {:body all-blocks-str :with-credentials? false}))
(rtc-log-and-state/rtc-log :rtc.log/upload {:sub-type :request-upload-graph
:message "requesting upload-graph"})
(let [aes-key (c.m/<? (crypt/<gen-aes-key))
aes-key-jwk (ldb/write-transit-str (c.m/<? (crypt/<export-key aes-key)))
upload-resp
(m/? (ws-util/send&recv get-ws-create-task {:action "upload-graph"
:s3-key key
:schema-version (str major-schema-version)
:graph-name remote-graph-name}))]
(if-let [graph-uuid (:graph-uuid upload-resp)]
(let [schema-version (ldb/get-graph-schema-version @conn)]
(ldb/transact! conn
[(ldb/kv :logseq.kv/graph-uuid graph-uuid)
(ldb/kv :logseq.kv/graph-local-tx "0")
(ldb/kv :logseq.kv/remote-schema-version schema-version)])
(client-op/update-graph-uuid repo graph-uuid)
(client-op/remove-local-tx repo)
(client-op/update-local-tx repo 1)
(client-op/add-all-exists-asset-as-ops repo)
(crypt/store-graph-keys-jwk repo aes-key-jwk)
(c.m/<? (worker-db-metadata/<store repo (pr-str {:kv/value graph-uuid})))
(rtc-log-and-state/rtc-log :rtc.log/upload {:sub-type :upload-completed
:message "upload-graph completed"})
{:graph-uuid graph-uuid})
(throw (ex-info "upload-graph failed" {:upload-resp upload-resp})))))))
(rtc-log-and-state/rtc-log :rtc.log/upload {:sub-type :generate-aes-key
:message "generate aes-encrypt-key"})
(let [aes-key (m/? (rtc-crypt/task--generate-graph-aes-key))
user-uuid (some-> (worker-state/get-id-token)
worker-util/parse-jwt
:sub)
public-key (when user-uuid
(:public-key (m/? (rtc-crypt/task--fetch-user-rsa-key-pair get-ws-create-task user-uuid))))]
(when-not public-key
(throw (ex-info "user public-key not found" {:type :rtc.exception/not-found-user-rsa-key-pair
:user-uuid user-uuid})))
(let [encrypted-aes-key (c.m/<? (crypt/<encrypt-aes-key public-key aes-key))
_ (ldb/transact! conn [(ldb/kv :logseq.kv/graph-rtc-e2ee? true)])
_ (rtc-log-and-state/rtc-log :rtc.log/upload {:sub-type :fetching-presigned-put-url
:message "fetching presigned put-url"})
[{:keys [url key]} all-blocks-str]
(m/?
(m/join
vector
(ws-util/send&recv get-ws-create-task {:action "presign-put-temp-s3-obj"})
(m/sp
(let [all-blocks (export-as-blocks
@conn
:ignore-attr-set rtc-const/ignore-attrs-when-init-upload
:ignore-entity-set rtc-const/ignore-entities-when-init-upload)
encrypted-blocks (c.m/<? (task--encrypt-blocks aes-key rtc-const/encrypt-attr-set all-blocks))]
(ldb/write-transit-str encrypted-blocks)))))]
(rtc-log-and-state/rtc-log :rtc.log/upload {:sub-type :upload-data
:message "uploading data"})
(m/? (http/put url {:body all-blocks-str :with-credentials? false}))
(rtc-log-and-state/rtc-log :rtc.log/upload {:sub-type :request-upload-graph
:message "requesting upload-graph"})
(let [upload-resp
(m/? (ws-util/send&recv get-ws-create-task {:action "upload-graph"
:s3-key key
:schema-version (str major-schema-version)
:graph-name remote-graph-name
:encrypted-aes-key
(ldb/write-transit-str encrypted-aes-key)}))]
(if-let [graph-uuid (:graph-uuid upload-resp)]
(let [schema-version (ldb/get-graph-schema-version @conn)]
(ldb/transact! conn
[(ldb/kv :logseq.kv/graph-uuid graph-uuid)
(ldb/kv :logseq.kv/graph-local-tx "0")
(ldb/kv :logseq.kv/remote-schema-version schema-version)])
(client-op/update-graph-uuid repo graph-uuid)
(client-op/remove-local-tx repo)
(client-op/update-local-tx repo 1)
(client-op/add-all-exists-asset-as-ops repo)
(c.m/<? (worker-db-metadata/<store repo (pr-str {:kv/value graph-uuid})))
(m/? (rtc-crypt/task--persist-graph-encrypted-aes-key graph-uuid encrypted-aes-key))
(rtc-log-and-state/rtc-log :rtc.log/upload {:sub-type :upload-completed
:message "upload-graph completed"})
{:graph-uuid graph-uuid})
(throw (ex-info "upload-graph failed" {:upload-resp upload-resp}))))))))
(defn- fill-block-fields
[blocks]
@@ -356,6 +391,29 @@
:init-tx-data init-tx-data
:tx-data tx-data}))
(defn- task--decrypt-blocks-aux
[aes-key encrypt-attr-set blocks]
(m/sp
(loop [[block & rest-blocks] blocks
result []]
(if-not block
result
(let [block* (c.m/<? (crypt/<decrypt-map aes-key encrypt-attr-set block))]
(recur rest-blocks (conj result block*)))))))
(defn- task--decrypt-blocks
[graph-uuid blocks]
(m/sp
(let [token (worker-state/get-id-token)
user-uuid (:sub (worker-util/parse-jwt token))
_ (assert (and token user-uuid))
{:keys [get-ws-create-task]}
(ws-util/gen-get-ws-create-map--memoized (ws-util/get-ws-url token))
aes-key (m/? (rtc-crypt/task--get-aes-key get-ws-create-task user-uuid graph-uuid))]
(if aes-key
(m/? (task--decrypt-blocks-aux aes-key rtc-const/encrypt-attr-set blocks))
blocks))))
(defn- new-task--transact-remote-all-blocks!
[all-blocks repo graph-uuid]
(let [{:keys [remote-t init-tx-data tx-data]}
@@ -455,9 +513,11 @@
(rtc-log-and-state/rtc-log :rtc.log/download {:sub-type :transact-graph-data-to-db
:message "transacting graph data to local db"
:graph-uuid graph-uuid})
(let [all-blocks (ldb/read-transit-str body)]
(let [all-blocks (ldb/read-transit-str body)
blocks* (m/? (task--decrypt-blocks graph-uuid (:blocks all-blocks)))
all-blocks* (assoc all-blocks :blocks blocks*)]
(worker-state/set-rtc-downloading-graph! true)
(m/? (new-task--transact-remote-all-blocks! all-blocks repo graph-uuid))
(m/? (new-task--transact-remote-all-blocks! all-blocks* repo graph-uuid))
(rtc-log-and-state/rtc-log :rtc.log/download {:sub-type :transacted-all-blocks
:message "transacted all blocks"
:graph-uuid graph-uuid})
@@ -491,9 +551,7 @@
(m/? (http/put url {:body all-blocks-str :with-credentials? false}))
(rtc-log-and-state/rtc-log :rtc.log/branch-graph {:sub-type :request-branch-graph
:message "requesting branch-graph"})
(let [aes-key (c.m/<? (crypt/<gen-aes-key))
aes-key-jwk (ldb/write-transit-str (c.m/<? (crypt/<export-key aes-key)))
resp (m/? (ws-util/send&recv get-ws-create-task {:action "branch-graph"
(let [resp (m/? (ws-util/send&recv get-ws-create-task {:action "branch-graph"
:s3-key key
:schema-version (str major-schema-version)
:graph-uuid graph-uuid}))]
@@ -506,7 +564,6 @@
(client-op/update-graph-uuid repo graph-uuid)
(client-op/remove-local-tx repo)
(client-op/add-all-exists-asset-as-ops repo)
(crypt/store-graph-keys-jwk repo aes-key-jwk)
(c.m/<? (worker-db-metadata/<store repo (pr-str {:kv/value graph-uuid})))
(rtc-log-and-state/rtc-log :rtc.log/branch-graph {:sub-type :completed
:message "branch-graph completed"})

70
src/main/frontend/worker/rtc/malli_schema.cljs
View File

@@ -132,6 +132,7 @@
[:map
[:t :int]
[:t-before :int]
[:t-query-end {:optional true} :int] ;TODO: remove 'optional' later, be compatible with old-clients for now
[:affected-blocks
[:map-of :uuid
[:multi {:dispatch :op :decode/string #(update % :op keyword)}
@@ -243,6 +244,27 @@
[:graph<->user/user-type :keyword]
[:user/online? :boolean]]]]]]
["inject-users-info" [:map]]
;; keys manage
["fetch-user-rsa-key-pair"
[:map
[:public-key [:maybe :string]]
[:encrypted-private-key [:maybe :string]]]]
["fetch-graph-encrypted-aes-key"
[:map
[:encrypted-aes-key [:maybe :string]]]]
["fetch-user-rsa-public-key"
[:map
[:public-key [:maybe :string]]]]
["upload-user-rsa-key-pair"
[:map
[:public-key :string]
[:encrypted-private-key :string]]]
["reset-user-rsa-key-pair"
[:map
[:public-key :string]
[:encrypted-private-key :string]]]
[nil data-from-ws-schema-fallback]]))
(def data-from-ws-coercer (m/coercer data-from-ws-schema mt/string-transformer nil
@@ -261,6 +283,7 @@
["init-request"
[:map
[:graph-uuid :uuid]
[:api-version :string]
[:schema-version db-schema/major-schema-version-string-schema]
[:t-before :int]
[:get-graph-skeleton :boolean]]]
@@ -273,6 +296,7 @@
[:map
[:req-id :string]
[:action :string]
[:api-version :string]
[:profile {:optional true} :boolean]
[:graph-uuid :uuid]
[:schema-version db-schema/major-schema-version-string-schema]
@@ -289,7 +313,8 @@
[:map
[:s3-key :string]
[:graph-name :string]
[:schema-version db-schema/major-schema-version-string-schema]]]
[:schema-version db-schema/major-schema-version-string-schema]
[:encrypted-aes-key {:optional true} :string]]]
["branch-graph"
[:map
[:s3-key :string]
@@ -306,8 +331,11 @@
["grant-access"
[:map
[:graph-uuid :uuid]
[:target-user-uuids {:optional true} [:sequential :uuid]]
[:target-user-emails {:optional true} [:sequential :string]]]]
[:target-user-email+encrypted-aes-key-coll
[:sequential
[:map
[:user/email :string]
[:encrypted-aes-key [:maybe :string]]]]]]]
["get-users-info"
[:map
[:graph-uuid :uuid]]]
@@ -349,31 +377,27 @@
[:graph-uuid :uuid]
[:schema-version db-schema/major-schema-version-string-schema]
[:asset-uuids [:sequential :uuid]]]]
["get-user-devices"
[:map]]
["add-user-device"
;; ================================================================
["upload-user-rsa-key-pair"
[:map
[:device-name :string]]]
["remove-user-device"
[:user-uuid :uuid]
[:public-key {:optional true} :string]
[:encrypted-private-key :string]
[:reset-private-key {:optional true} :boolean]]]
["reset-user-rsa-key-pair"
[:map
[:device-uuid :uuid]]]
["update-user-device-name"
[:user-uuid :uuid]
[:public-key :string]
[:encrypted-private-key :string]]]
["fetch-user-rsa-key-pair"
[:map
[:device-uuid :uuid]
[:device-name :string]]]
["add-device-public-key"
[:user-uuid :uuid]]]
["fetch-graph-encrypted-aes-key"
[:map
[:device-uuid :uuid]
[:key-name :string]
[:public-key :string]]]
["remove-device-public-key"
[:graph-uuid :uuid]]]
["fetch-user-rsa-public-key"
[:map
[:device-uuid :uuid]
[:key-name :string]]]
["sync-encrypted-aes-key"
[:map
[:device-uuid->encrypted-aes-key [:map-of :uuid :string]]
[:graph-uuid :uuid]]]])))
[:user/email :string]]]])))
(def data-to-ws-encoder (m/encoder data-to-ws-schema (mt/transformer
mt/string-transformer

135
src/main/frontend/worker/rtc/remote_update.cljs
View File

@@ -4,6 +4,8 @@
[clojure.set :as set]
[clojure.string :as string]
[datascript.core :as d]
[frontend.common.crypt :as crypt]
[frontend.common.missionary :as c.m]
[frontend.worker-common.util :as worker-util]
[frontend.worker.handler.page :as worker-page]
[frontend.worker.rtc.asset :as r.asset]
@@ -14,7 +16,6 @@
[frontend.worker.state :as worker-state]
[lambdaisland.glogi :as log]
[logseq.clj-fractional-indexing :as index]
[logseq.common.defkeywords :refer [defkeywords]]
[logseq.common.util :as common-util]
[logseq.db :as ldb]
[logseq.db.common.property-util :as db-property-util]
@@ -22,12 +23,8 @@
[logseq.graph-parser.whiteboard :as gp-whiteboard]
[logseq.outliner.batch-tx :as batch-tx]
[logseq.outliner.core :as outliner-core]
[logseq.outliner.transaction :as outliner-tx]))
(defkeywords
::need-pull-remote-data {:doc "
remote-update's :remote-t-before > :local-tx,
so need to pull earlier remote-data from websocket."})
[logseq.outliner.transaction :as outliner-tx]
[missionary.core :as m]))
(defmulti ^:private transact-db! (fn [action & _args] action))
@@ -600,13 +597,38 @@ so need to pull earlier remote-data from websocket."})
:parents [(:block/parent refed-block)])
(dissoc :block/uuid))])))))))
(defn apply-remote-update
"Apply remote-update(`remote-update-event`)"
[graph-uuid repo conn date-formatter remote-update-event add-log-fn]
(defn task--decrypt-blocks-in-remote-update-data
[aes-key encrypt-attr-set remote-update-data]
(assert aes-key)
(m/sp
(let [{affected-blocks-map :affected-blocks refed-blocks :refed-blocks} remote-update-data
affected-blocks-map'
(loop [[[block-uuid affected-block] & rest-affected-blocks] affected-blocks-map
affected-blocks-map-result {}]
(if-not block-uuid
affected-blocks-map-result
(let [affected-block' (c.m/<? (crypt/<decrypt-map aes-key encrypt-attr-set affected-block))]
(recur rest-affected-blocks (assoc affected-blocks-map-result block-uuid affected-block')))))
refed-blocks'
(loop [[refed-block & rest-refed-blocks] refed-blocks
refed-blocks-result []]
(if-not refed-block
refed-blocks-result
(let [refed-block' (c.m/<? (crypt/<decrypt-map aes-key encrypt-attr-set refed-block))]
(recur rest-refed-blocks (conj refed-blocks-result refed-block')))))]
(assoc remote-update-data
:affected-blocks affected-blocks-map'
:refed-blocks refed-blocks'))))
(defn apply-remote-update-check
"If the check passes, return true"
[repo remote-update-event add-log-fn]
(let [remote-update-data (:value remote-update-event)]
(assert (rtc-schema/data-from-ws-validator remote-update-data) remote-update-data)
(let [remote-t (:t remote-update-data)
remote-t-before (:t-before remote-update-data)
(let [{remote-latest-t :t
remote-t-before :t-before
remote-t :t-query-end} remote-update-data
remote-t (or remote-t remote-latest-t) ;TODO: remove this, be compatible with old-clients for now
local-tx (client-op/get-local-tx repo)]
(cond
(not (and (pos? remote-t)
@@ -614,47 +636,70 @@ so need to pull earlier remote-data from websocket."})
(throw (ex-info "invalid remote-data" {:data remote-update-data}))
(<= remote-t local-tx)
(add-log-fn :rtc.log/apply-remote-update {:sub-type :skip :remote-t remote-t :local-t local-tx})
(do (add-log-fn :rtc.log/apply-remote-update
{:sub-type :skip
:remote-t remote-t
:remote-latest-t remote-latest-t
:local-t local-tx})
false)
(< local-tx remote-t-before)
(do (add-log-fn :rtc.log/apply-remote-update {:sub-type :need-pull-remote-data
:remote-t remote-t :local-t local-tx
:remote-latest-t remote-latest-t
:remote-t remote-t
:local-t local-tx
:remote-t-before remote-t-before})
(throw (ex-info "need pull earlier remote-data"
{:type ::need-pull-remote-data
{:type :rtc.exception/local-graph-too-old
:local-tx local-tx})))
(<= remote-t-before local-tx remote-t)
(let [{affected-blocks-map :affected-blocks refed-blocks :refed-blocks} remote-update-data
{:keys [remove-ops-map move-ops-map update-ops-map update-page-ops-map remove-page-ops-map]}
(affected-blocks->diff-type-ops repo affected-blocks-map)
remove-ops (vals remove-ops-map)
sorted-move-ops (move-ops-map->sorted-move-ops move-ops-map)
update-ops (vals update-ops-map)
update-page-ops (vals update-page-ops-map)
remove-page-ops (vals remove-page-ops-map)
db-before @conn]
(rtc-log-and-state/update-remote-t graph-uuid remote-t)
(js/console.groupCollapsed "rtc/apply-remote-ops-log")
(batch-tx/with-batch-tx-mode conn {:rtc-tx? true
:persist-op? false
:gen-undo-ops? false}
(worker-util/profile :ensure-refed-blocks-exist (ensure-refed-blocks-exist repo conn refed-blocks))
(worker-util/profile :apply-remote-update-page-ops (apply-remote-update-page-ops repo conn update-page-ops))
(worker-util/profile :apply-remote-move-ops (apply-remote-move-ops repo conn sorted-move-ops))
(worker-util/profile :apply-remote-update-ops (apply-remote-update-ops repo conn update-ops))
(worker-util/profile :apply-remote-remove-page-ops (apply-remote-remove-page-ops repo conn remove-page-ops)))
;; NOTE: we cannot set :persist-op? = true when batch-tx/with-batch-tx-mode (already set to false)
;; and there're some transactions in `apply-remote-remove-ops` need to :persist-op?=true
(worker-util/profile :apply-remote-remove-ops (apply-remote-remove-ops repo conn date-formatter remove-ops))
;; wait all remote-ops transacted into db,
;; then start to check any asset-updates in remote
(let [db-after @conn]
(r.asset/emit-remote-asset-updates-from-block-ops db-before db-after remove-ops update-ops))
(js/console.groupEnd)
(<= remote-t-before local-tx remote-t) true
(client-op/update-local-tx repo remote-t)
(rtc-log-and-state/update-local-t graph-uuid remote-t))
:else (throw (ex-info "unreachable" {:remote-t remote-t
:remote-t-before remote-t-before
:remote-latest-t remote-latest-t
:local-t local-tx}))))))
(defn task--apply-remote-update
"Apply remote-update(`remote-update-event`)"
[graph-uuid repo conn date-formatter remote-update-event aes-key add-log-fn]
(m/sp
(when (apply-remote-update-check repo remote-update-event add-log-fn)
(let [remote-update-data (:value remote-update-event)
remote-update-data (if aes-key
(m/? (task--decrypt-blocks-in-remote-update-data
aes-key rtc-const/encrypt-attr-set
remote-update-data))
remote-update-data)
;; TODO: remove this 'or', be compatible with old-clients for now
remote-t (or (:t-query-end remote-update-data) (:t remote-update-data))
{affected-blocks-map :affected-blocks refed-blocks :refed-blocks} remote-update-data
{:keys [remove-ops-map move-ops-map update-ops-map update-page-ops-map remove-page-ops-map]}
(affected-blocks->diff-type-ops repo affected-blocks-map)
remove-ops (vals remove-ops-map)
sorted-move-ops (move-ops-map->sorted-move-ops move-ops-map)
update-ops (vals update-ops-map)
update-page-ops (vals update-page-ops-map)
remove-page-ops (vals remove-page-ops-map)
db-before @conn]
(rtc-log-and-state/update-remote-t graph-uuid remote-t)
(js/console.groupCollapsed "rtc/apply-remote-ops-log")
(batch-tx/with-batch-tx-mode conn {:rtc-tx? true
:persist-op? false
:gen-undo-ops? false}
(worker-util/profile :ensure-refed-blocks-exist (ensure-refed-blocks-exist repo conn refed-blocks))
(worker-util/profile :apply-remote-update-page-ops (apply-remote-update-page-ops repo conn update-page-ops))
(worker-util/profile :apply-remote-move-ops (apply-remote-move-ops repo conn sorted-move-ops))
(worker-util/profile :apply-remote-update-ops (apply-remote-update-ops repo conn update-ops))
(worker-util/profile :apply-remote-remove-page-ops (apply-remote-remove-page-ops repo conn remove-page-ops)))
;; NOTE: we cannot set :persist-op? = true when batch-tx/with-batch-tx-mode (already set to false)
;; and there're some transactions in `apply-remote-remove-ops` need to :persist-op?=true
(worker-util/profile :apply-remote-remove-ops (apply-remote-remove-ops repo conn date-formatter remove-ops))
;; wait all remote-ops transacted into db,
;; then start to check any asset-updates in remote
(let [db-after @conn]
(r.asset/emit-remote-asset-updates-from-block-ops db-before db-after remove-ops update-ops))
(js/console.groupEnd)
(client-op/update-local-tx repo remote-t)
(rtc-log-and-state/update-local-t graph-uuid remote-t)))))

37
src/main/frontend/worker/rtc/ws_util.cljs
View File

@@ -3,7 +3,6 @@
(:require [cljs-http-missionary.client :as http]
[frontend.worker-common.util :as worker-util]
[frontend.worker.rtc.db :as rtc-db]
[frontend.worker.rtc.exception :as r.ex]
[frontend.worker.rtc.malli-schema :as rtc-schema]
[frontend.worker.rtc.ws :as ws]
[frontend.worker.state :as worker-state]
@@ -11,17 +10,26 @@
[logseq.graph-parser.utf8 :as utf8]
[missionary.core :as m]))
(def ^:private remote-e-type->ex-info
{:ws-conn-already-disconnected
(ex-info "websocket conn is already disconnected" {:type :rtc.exception/ws-already-disconnected})
:graph-not-exist
(ex-info "remote graph not exist" {:type :rtc.exception/remote-graph-not-exist})
:graph-not-ready
(ex-info "remote graph still creating" {:type :rtc.exception/remote-graph-not-ready})
:bad-request-body
(ex-info "bad request body" {:type :rtc.exception/bad-request-body})
:not-allowed
(ex-info "not allowed" {:type :rtc.exception/not-allowed})
:client-graph-too-old
(ex-info "local graph too old" {:type :rtc.exception/local-graph-too-old})})
(defn- handle-remote-ex
[resp]
(when (= :graph-not-exist (:type (:ex-data resp)))
(rtc-db/remove-rtc-data-in-conn! (worker-state/get-current-repo))
(worker-util/post-message :remote-graph-gone []))
(if-let [e ({:ws-conn-already-disconnected r.ex/ex-ws-already-disconnected
:graph-not-exist r.ex/ex-remote-graph-not-exist
:graph-not-ready r.ex/ex-remote-graph-not-ready
:bad-request-body r.ex/ex-bad-request-body
:not-allowed r.ex/ex-not-allowed}
(:type (:ex-data resp)))]
(if-let [e (get remote-e-type->ex-info (:type (:ex-data resp)))]
(throw e)
resp))
@@ -31,8 +39,9 @@
{:pre [(= "apply-ops" (:action message))]}
(m/sp
(let [decoded-message (rtc-schema/data-to-ws-coercer (assoc message :req-id "temp-id"))
message-str (js/JSON.stringify (clj->js (select-keys (rtc-schema/data-to-ws-encoder decoded-message)
["graph-uuid" "ops" "t-before" "schema-version"])))
message-str (js/JSON.stringify
(clj->js (select-keys (rtc-schema/data-to-ws-encoder decoded-message)
["graph-uuid" "ops" "t-before" "schema-version" "api-version"])))
len (.-length (utf8/encode message-str))]
(when (< 100000 len)
(let [{:keys [url key]} (m/? (ws/send&recv ws {:action "presign-put-temp-s3-obj"}))
@@ -46,16 +55,12 @@
This function will attempt to reconnect and retry once after the ws closed(js/CloseEvent).
For huge apply-ops request(>100KB),
- upload its request message to s3 first,
then add `s3-key` key to request message map
For huge apply-ops request(> 400 ops)
- adjust its timeout to 20s"
[get-ws-create-task message]
then add `s3-key` key to request message map"
[get-ws-create-task message & {:keys [timeout-ms] :or {timeout-ms 10000}}]
(let [task--helper
(m/sp
(let [ws (m/? get-ws-create-task)
opts (when (and (= "apply-ops" (:action message))
(< 400 (count (:ops message))))
{:timeout-ms 20000})
opts {:timeout-ms timeout-ms}
s3-key (when (= "apply-ops" (:action message))
(m/? (put-apply-ops-message-on-s3-if-too-huge ws message)))
message* (if s3-key

12
src/main/logseq/api.cljs
View File

@@ -16,6 +16,7 @@
[logseq.api.editor :as api-editor]
[logseq.api.file-based :as file-based-api]
[logseq.api.plugin :as api-plugin]
[logseq.db.sqlite.util :as sqlite-util]
[logseq.sdk.assets :as sdk-assets]
[logseq.sdk.core]
[logseq.sdk.experiments]
@@ -208,12 +209,21 @@
(def ^:export remove_tag_property db-based-api/tag-remove-property)
;; Internal db-based CLI APIs
;; CLI APIs should use ensure-db-graph unless they have a nested check in cli-common-mcp-tools ns
(defn- ensure-db-graph
[f]
(fn ensure-db-graph-wrapper [& args]
(when-not (sqlite-util/db-based-graph? (state/get-current-repo))
(throw (ex-info "This endpoint must be called on a DB graph" {})))
(apply f args)))
(def ^:export list_tags cli-based-api/list-tags)
(def ^:export list_properties cli-based-api/list-properties)
(def ^:export list_pages cli-based-api/list-pages)
(def ^:export get_page_data cli-based-api/get-page-data)
(def ^:export upsert_nodes cli-based-api/upsert-nodes)
(def ^:export import_edn cli-based-api/import-edn)
(def ^:export import_edn (ensure-db-graph cli-based-api/import-edn))
(def ^:export export_edn (ensure-db-graph cli-based-api/export-edn))
;; file based graph APIs
(def ^:export get_current_graph_templates file-based-api/get_current_graph_templates)

22
src/main/logseq/api/db_based/cli.cljs
View File

@@ -1,12 +1,14 @@
(ns logseq.api.db-based.cli
"API fns for CLI"
(:require [frontend.handler.ui :as ui-handler]
(:require [clojure.string :as string]
[frontend.handler.ui :as ui-handler]
[frontend.modules.outliner.op :as outliner-op]
[frontend.modules.outliner.ui :as ui-outliner-tx]
[frontend.state :as state]
[logseq.cli.common.mcp.tools :as cli-common-mcp-tools]
[promesa.core :as p]
[logseq.db.sqlite.util :as sqlite-util]))
[logseq.common.config :as common-config]
[logseq.db.sqlite.util :as sqlite-util]
[promesa.core :as p]))
(defn list-tags
[options]
@@ -59,4 +61,16 @@
{:outliner-op :batch-import-edn}
(outliner-op/batch-import-edn! edn-data {}))]
(when error (throw (ex-info error {})))
(ui-handler/re-render-root!)))
(ui-handler/re-render-root!)))
(defn export-edn
"Given sqlite.export options, exports the current graph as a json map with the
:export-body key containing a transit string of the export EDN"
[options*]
(p/let [options (-> (js->clj options* :keywordize-keys true)
(update :export-type (fnil keyword :graph)))
result (state/<invoke-db-worker :thread-api/export-edn (state/get-current-repo) options)]
(when (:export-edn-error result)
(throw (ex-info (str "Export EDN Error: " (:export-edn-error result)) {})))
{:export-body (sqlite-util/transit-write result)
:graph (string/replace-first (state/get-current-repo) common-config/db-version-prefix "")}))

6
src/main/logseq/api/editor.cljs
View File

@@ -491,8 +491,7 @@
db-based? (config/db-based-graph?)
key-ns? (namespace (keyword key))
key (if (and db-based? (not key-ns?))
(api-block/get-db-ident-from-property-name
key (api-block/resolve-property-prefix-for-db this))
(api-block/get-db-ident-from-property-name key this)
key)]
(property-handler/remove-block-property!
(state/get-current-repo)
@@ -506,8 +505,7 @@
(when-let [properties (some-> block-uuid (db-model/get-block-by-uuid) (:block/properties))]
(when (seq properties)
(let [property-name (api-block/sanitize-user-property-name key)
ident (api-block/get-db-ident-from-property-name
property-name (api-block/resolve-property-prefix-for-db this))
ident (api-block/get-db-ident-from-property-name property-name this)
property-value (or (get properties property-name)
(get properties (keyword property-name))
(get properties ident))

47
src/main/logseq/sdk/experiments.cljs
View File

@@ -1,55 +1,46 @@
(ns logseq.sdk.experiments
(:require [frontend.state :as state]
[frontend.components.page :as page]
(:require [frontend.components.page :as page]
[frontend.handler.plugin :as plugin-handler]
[frontend.state :as state]
[frontend.util :as util]
[logseq.sdk.utils :as sdk-util]
[frontend.handler.plugin :as plugin-handler]))
[logseq.sdk.utils :as sdk-util]))
(defn ^:export cp_page_editor
[^js props]
(let [props1 (sdk-util/jsx->clj props)
page-name (some-> props1 :page)
linked-refs? (some-> props1 :include-linked-refs)
unlinked-refs? (some-> props1 :include-unlinked-refs)
config (some-> props1 (dissoc :page :include-linked-refs :include-unlinked-refs))]
(when-let [_entity (page/get-page-entity page-name)]
(page/page-cp
{:repo (state/get-current-repo)
:page-name page-name
:preview? false
:sidebar? false
:linked-refs? (not (false? linked-refs?))
:unlinked-refs? (not (false? unlinked-refs?))
:config config}))))
page-name (some-> props1 :page)]
(when-let [entity (page/get-page-entity page-name)]
(page/page-blocks-cp
entity {:container-id (state/get-next-container-id)}))))
(defn ^:export register_fenced_code_renderer
[pid type ^js opts]
(when-let [^js _pl (plugin-handler/get-plugin-inst pid)]
(plugin-handler/register-fenced-code-renderer
(keyword pid) type (reduce #(assoc %1 %2 (aget opts (name %2))) {}
[:edit :before :subs :render]))))
(keyword pid) type (reduce #(assoc %1 %2 (aget opts (name %2))) {}
[:edit :before :subs :render]))))
(defn ^:export register_route_renderer
[pid key ^js opts]
(when-let [^js _pl (plugin-handler/get-plugin-inst pid)]
(let [key (util/safe-keyword key)]
(plugin-handler/register-route-renderer
(keyword pid) key
(reduce (fn [r k]
(assoc r k (cond-> (aget opts (name k))
(= :name k)
(#(if % (util/safe-keyword %) key)))))
{} [:v :name :path :subs :render])))))
(keyword pid) key
(reduce (fn [r k]
(assoc r k (cond-> (aget opts (name k))
(= :name k)
(#(if % (util/safe-keyword %) key)))))
{} [:v :name :path :subs :render])))))
(defn ^:export register_daemon_renderer
[pid key ^js opts]
(when-let [^js _pl (plugin-handler/get-plugin-inst pid)]
(plugin-handler/register-daemon-renderer
(keyword pid) key (reduce #(assoc %1 %2 (aget opts (name %2))) {}
[:before :subs :render]))))
(keyword pid) key (reduce #(assoc %1 %2 (aget opts (name %2))) {}
[:before :subs :render]))))
(defn ^:export register_extensions_enhancer
[pid type enhancer]
(when-let [^js _pl (and (fn? enhancer) (plugin-handler/get-plugin-inst pid))]
(plugin-handler/register-extensions-enhancer
(keyword pid) type {:enhancer enhancer})))
(keyword pid) type {:enhancer enhancer})))

1
src/resources/dicts/en.edn
View File

@@ -278,6 +278,7 @@
:settings-page/tab-assets "Assets"
:settings-page/tab-features "Features"
:settings-page/tab-collaboration "Collaboration"
:settings-page/tab-encryption "End-to-end encryption"
:settings-page/plugin-system "Plugins"
:settings-page/enable-flashcards "Flashcards"
:settings-page/network-proxy "Network proxy"

Some files were not shown because too many files have changed in this diff Show More