From 517994990293dfd16255e6511e8081dede96837a Mon Sep 17 00:00:00 2001 From: rcmerci Date: Tue, 5 Aug 2025 00:48:29 +0800 Subject: [PATCH] Squashed commit of the following: commit a2eee34d80150712be8094a6fbc0163d48e6bac3 Merge: 3025c94d2 af41e7ea3 Author: Tienson Qin Date: Mon Aug 4 23:08:38 2025 +0800 Merge pull request #12009 from logseq/feat/hnswlib+transformer-js feat: semantic search commit af41e7ea3dbf9fb745a63e177dfda1c4e3f0815b Merge: a407035d8 3025c94d2 Author: Tienson Qin Date: Mon Aug 4 23:08:23 2025 +0800 Merge branch 'master' into feat/hnswlib+transformer-js commit a407035d80e75abfa470bafb989fdcb8afaba429 Author: Gabriel Horner Date: Mon Aug 4 10:41:55 2025 -0400 fix: app and other builds no longer require shadow-cljs config hack Reverts bug fix for https://github.com/logseq/logseq/commit/347323b7d6c8a2356968f8b42143721e5a171349 and for other builds. This bug no longer applies b/c :shared is no longer used or b/c of shadow-cljs update commit 3025c94d28c888708a23653417c57a4514ade376 Author: Tienson Qin Date: Mon Aug 4 20:40:53 2025 +0800 fix: checkbox property value should be aligned center commit a918e5b6f1b0855b138f8f3387cf61b97549f990 Author: Tienson Qin Date: Sun Aug 3 16:58:19 2025 +0800 enhance(ux): show edit button instead of 00:00 for datetime property commit c2c8727f6c2159261e3b1ef83670da95ac95b1c0 Merge: be6929234 c9e446fd8 Author: Tienson Qin Date: Sun Aug 3 16:40:22 2025 +0800 Merge branch 'master' into feat/hnswlib+transformer-js commit be692923469090b1dfb910d970569beb4ec67a8d Author: Tienson Qin Date: Sun Aug 3 16:37:45 2025 +0800 set *publishing? commit 11eb0f38a6e57e7409d1b2321fb559bd5c461a13 Author: Tienson Qin Date: Sun Aug 3 14:33:47 2025 +0800 use official @sqlite.org/sqlite-wasm instead of our fork commit 74fa24cb1ef4947840c5eee0f8b08924f58ece06 Author: Gabriel Horner Date: Fri Aug 1 19:20:59 2025 -0400 fix: publishing no longer has bug requiring shadow-cljs config hack Reverts bug fix from https://github.com/logseq/logseq/commit/a80a182a8f8f56815b675e04f197e9f6da81914a. Confirmed dev and release work for publishing. This bug may no longer apply b/c :shared is no longer used or b/c of shadow-cljs update commit 182a51a85e55c76bad39ce897be00c1617ada803 Author: Gabriel Horner Date: Fri Aug 1 17:56:39 2025 -0400 fix: publishing app in prod mode Also fix watch-publishing-frontend which had stopped working awhile back with shadow-cljs update commit f7e32108f20a8c8b9dc458f26e2f3cc25a742817 Author: Gabriel Horner Date: Fri Aug 1 14:37:18 2025 -0400 enhance(dev): add more workers and db ns to linters commit cf4153b1cd02cb20282d062ecca70c90054022a7 Author: Gabriel Horner Date: Fri Aug 1 13:49:31 2025 -0400 fix: yarn watch commit 1fe46682e426aa998f046642729010c5f41384e1 Merge: 1e8d7de44 8c402676a Author: Tienson Qin Date: Fri Aug 1 00:14:28 2025 +0800 Merge branch 'master' into feat/hnswlib+transformer-js commit 1e8d7de446dc450d4e88e6ccf1664d4c64d784db Author: Tienson Qin Date: Fri Aug 1 00:03:22 2025 +0800 fix: load progress in shared worker commit 1e84a2104272f42694a5ad7b9a9daf2befeea674 Author: Tienson Qin Date: Thu Jul 31 22:38:44 2025 +0800 enhance: use sharedworker for inference to reduce memory usage commit e8d6ae1a2fec99f3d405bffd8448b679dbfcf88b Author: Tienson Qin Date: Thu Jul 31 22:02:13 2025 +0800 disable qwen3 embedding commit 67a0f7cd8c67456305f76e17be9be09152d7704c Merge: 7e04733ac 0bfb458d9 Author: Tienson Qin Date: Thu Jul 31 20:08:29 2025 +0800 Merge branch 'master' into feat/hnswlib+transformer-js commit 7e04733acf79c26b1a1eff162494ac4916ed7e7d Author: Tienson Qin Date: Tue Jul 29 01:21:49 2025 +0800 fix: lint commit 5ee795171caa4f561408440af2c044a357c8e494 Author: Tienson Qin Date: Tue Jul 29 01:19:26 2025 +0800 enhance: display show more by default commit a9e91493503ea31062d4852ea073c2cb5b027fd8 Author: Tienson Qin Date: Tue Jul 29 01:17:22 2025 +0800 fix: add score for non-matched results commit d1cabafcc203984dfee9672426b1bddef5c6d68e Author: Tienson Qin Date: Tue Jul 29 00:53:24 2025 +0800 ensure db/id is not great than 2147483647 for embedding commit 0816c6a9854a4fb1cbe747b0420169752f2df899 Author: Tienson Qin Date: Tue Jul 29 00:15:51 2025 +0800 remove jina embeddings for now commit 74fb1194e9afd9ee6c86c49d41796ceaedb47dff Author: Tienson Qin Date: Tue Jul 29 00:08:46 2025 +0800 adjust weights for keyword search and semantic search commit 6e95bb1801d3ab48853bf1d9178597df84492d21 Author: Tienson Qin Date: Mon Jul 28 19:38:53 2025 +0800 re-enable malli dev commit 47132b395eefebdde2026724e36aac56a157bdc5 Author: Tienson Qin Date: Mon Jul 28 19:23:04 2025 +0800 fix: tweet embed commit 238b071b622c15bb9f6afe415162346c14306dd2 Author: Tienson Qin Date: Mon Jul 28 19:11:41 2025 +0800 fix: workflows commit 44fb03b0ae0e67cc0807da9183bea08e29851f5b Merge: 88dfaf8eb cb65cb88a Author: Tienson Qin Date: Mon Jul 28 18:35:20 2025 +0800 Merge branch 'master' into feat/hnswlib+transformer-js commit 88dfaf8ebd9453ae474b9ec00d7830aed2fce08f Author: Tienson Qin Date: Mon Jul 28 18:31:29 2025 +0800 fix: mobile build commit 7f18e941d0f7f369bd62e1d20b7c233d06f6c09a Author: Tienson Qin Date: Mon Jul 28 16:26:56 2025 +0800 separate config for app and mobile commit 0c3697e972319c5bb07c2b2347751a7340d144f6 Author: Tienson Qin Date: Mon Jul 28 12:50:06 2025 +0800 fix: add workers to rtc test workflow commit 18914d2fa2684f4d43cb5a265e4bda521e4f6ff8 Author: Tienson Qin Date: Mon Jul 28 12:43:24 2025 +0800 fix: add workers to mobile and electron release commit af33e6964bb75fc6081878b9d01135832a63553d Author: Tienson Qin Date: Mon Jul 28 12:30:30 2025 +0800 fix: ns commit 110f5b18e51f0fc35181c2f3186cc4de9e12466c Merge: dcd6d7414 654e6d1b7 Author: Tienson Qin Date: Mon Jul 28 12:29:32 2025 +0800 Merge branch 'master' into feat/hnswlib+transformer-js commit dcd6d741460a2e94c3a8a436557b56d63790530d Author: Tienson Qin Date: Mon Jul 28 12:26:49 2025 +0800 fix: typo commit c71e51bc785e4c3fd047c3293e4665b04b981e5b Author: Tienson Qin Date: Mon Jul 28 12:23:56 2025 +0800 add webpack to release-mobile commit 44d6c127310098d9dd2ac3a8d459f00051f0e86d Author: Tienson Qin Date: Mon Jul 28 12:21:35 2025 +0800 update output-feature-set to es-next-in commit 65afa1e76275457038d01dfae576b6282a77cb28 Author: Tienson Qin Date: Mon Jul 28 12:20:11 2025 +0800 fix: lint commit d488c53fd697dff4013193e59eebeca9360f2890 Author: Tienson Qin Date: Wed Jul 23 22:12:20 2025 +0800 adjust page/object sorting weight commit 6a22385f0beae521d87bc207e3b608e4231cfd4b Author: Tienson Qin Date: Wed Jul 23 20:54:29 2025 +0800 replace uuid ref with block title when embedding commit 0f48207329fc0e259393cb94af53622b9cf52afb Author: Tienson Qin Date: Wed Jul 23 20:48:55 2025 +0800 add tags to embedding commit 48ae054ce88e94ff9e0a48d568e8852a3484f9be Author: Tienson Qin Date: Wed Jul 23 20:22:22 2025 +0800 fix: lint commit aeeaad3eb0f4551b2dfbf22693222380d241e585 Author: Tienson Qin Date: Wed Jul 23 20:12:19 2025 +0800 fix: embedding status commit 9335281d4af4fc9f816c398f0bb031c5bdac9889 Author: Tienson Qin Date: Wed Jul 23 15:25:50 2025 +0800 fix: webpack release failed to require path commit 4b935df5d3ad1d799de8cf496a1c3160a8bf8317 Author: Tienson Qin Date: Wed Jul 23 14:42:54 2025 +0800 fix: katex path commit 5d7f56361ac45197623dbca4dc6ab77ee49c8daf Author: Tienson Qin Date: Wed Jul 23 14:36:10 2025 +0800 downgrade shadow-cljs commit f86be0f3a8f5688ae51f51f44ea9cf70ce274e2a Author: Tienson Qin Date: Wed Jul 23 13:11:15 2025 +0800 add more externs commit 31004ee371709bdf3927ecc8f4b285daea56e29e Author: Tienson Qin Date: Wed Jul 23 12:40:25 2025 +0800 wip: webpack release build commit dbe2708e5e00fddf4b5ed29a9dff676408ff8bf7 Author: Tienson Qin Date: Mon Jul 21 22:46:02 2025 +0800 Add webpack build commit 0722688fd8680b35865d57a2563f9aa9b4cc79f2 Author: Tienson Qin Date: Mon Jul 21 21:47:36 2025 +0800 feat: hybrid search commit 337b28e5d4553b14992e1f41148b2cf012622295 Author: Tienson Qin Date: Mon Jul 21 20:53:35 2025 +0800 set batch size to 500 commit 2c27aee09d1b2c9c3e0aac6e633fa7283b05feaf Author: Tienson Qin Date: Mon Jul 21 20:14:14 2025 +0800 use db/id for hnsw label Because db/id is unique, auto-increment so that old ids are not re-used. commit f1d8e526fc0bc12fec80255b8bf600b26319a5c2 Author: Tienson Qin Date: Mon Jul 21 17:44:45 2025 +0800 enhance: set embedding distance to 0.3 to keep search result related commit bf3af0fedd3a02af7f999563be4b45a1691d41ce Author: Tienson Qin Date: Mon Jul 21 17:21:20 2025 +0800 perf: skip refresh ui for embedding data transactions commit 690b92c975e0e234429e45b287f43f73dc2c2f70 Author: Tienson Qin Date: Mon Jul 21 17:15:06 2025 +0800 enhance: adjust batch embedding size to cut down resource usage commit 3f20f447909dce13b14c6327aae51081c550e479 Author: Tienson Qin Date: Mon Jul 21 17:06:17 2025 +0800 enhance: add embedding indicator commit 2e65b639dc982bc077244354b7e06a4dcda1317e Author: Tienson Qin Date: Mon Jul 21 16:37:35 2025 +0800 fix: cancel index when switching to another model commit ae78fc65c49e98db158b424877e009e07571294c Author: Tienson Qin Date: Mon Jul 21 16:22:17 2025 +0800 enhance: start embedding after selected a model commit 910c8f448fae39530a50a673b419907f93e06ed0 Author: Tienson Qin Date: Mon Jul 21 16:13:17 2025 +0800 Use missionary clock for auto embedding commit b23f9e28a27210a8ee0a93800186d0e6bde7224c Author: Tienson Qin Date: Mon Jul 21 13:30:45 2025 +0800 feat: add semantic search result when searching blocks commit f296c1e67d53364d15af620cd1fbb3a503a868c0 Author: Tienson Qin Date: Sat Jul 19 15:32:16 2025 +0800 fix: auto embedding stops when switching graph commit a5270a05e1255c770a6ab088fa0749cc6711664b Author: Tienson Qin Date: Sat Jul 19 15:17:11 2025 +0800 fix: inference worker js path commit 91d3ab70fb7f21abfdf858252da615f58459055e Author: Tienson Qin Date: Sat Jul 19 15:15:24 2025 +0800 enhance: don't update index info if there's no block update commit de80615c5f6b3519a937d206ad3f2d6fa4ec1818 Author: Tienson Qin Date: Sat Jul 19 14:36:11 2025 +0800 enhance: index embedding for updated blocks every 30s commit 133b5e497a06249063d8d1853d5a30235d0d9e77 Author: Tienson Qin Date: Wed Jul 16 18:59:38 2025 +0800 enhance(ux): show embedding model download progress commit 00405d98d39e19193f6e6439788f8279a141d67e Author: Tienson Qin Date: Wed Jul 16 15:59:22 2025 +0800 add local embedding model settings commit 88f62e86ec2d8a5e27bed9bc7ef4c0ab5b503a80 Author: Tienson Qin Date: Wed Jul 16 14:40:18 2025 +0800 perf improvement commit 4b0cb79d0d4d0da11e0f345edbdb7268a70c28db Author: Tienson Qin Date: Wed Jul 16 04:02:22 2025 +0800 fix: vector search commit bfed2e3bc6cbaa85f1aef81907263ac4db2404a7 Author: Tienson Qin Date: Tue Jul 15 17:49:01 2025 +0800 Add qwen3 embedding commit f3e1da67c713b9b122ecaa64cfb11eedbbf3cf75 Author: Tienson Qin Date: Tue Jul 15 16:52:30 2025 +0800 fix: infer worker doesn't work commit e1147c3264c67ea8691c311deb1f2c7daa70d65a Author: Tienson Qin Date: Tue Jul 15 15:59:01 2025 +0800 fix: c.m/run-task args commit 2d50ca5ce2d7236534dd1377215ea7a6035b4159 Merge: ca558450c 24e3a08b0 Author: Tienson Qin Date: Fri Jul 11 18:15:33 2025 +0800 Merge branch 'master' into feat/hnswlib+transformer-js commit ca558450c19553d128b1c87bff26444ad4464e8a Merge: 1f31f08bf 1565e7541 Author: rcmerci Date: Fri Mar 21 15:42:50 2025 +0800 Merge remote-tracking branch 'origin/feat/db' into feat/hnswlib+transformer-js commit 1f31f08bf6818e52845c1cf27fce49468a3e3036 Author: rcmerci Date: Fri Mar 21 15:37:34 2025 +0800 fix(vec-search): new-hnsw-index when force-reset-index commit bb226a4147e237887afac2f440bc9833ddf46b0b Author: rcmerci Date: Fri Mar 21 01:38:39 2025 +0800 feat(vec-search): debug ui show load-model-progress commit a7273847bc3d2bfb7a4891f0ec4ff0b5f0053d7d Author: rcmerci Date: Thu Mar 20 22:27:56 2025 +0800 feat(vec-search): debug ui support selecting model commit 47c24177a69b2817380acea94eda38b8b3166632 Author: rcmerci Date: Tue Mar 18 23:37:10 2025 +0800 feat(vec-search): update debug ui commit cd44e77ac925b7f09c2ba064fce39c924405773f Author: rcmerci Date: Tue Mar 18 21:55:51 2025 +0800 feat(vec-search): add debug-ui at sidebar commit 328b38db3058b692a62e63737f23cbb428335118 Author: rcmerci Date: Tue Mar 18 18:37:25 2025 +0800 feat(embedding): add vector-search-state-flows commit 6b8c1dde93396a32506f4eede244fd4e1aa689a3 Author: rcmerci Date: Tue Mar 18 15:43:44 2025 +0800 feat(embedding): check webgpu available commit 624f839bbf8153db77ac1c0aa422c8266e79077d Author: Tienson Qin Date: Sun Mar 16 12:08:11 2025 +0800 fix: can't use fuse.js commit 38e4b0d9b2c68341a4ecde06634f3571ff4a00c5 Merge: b19f0d91c a55137049 Author: Tienson Qin Date: Sun Mar 16 11:52:32 2025 +0800 Merge branch 'feat/db' into feat/hnswlib+transformer-js commit b19f0d91cac4b7137b53af3dc2be35d27786bc66 Author: rcmerci Date: Sat Mar 15 23:48:26 2025 +0800 enhance(embedding): add user-defined class/prop commit 0c82a420c791d4e581f6ef6e3c1c5a72e5f7b792 Author: rcmerci Date: Sat Mar 15 22:56:45 2025 +0800 fix: use d/entity instead of d/pull because :block/title in entity will replace :block/uuid by its block/title commit 2c88070fe02a642727b831e0100ad573b58146b8 Author: rcmerci Date: Sat Mar 15 22:43:28 2025 +0800 fix: remove outdated hnsw-label commit a5db52bf3791512cf6f1cb072a8ac37da5b1345c Author: rcmerci Date: Sat Mar 15 20:56:22 2025 +0800 feat(embedding): text-embedding graph blocks and query hnsw new properties: :logseq.property.embedding/hnsw-label, :logseq.property.embedding/hnsw-label-updated-at add ns frontend.worker.embedding commit 5f02b7c863ee88064b26cb21e479b72ac1b0a0d5 Author: rcmerci Date: Sat Mar 15 15:30:44 2025 +0800 dev: update cljfmt.edn commit d48a30000dea7b6741dc517945258ad0905f9549 Author: rcmerci Date: Fri Mar 14 17:30:25 2025 +0800 enhance(text-embedding): add api delete-labels commit eda39289371c8c58dcfc32daeb4359b068f7b714 Author: rcmerci Date: Fri Mar 14 16:45:14 2025 +0800 enhance(text-embedding): connect db-worker and infer-worker commit 53cc7485d8caab403a3d23aa7cba1d42910b868c Author: rcmerci Date: Thu Mar 13 22:42:31 2025 +0800 enhance(text-embedding): smaller init-max-elems, dynamically resizeIndex commit 95c0f2ef1c7761ea23076305d40c82d3280bf20f Author: rcmerci Date: Thu Mar 13 22:07:30 2025 +0800 feat: init add frontend.inference-worker.text-embedding commit e2271dd03da0d19fc8a675794fe889c73dbba143 Author: Tienson Qin Date: Thu Mar 13 21:00:35 2025 +0800 use forked sqlite-wasm commit c269046b443832f903e95f15ed63dcb42a9727f6 Author: Tienson Qin Date: Tue Mar 11 16:21:44 2025 +0800 Revert "disable js-toast temporally to open the app" This reverts commit da1c4dd465661758bf794071c0bd0cfd21ed852a. commit ebeeb280ef436c36bf391050005f0cca5a194212 Author: charlie Date: Tue Mar 11 16:16:37 2025 +0800 enhance(webpack): add externals for React and ReactDOM commit da1c4dd465661758bf794071c0bd0cfd21ed852a Author: Tienson Qin Date: Tue Mar 11 15:57:14 2025 +0800 disable js-toast temporally to open the app commit 8f1b61036175d7b77bd89b5de551b6f3973a36dc Author: Tienson Qin Date: Tue Mar 11 15:51:22 2025 +0800 chore: remove unused package commit 68426d54eb51afb8fc457d4462bdfbb3959a43d1 Author: Tienson Qin Date: Tue Mar 11 15:30:24 2025 +0800 get rid of shared module commit 0ecf65aefb9e41b8e6012899690497257b48f8c9 Author: Tienson Qin Date: Tue Mar 11 13:37:42 2025 +0800 chore: use official sqlite-wasm commit e36543256fbd0808797fc9f8aed0326d1e3fdc69 Author: Tienson Qin Date: Mon Mar 10 17:23:14 2025 +0800 fix: worker repl commit b122d453c23952e9e1e7f3380a2fdb08e823d7fc Author: Tienson Qin Date: Mon Mar 10 17:20:39 2025 +0800 separate :workers build commit 849ffd33291cae89079d821910988175ca1db20f Author: Tienson Qin Date: Mon Mar 10 16:23:52 2025 +0800 prepend bundle to worker js output commit 5632c29ba03d6a2af12a14891d40ff1eee8d56fc Author: Tienson Qin Date: Mon Mar 10 15:05:19 2025 +0800 wip: webpack integration commit 1c46e0205e6b076ca754727c74394db5b18b9130 Author: rcmerci Date: Mon Mar 10 12:08:44 2025 +0800 init add inference-worker --- .cljfmt.edn | 1 + .github/workflows/clj-e2e.yml | 2 +- .github/workflows/clj-rtc-e2e.yml | 2 +- .github/workflows/deploy-db-test-pages.yml | 2 +- clj-e2e/src/logseq/e2e/page.clj | 6 +- .../test/logseq/e2e/plugins_basic_test.clj | 12 +- deps.edn | 9 +- deps/common/src/logseq/common/path.cljs | 2 +- .../src/logseq/db/common/delete_blocks.cljs | 2 +- deps/db/src/logseq/db/common/entity_plus.cljc | 4 +- .../db/src/logseq/db/common/initial_data.cljs | 1 + deps/db/src/logseq/db/file_based/schema.cljs | 4 +- deps/db/src/logseq/db/frontend/db_ident.cljc | 26 +- deps/db/src/logseq/db/frontend/kv_entity.cljs | 7 +- deps/db/src/logseq/db/frontend/property.cljs | 11 +- deps/db/src/logseq/db/frontend/schema.cljs | 2 +- .../logseq/db/common/initial_data_test.cljs | 2 +- deps/db/test/logseq/db/sqlite/build_test.cljs | 4 +- deps/outliner/src/logseq/outliner/core.cljs | 2 +- .../src/logseq/outliner/pipeline.cljs | 13 +- .../src/logseq/publishing/export.cljs | 6 +- .../src/logseq/publishing/html.cljs | 2 +- deps/shui/src/logseq/shui/util.cljs | 66 +- externs/app.txt | 86 + externs/mobile.txt | 87 + gulpfile.js | 4 +- package.json | 67 +- public/index.html | 69 + resources/index.html | 2 +- resources/mobile/index.html | 2 +- .../create_graph_with_schema_org.cljs | 10 +- scripts/src/logseq/tasks/dev.clj | 22 +- .../logseq/tasks/dev/db_and_file_graphs.clj | 5 +- scripts/src/logseq/tasks/dev/lint.clj | 18 +- shadow-cljs.edn | 127 +- src/dev-cljs/shadow/hooks.clj | 18 - src/dev-cljs/shadow/user.clj | 6 +- src/electron/electron/fs_watcher.cljs | 10 +- src/electron/electron/utils.cljs | 4 +- src/electron/electron/window.cljs | 66 +- src/main/frontend/common/file/util.cljs | 20 +- src/main/frontend/common/search_fuzzy.cljs | 43 +- src/main/frontend/common/thread_api.cljc | 6 +- .../frontend/components/block/macros.cljs | 10 +- src/main/frontend/components/cmdk/core.cljs | 6 +- src/main/frontend/components/content.cljs | 2 +- src/main/frontend/components/file.cljs | 3 +- .../frontend/components/file_based/block.cljs | 28 +- .../frontend/components/file_based/query.cljs | 20 +- src/main/frontend/components/header.cljs | 37 +- src/main/frontend/components/property.cljs | 5 +- .../frontend/components/property/value.cljs | 8 +- .../frontend/components/right_sidebar.cljs | 11 + src/main/frontend/components/settings.cljs | 93 +- src/main/frontend/components/settings.css | 2 +- src/main/frontend/components/shortcut.cljs | 8 +- .../components/vector_search/sidebar.cljs | 119 ++ src/main/frontend/core.cljs | 1 + src/main/frontend/db/file_based/model.cljs | 2 +- src/main/frontend/db/react.cljs | 54 +- src/main/frontend/db/utils.cljs | 4 +- src/main/frontend/extensions/latex.cljs | 24 +- src/main/frontend/fs/diff_merge.cljs | 6 +- src/main/frontend/fs/watcher_handler.cljs | 2 +- src/main/frontend/handler.cljs | 10 +- .../handler/db_based/property/util.cljs | 30 +- .../vector_search_background_tasks.cljs | 28 + .../handler/db_based/vector_search_flows.cljs | 22 + src/main/frontend/handler/events.cljs | 6 + src/main/frontend/handler/export/opml.cljs | 26 +- src/main/frontend/handler/export/text.cljs | 38 +- .../handler/file_based/page_property.cljs | 4 +- src/main/frontend/handler/plugin.cljs | 2 +- src/main/frontend/handler/worker.cljs | 8 +- .../inference_worker/inference_worker.cljs | 72 + src/main/frontend/inference_worker/state.cljs | 10 + .../inference_worker/text_embedding.cljs | 237 +++ src/main/frontend/mixins.cljs | 3 +- src/main/frontend/mobile/intent.cljs | 2 +- .../frontend/modules/outliner/pipeline.cljs | 3 +- src/main/frontend/modules/shortcut/core.cljs | 8 +- src/main/frontend/persist_db/browser.cljs | 36 +- src/main/frontend/state.cljs | 7 +- src/main/frontend/template.cljs | 2 +- src/main/frontend/ui.cljs | 18 +- src/main/frontend/util.cljc | 2 +- src/main/frontend/utils.js | 4 +- src/main/frontend/worker/db/migrate.cljs | 5 +- src/main/frontend/worker/db_worker.cljs | 106 +- src/main/frontend/worker/embedding.cljs | 295 +++ src/main/frontend/worker/file.cljs | 2 +- src/main/frontend/worker/file/reset.cljs | 2 +- src/main/frontend/worker/pipeline.cljs | 2 +- src/main/frontend/worker/rtc/core.cljs | 7 +- .../rtc/full_upload_download_graph.cljs | 2 +- .../frontend/worker/rtc/remote_update.cljs | 2 +- src/main/frontend/worker/rtc/ws_util.cljs | 2 +- src/main/frontend/worker/search.cljs | 212 ++- src/main/frontend/worker/state.cljs | 1 + .../{worker => worker_common}/util.cljc | 4 +- src/resources/dicts/en.edn | 2 + webpack.config.js | 61 + yarn.lock | 1657 +++++++++++++---- 103 files changed, 3255 insertions(+), 990 deletions(-) create mode 100644 externs/app.txt create mode 100644 externs/mobile.txt create mode 100644 public/index.html create mode 100644 src/main/frontend/components/vector_search/sidebar.cljs create mode 100644 src/main/frontend/handler/db_based/vector_search_background_tasks.cljs create mode 100644 src/main/frontend/handler/db_based/vector_search_flows.cljs create mode 100644 src/main/frontend/inference_worker/inference_worker.cljs create mode 100644 src/main/frontend/inference_worker/state.cljs create mode 100644 src/main/frontend/inference_worker/text_embedding.cljs create mode 100644 src/main/frontend/worker/embedding.cljs rename src/main/frontend/{worker => worker_common}/util.cljc (93%) create mode 100644 webpack.config.js diff --git a/.cljfmt.edn b/.cljfmt.edn index a32163e909..fc0b474b80 100644 --- a/.cljfmt.edn +++ b/.cljfmt.edn @@ -1,4 +1,5 @@ {:extra-indents {missionary.core/sp [[:block 0]] missionary.core/ap [[:block 0]] + frontend.util/profile [[:inner 0]] frontend.common.missionary/run-task [[:inner 0]]} :sort-ns-references? true} diff --git a/.github/workflows/clj-e2e.yml b/.github/workflows/clj-e2e.yml index 4b66342063..7c4ac906f4 100644 --- a/.github/workflows/clj-e2e.yml +++ b/.github/workflows/clj-e2e.yml @@ -78,7 +78,7 @@ jobs: # NOTE: require the app to be build with DEV-RELEASE flag - name: Prepare E2E test build run: | - yarn gulp:build && clojure -M:cljs release app --config-merge "{:closure-defines {frontend.config/DEV-RELEASE true}}" --debug + yarn gulp:build && clojure -M:cljs release app workers --config-merge "{:closure-defines {frontend.config/DEV-RELEASE true}}" --debug && yarn webpack-app-build - name: Run e2e tests run: cd clj-e2e && timeout 30m bb dev diff --git a/.github/workflows/clj-rtc-e2e.yml b/.github/workflows/clj-rtc-e2e.yml index 2e1d195270..06cba77607 100644 --- a/.github/workflows/clj-rtc-e2e.yml +++ b/.github/workflows/clj-rtc-e2e.yml @@ -79,7 +79,7 @@ jobs: # NOTE: require the app to be build with DEV-RELEASE flag - name: Prepare E2E test build run: | - yarn gulp:build && clojure -M:cljs release app --config-merge "{:closure-defines {frontend.config/DEV-RELEASE true}}" --debug + yarn gulp:build && clojure -M:cljs release app workers --config-merge "{:closure-defines {frontend.config/DEV-RELEASE true}}" --debug && yarn webpack-app-build rsync -avz --exclude node_modules --exclude android --exclude ios ./static/ ./public/ ls -lR ./public diff --git a/.github/workflows/deploy-db-test-pages.yml b/.github/workflows/deploy-db-test-pages.yml index 00e74dbffe..fa90c768bc 100644 --- a/.github/workflows/deploy-db-test-pages.yml +++ b/.github/workflows/deploy-db-test-pages.yml @@ -42,7 +42,7 @@ jobs: - name: Build Released-Web run: | - yarn gulp:build && clojure -M:cljs release app --config-merge '{:compiler-options {:source-map true :source-map-include-sources-content true :source-map-detail-level :symbols}}' + yarn gulp:build && clojure -M:cljs release app workers --config-merge '{:compiler-options {:source-map true :source-map-include-sources-content true :source-map-detail-level :symbols}}' && yarn webpack-app-build rsync -avz --exclude node_modules --exclude android --exclude ios ./static/ ./public/ ls -lR ./public && mkdir r2 && mv ./public/js/main.js.map ./r2/db-test.main.js.map sed -i 's/=main.js.map/=https:\/\/assets.logseq.io\/db-test.main.js.map/g' ./public/js/main.js diff --git a/clj-e2e/src/logseq/e2e/page.clj b/clj-e2e/src/logseq/e2e/page.clj index e48aa0a36a..cf9e274f96 100644 --- a/clj-e2e/src/logseq/e2e/page.clj +++ b/clj-e2e/src/logseq/e2e/page.clj @@ -1,9 +1,9 @@ (ns logseq.e2e.page - (:require [logseq.e2e.keyboard :as k] + (:require [logseq.e2e.block :as b] + [logseq.e2e.keyboard :as k] [logseq.e2e.util :as util] [wally.main :as w] - [wally.selectors :as ws] - [logseq.e2e.block :as b]) + [wally.selectors :as ws]) (:import (com.microsoft.playwright TimeoutError))) (defn goto-page diff --git a/clj-e2e/test/logseq/e2e/plugins_basic_test.clj b/clj-e2e/test/logseq/e2e/plugins_basic_test.clj index 6eaeff8fdd..3b1fac5fa2 100644 --- a/clj-e2e/test/logseq/e2e/plugins_basic_test.clj +++ b/clj-e2e/test/logseq/e2e/plugins_basic_test.clj @@ -25,14 +25,14 @@ (when (string? s) (-> s ;; Normalize input: replace hyphens/spaces with underscores, collapse multiple underscores - (clojure.string/replace #"[-\s]+" "_") + (clojure.string/replace #"[-\s]+" "_") ;; Split on uppercase letters (except at start) and join with underscore - (clojure.string/replace #"(? (ls-api-call! :editor.insertBlock uuid' "insert-0") - (assert-api-ls-block!)) + (assert-api-ls-block!)) (ls-api-call! :editor.updateBlock uuid' "append-but-updated-0") (k/esc) (w/wait-for ".block-title-wrap:text('append-but-updated-0')") diff --git a/deps.edn b/deps.edn index cf32722fc9..ac22ad20b0 100644 --- a/deps.edn +++ b/deps.edn @@ -53,16 +53,14 @@ :git/sha "d61ce7e29186de021a2a453a8cee68efb5a88440"}} :aliases {:cljs {:extra-paths ["src/dev-cljs/" "src/test/" "src/electron/"] - :extra-deps {org.clojure/clojurescript {:mvn/version "1.11.132"} - org.clojure/tools.namespace {:mvn/version "0.2.11"} + :extra-deps {org.clojure/tools.namespace {:mvn/version "0.2.11"} cider/cider-nrepl {:mvn/version "0.55.1"} org.clojars.knubie/cljs-run-test {:mvn/version "1.0.1"} tortue/spy {:mvn/version "2.14.0"}} :main-opts ["-m" "shadow.cljs.devtools.cli"]} :test {:extra-paths ["src/test/"] - :extra-deps {org.clojure/clojurescript {:mvn/version "1.11.132"} - org.clojure/test.check {:mvn/version "1.1.1"} + :extra-deps {org.clojure/test.check {:mvn/version "1.1.1"} pjstadig/humane-test-output {:mvn/version "0.11.0"} org.clojars.knubie/cljs-run-test {:mvn/version "1.0.1"} tortue/spy {:mvn/version "2.14.0"} @@ -70,8 +68,7 @@ :main-opts ["-m" "shadow.cljs.devtools.cli"]} :rtc-e2e-test {:extra-paths ["src/rtc_e2e_test"] - :extra-deps {org.clojure/clojurescript {:mvn/version "1.11.132"} - cider/cider-nrepl {:mvn/version "0.50.2"}} + :extra-deps {cider/cider-nrepl {:mvn/version "0.50.2"}} :main-opts ["-m" "shadow.cljs.devtools.cli"]} :bench {:extra-paths ["src/bench/"] diff --git a/deps/common/src/logseq/common/path.cljs b/deps/common/src/logseq/common/path.cljs index d78f81b8f8..e722f0e5a2 100644 --- a/deps/common/src/logseq/common/path.cljs +++ b/deps/common/src/logseq/common/path.cljs @@ -148,7 +148,7 @@ scheme (.-protocol url) path (.-pathname url) domain (or (not-empty (.-host url)) - (if (string/starts-with? path "/") "" "/")) + (if (string/starts-with? path "/") "" "/")) encoded-new-path (apply uri-path-join-internal path segments)] (str scheme "//" domain encoded-new-path))) diff --git a/deps/db/src/logseq/db/common/delete_blocks.cljs b/deps/db/src/logseq/db/common/delete_blocks.cljs index ef2ea8b5b9..eccfb479a4 100644 --- a/deps/db/src/logseq/db/common/delete_blocks.cljs +++ b/deps/db/src/logseq/db/common/delete_blocks.cljs @@ -5,8 +5,8 @@ [logseq.common.util :as common-util] [logseq.common.util.block-ref :as block-ref] [logseq.common.util.page-ref :as page-ref] - [logseq.db.common.entity-util :as common-entity-util] [logseq.db.common.entity-plus :as entity-plus] + [logseq.db.common.entity-util :as common-entity-util] [logseq.db.frontend.entity-util :as entity-util])) (defn- replace-ref-with-deleted-block-title diff --git a/deps/db/src/logseq/db/common/entity_plus.cljc b/deps/db/src/logseq/db/common/entity_plus.cljc index 4a1d0bbc58..4f33540a74 100644 --- a/deps/db/src/logseq/db/common/entity_plus.cljc +++ b/deps/db/src/logseq/db/common/entity_plus.cljc @@ -95,8 +95,8 @@ (get (.-kv e) k) (if db-based? (let [result (lookup-entity e k default-value) - ;; Replace title for pages only, otherwise it'll recursively - ;; replace block id refs if there're cycle references of blocks + ;; Replace title for pages only, otherwise it'll recursively + ;; replace block id refs if there're cycle references of blocks refs (:block/refs e) result' (if (and (string? result) refs) (db-content/id-ref->title-ref result refs diff --git a/deps/db/src/logseq/db/common/initial_data.cljs b/deps/db/src/logseq/db/common/initial_data.cljs index a0aeefae34..c61f221513 100644 --- a/deps/db/src/logseq/db/common/initial_data.cljs +++ b/deps/db/src/logseq/db/common/initial_data.cljs @@ -344,6 +344,7 @@ :logseq.kv/graph-uuid :logseq.kv/latest-code-lang :logseq.kv/graph-backup-folder + :logseq.kv/graph-text-embedding-model-name :logseq.property/empty-placeholder]) favorites (when db-graph? (get-favorites db)) views (when db-graph? (get-views-data db)) diff --git a/deps/db/src/logseq/db/file_based/schema.cljs b/deps/db/src/logseq/db/file_based/schema.cljs index 32da7a81a5..b00c15c473 100644 --- a/deps/db/src/logseq/db/file_based/schema.cljs +++ b/deps/db/src/logseq/db/file_based/schema.cljs @@ -32,8 +32,8 @@ :block/path-refs {:db/valueType :db.type/ref :db/cardinality :db.cardinality/many} - :block/tags {:db/valueType :db.type/ref - :db/cardinality :db.cardinality/many} + :block/tags {:db/valueType :db.type/ref + :db/cardinality :db.cardinality/many} ;; which block this block links to, used for tag, embeds :block/link {:db/valueType :db.type/ref diff --git a/deps/db/src/logseq/db/frontend/db_ident.cljc b/deps/db/src/logseq/db/frontend/db_ident.cljc index ac05c1aba4..462487eb41 100644 --- a/deps/db/src/logseq/db/frontend/db_ident.cljc +++ b/deps/db/src/logseq/db/frontend/db_ident.cljc @@ -68,24 +68,24 @@ ([user-namespace name-string random-suffix?] {:pre [(or (keyword? user-namespace) (string? user-namespace)) (string? name-string) (boolean? random-suffix?)]} (assert (not (re-find #"^(logseq|block)(\.|$)" (name user-namespace))) - "New ident is not allowed to use an internal namespace") + "New ident is not allowed to use an internal namespace") (if #?(:org.babashka/nbb true :cljs (or (false? random-suffix?) - (and (exists? js/process) - (or js/process.env.REPEATABLE_IDENTS js/process.env.DB_GRAPH))) + (and (exists? js/process) + (or js/process.env.REPEATABLE_IDENTS js/process.env.DB_GRAPH))) :default false) ;; Used for contexts where we want repeatable idents e.g. tests and CLIs (keyword user-namespace - (->> (string/replace-first name-string #"^(\d)" "NUM-$1") + (->> (string/replace-first name-string #"^(\d)" "NUM-$1") ;; '-' must go last in char class - (filter #(re-find #"[0-9a-zA-Z*+!_'?<>=-]{1}" %)) - (apply str))) + (filter #(re-find #"[0-9a-zA-Z*+!_'?<>=-]{1}" %)) + (apply str))) (keyword user-namespace - (str - (->> (string/replace-first name-string #"^(\d)" "NUM-$1") + (str + (->> (string/replace-first name-string #"^(\d)" "NUM-$1") ;; '-' must go last in char class - (filter #(re-find #"[0-9a-zA-Z*+!_'?<>=-]{1}" %)) - (apply str)) - "-" - (rand-nth non-int-char-range) - (nano-id 7)))))) + (filter #(re-find #"[0-9a-zA-Z*+!_'?<>=-]{1}" %)) + (apply str)) + "-" + (rand-nth non-int-char-range) + (nano-id 7)))))) diff --git a/deps/db/src/logseq/db/frontend/kv_entity.cljs b/deps/db/src/logseq/db/frontend/kv_entity.cljs index 404b373e4a..89bea4c40a 100644 --- a/deps/db/src/logseq/db/frontend/kv_entity.cljs +++ b/deps/db/src/logseq/db/frontend/kv_entity.cljs @@ -26,6 +26,11 @@ RTC won't start when major-schema-versions don't match" :rtc {:rtc/ignore-entity-when-init-upload true :rtc/ignore-entity-when-init-download true}} :logseq.kv/graph-initial-schema-version {:doc "Graph's schema version when created"} + :logseq.kv/graph-last-gc-at {:doc "Last time graph gc at" :rtc {:rtc/ignore-entity-when-init-upload true - :rtc/ignore-entity-when-init-download true}}) + :rtc/ignore-entity-when-init-download true}} + + :logseq.kv/graph-text-embedding-model-name {:doc "Graph's text-embedding model name" + :rtc {:rtc/ignore-entity-when-init-upload true + :rtc/ignore-entity-when-init-download true}}) diff --git a/deps/db/src/logseq/db/frontend/property.cljs b/deps/db/src/logseq/db/frontend/property.cljs index de3be077c0..08194764ef 100644 --- a/deps/db/src/logseq/db/frontend/property.cljs +++ b/deps/db/src/logseq/db/frontend/property.cljs @@ -559,7 +559,14 @@ :schema {:type :class :cardinality :many :public? true} - :queryable? true}))) + :queryable? true} + :logseq.property.embedding/hnsw-label-updated-at {:title "HNSW label updated-at" + :schema {:type :datetime + :public? false + :hide? true} + :rtc {:rtc/ignore-attr-when-init-upload true + :rtc/ignore-attr-when-init-download true + :rtc/ignore-attr-when-syncing true}}))) (def db-attribute-properties "Internal properties that are also db schema attributes" @@ -611,7 +618,7 @@ "logseq.property.linked-references" "logseq.property.asset" "logseq.property.table" "logseq.property.node" "logseq.property.code" "logseq.property.repeat" "logseq.property.journal" "logseq.property.class" "logseq.property.view" - "logseq.property.user" "logseq.property.history"}) + "logseq.property.user" "logseq.property.history" "logseq.property.embedding"}) (defn logseq-property? "Determines if keyword is a logseq property" diff --git a/deps/db/src/logseq/db/frontend/schema.cljs b/deps/db/src/logseq/db/frontend/schema.cljs index 3970d568ca..99279b7017 100644 --- a/deps/db/src/logseq/db/frontend/schema.cljs +++ b/deps/db/src/logseq/db/frontend/schema.cljs @@ -37,7 +37,7 @@ (map (juxt :major :minor) [(parse-schema-version x) (parse-schema-version y)]))) -(def version (parse-schema-version "65.8")) +(def version (parse-schema-version "65.9")) (defn major-version "Return a number. diff --git a/deps/db/test/logseq/db/common/initial_data_test.cljs b/deps/db/test/logseq/db/common/initial_data_test.cljs index ff7230083f..eae8818106 100644 --- a/deps/db/test/logseq/db/common/initial_data_test.cljs +++ b/deps/db/test/logseq/db/common/initial_data_test.cljs @@ -6,8 +6,8 @@ [cljs.test :refer [deftest async use-fixtures is testing]] [datascript.core :as d] [logseq.db.common.initial-data :as common-initial-data] - [logseq.db.sqlite.build :as sqlite-build] [logseq.db.common.sqlite-cli :as sqlite-cli] + [logseq.db.sqlite.build :as sqlite-build] [logseq.db.sqlite.create-graph :as sqlite-create-graph] [logseq.db.test.helper :as db-test])) diff --git a/deps/db/test/logseq/db/sqlite/build_test.cljs b/deps/db/test/logseq/db/sqlite/build_test.cljs index e1e21ecff1..01fffe8a82 100644 --- a/deps/db/test/logseq/db/sqlite/build_test.cljs +++ b/deps/db/test/logseq/db/sqlite/build_test.cljs @@ -3,10 +3,10 @@ [datascript.core :as d] [logseq.common.util.page-ref :as page-ref] [logseq.db :as ldb] + [logseq.db.frontend.entity-util :as entity-util] [logseq.db.frontend.property :as db-property] [logseq.db.sqlite.build :as sqlite-build] - [logseq.db.test.helper :as db-test] - [logseq.db.frontend.entity-util :as entity-util])) + [logseq.db.test.helper :as db-test])) (deftest build-tags (let [conn (db-test/create-conn) diff --git a/deps/outliner/src/logseq/outliner/core.cljs b/deps/outliner/src/logseq/outliner/core.cljs index 8893abd98d..b78afb9caa 100644 --- a/deps/outliner/src/logseq/outliner/core.cljs +++ b/deps/outliner/src/logseq/outliner/core.cljs @@ -251,7 +251,7 @@ collapse-or-expand? (= outliner-op :collapse-expand-blocks) m* (cond-> (-> data' - (dissoc :block/children :block/meta :block/unordered + (dissoc :block/children :block/meta :block/unordered :block/path-refs :block.temp/ast-title :block.temp/ast-body :block/level :block.temp/load-status :block.temp/has-children?) common-util/remove-nils diff --git a/deps/outliner/src/logseq/outliner/pipeline.cljs b/deps/outliner/src/logseq/outliner/pipeline.cljs index 9dbb1f80ee..a232d245e2 100644 --- a/deps/outliner/src/logseq/outliner/pipeline.cljs +++ b/deps/outliner/src/logseq/outliner/pipeline.cljs @@ -50,7 +50,7 @@ (recur (into parent-refs (:block-ref-ids parent)) (:parent-id parent)) ;; exits when top-level parent is reached - parent-refs)))}) + (remove nil? parent-refs))))}) children-maps)] children-refs)) @@ -87,10 +87,13 @@ old-refs (if db-before (set (map :db/id (:block/path-refs (d/entity db-before (:db/id block))))) #{}) - new-refs (set (concat - (some-> (:db/id (:block/page block)) vector) - (map :db/id (:block/refs block)) - parents-refs)) + new-refs (->> + (concat + (some-> (:db/id (:block/page block)) vector) + (map :db/id (:block/refs block)) + parents-refs) + (remove nil?) + set) refs-changed? (not= old-refs new-refs) children (when refs-changed? (when-not page? diff --git a/deps/publishing/src/logseq/publishing/export.cljs b/deps/publishing/src/logseq/publishing/export.cljs index 44a0d53537..d764f3da14 100644 --- a/deps/publishing/src/logseq/publishing/export.cljs +++ b/deps/publishing/src/logseq/publishing/export.cljs @@ -1,14 +1,14 @@ (ns ^:node-only logseq.publishing.export "This electron only ns (for the main process) exports files from multiple locations to provide a complete publishing app" - (:require ["fs-extra" :as fse] + (:require ["fs" :as fs] + ["fs-extra" :as fse] ["path" :as node-path] - ["fs" :as fs] [promesa.core :as p])) (def ^:api js-files "js files from publishing release build" - (->> ["shared.js" "main.js" "code-editor.js" "excalidraw.js" "tldraw.js" "db-worker.js"] + (->> ["main.js" "code-editor.js" "excalidraw.js" "tldraw.js"] ;; Add source maps for all js files as it doesn't affect initial load time (mapcat #(vector % (str % ".map"))) vec)) diff --git a/deps/publishing/src/logseq/publishing/html.cljs b/deps/publishing/src/logseq/publishing/html.cljs index c45755f1d6..26df77563a 100644 --- a/deps/publishing/src/logseq/publishing/html.cljs +++ b/deps/publishing/src/logseq/publishing/html.cljs @@ -124,7 +124,7 @@ necessary db filtering" [:script {:src "static/js/react.production.min.js"}] [:script {:src "static/js/react-dom.production.min.js"}] [:script {:src "static/js/ui.js"}] - [:script {:src "static/js/shared.js"}] + [:script {:src "static/js/main-bundle.js"}] [:script {:src "static/js/main.js"}] ;; Deferring scripts above results in errors [:script {:defer true :src "static/js/interact.min.js"}] diff --git a/deps/shui/src/logseq/shui/util.cljs b/deps/shui/src/logseq/shui/util.cljs index 6ca81d2904..9e451a34fb 100644 --- a/deps/shui/src/logseq/shui/util.cljs +++ b/deps/shui/src/logseq/shui/util.cljs @@ -1,13 +1,13 @@ (ns logseq.shui.util (:require - [clojure.string :as s] - [rum.core :refer [use-state use-effect!] :as rum] - [logseq.shui.rum :as shui-rum] - [goog.object :refer [getValueByKeys] :as gobj] - [clojure.set :refer [rename-keys]] - [clojure.walk :as w] [cljs-bean.core :as bean] - [goog.dom :as gdom])) + [clojure.set :refer [rename-keys]] + [clojure.string :as s] + [clojure.walk :as w] + [goog.dom :as gdom] + [goog.object :refer [getValueByKeys] :as gobj] + [logseq.shui.rum :as shui-rum] + [rum.core :refer [use-state use-effect!] :as rum])) (goog-define NODETEST false) @@ -16,7 +16,7 @@ [input] (let [words (s/split input #"-") capitalize (->> (rest words) - (map #(apply str (s/upper-case (first %)) (rest %))))] + (map #(apply str (s/upper-case (first %)) (rest %))))] (apply str (first words) capitalize))) (defn map-keys->camel-case @@ -28,7 +28,7 @@ (let [convert-to-camel (fn [[key value]] (let [k (name key)] [(if-not (or (s/starts-with? k "data-") - (s/starts-with? k "aria-")) + (s/starts-with? k "aria-")) (kebab-case->camel-case k) k) value]))] (w/postwalk (fn [x] (if (map? x) @@ -37,7 +37,7 @@ x)] (into {} (map convert-to-camel new-map))) x)) - data))) + data))) (defn $LSUtils [] (aget js/window "LSUtils")) (def dev? (some-> ($LSUtils) (aget "isDev"))) @@ -45,11 +45,11 @@ (defn uuid-color [uuid-str] (some-> ($LSUtils) (aget "uniqolor") - (apply [uuid-str - #js {:saturation #js [55, 70], - :lightness 70, - :differencePoint 60}]) - (aget "color"))) + (apply [uuid-str + #js {:saturation #js [55, 70], + :lightness 70, + :differencePoint 60}]) + (aget "color"))) (defn get-path "Returns the component path." @@ -67,10 +67,10 @@ ;; we have to make sure to check if the children is sequential ;; as a list can be returned, eg: from a (for) new-children (if (and (not (nil? children#)) - (not (empty? children)) - (or (not (array? children#)) + (not (empty? children)) + (or (not (array? children#)) ;; maybe list children - (not (vector? type#)))) + (not (vector? type#)))) [children#] children#) ;; convert any options key value to a React element, if @@ -83,22 +83,22 @@ react-class (if dev? (react-class) react-class)] (apply js/React.createElement react-class ;; sablono html-to-dom-attrs does not work for nested hash-maps - (bean/->js (map-keys->camel-case new-options :html-props true)) - new-children))) + (bean/->js (map-keys->camel-case new-options :html-props true)) + new-children))) (defn use-atom-fn [a getter-fn setter-fn] (let [[val set-val] (use-state (getter-fn @a))] (use-effect! - (fn [] - (let [id (str (random-uuid))] - (add-watch a id (fn [_ _ prev-state next-state] - (let [prev-value (getter-fn prev-state) - next-value (getter-fn next-state)] - (when-not (= prev-value next-value) - (set-val next-value))))) - #(remove-watch a id))) - []) + (fn [] + (let [id (str (random-uuid))] + (add-watch a id (fn [_ _ prev-state next-state] + (let [prev-value (getter-fn prev-state) + next-value (getter-fn next-state)] + (when-not (= prev-value next-value) + (set-val next-value))))) + #(remove-watch a id))) + []) [val #(swap! a setter-fn %)])) (defn use-atom @@ -110,10 +110,10 @@ [] (let [*mounted (rum/use-ref false)] (use-effect! - (fn [] - (rum/set-ref! *mounted true) - #(rum/set-ref! *mounted false)) - []) + (fn [] + (rum/set-ref! *mounted true) + #(rum/set-ref! *mounted false)) + []) #(rum/deref *mounted))) (defn react->rum [c static?] diff --git a/externs/app.txt b/externs/app.txt new file mode 100644 index 0000000000..3591345ce0 --- /dev/null +++ b/externs/app.txt @@ -0,0 +1,86 @@ +require +DEBUG +convertFileSrc +getPlatform +handleError +isLoggingEnabled +isNative +isNativePlatform +isPluginAvailable +platform +pluginMethodNoop +registerPlugin +registerHelper +Pass +showHint +createRef +call +setState +forceUpdate +memo +createElement +render +unmount +createContext +createPortal +createRef +createElement +ref +path +current +useState +useReducer +useEffect +useLayoutEffect +useCallback +useMemo +useRef +renderToString +idb-keyval +newStore +isError +value +CodeMirror +over +forEachNode +on +getTextArea +setUser +nativeEvent +which +multipliedBy +minus +plus +isInteger +isPositive +isLessThan +toNumber +modulo +exponentiatedBy +dividedBy +multipliedBy +negated +isZero +isGreaterThanOrEqualTo +isLessThan +shiftedBy +s +dividedToIntegerBy +abs +toFraction +e +getCursor +getLineTokens +getTokenAt +getValue +completionActive +state +mime +getWrapperElement +doc +lastLine +getLine +setOption +string +dispose +path diff --git a/externs/mobile.txt b/externs/mobile.txt new file mode 100644 index 0000000000..c26926d170 --- /dev/null +++ b/externs/mobile.txt @@ -0,0 +1,87 @@ +require +DEBUG +convertFileSrc +getPlatform +handleError +isLoggingEnabled +isNative +isNativePlatform +isPluginAvailable +platform +pluginMethodNoop +registerPlugin +registerHelper +Pass +showHint +createRef +call +setState +forceUpdate +memo +createElement +render +unmount +createContext +createPortal +createRef +createElement +ref +path +current +useState +useReducer +useEffect +useLayoutEffect +useCallback +useMemo +useRef +renderToString +idb-keyval +newStore +isError +value +CodeMirror +over +forEachNode +on +getTextArea +setUser +nativeEvent +which +multipliedBy +minus +plus +isInteger +isPositive +isLessThan +toNumber +modulo +exponentiatedBy +dividedBy +multipliedBy +negated +isZero +isGreaterThanOrEqualTo +isLessThan +shiftedBy +s +dividedToIntegerBy +abs +toFraction +e +getCursor +getLineTokens +getTokenAt +getValue +completionActive +state +mime +getWrapperElement +doc +lastLine +getLine +setOption +string +dispose +path +base64String diff --git a/gulpfile.js b/gulpfile.js index 4fed6854f5..d1b72420fa 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -78,7 +78,7 @@ const common = { 'node_modules/@isomorphic-git/lightning-fs/dist/lightning-fs.min.js', 'packages/amplify/dist/amplify.js', 'packages/ui/dist/ui/ui.js', - 'node_modules/@logseq/sqlite-wasm/sqlite-wasm/jswasm/sqlite3.wasm', + 'node_modules/@sqlite.org/sqlite-wasm/sqlite-wasm/jswasm/sqlite3.wasm', 'node_modules/react/umd/react.production.min.js', 'node_modules/react/umd/react.development.js', 'node_modules/react-dom/umd/react-dom.production.min.js', @@ -129,7 +129,7 @@ const common = { 'node_modules/photoswipe/dist/umd/*.js', 'packages/amplify/dist/amplify.js', 'packages/ui/dist/ui/ui.js', - 'node_modules/@logseq/sqlite-wasm/sqlite-wasm/jswasm/sqlite3.wasm', + 'node_modules/@sqlite.org/sqlite-wasm/sqlite-wasm/jswasm/sqlite3.wasm', ]).pipe(gulp.dest(path.join(outputPath, 'mobile', 'js'))), () => gulp.src([ 'packages/ui/dist/ionic/*.js', diff --git a/package.json b/package.json index 97814c2cee..e5ea189bde 100644 --- a/package.json +++ b/package.json @@ -36,27 +36,36 @@ "postcss-import": "15.0.0", "postcss-import-ext-glob": "2.0.1", "postcss-nested": "6.0.0", + "process": "^0.11.10", "purgecss": "4.0.2", "semver": "7.5.2", - "shadow-cljs": "2.26.0", + "shadow-cljs": "2.28.23", "stylelint": "^13.8.0", "stylelint-config-standard": "^20.0.0", "tailwindcss": "3.3.5", "tailwindcss-animate": "^1.0.7", - "typescript": "^4.4.3" + "typescript": "^4.4.3", + "webpack": "^5.98.0", + "webpack-cli": "^6.0.1" }, "scripts": { - "watch": "run-p gulp:watch cljs:watch", - "electron-watch": "run-p gulp:watch cljs:electron-watch", - "app-watch": "run-p gulp:watch cljs:app-watch", - "dev": "run-p gulp:watch gulp:mobile-watch cljs:dev-watch", - "release": "run-s gulp:build cljs:release", - "release-app": "run-s gulp:build cljs:release-app", - "release-mobile": "run-s gulp:buildMobile cljs:release-mobile", - "dev-release-app": "run-s gulp:build cljs:dev-release-app", + "watch": "run-p gulp:watch cljs:watch webpack-app-watch", + "electron-watch": "run-p gulp:watch cljs:electron-watch webpack-app-watch", + "app-watch": "run-p gulp:watch cljs:app-watch webpack-app-watch", + "mobile-watch": "run-p gulp:mobile-watch cljs:mobile-watch webpack-mobile-watch", + "dev": "run-p gulp:watch gulp:mobile-watch cljs:dev-watch webpack", + "release": "run-s gulp:build cljs:release webpack-app-build", + "release-app": "run-s gulp:build cljs:release-app webpack-app-build", + "release-mobile": "run-s gulp:buildMobile cljs:release-mobile webpack-mobile-build", + "dev-release-app": "run-s gulp:build cljs:dev-release-app webpack-app-build", "dev-electron-app": "gulp electron", - "release-electron": "run-s gulp:build && gulp electronMaker", + "release-electron": "run-s gulp:build && gulp electronMaker webpack-app-build", "debug-electron": "cd static/ && yarn electron:debug", + "webpack": "webpack --watch", + "webpack-app-watch": "npx webpack --watch --config-name app", + "webpack-mobile-watch": "npx webpack --watch --config-name mobile", + "webpack-app-build": "npx webpack build --config-name app", + "webpack-mobile-build": "npx webpack build --config-name mobile", "sync-android-release": "yarn clean && yarn release-mobile && rm -rf ./static/mobile/**/*.map && npx cap sync android", "sync-ios-release": "yarn clean && yarn release-mobile && rm -rf ./static/mobile/**/*.map && npx cap sync ios", "clean": "gulp clean", @@ -68,27 +77,27 @@ "gulp:buildMobile": "cross-env NODE_ENV=production gulp buildMobile", "css:build": "postcss tailwind.all.css -o static/css/style.css --verbose --env production", "css:watch": "cross-env TAILWIND_MODE=watch postcss tailwind.all.css -o static/css/style.css --verbose --watch", - "cljs:watch": "clojure -M:cljs watch app electron", + "cljs:watch": "clojure -M:cljs watch app workers electron", "cljs:storybook-watch": "clojure -M:cljs watch stories-dev", "gulp:mobile-watch": "gulp watchMobile", "css:mobile-build": "postcss tailwind.mobile.css -o static/mobile/style.css --verbose --env production", "css:mobile-watch": "cross-env TAILWIND_MODE=watch postcss tailwind.mobile.css -o static/mobile/style.css --verbose --watch", - "cljs:mobile-watch": "clojure -M:cljs watch mobile", - "cljs:release-mobile": "clojure -M:cljs release mobile", - "cljs:dev-watch": "clojure -M:cljs watch app electron mobile", - "cljs:app-watch": "clojure -M:cljs watch app", - "cljs:electron-watch": "clojure -M:cljs watch app electron --config-merge \"{:asset-path \\\"./js\\\"}\"", - "cljs:release": "clojure -M:cljs release app publishing electron", - "cljs:release-electron": "clojure -M:cljs release app electron --debug && clojure -M:cljs release publishing", - "cljs:release-app": "clojure -M:cljs release app", - "cljs:release-publishing": "clojure -M:cljs release publishing", + "cljs:mobile-watch": "clojure -M:cljs watch mobile workers --config-merge \"{:output-dir \\\"./static/mobile/js\\\" :asset-path \\\"/static/mobile/js\\\" :release {:asset-path \\\"http://localhost\\\"}}\"", + "cljs:release-mobile": "clojure -M:cljs release mobile workers --config-merge \"{:output-dir \\\"./static/mobile/js\\\" :asset-path \\\"/static/mobile/js\\\" :release {:asset-path \\\"http://localhost\\\"}}\"", + "cljs:dev-watch": "clojure -M:cljs watch app workers electron mobile", + "cljs:app-watch": "clojure -M:cljs watch app workers", + "cljs:electron-watch": "clojure -M:cljs watch app workers electron --config-merge \"{:asset-path \\\"./js\\\"}\"", + "cljs:release": "clojure -M:cljs release app workers publishing electron", + "cljs:release-electron": "clojure -M:cljs release app workers electron --debug && clojure -M:cljs release publishing", + "cljs:release-app": "clojure -M:cljs release app workers", + "cljs:release-publishing": "clojure -M:cljs release app publishing", "cljs:test": "clojure -M:test compile test", "cljs:run-test": "node static/tests.js", - "cljs:dev-release-app": "clojure -M:cljs release app --config-merge \"{:closure-defines {frontend.config/DEV-RELEASE true}}\"", - "cljs:dev-release-electron": "clojure -M:cljs release app electron --debug --config-merge \"{:closure-defines {frontend.config/DEV-RELEASE true}}\" && clojure -M:cljs release publishing", - "cljs:debug": "clojure -M:cljs release app --debug", - "cljs:report": "clojure -M:cljs run shadow.cljs.build-report app report.html", - "cljs:build-electron": "clojure -A:cljs compile app electron", + "cljs:dev-release-app": "clojure -M:cljs release app workers --config-merge \"{:closure-defines {frontend.config/DEV-RELEASE true}}\"", + "cljs:dev-release-electron": "clojure -M:cljs release app workers electron --debug --config-merge \"{:closure-defines {frontend.config/DEV-RELEASE true}}\" && clojure -M:cljs release publishing", + "cljs:debug": "clojure -M:cljs release app workers --debug", + "cljs:report": "clojure -M:cljs run shadow.cljs.build-report app workers report.html", + "cljs:build-electron": "clojure -A:cljs compile app workers electron", "cljs:lint": "clojure -M:clj-kondo --parallel --lint src --cache false", "ios:dev": "cross-env PLATFORM=ios gulp cap", "android:dev": "cross-env PLATFORM=android gulp cap", @@ -121,6 +130,7 @@ "@excalidraw/excalidraw": "0.16.1", "@glidejs/glide": "^3.6.0", "@highlightjs/cdn-assets": "10.4.1", + "@huggingface/transformers": "^3.6.3", "@ionic/core": "^8.5.4", "@ionic/react": "^8.5.4", "@isomorphic-git/lightning-fs": "^4.6.0", @@ -129,10 +139,10 @@ "@js-joda/timezone": "2.5.0", "@logseq/diff-merge": "^0.2.2", "@logseq/react-tweet-embed": "1.3.1-1", - "@logseq/sqlite-wasm": "=0.1.0", "@radix-ui/colors": "^0.1.8", "@sentry/react": "^6.18.2", "@sentry/tracing": "^6.18.2", + "@sqlite.org/sqlite-wasm": "^3.50.3-build1", "@tabler/icons-react": "^2.47.0", "@tabler/icons-webfont": "^2.47.0", "@tippyjs/react": "4.2.5", @@ -154,6 +164,7 @@ "fuse.js": "6.4.6", "grapheme-splitter": "1.0.4", "graphology": "0.20.0", + "hnswlib-wasm": "^0.8.2", "html2canvas": "^1.4.1", "ignore": "5.1.8", "inter-ui": "^3.19.3", @@ -174,7 +185,6 @@ "react-dom": "18.3.1", "react-grid-layout": "0.16.6", "react-intersection-observer": "^9.3.5", - "react-resize-context": "3.0.0", "react-textarea-autosize": "8.3.3", "react-transition-group": "4.3.0", "react-virtuoso": "4.12.5", @@ -185,6 +195,7 @@ "tailwind-capitalize-first-letter": "^1.0.4", "threads": "1.6.5", "url": "^0.11.0", + "util": "^0.12.5", "yargs-parser": "20.2.4" }, "resolutions": { diff --git a/public/index.html b/public/index.html new file mode 100644 index 0000000000..73d3015898 --- /dev/null +++ b/public/index.html @@ -0,0 +1,69 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + Logseq: "A privacy-first platform for knowledge management and collaboration." + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/resources/index.html b/resources/index.html index 3ca6e1f172..67e94ede31 100644 --- a/resources/index.html +++ b/resources/index.html @@ -58,8 +58,8 @@ const portal = new MagicPortal(worker); + - diff --git a/resources/mobile/index.html b/resources/mobile/index.html index 78543706d3..b175f4782b 100644 --- a/resources/mobile/index.html +++ b/resources/mobile/index.html @@ -36,7 +36,7 @@ const portal = new MagicPortal(worker); - +