Bad3r:feat/db

.carve/config.edn

@@ -1,9 +1,9 @@
 {;; Only lint production namespaces as most dev
  ;; namespaces are unused
  :paths ["src/main" "src/electron" "src/test"]
- :api-namespaces [
-                  ;; Ignore b/c too many false positives
+ :api-namespaces [;; Ignore b/c too many false positives
                   frontend.db
                   ;; Used for debugging
-                  frontend.db.debug]
+                  frontend.db.debug
+                  frontend.worker.debug]
  :report {:format :ignore}}

.carve/ignore

@@ -37,24 +37,13 @@ frontend.fs/readdir
 ;; Referenced in TODO
 frontend.handler.metadata/update-properties!
 ;; Referenced in comment
-frontend.handler.route/toggle-between-page-and-file!
-;; Referenced in comment
-frontend.handler.shell/run-pandoc-command!
-;; Referenced in comment
 frontend.image/get-orientation
 ;; For debugging
 frontend.mixins/perf-measure-mixin
 ;; Previously useful fn
 frontend.mobile.util/get-idevice-statusbar-height
-;; Used in macro
-frontend.modules.outliner.datascript/transact!
-frontend.modules.outliner.core/*transaction-opts*
-;; Referenced in comment
-frontend.page/route-view
 ;; placeholder fn
 frontend.publishing/stop
-;; Referenced in comment
-frontend.state/set-db-persisted!
 ;; Future use
 frontend.storage/get-transit
 ;; repl fn
@@ -74,7 +63,7 @@ frontend.util/trace!
 ;; Repl fn
 frontend.util.pool/terminate-pool!
 ;; Repl fn
-frontend.util.property/add-page-properties
+frontend.handler.file-based.property.util/add-page-properties
 ;; Test runners used by shadow
 frontend.test.node-test-runner/main
 frontend.test.frontend-node-test-runner/main
@@ -85,3 +74,19 @@ frontend.fs.sync/debug-print-sync-events-loop
 frontend.fs.sync/stop-debug-print-sync-events-loop
 ;; Used in macro
 frontend.state/get-current-edit-block-and-position
+;; For debugging
+frontend.db.model/get-all-classes
+;; Initial loaded
+frontend.ui/_emoji-init-data
+;; placeholder var for defonce
+frontend.worker.rtc.op-mem-layer/_sync-loop-canceler
+;; Used by shadow.cljs
+frontend.worker.db-worker/init
+;; WIP fn, remove when it's ready
+frontend.worker.rtc.asset-sync/<loop-for-assets-sync
+;; Future use?
+frontend.worker.rtc.hash/hash-blocks
+;; Repl fn
+frontend.rum/use-atom-in
+;; missionary utils
+frontend.common.missionary-util/<!

.clj-kondo/config.edn

@@ -7,7 +7,7 @@
   ;; false positive with match/match and _
   frontend.handler.paste {:linters {:used-underscored-binding {:level :off}}}
   frontend.db {:linters {:aliased-namespace-symbol
-                         {:exclude [frontend.db.conn frontend.db.react logseq.db.default]}}}}
+                         {:exclude [frontend.db.conn frontend.db.react]}}}}
 
  :linters
  {:path-invalid-construct/string-join {:level :info}
@@ -21,6 +21,14 @@
                                 goog.string.unescapeEntities
                                 ;; TODO:lint: Fix when fixing all type hints
                                 object]}
+
+  :discouraged-namespace
+  {logseq.db.sqlite.cli {:message "frontend should not depend on CLI namespace with sqlite3 dependency"}
+   logseq.outliner.cli {:message "frontend should not depend on CLI namespace with sqlite3 dependency"}}
+
+  :unused-namespace {:level :warning
+                     :exclude [logseq.db.frontend.entity-plus]}
+
   ;; TODO:lint: Remove node-path excludes once we have a cleaner api
   :unresolved-var {:exclude [frontend.util/node-path.basename
                              frontend.util/node-path.dirname
@@ -44,12 +52,21 @@
              frontend.components.block.macros block-macros
              frontend.components.query query
              frontend.components.query.result query-result
+             frontend.components.class class-component
+             frontend.components.property property-component
+             frontend.common.date common-date
+             frontend.common.file.core common-file
+             frontend.common.file.util wfu
+             frontend.common.missionary-util c.m
+             frontend.common.schema-register sr
+             frontend.common.search-fuzzy fuzzy
              frontend.config config
              frontend.date date
              frontend.db db
              frontend.db-mixins db-mixins
              frontend.db.query-dsl query-dsl
              frontend.db.react react
+             frontend.db.query-custom query-custom
              frontend.db.query-react query-react
              frontend.db.util db-utils
              frontend.diff diff
@@ -66,15 +83,28 @@
              frontend.handler.common.file file-common-handler
              frontend.handler.common.plugin plugin-common-handler
              frontend.handler.common.developer dev-common-handler
+             frontend.handler.common.page page-common-handler
              frontend.handler.config config-handler
              frontend.handler.editor.property editor-property
              frontend.handler.events events
              frontend.handler.global-config global-config-handler
              frontend.handler.ui ui-handler
              frontend.handler.notification notification
              frontend.handler.page page-handler
+             frontend.handler.db-based.editor db-editor-handler
+             frontend.handler.db-based.page db-page-handler
+             frontend.handler.db-based.property db-property-handler
+             frontend.handler.db-based.property.util db-pu
+             frontend.handler.file-based.page file-page-handler
+             frontend.handler.file-based.property file-property-handler
+             frontend.handler.file-based.page-property file-page-property
+             frontend.handler.file-based.property.util property-util
+             frontend.handler.file-based.recent file-recent-handler
+             frontend.handler.file-based.repo file-repo-handler
              frontend.handler.plugin plugin-handler
              frontend.handler.plugin-config plugin-config-handler
+             frontend.handler.property.file property-file
+             frontend.handler.property.util pu
              frontend.handler.query.builder query-builder
              frontend.handler.repo repo-handler
              frontend.handler.repo-config repo-config-handler
@@ -83,8 +113,10 @@
              frontend.idb idb
              frontend.loader loader
              frontend.mixins mixins
+             frontend.modules.outliner.ui ui-outliner-tx
              frontend.mobile.util mobile-util
              frontend.page page
+             frontend.persist-db persist-db
              frontend.schema.handler.common-config common-config-schema
              frontend.search search
              frontend.state state
@@ -93,27 +125,56 @@
              frontend.ui ui
              frontend.util util
              frontend.util.clock clock
+             frontend.util.drawer drawer
              frontend.util.page page-util
              frontend.util.property property
              frontend.util.persist-var persist-var
              frontend.util.text text-util
              frontend.util.url url-util
              frontend.util.thingatpt thingatpt
+             frontend.worker.pipeline worker-pipeline
+             frontend.worker.util worker-util
+             frontend.worker.state worker-state
+             frontend.worker.handler.page worker-page
              lambdaisland.glogi log
-             logseq.common.path path
-             logseq.common.graph common-graph
              logseq.common.config common-config
+             logseq.common.graph common-graph
+             logseq.common.date-time-util date-time-util
+             logseq.common.path path
+             logseq.common.util common-util
+             logseq.common.util.page-ref page-ref
+             logseq.common.util.block-ref block-ref
+             logseq.common.util.macro macro-util
+             logseq.db ldb
+             logseq.db.frontend.content db-content
+             logseq.db.frontend.class db-class
+             logseq.db.frontend.db-ident db-ident
+             logseq.db.frontend.inputs db-inputs
+             logseq.db.frontend.order db-order
+             logseq.db.frontend.property db-property
+             logseq.db.frontend.property.build db-property-build
+             logseq.db.frontend.property.type db-property-type
+             logseq.db.frontend.property.util db-property-util
+             logseq.db.frontend.entity-plus entity-plus
+             logseq.db.frontend.entity-util entity-util
+             logseq.db.frontend.rules rules
+             logseq.db.frontend.schema db-schema
+             logseq.db.frontend.validate db-validate
+             logseq.db.sqlite.cli sqlite-cli
+             logseq.db.sqlite.util sqlite-util
              logseq.graph-parser graph-parser
              logseq.graph-parser.text text
+             logseq.graph-parser.db gp-db
              logseq.graph-parser.block gp-block
              logseq.graph-parser.mldoc gp-mldoc
-             logseq.graph-parser.util gp-util
              logseq.graph-parser.property gp-property
-             logseq.graph-parser.config gp-config
-             logseq.graph-parser.util.page-ref page-ref
-             logseq.graph-parser.util.block-ref block-ref
-             logseq.graph-parser.util.db db-util
-             logseq.graph-parser.date-time-util date-time-util
+             logseq.outliner.batch-tx batch-tx
+             logseq.outliner.core outliner-core
+             logseq.outliner.op outliner-op
+             logseq.outliner.pipeline outliner-pipeline
+             logseq.outliner.datascript-report ds-report
+             logseq.shui.ui shui
+             logseq.shui.popup.core shui-popup
              medley.core medley
              "path" node-path
              promesa.core p}}
@@ -134,11 +195,13 @@
            clojure.test.check.clojure-test/defspec clojure.core/def
            clojure.test.check.properties/for-all clojure.core/for
            ;; src/main
-           frontend.modules.outliner.datascript/auto-transact! clojure.core/let
            frontend.namespaces/import-vars potemkin/import-vars
            ;; src/test
            frontend.test.helper/deftest-async clojure.test/deftest
            frontend.test.helper/with-reset cljs.test/async
-           frontend.react/defc clojure.core/defn}
+           frontend.worker.rtc.idb-keyval-mock/with-reset-idb-keyval-mock cljs.test/async
+           frontend.react/defc clojure.core/defn
+           frontend.common.schema-register/defkeyword cljs.spec.alpha/def}
  :skip-comments true
- :output {:progress true}}
+ :output {:progress true
+          :exclude-files ["src/test/docs-0.10.9/"]}}

.github/workflows/build.yml

@@ -23,7 +23,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Actions Repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Check spelling with custom config file
         uses: crate-ci/[email protected]
         with:
@@ -38,7 +38,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c  # v3.3.0
+        uses: actions/checkout@v4
 
       - name: Set up Node
         uses: actions/setup-node@v3
@@ -77,17 +77,21 @@ jobs:
       - name: Fetch yarn deps
         run: yarn install --frozen-lockfile
 
+      - name: Build test asset
+        run: clojure -M:test compile test
+
+      - name: Run some ClojureScript tests against DB version
+        run: DB_GRAPH=1 node static/tests.js -r frontend.db.query-dsl-test
+
       - name: Run ClojureScript tests
-        run: |
-          yarn cljs:test
-          node static/tests.js
+        run: node static/tests.js -e fix-me
 
   lint:
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c  #v3
+        uses: actions/checkout@v4
 
       - name: Set up Java
         uses: actions/setup-java@v3
@@ -116,12 +120,66 @@ jobs:
       - name: Lint invalid translation entries
         run: bb lang:validate-translations
 
+      - name: Lint to keep worker independent of frontend
+        run: bb lint:worker-and-frontend-separate
+
+      - name: Lint to keep db and file graph code separate
+        run: bb lint:db-and-file-graphs-separate
+
+  db-graph-test:
+    strategy:
+      matrix:
+        operating-system: [ubuntu-latest]
+
+    runs-on: ${{ matrix.operating-system }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          cache: 'yarn'
+          cache-dependency-path: |
+            deps/db/yarn.lock
+            scripts/yarn.lock
+
+      - name: Set up Java
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: ${{ env.JAVA_VERSION }}
+
+      - name: Set up Clojure
+        uses: DeLaGuardo/[email protected]
+        with:
+          cli: ${{ env.CLOJURE_VERSION }}
+          bb: ${{ env.BABASHKA_VERSION }}
+
+      - name: Fetch scripts yarn deps
+        run: cd scripts && yarn install --frozen-lockfile
+
+      - name: Create DB graph with properties
+        run: cd scripts && yarn nbb-logseq src/logseq/tasks/db_graph/create_graph_with_properties.cljs ./db-graph-with-props
+
+      # TODO: Use a smaller, test-focused graph to test classes
+      - name: Create DB graph with classes
+        run: cd scripts && yarn nbb-logseq src/logseq/tasks/db_graph/create_graph_with_schema_org.cljs ./db-graph-with-schema
+
+      - name: Fetch deps/db yarn deps
+        run: cd deps/db && yarn install --frozen-lockfile
+
+      - name: Validate created DB graphs
+        run: cd deps/db && yarn nbb-logseq script/validate_client_db.cljs ../../scripts/db-graph-with-props ../../scripts/db-graph-with-schema --closed-maps --group-errors
+
   e2e-test:
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
+        uses: actions/checkout@v4
 
       - name: Set up Node
         uses: actions/setup-node@v3

.github/workflows/db.yml

@@ -56,6 +56,9 @@ jobs:
       - name: Fetch yarn deps
         run: yarn install --frozen-lockfile
 
+      - name: Run nbb-logseq tests
+        run: yarn test
+
       # In this job because it depends on an npm package
       - name: Load namespaces into nbb-logseq
         run: bb test:load-all-namespaces-with-nbb .
@@ -80,7 +83,7 @@ jobs:
           bb: ${{ env.BABASHKA_VERSION }}
 
       - name: Run clj-kondo lint
-        run: clojure -M:clj-kondo --parallel --lint src
+        run: clojure -M:clj-kondo --lint src test
 
       - name: Carve lint for unused vars
         run: bb lint:carve 2>/dev/null