Redefine the Catalog Hierarchy using Closure Tables (#954)

* WIP: made changes to orm and adapter

* MNT: fix typo

* ENH: upgrade path in Alembic migration

* ENH: add triggers to alembig migration

* MNT: add alembic downgrade path

* FIX: add include_data_sources to parent

* DOC: update the description for the Nodes tables

* MNT: clean-up ORM definitions

* ENH: add indices

* TST: update fts5 script

* FIX: prevent the root node from being deleted

* TST: fix the join operation in aliased ORMs when applying conditions

* ENH: fix recursive joins

* TST: fix tests

* REV: revert changes

* FIX: key attribute in CatalogNodeAdapter

* MNT: dependency and linting

* MNT: add PG dependency

* TST: use sqlite or pg uri

* TST: use sqlite or pg uri

* Update CHANGELOG.md

* ENH: add index on parent column

* FIX: create_index signature

* FIX: cartesian product warning

* Cleanup rebase mistake

* Drop synchronous PG dependency (we use asyncpg)

* MNT: changelog

* FIX: move mounting a node to async engine

---------

Co-authored-by: Dan Allan <[email protected]>

Author: genematx

CHANGELOG.md

@@ -17,6 +17,11 @@ continuous deployment processes.
 
 ## v0.1.0-b31 (2025-08-01)
 
+### Changed
+
+- The logic of hierarchical organization of the Nodes table in Catalog: use the concept
+  of Closure Table to track ancestors and descendands of the nodes.
+
 ### Added
 
 - Pooling of ADBC connections to storage databases.
@@ -39,13 +44,16 @@ continuous deployment processes.
   dependencies injection
 - Updated front-end dependencies, and updated node version used for building
   front-end.
+- The logic of hierarchical organization of the Nodes table in Catalog: use the
+  concept of Closure Table to track ancestors and descendants of the nodes.
 
 ### Fixed
 
 - Restored authentication check for API key
 - Updated usage for change in Zarr 3.x API.
 - Improved error message if config location is non-file
 
+
 ## v0.1.0-b29 (2025-06-06)
 
 ### Added

docs/source/explanations/catalog.md

@@ -31,10 +31,11 @@ presents the data to clients. Each row represents one node in the logical
 "tree" of data represented by Tiled.
 
 - `metadata` --- user-controlled JSON object, with arbitrary metadata
-- `ancestors` and `key` --- together specify the unique path of the data
+- `key` --- the name of the Node; together with ancestors specify the unique path of the data
 - `structre_family` --- enum of structure types (`"container"`, `"array"`, `"table"`, ...)
 - `specs` --- user-controlled JSON list of specs, such as `[{"name": "XDI", "version": "1"}]`
-- `id` an internal integer primary key, not exposed by the API
+- `id` --- an internal integer primary key, not exposed by the API
+- `parent` --- the `id` of the node's parent
 - `time_created` and `time_updated` --- for forensics, not exposed by the API
 
 The `time_created` and `time_updated` columns, which appear in this table and

tiled/_tests/test_access_control.py

@@ -468,7 +468,7 @@ def test_public_access(
         public_client_g["g", "A3"]
 
 
-def test_service_principal_access(tmpdir):
+def test_service_principal_access(tmpdir, sqlite_or_postgres_uri):
     "Test that a service principal can work with SimpleAccessPolicy."
     config = {
         "authentication": {
@@ -494,7 +494,7 @@ def test_service_principal_access(tmpdir):
             {
                 "tree": "catalog",
                 "args": {
-                    "uri": f"sqlite:///{tmpdir}/catalog.db",
+                    "uri": sqlite_or_postgres_uri,
                     "writable_storage": f"file://localhost{tmpdir}/data",
                     "init_if_not_exists": True,
                 },

tiled/_tests/test_catalog.py

@@ -60,8 +60,8 @@ async def test_nested_node_creation(a):
         specs=[],
     )
     c = await b.lookup_adapter(["c"])
-    assert b.segments == ["b"]
-    assert c.segments == ["b", "c"]
+    assert await b.path_segments() == ["b"]
+    assert await c.path_segments() == ["b", "c"]
     assert (await a.keys_range(0, 1)) == ["b"]
     assert (await b.keys_range(0, 1)) == ["c"]
     # smoke test
@@ -344,7 +344,7 @@ async def test_delete_tree(tmpdir):
         d.write_array([7, 8, 9])
 
         nodes_before_delete = (await tree.context.execute("SELECT * from nodes")).all()
-        assert len(nodes_before_delete) == 7
+        assert len(nodes_before_delete) == 7 + 1  # +1 for the root node
         data_sources_before_delete = (
             await tree.context.execute("SELECT * from data_sources")
         ).all()
@@ -361,7 +361,7 @@ async def test_delete_tree(tmpdir):
         await tree.delete_tree(external_only=False)
 
         nodes_after_delete = (await tree.context.execute("SELECT * from nodes")).all()
-        assert len(nodes_after_delete) == 0
+        assert len(nodes_after_delete) == 0 + 1  # the root node that should remain
         data_sources_after_delete = (
             await tree.context.execute("SELECT * from data_sources")
         ).all()
@@ -371,7 +371,7 @@ async def test_delete_tree(tmpdir):
 
 
 @pytest.mark.asyncio
-async def test_access_control(tmpdir):
+async def test_access_control(tmpdir, sqlite_or_postgres_uri):
     config = {
         "authentication": {
             "allow_anonymous_access": True,
@@ -410,7 +410,7 @@ async def test_access_control(tmpdir):
                 "tree": "catalog",
                 "path": "/",
                 "args": {
-                    "uri": f"sqlite:///{tmpdir}/catalog.db",
+                    "uri": sqlite_or_postgres_uri,
                     "writable_storage": str(tmpdir / "data"),
                     "init_if_not_exists": True,
                 },
@@ -536,7 +536,7 @@ async def test_constraints_on_parameter_and_num(a, assets):
 
 
 @pytest.mark.asyncio
-async def test_init_db_logging(tmpdir, caplog):
+async def test_init_db_logging(sqlite_or_postgres_uri, tmpdir, caplog):
     config = {
         "database": {
             "uri": "sqlite://",  # in-memory
@@ -546,7 +546,7 @@ async def test_init_db_logging(tmpdir, caplog):
                 "tree": "catalog",
                 "path": "/",
                 "args": {
-                    "uri": f"sqlite:///{tmpdir}/catalog.db",
+                    "uri": sqlite_or_postgres_uri,
                     "writable_storage": str(tmpdir / "data"),
                     "init_if_not_exists": True,
                 },

tiled/_tests/test_cli.py

@@ -93,7 +93,7 @@ def test_serve_catalog_temp(args, tmp_path):
         "",
     ],
 )
-def test_serve_config(args, tmp_path):
+def test_serve_config(args, tmp_path, sqlite_or_postgres_uri):
     "Test 'tiled serve config' with a tmp config file."
     (tmp_path / "data").mkdir()
     (tmp_path / "config").mkdir()
@@ -107,7 +107,7 @@ def test_serve_config(args, tmp_path):
   - path: /
     tree: catalog
     args:
-      uri: sqlite:///{tmp_path / 'catalog.db'}
+      uri: {sqlite_or_postgres_uri}
       writable_storage: {tmp_path / 'data'}
       init_if_not_exists: true
 """

tiled/_tests/test_writing.py

@@ -510,7 +510,7 @@ async def test_delete(tree):
             key="delete_me",
         )
         nodes_before_delete = (await tree.context.execute("SELECT * from nodes")).all()
-        assert len(nodes_before_delete) == 1
+        assert len(nodes_before_delete) == 1 + 1  # +1 for the root node
         data_sources_before_delete = (
             await tree.context.execute("SELECT * from data_sources")
         ).all()
@@ -531,7 +531,7 @@ async def test_delete(tree):
         client.delete("delete_me")
 
         nodes_after_delete = (await tree.context.execute("SELECT * from nodes")).all()
-        assert len(nodes_after_delete) == 0
+        assert len(nodes_after_delete) == 0 + 1  # the root node should still exist
         data_sources_after_delete = (
             await tree.context.execute("SELECT * from data_sources")
         ).all()