Rollforward of lakefs-spec doc changes (#123)

This is the same change set for the docpage generation script, see
aai-institute/lakefs-spec#273.

Co-authored-by: Adrian Rumpold <[email protected]>

docs/_scripts/gen_api_ref_pages.py

@@ -1,10 +1,15 @@
 """Automatically generate API reference pages from source files.
 
 Source: https://mkdocstrings.github.io/recipes/#automatic-code-reference-pages
+
+Note: this script assumes a source layout with a `src/` folder.
 """
 
+import ast
+import logging
 from pathlib import Path
 
+import docstring_parser
 import mkdocs_gen_files
 
 nav = mkdocs_gen_files.Nav()
@@ -31,5 +36,30 @@
 
     mkdocs_gen_files.set_edit_path(full_doc_path, path)
 
+# Add links for top-level modules to root page
+root_page = next(it for it in nav.items() if it.level == 0)
+children = [it for it in nav.items() if it.level == 1]
+
+with mkdocs_gen_files.open(f"reference/{root_page.filename}", "a") as f:
+    f.write("## Modules\n")
+    for ch in children:
+        f.write(f"### [{ch.title}](../{ch.filename})\n")
+
+        try:
+            source_file = Path("src", ch.filename).with_suffix(".py")
+
+            # Index page for submodules maps to __init__.py of the module
+            if source_file.stem == "index":
+                source_file = source_file.with_stem("__init__")
+
+            tree = ast.parse(source_file.read_text())
+            docstring = ast.get_docstring(tree, clean=False)
+            doc = docstring_parser.parse(docstring)
+
+            if doc.short_description:
+                f.write(f"{doc.short_description}\n\n")
+        except Exception as e:
+            logging.warning(f"Could not parse module docstring: {ch.filename}", exc_info=True)
+
 with mkdocs_gen_files.open("reference/SUMMARY.md", "w") as nav_file:
     nav_file.writelines(nav.build_literate_nav())

pyproject.toml

@@ -52,6 +52,7 @@ dev = [
 ]
 docs = [
     "black",
+    "docstring-parser",
     "mike",
     "mkdocs",
     "mkdocs-callouts",

requirements-docs.txt

@@ -11,6 +11,7 @@ certifi==2024.2.2
 charset-normalizer==3.3.2
 click==8.1.7
 colorama==0.4.6
+docstring-parser==0.16
 essentials==1.1.5
 essentials-openapi==1.0.9
 ghp-import==2.1.0