fix(docs): fix reference doc generation

wandb · Jul 24, 2023 · b9ac6a0 · b9ac6a0
1 parent 8befc05
commit b9ac6a0
Show file tree

Hide file tree

Showing 4 changed files with 82 additions and 42 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,23 @@
+# Install pre-commit hooks via `pip install pre-commit && pre-commit install`
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-merge-conflict
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: "v0.0.280"
+    hooks:
+      - id: ruff
+        args: [ --fix]
+  - repo: https://github.com/psf/black
+    rev: 23.7.0
+    hooks:
+      - id: black
+  - repo: https://github.com/asottile/blacken-docs
+    rev: 1.15.0
+    hooks:
+      - id: blacken-docs
diff --git a/cli.py b/cli.py
@@ -5,7 +5,6 @@
 
 from library import format_readme_titles
 
-import fileinput
 
 PATTERN = re.compile(r"(.*?)  +(.*)")
 # (           - Start of capture
@@ -20,9 +19,8 @@
 TEMPLATE = "# {}\n\n{}\n\n{}\n{}\n{}"
 
 # Replace auto-genearted title as a key, provide the preferred title as the value
-MARKDOWN_TITLES = {
-    'wandb' : 'Command Line Interface'
-}
+MARKDOWN_TITLES = {"wandb": "Command Line Interface"}
+
 
 def build(output_dir: str = None):
     """
@@ -186,17 +184,21 @@ def pre_process(help_page: str) -> str:
                               nvidia-docker is present
     ```
     We find two types of overflow problems:
-    - new description: Due to the overflow of the option `--sync-tensorboard / --no-sync-tensorboard`
-        the description `Stream tfevent files to wandb.` is sent to the next line.
-    - wrapped description: Due to the overflow of the description `Use the nvidia runtime, defaults to nvidia if`
+    - new description: Due to the overflow of the option
+        `--sync-tensorboard / --no-sync-tensorboard`
+        the description `Stream tfevent files to wandb.`
+        is sent to the next line.
+    - wrapped description: Due to the overflow of the description
+        `Use the nvidia runtime, defaults to nvidia if`
         the rest of the description is wrapped to the next line.
 
     This breaks our parser that looks for lines with `option   description`.
     With this function we transform these lines into the suitable format that our parser
     expects.
     ```bash
     --sync-tensorboard / --no-sync-tensorboard   Stream tfevent files to wandb.
-    --nvidia / --no-nvidia   Use the nvidia runtime, defaults to nvidia if nvidia-docker is present.
+    --nvidia / --no-nvidia   Use the nvidia runtime,
+       defaults to nvidia if nvidia-docker is present.
     ```
 
     Args:

diff --git a/generate.py b/generate.py
@@ -7,32 +7,28 @@
 python generate.py --help
 """
 import argparse
-import configparser
 import os
-from pathlib import Path
 import shutil
 
 import wandb
 
 import cli
 import library
 
-import util
-import fileinput
-import markdownify
 
-# Replace auto-genearted title as a key, provide the preferred title as the value
+# Replace auto-generated title as a key, provide the preferred title as the value
 MARKDOWN_TITLES = {
-    "python" : "Python Library",
-    'data-types' : 'Data Types',
-    'public-api' : 'Import & Export API',
-    'integrations' : 'Integrations',
-    'ref' : 'Reference',
-    'java' : 'Java Library [Beta]',
-    'keras' : 'Keras',
-    'weave' : 'Weave'
+    "python": "Python Library",
+    "data-types": "Data Types",
+    "public-api": "Import & Export API",
+    "integrations": "Integrations",
+    "ref": "Reference",
+    "java": "Java Library [Beta]",
+    "keras": "Keras",
+    "weave": "Weave",
 }
 
+
 def main(args):
     commit_id = args.commit_id
 
@@ -67,7 +63,7 @@ def main(args):
 
 
 def rename_to_readme(directory):
-    """Moves all the folder-level markdown files into their respective folders, as a README."""
+    """Moves all folder-level markdown files into respective folders, as a README."""
     for root, folders, file_names in os.walk(directory):
         for file_name in file_names:
             raw_file_name, suffix = file_name[:-3], file_name[-3:]
@@ -77,7 +73,10 @@ def rename_to_readme(directory):
                     os.path.join(f"{root}", raw_file_name, "README.md"),
                 )
                 # Format README doc titles to perferred title
-                library.format_readme_titles(os.path.join(f"{root}", raw_file_name, "README.md"), MARKDOWN_TITLES)
+                library.format_readme_titles(
+                    os.path.join(f"{root}", raw_file_name, "README.md"), MARKDOWN_TITLES
+                )
+
 
 def clean_names(directory):
     """Converts names to lower case and removes spaces."""
@@ -92,8 +91,9 @@ def clean_names(directory):
                 os.path.join(f"{root}", f"{short_name}"),
             )
 
+
 def single_folder_format(directory):
-    """Converts all sub-folders that only contain README.md to single files, as expected by GitBook.
+    """Converts all sub-folders that only contain README.md to single files.
 
     So the tree:
         - folder
@@ -119,7 +119,6 @@ def single_folder_format(directory):
                 os.rmdir(root)
 
 
-
 def get_args():
     parser = argparse.ArgumentParser(
         description="Generate documentation for the wandb library and CLI."
@@ -151,8 +150,8 @@ def get_args():
         "--output_dir",
         type=str,
         default=os.getcwd(),
-        help=f"Folder into which to place folder {library.DIRNAME}/ containing results. "
-        + "Defaults to current directory.",
+        help="Folder into which to place folder "
+        "{library.DIRNAME}/ containing results. " + "Defaults to current directory.",
     )
     return parser.parse_args()
 

diff --git a/library.py b/library.py
@@ -27,29 +27,31 @@
 
 WANDB_CORE, WANDB_DATATYPES, WANDB_API, WANDB_INTEGRATIONS = subconfigs
 
+
 def format_readme_titles(readme_file_path: str, markdown_titles: dict):
     """
-    Reads in README files and changes the title based on W&B Eng defined 
+    Reads in README files and changes the title based on W&B Eng defined
     markdown titles passed by a global dictionary.
 
     Args:
         readme_file_path (str): filepath to README
-        markdown_titles (dict): the default titles generated by docugen/generate.py are 
-                                listed as keys. Values are our preferred README filename.
+        markdown_titles (dict): the default titles generated by docugen/generate.py are
+            listed as keys. Values are our preferred README filename.
     """
-    
-    with open(readme_file_path, 'r') as f:
-        first_line = f.readline().strip('# ').strip('\n')
+
+    with open(readme_file_path, "r") as f:
+        first_line = f.readline().strip("# ").strip("\n")
 
         if first_line in markdown_titles:
-            new_title = '# ' + markdown_titles[first_line]
+            new_title = "# " + markdown_titles[first_line]
 
-            with fileinput.FileInput(readme_file_path, inplace = True) as f:    
+            with fileinput.FileInput(readme_file_path, inplace=True) as f:
                 for line in f:
                     if f.isfirstline():
-                        print(new_title, end='\n')
+                        print(new_title, end="\n")
                     else:
-                        print(line, end='')
+                        print(line, end="")
+
 
 def build(commit_id, code_url_prefix, output_dir):
     """Builds docs in stages: main library, then subcomponents."""
@@ -59,9 +61,19 @@ def build(commit_id, code_url_prefix, output_dir):
     build_docs_from_config(WANDB_CORE, commit_id, code_url_prefix, output_dir)
 
     modules_output_dir = os.path.join(output_dir, LIBRARY_DIRNAME)
-    build_docs_from_config(WANDB_DATATYPES, commit_id, code_url_prefix, modules_output_dir)
+    build_docs_from_config(
+        WANDB_DATATYPES,
+        commit_id,
+        code_url_prefix,
+        modules_output_dir,
+    )
     build_docs_from_config(WANDB_API, commit_id, code_url_prefix, modules_output_dir)
-    build_docs_from_config(WANDB_INTEGRATIONS, commit_id, code_url_prefix, modules_output_dir)
+    build_docs_from_config(
+        WANDB_INTEGRATIONS,
+        commit_id,
+        code_url_prefix,
+        modules_output_dir,
+    )
 
 
 def build_docs_from_config(config, commit_id, code_url_prefix, output_dir):
@@ -105,7 +117,7 @@ def handle_additions(add_from, add_elements):
     except AttributeError:  # if it's not an attribute, assume it needs to be imported
         module = import_module("." + add_from, "wandb")
     except ModuleNotFoundError as e:
-        raise(e)
+        raise (e)
     for element in add_elements:
         setattr(wandb, element, getattr(module, element))
 
@@ -140,6 +152,10 @@ def configure_doc_hiding():
         )
 
     from tensorflow import keras
+
     deco = doc_controls.do_not_doc_in_subclasses
     doc_controls.decorate_all_class_attributes(
-        decorator=deco, cls=keras.callbacks.Callback, skip=["__init__", "set_model", "set_params"])
+        decorator=deco,
+        cls=keras.callbacks.Callback,
+        skip=["__init__", "set_model", "set_params"],
+    )