Change the logic to combine subpackages' documentation (#429)

Author: regisss

docs/combine_docs.py

@@ -20,7 +20,7 @@
 
 def rename_subpackage_toc(subpackage: str, toc: Dict):
     """
-    Extend table of contents sections with subpackage name prefix.
+    Extend table of contents sections with the subpackage name as the parent folder.
 
     Args:
         subpackage (str): subpackage name.
@@ -29,21 +29,73 @@ def rename_subpackage_toc(subpackage: str, toc: Dict):
     for item in toc:
         for file in item["sections"]:
             if "local" in file:
-                file["local"] = f"{subpackage}_" + file["local"]
+                file["local"] = f"{subpackage}/" + file["local"]
             else:
                 # if "local" is not in file, it means we have a subsection, hence the recursive call
                 rename_subpackage_toc(subpackage, [file])
 
 
+def rename_copy_subpackage_html_paths(subpackage: str, subpackage_path: Path, optimum_path: Path, version: str):
+    """
+    Copy and rename the files from the given subpackage's documentation to Optimum's documentation.
+    In Optimum's documentation, the subpackage files are organized as follows:
+
+    optimum_doc
+        ├──────── subpackage_1
+        │               ├────── folder_1
+        │               │          └───── ...
+        │               ├────── ...
+        │               └────── folder_x
+        │                          └───── ...
+        ├──────── ...
+        ├──────── subpackage_n
+        │               ├────── folder_1
+        │               │          └───── ...
+        │               ├────── ...
+        │               └────── folder_y
+        │                          └───── ...
+        └──────── usual_optimum_doc
+
+    Args:
+        subpackage (str): subpackage name
+        subpackage_path (Path): path to the subpackage's documentation
+        optimum_path (Path): path to Optimum's documentation
+        version (str): Optimum's version
+    """
+    subpackage_html_paths = list(subpackage_path.rglob("*.html"))
+    # The language folder is the 4th folder in the subpackage HTML paths
+    language_folder_level = 3
+
+    for html_path in subpackage_html_paths:
+        language_folder = html_path.parts[language_folder_level]
+        # Build the relative path from the language folder
+        relative_path_from_language_folder = Path(*html_path.parts[language_folder_level + 1 :])
+        # New path in Optimum's doc
+        new_path_in_optimum = Path(
+            f"{optimum_path}/optimum/{version}/{language_folder}/{subpackage}/{relative_path_from_language_folder}"
+        )
+        # Build the parent folders if necessary
+        new_path_in_optimum.parent.mkdir(parents=True, exist_ok=True)
+
+        shutil.copyfile(html_path, new_path_in_optimum)
+
+
 def main():
     args = parser.parse_args()
     optimum_path = Path("optimum-doc-build")
+
+    # Copy and rename all files from subpackages' docs to Optimum doc
     for subpackage in args.subpackages:
         subpackage_path = Path(f"{subpackage}-doc-build")
+
         # Copy all HTML files from subpackage into optimum
-        subpackage_html_paths = list(subpackage_path.rglob("*.html"))
-        for html_path in subpackage_html_paths:
-            shutil.copyfile(html_path, f"{optimum_path}/optimum/{args.version}/en/{subpackage}_{html_path.name}")
+        rename_copy_subpackage_html_paths(
+            subpackage,
+            subpackage_path,
+            optimum_path,
+            args.version,
+        )
+
         # Load optimum table of contents
         base_toc_path = next(optimum_path.rglob("_toctree.yml"))
         with open(base_toc_path, "r") as f:
@@ -52,7 +104,7 @@ def main():
         subpackage_toc_path = next(subpackage_path.rglob("_toctree.yml"))
         with open(subpackage_toc_path, "r") as f:
             subpackage_toc = yaml.safe_load(f)
-        # Extend table of contents sections with subpackage name prefix
+        # Extend table of contents sections with the subpackage name as the parent folder
         rename_subpackage_toc(subpackage, subpackage_toc)
         # Update optimum table of contents
         base_toc.extend(subpackage_toc)

docs/source/index.mdx

@@ -21,15 +21,15 @@ As such, 🤗 Optimum enables developers to efficiently use any of these platfor
 
 <div class="mt-10">
   <div class="w-full flex flex-col space-y-4 md:space-y-0 md:grid md:grid-cols-3 md:gap-y-4 md:gap-x-5">
-    <a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="./graphcore_index"
+    <a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="./graphcore/index"
       ><div class="w-full text-center bg-gradient-to-br from-blue-400 to-blue-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed">Optimum Graphcore</div>
       <p class="text-gray-700">Train transformers on <a href="https://www.graphcore.ai/products/ipu">Graphcore IPUs</a>, a completely new kind of massively parallel processor to accelerate machine intelligence.</p>
     </a>
-    <a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="./habana_index"
+    <a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="./habana/index"
       ><div class="w-full text-center bg-gradient-to-br from-indigo-400 to-indigo-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed">Optimum Habana</div>
       <p class="text-gray-700">Maximize training throughput and efficiency with <a href="https://docs.habana.ai/en/latest/Gaudi_Overview/Gaudi_Architecture.html">Habana's Gaudi processor</a>.</p>
     </a>
-    <a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="./intel_index"
+    <a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="./intel/index"
       ><div class="w-full text-center bg-gradient-to-br from-pink-400 to-pink-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed">Optimum Intel</div>
       <p class="text-gray-700">Use Intel's <a href="https://www.intel.com/content/www/us/en/developer/tools/oneapi/neural-compressor.html">Neural Compressor</a> and <a href="https://docs.openvino.ai/latest/index.html">OpenVINO</a> frameworks to accelerate transformer inference.</p>
     </a>