Automate JSON generation

src/docbuild/cli/cmd_metadata/__init__.py

@@ -13,7 +13,8 @@
 from .metaprocess import process
 
 # Set up rich consoles for output
-console_out = Console()
+stdout = Console()
+console_err = Console(stderr=True, style='red')
 
 
 @click.command(help=__doc__)
@@ -22,10 +23,18 @@
     nargs=-1,
     callback=validate_doctypes,
 )
+@click.option(
+    '--exitfirst',
+    is_flag=True,
+    default=False,
+    show_default=True,
+    help='Exit on first failed deliverable.',
+)
 @click.pass_context
 def metadata(
     ctx: click.Context,
     doctypes: tuple[Doctype],
+    exitfirst: bool,
 ) -> None:
     """Subcommand to create metadata files.
 
@@ -44,9 +53,9 @@ def metadata(
     t = None
     try:
         with timer() as t:
-            result = asyncio.run(process(context, doctypes))
+            result = asyncio.run(process(context, doctypes, exitfirst=exitfirst))
     finally:
         if t and not math.isnan(t.elapsed):
-            console_out.print(f'Elapsed time {t.elapsed:0.2f}s')
+            stdout.print(f'Elapsed time {t.elapsed:0.2f}s')
 
     ctx.exit(result)

src/docbuild/cli/cmd_metadata/metaprocess.py

@@ -14,11 +14,12 @@
 from ...models.deliverable import Deliverable
 from ...models.doctype import Doctype
 from ...utils.contextmgr import PersistentOnErrorTemporaryDirectory
+from ...utils.git import ManagedGitRepo
 from ..context import DocBuildContext
 
 # Set up rich consoles for output
-console_out = Console()
-console_err = Console(stderr=True)
+stdout = Console()
+console_err = Console(stderr=True, style='red')
 
 # Set up logging
 log = logging.getLogger(__name__)
@@ -36,8 +37,8 @@ def get_deliverable_from_doctype(
     :param doctype: The Doctype object to process.
     :return: A list of deliverables for the given doctype.
     """
-    console_out.print(f'Getting deliverable for doctype: {doctype}')
-    console_out.print(f'XPath for {doctype}: {doctype.xpath()}')
+    # stdout.print(f'Getting deliverable for doctype: {doctype}')
+    # stdout.print(f'XPath for {doctype}: {doctype.xpath()}')
     languages = root.getroot().xpath(f'./{doctype.xpath()}')
 
     return [
@@ -56,7 +57,7 @@ async def process_deliverable(
     meta_cache_dir: Path,
     dapstmpl: str,
 ) -> bool:
-    """Process a single deliverable.
+    """Process a single deliverable asynchronously.
 
     This function creates a temporary clone of the deliverable's repository,
     checks out the correct branch, and then executes the DAPS command to
@@ -76,7 +77,7 @@ async def process_deliverable(
     :return: True if successful, False otherwise.
     :raises ValueError: If required configuration paths are missing.
     """
-    console_out.print(f'> Processing deliverable: {deliverable.full_id}')
+    log.info('> Processing deliverable: %s', deliverable.full_id)
 
     meta_cache_dir = Path(meta_cache_dir)
 
@@ -132,21 +133,22 @@ async def process_deliverable(
                     output=str(outputdir / deliverable.dcfile),
                 )
             )
-            console_out.print(f'  command: {cmd}')
+            # stdout.print(f'  command: {cmd}')
             daps_process = await asyncio.create_subprocess_exec(
                 *cmd,
                 stdin=asyncio.subprocess.DEVNULL,
+                stdout=asyncio.subprocess.PIPE,
                 stderr=asyncio.subprocess.PIPE,
             )
             _, stderr = await daps_process.communicate()
             if daps_process.returncode != 0:
                 # Raise an exception on failure.
                 raise RuntimeError(
-                    f'DAPS command failed for {deliverable.full_id}: '
+                    f'DAPS command {" ".join(cmd)!r} failed for {deliverable.full_id}: '
                     f'{stderr.decode().strip()}'
                 )
 
-        console_out.print(f'> Processed deliverable: {deliverable.pdlangdc}')
+        # stdout.print(f'> Processed deliverable: {deliverable.pdlangdc}')
         return True
 
     except RuntimeError as e:
@@ -155,64 +157,99 @@ async def process_deliverable(
         return False
 
 
+async def update_repositories(
+    deliverables: list[Deliverable], bare_repo_dir: Path
+) -> bool:
+    """Update all Git repositories associated with the deliverables.
+
+    :param deliverables: A list of Deliverable objects.
+    :param bare_repo_dir: The root directory for storing permanent bare clones.
+    """
+    log.info('Updating Git repositories...')
+    unique_urls = {d.git.url for d in deliverables}
+    repos = [ManagedGitRepo(url, bare_repo_dir) for url in unique_urls]
+
+    tasks = [repo.clone_bare() for repo in repos]
+    results = await asyncio.gather(*tasks, return_exceptions=True)
+
+    res = True
+    for repo, result in zip(repos, results):
+        if isinstance(result, Exception) or not result:
+            log.error('Failed to update repository %s', repo.slug)
+            res = False
+
+    return res
+
+
 async def process_doctype(
     root: etree._ElementTree,
     context: DocBuildContext,
     doctype: Doctype,
-) -> bool:
+    *,
+    exitfirst: bool = False,
+) -> list[Deliverable]:
     """Process the doctypes and create metadata files.
 
     :param root: The stitched XML node containing configuration.
     :param context: The DocBuildContext containing environment configuration.
     :param doctypes: A tuple of Doctype objects to process.
+    :param exitfirst: If True, stop processing on the first failure.
+    :return: True if all files passed validation, False otherwise
     """
     # Here you would implement the logic to process the doctypes
     # and create metadata files based on the stitchnode and context.
     # This is a placeholder for the actual implementation.
-    console_out.print(f'Processing doctypes: {doctype}')
+    # stdout.print(f'Processing doctypes: {doctype}')
     # xpath = doctype.xpath()
     # print("XPath: ", xpath)
-    console_out.print(f'XPath: {doctype.xpath()}', markup=False)
-
-    deliverables = await asyncio.to_thread(
-        get_deliverable_from_doctype,
-        root,
-        context,
-        doctype,
-    )
-    console_out.print(f'Found deliverables: {len(deliverables)}')
-    dapsmetatmpl = context.envconfig.get('build', {}).get('daps', {}).get('meta', None)
+    # stdout.print(f'XPath: {doctype.xpath()}', markup=False)
 
-    console_out.print(f'daps command: {dapsmetatmpl}', markup=False)
+    if not context.envconfig:
+        raise RuntimeError('No envconfig found in context. Maybe no config provided?')
+    else:
+        env = context.envconfig
 
-    repo_dir = context.envconfig.get('paths', {}).get('repo_dir', None)
-    base_cache_dir = context.envconfig.get('paths', {}).get('base_cache_dir', None)
+    repo_dir_str = env.get('paths', {}).get('repo_dir', None)
+    base_cache_dir_str = env.get('paths', {}).get('base_cache_dir', None)
     # We retrieve the path.meta_cache_dir and fall back to path.base_cache_dir
     # if not available:
-    meta_cache_dir = context.envconfig.get('paths', {}).get(
-        'meta_cache_dir', base_cache_dir
+    meta_cache_dir_str = env.get('paths', {}).get(
+        'meta_cache_dir', base_cache_dir_str
     )
     # Cloned temporary repo:
-    temp_repo_dir = context.envconfig.get('paths', {}).get('temp_repo_dir', None)
+    temp_repo_dir_str = env.get('paths', {}).get('temp_repo_dir', None)
 
     # Check all paths:
-    if not all((repo_dir, base_cache_dir, temp_repo_dir, meta_cache_dir)):
+    if not all((repo_dir_str, base_cache_dir_str, temp_repo_dir_str, meta_cache_dir_str)):
         raise ValueError(
             'Missing required paths in configuration: '
-            f'{repo_dir=}, {base_cache_dir=}, {temp_repo_dir=}, {meta_cache_dir=}'
+            f'repo_dir={repo_dir_str}, base_cache_dir={base_cache_dir_str}, '
+            f'temp_repo_dir={temp_repo_dir_str}, meta_cache_dir={meta_cache_dir_str}'
         )
 
     # Ensure base directories exist
-    temp_repo_dir = Path(temp_repo_dir)
-    meta_cache_dir = Path(meta_cache_dir)
-    base_cache_dir = Path(base_cache_dir)
-    repo_dir = Path(repo_dir)
+    temp_repo_dir = Path(temp_repo_dir_str)
+    meta_cache_dir = Path(meta_cache_dir_str)
+    base_cache_dir = Path(base_cache_dir_str)
+    repo_dir = Path(repo_dir_str)
     temp_repo_dir.mkdir(parents=True, exist_ok=True)
     meta_cache_dir.mkdir(parents=True, exist_ok=True)
     base_cache_dir.mkdir(parents=True, exist_ok=True)
     repo_dir.mkdir(parents=True, exist_ok=True)
 
-    tasks = [
+    deliverables = await asyncio.to_thread(
+         get_deliverable_from_doctype,
+         root,
+         context,
+         doctype,
+     )
+
+    await update_repositories(deliverables, repo_dir)
+
+    # stdout.print(f'Found deliverables: {len(deliverables)}')
+    dapsmetatmpl = env.get('build', {}).get('daps', {}).get('meta', None)
+
+    coroutines = [
         process_deliverable(
             deliverable,
             repo_dir=repo_dir,
@@ -223,19 +260,53 @@ async def process_doctype(
         )
         for deliverable in deliverables
     ]
-    results = await asyncio.gather(*tasks)
 
-    return all(results)
+    tasks = [asyncio.create_task(coro) for coro in coroutines]
+    failed_deliverables: list[Deliverable] = []
+
+    if exitfirst:
+        # Fail-fast behavior
+        for task in asyncio.as_completed(tasks):
+            result = await task
+            if not result:
+                # The task failed, so we find which deliverable it was
+                # This is a bit indirect but works with the current `process_deliverable`
+                # A better approach would be for `process_deliverable` to
+                # return the deliverable on failure
+                failed_deliverables.extend(
+                    d for d, t in zip(deliverables, tasks) if t is task
+                )
+                # Cancel remaining tasks
+                for t in tasks:
+                    if not t.done():
+                        t.cancel()
+                # Wait for cancellations to propagate
+                await asyncio.gather(*tasks, return_exceptions=True)
+                break  # Exit the loop on first failure
+
+    else:
+        # Run all and collect all results
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        failed_deliverables.extend(
+            deliverable
+            for deliverable, success in zip(deliverables, results)
+            if not success
+        )
+
+    return failed_deliverables
 
 
 async def process(
     context: DocBuildContext,
     doctypes: tuple[Doctype],
+    *,
+    exitfirst: bool=False,
 ) -> int:
     """Asynchronous function to process metadata retrieval.
 
     :param context: The DocBuildContext containing environment configuration.
-    :param xmlfiles: A tuple or iterator of XML file paths to validate.
+    :param doctype: A Doctype object to process.
+    :param exitfirst: If True, stop processing on the first failure.
     :raises ValueError: If no envconfig is found or if paths are not
         configured correctly.
     :return: 0 if all files passed validation, 1 if any failures occurred.
@@ -247,16 +318,27 @@ async def process(
     if configdir is None:
         raise ValueError('Could not get a value from envconfig.paths.config_dir')
     configdir = Path(configdir).expanduser()
-    console_out.print(f'Config path: {configdir}')
+    # stdout.print(f'Config path: {configdir}')
     xmlconfigs = tuple(configdir.rglob('[a-z]*.xml'))
     stitchnode = await create_stitchfile(xmlconfigs)
-    console_out.print(f'Stitch node: {stitchnode.getroot().tag}')
-    console_out.print(f'Deliverables: {len(stitchnode.xpath(".//deliverable"))}')
+    # stdout.print(f'Stitch node: {stitchnode.getroot().tag}')
+    # stdout.print(f'Deliverables: {len(stitchnode.xpath(".//deliverable"))}')
 
     if not doctypes:
         doctypes = [Doctype.from_str(DEFAULT_DELIVERABLES)]
 
-    tasks = [process_doctype(stitchnode, context, dt) for dt in doctypes]
-    results = await asyncio.gather(*tasks)
-    console_out.print(f'Results: {results}')
+    tasks = [process_doctype(stitchnode, context, dt, exitfirst=exitfirst)
+             for dt in doctypes]
+    results_per_doctype = await asyncio.gather(*tasks)
+
+    all_failed_deliverables = [
+        d for failed_list in results_per_doctype for d in failed_list
+    ]
+
+    if all_failed_deliverables:
+        console_err.print(f'Found {len(all_failed_deliverables)} failed deliverables:')
+        for d in all_failed_deliverables:
+            console_err.print(f'- {d.full_id}')
+        return 1
+
     return 0

src/docbuild/cli/cmd_validate/process.py

@@ -277,8 +277,9 @@ async def process_file(
     # Run all checks for this file
     check_results = await run_python_checks(tree)
 
-    # Display results based on verbosity level
-    display_results(shortname, check_results, context.verbose, max_len)
+    if check_results:
+        # Display results based on verbosity level
+        display_results(shortname, check_results, context.verbose, max_len)
 
     return 0 if all(result.success for _, result in check_results) else 1
 
@@ -324,7 +325,8 @@ async def process(
 
     # Filter for files that passed the initial validation
     successful_files_paths = [
-        xmlfile for xmlfile, result in zip(xmlfiles, results, strict=False) if result == 0
+        xmlfile for xmlfile, result in zip(xmlfiles, results, strict=False)
+        if result == 0
     ]
 
     # After validating individual files, perform a stitch validation to
@@ -356,11 +358,14 @@ async def process(
     # Display summary
     successful_part = f'[green]{successful_files}/{total_files} files(s)[/green]'
     failed_part = f'[red]{failed_files} file(s)[/red]'
-    summary_msg = f'{successful_part} successfully validated, {failed_part} failed.'
+    summary_msg = (
+        f'{successful_part} successfully validated, {failed_part} failed. '
+        f'Stitch validation [blue]{stitch_success}[/blue]'
+    )
+
+    final_success = (failed_files == 0) and stitch_success
 
     if context.verbose > 0:  # pragma: no cover
         console_out.print(f'Result: {summary_msg}')
 
-    final_success = (failed_files == 0) and stitch_success
-
     return 0 if final_success else 1