Merge pull request #429 from lomnido/feat-tmp_bare_repo

new feature: tmp bare repo: show position for DM|FM if SHA1

Author: lomnido

tsrc/cli/__init__.py

@@ -263,10 +263,26 @@ def resolve_repos_without_workspace(
     return repos
 
 
+def is_match_repo_dest_on_inc_excl(
+    gac: GroupsAndConstraints,
+    i_r_d: str,
+) -> bool:
+    if (
+        (gac.include_regex and re.search(gac.include_regex, i_r_d))  # noqa: W503
+        or not gac.include_regex  # noqa: W503
+    ) and (
+        (gac.exclude_regex and not re.search(gac.exclude_regex, i_r_d))  # noqa: W503
+        or not gac.exclude_regex  # noqa: W503
+    ):
+        return True
+    return False
+
+
 def resolve_repos_apply_constraints(
     repos: List[Repo],
     gac: GroupsAndConstraints,
 ) -> List[Repo]:
+    # NOTE: code duplication, see Fn above, and above above
     """
     Use just constraints on Repos in GroupAndConstraints class
     to filter Repos. Consider:

tsrc/cli/dump_manifest.py

@@ -10,7 +10,7 @@
     So we do not have '.tscr/config' or any other '.tsrc' data
     (not even Groups if they are not present in Manifest when we updating it)
 
-Siplest way to start a new 'tsrc' project by creating Manifest is to prepare
+Simplest way to start a new 'tsrc' project by creating Manifest is to prepare
 every repository into some dedicated directory and from there call:
 
 'tsrc dump-manifest --raw .'
@@ -26,7 +26,7 @@
 import argparse
 import io
 from pathlib import Path
-from typing import Dict, List, Tuple, Union
+from typing import Dict, List, Tuple, Union, cast
 
 import cli_ui as ui
 from ruamel.yaml import YAML
@@ -50,7 +50,7 @@
 from tsrc.executor import process_items
 from tsrc.file_system import make_relative
 from tsrc.repo import Repo
-from tsrc.status_endpoint import CollectedStatuses, StatusCollector
+from tsrc.status_endpoint import CollectedStatuses, Status, StatusCollector
 from tsrc.utils import erase_last_line
 
 
@@ -86,6 +86,28 @@ def configure_parser(subparser: argparse._SubParsersAction) -> None:
         help="Disallow to delete any Repo record from existing Manifest. This have only meaning when on UPDATE operation mode",  # noqa: E501
         dest="no_repo_delete",
     )
+    parser.add_argument(
+        "--sha1-only",
+        action="store_true",
+        help="Use SHA1 as only value (with branch if available) for every considered Repo. This is particulary useful when we want to point to exact point of Repos states",  # noqa: E501
+        dest="sha1_only",
+    )
+    parser.add_argument(
+        "-X",
+        "--skip-manifest",
+        help="Skip manifest repository if found. If not, it is ignored. For this filter to work, the Workspace needs to be present. And it is only applied after the processing of the Repositories",  # noqa: E501
+        dest="skip_manifest",
+        default=False,
+        action="store_true",
+    )
+    parser.add_argument(
+        "-M",
+        "--only-manifest",
+        help="Only work with manifest repository if found. If not, the Error is thrown that list of Repositories ends up empty. For this filter to work, the Workspace needs to be present. And it is only applied after the processing of the Repositories",  # noqa: E501
+        dest="only_manifest",
+        default=False,
+        action="store_true",
+    )
     parser.add_argument(
         "-p",
         "--preview",
@@ -150,8 +172,6 @@ def __init__(self, a: DumpManifestArgs, num_jobs: int) -> None:
 
         # output data
         self.yy: Union[Dict, List, None] = None
-        self.load_path: Union[Path, None] = None  # may be useless
-        self.save_path: Union[Path, None] = None  # may be useless
         self.is_updated: Union[bool, None] = None
 
         # everything in regard of args
@@ -177,7 +197,7 @@ def get_data(self) -> None:
                 self.mris_h = MRISHelpers(repos=repos)
             elif self.a.dmod.source_mode == SourceModeEnum.WORKSPACE_DUMP:
                 statuses, w_repos = self._get_data_get_statuses()
-                for status in statuses:
+                for _, status in statuses.items():
                     if not (
                         isinstance(status, MissingRepoError)
                         or isinstance(status, Exception)  # noqa: W503
@@ -203,6 +223,27 @@ def _get_data_get_repos(self) -> List[Repo]:
             if self.a.dmod.source_path:
                 mgr = ManifestRawGrabber(self.a, self.a.dmod.source_path)
                 repos, self.a = mgr.grab(self.num_jobs)
+                if (
+                    self.a.args.skip_manifest is True
+                    or self.a.args.only_manifest is True  # noqa: W503
+                ):
+                    if self.a.dmod.workspace:
+                        repos, _ = self.m_du.filter_repos_bo_manifest(
+                            self.a.dmod.workspace,
+                            self.a.args.skip_manifest,
+                            self.a.args.only_manifest,
+                            repos,
+                        )
+                    else:
+                        if self.a.args.skip_manifest is True:
+                            ui.warning(
+                                "Cannot skip Deep Manifest if there is no Workspace"
+                            )
+                        elif self.a.args.only_manifest is True:
+                            ui.warning(
+                                "Cannot look for Deep Manifest if there is no Workspace"
+                            )
+                            repos = []
         except Exception as e:
             raise (e)
 
@@ -212,12 +253,24 @@ def _get_data_get_statuses(self) -> Tuple[CollectedStatuses, List[Repo]]:
         if self.a.dmod.workspace:
             status_collector = StatusCollector(self.a.dmod.workspace)
             w_repos = self.a.dmod.workspace.repos
+            if self.a.args.skip_manifest is True or self.a.args.only_manifest is True:
+                w_repos, _ = self.m_du.filter_repos_bo_manifest(
+                    self.a.dmod.workspace,
+                    self.a.args.skip_manifest,
+                    self.a.args.only_manifest,
+                    w_repos,
+                )
+
             if not w_repos:
                 raise Exception("Workspace is empty, therefore no valid data")
             ui.info_1(f"Collecting statuses of {len(w_repos)} repo(s)")
             process_items(w_repos, status_collector, num_jobs=self.num_jobs)
             erase_last_line()
-            return status_collector.statuses, w_repos
+            # TODO: we may want to get rid of BareStatus, but there should not be one in any anyway
+            return (
+                cast(Dict[str, Union[Status, Exception]], status_collector.statuses),
+                w_repos,
+            )
         return {}, []
 
     def _get_yaml_data(self) -> None:
@@ -240,17 +293,21 @@ def _get_yaml_data(self) -> None:
                 self.yy, self.is_updated = self.m_du.on_update(
                     y,
                     self.mris_h.mris,
+                    self.a.dmod.workspace,
+                    self.a.dmod.manifest_data_options,
                     self.a.mdo,
                     self.a.gac,
                 )
 
             if not self.yy:
                 raise Exception(
-                    f"Not able to load YAML data from file: '{self.load_path}'"
+                    f"Not able to load YAML data from file: '{self.a.dmod.final_output_path_list.update_on_path}'"  # noqa: E501
                 )
 
         else:  # decided: create Manifest YAML data (not loading YAML data)
-            self.yy = self.m_du.do_create(self.mris_h.mris)
+            self.yy = self.m_du.do_create(
+                self.mris_h.mris, self.a.dmod.manifest_data_options
+            )
             if self.yy:
                 self.is_updated = True
 

tsrc/cli/manifest.py

@@ -157,7 +157,8 @@ def run(args: argparse.Namespace) -> None:
         wrs.ready_data(
             statuses,
         )
-        wrs.separate_leftover_statuses(workspace.repos)
+        wrs.separate_statuses(workspace.repos)
+        wrs.calculate_fields_len()
 
         # only calculate summary when there are some Workspace repos
         if workspace.repos:

tsrc/cli/status.py

@@ -2,7 +2,7 @@
 
 import argparse
 from copy import deepcopy
-from typing import Union
+from typing import Dict, List, Union, cast
 
 from tsrc.cli import (
     add_num_jobs_arg,
@@ -15,8 +15,20 @@
 from tsrc.executor import process_items
 from tsrc.groups import GroupNotFound
 from tsrc.groups_to_find import GroupsToFind
+from tsrc.local_tmp_bare_repos import (
+    prepare_tmp_bare_dm_repos,
+    process_bare_repos,
+    ready_tmp_bare_repos,
+)
+from tsrc.manifest_common_data import ManifestsTypeOfData
 from tsrc.pcs_repo import get_deep_manifest_from_local_manifest_pcsrepo
-from tsrc.status_endpoint import StatusCollector, StatusCollectorLocalOnly
+from tsrc.repo import Repo
+from tsrc.status_endpoint import (
+    BareStatus,
+    Status,
+    StatusCollector,
+    StatusCollectorLocalOnly,
+)
 from tsrc.status_header import StatusHeader, StatusHeaderDisplayMode
 from tsrc.utils import erase_last_line
 
@@ -32,6 +44,12 @@ def configure_parser(subparser: argparse._SubParsersAction) -> None:
     add_workspace_arg(parser)
     add_repos_selection_args(parser)
     add_num_jobs_arg(parser)
+    parser.add_argument(
+        "--show-leftovers-status",
+        action="store_true",
+        help="show full GIT status also for leftovers, if there are some, that have valid repository on the filesystem. here hard error about the repository is ignored and no status is displayed",  # noqa: E501
+        dest="show_leftovers_status",
+    )
     parser.add_argument(
         "--no-mm",
         action="store_false",
@@ -105,12 +123,19 @@ def run(args: argparse.Namespace) -> None:
             ignore_group_item=args.ignore_group_item,
         )
 
+    # DM (if present) + bare DM (if DM and present)
     dm = None
+    bare_dm_repos: List[Repo] = []
     if args.use_deep_manifest is True:
         dm, gtf = get_deep_manifest_from_local_manifest_pcsrepo(
             workspace,
             gtf,
         )
+        if dm and args.local_git_only is False:
+            # this require to check remote
+            bare_dm_repos = prepare_tmp_bare_dm_repos(
+                workspace, dm, gtf, num_jobs=get_num_jobs(args)
+            )
 
     wrs = WorkspaceReposSummary(
         workspace,
@@ -119,6 +144,7 @@ def run(args: argparse.Namespace) -> None:
         manifest_marker=args.use_manifest_marker,
         future_manifest=args.use_future_manifest,
         use_same_future_manifest=args.use_same_future_manifest,
+        show_leftovers_status=args.show_leftovers_status,
     )
 
     status_header = StatusHeader(
@@ -137,15 +163,27 @@ def run(args: argparse.Namespace) -> None:
         )
 
     repos = deepcopy(workspace.repos)
+    bare_fm_repos = wrs.get_bare_fm_repos()
+    bare_fm_repos = ready_tmp_bare_repos(
+        workspace, ManifestsTypeOfData.FUTURE, bare_fm_repos
+    )
+    bare_repos = bare_fm_repos + bare_dm_repos
+    bare_repos = process_bare_repos(workspace, bare_repos, num_jobs=get_num_jobs(args))
+    repos += bare_repos
 
     wrs.prepare_repos()
 
+    leftovers_repos: List[Repo] = []
     if args.strict_on_git_desc is False:
-        repos += wrs.obtain_leftovers_repos(repos)
+        leftovers_repos = wrs.obtain_leftovers_repos(repos)
+        repos += leftovers_repos
 
     if repos:
 
-        status_header.report_collecting(len(repos))
+        # status_header.report_collecting(len(repos))
+        status_header.report_collecting(
+            len(workspace.repos), len(leftovers_repos), len(bare_repos)
+        )
 
         num_jobs = get_num_jobs(args)
         process_items(repos, status_collector, num_jobs=num_jobs)
@@ -154,9 +192,11 @@ def run(args: argparse.Namespace) -> None:
         statuses = status_collector.statuses
 
         wrs.ready_data(
-            statuses,
+            # TODO: this crazines is there due to 'StatusCollectorLocalOnly' is possible
+            cast(Dict[str, Union[Status, Exception, BareStatus, Exception]], statuses),
         )
-        wrs.separate_leftover_statuses(workspace.repos)
+        wrs.separate_statuses(bare_repos)
+        wrs.calculate_fields_len()
 
         # only calculate summary when there are some Workspace repos
         if workspace.repos: