Merge pull request #432 from lomnido/feat-clean_sync

sync '--clean', fix bare_repos, additional tests

Author: lomnido

tsrc/cleaner.py

@@ -0,0 +1,40 @@
+from pathlib import Path
+from typing import List
+
+import cli_ui as ui
+
+from tsrc.executor import Outcome, Task
+from tsrc.repo import Repo
+
+
+class Cleaner(Task[Repo]):
+    def __init__(
+        self,
+        workspace_path: Path,
+        *,
+        do_hard_clean: bool = False,
+    ) -> None:
+        self.workspace_path = workspace_path
+        self.do_hard_clean = do_hard_clean
+
+    def describe_item(self, item: Repo) -> str:
+        return item.dest
+
+    def describe_process_start(self, item: Repo) -> List[ui.Token]:
+        return ["Cleaning", item.dest]
+
+    def describe_process_end(self, item: Repo) -> List[ui.Token]:
+        return [ui.green, "ok", ui.reset, item.dest]
+
+    def process(self, index: int, count: int, repo: Repo) -> Outcome:
+        """
+        Clean each repo so it will be ready for next 'sync'
+        """
+        self.info_count(index, count, "Cleaning", repo.dest)
+
+        repo_path = self.workspace_path / repo.dest
+        self.run_git(repo_path, "clean", "-f", "-d")
+        if self.do_hard_clean is True:
+            self.run_git(repo_path, "clean", "-f", "-X", "-d")
+
+        return Outcome.empty()

tsrc/cli/__init__.py

@@ -334,7 +334,7 @@ def repos_from_config(
                 ", ".join(repo_groups),
             )
         # fmt: on
-        return manifest.get_repos(groups=repo_groups)
+        return manifest.get_repos(groups=repo_groups, ignore_if_group_not_found=silent)
     else:
         # workspace config does not specify clone_all_repos nor
         # a list of groups, ask the manifest for the list of default

tsrc/cli/dump_manifest.py

@@ -76,7 +76,7 @@ def configure_parser(subparser: argparse._SubParsersAction) -> None:
     parser.add_argument(
         "-U",
         "--update-on",
-        help="Set UPDATE operation mode, by setting the UPDATE source and DESTINATION default to provided UPDATE_AT path to YAML file. Such path must exists",  # noqa: E501
+        help="Set UPDATE operation mode, by setting the UPDATE source and DESTINATION default to provided UPDATE_ON path to YAML file. Such path must exists",  # noqa: E501
         type=Path,
         dest="update_on",
     )
@@ -87,22 +87,28 @@ def configure_parser(subparser: argparse._SubParsersAction) -> None:
         dest="no_repo_delete",
     )
     parser.add_argument(
-        "--sha1-only",
+        "--sha1-on",
         action="store_true",
-        help="Use SHA1 as only value (with branch if available) for every considered Repo. This is particulary useful when we want to point to exact point of Repos states",  # noqa: E501
-        dest="sha1_only",
+        help="Explicitly set SHA1 for every considered Repo. This is particulary useful when we want to point to exact commit in Repos",  # noqa: E501
+        dest="sha1_on",
+    )
+    parser.add_argument(
+        "--sha1-off",
+        action="store_true",
+        help="Tell dumping mechanism that we do not care about excat Repo commit, as long as it keep 'branch' and 'tag's. This option is ignored, when there is no 'branch' or 'tag'",  # noqa: E501
+        dest="sha1_off",
     )
     parser.add_argument(
         "-X",
-        "--skip-manifest",
+        "--skip-manifest-repo",
         help="Skip manifest repository if found. If not, it is ignored. For this filter to work, the Workspace needs to be present. And it is only applied after the processing of the Repositories",  # noqa: E501
         dest="skip_manifest",
         default=False,
         action="store_true",
     )
     parser.add_argument(
         "-M",
-        "--only-manifest",
+        "--only-manifest-repo",
         help="Only work with manifest repository if found. If not, the Error is thrown that list of Repositories ends up empty. For this filter to work, the Workspace needs to be present. And it is only applied after the processing of the Repositories",  # noqa: E501
         dest="only_manifest",
         default=False,

tsrc/cli/manifest.py

@@ -84,7 +84,7 @@ def configure_parser(subparser: argparse._SubParsersAction) -> None:
 
 
 def run(args: argparse.Namespace) -> None:
-    gtf = GroupsToFind(args.groups)
+    gtf = GroupsToFind(args.groups, args.ignore_if_group_not_found)
     groups_seen = simulate_get_workspace_with_repos(args)
     gtf.found_these(groups_seen)
 
@@ -129,9 +129,13 @@ def run(args: argparse.Namespace) -> None:
     )
     if args.manifest_branch:
         cfg_update_data = ConfigUpdateData(manifest_branch=args.manifest_branch)
-        status_header.register_change(
-            cfg_update_data, [ConfigUpdateType.MANIFEST_BRANCH]
-        )
+        if (
+            status_header.register_change(
+                cfg_update_data, [ConfigUpdateType.MANIFEST_BRANCH]
+            )
+            is False
+        ):
+            return
     status_header.display()
     status_collector = StatusCollector(
         workspace, ignore_group_item=args.ignore_group_item

tsrc/cli/status.py

@@ -21,7 +21,7 @@
     ready_tmp_bare_repos,
 )
 from tsrc.manifest_common_data import ManifestsTypeOfData
-from tsrc.pcs_repo import get_deep_manifest_from_local_manifest_pcsrepo
+from tsrc.pcs_repo import PCSRepo, get_deep_manifest_from_local_manifest_pcsrepo
 from tsrc.repo import Repo
 from tsrc.status_endpoint import (
     BareStatus,
@@ -102,7 +102,7 @@ def configure_parser(subparser: argparse._SubParsersAction) -> None:
 
 
 def run(args: argparse.Namespace) -> None:
-    gtf = GroupsToFind(args.groups)
+    gtf = GroupsToFind(args.groups, args.ignore_if_group_not_found)
     groups_seen = simulate_get_workspace_with_repos(args)
     gtf.found_these(groups_seen)
 
@@ -124,23 +124,23 @@ def run(args: argparse.Namespace) -> None:
         )
 
     # DM (if present) + bare DM (if DM and present)
-    dm = None
+    dm_pcsr: Union[PCSRepo, None] = None
     bare_dm_repos: List[Repo] = []
     if args.use_deep_manifest is True:
-        dm, gtf = get_deep_manifest_from_local_manifest_pcsrepo(
+        dm_pcsr, gtf = get_deep_manifest_from_local_manifest_pcsrepo(
             workspace,
             gtf,
         )
-        if dm and args.local_git_only is False:
+        if dm_pcsr and args.local_git_only is False:
             # this require to check remote
             bare_dm_repos = prepare_tmp_bare_dm_repos(
-                workspace, dm, gtf, num_jobs=get_num_jobs(args)
+                workspace, dm_pcsr, gtf, num_jobs=get_num_jobs(args)
             )
 
     wrs = WorkspaceReposSummary(
         workspace,
         gtf,
-        dm,
+        dm_pcsr,
         manifest_marker=args.use_manifest_marker,
         future_manifest=args.use_future_manifest,
         use_same_future_manifest=args.use_same_future_manifest,

tsrc/cli/sync.py

@@ -50,6 +50,18 @@ def configure_parser(subparser: argparse._SubParsersAction) -> None:
         dest="ignore_group_item",
         help="ignore group element if it is not found among Manifest's Repos. WARNING: If you end up in need of this option, you have to understand that you end up with useles Manifest. Warnings will be printed for each Group element that is missing, so it may be easier to fix that. Using this option is NOT RECOMMENDED for normal use",  # noqa: E501
     )
+    parser.add_argument(
+        "--clean",
+        action="store_true",
+        dest="do_clean",
+        help="WARNING: you may loose files that are not under the version control. like such files that are ignored by '.gitignore'. sync to clean state, so the next sync can run smoothly. use with care.",  # noqa: E501
+    )
+    parser.add_argument(
+        "--hard-clean",
+        action="store_true",
+        dest="do_hard_clean",
+        help="WARNING: you may loose files that are not under the version control and also files ignored by '.gitignore'. sync to clean state, that does not even contain ignored files. use with care.",  # noqa: E501
+    )
     parser.add_argument(
         "--no-correct-branch",
         action="store_false",
@@ -77,6 +89,8 @@ def run(args: argparse.Namespace) -> None:
     correct_branch = args.correct_branch
     workspace = get_workspace(args)
     num_jobs = get_num_jobs(args)
+    do_clean = args.do_clean
+    do_hard_clean = args.do_hard_clean
 
     ignore_if_group_not_found: bool = False
     report_update_repo_groups: bool = False
@@ -93,10 +107,11 @@ def run(args: argparse.Namespace) -> None:
                 found_groups = list(
                     set(groups).intersection(local_manifest.group_list.groups)
                 )
-                workspace.update_config_repo_groups(
-                    groups=found_groups, ignore_group_item=args.ignore_group_item
-                )
-                report_update_repo_groups = True
+                if update_config_repo_groups is True:
+                    workspace.update_config_repo_groups(
+                        groups=found_groups, ignore_group_item=args.ignore_group_item
+                    )
+                    report_update_repo_groups = True
 
         if update_config_repo_groups is True:
             if args.ignore_if_group_not_found is True:
@@ -112,6 +127,8 @@ def run(args: argparse.Namespace) -> None:
             ui.info_2("Leaving repo_groups intact")
     else:
         ui.info_2("Not updating manifest")
+    if args.ignore_if_group_not_found is True:
+        ignore_if_group_not_found = True
 
     workspace.repos = resolve_repos(
         workspace,
@@ -123,7 +140,9 @@ def run(args: argparse.Namespace) -> None:
         ignore_if_group_not_found=ignore_if_group_not_found,
         ignore_group_item=args.ignore_group_item,
     )
-
+    if len(workspace.repos) == 0:
+        ui.info_1("Nothing to synchronize, skipping")
+        return
     workspace.clone_missing(num_jobs=num_jobs)
     workspace.set_remotes(num_jobs=num_jobs)
     workspace.sync(
@@ -132,5 +151,6 @@ def run(args: argparse.Namespace) -> None:
         correct_branch=correct_branch,
         num_jobs=num_jobs,
     )
+    workspace.clean(do_clean=do_clean, do_hard_clean=do_hard_clean, num_jobs=num_jobs)
     workspace.perform_filesystem_operations(ignore_group_item=args.ignore_group_item)
     ui.info_1("Workspace synchronized")

tsrc/cloner.py

@@ -180,31 +180,36 @@ def _choose_remote(self, repo: Repo) -> Remote:
 
         return repo.remotes[0]
 
-    def bare_clone_repo(self, repo: Repo) -> None:
+    def bare_clone_repo(self, repo: Repo) -> Path:
         # check if our Repo is bare
         repo_path = self.workspace_path / repo.dest
         parent = repo_path.parent
         parent.mkdir(parents=True, exist_ok=True)
         remote = self._choose_remote(repo)
         remote_url = remote.url
         if Path(str(repo_path) + os.sep + ".git").is_dir():
-            return
-        if repo._bare_clone_path:
-            clone_args = [
-                "clone",
-                "--mirror",
-                str(repo._bare_clone_path),
-                str(repo_path) + os.sep + ".git",
-            ]
-        else:
-            clone_args = [
-                "clone",
-                "--mirror",
-                remote_url,
-                str(repo_path) + os.sep + ".git",
-            ]
+            return repo_path
+        clone_args = [
+            "clone",
+            "--mirror",
+            remote_url,
+            str(repo_path) + os.sep + ".git",
+        ]
 
-        self.run_git(parent, *clone_args)
+        run_git_captured(parent, *clone_args)
+
+        # make sure from this moment on to use 'repo_path'
+
+        run_git_captured(
+            repo_path, "config", "--bool", "core.bare", "false", check=False
+        )
+
+        run_git_captured(repo_path, "remote", "remove", remote.name, check=False)
+        run_git_captured(
+            repo_path, "remote", "add", remote.name, remote_url, check=False
+        )
+
+        return repo_path
 
     def bare_set_branch(self, repo: Repo) -> bool:
 
@@ -244,7 +249,8 @@ def bare_reset_repo(self, repo: Repo) -> None:
     def process(self, index: int, count: int, repo: Repo) -> Outcome:
 
         self.info_count(index, count, repo.dest, end="\r")
-        self.bare_clone_repo(repo)
+        repo_path = self.bare_clone_repo(repo)
+        self.run_git(repo_path, "fetch", "--all", "--prune")
         if self.bare_set_branch(repo) is True:
             self.bare_reset_repo(repo)
         # NOTE: not considering submodules (not useful for bare Repo)

tsrc/config_status.py

@@ -15,6 +15,7 @@
 from tsrc.config_data import ConfigUpdateData, ConfigUpdateType
 from tsrc.config_status_rc import ConfigStatusReturnCode
 from tsrc.config_tools import ConfigTools
+from tsrc.errors import Error
 from tsrc.status_header_dm import StatusHeaderDisplayMode
 from tsrc.workspace import Workspace
 
@@ -110,6 +111,7 @@ def _manifest_branch_report_issue(
                 "ignoring",
                 ui.reset,
             )
+            raise Error("aborting Manifest branch change")
         if rc == ConfigStatusReturnCode.CANCEL:
             branch_0 = self.workspace.config.manifest_branch_0
             if branch == branch_0:

tsrc/dump_manifest.py

@@ -584,9 +584,7 @@ def _add_repos_based_on_mris(
                     rr["branch"] = mri.branch
                 if mri.tag:
                     rr["tag"] = mri.tag
-                if (
-                    not mri.branch and not mri.tag and mri.sha1
-                ) or mdo.sha1_only is True:
+                if (not mri.branch and not mri.tag and mri.sha1) or mdo.sha1_on is True:
                     rr["sha1"] = mri.sha1
 
                 # TODO: add comment in form of '\n' just to better separate Repos
@@ -718,7 +716,12 @@ def _update_on_update_on_items_on_repo(
                     and mri.sha1  # noqa: W503
                     and y[u_i] != mri.sha1  # noqa: W503
                 )
-                or (mdo.sha1_only is True and y[u_i] != mri.sha1)  # noqa: W503
+                or (mdo.sha1_on is True and y[u_i] != mri.sha1)  # noqa: W503
+                or (
+                    mdo.sha1_off is False
+                    and (mri.ahead > 0 or mri.behind > 0)
+                    and y[u_i] != mri.sha1
+                )
             ):
                 y[u_i] = mri.sha1
                 ret_updated = True
@@ -833,7 +836,10 @@ def _update_on_items_on_repo(
         if not s_item:
             if mri.sha1:
                 s_item.append("sha1")
-        if "sha1" not in s_item and mdo.sha1_only is True:
+        if "sha1" not in s_item and (
+            mdo.sha1_on is True
+            or (mdo.sha1_off is False and (mri.behind > 0 or mri.ahead > 0))
+        ):
             s_item.append("sha1")
 
         # add these on item
@@ -1145,8 +1151,10 @@ def do_create(
             if items.tag:
                 rr["tag"] = items.tag
             if (
-                not items.branch and not items.tag and items.sha1
-            ) or mdo.sha1_only is True:
+                (not items.branch and not items.tag and items.sha1)
+                or mdo.sha1_on is True
+                or (mdo.sha1_off is False and (items.ahead > 0 or items.behind > 0))
+            ):
                 rr["sha1"] = items.sha1
 
             y["repos"].append(rr)

tsrc/dump_manifest_args.py

@@ -67,15 +67,19 @@ def __init__(self, args: argparse.Namespace) -> None:
 
     def _get_manifest_data_options(self) -> ManifestDataOptions:
         mdo = ManifestDataOptions()
-        if self.args.sha1_only is True:
-            mdo.sha1_only = True
+        if self.args.sha1_on is True and self.args.sha1_off is True:
+            raise Exception("'--sha1-on' and '--sha1-off' are mutually exclusive")
+        elif self.args.sha1_on is True:
+            mdo.sha1_on = True
+        elif self.args.sha1_off is True:
+            mdo.sha1_off = True
         if self.args.skip_manifest is True:
             mdo.skip_manifest = True
         if self.args.only_manifest is True:
             mdo.only_manifest = True
         if self.args.skip_manifest is True and self.args.only_manifest is True:
             raise Exception(
-                "'--skip-manifest' and '--only-manifest' are mutually exclusive"
+                "'--skip-manifest-repo' and '--only-manifest-repo' are mutually exclusive"
             )
         return mdo