Merge pull request #71 from sclorg/support_gitlab

Use GitLab for syncing upstream->downstream - gitlab_git_changes

Author: phracek

container_workflow_tool/constants.py

@@ -1,6 +1,6 @@
 action_map = {}
 action_map['git'] = {
-    'pullupstream': 'dist_git_changes',
+    'pullupstream': 'dist_git_merge_changes',
     'clonedownstream': 'pull_downstream',
     'cloneupstream': 'pull_upstream',
     'rebase': 'dist_git_rebase',

container_workflow_tool/distgit.py

@@ -5,7 +5,7 @@
 from git import Repo
 from git.exc import GitCommandError
 
-import container_workflow_tool.utility as u
+from container_workflow_tool import utility
 from container_workflow_tool.utility import RebuilderError
 from container_workflow_tool.dockerfile import DockerfileHandler
 from container_workflow_tool.sync import SyncHandler
@@ -49,21 +49,22 @@ def check_script(self, component, script_path, component_path):
             self.logger.info(template.format(name=component, status="Affected"))
             err = ret.stderr.decode('utf-8').strip()
             if err:
-                self.logger.error(u._2sp(err))
+                self.logger.error(utility._2sp(err))
         else:
             self.logger.info(template.format(name=component, status="OK"))
 
-    def dist_git_changes(self, images, rebase=False):
+    def dist_git_merge_changes(self, images, rebase=False):
         """Method to merge changes from upstream into downstream
 
         Pulls both downstream and upstream repositories into a temporary dir.
         Merge is done by copying tracked files from upstream into downstream.
 
         Args:
+            images (list): List of images to sync
             rebase (bool, optional): Specify if a rebase should be done instead
         """
         try:
-            for image in (images):
+            for image in images:
                 name = image["name"]
                 component = image["component"]
                 branch = image["git_branch"]
@@ -92,18 +93,17 @@ def dist_git_changes(self, images, rebase=False):
                     ups_name = name.split('-')[0]
                     # Clone upstream repository
                     ups_path = os.path.join('upstreams/', ups_name)
-                    self._clone_upstream(url, ups_path, commands=commands)
+                    self.clone_upstream(url, ups_path, commands=commands)
                     # Save the upstream commit hash
                     ups_hash = Repo(ups_path).commit().hexsha
-                    self._pull_upstream(component, path, url, repo, ups_name, commands)
+                    self.pull_upstream(component, path, url, repo, ups_name, commands)
                     self.df_handler.update_dockerfile(
                         df_path, from_tag, downstream_from=downstream_from
                     )
                     repo.git.add("Dockerfile")
                     # It is possible for the git repository to have no changes
                     if repo.is_dirty():
-                        commit = self.get_commit_msg(rebase, image, ups_hash
-                        )
+                        commit = self.get_commit_msg(rebase, image, ups_hash)
                         if commit:
                             repo.git.commit("-m", commit)
                         else:
@@ -122,8 +122,8 @@ def _clone_downstream(self, component, branch):
             self.logger.info("Using existing downstream repo: " + component)
             repo = Repo(component)
         else:
-            hostname_url = u._get_hostname_url(self.conf)
-            packager = u._get_packager(self.conf)
+            hostname_url = utility._get_hostname_url(self.conf)
+            packager = utility._get_packager(self.conf)
             # if packager is fedpkg then namespace is `container` else `containers`
             namespace = "container" if packager == "fedpkg" else "containers"
             component_path = f"{namespace}/{component}"
@@ -162,7 +162,7 @@ def push_changes(self, tmp, images):
                     # commit_msg is set so it is always returned
                     commit = self.get_commit_msg(None, image)
                     repo.git.commit("-am", commit)
-                if self._get_unpushed_commits(repo):
+                if self.are_unpushed_commits_available(repo):
                     self.logger.info("Pushing: " + component)
 
                     repo.git.push()
@@ -176,7 +176,7 @@ def push_changes(self, tmp, images):
         if failed:
             self.logger.error("Failed pushing images:")
             for image in failed:
-                self.logger.error(u._2sp(image["component"]))
+                self.logger.error(utility._2sp(image["component"]))
             self.logger.error("Please check the failures and push the changes manually.")
 
     # TODO: Multiple future branches?
@@ -204,5 +204,5 @@ def merge_future_branches(self, images):
         if failed:
             self.logger.error("Failed merging images:")
             for image in failed:
-                self.logger.error(u._2sp(image["component"]))
+                self.logger.error(utility._2sp(image["component"]))
             self.logger.error("Please check the failures and push the changes manually.")

container_workflow_tool/git_operations.py

@@ -1,6 +1,6 @@
 # MIT License
 #
-# Copyright (c) 2020 SCL team at Red Hat
+# Copyright (c) 2023 SCL team at Red Hat
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -55,7 +55,7 @@ def set_commit_msg(self, msg):
         """
         self.commit_msg = msg
 
-    def _do_git_reset(self, repo):
+    def do_git_reset(self, repo):
         file_list = ['--', '.gitignore'] + self.conf.ignore_files
         repo.git.reset(file_list)
         # One file at a time to make sure all files get reset even on error
@@ -68,7 +68,7 @@ def _do_git_reset(self, repo):
                 repo.git.clean('-xfd', f)
                 self.logger.debug("Removing untracked ignored file: " + f)
 
-    def _clone_upstream(self, url, ups_path, commands=None):
+    def clone_upstream(self, url, ups_path, commands=None):
         """
         :params: url is URL to repofile from upstream. https://github.com/sclorg
         :param: ups_path is path where URL is cloned locally
@@ -113,20 +113,23 @@ def _clone_upstream(self, url, ups_path, commands=None):
         os.chdir(oldcwd)
         return repo
 
-    def are_unpushed_commits_available(self, repo) -> bool:
+    def are_unpushed_commits_available(self, repo, branch_name="") -> bool:
         """
         Get unpushed commits
         :param repo: repo object to check for unpushed commits
+        :param branch_name: In case of gitlab, branch_name has to be defined.
+        :param branch_name: In case of gitlab, branch_name has to be defined.
+                            branch_name is e.g. rhel-8.7.0 and 'repo.active_branch.name' is 'rhel-8.7.0-<ubi_name>'
         :return: List of commits or empty array
         """
         branch = repo.active_branch.name
         # Get a list of commits that have not been pushed to remote
         select = "origin/" + branch + ".." + branch
-        if len(list(repo.iter_commits(select))) == 0:
-            return False
-        return True
+        if branch_name != "":
+            select = "origin/" + branch_name + ".." + branch
+        return bool(list(repo.iter_commits(select)))
 
-    def show_git_changes(self, tmp, components=None, diff=False):
+    def show_git_changes(self, tmp, components=None, diff=False, branch_name=""):
         """Shows changes made to tracked files in local downstream repositories
 
         Walks through all repositories and calls 'git-show' or 'git-diff' on each of them.
@@ -135,6 +138,8 @@ def show_git_changes(self, tmp, components=None, diff=False):
             tmp (str): Path to the directory that is used to store git repositories
             components (list of str, optional): List of components to show changes for
             diff (boolean, optional): Controls whether the method calls git-show or git-diff
+            branch_name (str, optional): In case of gitlab, branch_name has to be defined.
+                            branch_name is e.g. rhel-8.7.0 and 'repo.active_branch.name' is 'rhel-8.7.0-<ubi_name>'
         """
         # Function to check if a path contains a git repository
         def is_git(x): return os.path.isdir(os.path.join(x, '.git'))
@@ -155,7 +160,7 @@ def is_git(x): return os.path.isdir(os.path.join(x, '.git'))
             repo = Repo(path)
             # Only show changes if there are unpushed commits to show
             # or we only want the diff of unstaged changes
-            if self.are_unpushed_commits_available(repo) or diff:
+            if self.are_unpushed_commits_available(repo, branch_name=branch_name) or diff:
                 # Clears the screen
                 print(chr(27) + "[2J")
                 # Force pager for short git diffs
@@ -229,7 +234,7 @@ def get_commit_msg(self, rebase, image=None, ups_hash=None):
             commit += "\n created from upstream commit: " + ups_hash
         return commit
 
-    def _pull_upstream(self, component, path, url, repo, ups_name, commands):
+    def pull_upstream(self, component, path, url, repo, ups_name, commands):
         """Pulls an upstream repo and copies it into downstream"""
         ups_path = os.path.join('upstreams/', ups_name)
         cp_path = os.path.join(ups_path, path)
@@ -268,7 +273,7 @@ def _pull_upstream(self, component, path, url, repo, ups_name, commands):
             self.update_test_openshift_yaml(test_openshift_yaml_file, path, short_name=ups_name)
 
         repo.git.add("*")
-        self._do_git_reset(repo)
+        self.do_git_reset(repo)
         # TODO: Configurable?
         df_ext = self.df_ext
         df_path = os.path.join(component, "Dockerfile")

container_workflow_tool/main.py

@@ -9,7 +9,6 @@
 import re
 import tempfile
 import pprint
-import getpass
 import logging
 
 from git import Repo, GitError
@@ -53,6 +52,7 @@ def __init__(self,
 
         self.conf_name = config
         self.rebuild_reason = rebuild_reason
+        self.gitlab_usage = None
         self.do_image = None
         self.exclude_image = None
         self.do_set = None
@@ -119,10 +119,8 @@ def _setup_args(self, args):
             self.rebuild_reason = args.rebuild_reason
         if getattr(args, 'check_script', None) is not None and args.check_script:
             self.check_script = args.check_script
-        if getattr(args, 'disable_klist', None) is not None and args.disable_klist:
-            self.disable_klist = args.disable_klist
-        if getattr(args, 'latest_release', None) is not None and args.latest_release:
-            self.latest_release = args.latest_release
+        self.disable_klist = args.disable_klist
+        self.latest_release = args.latest_release
         if getattr(args, 'output_file', None) is not None and args.output_file:
             self.output_file = args.output_file
 
@@ -150,7 +148,7 @@ def git_ops(self):
         if not self._git_ops:
             self._git_ops = GitOperations(self.base_image, self.conf,
                                           self.rebuild_reason,
-                                          self.logger.getChild("-git-ops"))
+                                          self.logger.getChild("git-ops"))
         return self._git_ops
 
     @property
@@ -449,16 +447,16 @@ def set_repo_url(self, repo_url):
 
     def list_images(self):
         """Prints list of images that we work with"""
-        for i in self._get_images():
-            self.logger.info(i["component"])
+        for image in self._get_images():
+            self.logger.info(image["component"])
 
     def print_upstream(self):
         """Prints the upstream name and url for images used in config"""
-        for i in self._get_images():
+        for image in self._get_images():
             ups_name = re.search(r".*\/([a-zA-Z0-9-]+).git",
-                                 i["git_url"]).group(1)
-            msg = f"{i.get('component')} {i.get('name')} {ups_name} " \
-                  f"{i.get('git_url')} {i.get('git_path')} {i.get('git_branch')}"
+                                 image["git_url"]).group(1)
+            msg = f"{image.get('component')} {image.get('name')} {ups_name} " \
+                  f"{image.get('git_url')} {image.get('git_path')} {image.get('git_branch')}"
             self.logger.info(msg)
 
     def show_config_contents(self):
@@ -502,17 +500,14 @@ def pull_downstream(self):
         Additionally runs a script against each repository if check_script is set,
         checking its exit value.
         """
-        self._check_kerb_ticket()
-        tmp = self._get_tmp_workdir()
-        self._change_workdir(tmp)
-        images = self._get_images()
-        for i in images:
-            self.distgit._clone_downstream(i["component"], i["git_branch"])
+        tmp, images = self.preparation()
+        for image in images:
+            self.distgit._clone_downstream(image["component"], image["git_branch"])
         # If check script is set, run the script provided for each config entry
         if self.check_script:
-            for i in images:
-                self.distgit.check_script(i["component"], self.check_script,
-                                          i["git_branch"])
+            for image in images:
+                self.distgit.check_script(image["component"], self.check_script,
+                                          image["git_branch"])
 
     def pull_upstream(self):
         """
@@ -521,24 +516,19 @@ def pull_upstream(self):
         Additionally runs a script against each repository if check_script is set,
         checking its exit value.
         """
-        tmp = self._get_tmp_workdir()
-        self._change_workdir(tmp)
-        images = self._get_images()
-        for i in images:
+        tmp, images = self.preparation()
+        for image in images:
             # Use unversioned name as a path for the repository
-            ups_name = i["name"].split('-')[0]
-            self.distgit._clone_upstream(i["git_url"],
-                                         ups_name,
-                                         commands=i["commands"])
+            ups_name = image["name"].split('-')[0]
+            self.git_ops.clone_upstream(image["git_url"], ups_name, commands=image["commands"])
         # If check script is set, run the script provided for each config entry
         if self.check_script:
-            for i in images:
-                ups_name = i["name"].split('-')[0]
-                self.distgit.check_script(i["component"], self.check_script,
-                                          os.path.join(ups_name, i["git_path"]))
+            for image in images:
+                ups_name = image["name"].split('-')[0]
+                self.distgit.check_script(image["component"], self.check_script,
+                                          os.path.join(ups_name, image["git_path"]))
 
-    def push_changes(self):
-        """Pushes changes for all components into downstream dist-git repository"""
+    def preparation(self):
         # Check for kerberos ticket
         self._check_kerb_ticket()
         tmp = self._get_tmp_workdir(setup_dir=False)
@@ -547,31 +537,22 @@ def push_changes(self):
             raise RebuilderError(msg)
         self._change_workdir(tmp)
         images = self._get_images()
+        return tmp, images
+
+    def push_changes(self):
+        """Pushes changes for all components into downstream dist-git repository"""
 
+        tmp, images = self.preparation()
         self.distgit.push_changes(tmp, images)
 
     def dist_git_rebase(self):
         """
         Do a rebase against a new base/s2i image.
         Does not pull in upstream changes of layered images.
         """
-        self.dist_git_changes(rebase=True)
-
-    def dist_git_changes(self, rebase: bool = False):
-        """Method to merge changes from upstream into downstream
+        self.dist_git_merge_changes(rebase=True)
 
-        Pulls both downstream and upstream repositories into a temporary directory.
-        Merge is done by copying tracked files from upstream into downstream.
-
-        Args:
-            rebase (bool, optional): Specifies whether a rebase should be done instead.
-        """
-        # Check for kerberos ticket
-        self._check_kerb_ticket()
-        tmp = self._get_tmp_workdir()
-        self._change_workdir(tmp)
-        images = self._get_images()
-        self.distgit.dist_git_changes(images, rebase)
+    def git_changes_report(self, tmp):
         self.logger.info("\nGit location: " + tmp)
         if self.args:
             tmp_str = ' --tmp ' + self.tmp_workdir if self.tmp_workdir else '"'
@@ -583,13 +564,23 @@ def dist_git_changes(self, rebase: bool = False):
                 "cwt git push && cwt build"
                 "[base/core/s2i] --repo-url link-to-repo-file")
 
+    def dist_git_merge_changes(self, rebase: bool = False):
+        """Method to merge changes from upstream into downstream
+
+        Pulls both downstream and upstream repositories into a temporary directory.
+        Merge is done by copying tracked files from upstream into downstream.
+
+        Args:
+            rebase (bool, optional): Specifies whether a rebase should be done instead.
+        """
+        tmp, images = self.preparation()
+        self.distgit.dist_git_merge_changes(images, rebase)
+        self.git_changes_report(tmp=tmp)
+
     def merge_future_branches(self):
         """Merges current branch with future branches"""
         # Check for kerberos ticket
-        self._check_kerb_ticket()
-        tmp = self._get_tmp_workdir()
-        self._change_workdir(tmp)
-        images = self._get_images()
+        tmp, images = self.preparation()
         self.distgit.merge_future_branches(images)
 
     def show_git_changes(self, components: List = None):
@@ -599,9 +590,8 @@ def show_git_changes(self, components: List = None):
             components (list of str, optional): List of components to show changes for
         Walks through all downstream repositories and calls 'git-show' on each of them.
         """
+        tmp, _ = self.preparation()
         if not components:
             images = self._get_images()
-            components = [i["component"] for i in images]
-        tmp = self._get_tmp_workdir()
-        self._change_workdir(tmp)
+            components = [image["component"] for image in images]
         self.distgit.show_git_changes(tmp, components)