Merge pull request #514 from M0ses/refactor_archive

Refactor archive

Author: M0ses

.pylintrc

@@ -432,4 +432,4 @@ exclude-protected=_asdict,_fields,_replace,_source,_make
 
 # Exceptions that will emit a warning when being caught. Defaults to
 # "Exception"
-overgeneral-exceptions=Exception
+overgeneral-exceptions=builtins.Exception

.pylinttestsrc

@@ -430,4 +430,4 @@ exclude-protected=_asdict,_fields,_replace,_source,_make
 
 # Exceptions that will emit a warning when being caught. Defaults to
 # "Exception"
-overgeneral-exceptions=Exception
+overgeneral-exceptions=builtins.Exception

GNUmakefile

@@ -113,9 +113,9 @@ flake8:
 		echo "Skipping flake8 - python2 in CI" \
 	;else \
 		if [ "x$(FLAKE83)" != "x" ]; then \
-			echo "Running flake83";\
-			$(FLAKE83);\
-			echo "Finished flake83";\
+			echo "Running flake83"\
+			$(FLAKE83)\
+			echo "Finished flake83"\
 		else \
 			echo "flake8 for python3 not found or python major version == 2 ($(python_version_major))";\
 		fi \

TarSCM/archive.py

@@ -46,42 +46,12 @@ def extract_from_archive(self, repodir, files, outdir):
 
                 shutil.copy2(src, outdir)
 
-
-class ObsCpio(BaseArchive):
-    def create_archive(self, scm_object, **kwargs):
-        """Create an OBS cpio archive of repodir in destination directory.
+    def filter_files(self, filelist, topdir, args):
+        """
+        Filter filelist by exclude/include parameters
         """
-        basename         = kwargs['basename']
-        dstname          = kwargs['dstname']
-        version          = kwargs['version']
-        args             = kwargs['cli']
-        commit           = scm_object.get_current_commit()
         package_metadata = args.package_meta
 
-        (workdir, topdir) = os.path.split(scm_object.arch_dir)
-        extension = 'obscpio'
-
-        cwd = os.getcwd()
-        os.chdir(workdir)
-
-        archivefilename = os.path.join(args.outdir, dstname + '.' + extension)
-        archivefile     = open(archivefilename, "w")
-
-        # detect reproducible support
-        params = ['cpio', '--create', '--format=newc', '--owner', '0:0']
-        chkcmd = "cpio --create --format=newc --reproducible "
-        chkcmd += "</dev/null >/dev/null 2>&1"
-        if os.system(chkcmd) == 0:
-            params.append('--reproducible')
-
-        proc = subprocess.Popen(
-            params,
-            shell  = False,
-            stdin  = subprocess.PIPE,
-            stdout = archivefile,
-            stderr = subprocess.STDOUT
-        )
-
         # transform glob patterns to regular expressions
         includes  = ''
         excludes  = r'$.'
@@ -96,7 +66,7 @@ def create_archive(self, scm_object, **kwargs):
 
         # add topdir without filtering for now
         cpiolist = []
-        for root, dirs, files in os.walk(topdir, topdown=False):
+        for root, dirs, files in filelist:
             # excludes
             dirs[:] = [os.path.join(root, d) for d in dirs]
             dirs[:] = [d for d in dirs if not re.match(excludes, d)]
@@ -114,9 +84,44 @@ def create_archive(self, scm_object, **kwargs):
             for name in files:
                 if not METADATA_PATTERN.match(name) or package_metadata:
                     cpiolist.append(name)
+        return sorted(cpiolist)
 
+class ObsCpio(BaseArchive):
+    def create_archive(self, scm_object, **kwargs):
+        """Create an OBS cpio archive of repodir in destination directory.
+        """
+        basename         = kwargs['basename']
+        dstname          = kwargs['dstname']
+        version          = kwargs['version']
+        args             = kwargs['cli']
+        commit           = scm_object.get_current_commit()
+
+        (workdir, topdir) = os.path.split(scm_object.arch_dir)
+        extension = 'obscpio'
+
+        cwd = os.getcwd()
+        os.chdir(workdir)
+
+        archivefilename = os.path.join(args.outdir, dstname + '.' + extension)
+        archivefile     = open(archivefilename, "w")
+
+        # detect reproducible support
+        params = ['cpio', '--create', '--format=newc', '--owner', '0:0']
+        chkcmd = "cpio --create --format=newc --reproducible "
+        chkcmd += "</dev/null >/dev/null 2>&1"
+        if os.system(chkcmd) == 0:
+            params.append('--reproducible')
+
+        proc = subprocess.Popen(
+            params,
+            shell  = False,
+            stdin  = subprocess.PIPE,
+            stdout = archivefile,
+            stderr = subprocess.STDOUT
+        )
+        filelist = os.walk(topdir, topdown=False)
         tstamp = self.helpers.get_timestamp(scm_object, args, topdir)
-        for name in sorted(cpiolist):
+        for name in self.filter_files(filelist, topdir, args):
             try:
                 os.utime(name, (tstamp, tstamp), follow_symlinks=False)
             except OSError:
@@ -186,24 +191,6 @@ def create_archive(self, scm_object, **kwargs):
             pat = fnmatch.translate(os.path.join(topdir, exc))
             excl_patterns.append(re.compile(pat))
 
-        def tar_exclude(filename):
-            """
-            Exclude (return True) or add (return False) file to tar achive.
-            """
-            if not package_metadata and METADATA_PATTERN.match(filename):
-                return True
-
-            if incl_patterns:
-                for pat in incl_patterns:
-                    if pat.match(filename):
-                        return False
-                return True
-
-            for pat in excl_patterns:
-                if pat.match(filename):
-                    return True
-            return False
-
         def reset(tarinfo):
             """Python 2.7 only: reset uid/gid to 0/0 (root)."""
             tarinfo.uid = tarinfo.gid = 0
@@ -212,31 +199,32 @@ def reset(tarinfo):
                 tarinfo.mtime = timestamp
             return tarinfo
 
-        def tar_filter(tarinfo):
-            if tar_exclude(tarinfo.name):
-                return None
-
-            return reset(tarinfo)
-
         cwd = os.getcwd()
         os.chdir(workdir)
         enc = locale.getpreferredencoding()
 
         out_file = os.path.join(outdir, dstname + '.' + extension)
+        filelist = os.walk(topdir, topdown=False)
+
+        files_added = dict()
 
         with tarfile.open(out_file, "w", encoding=enc) as tar:
             try:
                 tar.add(topdir, recursive=False, filter=reset)
             except TypeError:
                 # Python 2.6 compatibility
                 tar.add(topdir, recursive=False)
-            for entry in map(lambda x: os.path.join(topdir, x),
-                             sorted(os.listdir(topdir))):
-                try:
-                    tar.add(entry, filter=tar_filter)
-                except TypeError:
-                    # Python 2.6 compatibility
-                    tar.add(entry, exclude=tar_exclude)
+            for entry in self.filter_files(filelist, topdir, args):
+                logging.debug("Filtered file: %s", entry)
+                if not files_added.get(entry, False):
+                    logging.debug("Adding filtered file: %s", entry)
+                    try:
+                        tar.add(entry, recursive=False, filter=reset)
+                    except TypeError:
+                        # Python 2.6 compatibility
+                        tar.add(entry, exclude=tar_exclude)
+                    files_added[entry] = True
+                    logging.debug("Added filtered file: %s", entry)
 
         self.archivefile    = tar.name
 

TarSCM/scm/base.py

@@ -55,6 +55,8 @@ def __init__(self, args, task):
         self.task           = task
         self.url            = args.url
 
+        self.in_osc = bool(os.getenv('OSC_VERSION'))
+
         # optional arguments
         self.revision       = args.revision
         if args.user and args.keyring_passphrase:
@@ -270,9 +272,8 @@ def prepare_clone_dir(self):
         # of the git repo and fetch local changes
         is_snap = sys.argv[0].endswith("snapcraft")
         is_obs_scm = self.args.use_obs_scm
-        in_osc = bool(os.getenv('OSC_VERSION'))
         in_git = os.path.isdir('.git')
-        if is_snap or (is_obs_scm and in_osc and in_git):
+        if is_snap or (is_obs_scm and self.in_osc and in_git):
             self.repodir = os.getcwd()
 
         # construct repodir (the parent directory of the checkout)
@@ -299,7 +300,7 @@ def _calc_dir_to_clone_to(self, prefix):
         osc_version = 0
 
         logging.debug(" - SUBDIR: %s", self.args.subdir)
-        if not self.args.subdir and self.scm == 'git':
+        if not self.args.subdir and self.scm == 'git' and not self.in_osc:
             self.partial_clone = True
             logging.debug("NO SUBDIR FOUND - USING PARTIAL CLONE")
             if self.repocachedir:

tests/commontests.py

@@ -62,6 +62,16 @@ def test_tar_exclude(self):
                     self.basename() + '/subdir/b']
         self.assertTrue(tarents == expected)
 
+    def test_tar_exclude_re(self):
+        self.tar_scm_std('--exclude', '(a|c)')
+        tar_file = os.path.join(self.outdir, self.basename()+'.tar')
+        tar      = tarfile.open(tar_file)
+        tarents  = tar.getnames()
+        expected = [self.basename(),
+                    self.basename() + '/subdir',
+                    self.basename() + '/subdir/b']
+        self.assertTrue(tarents == expected)
+
     def test_tar_include(self):
         self.tar_scm_std('--include', self.fixtures.subdir)
         tar_file = os.path.join(self.outdir, self.basename()+'.tar')