From 95088964ea076bed34b9c8a1065edf765d21a76f Mon Sep 17 00:00:00 2001 From: Jake Fennick Date: Mon, 15 Aug 2022 21:49:42 -0600 Subject: [PATCH 1/7] Fixed another symlinking issue with Docker + staged files from InitialWorkDirRequirement --- cwltool/docker.py | 4 +++- cwltool/job.py | 13 ++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/cwltool/docker.py b/cwltool/docker.py index 7e7c93507..ff463a853 100644 --- a/cwltool/docker.py +++ b/cwltool/docker.py @@ -236,7 +236,9 @@ def add_file_or_directory_volume( """Append volume a file/dir mapping to the runtime option list.""" if not volume.resolved.startswith("_:"): _check_docker_machine_path(volume.resolved) - self.append_volume(runtime, volume.resolved, volume.target) + self.append_volume( + runtime, volume.resolved, volume.target, writable=volume.staged + ) def add_writable_file_volume( self, diff --git a/cwltool/job.py b/cwltool/job.py index 4011641cd..0decc2c06 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -100,18 +100,13 @@ def relink_initialworkdir( continue host_outdir_tgt = os.path.join(host_outdir, vol.target[len(container_outdir) + 1 :]) if os.path.islink(host_outdir_tgt) or os.path.isfile(host_outdir_tgt): - try: - os.remove(host_outdir_tgt) - except PermissionError: - pass + subprocess.run(["chmod", "777", host_outdir_tgt], check=True) + os.remove(host_outdir_tgt) elif os.path.isdir(host_outdir_tgt) and not vol.resolved.startswith("_:"): + subprocess.run(["chmod", "777", host_outdir_tgt], check=True) shutil.rmtree(host_outdir_tgt) if not vol.resolved.startswith("_:"): - try: - os.symlink(vol.resolved, host_outdir_tgt) - except FileExistsError: - pass - + os.symlink(vol.resolved, host_outdir_tgt) def neverquote(string: str, pos: int = 0, endpos: int = 0) -> Optional[Match[str]]: return None From 20e3eabcb3a595e8d1ae8e43d7ee53791cbbca54 Mon Sep 17 00:00:00 2001 From: Jake Fennick Date: Mon, 15 Aug 2022 23:43:11 -0600 Subject: [PATCH 2/7] use os.chmod and add write to existing mode --- cwltool/job.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/cwltool/job.py b/cwltool/job.py index 0decc2c06..eedbb1853 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -98,16 +98,25 @@ def relink_initialworkdir( # directory, so therefore ineligable for being an output file. # Thus, none of our business continue - host_outdir_tgt = os.path.join(host_outdir, vol.target[len(container_outdir) + 1 :]) + host_outdir_tgt = os.path.join( + host_outdir, vol.target[len(container_outdir) + 1 :] + ) + mode = ( + os.stat(host_outdir_tgt).st_mode + | stat.S_IWUSR + | stat.S_IWGRP + | stat.S_IWOTH + ) if os.path.islink(host_outdir_tgt) or os.path.isfile(host_outdir_tgt): - subprocess.run(["chmod", "777", host_outdir_tgt], check=True) + os.chmod(host_outdir_tgt, mode) os.remove(host_outdir_tgt) elif os.path.isdir(host_outdir_tgt) and not vol.resolved.startswith("_:"): - subprocess.run(["chmod", "777", host_outdir_tgt], check=True) + os.chmod(host_outdir_tgt, mode) shutil.rmtree(host_outdir_tgt) if not vol.resolved.startswith("_:"): os.symlink(vol.resolved, host_outdir_tgt) + def neverquote(string: str, pos: int = 0, endpos: int = 0) -> Optional[Match[str]]: return None From 08bb056b85c497f6e5425dded9e25c58d37d77d8 Mon Sep 17 00:00:00 2001 From: Jake Fennick Date: Tue, 16 Aug 2022 10:16:19 -0600 Subject: [PATCH 3/7] blasting CI --- cwltool/job.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/cwltool/job.py b/cwltool/job.py index eedbb1853..4db234177 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -107,14 +107,22 @@ def relink_initialworkdir( | stat.S_IWGRP | stat.S_IWOTH ) + mode = 0o664 # Doesn't work for my code + mode = 0o777 # works for my code if os.path.islink(host_outdir_tgt) or os.path.isfile(host_outdir_tgt): - os.chmod(host_outdir_tgt, mode) - os.remove(host_outdir_tgt) + try: + os.chmod(host_outdir_tgt, mode) + os.remove(host_outdir_tgt) + except PermissionError: + pass elif os.path.isdir(host_outdir_tgt) and not vol.resolved.startswith("_:"): os.chmod(host_outdir_tgt, mode) shutil.rmtree(host_outdir_tgt) if not vol.resolved.startswith("_:"): - os.symlink(vol.resolved, host_outdir_tgt) + try: + os.symlink(vol.resolved, host_outdir_tgt) + except FileExistsError: + pass def neverquote(string: str, pos: int = 0, endpos: int = 0) -> Optional[Match[str]]: From 806d6955f31f0c8577b25dd7c91378b4573b67ab Mon Sep 17 00:00:00 2001 From: Jake Fennick Date: Thu, 9 Mar 2023 10:27:23 -0700 Subject: [PATCH 4/7] fix pylint --- cwltool/docker.py | 4 +--- cwltool/job.py | 11 ++--------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/cwltool/docker.py b/cwltool/docker.py index ff463a853..62b83ca9a 100644 --- a/cwltool/docker.py +++ b/cwltool/docker.py @@ -236,9 +236,7 @@ def add_file_or_directory_volume( """Append volume a file/dir mapping to the runtime option list.""" if not volume.resolved.startswith("_:"): _check_docker_machine_path(volume.resolved) - self.append_volume( - runtime, volume.resolved, volume.target, writable=volume.staged - ) + self.append_volume(runtime, volume.resolved, volume.target, writable=volume.staged) def add_writable_file_volume( self, diff --git a/cwltool/job.py b/cwltool/job.py index 4db234177..2ac5bbc8f 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -98,15 +98,8 @@ def relink_initialworkdir( # directory, so therefore ineligable for being an output file. # Thus, none of our business continue - host_outdir_tgt = os.path.join( - host_outdir, vol.target[len(container_outdir) + 1 :] - ) - mode = ( - os.stat(host_outdir_tgt).st_mode - | stat.S_IWUSR - | stat.S_IWGRP - | stat.S_IWOTH - ) + host_outdir_tgt = os.path.join(host_outdir, vol.target[len(container_outdir) + 1 :]) + mode = os.stat(host_outdir_tgt).st_mode | stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH mode = 0o664 # Doesn't work for my code mode = 0o777 # works for my code if os.path.islink(host_outdir_tgt) or os.path.isfile(host_outdir_tgt): From f5a74129aea9581c15d3b87f78bf99a85c18a6c3 Mon Sep 17 00:00:00 2001 From: Jake Fennick Date: Thu, 9 Mar 2023 11:06:41 -0700 Subject: [PATCH 5/7] use ensure_writable --- cwltool/job.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cwltool/job.py b/cwltool/job.py index 2ac5bbc8f..6f11d048f 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -99,17 +99,14 @@ def relink_initialworkdir( # Thus, none of our business continue host_outdir_tgt = os.path.join(host_outdir, vol.target[len(container_outdir) + 1 :]) - mode = os.stat(host_outdir_tgt).st_mode | stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH - mode = 0o664 # Doesn't work for my code - mode = 0o777 # works for my code if os.path.islink(host_outdir_tgt) or os.path.isfile(host_outdir_tgt): try: - os.chmod(host_outdir_tgt, mode) + ensure_writable(host_outdir_tgt, include_root=True) os.remove(host_outdir_tgt) except PermissionError: pass elif os.path.isdir(host_outdir_tgt) and not vol.resolved.startswith("_:"): - os.chmod(host_outdir_tgt, mode) + ensure_writable(host_outdir_tgt, include_root=True) shutil.rmtree(host_outdir_tgt) if not vol.resolved.startswith("_:"): try: From 83b0882b61727afc1fbadaeeae145ba08da00d7a Mon Sep 17 00:00:00 2001 From: Jake Fennick Date: Thu, 9 Mar 2023 11:22:10 -0700 Subject: [PATCH 6/7] added PermissionError --- cwltool/job.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cwltool/job.py b/cwltool/job.py index 6f11d048f..8ddea7c1b 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -106,8 +106,11 @@ def relink_initialworkdir( except PermissionError: pass elif os.path.isdir(host_outdir_tgt) and not vol.resolved.startswith("_:"): - ensure_writable(host_outdir_tgt, include_root=True) - shutil.rmtree(host_outdir_tgt) + try: + ensure_writable(host_outdir_tgt, include_root=True) + shutil.rmtree(host_outdir_tgt) + except PermissionError: + pass if not vol.resolved.startswith("_:"): try: os.symlink(vol.resolved, host_outdir_tgt) From a6992968871ae98b274e563c95d632c05605c317 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 5 Dec 2023 13:08:51 +0100 Subject: [PATCH 7/7] try setting write bit separately from deletion --- cwltool/job.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cwltool/job.py b/cwltool/job.py index 8ddea7c1b..bc926d533 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -102,12 +102,18 @@ def relink_initialworkdir( if os.path.islink(host_outdir_tgt) or os.path.isfile(host_outdir_tgt): try: ensure_writable(host_outdir_tgt, include_root=True) + except PermissionError: + pass + try: os.remove(host_outdir_tgt) except PermissionError: pass elif os.path.isdir(host_outdir_tgt) and not vol.resolved.startswith("_:"): try: ensure_writable(host_outdir_tgt, include_root=True) + except PermissionError: + pass + try: shutil.rmtree(host_outdir_tgt) except PermissionError: pass