Support adding layers and context to container build

build.gradle

@@ -42,7 +42,7 @@ dependencies {
     implementation 'io.seqera:lib-random:1.0.0'
     implementation 'io.seqera:lib-activator:1.0.0'
     implementation 'io.seqera:wave-api:0.16.0'
-    implementation 'io.seqera:wave-utils:1.0.0'
+    implementation 'io.seqera:wave-utils:1.0.1-A3'
     implementation 'io.seqera:lib-crypto:1.0.0'
     implementation 'io.seqera:jedis-lock:1.0.0'
     implementation 'io.seqera:lib-data-queue-redis:1.1.0'

src/main/groovy/io/seqera/wave/controller/ContainerController.groovy

@@ -254,7 +254,7 @@ class ContainerController {
 
         if( v2 && req.packages ) {
             // generate the container file required to assemble the container
-            final generated = containerFileFromPackages(req.packages, req.formatSingularity())
+            final generated = containerFileFromPackages(req.packages, req.formatSingularity(), req.containerConfig?.layers)
             req = req.copyWith(containerFile: generated.bytes.encodeBase64().toString())
         }
         // make sure container platform is defined 

src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy

@@ -18,6 +18,8 @@
 
 package io.seqera.wave.service.builder
 
+import java.nio.file.Path
+
 import groovy.transform.CompileStatic
 import io.seqera.wave.configuration.BuildConfig
 import jakarta.inject.Inject

src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy

@@ -67,7 +67,11 @@ class DockerBuildStrategy extends BuildStrategy {
                     cmdToStr(buildCmd).bytes,
                     CREATE, WRITE, TRUNCATE_EXISTING)
         }
-
+
+        if( req.buildContext ) {
+            runExtractContext(req.workDir)
+        }
+
         final process = new ProcessBuilder()
             .command(buildCmd)
             .directory(req.workDir.toFile())
@@ -95,6 +99,17 @@ class DockerBuildStrategy extends BuildStrategy {
         return dockerCmd + launchCmd(req)
     }
 
+    protected static void runExtractContext(Path workDir) {
+        log.debug("Extracting context archive: $workDir/context/content")
+        def tarCmd = "tar -xzvf $workDir/compressedcontext -C $workDir/context".toString()
+        log.debug("Context extract command: $tarCmd")
+        def process = tarCmd.execute()
+        process.waitFor()
+        if (process.exitValue() != 0) {
+            throw new RuntimeException("Failed to extract context")
+        }
+    }
+
     protected List<String> cmdForBuildkit(String name, Path workDir, Path credsFile, ContainerPlatform platform ) {
         //checkout the documentation here to know more about these options https://github.com/moby/buildkit/blob/master/docs/rootless.md#docker
         final wrapper = ['docker',

src/main/groovy/io/seqera/wave/service/builder/FreezeServiceImpl.groovy

@@ -30,6 +30,7 @@ import jakarta.inject.Inject
 import jakarta.inject.Singleton
 import static io.seqera.wave.util.RegHelper.layerDir
 import static io.seqera.wave.util.RegHelper.layerName
+import static io.seqera.wave.util.RegHelper.layerMountDir
 /**
  * Implements helper methods to handle container build context
  *
@@ -131,7 +132,7 @@ class FreezeServiceImpl implements FreezeService {
         if( layers ) {
             result += '%files\n'
             for(int i=0; i<layers.size(); i++) {
-                result += "  {{wave_context_dir}}/${layerDir(layers[i])}/* /\n"
+                result += "  {{wave_context_dir}}/${layerName(layers[i])} ${layerMountDir(layers[i])}\n"
             }
         }
 

src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy

@@ -74,7 +74,7 @@ class KubeBuildStrategy extends BuildStrategy {
             final buildCmd = launchCmd(req)
             final timeout = req.maxDuration ?: buildConfig.defaultTimeout
             final selector= getSelectorLabel(req.platform, nodeSelectorMap)
-            k8sService.launchBuildJob(jobName, buildImage, buildCmd, req.workDir, configFile, timeout, selector)
+            k8sService.launchBuildJob(jobName, buildImage, buildCmd, req, configFile, timeout, selector)
         }
         catch (ApiException e) {
             throw new BadRequestException("Unexpected build failure - ${e.responseBody}", e)

src/main/groovy/io/seqera/wave/service/builder/impl/ContainerBuildServiceImpl.groovy

@@ -72,8 +72,6 @@ import static io.seqera.wave.util.RegHelper.layerName
 import static java.nio.file.StandardOpenOption.CREATE
 import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING
 import static java.nio.file.StandardOpenOption.WRITE
-import static java.nio.file.attribute.PosixFilePermission.OWNER_READ
-import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE
 /**
  * Implements container build service
  *
@@ -185,7 +183,7 @@ class ContainerBuildServiceImpl implements ContainerBuildService, JobHandler<Bui
             Files.write(containerFile, containerFile0(req, context).bytes, CREATE, WRITE, TRUNCATE_EXISTING)
             // save build context
             if( req.buildContext ) {
-                saveBuildContext(req.buildContext, context, req.identity)
+                saveBuildContext(req.buildContext, req.workDir, req.identity)
             }
             // save the conda file
             if( req.condaFile ) {
@@ -265,59 +263,34 @@ class ContainerBuildServiceImpl implements ContainerBuildService, JobHandler<Bui
         throw new IllegalStateException("Unable to determine build status for '$request.targetImage'")
     }
 
-    protected void saveLayersToContext(BuildRequest req, Path contextDir) {
-        if(req.formatDocker()) {
-            saveLayersToDockerContext0(req, contextDir)
-        }
-        else if(req.formatSingularity()) {
-            saveLayersToSingularityContext0(req, contextDir)
-        }
-        else
-            throw new IllegalArgumentException("Unknown container format: $req.format")
-    }
-
-    protected void saveLayersToDockerContext0(BuildRequest request, Path contextDir) {
+    protected void saveLayersToContext(BuildRequest request, Path contextDir) {
         final layers = request.containerConfig.layers
         for(int i=0; i<layers.size(); i++) {
             final it = layers[i]
             final target = contextDir.resolve(layerName(it))
-            final retryable = retry0("Unable to copy '${it.location}' to docker context '${contextDir}'")
-            // copy the layer to the build context
-            retryable.apply(()-> {
-                try (InputStream stream = streamService.stream(it.location, request.identity)) {
-                    Files.copy(stream, target, StandardCopyOption.REPLACE_EXISTING)
-                }
-                return
-            })
-        }
-    }
-
-    protected void saveLayersToSingularityContext0(BuildRequest request, Path contextDir) {
-        final layers = request.containerConfig.layers
-        for(int i=0; i<layers.size(); i++) {
-            final it = layers[i]
-            final target = contextDir.resolve(layerDir(it))
             try { Files.createDirectory(target) }
             catch (FileAlreadyExistsException e) { /* ignore */ }
-            // retry strategy
-            final retryable = retry0("Unable to copy '${it.location} to singularity context '${contextDir}'")
+            final retryable = retry0("Unable to copy '${it.location}' to context '${contextDir}'")
             // copy the layer to the build context
             retryable.apply(()-> {
                 try (InputStream stream = streamService.stream(it.location, request.identity)) {
-                    TarUtils.untarGzip(stream, target)
+                    Files.copy(stream, target, StandardCopyOption.REPLACE_EXISTING)
                 }
                 return
             })
         }
     }
 
-    protected void saveBuildContext(BuildContext buildContext, Path contextDir, PlatformId identity) {
+    protected void saveBuildContext(BuildContext buildContext, Path workDir, PlatformId identity) {
         // retry strategy
-        final retryable = retry0("Unable to copy '${buildContext.location} to build context '${contextDir}'")
+        final retryable = retry0("Unable to copy '${buildContext.location} to work directory '${workDir}'")
+        final target = workDir.resolve("context.tar.gz")
+        try { Files.createDirectory(target) }
+        catch (FileAlreadyExistsException e) { /* ignore */ }
         // copy the layer to the build context
         retryable.apply(()-> {
             try (InputStream stream = streamService.stream(buildContext.location, identity)) {
-                TarUtils.untarGzip(stream, contextDir)
+                Files.copy(stream, target, StandardCopyOption.REPLACE_EXISTING)
             }
             return
         })

src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy

@@ -26,6 +26,8 @@ import io.kubernetes.client.openapi.models.V1Pod
 import io.seqera.wave.configuration.BlobCacheConfig
 import io.seqera.wave.configuration.MirrorConfig
 import io.seqera.wave.configuration.ScanConfig
+import io.seqera.wave.service.builder.BuildRequest
+
 /**
  * Defines Kubernetes operations
  *
@@ -47,7 +49,7 @@ interface K8sService {
 
     V1Job launchTransferJob(String name, String containerImage, List<String> args, BlobCacheConfig blobConfig)
 
-    V1Job launchBuildJob(String name, String containerImage, List<String> args, Path workDir, Path creds, Duration timeout, Map<String,String> nodeSelector)
+    V1Job launchBuildJob(String name, String containerImage, List<String> args, BuildRequest request, Path creds, Duration timeout, Map<String,String> nodeSelector)
 
     V1Job launchScanJob(String name, String containerImage, List<String> args, Path workDir, Path creds, ScanConfig scanConfig)
 

src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy

@@ -58,6 +58,7 @@ import io.seqera.wave.configuration.BuildConfig
 import io.seqera.wave.configuration.MirrorConfig
 import io.seqera.wave.configuration.ScanConfig
 import io.seqera.wave.core.ContainerPlatform
+import io.seqera.wave.service.builder.BuildRequest
 import io.seqera.wave.service.scan.Trivy
 import jakarta.inject.Inject
 import jakarta.inject.Singleton
@@ -499,16 +500,17 @@ class K8sServiceImpl implements K8sService {
      */
     @Override
     @TraceElapsedTime(thresholdMillis = '${wave.trace.k8s.threshold:200}')
-    V1Job launchBuildJob(String name, String containerImage, List<String> args, Path workDir, Path creds, Duration timeout, Map<String,String> nodeSelector) {
-        final spec = buildJobSpec(name, containerImage, args, workDir, creds, timeout, nodeSelector)
+    V1Job launchBuildJob(String name, String containerImage, List<String> args, BuildRequest request, Path creds, Duration timeout, Map<String,String> nodeSelector) {
+        final spec = buildJobSpec(name, containerImage, args, request, creds, timeout, nodeSelector)
         return k8sClient
                 .batchV1Api()
                 .createNamespacedJob(namespace, spec)
                 .execute()
     }
 
-    V1Job buildJobSpec(String name, String containerImage, List<String> args, Path workDir, Path credsFile, Duration timeout, Map<String,String> nodeSelector) {
+    V1Job buildJobSpec(String name, String containerImage, List<String> args, BuildRequest request, Path credsFile, Duration timeout, Map<String,String> nodeSelector) {
 
+        final workDir = request.workDir
         // dirty dependency to avoid introducing another parameter
         final singularity = containerImage.contains('singularity')
 
@@ -524,6 +526,7 @@ class K8sServiceImpl implements K8sService {
         if( credsFile ){
             if( !singularity ) {
                 mounts.add(0, mountHostPath(credsFile, storageMountPath, '/home/user/.docker/config.json'))
+                initMounts.add(0,mountBuildStorage(workDir, storageMountPath, false))
             }
             else {
                 //emptydir volume for singularity
@@ -618,6 +621,20 @@ class K8sServiceImpl implements K8sService {
                             .withType("Unconfined")
                         .endAppArmorProfile()
                     .endSecurityContext()
+
+            if (request.buildContext) {
+                //init container to untar context
+                spec.withInitContainers(new V1ContainerBuilder()
+                        .withName("untar-context")
+                        .withImage("busybox")
+                        .withCommand(
+                                "sh",
+                                "-c",
+                                "[ -d ${request.workDir}/context ] || mkdir -p ${request.workDir}/context && tar -xzf ${request.workDir}/context.tar.gz -C ${request.workDir}/context")
+                        .withVolumeMounts(initMounts)
+                        .build()
+                )
+            }
         }
 
         // spec section

src/main/groovy/io/seqera/wave/util/ContainerHelper.groovy

@@ -23,6 +23,7 @@ import groovy.util.logging.Slf4j
 import io.micronaut.core.annotation.Nullable
 import io.seqera.wave.api.BuildContext
 import io.seqera.wave.api.ContainerConfig
+import io.seqera.wave.api.ContainerLayer
 import io.seqera.wave.api.ImageNameStrategy
 import io.seqera.wave.api.PackagesSpec
 import io.seqera.wave.api.SubmitContainerTokenRequest
@@ -41,6 +42,9 @@ import static io.seqera.wave.util.DockerHelper.condaFileToSingularityFile
 import static io.seqera.wave.util.DockerHelper.condaPackagesToCondaYaml
 import static io.seqera.wave.util.DockerHelper.condaPackagesToDockerFile
 import static io.seqera.wave.util.DockerHelper.condaPackagesToSingularityFile
+import static io.seqera.wave.util.RegHelper.layerMountDir
+import static io.seqera.wave.util.RegHelper.layerName
+
 /**
  * Container helper methods
  *
@@ -60,19 +64,27 @@ class ContainerHelper {
      * @return
      *      The corresponding Containerfile
      */
-    static String containerFileFromPackages(PackagesSpec spec, boolean formatSingularity) {
+    static String containerFileFromPackages(PackagesSpec spec, boolean formatSingularity, List<ContainerLayer> layers) {
         if( spec.type == PackagesSpec.Type.CONDA ) {
             final lockFile = condaLockFile(spec.entries)
             if( !spec.condaOpts )
                 spec.condaOpts = new CondaOpts()
+            List<String> unTarLayerCmd = null
+            if( formatSingularity && layers ) {
+                unTarLayerCmd = new ArrayList<>(layers.size())
+                for (def layer : layers) {
+                    def layerMountDir = layerMountDir(layer)
+                    unTarLayerCmd.add("cd / && tar -xzf $layerMountDir && rm $layerMountDir".toString())
+                }
+            }
             def result
             if ( lockFile ) {
                 result = formatSingularity
-                        ? condaPackagesToSingularityFile(lockFile, spec.channels, spec.condaOpts)
+                        ? condaPackagesToSingularityFile(lockFile, spec.channels, spec.condaOpts, unTarLayerCmd)
                         : condaPackagesToDockerFile(lockFile, spec.channels, spec.condaOpts)
             } else {
                 result = formatSingularity
-                        ? condaFileToSingularityFile(spec.condaOpts)
+                        ? condaFileToSingularityFile(spec.condaOpts, unTarLayerCmd)
                         : condaFileToDockerFile(spec.condaOpts)
             }
             return result

src/main/groovy/io/seqera/wave/util/RegHelper.groovy

@@ -236,6 +236,10 @@ class RegHelper {
         return layerName(layer).replace(/.tar.gz/,'')
     }
 
+    static String layerMountDir(ContainerLayer layer) {
+        return "/opt/layers/${layerName(layer)}"
+    }
+
     static void closeResponse(HttpResponse<?> response) {
         log.trace "Closing HttpClient response: $response"
         try {

src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy

@@ -33,7 +33,6 @@ import io.micronaut.context.annotation.Primary
 import io.micronaut.context.annotation.Requires
 import io.micronaut.context.event.ApplicationEventPublisher
 import io.micronaut.test.extensions.spock.annotation.MicronautTest
-import io.seqera.wave.api.BuildContext
 import io.seqera.wave.api.ContainerConfig
 import io.seqera.wave.api.ContainerLayer
 import io.seqera.wave.auth.RegistryCredentialsProvider
@@ -228,31 +227,6 @@ class ContainerBuildServiceTest extends Specification {
 
     }
 
-    def 'should untar build context' () {
-        given:
-        def folder = Files.createTempDirectory('test')
-        def source = folder.resolve('source')
-        def target = folder.resolve('target')
-        Files.createDirectory(source)
-        Files.createDirectory(target)
-        and:
-        source.resolve('foo.txt').text  = 'Foo'
-        source.resolve('bar.txt').text  = 'Bar'
-        and:
-        def layer = new Packer().layer(source)
-        def context = BuildContext.of(layer)
-
-        when:
-        service.saveBuildContext(context, target, Mock(PlatformId))
-        then:
-        target.resolve('foo.txt').text == 'Foo'
-        target.resolve('bar.txt').text == 'Bar'
-
-        cleanup:
-        folder?.deleteDir()
-    }
-
-
     def 'should save layers to context dir' () {
         given:
         def folder = Files.createTempDirectory('test')

src/test/groovy/io/seqera/wave/service/builder/FreezeServiceImplTest.groovy

@@ -127,8 +127,8 @@ class FreezeServiceImplTest extends Specification  {
             BootStrap: docker
             From: ubuntu:latest
             %files
-              {{wave_context_dir}}/layer-digest1/* /
-              {{wave_context_dir}}/layer-digest2/* /
+              {{wave_context_dir}}/layer-digest1.tar.gz /opt/layers/layer-digest1.tar.gz
+              {{wave_context_dir}}/layer-digest2.tar.gz /opt/layers/layer-digest2.tar.gz
             %environment
               export FOO=1 BAR=2
             %runscript