Allow copy specific files to docker compose (#8848)

This commit adds support for a `withCopyFilesInContainer` method on `ComposeContainer` and `DockerComposeContainer`. It allows to specify what files or directories should be copied, instead of just copying all files. If not used, the current behavior is preserved.

Fixes #8847

---------

Co-authored-by: Eddú Meléndez <eddu.melendez@gmail.com>

core/build.gradle

@@ -121,6 +121,7 @@ dependencies {
     testImplementation files('testlib/repo/fakejar/fakejar/0/fakejar-0.jar')
 
     testImplementation 'org.assertj:assertj-core:3.25.3'
+    testImplementation 'io.rest-assured:rest-assured:5.4.0'
 
     jarFileTestCompileOnly "org.projectlombok:lombok:${lombok.version}"
     jarFileTestAnnotationProcessor "org.projectlombok:lombok:${lombok.version}"

core/src/main/java/org/testcontainers/containers/ComposeContainer.java

@@ -67,6 +67,8 @@ public class ComposeContainer extends FailureDetectingExternalResource implement
 
     private String project;
 
+    private List<String> filesInDirectory = new ArrayList<>();
+
     public ComposeContainer(File... composeFiles) {
         this(Arrays.asList(composeFiles));
     }
@@ -134,7 +136,8 @@ public void start() {
                     this.options,
                     this.services,
                     this.scalingPreferences,
-                    this.env
+                    this.env,
+                    this.filesInDirectory
                 );
             this.composeDelegate.startAmbassadorContainer();
             this.composeDelegate.waitUntilServiceStarted(this.tailChildContainers);
@@ -165,7 +168,7 @@ public void stop() {
                 if (removeImages != null) {
                     cmd += " --rmi " + removeImages.dockerRemoveImagesType();
                 }
-                this.composeDelegate.runWithCompose(this.localCompose, cmd, this.env);
+                this.composeDelegate.runWithCompose(this.localCompose, cmd, this.env, this.filesInDirectory);
             } finally {
                 this.project = this.composeDelegate.randomProjectId();
             }
@@ -352,6 +355,11 @@ public ComposeContainer withStartupTimeout(Duration startupTimeout) {
         return this;
     }
 
+    public ComposeContainer withCopyFilesInContainer(String... fileCopyInclusions) {
+        this.filesInDirectory = Arrays.asList(fileCopyInclusions);
+        return this;
+    }
+
     public Optional<ContainerState> getContainerByServiceName(String serviceName) {
         return this.composeDelegate.getContainerByServiceName(serviceName);
     }

core/src/main/java/org/testcontainers/containers/ComposeDelegate.java

@@ -126,7 +126,8 @@ void createServices(
         final Set<String> options,
         final List<String> services,
         final Map<String, Integer> scalingPreferences,
-        Map<String, String> env
+        Map<String, String> env,
+        List<String> fileCopyInclusions
     ) {
         // services that have been explicitly requested to be started. If empty, all services should be started.
         final String serviceNameArgs = Stream
@@ -160,7 +161,7 @@ void createServices(
         }
 
         // Run the docker compose container, which starts up the services
-        runWithCompose(localCompose, command, env);
+        runWithCompose(localCompose, command, env, fileCopyInclusions);
     }
 
     private String getUpCommand(String options) {
@@ -237,18 +238,24 @@ private String getServiceNameFromContainer(com.github.dockerjava.api.model.Conta
     }
 
     public void runWithCompose(boolean localCompose, String cmd) {
-        runWithCompose(localCompose, cmd, Collections.emptyMap());
+        runWithCompose(localCompose, cmd, Collections.emptyMap(), Collections.emptyList());
     }
 
-    public void runWithCompose(boolean localCompose, String cmd, Map<String, String> env) {
+    public void runWithCompose(
+        boolean localCompose,
+        String cmd,
+        Map<String, String> env,
+        List<String> fileCopyInclusions
+    ) {
         Preconditions.checkNotNull(composeFiles);
         Preconditions.checkArgument(!composeFiles.isEmpty(), "No docker compose file have been provided");
 
         final DockerCompose dockerCompose;
         if (localCompose) {
             dockerCompose = new LocalDockerCompose(this.executable, composeFiles, project);
         } else {
-            dockerCompose = new ContainerisedDockerCompose(this.defaultImageName, composeFiles, project);
+            dockerCompose =
+                new ContainerisedDockerCompose(this.defaultImageName, composeFiles, project, fileCopyInclusions);
         }
 
         dockerCompose.withCommand(cmd).withEnv(env).invoke();

core/src/main/java/org/testcontainers/containers/ContainerisedDockerCompose.java

@@ -24,7 +24,12 @@ class ContainerisedDockerCompose extends GenericContainer<ContainerisedDockerCom
 
     public static final char UNIX_PATH_SEPARATOR = ':';
 
-    public ContainerisedDockerCompose(DockerImageName dockerImageName, List<File> composeFiles, String identifier) {
+    public ContainerisedDockerCompose(
+        DockerImageName dockerImageName,
+        List<File> composeFiles,
+        String identifier,
+        List<String> fileCopyInclusions
+    ) {
         super(dockerImageName);
         addEnv(ENV_PROJECT_NAME, identifier);
 
@@ -43,7 +48,22 @@ public ContainerisedDockerCompose(DockerImageName dockerImageName, List<File> co
         final String composeFileEnvVariableValue = Joiner.on(UNIX_PATH_SEPARATOR).join(absoluteDockerComposeFiles); // we always need the UNIX path separator
         logger().debug("Set env COMPOSE_FILE={}", composeFileEnvVariableValue);
         addEnv(ENV_COMPOSE_FILE, composeFileEnvVariableValue);
-        withCopyFileToContainer(MountableFile.forHostPath(pwd), containerPwd);
+        if (fileCopyInclusions.isEmpty()) {
+            logger().info("Copying all files in {} into the container", pwd);
+            withCopyFileToContainer(MountableFile.forHostPath(pwd), containerPwd);
+        } else {
+            // Always copy the compose file itself
+            logger().info("Copying docker compose file: {}", dockerComposeBaseFile.getAbsolutePath());
+            withCopyFileToContainer(
+                MountableFile.forHostPath(dockerComposeBaseFile.getAbsolutePath()),
+                convertToUnixFilesystemPath(dockerComposeBaseFile.getAbsolutePath())
+            );
+            for (String pathToCopy : fileCopyInclusions) {
+                String hostPath = pwd + "/" + pathToCopy;
+                logger().info("Copying inclusion file: {}", hostPath);
+                withCopyFileToContainer(MountableFile.forHostPath(hostPath), convertToUnixFilesystemPath(hostPath));
+            }
+        }
 
         // Ensure that compose can access docker. Since the container is assumed to be running on the same machine
         //  as the docker daemon, just mapping the docker control socket is OK.

core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java

@@ -68,6 +68,8 @@ public class DockerComposeContainer<SELF extends DockerComposeContainer<SELF>>
 
     private String project;
 
+    private List<String> filesInDirectory = new ArrayList<>();
+
     @Deprecated
     public DockerComposeContainer(File composeFile, String identifier) {
         this(identifier, composeFile);
@@ -140,7 +142,8 @@ public void start() {
                     this.options,
                     this.services,
                     this.scalingPreferences,
-                    this.env
+                    this.env,
+                    this.filesInDirectory
                 );
             this.composeDelegate.startAmbassadorContainer();
             this.composeDelegate.waitUntilServiceStarted(this.tailChildContainers);
@@ -172,7 +175,7 @@ public void stop() {
                 if (removeImages != null) {
                     cmd += " --rmi " + removeImages.dockerRemoveImagesType();
                 }
-                this.composeDelegate.runWithCompose(this.localCompose, cmd, this.env);
+                this.composeDelegate.runWithCompose(this.localCompose, cmd, this.env, this.filesInDirectory);
             } finally {
                 this.project = this.composeDelegate.randomProjectId();
             }
@@ -355,6 +358,11 @@ public SELF withStartupTimeout(Duration startupTimeout) {
         return self();
     }
 
+    public SELF withCopyFilesInContainer(String... fileCopyInclusions) {
+        this.filesInDirectory = Arrays.asList(fileCopyInclusions);
+        return self();
+    }
+
     public Optional<ContainerState> getContainerByServiceName(String serviceName) {
         return this.composeDelegate.getContainerByServiceName(serviceName);
     }

core/src/test/java/org/testcontainers/junit/ComposeContainerWithCopyFilesTest.java

@@ -0,0 +1,87 @@
+package org.testcontainers.junit;
+
+import io.restassured.RestAssured;
+import org.junit.Test;
+import org.testcontainers.containers.ComposeContainer;
+import org.testcontainers.containers.ContainerLaunchException;
+
+import java.io.File;
+import java.io.IOException;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
+
+public class ComposeContainerWithCopyFilesTest {
+
+    @Test
+    public void testShouldCopyAllFilesByDefault() throws IOException {
+        try (
+            ComposeContainer environment = new ComposeContainer(
+                new File("src/test/resources/compose-file-copy-inclusions/compose.yml")
+            )
+                .withExposedService("app", 8080)
+        ) {
+            environment.start();
+
+            String response = readStringFromURL(environment);
+            assertThat(response).isEqualTo("MY_ENV_VARIABLE: override");
+        }
+    }
+
+    @Test
+    public void testWithFileCopyInclusionUsingFilePath() throws IOException {
+        try (
+            ComposeContainer environment = new ComposeContainer(
+                new File("src/test/resources/compose-file-copy-inclusions/compose-root-only.yml")
+            )
+                .withExposedService("app", 8080)
+                .withCopyFilesInContainer("Dockerfile", "EnvVariableRestEndpoint.java", ".env")
+        ) {
+            environment.start();
+
+            String response = readStringFromURL(environment);
+
+            // The `test/.env` file is not copied, now so we get the original value
+            assertThat(response).isEqualTo("MY_ENV_VARIABLE: original");
+        }
+    }
+
+    @Test
+    public void testWithFileCopyInclusionUsingDirectoryPath() throws IOException {
+        try (
+            ComposeContainer environment = new ComposeContainer(
+                new File("src/test/resources/compose-file-copy-inclusions/compose-test-only.yml")
+            )
+                .withExposedService("app", 8080)
+                .withCopyFilesInContainer("Dockerfile", "EnvVariableRestEndpoint.java", "test")
+        ) {
+            environment.start();
+
+            String response = readStringFromURL(environment);
+            // The test directory (with its contents) is copied, so we get the override
+            assertThat(response).isEqualTo("MY_ENV_VARIABLE: override");
+        }
+    }
+
+    @Test
+    public void testShouldNotBeAbleToStartIfNeededEnvFileIsNotCopied() {
+        try (
+            ComposeContainer environment = new ComposeContainer(
+                new File("src/test/resources/compose-file-copy-inclusions/compose-test-only.yml")
+            )
+                .withExposedService("app", 8080)
+                .withCopyFilesInContainer("Dockerfile", "EnvVariableRestEndpoint.java")
+        ) {
+            assertThatExceptionOfType(ContainerLaunchException.class)
+                .isThrownBy(environment::start)
+                .withMessageContaining("Container startup failed for image docker");
+        }
+    }
+
+    private static String readStringFromURL(ComposeContainer container) throws IOException {
+        Integer servicePort = container.getServicePort("app-1", 8080);
+        String serviceHost = container.getServiceHost("app-1", 8080);
+        String requestURL = "http://" + serviceHost + ":" + servicePort + "/env";
+        return RestAssured.get(requestURL).thenReturn().body().asString();
+    }
+}

core/src/test/java/org/testcontainers/junit/DockerComposeContainerWithCopyFilesTest.java

@@ -0,0 +1,70 @@
+package org.testcontainers.junit;
+
+import io.restassured.RestAssured;
+import org.junit.Test;
+import org.testcontainers.containers.DockerComposeContainer;
+
+import java.io.File;
+import java.io.IOException;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class DockerComposeContainerWithCopyFilesTest {
+
+    @Test
+    public void testShouldCopyAllFilesByDefault() throws IOException {
+        try (
+            DockerComposeContainer environment = new DockerComposeContainer(
+                new File("src/test/resources/compose-file-copy-inclusions/compose.yml")
+            )
+                .withExposedService("app", 8080)
+        ) {
+            environment.start();
+
+            String response = readStringFromURL(environment);
+            assertThat(response).isEqualTo("MY_ENV_VARIABLE: override");
+        }
+    }
+
+    @Test
+    public void testWithFileCopyInclusionUsingFilePath() throws IOException {
+        try (
+            DockerComposeContainer environment = new DockerComposeContainer(
+                new File("src/test/resources/compose-file-copy-inclusions/compose-root-only.yml")
+            )
+                .withExposedService("app", 8080)
+                .withCopyFilesInContainer("Dockerfile", "EnvVariableRestEndpoint.java", ".env")
+        ) {
+            environment.start();
+
+            String response = readStringFromURL(environment);
+
+            // The `test/.env` file is not copied, now so we get the original value
+            assertThat(response).isEqualTo("MY_ENV_VARIABLE: original");
+        }
+    }
+
+    @Test
+    public void testWithFileCopyInclusionUsingDirectoryPath() throws IOException {
+        try (
+            DockerComposeContainer environment = new DockerComposeContainer(
+                new File("src/test/resources/compose-file-copy-inclusions/compose-test-only.yml")
+            )
+                .withExposedService("app", 8080)
+                .withCopyFilesInContainer("Dockerfile", "EnvVariableRestEndpoint.java", "test")
+        ) {
+            environment.start();
+
+            String response = readStringFromURL(environment);
+            // The test directory (with its contents) is copied, so we get the override
+            assertThat(response).isEqualTo("MY_ENV_VARIABLE: override");
+        }
+    }
+
+    private static String readStringFromURL(DockerComposeContainer container) throws IOException {
+        Integer servicePort = container.getServicePort("app_1", 8080);
+        String serviceHost = container.getServiceHost("app_1", 8080);
+        String requestURL = "http://" + serviceHost + ":" + servicePort + "/env";
+        return RestAssured.get(requestURL).thenReturn().body().asString();
+    }
+}

core/src/test/resources/compose-file-copy-inclusions/.env

@@ -0,0 +1 @@
+MY_ENV_VARIABLE=original

core/src/test/resources/compose-file-copy-inclusions/Dockerfile

@@ -0,0 +1,9 @@
+FROM jbangdev/jbang-action
+
+WORKDIR /app
+COPY EnvVariableRestEndpoint.java .
+
+RUN jbang export portable --force EnvVariableRestEndpoint.java
+
+EXPOSE 8080
+CMD ["./EnvVariableRestEndpoint.java"]

core/src/test/resources/compose-file-copy-inclusions/EnvVariableRestEndpoint.java

@@ -0,0 +1,45 @@
+///usr/bin/env jbang "$0" "$@" ; exit $?
+
+import com.sun.net.httpserver.HttpServer;
+import com.sun.net.httpserver.HttpHandler;
+import com.sun.net.httpserver.HttpExchange;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+
+public class EnvVariableRestEndpoint {
+    private static final String ENV_VARIABLE_NAME = "MY_ENV_VARIABLE";
+    private static final int PORT = 8080;
+
+    public static void main(String[] args) throws IOException {
+        HttpServer server = HttpServer.create(new InetSocketAddress(PORT), 0);
+        server.createContext("/env", new EnvVariableHandler());
+        server.setExecutor(null);
+        server.start();
+        System.out.println("Server started on port " + PORT);
+    }
+
+    static class EnvVariableHandler implements HttpHandler {
+        @Override
+        public void handle(HttpExchange exchange) throws IOException {
+            if ("GET".equals(exchange.getRequestMethod())) {
+                String envValue = System.getenv(ENV_VARIABLE_NAME);
+                String response = envValue != null
+                    ? ENV_VARIABLE_NAME + ": " + envValue
+                    : "Environment variable " + ENV_VARIABLE_NAME + " not found";
+
+                exchange.sendResponseHeaders(200, response.length());
+                try (OutputStream os = exchange.getResponseBody()) {
+                    os.write(response.getBytes());
+                }
+            } else {
+                String response = "Method not allowed";
+                exchange.sendResponseHeaders(405, response.length());
+                try (OutputStream os = exchange.getResponseBody()) {
+                    os.write(response.getBytes());
+                }
+            }
+        }
+    }
+}

core/src/test/resources/compose-file-copy-inclusions/compose-root-only.yml

@@ -0,0 +1,7 @@
+services:
+  app:
+    build: .
+    ports:
+      - "8080:8080"
+    env_file:
+      - '.env'

core/src/test/resources/compose-file-copy-inclusions/compose-test-only.yml

@@ -0,0 +1,7 @@
+services:
+  app:
+    build: .
+    ports:
+      - "8080:8080"
+    env_file:
+      - './test/.env'

core/src/test/resources/compose-file-copy-inclusions/compose.yml

@@ -0,0 +1,8 @@
+services:
+  app:
+    build: .
+    ports:
+      - "8080:8080"
+    env_file:
+      - '.env'
+      - './test/.env'

core/src/test/resources/compose-file-copy-inclusions/test/.env

@@ -0,0 +1 @@
+MY_ENV_VARIABLE=override

docs/modules/docker_compose.md

@@ -135,6 +135,26 @@ public static ComposeContainer environment =
 !!! note
     Make sure the service name use a `-` instead of `_` as separator using `ComposeContainer`.
 
+## Build working directory
+
+You can select what files should be copied only via `withCopyFilesInContainer`:
+
+```java
+public static ComposeContainer environment =
+    new ComposeContainer(new File("compose.yml"))
+        .withCopyFilesInContainer(".env");
+```
+
+In this example, only `compose.yml` and `.env` are copied over into the container that will run the Docker Compose file.  
+By default, all files in the same directory as the compose file are copied over.
+
+This can be used with `DockerComposeContainer` and `ComposeContainer`.
+You can use file and directory references.
+They are always resolved relative to the directory where the compose file resides.
+
+!!! note
+    This only work with containarized Compose, not with `Local Compose` mode.
+
 ## Using private repositories in Docker compose
 When Docker Compose is used in container mode (not local), it's needs to be made aware of Docker settings for private repositories. 
 By default, those setting are located in `$HOME/.docker/config.json`.