Merge pull request #33 from avast/WaitingForStartingContainers

Check health state of containers Closes #31

Author: jakubjanecek

README.md

@@ -3,7 +3,7 @@
 
 Simplifies usage of [Docker Compose](https://www.docker.com/docker-compose) for local development and integration testing in [Gradle](https://gradle.org/) environment.
 
-`composeUp` task starts the application and waits till all exposed TCP ports are open (so till the application is ready). It reads assigned host and ports of particular containers and stores them into `dockerCompose.servicesInfos` property.
+`composeUp` task starts the application and waits till all containers become [healthy](https://docs.docker.com/engine/reference/builder/#/healthcheck) and all exposed TCP ports are open (so till the application is ready). It reads assigned host and ports of particular containers and stores them into `dockerCompose.servicesInfos` property.
 
 `composeDown` task stops the application and removes the containers.
 
@@ -14,8 +14,8 @@ Simplifies usage of [Docker Compose](https://www.docker.com/docker-compose) for
 ## Why this plugin?
 You could easily ensure that `docker-compose up` is called before your tests but there are few gotchas that this plugin solves:
 
-1. If you execute `docker-compose up -d` (_detached_) then this command returns immediately and your application is probably not able to serve requests at this time. This plugin waits till all exported TCP ports of all services are open.
-  - If waiting for open TCP ports timeouts (default 15 minutes) then it prints log of related service. 
+1. If you execute `docker-compose up -d` (_detached_) then this command returns immediately and your application is probably not able to serve requests at this time. This plugin waits till all containers become [healthy](https://docs.docker.com/engine/reference/builder/#/healthcheck) and all exported TCP ports of all services are open.
+  - If waiting for healthy state or open TCP ports timeouts (default is 15 minutes) then it prints log of related service. 
 2. It's recommended not to assign fixed values of exposed ports in `docker-compose.yml` (i.e. `8888:80`) because it can cause ports collision on integration servers. If you don't assign a fixed value for exposed port (use just `80`) then the port is exposed as a random free port. This plugin reads assigned ports (and even IP addresses of containers) and stores them into `dockerCompose.servicesInfo` map.
 
 # Usage
@@ -63,7 +63,7 @@ test.doFirst {
 # Tips
 * You can call `dockerCompose.isRequiredBy(anyTask)` for any task, for example for your custom `integrationTest` task.
 * If some Dockerfile needs an artifact generated by Gradle then you can declare this dependency in a standard way, like `composeUp.dependsOn project(':my-app').distTar`
-* All properties in `dockerCompose` have meaningfull default values so you don't have to touch it. If you are interested then you can look at [ComposeExtension.groovy](/src/main/groovy/com/avast/gradle/dockercompose/ComposeExtension.groovy) for reference.
+* All properties in `dockerCompose` have meaningful default values so you don't have to touch it. If you are interested then you can look at [ComposeExtension.groovy](/src/main/groovy/com/avast/gradle/dockercompose/ComposeExtension.groovy) for reference.
 * `dockerCompose.servicesInfos` contains information about running containers so you must access this property after `composeUp` task is finished. So `doFirst` of your test task is perfect place where to access it.
 * Plugin honours a `docker-compose.override.yml` file, but only when no files are specified with `useComposeFiles` (conform command-line behavior).
 * Check [ServiceInfo.groovy](/src/main/groovy/com/avast/gradle/dockercompose/ServiceInfo.groovy) to see what you can know about running containers.

src/main/groovy/com/avast/gradle/dockercompose/ComposeExtension.groovy

@@ -19,6 +19,8 @@ class ComposeExtension {
     boolean waitForTcpPorts = true
     Duration waitAfterTcpProbeFailure = Duration.ofSeconds(1)
     Duration waitForTcpPortsTimeout = Duration.ofMinutes(15)
+    Duration waitAfterHealthyStateProbeFailure = Duration.ofSeconds(5)
+    Duration waitForHealthyStateTimeout = Duration.ofMinutes(15)
     List<String> useComposeFiles = []
 
     boolean stopContainers = true

src/main/groovy/com/avast/gradle/dockercompose/ServiceInfo.groovy

@@ -11,6 +11,7 @@ class ServiceInfo {
     String containerHostname
     /* Docker inspection */
     Map<String, Object> inspection
+    String getContainerId() { inspection.Id }
 
     String getHost() { serviceHost.host }
     Map<Integer, Integer> getPorts() { tcpPorts }

src/main/groovy/com/avast/gradle/dockercompose/tasks/ComposeUp.groovy

@@ -41,6 +41,7 @@ class ComposeUp extends DefaultTask {
         }
         try {
             servicesInfos = loadServicesInfo().collectEntries { [(it.name): (it)] }
+            waitForHealthyContainers(servicesInfos.values())
             if (extension.waitForTcpPorts) {
                 waitForOpenTcpPorts(servicesInfos.values())
             }
@@ -162,6 +163,30 @@ class ComposeUp extends DefaultTask {
         ports
     }
 
+    void waitForHealthyContainers(Iterable<ServiceInfo> servicesInfos) {
+        servicesInfos.forEach { service ->
+            def start = Instant.now()
+            while (true) {
+                Map<String, Object> inspectionState = getDockerInspection(service.getContainerId()).State
+                if (inspectionState.containsKey('Health')) {
+                    String healthStatus = inspectionState.Health.Status
+                    if (!"starting".equalsIgnoreCase(healthStatus)) {
+                        logger.lifecycle("${service.name} healh state reported as '$healthStatus' - continuing...")
+                        return
+                    }
+                } else {
+                    logger.debug("Service ${service.name} or this version of Docker doesn't support healtchecks")
+                    return
+                }
+                if (start.plus(extension.waitForHealthyStateTimeout) < Instant.now()) {
+                    throw new RuntimeException("Container ${service.containerId} of service ${service.name} is still reported as 'starting'. Logs:${System.lineSeparator()}${getServiceLogs(service.name)}")
+                }
+                logger.lifecycle("Waiting for ${service.name} to become healthy")
+                sleep(extension.waitAfterHealthyStateProbeFailure.toMillis())
+            }
+        }
+    }
+
     void waitForOpenTcpPorts(Iterable<ServiceInfo> servicesInfos) {
         servicesInfos.forEach { service ->
             service.tcpPorts.forEach { exposedPort, forwardedPort ->