Release 2.3.4

* Fix agent debugger * Security to SessionScoped * Minimum single thread for debugger * Validate position of list before every use to avoid IndexOutOfBoundsException after a item removal * AWSXRay: setUser to currentSegment at filter * update README.MD with badges * Fix kill instances across regions * filter instances by present in region before terminate/stop. * update libraries versions * java streams * requestInstances re-write * AgentStartup retry loop * backoff Strategy for supportFiles download * NonNull TPS results return * JSF to 2.3 * CloudWatchMetrics * Remove SimpleDB DataSource * Add CloudWatch Metrics DataSource * FIBONACCI retry backoff strategy
intuit · Apr 11, 2020 · d1c505b · d1c505b
1 parent 69e36f3
commit d1c505b
Show file tree

Hide file tree

Showing 150 changed files with 813 additions and 1,495 deletions.
diff --git a/agent/agent_common/pom.xml b/agent/agent_common/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>com.intuit.tank</groupId>
     <artifactId>agent-parent</artifactId>
-    <version>2.3.3</version>
+    <version>2.3.4</version>
   </parent>
 
   <artifactId>agent-common</artifactId>
@@ -33,7 +33,7 @@
     </dependency>
 
     <dependency>
-      <groupId>jdom</groupId>
+      <groupId>org.jdom</groupId>
       <artifactId>jdom</artifactId>
     </dependency>
   </dependencies>

diff --git a/agent/agent_common/src/main/java/com/intuit/tank/http/TankHttpUtil.java b/agent/agent_common/src/main/java/com/intuit/tank/http/TankHttpUtil.java
@@ -2,6 +2,7 @@
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLEncoder;
@@ -10,7 +11,6 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
 
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.fileupload.MultipartStream;
@@ -22,76 +22,48 @@
 import com.intuit.tank.http.json.JsonResponse;
 import com.intuit.tank.http.xml.XMLResponse;
 
+import static java.util.stream.Collectors.joining;
+
 /**
  * utitly methods for tank http clients
  * @author denisa
  *
  */
 public class TankHttpUtil {
-
     private static Logger LOG = LogManager.getLogger(TankHttpUtil.class);
 
     public static URL buildUrl(String protocol, String host, int port, String path, Map<String, String> urlVariables) {
-
         try {
-            // no default port specified for http
-            if (protocol.equalsIgnoreCase("http") && port == -1) {
-            	return new URL(protocol, host, path + getQueryString(urlVariables));
-            } else if (protocol.equalsIgnoreCase("https") && port == -1) {
-            	return new URL(protocol, host, path + getQueryString(urlVariables));
-            }
             // ensure that port 80 and 8080 requests use http and not https
             if (port == 80 || port == 8080) {
                 protocol = "http";
             }
 
-            return new URL(protocol, host, port, path + getQueryString(urlVariables));
+            // no default port specified for http
+            return (port == -1) ?
+                    new URL(protocol, host, path + getQueryString(urlVariables)) :
+                    new URL(protocol, host, port, path + getQueryString(urlVariables));
         } catch (MalformedURLException e) {
             throw new RuntimeException(e);
         }
     }
 
     public static String getQueryString(Map<String, String> urlVariables) {
-
-        StringBuilder queryString = new StringBuilder();
-
-        // Set the query string
-        if (urlVariables != null) {
-            if (!urlVariables.isEmpty()) {
-
-                queryString.append("?");
-
-                // Set<Map.Entry<String, String>> set = urlVariables.entrySet();
-                // Iterator<Map.Entry<String, String>> iterator =
-                // set.iterator();
-                for (Entry<String, String> entry : urlVariables.entrySet()) {
-                    try {
-                        StringBuilder nvp = new StringBuilder();
-                        nvp.append(URLEncoder.encode(entry.getKey(), StandardCharsets.UTF_8.toString()));
-                        if (entry.getValue() != null) {
-                            nvp.append("=");
-                            nvp.append(URLEncoder.encode(entry.getValue(), StandardCharsets.UTF_8.toString()));
+        if (urlVariables != null && !urlVariables.isEmpty()) {
+            return "?" + urlVariables.entrySet().stream()
+                    .map(entry -> {
+                        try {
+                            return URLEncoder.encode(entry.getKey(), StandardCharsets.UTF_8.name())
+                                    + "="
+                                    + URLEncoder.encode(entry.getValue(), StandardCharsets.UTF_8.name());
+                        } catch (UnsupportedEncodingException ex) {
+                            LOG.warn("Unable to set query string value: " + ex.getMessage());
                         }
-                        nvp.append("&");
-                        queryString.append(nvp.toString());
-
-                    } catch (Exception ex) {
-                        LOG.warn("Unable to set query string value: " + ex.getMessage());
-                    }
-                }
-            }
-        }
-
-        // Remove the last &
-        String reqQueryString = "";
-        if (queryString.length() > 0) {
-            if (queryString.charAt(queryString.length() - 1) == '&')
-                reqQueryString = queryString.deleteCharAt(queryString.length() - 1).toString();
-            else
-                reqQueryString = queryString.toString();
+                        return "";
+                    })
+                    .collect(joining("&"));
         }
-
-        return reqQueryString;
+        return "";
     }
 
     /**
@@ -101,13 +73,11 @@ public static String getQueryString(Map<String, String> urlVariables) {
      */
     public static BaseResponse newResponseObject(String contentTypeHeader) {
         String contentType = StringUtils.isNotBlank(contentTypeHeader) ? contentTypeHeader : "";
-        if (contentType.contains("xml")) {
-            return new XMLResponse();
-        } else if (contentType.contains("json")) {
-            return new JsonResponse();
-        } else {
-            return new BinaryResponse();
-        }
+        return contentType.contains("xml") ?
+                new XMLResponse() :
+                contentType.contains("json") ?
+                        new JsonResponse() :
+                        new BinaryResponse();
     }
 
     public static List<PartHolder> getPartsFromBody(BaseRequest request) {
@@ -196,20 +166,18 @@ public String getFileName() {
         // filename="diamond-sword.png"
         public String getContentType() {
             String ct = headerMap.get("Content-Type");
-            if (ct == null) {
-                ct = "text/plain";
-            }
-            return ct;
+            return ct == null ?
+                    "text/plain" :
+                    ct;
         }
 
         // Content-Disposition: form-data; name="uploadname1";
         // filename="diamond-sword.png"
         public String getContentDisposition() {
             String ct = headerMap.get("Content-Disposition");
-            if (ct == null) {
-                ct = "form-data";
-            }
-            return ct;
+            return ct == null ?
+                    "form-data" :
+                    ct;
         }
 
         /**

diff --git a/agent/agent_common/src/main/java/com/intuit/tank/http/xml/GenericXMLHandler.java b/agent/agent_common/src/main/java/com/intuit/tank/http/xml/GenericXMLHandler.java
@@ -27,14 +27,14 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import org.jdom.Attribute;
-import org.jdom.Document;
-import org.jdom.Element;
-import org.jdom.JDOMException;
-import org.jdom.Namespace;
-import org.jdom.input.SAXBuilder;
-import org.jdom.output.XMLOutputter;
-import org.jdom.xpath.XPath;
+import org.jdom2.Attribute;
+import org.jdom2.Document;
+import org.jdom2.Element;
+import org.jdom2.JDOMException;
+import org.jdom2.Namespace;
+import org.jdom2.input.SAXBuilder;
+import org.jdom2.output.XMLOutputter;
+import org.jdom2.xpath.XPath;
 import org.xml.sax.InputSource;
 
 /**
@@ -70,7 +70,7 @@ public GenericXMLHandler() {
     public GenericXMLHandler(File xmlFile) {
         try {
             this.xmlFile = xmlFile;
-            this.xmlDocument = new org.jdom.Document();
+            this.xmlDocument = new org.jdom2.Document();
             SAXBuilder builder = new SAXBuilder();
             builder.setValidation(false);
             this.xmlDocument = builder.build(this.xmlFile);
@@ -92,7 +92,7 @@ public GenericXMLHandler(String xmlFile) {
             this.xml = xmlFile;
             try {
                 this.xmlFile = null;
-                this.xmlDocument = new org.jdom.Document();
+                this.xmlDocument = new org.jdom2.Document();
                 SAXBuilder builder = new SAXBuilder();
                 builder.setValidation(false);
                 // LOG.debug("XML string to load: "+xmlFile);
@@ -155,7 +155,7 @@ private Element SetElementText(String xPathExpression, int currentNode) throws J
 
         if (xPathExists(currentPath)) {
             if (currentPath.equals(xPathExpression)) {
-                return (org.jdom.Element) XPath.selectSingleNode(this.xmlDocument, xPathExpression);
+                return (org.jdom2.Element) XPath.selectSingleNode(this.xmlDocument, xPathExpression);
             }
             else {
                 return SetElementText(xPathExpression, currentNode + 1);
@@ -225,7 +225,7 @@ public String GetElementText(String xPathExpression) {
      */
     public String GetElementAttr(String xPathExpression) {
         try {
-            org.jdom.Attribute node = (Attribute) XPath.selectSingleNode(this.xmlDocument, xPathExpression);
+            org.jdom2.Attribute node = (Attribute) XPath.selectSingleNode(this.xmlDocument, xPathExpression);
             return node.getValue();
         } catch (Exception ex) {
             LOG.error("Error in handler: " + ex.getMessage(), ex);

diff --git a/agent/agent_common/src/test/java/com/intuit/tank/http/TankHttpUtilTest.java b/agent/agent_common/src/test/java/com/intuit/tank/http/TankHttpUtilTest.java
@@ -1,5 +1,6 @@
 package com.intuit.tank.http;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 
 /*
@@ -146,9 +147,13 @@ public void testGetQueryString_1()
     public void testGetQueryString_2()
         throws Exception {
         Map<String, String> urlVariables = new HashMap();
+        urlVariables.put("val1", "val1");
+        urlVariables.put("val2", "val2");
+        urlVariables.put("val3", "val3");
 
         String result = TankHttpUtil.getQueryString(urlVariables);
         assertNotNull(result);
+        assertEquals( "?val3=val3&val2=val2&val1=val1", result);
     }
 
     /**
@@ -162,9 +167,11 @@ public void testGetQueryString_2()
     public void testGetQueryString_3()
         throws Exception {
         Map<String, String> urlVariables = new HashMap();
+        urlVariables.put("data", "{\"val1\":\"val1\", \"val2\":\"val2\", \"val3\":\"val3\"}");
 
         String result = TankHttpUtil.getQueryString(urlVariables);
         assertNotNull(result);
+        assertEquals( "?data=%7B%22val1%22%3A%22val1%22%2C+%22val2%22%3A%22val2%22%2C+%22val3%22%3A%22val3%22%7D", result);
     }
 
     /**

diff --git a/agent/agent_common/src/test/java/com/intuit/tank/http/xml/GenericXMLHandlerTest.java b/agent/agent_common/src/test/java/com/intuit/tank/http/xml/GenericXMLHandlerTest.java
@@ -16,7 +16,7 @@
 import java.io.File;
 import java.util.HashMap;
 
-import org.jdom.Document;
+import org.jdom2.Document;
 import org.junit.jupiter.api.*;
 
 import static org.junit.jupiter.api.Assertions.*;

diff --git a/agent/agent_standalone/pom.xml b/agent/agent_standalone/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>com.intuit.tank</groupId>
     <artifactId>agent-parent</artifactId>
-    <version>2.3.3</version>
+    <version>2.3.4</version>
   </parent>
 
   <artifactId>agent-standalone</artifactId>

diff --git a/agent/agent_standalone_pkg/pom.xml b/agent/agent_standalone_pkg/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>com.intuit.tank</groupId>
     <artifactId>agent-parent</artifactId>
-    <version>2.3.3</version>
+    <version>2.3.4</version>
   </parent>
 
   <artifactId>agent-standalone-pkg</artifactId>

diff --git a/agent/agent_startup/pom.xml b/agent/agent_startup/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>com.intuit.tank</groupId>
     <artifactId>agent-parent</artifactId>
-    <version>2.3.3</version>
+    <version>2.3.4</version>
   </parent>
 
   <artifactId>agent-startup</artifactId>

diff --git a/agent/agent_startup/src/main/java/com/intuit/tank/agent/AgentStartup.java b/agent/agent_startup/src/main/java/com/intuit/tank/agent/AgentStartup.java
@@ -13,12 +13,15 @@
  * #L%
  */
 
+import java.io.EOFException;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.InputStream;
+import java.net.ConnectException;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.util.zip.ZipEntry;
+import java.util.zip.ZipException;
 import java.util.zip.ZipInputStream;
 
 import com.amazonaws.util.StringUtils;
@@ -31,12 +34,12 @@
 import com.intuit.tank.vm.common.TankConstants;
 
 public class AgentStartup implements Runnable {
-
     private static Logger logger = LogManager.getLogger(AgentStartup.class);
     private static final String SERVICE_RELATIVE_PATH = "/rest/v1/agent-service";
     private static final String METHOD_SETTINGS = "/settings";
     private static final String API_HARNESS_COMMAND = "./startAgent.sh";
     private static final String METHOD_SUPPORT = "/supportFiles";
+    private static final int[] FIBONACCI = new int[] { 1, 1, 2, 3, 5, 8, 13 };
 
     private final String controllerBaseUrl;
 
@@ -55,25 +58,39 @@ public void run() {
                 String settings = IOUtils.toString(settingsStream, StandardCharsets.UTF_8);
                 FileUtils.writeStringToFile(new File("settings.xml"), settings, StandardCharsets.UTF_8);
                 logger.info("got settings file...");
+            } catch (ConnectException ce) {
+                logger.error("Error creating connection to "
+                        + controllerBaseUrl + " : this is normal during the bake : " + ce.getMessage());
             }
+            // Download Support Files
             url = new URL(controllerBaseUrl + SERVICE_RELATIVE_PATH + METHOD_SUPPORT);
             logger.info("Making call to tank service url to get support files " + url.toExternalForm());
-            try ( ZipInputStream zip = new ZipInputStream(url.openStream()) ){
-                ZipEntry entry = zip.getNextEntry();
-                while (entry != null) {
-                    String name = entry.getName();
-                    logger.info("Got file from controller: " + name);
-                    File f = new File(name);
-                    try ( FileOutputStream fout = FileUtils.openOutputStream(f) ) {
-                        IOUtils.copy(zip, fout);
+            int retryCount = 0;
+            while (true) {
+                try (ZipInputStream zip = new ZipInputStream(url.openStream())) {
+                    ZipEntry entry = zip.getNextEntry();
+                    while (entry != null) {
+                        String name = entry.getName();
+                        logger.info("Got file from controller: " + name);
+                        File f = new File(name);
+                        try (FileOutputStream fout = FileUtils.openOutputStream(f)) {
+                            IOUtils.copy(zip, fout);
+                        }
+                        entry = zip.getNextEntry();
                     }
-                    entry = zip.getNextEntry();
+                    break;
+                } catch (EOFException | ZipException e) {
+                    logger.error("Error unzipping support files : retryCount="
+                            + retryCount + " : " + e.getMessage());
+                    if (retryCount < FIBONACCI.length) {
+                        Thread.sleep( FIBONACCI[++retryCount] * 1000 );
+                    } else throw e;
                 }
             }
             // now start the harness
             String jvmArgs = AmazonUtil.getUserDataAsMap().get(TankConstants.KEY_JVM_ARGS);
-            logger.info("Starting apiharness with command: " + API_HARNESS_COMMAND + " -http=" + controllerBaseUrl + " "
-                    + jvmArgs);
+            logger.info("Starting apiharness with command: "
+                    + API_HARNESS_COMMAND + " -http=" + controllerBaseUrl + " " + jvmArgs);
             Runtime.getRuntime().exec(API_HARNESS_COMMAND + " -http=" + controllerBaseUrl + " " + jvmArgs);
         } catch (Exception e) {
             logger.error("Error in AgentStartup " + e, e);

diff --git a/agent/agent_startup_pkg/pom.xml b/agent/agent_startup_pkg/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>com.intuit.tank</groupId>
     <artifactId>agent-parent</artifactId>
-    <version>2.3.3</version>
+    <version>2.3.4</version>
   </parent>
 
   <artifactId>agent-startup-pkg</artifactId>