Release 2.3.3

* Remove decompress apache httpclient 4.5.x does it automatically * thread should end when loops end. * Enable compression request to tank clients * AWS X-ray instrumentation * Reduce pending to running wait from 1min to 30sec * Enable Referer and Authorization headers, Test and documentation * trace recent jobs query * Performance & Size Reduction * Cache agent-support-files.zip in /tmp * Replace deprecated hibernate index annotations with JPA index annotations * Library reorganization saved 43MB off deployment war * AWS Xray + DBQueries + Performance * Cleanup * RequestScoped bean to ApplicationScoped bean * AWSXray dummy segments to allow jdbc tracing to be enabled * LazyLoad User Table * BatchSize * userAdmin Fix + CriteriaBuilder * better error handling on UserDao
intuit · Feb 18, 2020 · 69e36f3 · 69e36f3
1 parent b75a901
commit 69e36f3
Show file tree

Hide file tree

Showing 193 changed files with 1,086 additions and 1,025 deletions.
diff --git a/agent/agent_common/pom.xml b/agent/agent_common/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>com.intuit.tank</groupId>
     <artifactId>agent-parent</artifactId>
-    <version>2.3.2</version>
+    <version>2.3.3</version>
   </parent>
 
   <artifactId>agent-common</artifactId>

diff --git a/agent/agent_common/src/main/java/com/intuit/tank/http/BaseResponse.java b/agent/agent_common/src/main/java/com/intuit/tank/http/BaseResponse.java
@@ -226,11 +226,13 @@ public void logResponse() {
     }
 
     public static final boolean isDataType(String contentType) {
-        boolean ret = false;
         if (!StringUtils.isBlank(contentType)) {
             contentType = contentType.toLowerCase();
-            ret = contentType.contains("html") || contentType.contains("text") || contentType.contains("json") || contentType.contains("xml");
+            return  contentType.contains("html") ||
+                    contentType.contains("text") ||
+                    contentType.contains("json") ||
+                    contentType.contains("xml");
         }
-        return ret;
+        return false;
     }
 }
diff --git a/agent/agent_common/src/main/java/com/intuit/tank/http/TankHttpUtil.java b/agent/agent_common/src/main/java/com/intuit/tank/http/TankHttpUtil.java
@@ -5,7 +5,6 @@
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLEncoder;
-import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -22,6 +21,7 @@
 import com.intuit.tank.http.binary.BinaryResponse;
 import com.intuit.tank.http.json.JsonResponse;
 import com.intuit.tank.http.xml.XMLResponse;
+
 /**
  * utitly methods for tank http clients
  * @author denisa
@@ -177,7 +177,7 @@ public byte[] getBody() {
          * @return the body as a string
          */
         public String getBodyAsString() {
-            return new String(body, Charset.forName("UTF-8"));
+            return new String(body, StandardCharsets.UTF_8);
         }
 
         // Content-Disposition: form-data; name="uploadname1";

diff --git a/agent/agent_standalone/pom.xml b/agent/agent_standalone/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>com.intuit.tank</groupId>
     <artifactId>agent-parent</artifactId>
-    <version>2.3.2</version>
+    <version>2.3.3</version>
   </parent>
 
   <artifactId>agent-standalone</artifactId>

diff --git a/agent/agent_standalone_pkg/pom.xml b/agent/agent_standalone_pkg/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>com.intuit.tank</groupId>
     <artifactId>agent-parent</artifactId>
-    <version>2.3.2</version>
+    <version>2.3.3</version>
   </parent>
 
   <artifactId>agent-standalone-pkg</artifactId>

diff --git a/agent/agent_startup/pom.xml b/agent/agent_startup/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>com.intuit.tank</groupId>
     <artifactId>agent-parent</artifactId>
-    <version>2.3.2</version>
+    <version>2.3.3</version>
   </parent>
 
   <artifactId>agent-startup</artifactId>

diff --git a/agent/agent_startup/src/main/java/com/intuit/tank/agent/AgentStartup.java b/agent/agent_startup/src/main/java/com/intuit/tank/agent/AgentStartup.java
@@ -21,6 +21,7 @@
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
+import com.amazonaws.util.StringUtils;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.logging.log4j.LogManager;
@@ -32,40 +33,30 @@
 public class AgentStartup implements Runnable {
 
     private static Logger logger = LogManager.getLogger(AgentStartup.class);
-    public static final String SERVICE_RELATIVE_PATH = "/rest/v1/agent-service";
-    public static final String METHOD_SETTINGS = "/settings";
-    private static String API_HARNESS_COMMAND = "./startAgent.sh";
-    public static final String METHOD_SUPPORT = "/supportFiles";
-    private static final long WAIT_FOR_RESTART_TIME = 00000; // zero minute
+    private static final String SERVICE_RELATIVE_PATH = "/rest/v1/agent-service";
+    private static final String METHOD_SETTINGS = "/settings";
+    private static final String API_HARNESS_COMMAND = "./startAgent.sh";
+    private static final String METHOD_SUPPORT = "/supportFiles";
 
-    private String controllerBase;
+    private final String controllerBaseUrl;
+
+    public AgentStartup (String controllerBaseUrl ) {
+        this.controllerBaseUrl = controllerBaseUrl;
+    }
 
     public void run() {
         logger.info("Starting up...");
-        if (AmazonUtil.usingEip()) {
-            try {
-                logger.info("Using EIP. Sleeping for " + WAIT_FOR_RESTART_TIME + " ms.");
-                Thread.sleep(WAIT_FOR_RESTART_TIME);
-            } catch (InterruptedException e1) {
-                logger.info("Exception waiting.");
-                System.exit(0);
-            }
-        }
         try {
-            if (controllerBase == null) {
-                controllerBase = AmazonUtil.getControllerBaseUrl();
-            }
-
-            logger.info("Starting up: ControllerBaseUrl=" + controllerBase);
-            URL url = new URL(controllerBase + SERVICE_RELATIVE_PATH + METHOD_SETTINGS);
+            logger.info("Starting up: ControllerBaseUrl=" + controllerBaseUrl);
+            URL url = new URL(controllerBaseUrl + SERVICE_RELATIVE_PATH + METHOD_SETTINGS);
             logger.info("Starting up: making call to tank service url to get settings.xml "
                     + url.toExternalForm());
             try ( InputStream settingsStream = url.openStream() ) {
                 String settings = IOUtils.toString(settingsStream, StandardCharsets.UTF_8);
                 FileUtils.writeStringToFile(new File("settings.xml"), settings, StandardCharsets.UTF_8);
                 logger.info("got settings file...");
             }
-            url = new URL(controllerBase + SERVICE_RELATIVE_PATH + METHOD_SUPPORT);
+            url = new URL(controllerBaseUrl + SERVICE_RELATIVE_PATH + METHOD_SUPPORT);
             logger.info("Making call to tank service url to get support files " + url.toExternalForm());
             try ( ZipInputStream zip = new ZipInputStream(url.openStream()) ){
                 ZipEntry entry = zip.getNextEntry();
@@ -81,17 +72,16 @@ public void run() {
             }
             // now start the harness
             String jvmArgs = AmazonUtil.getUserDataAsMap().get(TankConstants.KEY_JVM_ARGS);
-            logger.info("Starting apiharness with command: " + API_HARNESS_COMMAND + " -http=" + controllerBase + " "
+            logger.info("Starting apiharness with command: " + API_HARNESS_COMMAND + " -http=" + controllerBaseUrl + " "
                     + jvmArgs);
-            Runtime.getRuntime().exec(API_HARNESS_COMMAND + " -http=" + controllerBase + " " + jvmArgs);
+            Runtime.getRuntime().exec(API_HARNESS_COMMAND + " -http=" + controllerBaseUrl + " " + jvmArgs);
         } catch (Exception e) {
             logger.error("Error in AgentStartup " + e, e);
         }
     }
 
     public static void main(String[] args) {
-        AgentStartup agentStartup = new AgentStartup();
-
+        String controllerBaseUrl = null;
         for (String argument : args) {
             String[] values = argument.split("=");
 
@@ -100,10 +90,13 @@ public static void main(String[] args) {
                     usage();
                     return;
                 }
-                agentStartup.controllerBase = values[1];
+                controllerBaseUrl = values[1];
             }
-
         }
+        if (StringUtils.isNullOrEmpty(controllerBaseUrl)) {
+            controllerBaseUrl = AmazonUtil.getControllerBaseUrl();
+        }
+        AgentStartup agentStartup = new AgentStartup(controllerBaseUrl);
         agentStartup.run();
     }
 

diff --git a/agent/agent_startup_pkg/pom.xml b/agent/agent_startup_pkg/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>com.intuit.tank</groupId>
     <artifactId>agent-parent</artifactId>
-    <version>2.3.2</version>
+    <version>2.3.3</version>
   </parent>
 
   <artifactId>agent-startup-pkg</artifactId>

diff --git a/agent/apiharness/pom.xml b/agent/apiharness/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>com.intuit.tank</groupId>
     <artifactId>agent-parent</artifactId>
-    <version>2.3.2</version>
+    <version>2.3.3</version>
   </parent>
 
   <artifactId>agent</artifactId>
@@ -57,12 +57,6 @@
       <version>${project.version}</version>
     </dependency>
 
-    <dependency>
-      <groupId>${project.groupId}</groupId>
-      <artifactId>tank-common</artifactId>
-      <version>${project.version}</version>
-    </dependency>
-
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>agent-common</artifactId>

diff --git a/agent/apiharness/src/main/java/com/intuit/tank/harness/APITestHarness.java b/agent/apiharness/src/main/java/com/intuit/tank/harness/APITestHarness.java
@@ -658,7 +658,7 @@ public synchronized void threadComplete() {
         long count = doneSignal.getCount();
         // numCompletedThreads = (int) (agentRunData.getNumUsers() - count);
         if (isDebug() || count < 10) {
-            LOG.info(new ObjectMessage(ImmutableMap.of("Message", "User thread finished... Remaining->" + currentUsers)));
+            LOG.info(new ObjectMessage(ImmutableMap.of("Message", "User thread finished... Remaining = " + currentUsers)));
         }
     }
 

diff --git a/agent/apiharness/src/main/java/com/intuit/tank/harness/functions/JexlStringFunctions.java b/agent/apiharness/src/main/java/com/intuit/tank/harness/functions/JexlStringFunctions.java
@@ -19,7 +19,6 @@
 
 import java.net.URLDecoder;
 import java.net.URLEncoder;
-import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.text.SimpleDateFormat;
 import java.util.Collections;
@@ -487,7 +486,7 @@ public String randomAlphaMixedNumericSpecial(int length) {
     public String toBase64(String toEncode) {
         try {
             byte[] bytes = toEncode.getBytes();
-            return new String(Base64.encodeBase64(bytes), Charset.forName("utf-8")).trim();
+            return new String(Base64.encodeBase64(bytes), StandardCharsets.UTF_8).trim();
         } catch (Exception e) {
             LOG.error("Error base64 encoding " + toEncode + ": " + e);
         }
@@ -504,7 +503,7 @@ public String toBase64(String toEncode) {
     public String fromBase64(String toDecode) {
         try {
             byte[] bytes = Base64.decodeBase64(toDecode.trim());
-            return new String(bytes, Charset.forName("utf-8"));
+            return new String(bytes, StandardCharsets.UTF_8);
         } catch (Exception e) {
             LOG.error("Error base64 decoding " + toDecode + ": " + e);
         }

diff --git a/agent/apiharness/src/main/java/com/intuit/tank/runner/TestPlanRunner.java b/agent/apiharness/src/main/java/com/intuit/tank/runner/TestPlanRunner.java
@@ -121,7 +121,6 @@ public void run() {
         List<HDScriptGroup> group = testPlan.getGroup();
         try {
             int i = 0;
-            int loopCount = 1;
 
             mainLoop: while (i < group.size()) {
                 HDScriptGroup hdScriptGroup = group.get(i);
@@ -144,7 +143,6 @@ public void run() {
                         break;
                     } catch (RestartScriptException e) {
                         i = 0;
-                        loopCount = 1;
                         continue mainLoop;
                     }
                     if (isCompleted(RunPhase.group, finished)) {
@@ -156,16 +154,8 @@ public void run() {
                 if (group.get(group.size() - 1) != hdScriptGroup) {
                     i++;
                 } else {
-                    finished = true;
-                    if (isCompleted(RunPhase.test, finished)) {
-                        LOG.info(LogUtil.getLogMessage("finished or Stop set to test or less, exiting at test..."));
-                        return;
-                    }
-                    i = 0;
-                    loopCount++;
-                    LOG.info(LogUtil.getLogMessage("Test for test plan " + testPlan.getTestPlanName()
-                            + " has finished and is now starting over for the  "
-                            + loopCount + " time"));
+                    // All Threads should end after loops complete.
+                    return;
                 }
             }
         } catch (Throwable e) {
@@ -176,8 +166,8 @@ public void run() {
         }
     }
 
-    private void runTestGroup(List<HDScript> scripts, HDScriptGroup parent) throws KillScriptException,
-            RestartScriptException {
+    private void runTestGroup(List<HDScript> scripts, HDScriptGroup parent)
+            throws KillScriptException, RestartScriptException {
         MethodTimer mt = new MethodTimer(LOG, getClass(), "runScriptGroup(" + parent.getName() + ")");
         LogEvent logEvent = LogUtil.getLogEvent();
         try {
@@ -222,8 +212,7 @@ private void runTestGroup(List<HDScript> scripts, HDScriptGroup parent) throws K
                     APITestHarness.getInstance().getUserTracker().remove(hdscript.getName());
                 }
                 if (shouldStop(RunPhase.script)) {
-                    LOG.info(LogUtil.getLogMessage("Stop set to script or less, exiting at script " + hdscript.getName()));
-                    return;
+                    throw new KillScriptException("Stop set to script or less, exiting at script " + hdscript.getName());
                 }
             }
         } finally {

diff --git a/agent/apiharness_pkg/pom.xml b/agent/apiharness_pkg/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>com.intuit.tank</groupId>
     <artifactId>agent-parent</artifactId>
-    <version>2.3.2</version>
+    <version>2.3.3</version>
   </parent>
 
 

diff --git a/agent/http_client_3/pom.xml b/agent/http_client_3/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>com.intuit.tank</groupId>
     <artifactId>agent-parent</artifactId>
-    <version>2.3.2</version>
+    <version>2.3.3</version>
   </parent>
 
   <artifactId>http_client_commons_3</artifactId>

diff --git a/agent/http_client_3/src/main/java/com/intuit/tank/httpclient3/TankHttpClient3.java b/agent/http_client_3/src/main/java/com/intuit/tank/httpclient3/TankHttpClient3.java
@@ -15,7 +15,6 @@
  * #L%
  */
 
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
@@ -353,18 +352,11 @@ private void processResponse(byte[] bResponse, long waitTime, BaseRequest reques
                 }
             }
             response.setResponseTime(waitTime);
-            String contentType = response.getHttpHeader("Content-Type");
-            String contentEncode = response.getHttpHeader("Content-Encoding");
-            if (BaseResponse.isDataType(contentType) && contentEncode != null && contentEncode.toLowerCase().contains("gzip")) {
-                // decode gzip for data types
-                try (   GZIPInputStream in = new GZIPInputStream(new ByteArrayInputStream(bResponse));
-                        ByteArrayOutputStream out = new ByteArrayOutputStream() ) {
-                    IOUtils.copy(in, out);
-                    bResponse = out.toByteArray();
-                } catch (IOException | NullPointerException e) {
-                    LOG.warn(request.getLogUtil().getLogMessage("cannot decode gzip stream: " + e, LogEventType.System));
-                }
-            }
+
+            String contentEncoding = response.getHttpHeader("Content-Encoding");
+            bResponse = StringUtils.equalsIgnoreCase(contentEncoding, "gzip") ?
+                    IOUtils.toByteArray(new GZIPInputStream(new ByteArrayInputStream(bResponse))) :
+                    bResponse;
             response.setResponseBody(bResponse);
 
         } catch (Exception ex) {

diff --git a/agent/http_client_4/pom.xml b/agent/http_client_4/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>com.intuit.tank</groupId>
     <artifactId>agent-parent</artifactId>
-    <version>2.3.2</version>
+    <version>2.3.3</version>
   </parent>
 
   <artifactId>http_client_commons_4</artifactId>