From fc7de94eed293c99b49e68c3c0f24adea19f4225 Mon Sep 17 00:00:00 2001 From: Varun Saxena Date: Tue, 16 Oct 2018 15:41:03 +0530 Subject: [PATCH] Revert "Revert "Merge pull request #397 from arpang/customSHSWork"" This reverts commit 9030a59f99dda354429ce7e09d567a036881e09b. --- .../tuning/AutoTuningAPIHelper.java | 562 +++++++++--------- .../tuning/AzkabanJobCompleteDetector.java | 59 +- .../tuning/BaselineComputeUtil.java | 13 +- .../drelephant/tuning/FitnessComputeUtil.java | 403 ++++++++----- .../tuning/JobCompleteDetector.java | 87 +-- .../drelephant/tuning/JobTuningInfo.java | 2 - .../drelephant/tuning/PSOParamGenerator.java | 20 +- .../drelephant/tuning/ParamGenerator.java | 183 +++--- .../linkedin/drelephant/tuning/Particle.java | 1 - .../drelephant/tuning/TuningInput.java | 48 +- app/controllers/Application.java | 41 +- app/models/FlowDefinition.java | 24 + app/models/FlowExecution.java | 24 + app/models/JobDefinition.java | 17 +- app/models/JobExecution.java | 31 +- app/models/JobSavedState.java | 14 + app/models/JobSuggestedParamSet.java | 118 ++++ app/models/JobSuggestedParamValue.java | 30 +- app/models/TuningAlgorithm.java | 15 + app/models/TuningJobDefinition.java | 16 +- ...n.java => TuningJobExecutionParamSet.java} | 60 +- app/models/TuningParameter.java | 16 +- conf/evolutions/default/2.sql | 18 +- conf/evolutions/default/3.sql | 16 + conf/evolutions/default/4.sql | 16 + conf/evolutions/default/5.sql | 109 ++-- conf/evolutions/default/6.sql | 11 - .../tuning/PSOParamGeneratorTest.java | 16 +- test/resources/AutoTuningConf.xml | 4 +- test/resources/test-init.sql | 39 +- test/resources/tunein-test1.sql | 56 +- test/rest/RestAPITest.java | 43 +- 32 files changed, 1269 insertions(+), 843 deletions(-) create mode 100644 app/models/JobSuggestedParamSet.java rename app/models/{TuningJobExecution.java => TuningJobExecutionParamSet.java} (50%) delete mode 100644 conf/evolutions/default/6.sql diff --git a/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java b/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java index 14346fa17..b6b5b68ef 100644 --- a/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java +++ b/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java @@ -19,21 +19,27 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.drelephant.ElephantContext; import com.linkedin.drelephant.util.Utils; + import controllers.AutoTuningMetricsController; + +import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; + import models.FlowDefinition; import models.FlowExecution; import models.JobDefinition; import models.JobExecution; import models.JobExecution.ExecutionState; +import models.JobSuggestedParamSet; +import models.JobSuggestedParamSet.ParamSetStatus; import models.JobSuggestedParamValue; import models.TuningAlgorithm; import models.TuningJobDefinition; -import models.TuningJobExecution; -import models.TuningJobExecution.ParamSetStatus; +import models.TuningJobExecutionParamSet; import models.TuningParameter; + import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.log4j.Logger; @@ -44,96 +50,46 @@ */ public class AutoTuningAPIHelper { - public static final String ALLOWED_MAX_RESOURCE_USAGE_PERCENT_DEFAULT = + private static final String ALLOWED_MAX_RESOURCE_USAGE_PERCENT_DEFAULT = "autotuning.default.allowed_max_resource_usage_percent"; - public static final String ALLOWED_MAX_EXECUTION_TIME_PERCENT_DEFAULT = + private static final String ALLOWED_MAX_EXECUTION_TIME_PERCENT_DEFAULT = "autotuning.default.allowed_max_execution_time_percent"; private static final Logger logger = Logger.getLogger(AutoTuningAPIHelper.class); /** - * For a job, returns the execution with the best parameter set if available else the one with the default parameter set. + * For a job, returns the best parameter set of the given job if it exists else the default parameter set * @param jobDefId Sting JobDefId of the job - * @return TuningJobExecution with the best parameter set if available else the one with the default parameter set. + * @return JobSuggestedParamSet the best parameter set of the given job if it exists else the default parameter set */ - - private TuningJobExecution getBestParamSetTuningJobExecution(String jobDefId) { - TuningJobExecution tuningJobExecutionBestParamSet = TuningJobExecution.find.select("*") + private JobSuggestedParamSet getBestParamSet(String jobDefId) { + JobSuggestedParamSet jobSuggestedParamSetBestParamSet = JobSuggestedParamSet.find.select("*") .where() - .eq(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.job + "." + JobDefinition.TABLE.jobDefId, - jobDefId) - .eq(TuningJobExecution.TABLE.isParamSetBest, true) + .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.jobDefId, jobDefId) + .eq(JobSuggestedParamSet.TABLE.isParamSetBest, true) .setMaxRows(1) .findUnique(); - if (tuningJobExecutionBestParamSet == null) { - tuningJobExecutionBestParamSet = TuningJobExecution.find.select("*") + if (jobSuggestedParamSetBestParamSet == null) { + jobSuggestedParamSetBestParamSet = JobSuggestedParamSet.find.select("*") .where() - .eq(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.job + "." + JobDefinition.TABLE.jobDefId, - jobDefId) - .eq(TuningJobExecution.TABLE.isDefaultExecution, true) + .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.jobDefId, jobDefId) + .eq(JobSuggestedParamSet.TABLE.isParamSetDefault, true) .setMaxRows(1) .findUnique(); } - return tuningJobExecutionBestParamSet; + return jobSuggestedParamSetBestParamSet; } /** - * Returns the param set corresponding to a given job execution id - * @param jobExecutionId Long job execution id of the execution - * @return List: list of parameters + * Returns the param values corresponding to the given param set id + * @param paramSetId Long parameter set id + * @return List list of parameters */ - private List getJobExecutionParamSet(Long jobExecutionId) { - return JobSuggestedParamValue.find.where() - .eq(JobSuggestedParamValue.TABLE.jobExecution + "." + JobExecution.TABLE.id, jobExecutionId) + private List getParamSetValues(Long paramSetId) { + List jobSuggestedParamValues = JobSuggestedParamValue.find.where() + .eq(JobSuggestedParamValue.TABLE.jobSuggestedParamSet + '.' + JobSuggestedParamSet.TABLE.id, paramSetId) .findList(); - } - - /** - * This method creates a job execution with best parameter set. This is required when there is no parameters set or tuning has been switch off for the job - * remains for suggestion. - * @param tuningJobDefinition Job definition - * @return Tuning Job Execution with best parameters - */ - private TuningJobExecution cloneBestParamSetTuningJobExecution(TuningJobDefinition tuningJobDefinition) { - logger.info("Searching for best param set for job: " + tuningJobDefinition.job.jobName); - - TuningJobExecution bestParamSetTuningJobExecution = - getBestParamSetTuningJobExecution(tuningJobDefinition.job.jobDefId); - List jobSuggestedParamValueList = - getJobExecutionParamSet(bestParamSetTuningJobExecution.jobExecution.id); - - TuningJobExecution tuningJobExecution = new TuningJobExecution(); - JobExecution jobExecution = new JobExecution(); - jobExecution.id = 0L; - jobExecution.job = bestParamSetTuningJobExecution.jobExecution.job; - jobExecution.executionState = ExecutionState.NOT_STARTED; - jobExecution.save(); - - tuningJobExecution.jobExecution = jobExecution; - tuningJobExecution.isDefaultExecution = bestParamSetTuningJobExecution.isDefaultExecution; - tuningJobExecution.tuningAlgorithm = bestParamSetTuningJobExecution.tuningAlgorithm; - tuningJobExecution.paramSetState = ParamSetStatus.CREATED; - tuningJobExecution.save(); - - logger.debug("Execution with default parameter created with execution id: " + tuningJobExecution.jobExecution.id); - - //Save default parameters corresponding to new default execution - for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValueList) { - JobSuggestedParamValue jobSuggestedParamValue1 = new JobSuggestedParamValue(); - jobSuggestedParamValue1.id = 0; - jobSuggestedParamValue1.jobExecution = jobExecution; - jobSuggestedParamValue1.paramValue = jobSuggestedParamValue.paramValue; - jobSuggestedParamValue1.tuningParameter = jobSuggestedParamValue.tuningParameter; - jobSuggestedParamValue1.save(); - } - - tuningJobExecution = TuningJobExecution.find.select("*") - .where() - .eq(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.id, tuningJobExecution.jobExecution.id) - .setMaxRows(1) - .findUnique(); - - return tuningJobExecution; + return jobSuggestedParamValues; } /** @@ -158,27 +114,39 @@ private void setMaxAllowedMetricIncreasePercentage(TuningInput tuningInput) { * Sets the tuning algorithm based on the job type and optimization metric * @param tuningInput TuningInput for which tuning algorithm is to be set */ - private void setTuningAlgorithm(TuningInput tuningInput) { + private void setTuningAlgorithm(TuningInput tuningInput) throws IllegalArgumentException { //Todo: Handle algorithm version later TuningAlgorithm tuningAlgorithm = TuningAlgorithm.find.select("*") .where() .eq(TuningAlgorithm.TABLE.jobType, tuningInput.getJobType()) .eq(TuningAlgorithm.TABLE.optimizationMetric, tuningInput.getOptimizationMetric()) .findUnique(); + if (tuningAlgorithm == null) { + throw new IllegalArgumentException( + "Wrong job type or optimization metric. Job Type " + tuningInput.getJobType() + ". Optimization Metrics: " + + tuningInput.getOptimizationMetric()); + } tuningInput.setTuningAlgorithm(tuningAlgorithm); } /** - * Applies penalty to the given execution - * @param jobExecId String jobExecId of the execution to which penalty has to be applied + * Applies penalty to the param set corresponding to the given execution + * @param jobExecId String job execution id/url of the execution whose parameter set has to be penalized + * Assumption: Best param set will never be penalized */ private void applyPenalty(String jobExecId) { Integer penaltyConstant = 3; logger.info("Execution " + jobExecId + " failed/cancelled. Applying penalty"); - TuningJobExecution tuningJobExecution = TuningJobExecution.find.where() - .eq(TuningJobExecution.TABLE.jobExecution + '.' + JobExecution.TABLE.jobExecId, jobExecId) + + TuningJobExecutionParamSet tuningJobExecutionParamSet = TuningJobExecutionParamSet.find.where() + .eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.jobExecId, jobExecId) + .setMaxRows(1) .findUnique(); - JobDefinition jobDefinition = tuningJobExecution.jobExecution.job; + + JobSuggestedParamSet jobSuggestedParamSet = tuningJobExecutionParamSet.jobSuggestedParamSet; + JobExecution jobExecution = tuningJobExecutionParamSet.jobExecution; + JobDefinition jobDefinition = jobExecution.job; + TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.where() .eq(TuningJobDefinition.TABLE.job + '.' + JobDefinition.TABLE.id, jobDefinition.id) .findUnique(); @@ -186,11 +154,12 @@ private void applyPenalty(String jobExecId) { tuningJobDefinition.averageResourceUsage * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; Double maxDesiredResourceUsagePerGBInput = averageResourceUsagePerGBInput * tuningJobDefinition.allowedMaxResourceUsagePercent / 100.0; - tuningJobExecution.fitness = penaltyConstant * maxDesiredResourceUsagePerGBInput; - tuningJobExecution.paramSetState = ParamSetStatus.FITNESS_COMPUTED; - tuningJobExecution.update(); - JobExecution jobExecution = tuningJobExecution.jobExecution; + jobSuggestedParamSet.fitness = penaltyConstant * maxDesiredResourceUsagePerGBInput; + jobSuggestedParamSet.paramSetState = ParamSetStatus.FITNESS_COMPUTED; + jobSuggestedParamSet.fitnessJobExecution = jobExecution; + jobSuggestedParamSet.update(); + jobExecution.resourceUsage = 0D; jobExecution.executionTime = 0D; jobExecution.inputSizeInBytes = 1D; @@ -198,128 +167,30 @@ private void applyPenalty(String jobExecId) { } /** - * Handles the api request and returns param suggestions as response - * @param tuningInput Rest api parameters - * @return Parameter Suggestion + * Returns flow definition corresponding to the given tuning input if it exists, else creates one and returns it + * @param tuningInput TuningInput containing the flow definition id corresponding to which flow definition + * is to be returned + * @return FlowDefinition flow definition */ - public Map getCurrentRunParameters(TuningInput tuningInput) { - logger.info("Parameter suggestion request for execution: " + tuningInput.getJobExecId()); - List jobSuggestedParamValues; - - if (tuningInput.getAllowedMaxExecutionTimePercent() == null - || tuningInput.getAllowedMaxResourceUsagePercent() == null) { - setMaxAllowedMetricIncreasePercentage(tuningInput); - } - setTuningAlgorithm(tuningInput); - String jobDefId = tuningInput.getJobDefId(); - - if (tuningInput.getRetry()) { - applyPenalty(tuningInput.getJobExecId()); - TuningJobExecution bestParamSetTuningJobExecution = getBestParamSetTuningJobExecution(jobDefId); - jobSuggestedParamValues = getJobExecutionParamSet(bestParamSetTuningJobExecution.jobExecution.id); - } else { - boolean isJobNewToTuning = false; - boolean isTuningEnabledForJob; - - TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.select("*") - .fetch(TuningJobDefinition.TABLE.job, "*") - .where() - .eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.jobDefId, jobDefId) - .eq(TuningJobDefinition.TABLE.tuningEnabled, 1) - .findUnique(); - - isTuningEnabledForJob = tuningJobDefinition != null; - - if (!isTuningEnabledForJob) { - //Tuning not enabled for the job currently. Either the job is new to tuning or tuning has been turned off for the job - //TuningJobDefinition will have a unique entry for every time a job is turned on for tuning - tuningJobDefinition = TuningJobDefinition.find.select("*") - .fetch(TuningJobDefinition.TABLE.job, "*") - .where() - .eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.jobDefId, jobDefId) - .setMaxRows(1) - .orderBy(TuningJobDefinition.TABLE.createdTs + " desc") - .findUnique(); - - isJobNewToTuning = tuningJobDefinition == null; - - if (isJobNewToTuning) { - //The job is new to tuning - logger.debug("Registering job: " + tuningInput.getJobName() + " for auto tuning tuning"); - AutoTuningMetricsController.markNewAutoTuningJob(); - tuningJobDefinition = addNewJobForTuning(tuningInput); - } - } - - TuningJobExecution tuningJobExecution; - if (isJobNewToTuning || isTuningEnabledForJob) { - logger.debug("Finding parameter suggestion for job: " + tuningJobDefinition.job.jobName); - tuningJobExecution = getNewTuningJobExecution(tuningJobDefinition); - } else { - //Tuning has been switched off for the job. Returning best param set - tuningJobExecution = cloneBestParamSetTuningJobExecution(tuningJobDefinition); - } - updateJobExecutionParameter(tuningJobExecution, tuningInput); - jobSuggestedParamValues = getJobExecutionParamSet(tuningJobExecution.jobExecution.id); - } - logger.debug("Number of output parameters : " + jobSuggestedParamValues.size()); - logger.info("Finishing getCurrentRunParameters"); - return jobSuggestedParamValueListToMap(jobSuggestedParamValues); - } - - /** - * Returns an execution with unsent parameters corresponding to the given job definition - * @param tuningJobDefinition TuningJobDefinition corresponding to which execution is to be returned - * @return TuningJobExecution corresponding to the given job definition - */ - private TuningJobExecution getNewTuningJobExecution(TuningJobDefinition tuningJobDefinition) { - TuningJobExecution tuningJobExecution = TuningJobExecution.find.select("*") - .fetch(TuningJobExecution.TABLE.jobExecution, "*") - .fetch(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.job, "*") - .where() - .eq(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.job + "." + JobDefinition.TABLE.id, - tuningJobDefinition.job.id) - .eq(TuningJobExecution.TABLE.paramSetState, ParamSetStatus.CREATED) - .order() - .asc(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.createdTs) - .setMaxRows(1) - .findUnique(); - - //If no new parameter set for suggestion, create a new suggestion with best parameter set - if (tuningJobExecution == null) { - logger.info( - "Returning best parameter set as no parameter suggestion found for job: " + tuningJobDefinition.job.jobName); - AutoTuningMetricsController.markParamSetNotFound(); - tuningJobExecution = cloneBestParamSetTuningJobExecution(tuningJobDefinition); - } - return tuningJobExecution; - } - - /** - * Returns the list of JobSuggestedParamValue as Map of String to Double - * @param jobSuggestedParamValues List of JobSuggestedParamValue - * @return Map of string to double containing the parameter name and corresponding value - */ - private Map jobSuggestedParamValueListToMap(List jobSuggestedParamValues) { - Map paramValues = new HashMap(); - if (jobSuggestedParamValues != null) { - for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValues) { - logger.debug("Param Name is " + jobSuggestedParamValue.tuningParameter.paramName + " And value is " - + jobSuggestedParamValue.paramValue); - paramValues.put(jobSuggestedParamValue.tuningParameter.paramName, jobSuggestedParamValue.paramValue); - } + private FlowDefinition getFlowDefinition(TuningInput tuningInput) { + FlowDefinition flowDefinition = + FlowDefinition.find.where().eq(FlowDefinition.TABLE.flowDefId, tuningInput.getFlowDefId()).findUnique(); + if (flowDefinition == null) { + flowDefinition = new FlowDefinition(); + flowDefinition.flowDefId = tuningInput.getFlowDefId(); + flowDefinition.flowDefUrl = tuningInput.getFlowDefUrl(); + flowDefinition.save(); } - return paramValues; + return flowDefinition; } /** - *This is to update job execution with IN_PROGRESS and parameter set with IN_PROGRESS. Also update flow_exec_id - *, flowExecURL, JobExecID and jobExecURL - * @param tuningJobExecution TuningJobExecution which is to be updated - * @param tuningInput TuningInput corresponding to the TuningJobExecution + * Returns flow execution corresponding to the given tuning input if it exists, else creates one and returns it + * @param tuningInput TuningInput containing the flow execution id corresponding to which flow execution + * is to be returned + * @return FlowExecution flow execution */ - private void updateJobExecutionParameter(TuningJobExecution tuningJobExecution, TuningInput tuningInput) { - + private FlowExecution getFlowExecution(TuningInput tuningInput) { FlowExecution flowExecution = FlowExecution.find.where().eq(FlowExecution.TABLE.flowExecId, tuningInput.getFlowExecId()).findUnique(); @@ -327,47 +198,23 @@ private void updateJobExecutionParameter(TuningJobExecution tuningJobExecution, flowExecution = new FlowExecution(); flowExecution.flowExecId = tuningInput.getFlowExecId(); flowExecution.flowExecUrl = tuningInput.getFlowExecUrl(); - flowExecution.flowDefinition = tuningJobExecution.jobExecution.job.flowDefinition; + flowExecution.flowDefinition = getFlowDefinition(tuningInput); flowExecution.save(); } - - JobExecution jobExecution = tuningJobExecution.jobExecution; - jobExecution.jobExecId = tuningInput.getJobExecId(); - jobExecution.jobExecUrl = tuningInput.getJobExecUrl(); - jobExecution.executionState = ExecutionState.IN_PROGRESS; - jobExecution.flowExecution = flowExecution; - - logger.debug("Saving job execution" + jobExecution.jobExecId); - - jobExecution.save(); - - tuningJobExecution.jobExecution = jobExecution; - tuningJobExecution.paramSetState = ParamSetStatus.SENT; - tuningJobExecution.save(); + return flowExecution; } /** - * Add new job for tuning + * Adds new job for tuning * @param tuningInput Tuning input parameters * @return Job */ private TuningJobDefinition addNewJobForTuning(TuningInput tuningInput) { - logger.info("Adding new job for tuning, job id: " + tuningInput.getJobDefId()); - + FlowDefinition flowDefinition = getFlowDefinition(tuningInput); JobDefinition job = JobDefinition.find.select("*").where().eq(JobDefinition.TABLE.jobDefId, tuningInput.getJobDefId()).findUnique(); - FlowDefinition flowDefinition = - FlowDefinition.find.where().eq(FlowDefinition.TABLE.flowDefId, tuningInput.getFlowDefId()).findUnique(); - - if (flowDefinition == null) { - flowDefinition = new FlowDefinition(); - flowDefinition.flowDefId = tuningInput.getFlowDefId(); - flowDefinition.flowDefUrl = tuningInput.getFlowDefUrl(); - flowDefinition.save(); - } - if (job == null) { job = new JobDefinition(); job.jobDefId = tuningInput.getJobDefId(); @@ -379,99 +226,231 @@ private TuningJobDefinition addNewJobForTuning(TuningInput tuningInput) { job.save(); } - String flowExecId = tuningInput.getFlowExecId(); - String jobExecId = tuningInput.getJobExecId(); - String flowExecUrl = tuningInput.getFlowExecUrl(); - String jobExecUrl = tuningInput.getJobExecUrl(); String client = tuningInput.getClient(); - String defaultParams = tuningInput.getDefaultParams(); - + Map defaultParams = null; + try { + defaultParams = tuningInput.getDefaultParams(); + } catch (IOException e) { + logger.error("Error in getting default parameters from request. ", e); + } TuningJobDefinition tuningJobDefinition = new TuningJobDefinition(); tuningJobDefinition.job = job; tuningJobDefinition.client = client; tuningJobDefinition.tuningAlgorithm = tuningInput.getTuningAlgorithm(); - tuningJobDefinition.tuningEnabled = 1; + tuningJobDefinition.tuningEnabled = true; tuningJobDefinition.allowedMaxExecutionTimePercent = tuningInput.getAllowedMaxExecutionTimePercent(); tuningJobDefinition.allowedMaxResourceUsagePercent = tuningInput.getAllowedMaxResourceUsagePercent(); tuningJobDefinition.save(); - TuningJobExecution tuningJobExecution = - insertDefaultJobExecution(job, flowExecId, jobExecId, flowExecUrl, jobExecUrl, flowDefinition, - tuningInput.getTuningAlgorithm()); - insertDefaultParameters(tuningJobExecution, defaultParams); + insertParamSet(job, tuningInput.getTuningAlgorithm(), defaultParams); logger.info("Added job: " + tuningInput.getJobDefId() + " for tuning"); return tuningJobDefinition; } /** - * Inserts default job execution in database - * @param job Job - * @param flowExecId Flow execution id - * @param jobExecId Job execution id - * @param flowExecUrl Flow execution url - * @param jobExecUrl Job execution url - * @return default job execution + * Returns the job definition corresponding to the given tuning input if it exists, else creates one and returns it + * @param tuningInput Tuning Input corresponding to which job definition is to be returned + * @return JobDefinition corresponding to the given tuning input */ - private TuningJobExecution insertDefaultJobExecution(JobDefinition job, String flowExecId, String jobExecId, - String flowExecUrl, String jobExecUrl, FlowDefinition flowDefinition, TuningAlgorithm tuningAlgorithm) { - logger.debug("Starting insertDefaultJobExecution"); + private JobDefinition getJobDefinition(TuningInput tuningInput) { - FlowExecution flowExecution = - FlowExecution.find.where().eq(FlowExecution.TABLE.flowExecId, flowExecId).findUnique(); + String jobDefId = tuningInput.getJobDefId(); - if (flowExecution == null) { - flowExecution = new FlowExecution(); - flowExecution.flowExecId = flowExecId; - flowExecution.flowExecUrl = flowExecUrl; - flowExecution.flowDefinition = flowDefinition; - flowExecution.save(); + TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.select("*") + .fetch(TuningJobDefinition.TABLE.job, "*") + .where() + .eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.jobDefId, jobDefId) + .setMaxRows(1) + .orderBy(TuningJobDefinition.TABLE.createdTs + " desc") + .findUnique(); + + if (tuningJobDefinition == null) { + // Job new to tuning + logger.debug("Registering job: " + tuningInput.getJobName() + " for auto tuning tuning"); + AutoTuningMetricsController.markNewAutoTuningJob(); + tuningJobDefinition = addNewJobForTuning(tuningInput); } + return tuningJobDefinition.job; + } + + /** + * Creates a new job execution entry corresponding to the given tuning input + * @param tuningInput Input corresponding to which job execution is to be created + * @return JobExecution: the newly created job execution + */ + private JobExecution addNewExecution(TuningInput tuningInput) { + JobDefinition jobDefinition = getJobDefinition(tuningInput); + FlowExecution flowExecution = getFlowExecution(tuningInput); + + JobExecution jobExecution = new JobExecution(); + jobExecution.jobExecId = tuningInput.getJobExecId(); + jobExecution.jobExecUrl = tuningInput.getJobExecUrl(); + jobExecution.job = jobDefinition; + jobExecution.executionState = ExecutionState.IN_PROGRESS; + jobExecution.flowExecution = flowExecution; + jobExecution.save(); + return jobExecution; + } - JobExecution jobExecution = JobExecution.find.where().eq(JobExecution.TABLE.jobExecId, jobExecId).findUnique(); + /** + * Returns the job execution corresponding to the given tuning input if it exists, else creates one and returns it + * @param tuningInput Tuning Input corresponding to which job execution is to be returned + * @return JobExecution corresponding to the given tuning input + */ + private JobExecution getJobExecution(TuningInput tuningInput) { + + JobExecution jobExecution = JobExecution.find.select("*") + .fetch(JobExecution.TABLE.job, "*") + .where() + .eq(JobExecution.TABLE.jobExecId, tuningInput.getJobExecId()) + .findUnique(); if (jobExecution == null) { - jobExecution = new JobExecution(); - jobExecution.job = job; - jobExecution.executionState = ExecutionState.NOT_STARTED; - jobExecution.jobExecId = jobExecId; - jobExecution.jobExecUrl = jobExecUrl; - jobExecution.flowExecution = flowExecution; - jobExecution.save(); + jobExecution = addNewExecution(tuningInput); + } + return jobExecution; + } + + /** + * Handles the api request and returns param suggestions as response + * @param tuningInput Rest api parameters + * @return Parameter Suggestion + */ + public Map getCurrentRunParameters(TuningInput tuningInput) throws Exception { + logger.info("Parameter set request received from execution: " + tuningInput.getJobExecId()); + + if (tuningInput.getAllowedMaxExecutionTimePercent() == null + || tuningInput.getAllowedMaxResourceUsagePercent() == null) { + setMaxAllowedMetricIncreasePercentage(tuningInput); } + setTuningAlgorithm(tuningInput); - TuningJobExecution tuningJobExecution = new TuningJobExecution(); - tuningJobExecution.jobExecution = jobExecution; - tuningJobExecution.tuningAlgorithm = tuningAlgorithm; - tuningJobExecution.paramSetState = ParamSetStatus.CREATED; - tuningJobExecution.isDefaultExecution = true; - tuningJobExecution.save(); + JobSuggestedParamSet jobSuggestedParamSet; + JobExecution jobExecution = getJobExecution(tuningInput); - logger.debug("Finishing insertDefaultJobExecution. Job Execution ID " + jobExecution.jobExecId); + if (tuningInput.getRetry()) { + applyPenalty(tuningInput.getJobExecId()); + jobSuggestedParamSet = getBestParamSet(tuningInput.getJobDefId()); + } else { + logger.debug("Finding parameter suggestion for job: " + jobExecution.job.jobName); + jobSuggestedParamSet = getNewSuggestedParamSet(jobExecution.job); + markParameterSetSent(jobSuggestedParamSet); + } + + addNewTuningJobExecutionParamSet(jobSuggestedParamSet, jobExecution); - return tuningJobExecution; + List jobSuggestedParamValues = getParamSetValues(jobSuggestedParamSet.id); + logger.debug("Number of output parameters for execution " + tuningInput.getJobExecId() + " = " + + jobSuggestedParamValues.size()); + logger.info("Finishing getCurrentRunParameters"); + return jobSuggestedParamValueListToMap(jobSuggestedParamValues); } /** - * Inserts default execution parameters in database - * @param tuningJobExecution Tuning Job Execution - * @param defaultParams Default parameters map as string + * Adds a new entry to the "tuning_job_execution_param_set" for the given param set and job execution + * @param jobSuggestedParamSet JobSuggestedParamSet: param set + * @param jobExecution JobExecution */ - @SuppressWarnings("unchecked") - private void insertDefaultParameters(TuningJobExecution tuningJobExecution, String defaultParams) { - JobExecution jobExecution = tuningJobExecution.jobExecution; - TuningAlgorithm.JobType jobType = tuningJobExecution.tuningAlgorithm.jobType; + private void addNewTuningJobExecutionParamSet(JobSuggestedParamSet jobSuggestedParamSet, JobExecution jobExecution) { + TuningJobExecutionParamSet tuningJobExecutionParamSet = new TuningJobExecutionParamSet(); + tuningJobExecutionParamSet.jobSuggestedParamSet = jobSuggestedParamSet; + tuningJobExecutionParamSet.jobExecution = jobExecution; - ObjectMapper mapper = new ObjectMapper(); - Map paramValueMap = null; - try { - paramValueMap = (Map) mapper.readValue(defaultParams, Map.class); - } catch (Exception e) { - logger.error(e); + TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.where() + .eq(TuningJobDefinition.TABLE.job + '.' + JobDefinition.TABLE.id, jobExecution.job.id) + .order() + .desc(TuningJobDefinition.TABLE.createdTs) + .setMaxRows(1) + .findUnique(); + tuningJobExecutionParamSet.tuningEnabled = tuningJobDefinition.tuningEnabled; + tuningJobExecutionParamSet.save(); + } + + /** + * Returns a parameter set in "CREATED" state corresponding to the given job definition if it exists, else returns + * the best parameter set + * @param jobDefinition jobDefinition for which param set is to be returned + * @return JobSuggestedParamSet corresponding to the given job definition + */ + private JobSuggestedParamSet getNewSuggestedParamSet(JobDefinition jobDefinition) { + JobSuggestedParamSet jobSuggestedParamSet = JobSuggestedParamSet.find.select("*") + .fetch(JobSuggestedParamSet.TABLE.jobDefinition, "*") + .where() + .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.id, jobDefinition.id) + .eq(JobSuggestedParamSet.TABLE.paramSetState, ParamSetStatus.CREATED) + .order() + .asc(JobSuggestedParamSet.TABLE.id) + .setMaxRows(1) + .findUnique(); + + if (jobSuggestedParamSet == null) { + //No new parameter set exists, returning the best parameter set + logger.info("Returning best parameter set as no parameter suggestion found for job: " + jobDefinition.jobName); + AutoTuningMetricsController.markParamSetNotFound(); + jobSuggestedParamSet = getBestParamSet(jobDefinition.jobDefId); } + return jobSuggestedParamSet; + } + + /** + * Returns the list of JobSuggestedParamValue as Map of String to Double + * @param jobSuggestedParamValues List of JobSuggestedParamValue + * @return Map of string to double containing the parameter name and corresponding value + */ + private Map jobSuggestedParamValueListToMap(List jobSuggestedParamValues) { + Map paramValues = new HashMap(); + if (jobSuggestedParamValues != null) { + for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValues) { + logger.debug("Param Name is " + jobSuggestedParamValue.tuningParameter.paramName + " And value is " + + jobSuggestedParamValue.paramValue); + paramValues.put(jobSuggestedParamValue.tuningParameter.paramName, jobSuggestedParamValue.paramValue); + } + } + return paramValues; + } + + /** + *Updates parameter set state to SENT if it is in CREATED state + * @param jobSuggestedParamSet JobSuggestedParamSet which is to be updated + */ + private void markParameterSetSent(JobSuggestedParamSet jobSuggestedParamSet) { + if (jobSuggestedParamSet.paramSetState.equals(ParamSetStatus.CREATED)) { + logger.info("Marking paramSetID: " + jobSuggestedParamSet.id + " SENT"); + jobSuggestedParamSet.paramSetState = ParamSetStatus.SENT; + jobSuggestedParamSet.save(); + } + } + + /** + * Inserts a parameter set in database + * @param job Job + */ + private void insertParamSet(JobDefinition job, TuningAlgorithm tuningAlgorithm, Map paramValueMap) { + logger.debug("Inserting default parameter set for job: " + job.jobName); + JobSuggestedParamSet jobSuggestedParamSet = new JobSuggestedParamSet(); + jobSuggestedParamSet.jobDefinition = job; + jobSuggestedParamSet.tuningAlgorithm = tuningAlgorithm; + jobSuggestedParamSet.paramSetState = ParamSetStatus.CREATED; + jobSuggestedParamSet.isParamSetDefault = true; + jobSuggestedParamSet.areConstraintsViolated = false; + jobSuggestedParamSet.isParamSetBest = false; + jobSuggestedParamSet.save(); + insertParameterValues(jobSuggestedParamSet, paramValueMap); + logger.debug("Default parameter set inserted for job: " + job.jobName); + } + + /** + * Inserts parameter values in database + * @param jobSuggestedParamSet Set of the parameters which is to be inserted + * @param paramValueMap Map of parameter values as string + */ + @SuppressWarnings("unchecked") + private void insertParameterValues(JobSuggestedParamSet jobSuggestedParamSet, Map paramValueMap) { + ObjectMapper mapper = new ObjectMapper(); if (paramValueMap != null) { for (Map.Entry paramValue : paramValueMap.entrySet()) { - insertExecutionParameter(jobExecution, paramValue.getKey(), paramValue.getValue()); + insertParameterValue(jobSuggestedParamSet, paramValue.getKey(), paramValue.getValue()); } } else { logger.warn("ParamValueMap is null "); @@ -479,24 +458,21 @@ private void insertDefaultParameters(TuningJobExecution tuningJobExecution, Stri } /** - * Inserts parameter of an execution in database - * @param jobExecution Job execution + * Inserts parameter value in database + * @param jobSuggestedParamSet Parameter set to which the parameter belongs * @param paramName Parameter name * @param paramValue Parameter value */ - private void insertExecutionParameter(JobExecution jobExecution, String paramName, Double paramValue) { - logger.debug("Starting insertExecutionParameter"); + private void insertParameterValue(JobSuggestedParamSet jobSuggestedParamSet, String paramName, Double paramValue) { + logger.debug("Starting insertParameterValue"); JobSuggestedParamValue jobSuggestedParamValue = new JobSuggestedParamValue(); - jobSuggestedParamValue.jobExecution = jobExecution; + jobSuggestedParamValue.jobSuggestedParamSet = jobSuggestedParamSet; TuningParameter tuningParameter = TuningParameter.find.where().eq(TuningParameter.TABLE.paramName, paramName).findUnique(); if (tuningParameter != null) { jobSuggestedParamValue.tuningParameter = tuningParameter; jobSuggestedParamValue.paramValue = paramValue; jobSuggestedParamValue.save(); - logger.debug( - "Finishing insertDefaultJobExecution. Job Execution ID. Param ID " + jobSuggestedParamValue.tuningParameter.id - + " Param Name: " + jobSuggestedParamValue.tuningParameter.paramName); } else { logger.warn("TuningAlgorithm param null " + paramName); } diff --git a/app/com/linkedin/drelephant/tuning/AzkabanJobCompleteDetector.java b/app/com/linkedin/drelephant/tuning/AzkabanJobCompleteDetector.java index 9e527791f..4bd0113cc 100644 --- a/app/com/linkedin/drelephant/tuning/AzkabanJobCompleteDetector.java +++ b/app/com/linkedin/drelephant/tuning/AzkabanJobCompleteDetector.java @@ -24,8 +24,9 @@ import java.util.Map; import models.JobExecution; import models.JobExecution.ExecutionState; -import models.TuningJobExecution; -import models.TuningJobExecution.ParamSetStatus; +import models.JobSuggestedParamSet; +import models.JobSuggestedParamSet.ParamSetStatus; +import models.TuningJobExecutionParamSet; import org.apache.log4j.Logger; @@ -39,26 +40,27 @@ public class AzkabanJobCompleteDetector extends JobCompleteDetector { private AzkabanJobStatusUtil _azkabanJobStatusUtil; public enum AzkabanJobStatus { - FAILED, CANCELLED, KILLED, SUCCEEDED + FAILED, CANCELLED, KILLED, SUCCEEDED, SKIPPED } /** * Returns the list of completed executions - * @param jobExecutions Started Execution list + * @param inProgressExecutionParamSet List of executions (with corresponding param set) in progress * @return List of completed executions - * @throws MalformedURLException - * @throws URISyntaxException + * @throws MalformedURLException MalformedURLException + * @throws URISyntaxException URISyntaxException */ - protected List getCompletedExecutions(List jobExecutions) + protected List getCompletedExecutions(List inProgressExecutionParamSet) throws MalformedURLException, URISyntaxException { logger.info("Fetching the list of executions completed since last iteration"); - List completedExecutions = new ArrayList(); + List completedExecutions = new ArrayList(); try { - for (TuningJobExecution tuningJobExecution : jobExecutions) { + for (TuningJobExecutionParamSet tuningJobExecutionParamSet : inProgressExecutionParamSet) { - JobExecution jobExecution = tuningJobExecution.jobExecution; + JobSuggestedParamSet jobSuggestedParamSet = tuningJobExecutionParamSet.jobSuggestedParamSet; + JobExecution jobExecution = tuningJobExecutionParamSet.jobExecution; - logger.info("Checking current status of started execution: " + tuningJobExecution.jobExecution.jobExecId); + logger.info("Checking current status of started execution: " + jobExecution.jobExecId); if (_azkabanJobStatusUtil == null) { logger.info("Initializing AzkabanJobStatusUtil"); @@ -72,23 +74,32 @@ protected List getCompletedExecutions(List tuningJobDefinitions) { */ private List getJobForBaselineComputation() { logger.info("Fetching jobs for which baseline metrics need to be computed"); - List tuningJobDefinitions = new ArrayList(); - try { - tuningJobDefinitions = - TuningJobDefinition.find.where().eq(TuningJobDefinition.TABLE.averageResourceUsage, null).findList(); - } catch (NullPointerException e) { - logger.info("There are no jobs for which baseline has to be computed", e); - } + List tuningJobDefinitions = + TuningJobDefinition.find.where().eq(TuningJobDefinition.TABLE.averageResourceUsage, null).findList(); return tuningJobDefinitions; } diff --git a/app/com/linkedin/drelephant/tuning/FitnessComputeUtil.java b/app/com/linkedin/drelephant/tuning/FitnessComputeUtil.java index 658ecbbee..bf6f64d66 100644 --- a/app/com/linkedin/drelephant/tuning/FitnessComputeUtil.java +++ b/app/com/linkedin/drelephant/tuning/FitnessComputeUtil.java @@ -34,11 +34,12 @@ import models.AppResult; import models.JobDefinition; import models.JobExecution; +import models.JobSuggestedParamSet; +import models.JobSuggestedParamSet.ParamSetStatus; import models.JobSuggestedParamValue; import models.TuningAlgorithm; import models.TuningJobDefinition; -import models.TuningJobExecution; -import models.TuningJobExecution.ParamSetStatus; +import models.TuningJobExecutionParamSet; import models.TuningParameter; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; @@ -48,20 +49,53 @@ /** * This class computes the fitness of the suggested parameters after the execution is complete. This uses * Dr Elephant's DB to compute the fitness. - * Fitness is : Resource Usage/Input Size in GB + * Fitness is : Resource Usage/(Input Size in GB) * In case there is failure or resource usage/execution time goes beyond configured limit, fitness is computed by * adding a penalty. */ public class FitnessComputeUtil { private static final Logger logger = Logger.getLogger(FitnessComputeUtil.class); private static final String FITNESS_COMPUTE_WAIT_INTERVAL = "fitness.compute.wait_interval.ms"; - private static final int MAX_TUNING_EXECUTIONS = 39; - private static final int MIN_TUNING_EXECUTIONS = 18; - private Long waitInterval; + private static final String IGNORE_EXECUTION_WAIT_INTERVAL = "ignore.execution.wait.interval.ms"; + private static final String MAX_TUNING_EXECUTIONS = "max.tuning.executions"; + private static final String MIN_TUNING_EXECUTIONS = "min.tuning.executions"; + private int maxTuningExecutions; + private int minTuningExecutions; + private Long fitnessComputeWaitInterval; + private Long ignoreExecutionWaitInterval; public FitnessComputeUtil() { Configuration configuration = ElephantContext.instance().getAutoTuningConf(); - waitInterval = Utils.getNonNegativeLong(configuration, FITNESS_COMPUTE_WAIT_INTERVAL, 5 * AutoTuner.ONE_MIN); + + // Time duration to wait for computing the fitness of a param set once the corresponding execution is completed + fitnessComputeWaitInterval = + Utils.getNonNegativeLong(configuration, FITNESS_COMPUTE_WAIT_INTERVAL, 5 * AutoTuner.ONE_MIN); + + // Time duration to wait for metrics (resource usage, execution time) of an execution to be computed before + // discarding it for fitness computation + ignoreExecutionWaitInterval = + Utils.getNonNegativeLong(configuration, IGNORE_EXECUTION_WAIT_INTERVAL, 2 * 60 * AutoTuner.ONE_MIN); + + // #executions after which tuning will stop even if parameters don't converge + maxTuningExecutions = + Utils.getNonNegativeInt(configuration, MAX_TUNING_EXECUTIONS, 39); + + // #executions before which tuning cannot stop even if parameters converge + minTuningExecutions = + Utils.getNonNegativeInt(configuration, MIN_TUNING_EXECUTIONS, 18); + } + + private boolean isTuningEnabled(Integer jobDefinitionId) { + TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.where() + .eq(TuningJobDefinition.TABLE.job + '.' + JobDefinition.TABLE.id, jobDefinitionId) + .order() + // There can be multiple entries in tuningJobDefinition if the job is switch on/off multiple times. + // The latest entry gives the information regarding whether tuning is enabled or not + .desc(TuningJobDefinition.TABLE.createdTs) + .setMaxRows(1) + .findUnique(); + + return tuningJobDefinition != null && tuningJobDefinition.tuningEnabled; } /** @@ -70,59 +104,69 @@ public FitnessComputeUtil() { */ public void updateFitness() { logger.info("Computing and updating fitness for completed executions"); - List completedExecutions = getCompletedExecutions(); - updateExecutionMetrics(completedExecutions); - updateMetrics(completedExecutions); + List completedJobExecutionParamSets = getCompletedJobExecutionParamSets(); + updateExecutionMetrics(completedJobExecutionParamSets); + updateMetrics(completedJobExecutionParamSets); Set jobDefinitionSet = new HashSet(); - for (TuningJobExecution tuningJobExecution : completedExecutions) { - jobDefinitionSet.add(tuningJobExecution.jobExecution.job); + for (TuningJobExecutionParamSet completedJobExecutionParamSet : completedJobExecutionParamSets) { + JobDefinition jobDefinition = completedJobExecutionParamSet.jobSuggestedParamSet.jobDefinition; + if (isTuningEnabled(jobDefinition.id)) { + jobDefinitionSet.add(jobDefinition); + } } checkToDisableTuning(jobDefinitionSet); } /** * Checks if the tuning parameters converge - * @param jobExecutions List of previous executions on which parameter convergence is to be checked + * @param tuningJobExecutionParamSets List of previous executions and corresponding param sets * @return true if the parameters converge, else false */ - private boolean doesParameterSetConverge(List jobExecutions) { + private boolean didParameterSetConverge(List tuningJobExecutionParamSets) { boolean result = false; - int num_param_set_for_convergence = 3; + int numParamSetForConvergence = 3; - TuningJobExecution tuningJobExecution = TuningJobExecution.find.where() - .eq(TuningJobExecution.TABLE.jobExecution + '.' + JobExecution.TABLE.id, jobExecutions.get(0).id) - .findUnique(); - TuningAlgorithm.JobType jobType = tuningJobExecution.tuningAlgorithm.jobType; + if (tuningJobExecutionParamSets.size() < numParamSetForConvergence) { + return false; + } + + TuningAlgorithm.JobType jobType = tuningJobExecutionParamSets.get(0).jobSuggestedParamSet.tuningAlgorithm.jobType; if (jobType == TuningAlgorithm.JobType.PIG) { + Map> paramValueSet = new HashMap>(); - for (JobExecution jobExecution : jobExecutions) { - List jobSuggestedParamValueList = new ArrayList(); - try { - jobSuggestedParamValueList = JobSuggestedParamValue.find.where() - .eq(JobSuggestedParamValue.TABLE.jobExecution + '.' + JobExecution.TABLE.id, jobExecution.id) - .or(Expr.eq(JobSuggestedParamValue.TABLE.tuningParameter + '.' + TuningParameter.TABLE.id, 2), - Expr.eq(JobSuggestedParamValue.TABLE.tuningParameter + '.' + TuningParameter.TABLE.id, 5)) - .findList(); - } catch (NullPointerException e) { - logger.info("Checking param convergence: Map memory and reduce memory parameter not found"); - } - if (jobSuggestedParamValueList.size() > 0) { - num_param_set_for_convergence -= 1; + + for (TuningJobExecutionParamSet tuningJobExecutionParamSet : tuningJobExecutionParamSets) { + + JobSuggestedParamSet jobSuggestedParamSet = tuningJobExecutionParamSet.jobSuggestedParamSet; + + List jobSuggestedParamValueList = JobSuggestedParamValue.find.where() + .eq(JobSuggestedParamValue.TABLE.jobSuggestedParamSet + '.' + JobSuggestedParamSet.TABLE.id, + jobSuggestedParamSet.id) + .or(Expr.eq(JobSuggestedParamValue.TABLE.tuningParameter + '.' + TuningParameter.TABLE.paramName, + "mapreduce.map.memory.mb"), + Expr.eq(JobSuggestedParamValue.TABLE.tuningParameter + '.' + TuningParameter.TABLE.paramName, + "mapreduce.reduce.memory.mb")) + .findList(); + + // if jobSuggestedParamValueList contains both mapreduce.map.memory.mb and mapreduce.reduce.memory.mb + // ie, if the size of jobSuggestedParamValueList is 2 + if (jobSuggestedParamValueList != null && jobSuggestedParamValueList.size() == 2) { + numParamSetForConvergence -= 1; for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValueList) { - Set tmp; + Set tmp; if (paramValueSet.containsKey(jobSuggestedParamValue.id)) { tmp = paramValueSet.get(jobSuggestedParamValue.id); } else { - tmp = new HashSet(); + tmp = new HashSet(); } tmp.add(jobSuggestedParamValue.paramValue); paramValueSet.put(jobSuggestedParamValue.id, tmp); } } - if (num_param_set_for_convergence == 0) { + if (numParamSetForConvergence == 0) { break; } } @@ -136,8 +180,8 @@ private boolean doesParameterSetConverge(List jobExecutions) { } if (result) { - logger.info( - "Switching off tuning for job: " + jobExecutions.get(0).job.jobName + " Reason: parameter set converged"); + logger.info("Switching off tuning for job: " + tuningJobExecutionParamSets.get( + 0).jobSuggestedParamSet.jobDefinition.jobName + " Reason: parameter set converged"); } return result; } @@ -147,22 +191,25 @@ private boolean doesParameterSetConverge(List jobExecutions) { * Last 6 executions constitutes 2 iterations of PSO (given the swarm size is three). Negative average gains in * latest 2 algorithm iterations (after a fixed number of minimum iterations) imply that either the algorithm hasn't * converged or there isn't enough scope for tuning. In both the cases, switching tuning off is desired - * @param jobExecutions List of previous executions + * @param tuningJobExecutionParamSets List of previous executions * @return true if the median gain is negative, else false */ - private boolean isMedianGainNegative(List jobExecutions) { - int num_fitness_for_median = 6; - Double[] fitnessArray = new Double[num_fitness_for_median]; + private boolean isMedianGainNegative(List tuningJobExecutionParamSets) { + int numFitnessForMedian = 6; + Double[] fitnessArray = new Double[numFitnessForMedian]; int entries = 0; - for (JobExecution jobExecution : jobExecutions) { - TuningJobExecution tuningJobExecution = TuningJobExecution.find.where() - .eq(TuningJobExecution.TABLE.jobExecution + '.' + JobExecution.TABLE.id, jobExecution.id) - .findUnique(); + + if (tuningJobExecutionParamSets.size() < numFitnessForMedian) { + return false; + } + for (TuningJobExecutionParamSet tuningJobExecutionParamSet : tuningJobExecutionParamSets) { + JobSuggestedParamSet jobSuggestedParamSet = tuningJobExecutionParamSet.jobSuggestedParamSet; + JobExecution jobExecution = tuningJobExecutionParamSet.jobExecution; if (jobExecution.executionState == JobExecution.ExecutionState.SUCCEEDED - && tuningJobExecution.paramSetState == ParamSetStatus.FITNESS_COMPUTED) { - fitnessArray[entries] = tuningJobExecution.fitness; + && jobSuggestedParamSet.paramSetState == ParamSetStatus.FITNESS_COMPUTED) { + fitnessArray[entries] = jobSuggestedParamSet.fitness; entries += 1; - if (entries == num_fitness_for_median) { + if (entries == numFitnessForMedian) { break; } } @@ -175,15 +222,14 @@ private boolean isMedianGainNegative(List jobExecutions) { medianFitness = fitnessArray[fitnessArray.length / 2]; } - JobDefinition jobDefinition = jobExecutions.get(0).job; + JobDefinition jobDefinition = tuningJobExecutionParamSets.get(0).jobSuggestedParamSet.jobDefinition; TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.where(). eq(TuningJobDefinition.TABLE.job + '.' + JobDefinition.TABLE.id, jobDefinition.id).findUnique(); double baselineFitness = tuningJobDefinition.averageResourceUsage * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; if (medianFitness > baselineFitness) { - logger.info( - "Switching off tuning for job: " + jobExecutions.get(0).job.jobName + " Reason: unable to tune enough"); + logger.info("Switching off tuning for job: " + jobDefinition.jobName + " Reason: unable to tune enough"); return true; } else { return false; @@ -198,9 +244,8 @@ private void disableTuning(JobDefinition jobDefinition, String reason) { TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.where() .eq(TuningJobDefinition.TABLE.job + '.' + JobDefinition.TABLE.id, jobDefinition.id) .findUnique(); - if (tuningJobDefinition.tuningEnabled == 1) { - logger.info("Disabling tuning for job: " + tuningJobDefinition.job.jobDefId); - tuningJobDefinition.tuningEnabled = 0; + if (tuningJobDefinition.tuningEnabled) { + tuningJobDefinition.tuningEnabled = false; tuningJobDefinition.tuningDisabledReason = reason; tuningJobDefinition.save(); } @@ -209,45 +254,46 @@ private void disableTuning(JobDefinition jobDefinition, String reason) { /** * Checks and disables tuning for the given job definitions. * Tuning can be disabled if: - * - Number of tuning executions >= MAX_TUNING_EXECUTIONS - * - or number of tuning executions >= MIN_TUNING_EXECUTIONS and parameters converge - * - or number of tuning executions >= MIN_TUNING_EXECUTIONS and median gain (in cost function) in last 6 executions is negative + * - Number of tuning executions >= maxTuningExecutions + * - or number of tuning executions >= minTuningExecutions and parameters converge + * - or number of tuning executions >= minTuningExecutions and median gain (in cost function) in last 6 executions is negative * @param jobDefinitionSet Set of jobs to check if tuning can be switched off for them */ private void checkToDisableTuning(Set jobDefinitionSet) { for (JobDefinition jobDefinition : jobDefinitionSet) { - try { - List jobExecutions = JobExecution.find.where() - .eq(JobExecution.TABLE.job + '.' + JobDefinition.TABLE.id, jobDefinition.id) - .isNotNull(JobExecution.TABLE.jobExecId) - .orderBy("id desc") - .findList(); - if (jobExecutions.size() >= MIN_TUNING_EXECUTIONS) { - if (doesParameterSetConverge(jobExecutions)) { + List tuningJobExecutionParamSets = + TuningJobExecutionParamSet.find.fetch(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet, "*") + .fetch(TuningJobExecutionParamSet.TABLE.jobExecution, "*") + .where() + .eq(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet + '.' + + JobSuggestedParamSet.TABLE.jobDefinition + '.' + JobDefinition.TABLE.id, jobDefinition.id) + .order() + .desc("job_execution_id") + .findList(); + + if (tuningJobExecutionParamSets.size() >= minTuningExecutions) { + if (didParameterSetConverge(tuningJobExecutionParamSets)) { logger.info("Parameters converged. Disabling tuning for job: " + jobDefinition.jobName); disableTuning(jobDefinition, "Parameters converged"); - } else if (isMedianGainNegative(jobExecutions)) { + } else if (isMedianGainNegative(tuningJobExecutionParamSets)) { logger.info("Unable to get gain while tuning. Disabling tuning for job: " + jobDefinition.jobName); disableTuning(jobDefinition, "Unable to get gain"); - } else if (jobExecutions.size() >= MAX_TUNING_EXECUTIONS) { + } else if (tuningJobExecutionParamSets.size() >= maxTuningExecutions) { logger.info("Maximum tuning executions limit reached. Disabling tuning for job: " + jobDefinition.jobName); disableTuning(jobDefinition, "Maximum executions reached"); } } - } catch (NullPointerException e) { - logger.info("No execution found for job: " + jobDefinition.jobName); - } } } /** * This method update metrics for auto tuning monitoring for fitness compute daemon - * @param completedExecutions List of completed tuning job executions + * @param completedJobExecutionParamSets List of completed tuning job executions */ - private void updateMetrics(List completedExecutions) { + private void updateMetrics(List completedJobExecutionParamSets) { int fitnessNotUpdated = 0; - for (TuningJobExecution tuningJobExecution : completedExecutions) { - if (!tuningJobExecution.paramSetState.equals(ParamSetStatus.FITNESS_COMPUTED)) { + for (TuningJobExecutionParamSet completedJobExecutionParamSet : completedJobExecutionParamSets) { + if (!completedJobExecutionParamSet.jobSuggestedParamSet.paramSetState.equals(ParamSetStatus.FITNESS_COMPUTED)) { fitnessNotUpdated++; } else { AutoTuningMetricsController.markFitnessComputedJobs(); @@ -260,57 +306,61 @@ private void updateMetrics(List completedExecutions) { * Returns the list of completed executions whose metrics are not computed * @return List of job execution */ - private List getCompletedExecutions() { + private List getCompletedJobExecutionParamSets() { logger.info("Fetching completed executions whose fitness are yet to be computed"); - List jobExecutions = new ArrayList(); - List outputJobExecutions = new ArrayList(); + List completedJobExecutionParamSet = new ArrayList(); - try { - jobExecutions = TuningJobExecution.find.select("*") + List tuningJobExecutionParamSets = TuningJobExecutionParamSet.find.select("*") + .fetch(TuningJobExecutionParamSet.TABLE.jobExecution, "*") + .fetch(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet, "*") .where() - .eq(TuningJobExecution.TABLE.paramSetState, ParamSetStatus.EXECUTED) + .or(Expr.or(Expr.eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.executionState, + JobExecution.ExecutionState.SUCCEEDED), + Expr.eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.executionState, + JobExecution.ExecutionState.FAILED)), + Expr.eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.executionState, + JobExecution.ExecutionState.CANCELLED)) + .isNull(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.resourceUsage) .findList(); - for (TuningJobExecution tuningJobExecution : jobExecutions) { - long diff = System.currentTimeMillis() - tuningJobExecution.jobExecution.updatedTs.getTime(); - logger.debug("Current Time in millis: " + System.currentTimeMillis() + ", Job execution last updated time " - + tuningJobExecution.jobExecution.updatedTs.getTime()); - if (diff < waitInterval) { - logger.debug("Delaying fitness compute for execution: " + tuningJobExecution.jobExecution.jobExecId); + logger.info("#completed executions whose metrics are not computed: " + tuningJobExecutionParamSets.size()); + + for (TuningJobExecutionParamSet tuningJobExecutionParamSet : tuningJobExecutionParamSets) { + JobExecution jobExecution = tuningJobExecutionParamSet.jobExecution; + long diff = System.currentTimeMillis() - jobExecution.updatedTs.getTime(); + logger.info("Current Time in millis: " + System.currentTimeMillis() + ", Job execution last updated time " + + jobExecution.updatedTs.getTime()); + if (diff < fitnessComputeWaitInterval) { + logger.info("Delaying fitness compute for execution: " + jobExecution.jobExecId); } else { - logger.debug("Adding execution " + tuningJobExecution.jobExecution.jobExecId + " for fitness computation"); - outputJobExecutions.add(tuningJobExecution); + logger.info("Adding execution " + jobExecution.jobExecId + " to fitness computation queue"); + completedJobExecutionParamSet.add(tuningJobExecutionParamSet); } } - } catch (NullPointerException e) { - logger.error("No completed execution found for which fitness is to be computed", e); - } - logger.info("Number of completed execution fetched for fitness computation: " + outputJobExecutions.size()); - logger.debug("Finished fetching completed executions for fitness computation"); - return outputJobExecutions; + logger.info( + "Number of completed execution fetched for fitness computation: " + completedJobExecutionParamSet.size()); + return completedJobExecutionParamSet; } /** * Updates the execution metrics - * @param completedExecutions List of completed executions + * @param completedJobExecutionParamSets List of completed executions */ - private void updateExecutionMetrics(List completedExecutions) { + private void updateExecutionMetrics(List completedJobExecutionParamSets) { + for (TuningJobExecutionParamSet completedJobExecutionParamSet : completedJobExecutionParamSets) { - //To artificially increase the cost function value 3 times (as a penalty) in case of metric value violation - Integer penaltyConstant = 3; + JobExecution jobExecution = completedJobExecutionParamSet.jobExecution; + JobSuggestedParamSet jobSuggestedParamSet = completedJobExecutionParamSet.jobSuggestedParamSet; + JobDefinition job = jobExecution.job; - for (TuningJobExecution tuningJobExecution : completedExecutions) { - logger.info("Updating execution metrics and fitness for execution: " + tuningJobExecution.jobExecution.jobExecId); + logger.info("Updating execution metrics and fitness for execution: " + jobExecution.jobExecId); try { - JobExecution jobExecution = tuningJobExecution.jobExecution; - JobDefinition job = jobExecution.job; - - // job id match and tuning enabled TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.select("*") .fetch(TuningJobDefinition.TABLE.job, "*") .where() .eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.id, job.id) - .eq(TuningJobDefinition.TABLE.tuningEnabled, 1) + .order() + .desc(TuningJobDefinition.TABLE.createdTs) .findUnique(); List results = AppResult.find.select("*") @@ -323,12 +373,9 @@ private void updateExecutionMetrics(List completedExecutions .findList(); if (results != null && results.size() > 0) { - Long totalExecutionTime = 0L; Double totalResourceUsed = 0D; Double totalInputBytesInBytes = 0D; - Map counterValuesMap = new HashMap(); - for (AppResult appResult : results) { totalResourceUsed += appResult.resourceUsed; totalInputBytesInBytes += getTotalInputBytes(appResult); @@ -336,13 +383,13 @@ private void updateExecutionMetrics(List completedExecutions Long totalRunTime = Utils.getTotalRuntime(results); Long totalDelay = Utils.getTotalWaittime(results); - totalExecutionTime = totalRunTime - totalDelay; + Long totalExecutionTime = totalRunTime - totalDelay; if (totalExecutionTime != 0) { jobExecution.executionTime = totalExecutionTime * 1.0 / (1000 * 60); jobExecution.resourceUsage = totalResourceUsed * 1.0 / (1024 * 3600); jobExecution.inputSizeInBytes = totalInputBytesInBytes; - + jobExecution.update(); logger.info( "Metric Values for execution " + jobExecution.jobExecId + ": Execution time = " + totalExecutionTime + ", Resource usage = " + totalResourceUsed + " and total input size = " + totalInputBytesInBytes); @@ -356,54 +403,112 @@ private void updateExecutionMetrics(List completedExecutions } //Compute fitness - Double averageResourceUsagePerGBInput = - tuningJobDefinition.averageResourceUsage * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; - Double maxDesiredResourceUsagePerGBInput = - averageResourceUsagePerGBInput * tuningJobDefinition.allowedMaxResourceUsagePercent / 100.0; - Double averageExecutionTimePerGBInput = - tuningJobDefinition.averageExecutionTime * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; - Double maxDesiredExecutionTimePerGBInput = - averageExecutionTimePerGBInput * tuningJobDefinition.allowedMaxExecutionTimePercent / 100.0; - Double resourceUsagePerGBInput = - jobExecution.resourceUsage * FileUtils.ONE_GB / jobExecution.inputSizeInBytes; - Double executionTimePerGBInput = - jobExecution.executionTime * FileUtils.ONE_GB / jobExecution.inputSizeInBytes; - - if (resourceUsagePerGBInput > maxDesiredResourceUsagePerGBInput - || executionTimePerGBInput > maxDesiredExecutionTimePerGBInput) { - logger.info("Execution " + jobExecution.jobExecId + " violates constraint on resource usage per GB input"); - tuningJobExecution.fitness = penaltyConstant * maxDesiredResourceUsagePerGBInput; - } else { - tuningJobExecution.fitness = resourceUsagePerGBInput; - } - tuningJobExecution.paramSetState = ParamSetStatus.FITNESS_COMPUTED; - jobExecution.update(); - tuningJobExecution.update(); - } - TuningJobExecution currentBestTuningJobExecution; - try { - currentBestTuningJobExecution = - TuningJobExecution.find.where().eq("jobExecution.job.id", tuningJobExecution.jobExecution.job.id). - eq(TuningJobExecution.TABLE.isParamSetBest, 1).findUnique(); - if (currentBestTuningJobExecution.fitness > tuningJobExecution.fitness) { - currentBestTuningJobExecution.isParamSetBest = false; - tuningJobExecution.isParamSetBest = true; - currentBestTuningJobExecution.save(); - tuningJobExecution.save(); + if (!jobSuggestedParamSet.paramSetState.equals(ParamSetStatus.FITNESS_COMPUTED)) { + if (jobExecution.executionState.equals(JobExecution.ExecutionState.SUCCEEDED)) { + logger.info("Execution id: " + jobExecution.id + " succeeded"); + updateJobSuggestedParamSetSucceededExecution(jobExecution, jobSuggestedParamSet, tuningJobDefinition); + } else { + // Resetting param set to created state because this case captures the scenarios when + // either the job failed for reasons other than auto tuning or was killed/cancelled/skipped etc. + // In all the above scenarios, fitness cannot be computed for the param set correctly. + // Note that the penalty on failures caused by auto tuning is applied when the job execution is retried + // after failure. + logger.info("Execution id: " + jobExecution.id + " was not successful for reason other than tuning." + + "Resetting param set: " + jobSuggestedParamSet.id + " to CREATED state"); + resetParamSetToCreated(jobSuggestedParamSet); + } + } + } else { + long diff = System.currentTimeMillis() - jobExecution.updatedTs.getTime(); + logger.debug("Current Time in millis: " + System.currentTimeMillis() + ", job execution last updated time " + + jobExecution.updatedTs.getTime()); + if (diff > ignoreExecutionWaitInterval) { + logger.info("Fitness of param set " + jobSuggestedParamSet.id + " corresponding to execution id: " + + jobExecution.id + " not computed for more than the maximum duration specified to compute fitness. " + + "Resetting the param set to CREATED state"); + resetParamSetToCreated(jobSuggestedParamSet); } - } catch (NullPointerException e) { - tuningJobExecution.isParamSetBest = true; - tuningJobExecution.save(); } } catch (Exception e) { - logger.error("Error updating fitness of execution: " + tuningJobExecution.jobExecution.id + "\n Stacktrace: ", - e); + logger.error("Error updating fitness of execution: " + jobExecution.id + "\n Stacktrace: ", e); } } logger.info("Execution metrics updated"); } + /** + * Resets the param set to CREATED state if its fitness is not already computed + * @param jobSuggestedParamSet Param set which is to be reset + */ + private void resetParamSetToCreated(JobSuggestedParamSet jobSuggestedParamSet) { + if (!jobSuggestedParamSet.paramSetState.equals(ParamSetStatus.FITNESS_COMPUTED)) { + logger.info("Resetting parameter set to created: " + jobSuggestedParamSet.id); + jobSuggestedParamSet.paramSetState = ParamSetStatus.CREATED; + jobSuggestedParamSet.save(); + } + } + + /** + * Updates the job suggested param set when the corresponding execution was succeeded + * @param jobExecution JobExecution: succeeded job execution corresponding to the param set which is to be updated + * @param jobSuggestedParamSet param set which is to be updated + * @param tuningJobDefinition TuningJobDefinition of the job to which param set corresponds + */ + private void updateJobSuggestedParamSetSucceededExecution(JobExecution jobExecution, + JobSuggestedParamSet jobSuggestedParamSet, TuningJobDefinition tuningJobDefinition) { + int penaltyConstant = 3; + Double averageResourceUsagePerGBInput = + tuningJobDefinition.averageResourceUsage * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; + Double maxDesiredResourceUsagePerGBInput = + averageResourceUsagePerGBInput * tuningJobDefinition.allowedMaxResourceUsagePercent / 100.0; + Double averageExecutionTimePerGBInput = + tuningJobDefinition.averageExecutionTime * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; + Double maxDesiredExecutionTimePerGBInput = + averageExecutionTimePerGBInput * tuningJobDefinition.allowedMaxExecutionTimePercent / 100.0; + Double resourceUsagePerGBInput = jobExecution.resourceUsage * FileUtils.ONE_GB / jobExecution.inputSizeInBytes; + Double executionTimePerGBInput = jobExecution.executionTime * FileUtils.ONE_GB / jobExecution.inputSizeInBytes; + + if (resourceUsagePerGBInput > maxDesiredResourceUsagePerGBInput + || executionTimePerGBInput > maxDesiredExecutionTimePerGBInput) { + logger.info("Execution " + jobExecution.jobExecId + " violates constraint on resource usage per GB input"); + jobSuggestedParamSet.fitness = penaltyConstant * maxDesiredResourceUsagePerGBInput; + } else { + jobSuggestedParamSet.fitness = resourceUsagePerGBInput; + } + jobSuggestedParamSet.paramSetState = ParamSetStatus.FITNESS_COMPUTED; + jobSuggestedParamSet.fitnessJobExecution = jobExecution; + jobSuggestedParamSet = updateBestJobSuggestedParamSet(jobSuggestedParamSet); + jobSuggestedParamSet.update(); + } + + /** + * Updates the given job suggested param set to be the best param set if its fitness is less than the current best param set + * (since the objective is to minimize the fitness, the param set with the lowest fitness is the best) + * @param jobSuggestedParamSet JobSuggestedParamSet + */ + private JobSuggestedParamSet updateBestJobSuggestedParamSet(JobSuggestedParamSet jobSuggestedParamSet) { + logger.info("Checking if a new best param set is found for job: " + jobSuggestedParamSet.jobDefinition.jobDefId); + JobSuggestedParamSet currentBestJobSuggestedParamSet = JobSuggestedParamSet.find.where() + .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.id, + jobSuggestedParamSet.jobDefinition.id) + .eq(JobSuggestedParamSet.TABLE.isParamSetBest, 1) + .findUnique(); + if (currentBestJobSuggestedParamSet != null) { + if (currentBestJobSuggestedParamSet.fitness > jobSuggestedParamSet.fitness) { + logger.info("Param set: " + jobSuggestedParamSet.id + " is the new best param set for job: " + jobSuggestedParamSet.jobDefinition.jobDefId); + currentBestJobSuggestedParamSet.isParamSetBest = false; + jobSuggestedParamSet.isParamSetBest = true; + currentBestJobSuggestedParamSet.save(); + } + } else { + logger.info("No best param set found for job: " + jobSuggestedParamSet.jobDefinition.jobDefId + + ". Marking current param set " + jobSuggestedParamSet.id + " as best"); + jobSuggestedParamSet.isParamSetBest = true; + } + return jobSuggestedParamSet; + } + /** * Returns the total input size * @param appResult appResult diff --git a/app/com/linkedin/drelephant/tuning/JobCompleteDetector.java b/app/com/linkedin/drelephant/tuning/JobCompleteDetector.java index 694cfb329..21b8a909b 100644 --- a/app/com/linkedin/drelephant/tuning/JobCompleteDetector.java +++ b/app/com/linkedin/drelephant/tuning/JobCompleteDetector.java @@ -16,21 +16,15 @@ package com.linkedin.drelephant.tuning; +import controllers.AutoTuningMetricsController; import java.net.MalformedURLException; import java.net.URISyntaxException; -import java.util.ArrayList; import java.util.List; - import models.JobExecution; import models.JobExecution.ExecutionState; -import models.TuningJobExecution; -import models.TuningJobExecution.ParamSetStatus; - +import models.TuningJobExecutionParamSet; import org.apache.log4j.Logger; -import controllers.AutoTuningMetricsController; -import play.libs.Json; - /** * This class pools the scheduler for completion status of execution and updates the database with current status @@ -41,77 +35,54 @@ public abstract class JobCompleteDetector { /** * Updates the status of completed executions - * @return List of completed executions * @throws MalformedURLException MalformedURLException * @throws URISyntaxException URISyntaxException */ - public List updateCompletedExecutions() throws MalformedURLException, URISyntaxException { - logger.info("Checking execution status"); - List runningExecutions = getStartedExecutions(); - List completedExecutions = getCompletedExecutions(runningExecutions); - updateExecutionStatus(completedExecutions); + public void updateCompletedExecutions() throws MalformedURLException, URISyntaxException { + logger.info("Updating execution status"); + List inProgressExecutionParamSet = getExecutionsInProgress(); + List completedExecutions = getCompletedExecutions(inProgressExecutionParamSet); updateMetrics(completedExecutions); logger.info("Finished updating execution status"); - return completedExecutions; } /** - * This method is for updating metrics for auto tuning monitoring for job completion daemon + * Updates metrics for auto tuning monitoring for job completion daemon * @param completedExecutions List completed job executions */ - private void updateMetrics(List completedExecutions) { - for (TuningJobExecution tuningJobExecution : completedExecutions) { - if (tuningJobExecution.paramSetState.equals(ParamSetStatus.EXECUTED)) { - if (tuningJobExecution.jobExecution.executionState.equals(ExecutionState.SUCCEEDED)) { - AutoTuningMetricsController.markSuccessfulJobs(); - } else if (tuningJobExecution.jobExecution.executionState.equals(ExecutionState.FAILED)) { - AutoTuningMetricsController.markFailedJobs(); - } + private void updateMetrics(List completedExecutions) { + for (JobExecution jobExecution : completedExecutions) { + if (jobExecution.executionState.equals(ExecutionState.SUCCEEDED)) { + AutoTuningMetricsController.markSuccessfulJobs(); + } else if (jobExecution.executionState.equals(ExecutionState.FAILED)) { + AutoTuningMetricsController.markFailedJobs(); } } } /** - * Returns the list of executions which have already received param suggestion + * Returns the executions in progress * @return JobExecution list */ - private List getStartedExecutions() { - logger.info("Fetching the executions which were running"); - List tuningJobExecutionList = new ArrayList(); - try { - tuningJobExecutionList = TuningJobExecution.find.select("*") - .where() - .eq(TuningJobExecution.TABLE.paramSetState, ParamSetStatus.SENT) - .findList(); - } catch (NullPointerException e) { - logger.info("None of the executions were running ", e); - } - logger.info("Number of executions which were in running state: " + tuningJobExecutionList.size()); - return tuningJobExecutionList; + private List getExecutionsInProgress() { + logger.info("Fetching the executions which are in progress"); + List tuningJobExecutionParamSets = TuningJobExecutionParamSet.find.fetch(TuningJobExecutionParamSet.TABLE.jobExecution) + .fetch(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet) + .where() + .eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.executionState, + ExecutionState.IN_PROGRESS) + .findList(); + logger.info("Number of executions which are in progress: " + tuningJobExecutionParamSets.size()); + return tuningJobExecutionParamSets; } /** * Returns the list of completed executions. - * @param jobExecutions Started Execution list + * @param inProgressExecutionParamSet List of executions (with corresponding param set) in progress * @return List of completed executions - * @throws MalformedURLException - * @throws URISyntaxException - */ - protected abstract List getCompletedExecutions(List jobExecutions) - throws MalformedURLException, URISyntaxException; - - /** - * Updates the job execution status - * @param jobExecutions JobExecution list - * @return Update status + * @throws MalformedURLException MalformedURLException + * @throws URISyntaxException URISyntaxException */ - private void updateExecutionStatus(List jobExecutions) { - logger.info("Updating status of executions completed since last iteration"); - for (TuningJobExecution tuningJobExecution : jobExecutions) { - JobExecution jobExecution = tuningJobExecution.jobExecution; - logger.info("Updating execution status to EXECUTED for the execution: " + jobExecution.jobExecId); - jobExecution.update(); - tuningJobExecution.update(); - } - } + protected abstract List getCompletedExecutions( + List inProgressExecutionParamSet) throws MalformedURLException, URISyntaxException; } diff --git a/app/com/linkedin/drelephant/tuning/JobTuningInfo.java b/app/com/linkedin/drelephant/tuning/JobTuningInfo.java index 3da56b6ab..36ce00f6a 100644 --- a/app/com/linkedin/drelephant/tuning/JobTuningInfo.java +++ b/app/com/linkedin/drelephant/tuning/JobTuningInfo.java @@ -18,9 +18,7 @@ import models.TuningParameter; import models.JobDefinition; - import java.util.List; - import models.TuningAlgorithm.JobType; /** diff --git a/app/com/linkedin/drelephant/tuning/PSOParamGenerator.java b/app/com/linkedin/drelephant/tuning/PSOParamGenerator.java index b553f8772..1d1651cad 100644 --- a/app/com/linkedin/drelephant/tuning/PSOParamGenerator.java +++ b/app/com/linkedin/drelephant/tuning/PSOParamGenerator.java @@ -18,17 +18,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.linkedin.drelephant.ElephantContext; - -import org.apache.hadoop.conf.Configuration; -import org.apache.log4j.Logger; - -import play.libs.Json; - import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.log4j.Logger; +import play.libs.Json; /** @@ -36,12 +33,11 @@ */ public class PSOParamGenerator extends ParamGenerator { - private final Logger logger = Logger.getLogger(PSOParamGenerator.class); private static final String PARAMS_TO_TUNE_FIELD_NAME = "parametersToTune"; private static final String PYTHON_PATH_CONF = "python.path"; private static final String PSO_DIR_PATH_ENV_VARIABLE = "PSO_DIR_PATH"; private static final String PYTHON_PATH_ENV_VARIABLE = "PYTHONPATH"; - + private final Logger logger = Logger.getLogger(PSOParamGenerator.class); private String PYTHON_PATH = null; private String TUNING_SCRIPT_PATH = null; @@ -78,9 +74,7 @@ public JobTuningInfo generateParamSet(JobTuningInfo jobTuningInfo) { newJobTuningInfo.setJobType(jobTuningInfo.getJobType()); JsonNode jsonJobTuningInfo = Json.toJson(jobTuningInfo); - logger.info("Job Tuning Info for " + jobTuningInfo.getTuningJob().jobName + ": " + jsonJobTuningInfo); String parametersToTune = jsonJobTuningInfo.get(PARAMS_TO_TUNE_FIELD_NAME).toString(); - logger.info("Parameters to tune for job: " + parametersToTune); String stringTunerState = jobTuningInfo.getTunerState(); stringTunerState = stringTunerState.replaceAll("\\s+", ""); String jobType = jobTuningInfo.getJobType().toString(); @@ -88,14 +82,12 @@ public JobTuningInfo generateParamSet(JobTuningInfo jobTuningInfo) { List error = new ArrayList(); try { - logger.info( - "Calling PSO with Job type = " + jobType + " StringTunerState= " + stringTunerState + "\nand Parameters to tune: " + parametersToTune); Process p = Runtime.getRuntime() - .exec(PYTHON_PATH + " " + TUNING_SCRIPT_PATH + " " + stringTunerState + " " + parametersToTune + " " + jobType); + .exec( + PYTHON_PATH + " " + TUNING_SCRIPT_PATH + " " + stringTunerState + " " + parametersToTune + " " + jobType); BufferedReader inputStream = new BufferedReader(new InputStreamReader(p.getInputStream())); BufferedReader errorStream = new BufferedReader(new InputStreamReader(p.getErrorStream())); String updatedStringTunerState = inputStream.readLine(); - logger.info("Output from PSO script: " + updatedStringTunerState); newJobTuningInfo.setTunerState(updatedStringTunerState); String errorLine; while ((errorLine = errorStream.readLine()) != null) { diff --git a/app/com/linkedin/drelephant/tuning/ParamGenerator.java b/app/com/linkedin/drelephant/tuning/ParamGenerator.java index 952a9e22b..4b3ea3508 100644 --- a/app/com/linkedin/drelephant/tuning/ParamGenerator.java +++ b/app/com/linkedin/drelephant/tuning/ParamGenerator.java @@ -17,37 +17,34 @@ package com.linkedin.drelephant.tuning; import com.avaje.ebean.Expr; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; - +import controllers.AutoTuningMetricsController; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; - -import models.*; - -import com.fasterxml.jackson.databind.JsonNode; - -import controllers.AutoTuningMetricsController; - +import models.JobDefinition; +import models.JobExecution; +import models.JobSavedState; +import models.JobSuggestedParamSet; +import models.JobSuggestedParamValue; +import models.TuningAlgorithm; +import models.TuningJobDefinition; +import models.TuningParameter; import org.apache.commons.io.FileUtils; import org.apache.log4j.Logger; - import play.libs.Json; -import java.util.ArrayList; -import java.util.List; - -import static java.lang.Math.*; - /** * This is an abstract class for generating parameter suggestions for jobs */ public abstract class ParamGenerator { - private final Logger logger = Logger.getLogger(getClass()); - private static final String JSON_CURRENT_POPULATION_KEY = "current_population"; + private final Logger logger = Logger.getLogger(getClass()); /** * Generates the parameters using tuningJobInfo and returns it in updated JobTuningInfo @@ -82,58 +79,45 @@ private List jsonToParticleList(JsonNode jsonParticleList) { * Fetches the list to job which need new parameter suggestion * @return Job list */ - private List fetchJobsForParamSuggestion() { - + private List getJobsForParamSuggestion() { // Todo: [Important] Change the logic. This is very rigid. Ideally you should look at the param set ids in the saved state, // todo: [continuation] if their fitness is computed, pso can generate new params for the job logger.info("Checking which jobs need new parameter suggestion"); List jobsForParamSuggestion = new ArrayList(); - List pendingParamExecutionList = new ArrayList(); - try { - pendingParamExecutionList = TuningJobExecution.find.select("*") - .fetch(TuningJobExecution.TABLE.jobExecution, "*") - .where() - .or(Expr.or(Expr.eq(TuningJobExecution.TABLE.paramSetState, TuningJobExecution.ParamSetStatus.CREATED), - Expr.eq(TuningJobExecution.TABLE.paramSetState, TuningJobExecution.ParamSetStatus.SENT)), - Expr.eq(TuningJobExecution.TABLE.paramSetState, TuningJobExecution.ParamSetStatus.EXECUTED)) - .eq(TuningJobExecution.TABLE.isDefaultExecution, 0) - .findList(); - } catch (NullPointerException e) { - logger.info("None of the non-default executions are in CREATED, SENT OR EXECUTED state"); - } + List pendingParamSetList = JobSuggestedParamSet.find.select("*") + .fetch(JobSuggestedParamSet.TABLE.jobDefinition, "*") + .where() + .or(Expr.or(Expr.eq(JobSuggestedParamSet.TABLE.paramSetState, JobSuggestedParamSet.ParamSetStatus.CREATED), + Expr.eq(JobSuggestedParamSet.TABLE.paramSetState, JobSuggestedParamSet.ParamSetStatus.SENT)), + Expr.eq(JobSuggestedParamSet.TABLE.paramSetState, JobSuggestedParamSet.ParamSetStatus.EXECUTED)) + .eq(JobSuggestedParamSet.TABLE.isParamSetDefault, 0) + .eq(JobSuggestedParamSet.TABLE.isParamSetBest, 0).findList(); List pendingParamJobList = new ArrayList(); - for (TuningJobExecution pendingParamExecution : pendingParamExecutionList) { - if (!pendingParamJobList.contains(pendingParamExecution.jobExecution.job)) { - pendingParamJobList.add(pendingParamExecution.jobExecution.job); + for (JobSuggestedParamSet pendingParamSet : pendingParamSetList) { + if (!pendingParamJobList.contains(pendingParamSet.jobDefinition)) { + pendingParamJobList.add(pendingParamSet.jobDefinition); } } - List tuningJobDefinitionList = new ArrayList(); + List tuningJobDefinitionList = TuningJobDefinition.find.select("*") + .fetch(TuningJobDefinition.TABLE.job, "*") + .where() + .eq(TuningJobDefinition.TABLE.tuningEnabled, 1) + .findList(); - try { - tuningJobDefinitionList = TuningJobDefinition.find.select("*") - .fetch(TuningJobDefinition.TABLE.job, "*") - .where() - .eq(TuningJobDefinition.TABLE.tuningEnabled, 1) - .findList(); - } catch (NullPointerException e) { + if (tuningJobDefinitionList.size() == 0) { logger.error("No auto-tuning enabled jobs found"); } for (TuningJobDefinition tuningJobDefinition : tuningJobDefinitionList) { if (!pendingParamJobList.contains(tuningJobDefinition.job)) { - jobsForParamSuggestion.add(tuningJobDefinition); - } - } - if (jobsForParamSuggestion.size() > 0) { - for (TuningJobDefinition tuningJobDefinition : jobsForParamSuggestion) { logger.info("New parameter suggestion needed for job: " + tuningJobDefinition.job.jobName); + jobsForParamSuggestion.add(tuningJobDefinition); } - } else { - logger.info("None of the jobs need new parameter suggestion"); } + logger.info("Number of job(s) which need new parameter suggestion: " + jobsForParamSuggestion.size()); return jobsForParamSuggestion; } @@ -171,19 +155,19 @@ private List getJobsTuningInfo(List tuningJo .eq(TuningParameter.TABLE.isDerived, 0) .findList(); - try { logger.info("Fetching default parameter values for job " + tuningJobDefinition.job.jobDefId); - TuningJobExecution defaultJobExecution = TuningJobExecution.find.where() - .eq(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.job + "." + JobDefinition.TABLE.id, - tuningJobDefinition.job.id) - .eq(TuningJobExecution.TABLE.isDefaultExecution, 1) - .orderBy(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.id + " desc") + JobSuggestedParamSet defaultJobParamSet = JobSuggestedParamSet.find.where() + .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.id, tuningJobDefinition.job.id) + .eq(JobSuggestedParamSet.TABLE.isParamSetDefault, 1) + .order() + .desc(JobSuggestedParamSet.TABLE.id) .setMaxRows(1) .findUnique(); - if (defaultJobExecution != null && defaultJobExecution.jobExecution != null) { + + if (defaultJobParamSet != null) { List jobSuggestedParamValueList = JobSuggestedParamValue.find.where() - .eq(JobSuggestedParamValue.TABLE.jobExecution + "." + JobExecution.TABLE.id, - defaultJobExecution.jobExecution.id) + .eq(JobSuggestedParamValue.TABLE.jobSuggestedParamSet + "." + JobExecution.TABLE.id, + defaultJobParamSet.id) .findList(); if (jobSuggestedParamValueList.size() > 0) { @@ -205,9 +189,6 @@ private List getJobsTuningInfo(List tuningJo } } } - } catch (NullPointerException e) { - logger.error("Error extracting default value of params for job " + tuningJobDefinition.job.jobDefId, e); - } JobTuningInfo jobTuningInfo = new JobTuningInfo(); jobTuningInfo.setTuningJob(job); @@ -225,16 +206,12 @@ private List getJobsTuningInfo(List tuningJo Long paramSetId = particle.getParamSetId(); logger.info("Param set id: " + paramSetId.toString()); - TuningJobExecution tuningJobExecution = TuningJobExecution.find.select("*") - .fetch(TuningJobExecution.TABLE.jobExecution, "*") - .where() - .eq(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.id, paramSetId) - .findUnique(); - - JobExecution jobExecution = tuningJobExecution.jobExecution; + JobSuggestedParamSet jobSuggestedParamSet = + JobSuggestedParamSet.find.select("*").where().eq(JobSuggestedParamSet.TABLE.id, paramSetId).findUnique(); - if (tuningJobExecution.fitness != null) { - particle.setFitness(tuningJobExecution.fitness); + if (jobSuggestedParamSet.paramSetState.equals(JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED) + && jobSuggestedParamSet.fitness != null) { + particle.setFitness(jobSuggestedParamSet.fitness); } else { validSavedState = false; logger.error("Invalid saved state: Fitness of previous execution not computed."); @@ -283,10 +260,8 @@ private List getParamValueList(Particle particle, List getParamValueList(Particle particle, List jobTuningInfoList) { JobDefinition job = jobTuningInfo.getTuningJob(); List paramList = jobTuningInfo.getParametersToTune(); String stringTunerState = jobTuningInfo.getTunerState(); - if (stringTunerState == null) { logger.error("Suggested parameter suggestion is empty for job id: " + job.jobDefId); continue; @@ -341,15 +315,12 @@ private void updateDatabase(List jobTuningInfoList) { .findUnique(); List derivedParameterList = new ArrayList(); - try { derivedParameterList = TuningParameter.find.where() .eq(TuningParameter.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.id, tuningJobDefinition.tuningAlgorithm.id) .eq(TuningParameter.TABLE.isDerived, 1) .findList(); - } catch (NullPointerException e) { - logger.info("No derived parameters for job: " + job.jobName); - } + logger.info("No. of derived tuning params for job " + tuningJobDefinition.job.jobName + ": " + derivedParameterList.size()); @@ -402,28 +373,30 @@ private void updateDatabase(List jobTuningInfoList) { } } - TuningJobExecution tuningJobExecution = new TuningJobExecution(); - JobExecution jobExecution = new JobExecution(); - jobExecution.job = job; - tuningJobExecution.jobExecution = jobExecution; - tuningJobExecution.tuningAlgorithm = tuningJobDefinition.tuningAlgorithm; - tuningJobExecution.isDefaultExecution = false; - if (isParamConstraintViolated(jobSuggestedParamValueList, tuningJobExecution.tuningAlgorithm.jobType, job.id)) { + JobSuggestedParamSet jobSuggestedParamSet = new JobSuggestedParamSet(); + jobSuggestedParamSet.jobDefinition = job; + jobSuggestedParamSet.tuningAlgorithm = tuningJobDefinition.tuningAlgorithm; + jobSuggestedParamSet.isParamSetDefault = false; + jobSuggestedParamSet.isParamSetBest = false; + if (isParamConstraintViolated(jobSuggestedParamValueList, jobSuggestedParamSet.tuningAlgorithm.jobType)) { logger.info("Parameter constraint violated. Applying penalty."); int penaltyConstant = 3; Double averageResourceUsagePerGBInput = - tuningJobDefinition.averageResourceUsage * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; + tuningJobDefinition.averageResourceUsage * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; Double maxDesiredResourceUsagePerGBInput = - averageResourceUsagePerGBInput * tuningJobDefinition.allowedMaxResourceUsagePercent / 100.0; - tuningJobExecution.fitness = penaltyConstant * maxDesiredResourceUsagePerGBInput; - tuningJobExecution.paramSetState = TuningJobExecution.ParamSetStatus.FITNESS_COMPUTED; + averageResourceUsagePerGBInput * tuningJobDefinition.allowedMaxResourceUsagePercent / 100.0; + + jobSuggestedParamSet.areConstraintsViolated = true; + jobSuggestedParamSet.fitness = penaltyConstant * maxDesiredResourceUsagePerGBInput; + jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED; } else { - tuningJobExecution.paramSetState = TuningJobExecution.ParamSetStatus.CREATED; + jobSuggestedParamSet.areConstraintsViolated = false; + jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.CREATED; } - Long paramSetId = saveSuggestedParamMetadata(tuningJobExecution); + Long paramSetId = saveSuggestedParamSet(jobSuggestedParamSet); for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValueList) { - jobSuggestedParamValue.jobExecution = jobExecution; + jobSuggestedParamValue.jobSuggestedParamSet = jobSuggestedParamSet; } suggestedParticle.setPramSetId(paramSetId); saveSuggestedParams(jobSuggestedParamValueList); @@ -445,11 +418,12 @@ private void updateDatabase(List jobTuningInfoList) { * Constraint 1: sort.mb > 60% of map.memory: To avoid heap memory failure * Constraint 2: map.memory - sort.mb < 768: To avoid heap memory failure * Constraint 3: pig.maxCombinedSplitSize > 1.8*mapreduce.map.memory.mb - * @param jobSuggestedParamValueList + * @param jobSuggestedParamValueList List of suggested param values + * @param jobType Job type * @return true if the constraint is violated, false otherwise */ private boolean isParamConstraintViolated(List jobSuggestedParamValueList, - TuningAlgorithm.JobType jobType, Integer jobDefinitionId) { + TuningAlgorithm.JobType jobType) { logger.info("Checking whether parameter values are within constraints"); Integer violations = 0; @@ -513,7 +487,7 @@ private void saveTunerState(List jobTuningInfoList) { } /** - * Saved the list of suggested parameter values to database + * Saves the list of suggested parameter values to database * @param jobSuggestedParamValueList Suggested Parameter Values List */ private void saveSuggestedParams(List jobSuggestedParamValueList) { @@ -523,21 +497,20 @@ private void saveSuggestedParams(List jobSuggestedParamV } /** - * Save the job execution in the database and returns the param set id - * @param tuningJobExecution JobExecution + * Saves the suggested param set in the database and returns the param set id + * @param jobSuggestedParamSet JobExecution * @return Param Set Id */ - - private Long saveSuggestedParamMetadata(TuningJobExecution tuningJobExecution) { - tuningJobExecution.save(); - return tuningJobExecution.jobExecution.id; + private Long saveSuggestedParamSet(JobSuggestedParamSet jobSuggestedParamSet) { + jobSuggestedParamSet.save(); + return jobSuggestedParamSet.id; } /** * Fetches job which need parameters, generates parameters and stores it in the database */ public void getParams() { - List jobsForSwarmSuggestion = fetchJobsForParamSuggestion(); + List jobsForSwarmSuggestion = getJobsForParamSuggestion(); List jobTuningInfoList = getJobsTuningInfo(jobsForSwarmSuggestion); List updatedJobTuningInfoList = new ArrayList(); for (JobTuningInfo jobTuningInfo : jobTuningInfoList) { diff --git a/app/com/linkedin/drelephant/tuning/Particle.java b/app/com/linkedin/drelephant/tuning/Particle.java index 978829285..961d839f5 100644 --- a/app/com/linkedin/drelephant/tuning/Particle.java +++ b/app/com/linkedin/drelephant/tuning/Particle.java @@ -18,7 +18,6 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.List; diff --git a/app/com/linkedin/drelephant/tuning/TuningInput.java b/app/com/linkedin/drelephant/tuning/TuningInput.java index b37efae92..7ccf80a23 100644 --- a/app/com/linkedin/drelephant/tuning/TuningInput.java +++ b/app/com/linkedin/drelephant/tuning/TuningInput.java @@ -16,6 +16,12 @@ package com.linkedin.drelephant.tuning; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import models.TuningAlgorithm; @@ -46,6 +52,15 @@ public class TuningInput { private Double _allowedMaxResourceUsagePercent; private Double _allowedMaxExecutionTimePercent; private TuningAlgorithm _tuningAlgorithm; + private Integer version; + + public Integer getVersion() { + return version; + } + + public void setVersion(Integer version) { + this.version = version; + } public TuningAlgorithm getTuningAlgorithm() { return _tuningAlgorithm; @@ -274,9 +289,36 @@ public void setScheduler(String scheduler) { /** * Returns the default parameters * @return default parameters - */ - public String getDefaultParams() { - return _defaultParams; + * @throws IOException + * @throws JsonMappingException + * @throws JsonParseException + */ + @SuppressWarnings("unchecked") + public Map getDefaultParams() throws JsonParseException, JsonMappingException, IOException { + ObjectMapper mapper = new ObjectMapper(); + Map paramValueMap; + if (version == 1) { + paramValueMap = (Map) mapper.readValue(this._defaultParams, Map.class); + } else { + paramValueMap = new HashMap(); + Map paramsStringMap = (Map) mapper.readValue(this._defaultParams, Map.class); + for (Map.Entry entry : paramsStringMap.entrySet()) { + String confKey = entry.getKey(); + String confVal = entry.getValue(); + Double confValDouble = null; + try { + if (confVal != null) { + confValDouble = Double.parseDouble(confVal); + } + } catch (NumberFormatException nfe) { + //Do Nothing + } + if (confValDouble != null) { + paramValueMap.put(confKey, confValDouble); + } + } + } + return paramValueMap; } /** diff --git a/app/controllers/Application.java b/app/controllers/Application.java index 1f3bff60f..1c1a60900 100644 --- a/app/controllers/Application.java +++ b/app/controllers/Application.java @@ -71,6 +71,7 @@ import views.html.page.searchPage; import views.html.results.*; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -914,6 +915,10 @@ public static Result getCurrentRunParameters() { String client = paramValueMap.get("client"); String scheduler = paramValueMap.get("scheduler"); String defaultParams = paramValueMap.get("defaultParams"); + Integer version = 1; + if (paramValueMap.containsKey("version")) { + version = Integer.parseInt(paramValueMap.get("version")); + } Boolean isRetry = false; if (paramValueMap.containsKey("isRetry")) { isRetry = Boolean.parseBoolean(paramValueMap.get("isRetry")); @@ -948,6 +953,7 @@ public static Result getCurrentRunParameters() { tuningInput.setClient(client); tuningInput.setScheduler(scheduler); tuningInput.setDefaultParams(defaultParams); + tuningInput.setVersion(version); tuningInput.setRetry(isRetry); tuningInput.setSkipExecutionForOptimization(skipExecutionForOptimization); tuningInput.setJobType(jobType); @@ -960,21 +966,44 @@ public static Result getCurrentRunParameters() { } catch (Exception e) { AutoTuningMetricsController.markGetCurrentRunParametersFailures(); logger.error("Exception parsing input: ", e); - return notFound("Error parsing input "); - }finally{ - if(context!=null) - { + return notFound("Error parsing input " + e.getMessage()); + } finally { + if (context != null) { context.stop(); } } } - private static Result getCurrentRunParameters(TuningInput tuningInput) { + private static JsonNode formatGetCurrentRunParametersOutput(Map outputParams, Integer version) { + if (version == 1) { + return Json.toJson(outputParams); + } else { + Map outputParamFormatted = new HashMap(); + + //Temporarily removing input split parameters + outputParams.remove("pig.maxCombinedSplitSize"); + outputParams.remove("mapreduce.input.fileinputformat.split.maxsize"); + + for (Map.Entry param : outputParams.entrySet()) { + if (param.getKey().equals("mapreduce.map.sort.spill.percent")) { + outputParamFormatted.put(param.getKey(), String.valueOf(param.getValue())); + } else if (param.getKey().equals("mapreduce.map.java.opts") + || param.getKey().equals("mapreduce.reduce.java.opts")) { + outputParamFormatted.put(param.getKey(), "-Xmx" + Math.round(param.getValue()) + "m"); + } else { + outputParamFormatted.put(param.getKey(), String.valueOf(Math.round(param.getValue()))); + } + } + return Json.toJson(outputParamFormatted); + } + } + + private static Result getCurrentRunParameters(TuningInput tuningInput) throws Exception { AutoTuningAPIHelper autoTuningAPIHelper = new AutoTuningAPIHelper(); Map outputParams = autoTuningAPIHelper.getCurrentRunParameters(tuningInput); if (outputParams != null) { logger.info("Output params " + outputParams); - return ok(Json.toJson(outputParams)); + return ok(formatGetCurrentRunParametersOutput(outputParams, tuningInput.getVersion())); } else { AutoTuningMetricsController.markGetCurrentRunParametersFailures(); return notFound("Unable to find parameters. Job id: " + tuningInput.getJobDefId() + " Flow id: " diff --git a/app/models/FlowDefinition.java b/app/models/FlowDefinition.java index e6023ad00..d01a347b1 100644 --- a/app/models/FlowDefinition.java +++ b/app/models/FlowDefinition.java @@ -16,6 +16,8 @@ package models; +import com.avaje.ebean.annotation.UpdatedTimestamp; +import java.sql.Timestamp; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.GeneratedValue; @@ -37,6 +39,8 @@ public static class TABLE { public static final String id = "id"; public static final String flowDefId = "flowDefId"; public static final String flowDefUrl = "flowDefUrl"; + public static final String createdTs = "createdTs"; + public static final String updatedTs = "updatedTs"; } @Id @@ -49,8 +53,28 @@ public static class TABLE { @Column(nullable = false) public String flowDefUrl; + @Column(nullable = false) + public Timestamp createdTs; + + @Column(nullable = false) + @UpdatedTimestamp + public Timestamp updatedTs; + public static Model.Finder find = new Model.Finder(Integer.class, FlowDefinition.class); + + @Override + public void save() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.save(); + } + + @Override + public void update() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.update(); + } + } diff --git a/app/models/FlowExecution.java b/app/models/FlowExecution.java index 3fb66ef72..0bcee835f 100644 --- a/app/models/FlowExecution.java +++ b/app/models/FlowExecution.java @@ -16,6 +16,8 @@ package models; +import com.avaje.ebean.annotation.UpdatedTimestamp; +import java.sql.Timestamp; import javax.persistence.CascadeType; import javax.persistence.Column; import javax.persistence.Entity; @@ -42,6 +44,8 @@ public static class TABLE { public static final String flowExecId = "flowExecId"; public static final String flowExecUrl = "flowExecUrl"; public static final String flowDefinition = "flowDefinition"; + public static final String createdTs = "createdTs"; + public static final String updatedTs = "updatedTs"; } @Id @@ -54,10 +58,30 @@ public static class TABLE { @Column(nullable = false) public String flowExecUrl; + @Column(nullable = false) @ManyToOne(cascade = CascadeType.ALL) @JoinTable(name = "flow_definition", joinColumns = {@JoinColumn(name = "flow_definition_id", referencedColumnName = "id")}) public FlowDefinition flowDefinition; + @Column(nullable = false) + public Timestamp createdTs; + + @Column(nullable = false) + @UpdatedTimestamp + public Timestamp updatedTs; + public static Model.Finder find = new Model.Finder(Integer.class, FlowExecution.class); + + @Override + public void save() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.save(); + } + + @Override + public void update() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.update(); + } } diff --git a/app/models/JobDefinition.java b/app/models/JobDefinition.java index 2b4d5732e..387b7ff12 100644 --- a/app/models/JobDefinition.java +++ b/app/models/JobDefinition.java @@ -74,17 +74,30 @@ public static class TABLE { @Column(length = JOB_NAME_LIMIT, nullable = false) public String jobDefUrl; + @Column(nullable = false) @ManyToOne(cascade = CascadeType.ALL) @JoinTable(name = "flow_definition", joinColumns = {@JoinColumn(name = "flow_definition_id", referencedColumnName = "id")}) public FlowDefinition flowDefinition; - @Column(nullable = true) + @Column(nullable = false) public Timestamp createdTs; - @Column(nullable = true) + @Column(nullable = false) @UpdatedTimestamp public Timestamp updatedTs; + @Override + public void save() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.save(); + } + + @Override + public void update() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.update(); + } + public static Finder find = new Finder(Integer.class, JobDefinition.class); } diff --git a/app/models/JobExecution.java b/app/models/JobExecution.java index 8c0c6a2d6..2f67f0c88 100644 --- a/app/models/JobExecution.java +++ b/app/models/JobExecution.java @@ -56,36 +56,29 @@ public static class TABLE { public static final String inputSizeInBytes = "inputSizeInBytes"; public static final String jobExecUrl = "jobExecUrl"; public static final String jobDefinition = "jobDefinition"; - public static final String createdTs = "createdTs"; - public static final String updatedTs = "updatedTs"; public static final String flowExecution = "flowExecution"; public static final String job = "job"; + public static final String createdTs = "createdTs"; + public static final String updatedTs = "updatedTs"; } @Id @GeneratedValue(strategy = GenerationType.IDENTITY) public Long id; - @Column(nullable = true) public String jobExecId; - @Column(nullable = true) + public String jobExecUrl; + @Enumerated(EnumType.STRING) public ExecutionState executionState; - @Column(nullable = true) public Double resourceUsage; - @Column(nullable = true) public Double executionTime; - @Column(nullable = true) public Double inputSizeInBytes; - @Column(nullable = true) - public String jobExecUrl; - - @Column(nullable = true) @ManyToOne(cascade = CascadeType.ALL) @JoinTable(name = "flow_execution", joinColumns = {@JoinColumn(name = "flow_execution_id", referencedColumnName = "id")}) public FlowExecution flowExecution; @@ -95,12 +88,24 @@ public static class TABLE { @JoinTable(name = "job_definition", joinColumns = {@JoinColumn(name = "job_definition_id", referencedColumnName = "id")}) public JobDefinition job; - @Column(nullable = true) + @Column(nullable = false) public Timestamp createdTs; - @Column(nullable = true) + @Column(nullable = false) @UpdatedTimestamp public Timestamp updatedTs; + @Override + public void save() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.save(); + } + + @Override + public void update() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.update(); + } + public static Finder find = new Finder(Long.class, JobExecution.class); } diff --git a/app/models/JobSavedState.java b/app/models/JobSavedState.java index 02f8688de..f7ca624ae 100644 --- a/app/models/JobSavedState.java +++ b/app/models/JobSavedState.java @@ -49,8 +49,10 @@ public static class TABLE { @Column(nullable = false) public byte[] savedState; + @Column(nullable = false) public Timestamp createdTs; + @Column(nullable = false) @UpdatedTimestamp public Timestamp updatedTs; @@ -60,4 +62,16 @@ public boolean isValid() { public static Finder find = new Finder(Integer.class, JobSavedState.class); + + @Override + public void save() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.save(); + } + + @Override + public void update() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.update(); + } } diff --git a/app/models/JobSuggestedParamSet.java b/app/models/JobSuggestedParamSet.java new file mode 100644 index 000000000..bad279932 --- /dev/null +++ b/app/models/JobSuggestedParamSet.java @@ -0,0 +1,118 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package models; + +import com.avaje.ebean.annotation.UpdatedTimestamp; +import java.sql.Timestamp; +import javax.persistence.CascadeType; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.JoinColumn; +import javax.persistence.JoinTable; +import javax.persistence.ManyToOne; +import javax.persistence.OneToOne; +import javax.persistence.Table; + +import play.db.ebean.Model; + + +@Entity +@Table(name = "job_suggested_param_set") +public class JobSuggestedParamSet extends Model { + + private static final long serialVersionUID = -294471313051608818L; + + public enum ParamSetStatus { + CREATED, SENT, EXECUTED, FITNESS_COMPUTED, DISCARDED + } + + public static class TABLE { + public static final String TABLE_NAME = "job_suggested_param_set"; + public static final String id = "id"; + public static final String jobDefinition = "jobDefinition"; + public static final String tuningAlgorithm = "tuningAlgorithm"; + public static final String paramSetState = "paramSetState"; + public static final String isParamSetDefault = "isParamSetDefault"; + public static final String fitness = "fitness"; + public static final String fitnessJobExecution = "fitnessJobExecution"; + public static final String isParamSetBest = "isParamSetBest"; + public static final String areConstraintsViolated = "areConstraintsViolated"; + public static final String createdTs = "createdTs"; + public static final String updatedTs = "updatedTs"; + + } + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + public Long id; + + @Column(nullable = false) + @OneToOne(cascade = CascadeType.ALL) + @JoinTable(name = "job_definition", joinColumns = {@JoinColumn(name = "job_definition_id", referencedColumnName = "id")}) + public JobDefinition jobDefinition; + + @OneToOne(cascade = CascadeType.ALL) + @JoinTable(name = "job_execution", joinColumns = {@JoinColumn(name = "fitness_job_execution_id", referencedColumnName = "id")}) + public JobExecution fitnessJobExecution; + + @Column(nullable = false) + @ManyToOne(cascade = CascadeType.ALL) + @JoinTable(name = "tuning_algorithm", joinColumns = {@JoinColumn(name = "tuning_algorithm_id", referencedColumnName = "id")}) + public TuningAlgorithm tuningAlgorithm; + + + @Enumerated(EnumType.STRING) + public ParamSetStatus paramSetState; + + @Column(nullable = false) + public Boolean isParamSetDefault; + + public Double fitness; + + @Column(nullable = false) + public Boolean isParamSetBest; + + @Column(nullable = false) + public Boolean areConstraintsViolated; + + @Column(nullable = false) + public Timestamp createdTs; + + @Column(nullable = false) + @UpdatedTimestamp + public Timestamp updatedTs; + + public static Model.Finder find = + new Model.Finder(Long.class, JobSuggestedParamSet.class); + + @Override + public void save() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.save(); + } + + @Override + public void update() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.update(); + } +} diff --git a/app/models/JobSuggestedParamValue.java b/app/models/JobSuggestedParamValue.java index 84dcef981..d78ba21ba 100644 --- a/app/models/JobSuggestedParamValue.java +++ b/app/models/JobSuggestedParamValue.java @@ -20,6 +20,7 @@ import java.sql.Timestamp; import javax.persistence.CascadeType; +import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; @@ -41,7 +42,7 @@ public class JobSuggestedParamValue extends Model { public static class TABLE { public static final String TABLE_NAME = "job_suggested_param_value"; public static final String id = "id"; - public static final String jobExecution = "jobExecution"; + public static final String jobSuggestedParamSet = "jobSuggestedParamSet"; public static final String tuningParameter = "tuningParameter"; public static final String paramValue = "paramValue"; public static final String createdTs = "createdTs"; @@ -52,19 +53,34 @@ public static class TABLE { @GeneratedValue(strategy = GenerationType.IDENTITY) public Integer id; public Double paramValue; - public Timestamp createdTs; - - @UpdatedTimestamp - public Timestamp updatedTs; @ManyToOne(cascade = CascadeType.ALL) - @JoinTable(name = "job_execution", joinColumns = {@JoinColumn(name = "job_execution_id", referencedColumnName = "id")}) - public JobExecution jobExecution; + @JoinTable(name = "job_suggested_param_set", joinColumns = {@JoinColumn(name = "job_suggested_param_set_id", referencedColumnName = "id")}) + public JobSuggestedParamSet jobSuggestedParamSet; @ManyToOne(cascade = CascadeType.ALL) @JoinTable(name = "tuning_parameter", joinColumns = {@JoinColumn(name = "tuning_parameter_id", referencedColumnName = "id")}) public TuningParameter tuningParameter; + @Column(nullable = false) + public Timestamp createdTs; + + @Column(nullable = false) + @UpdatedTimestamp + public Timestamp updatedTs; + public static Finder find = new Finder(Long.class, JobSuggestedParamValue.class); + + @Override + public void save() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.save(); + } + + @Override + public void update() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.update(); + } } diff --git a/app/models/TuningAlgorithm.java b/app/models/TuningAlgorithm.java index 4f2fdfb06..69b94c9b7 100644 --- a/app/models/TuningAlgorithm.java +++ b/app/models/TuningAlgorithm.java @@ -18,6 +18,7 @@ import java.sql.Timestamp; +import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.EnumType; import javax.persistence.Enumerated; @@ -75,11 +76,25 @@ public static class TABLE { @Enumerated(EnumType.STRING) public OptimizationMetric optimizationMetric; + @Column(nullable = false) public Timestamp createdTs; + @Column(nullable = false) @UpdatedTimestamp public Timestamp updatedTs; public static Finder find = new Finder(Integer.class, TuningAlgorithm.class); + + @Override + public void save() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.save(); + } + + @Override + public void update() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.update(); + } } diff --git a/app/models/TuningJobDefinition.java b/app/models/TuningJobDefinition.java index 1c0891f75..27c7c61d3 100644 --- a/app/models/TuningJobDefinition.java +++ b/app/models/TuningJobDefinition.java @@ -50,9 +50,9 @@ public static class TABLE { public static final String allowedMaxResourceUsagePercent = "allowedMaxResourceUsagePercent"; public static final String allowedMaxExecutionTimePercent = "allowedMaxExecutionTimePercent"; public static final String job = "job"; + public static final String tuningDisabledReason = "tuningDisabledReason"; public static final String createdTs = "createdTs"; public static final String updatedTs = "updatedTs"; - public static final String tuningDisabledReason = "tuningDisabledReason"; } @ManyToOne(cascade = CascadeType.ALL) @@ -67,7 +67,7 @@ public static class TABLE { public TuningAlgorithm tuningAlgorithm; @Column(nullable = false) - public int tuningEnabled; + public boolean tuningEnabled; @Column(nullable = true) public Double averageResourceUsage; @@ -105,4 +105,16 @@ public Double getAverageInputSizeInGB() { @Column(nullable = true) public String tuningDisabledReason; + + @Override + public void save() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.save(); + } + + @Override + public void update() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.update(); + } } diff --git a/app/models/TuningJobExecution.java b/app/models/TuningJobExecutionParamSet.java similarity index 50% rename from app/models/TuningJobExecution.java rename to app/models/TuningJobExecutionParamSet.java index 6f57acaa7..1be048e17 100644 --- a/app/models/TuningJobExecution.java +++ b/app/models/TuningJobExecutionParamSet.java @@ -16,58 +16,60 @@ package models; +import com.avaje.ebean.annotation.UpdatedTimestamp; +import java.sql.Timestamp; import javax.persistence.CascadeType; import javax.persistence.Column; import javax.persistence.Entity; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; import javax.persistence.JoinColumn; import javax.persistence.JoinTable; -import javax.persistence.ManyToOne; import javax.persistence.OneToOne; import javax.persistence.Table; - import play.db.ebean.Model; - @Entity -@Table(name = "tuning_job_execution") -public class TuningJobExecution extends Model { - - private static final long serialVersionUID = -294471313051608818L; - - public enum ParamSetStatus { - CREATED, SENT, EXECUTED, FITNESS_COMPUTED, DISCARDED - } +@Table(name = "tuning_job_execution_param_set") +public class TuningJobExecutionParamSet extends Model { + private static final long serialVersionUID = 1L; public static class TABLE { - public static final String TABLE_NAME = "tuning_job_execution"; - public static final String paramSetState = "paramSetState"; - public static final String isDefaultExecution = "isDefaultExecution"; - public static final String fitness = "fitness"; - public static final String isParamSetBest = "isParamSetBest"; + public static final String TABLE_NAME = "tuning_job_execution_param_set"; + public static final String jobSuggestedParamSet = "jobSuggestedParamSet"; public static final String jobExecution = "jobExecution"; - public static final String tuningAlgorithm = "tuningAlgorithm"; + public static final String tuningEnabled = "tuningEnabled"; + public static final String createdTs = "createdTs"; + public static final String updatedTs = "updatedTs"; } + @OneToOne(cascade = CascadeType.ALL) + @JoinTable(name = "job_suggested_param_set", joinColumns = {@JoinColumn(name = "job_suggested_param_set_id", referencedColumnName = "id")}) + public JobSuggestedParamSet jobSuggestedParamSet; + @OneToOne(cascade = CascadeType.ALL) @JoinTable(name = "job_execution", joinColumns = {@JoinColumn(name = "job_execution_id", referencedColumnName = "id")}) public JobExecution jobExecution; - @ManyToOne(cascade = CascadeType.ALL) - @JoinTable(name = "tuning_algorithm", joinColumns = {@JoinColumn(name = "tuning_algorithm_id", referencedColumnName = "id")}) - public TuningAlgorithm tuningAlgorithm; + public Boolean tuningEnabled; - @Enumerated(EnumType.STRING) @Column(nullable = false) - public ParamSetStatus paramSetState; + public Timestamp createdTs; - public Boolean isDefaultExecution; + @Column(nullable = false) + @UpdatedTimestamp + public Timestamp updatedTs; - public Double fitness; + public static Finder find = + new Finder(Long.class, TuningJobExecutionParamSet.class); - public Boolean isParamSetBest; + @Override + public void save() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.save(); + } - public static Model.Finder find = - new Model.Finder(Long.class, TuningJobExecution.class); + @Override + public void update() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.update(); + } } diff --git a/app/models/TuningParameter.java b/app/models/TuningParameter.java index d19e70b3b..7ecce1acc 100644 --- a/app/models/TuningParameter.java +++ b/app/models/TuningParameter.java @@ -52,10 +52,10 @@ public static class TABLE { public static final String minValue = "minValue"; public static final String maxValue = "maxValue"; public static final String stepSize = "stepSize"; - public static final String createdTs = "createdTs"; - public static final String updatedTs = "updatedTs"; public static final String tuningAlgorithm = "tuningAlgorithm"; public static final String isDerived = "isDerived"; + public static final String createdTs = "createdTs"; + public static final String updatedTs = "updatedTs"; } @Id @@ -93,4 +93,16 @@ public static class TABLE { public static Finder find = new Finder(Integer.class, TuningParameter.class); + + @Override + public void save() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.save(); + } + + @Override + public void update() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.update(); + } } diff --git a/conf/evolutions/default/2.sql b/conf/evolutions/default/2.sql index 9f9e39e51..4f3d899b7 100644 --- a/conf/evolutions/default/2.sql +++ b/conf/evolutions/default/2.sql @@ -1,4 +1,20 @@ -# --- Indexing on queue for seach by queue feature +# +# Copyright 2016 LinkedIn Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + +# --- Indexing on queue for search by queue feature # --- !Ups create index yarn_app_result_i8 on yarn_app_result (queue_name); diff --git a/conf/evolutions/default/3.sql b/conf/evolutions/default/3.sql index a8cd34093..e01a3a8c8 100644 --- a/conf/evolutions/default/3.sql +++ b/conf/evolutions/default/3.sql @@ -1,3 +1,19 @@ +# +# Copyright 2016 LinkedIn Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + # --- Indexing on queue for seach by queue feature # --- !Ups diff --git a/conf/evolutions/default/4.sql b/conf/evolutions/default/4.sql index c3d9b844b..c9852e7e8 100644 --- a/conf/evolutions/default/4.sql +++ b/conf/evolutions/default/4.sql @@ -1,3 +1,19 @@ +# +# Copyright 2016 LinkedIn Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + # --- Indexing on severity,finish_time for count on welcome page # --- !Ups diff --git a/conf/evolutions/default/5.sql b/conf/evolutions/default/5.sql index 174d42ee0..640c292be 100644 --- a/conf/evolutions/default/5.sql +++ b/conf/evolutions/default/5.sql @@ -28,10 +28,12 @@ CREATE TABLE IF NOT EXISTS tuning_algorithm ( optimization_metric enum('RESOURCE','EXECUTION_TIME') DEFAULT NULL COMMENT 'metric to be optimized', created_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , - PRIMARY KEY (id) + PRIMARY KEY (id), + UNIQUE KEY tuning_algorithm_u1 (optimization_algo, optimization_algo_version) ) ENGINE=InnoDB; -INSERT INTO tuning_algorithm VALUES (1, 'PIG', 'PSO', '1', 'RESOURCE', current_timestamp(0), current_timestamp(0)); +INSERT INTO tuning_algorithm(id, job_type, optimization_algo, optimization_algo_version, optimization_metric, created_ts, updated_ts) +VALUES (1, 'PIG', 'PSO', '1', 'RESOURCE', current_timestamp(0), current_timestamp(0)); /** * This table represents hadoop parameters to be optimized for each algo in tuning_algorithm. @@ -52,15 +54,16 @@ CREATE TABLE IF NOT EXISTS tuning_parameter ( CONSTRAINT tuning_parameter_ibfk_1 FOREIGN KEY (tuning_algorithm_id) REFERENCES tuning_algorithm (id) ) ENGINE=InnoDB; -INSERT INTO tuning_parameter VALUES (1,'mapreduce.task.io.sort.mb',1,100,50,1920,50, 0, current_timestamp(0), current_timestamp(0)); -INSERT INTO tuning_parameter VALUES (2,'mapreduce.map.memory.mb',1,2048,1536,8192,128, 0, current_timestamp(0), current_timestamp(0)); -INSERT INTO tuning_parameter VALUES (3,'mapreduce.task.io.sort.factor',1,10,10,150,10 ,0, current_timestamp(0), current_timestamp(0)); -INSERT INTO tuning_parameter VALUES (4,'mapreduce.map.sort.spill.percent',1,0.8,0.6,0.9,0.1, 0, current_timestamp(0), current_timestamp(0)); -INSERT INTO tuning_parameter VALUES (5,'mapreduce.reduce.memory.mb',1,2048,1536,8192,128, 0, current_timestamp(0), current_timestamp(0)); -INSERT INTO tuning_parameter VALUES (6,'pig.maxCombinedSplitSize',1,536870912,536870912,536870912,128, 0, current_timestamp(0), current_timestamp(0)); -INSERT INTO tuning_parameter VALUES (7,'mapreduce.reduce.java.opts',1,1536,1152,6144,128, 1, current_timestamp(0), current_timestamp(0)); -INSERT INTO tuning_parameter VALUES (8,'mapreduce.map.java.opts',1,1536,1152,6144,128, 1, current_timestamp(0), current_timestamp(0)); -INSERT INTO tuning_parameter VALUES (9,'mapreduce.input.fileinputformat.split.maxsize',1,536870912,536870912,536870912,128, 1, current_timestamp(0), current_timestamp(0)); +INSERT INTO tuning_parameter (id, param_name, tuning_algorithm_id, default_value, min_value, max_value, step_size, is_derived, created_ts, updated_ts) VALUES +(1,'mapreduce.task.io.sort.mb',1,100,50,1920,50, 0, current_timestamp(0), current_timestamp(0)), +(2,'mapreduce.map.memory.mb',1,2048,1536,8192,128, 0, current_timestamp(0), current_timestamp(0)), +(3,'mapreduce.task.io.sort.factor',1,10,10,150,10 ,0, current_timestamp(0), current_timestamp(0)), +(4,'mapreduce.map.sort.spill.percent',1,0.8,0.6,0.9,0.1, 0, current_timestamp(0), current_timestamp(0)), +(5,'mapreduce.reduce.memory.mb',1,2048,1536,8192,128, 0, current_timestamp(0), current_timestamp(0)), +(6,'pig.maxCombinedSplitSize',1,536870912,536870912,536870912,128, 0, current_timestamp(0), current_timestamp(0)), +(7,'mapreduce.reduce.java.opts',1,1536,1152,6144,128, 1, current_timestamp(0), current_timestamp(0)), +(8,'mapreduce.map.java.opts',1,1536,1152,6144,128, 1, current_timestamp(0), current_timestamp(0)), +(9,'mapreduce.input.fileinputformat.split.maxsize',1,536870912,536870912,536870912,128, 1, current_timestamp(0), current_timestamp(0)); create index index_tp_algo_id on tuning_parameter (tuning_algorithm_id); @@ -71,8 +74,10 @@ CREATE TABLE IF NOT EXISTS flow_definition ( id int(10) unsigned NOT NULL AUTO_INCREMENT COMMENT 'Auto increment unique id', flow_def_id varchar(700) NOT NULL COMMENT 'unique flow definition id from scheduler like azkaban, oozie, appworx etc', flow_def_url varchar(700) NOT NULL COMMENT 'flow definition URL from scheduler like azkaban, oozie, appworx etc', + created_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (id), - UNIQUE KEY flow_def_id (flow_def_id) + UNIQUE KEY flow_definition_u1 (flow_def_id) ) ENGINE=InnoDB AUTO_INCREMENT=10000; /** @@ -83,15 +88,15 @@ CREATE TABLE IF NOT EXISTS flow_definition ( CREATE TABLE IF NOT EXISTS job_definition ( id int(10) unsigned NOT NULL AUTO_INCREMENT COMMENT 'Auto increment unique id', job_def_id varchar(700) NOT NULL COMMENT 'unique job definition id from scheduler like azkaban, oozie etc', + job_def_url varchar(700) NOT NULL COMMENT 'job definition URL from scheduler like azkaban, oozie, appworx etc', flow_definition_id int(10) unsigned NOT NULL COMMENT 'foreign key from flow_definition table', job_name varchar(700) DEFAULT NULL COMMENT 'name of the job', - job_def_url varchar(700) NOT NULL COMMENT 'job definition URL from scheduler like azkaban, oozie, appworx etc', scheduler varchar(100) NOT NULL COMMENT 'name of the scheduler like azkaban. oozie ', username varchar(100) NOT NULL COMMENT 'name of the user', created_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (id), - UNIQUE KEY job_def_id (job_def_id) , + UNIQUE KEY job_definition_u1 (job_def_id) , CONSTRAINT job_definition_ibfk_1 FOREIGN KEY (flow_definition_id) REFERENCES flow_definition (id) ) ENGINE=InnoDB AUTO_INCREMENT=100000; @@ -110,6 +115,7 @@ CREATE TABLE IF NOT EXISTS tuning_job_definition ( average_input_size_in_bytes bigint(20) DEFAULT NULL COMMENT 'Average input size in bytes when optimization started on this job', allowed_max_resource_usage_percent double DEFAULT NULL COMMENT 'Limit on resource usage, For ex 150 means it should not go beyond 150% ', allowed_max_execution_time_percent double DEFAULT NULL COMMENT 'Limit on execution time, For ex 150 means it should not go beyond 150% ', + tuning_disabled_reason text NULL COMMENT 'reason for disabling tuning, if any', created_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, CONSTRAINT tuning_job_definition_ibfk_1 FOREIGN KEY (job_definition_id) REFERENCES job_definition (id), @@ -127,6 +133,8 @@ CREATE TABLE IF NOT EXISTS flow_execution ( flow_exec_id varchar(700) NOT NULL COMMENT 'unique flow execution id from scheduler like azkaban, oozie etc ', flow_exec_url varchar(700) NOT NULL COMMENT 'execution url from scheduler like azkaban, oozie etc', flow_definition_id int(10) unsigned NOT NULL COMMENT 'foreign key from flow_definition table', + created_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (id), CONSTRAINT flow_execution_ibfk_1 FOREIGN KEY (flow_definition_id) REFERENCES flow_definition (id) ) ENGINE=InnoDB AUTO_INCREMENT=1000; @@ -140,11 +148,11 @@ create index index_fe_flow_definition_id on flow_execution (flow_definition_id); */ CREATE TABLE IF NOT EXISTS job_execution ( id int(10) unsigned NOT NULL AUTO_INCREMENT COMMENT 'Auto increment unique id', - job_exec_id varchar(700) DEFAULT NULL COMMENT 'unique job execution id from scheduler like azkaban, oozie etc', - job_exec_url varchar(700) DEFAULT NULL COMMENT 'job execution url from scheduler like azkaban, oozie etc', + job_exec_id varchar(700) NOT NULL COMMENT 'unique job execution id from scheduler like azkaban, oozie etc', + job_exec_url varchar(700) NOT NULL COMMENT 'job execution url from scheduler like azkaban, oozie etc', job_definition_id int(10) unsigned NOT NULL COMMENT 'foreign key from job_definition table', - flow_execution_id int(10) unsigned DEFAULT NULL COMMENT 'foreign key from flow_execution table', - execution_state enum('SUCCEEDED','FAILED','NOT_STARTED','IN_PROGRESS','CANCELLED') DEFAULT NULL COMMENT 'current state of execution of the job ', + flow_execution_id int(10) unsigned NOT NULL COMMENT 'foreign key from flow_execution table', + execution_state enum('SUCCEEDED','FAILED','NOT_STARTED','IN_PROGRESS','CANCELLED') NOT NULL COMMENT 'current state of execution of the job ', resource_usage double DEFAULT NULL COMMENT 'resource usage in GB Hours for this execution of the job', execution_time double DEFAULT NULL COMMENT 'execution time excluding delay for this execution of the job', input_size_in_bytes bigint(20) DEFAULT NULL COMMENT 'input size in bytes for this execution of the job', @@ -164,19 +172,28 @@ create index index_je_flow_execution_id on job_execution (flow_execution_id); * This table represent jobs from one execution of a flow and contains auto tuning related information. * This one execution is corresponding to one set of parameters. */ -CREATE TABLE IF NOT EXISTS tuning_job_execution ( - job_execution_id int(10) unsigned NOT NULL COMMENT 'foreign key from job_execution table', +CREATE TABLE IF NOT EXISTS job_suggested_param_set ( + id int(10) unsigned NOT NULL AUTO_INCREMENT COMMENT 'Auto increment unique id', + job_definition_id int(10) unsigned NOT NULL COMMENT 'foreign key from job_definition table', tuning_algorithm_id int(10) unsigned NOT NULL COMMENT 'foreign key from tuning_algorithm table', - param_set_state enum('CREATED','SENT','EXECUTED','FITNESS_COMPUTED','DISCARDED') DEFAULT NULL COMMENT 'state of this execution parameter set', - is_default_execution tinyint(4) NOT NULL COMMENT 'Is this default execution', + param_set_state enum('CREATED','SENT','EXECUTED','FITNESS_COMPUTED','DISCARDED') NOT NULL COMMENT 'state of this execution parameter set', + are_constraints_violated tinyint(4) default 0 NOT NULL COMMENT 'are constraints violated for the parameter set', + is_param_set_default tinyint(4) DEFAULT 0 NOT NULL COMMENT 'Is parameter set default', + is_param_set_best tinyint(4) DEFAULT 0 NOT NULL COMMENT 'Is parameter set best', fitness double DEFAULT NULL COMMENT 'fitness of this parameter set', - UNIQUE KEY job_execution_id_2 (job_execution_id), - CONSTRAINT tuning_job_execution_ibfk_1 FOREIGN KEY (tuning_algorithm_id) REFERENCES tuning_algorithm (id), - CONSTRAINT tuning_job_execution_ibfk_2 FOREIGN KEY (job_execution_id) REFERENCES job_execution (id) -) ENGINE=InnoDB ; + fitness_job_execution_id int(10) unsigned NULL COMMENT 'foreign key from job_execution table', + created_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id), + CONSTRAINT job_suggested_param_set_f1 FOREIGN KEY (tuning_algorithm_id) REFERENCES tuning_algorithm (id), + -- The following statement is commented as it leads to unit test failures though it works fine when deployed. + -- This is happening because unlike mysql, h2 database doesn't support nullable foreign keys. + -- CONSTRAINT job_suggested_param_set_f2 FOREIGN KEY (fitness_job_execution_id) REFERENCES job_execution (id), + CONSTRAINT job_suggested_param_set_f3 FOREIGN KEY (job_definition_id) REFERENCES job_definition (id) +) ENGINE=InnoDB AUTO_INCREMENT=1000; -create index index_tje_job_execution_id on tuning_job_execution (job_execution_id); -create index index_tje_tuning_algorithm_id on tuning_job_execution (tuning_algorithm_id); +create index index_tje_job_definition_id on job_suggested_param_set (job_definition_id); +create index index_tje_tuning_algorithm_id on job_suggested_param_set (tuning_algorithm_id); /** * Internal table for optimization algorithm. Stores the current state of job to be optimized/ @@ -192,31 +209,47 @@ CREATE TABLE IF NOT EXISTS job_saved_state ( /** - * Suggested parameter value corresponding to one execution of the job. + * Stores the suggested parameter value corresponding to one execution of the job. */ CREATE TABLE IF NOT EXISTS job_suggested_param_value ( id int(10) unsigned NOT NULL AUTO_INCREMENT COMMENT 'Auto increment unique id', - job_execution_id int(10) unsigned NOT NULL COMMENT 'foreign key from job_execution table', + job_suggested_param_set_id int(10) unsigned NOT NULL COMMENT 'foreign key from job_suggested_param_set table', tuning_parameter_id int(10) unsigned NOT NULL COMMENT 'foreign key from tuning_parameter table', - param_value double NOT NULL COMMENT 'value of the parameter suggested by algo', + param_value double NOT NULL COMMENT 'suggested value of the parameter', created_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (id), - UNIQUE KEY job_execution_id (job_execution_id,tuning_parameter_id), - CONSTRAINT job_suggested_param_values_f1 FOREIGN KEY (job_execution_id) REFERENCES job_execution (id), - CONSTRAINT job_suggested_param_values_f2 FOREIGN KEY (tuning_parameter_id) REFERENCES tuning_parameter (id) -) ENGINE=InnoDB AUTO_INCREMENT=1000 ; + UNIQUE KEY job_suggested_param_value_u1 (job_suggested_param_set_id, tuning_parameter_id), + CONSTRAINT job_suggested_param_value_f1 FOREIGN KEY (job_suggested_param_set_id) REFERENCES job_suggested_param_set (id), + CONSTRAINT job_suggested_param_value_f2 FOREIGN KEY (tuning_parameter_id) REFERENCES tuning_parameter (id) +) ENGINE=InnoDB AUTO_INCREMENT=1000; create index index_jspv_tuning_parameter_id on job_suggested_param_value (tuning_parameter_id); +/** + * Stores the mapping of job execution and corresponding parameter set + */ + +CREATE TABLE IF NOT EXISTS tuning_job_execution_param_set ( + job_suggested_param_set_id int(10) unsigned NOT NULL COMMENT 'foreign key from job_suggested_param_set table', + job_execution_id int(10) unsigned NOT NULL COMMENT 'foreign key from job_execution table', + tuning_enabled tinyint(4) NOT NULL COMMENT 'Is tuning enabled for the execution', + created_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE KEY tuning_job_execution_param_set_u1 (job_suggested_param_set_id, job_execution_id), + CONSTRAINT tuning_job_execution_param_set_f1 FOREIGN KEY (job_suggested_param_set_id) REFERENCES job_suggested_param_set (id), + CONSTRAINT tuning_job_execution_param_set_f2 FOREIGN KEY (job_execution_id) REFERENCES job_execution (id) +) ENGINE=InnoDB; + # --- !Downs -drop table job_suggested_param_value ; +drop table tuning_job_execution_param_set; +drop table job_suggested_param_value; drop table job_saved_state; -drop table tuning_job_execution; +drop table job_suggested_param_set; drop table tuning_job_definition; drop table job_execution; drop table flow_execution; drop table job_definition; drop table flow_definition; drop table tuning_parameter; -drop table tuning_algorithm; \ No newline at end of file +drop table tuning_algorithm; diff --git a/conf/evolutions/default/6.sql b/conf/evolutions/default/6.sql deleted file mode 100644 index f16030727..000000000 --- a/conf/evolutions/default/6.sql +++ /dev/null @@ -1,11 +0,0 @@ -# --- Support for auto tuning spark -# --- !Ups - -ALTER TABLE tuning_algorithm ADD UNIQUE KEY tuning_algorithm_uk1(optimization_algo, optimization_algo_version); -ALTER TABLE tuning_job_execution ADD COLUMN is_param_set_best tinyint(4) default 0 NOT NULL; -ALTER TABLE tuning_job_definition ADD COLUMN tuning_disabled_reason text; - -# --- !Downs -ALTER TABLE tuning_job_definition DROP COLUMN tuning_disabled_reason; -ALTER TABLE tuning_job_execution DROP COLUMN is_param_set_best; -ALTER TABLE tuning_algorithm DROP INDEX tuning_algorithm_uk1; \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tuning/PSOParamGeneratorTest.java b/test/com/linkedin/drelephant/tuning/PSOParamGeneratorTest.java index 99aa1f0dd..255268255 100644 --- a/test/com/linkedin/drelephant/tuning/PSOParamGeneratorTest.java +++ b/test/com/linkedin/drelephant/tuning/PSOParamGeneratorTest.java @@ -25,10 +25,10 @@ import java.util.Map; import models.JobDefinition; import models.JobExecution; +import models.JobSuggestedParamSet; import models.JobSuggestedParamValue; import models.TuningAlgorithm; import models.TuningJobDefinition; -import models.TuningJobExecution; import models.TuningParameter; import org.junit.Before; import org.junit.Test; @@ -203,19 +203,19 @@ public void run() { PSOParamGenerator psoParamGenerator = new PSOParamGenerator(); psoParamGenerator.getParams(); - List tuningJobExecutionList = TuningJobExecution.find.where() - .eq(TuningJobExecution.TABLE.paramSetState, TuningJobExecution.ParamSetStatus.CREATED) + List jobSuggestedParamSetList = JobSuggestedParamSet.find.where() + .eq(JobSuggestedParamSet.TABLE.paramSetState, JobSuggestedParamSet.ParamSetStatus.CREATED) .findList(); - assertEquals("Swarm size did not match", SWARM_SIZE, tuningJobExecutionList.size()); + assertEquals("Swarm size did not match", SWARM_SIZE, jobSuggestedParamSetList.size()); - TuningJobExecution tuningJobExecution = tuningJobExecutionList.get(0); + JobSuggestedParamSet jobSuggestedParamSet = jobSuggestedParamSetList.get(0); List jobSuggestedParamValueList = JobSuggestedParamValue.find.where() - .eq(JobSuggestedParamValue.TABLE.jobExecution + '.' + JobExecution.TABLE.id, - tuningJobExecution.jobExecution.id) + .eq(JobSuggestedParamValue.TABLE.jobSuggestedParamSet + '.' + JobSuggestedParamSet.TABLE.id, + jobSuggestedParamSet.id) .findList(); - TuningAlgorithm tuningAlgorithm = tuningJobExecution.tuningAlgorithm; + TuningAlgorithm tuningAlgorithm = jobSuggestedParamSet.tuningAlgorithm; List tuningParameterList = TuningParameter.find.where() .eq(TuningParameter.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.id, tuningAlgorithm.id) .findList(); diff --git a/test/resources/AutoTuningConf.xml b/test/resources/AutoTuningConf.xml index b505e52ed..e2e6cacea 100644 --- a/test/resources/AutoTuningConf.xml +++ b/test/resources/AutoTuningConf.xml @@ -44,8 +44,8 @@ fitness.compute.wait_interval.ms - 180 - Wait time after the job is completed for fitness computation + 0 + Wait time after the job is completed till fitness is computer dr.elephant.api.url diff --git a/test/resources/test-init.sql b/test/resources/test-init.sql index 024cd9155..522247546 100644 --- a/test/resources/test-init.sql +++ b/test/resources/test-init.sql @@ -11,27 +11,28 @@ INSERT INTO flow_definition(id, flow_def_id, flow_def_url) VALUES (10003,'https: INSERT INTO job_definition(id, job_def_id, flow_definition_id, job_name, job_def_url, scheduler, username, created_ts, updated_ts) VALUES (100003,'https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry',10003,'countByCountryFlow_countByCountry','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry','azkaban','mkumar1','2018-02-12 08:40:42','2018-02-12 08:40:43'); ---INSERT INTO tuning_algorithm VALUES (1,'PIG','PSO',1,'RESOURCE','2018-02-06 17:03:23','2018-02-06 17:03:23'); - --- INSERT INTO tuning_parameter VALUES (1,'mapreduce.task.io.sort.mb',1,100,50,1920,50,0,'2018-02-06 17:03:23','2018-02-06 17:03:23'),(2,'mapreduce.map.memory.mb',1,2048,1536,8192,128,0,'2018-02-06 17:03:23','2018-02-06 17:03:23'),(3,'mapreduce.task.io.sort.factor',1,10,10,150,10,0,'2018-02-06 17:03:23','2018-02-06 17:03:23'),(4,'mapreduce.map.sort.spill.percent',1,0.8,0.6,0.9,0.1,0,'2018-02-06 17:03:24','2018-02-06 17:03:24'),(5,'mapreduce.reduce.memory.mb',1,2048,1536,8192,128,0,'2018-02-06 17:03:24','2018-02-06 17:03:24'),(6,'pig.maxCombinedSplitSize',1,536870912,536870912,536870912,128,0,'2018-02-06 17:03:24','2018-02-06 17:03:24'),(7,'mapreduce.reduce.java.opts',1,1536,1152,6144,128,1,'2018-02-06 17:03:24','2018-02-06 17:03:24'),(8,'mapreduce.map.java.opts',1,1536,1152,6144,128,1,'2018-02-06 17:03:24','2018-02-06 17:03:24'),(9,'mapreduce.input.fileinputformat.split.maxsize',1,536870912,536870912,536870912,128,1,'2018-02-06 17:03:24','2018-02-06 17:03:24'); - INSERT INTO tuning_job_definition(job_definition_id, client, tuning_algorithm_id, tuning_enabled, average_resource_usage, average_execution_time, average_input_size_in_bytes, allowed_max_resource_usage_percent, allowed_max_execution_time_percent, created_ts, updated_ts, tuning_disabled_reason) VALUES (100003,'azkaban',1,1,40.29456456163195,5.178423333333334,324168876088,150,150,'2018-02-12 08:40:42','2018-02-12 08:40:43', NULL); INSERT INTO flow_execution(id, flow_exec_id, flow_exec_url, flow_definition_id) VALUES (1496,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293',10003),(1497,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416389','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416389',10003),(1498,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416495','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416495',10003),(1499,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416589','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416589',10003),(1500,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416680','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416680',10003),(1501,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416818','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416818',10003),(1502,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5417057','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5417057',10003); -INSERT INTO job_execution(id, job_exec_id, job_exec_url, job_definition_id, flow_execution_id, execution_state, resource_usage, execution_time, input_size_in_bytes, created_ts, updated_ts) VALUES (1541,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0',100003,1496,'SUCCEEDED',21.132545572916666,3.2694833333333335,324713861757,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(1542,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416389&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416389&job=countByCountryFlow_countByCountry&attempt=0',100003,1497,'SUCCEEDED',23.334004991319443,3.6118166666666665,324713861757,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(1543,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416495&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416495&job=countByCountryFlow_countByCountry&attempt=0',100003,1498,'SUCCEEDED',21.28552951388889,3.2940833333333335,324713861757,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(1544,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416589&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416589&job=countByCountryFlow_countByCountry&attempt=0',100003,1499,'SUCCEEDED',21.630970052083335,3.9560833333333334,324713861757,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(1545,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416680&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416680&job=countByCountryFlow_countByCountry&attempt=0',100003,1500,'SUCCEEDED',22.328486328125,3.7285166666666667,324713861757,'2018-02-14 07:29:47','2018-02-14 07:29:48'),(1546,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416818&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416818&job=countByCountryFlow_countByCountry&attempt=0',100003,1501,'SUCCEEDED',32.16945149739583,5.203783333333333,324713861757,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(1547,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5417057&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5417057&job=countByCountryFlow_countByCountry&attempt=0',100003,1502,'SUCCEEDED', 27.2955078125, 4.047583333333334, 324713861757,'2018-02-14 07:29:48','2018-02-14 07:29:48'); - -INSERT INTO tuning_job_execution(job_execution_id, tuning_algorithm_id, param_set_state, is_default_execution, fitness, is_param_set_best) VALUES (1541,1,'FITNESS_COMPUTED',0,0.06987967161749142,0), -(1542,1,'FITNESS_COMPUTED',0,0.07715930864495756,0), -(1543,1,'FITNESS_COMPUTED',0,0.07038554856075895,0), -(1544,1,'FITNESS_COMPUTED',0,0.07152782795578526,0), -(1545,1,'FITNESS_COMPUTED',0,0.07383432757503201,0), -(1546,1,'FITNESS_COMPUTED',0,0.10637576523832741,0), -(1547,1,'FITNESS_COMPUTED',0,0.09025893809095505,0); - --- INSERT INTO job_saved_state (job_definition_id, saved_state) VALUES (100003,'{\"current_population\":[{\"birthdate\":1.518593387909695E9,\"maximize\":false,\"candidate\":[159.88928672056016,1536.0,10.0,0.770164839202443,2863.372720073011,5.36870912E8],\"fitness\":10000.0,\"paramSetId\":1545,\"_candidate\":[159.88928672056016,1536.0,10.0,0.770164839202443,2863.372720073011,5.36870912E8]},{\"birthdate\":1.518593387909696E9,\"maximize\":false,\"candidate\":[201.64225529876035,1536.0,10.29839988592941,0.7635183100860585,2789.189282499988,5.36870912E8],\"fitness\":10000.0,\"paramSetId\":1546,\"_candidate\":[201.64225529876035,1536.0,10.29839988592941,0.7635183100860585,2789.189282499988,5.36870912E8]},{\"birthdate\":1.518593387909698E9,\"maximize\":false,\"candidate\":[149.52419594024295,1536.0,10.0,0.7630834894363029,2844.1716734703073,5.36870912E8],\"fitness\":10000.0,\"paramSetId\":1547,\"_candidate\":[149.52419594024295,1536.0,10.0,0.7630834894363029,2844.1716734703073,5.36870912E8]}],\"prev_population\":[{\"_candidate\":[162.7907384046847,1536.0,10.0,0.7632081784681852,2809.5806453243313,5.36870912E8],\"maximize\":false,\"birthdate\":1.518593387909403E9,\"fitness\":0.07715930864495756},{\"_candidate\":[124.05878355054111,1536.0,12.521341191290857,0.7622909149004323,2041.562366831904,5.36870912E8],\"maximize\":false,\"birthdate\":1.518593387909406E9,\"fitness\":0.07038554856075895},{\"_candidate\":[149.51252503919468,1536.0,10.0,0.7619961998308155,2844.326081249364,5.36870912E8],\"maximize\":false,\"birthdate\":1.518593387909407E9,\"fitness\":0.07152782795578526}],\"archive\":[{\"birthday\":1.518589785163133E9,\"_candidate\":[214.91248065459718,1536.0,10.0,0.7684337983774014,2810.072296417105,5.36870912E8],\"maximize\":false,\"birthdate\":1.518593387904912E9,\"fitness\":0.06798720858096566},{\"_candidate\":[124.05878355054111,1536.0,12.521341191290857,0.7622909149004323,2041.562366831904,5.36870912E8],\"maximize\":false,\"birthdate\":1.518593387909406E9,\"fitness\":0.07038554856075895},{\"birthday\":1.518589785163142E9,\"_candidate\":[149.5180195836781,1536.0,10.0,0.7624156183109899,2844.240241069899,5.36870912E8],\"maximize\":false,\"birthdate\":1.518593387904921E9,\"fitness\":0.06753575007445276}],\"rnd_state\":\"[3, [436764630, 3845696172, 1536093292, 3091223851, 1875971029, 1714861982, 2792470647, 3065135847, 2614095987, 3861985834, 3271806303, 1346964832, 3502746452, 2479553754, 2365815367, 4052784148, 98894269, 865838453, 378106826, 223993858, 4003299431, 4066675665, 203485080, 591015950, 2009306943, 209194928, 1127412241, 780814913, 2127898994, 1397125766, 2388957036, 2132624000, 2298888066, 2872527307, 2469348483, 1291602844, 3055370485, 2875413348, 1561776770, 44416253, 2399434358, 3500131466, 921216978, 4241064332, 436769369, 3842353021, 307088723, 1108438886, 3147936013, 2878320949, 306583468, 2017374956, 3295572824, 3273465645, 1897933257, 319521868, 2207782720, 1273667531, 372255185, 4062657208, 411612960, 282273738, 1946543237, 3282301954, 441294767, 4284945843, 1049714129, 2018344119, 1501407651, 3952558232, 3193503080, 3824343591, 3881423919, 77231396, 3043228678, 4105014263, 1763641998, 2079208800, 4077989659, 3535532484, 266896722, 3238452269, 3389015947, 3300573445, 400677334, 86727356, 492103313, 2873593589, 3349834695, 2913301047, 1420741021, 50847724, 1599203833, 3743114912, 2826729030, 3192901431, 2123130625, 392187268, 3073037125, 929590156, 4037814058, 1884481338, 2743891587, 4008113834, 258439194, 1605236424, 65715142, 1777541766, 2364169967, 3153165247, 3110291887, 1630800170, 1838776728, 3106288739, 430636498, 2880404702, 1852472028, 1552499468, 2539957287, 3785077327, 3823438269, 3242387062, 2983664566, 3249832040, 865349220, 1794595840, 2821840008, 3923234509, 1623391669, 2809627269, 2617933073, 3106890857, 3058957671, 2708135756, 1562894481, 2523137886, 527517498, 2114931269, 4254571978, 568227586, 93804775, 2796328419, 558871677, 2758944023, 1124005519, 1183509574, 335486627, 504724538, 1826353090, 1127161878, 1410745185, 4037978787, 3102147541, 398636727, 4216182187, 4099107742, 3961538036, 3264622209, 2654911086, 90501146, 3002202423, 190590765, 2420050097, 2767758974, 4057723606, 252185447, 2305162223, 3613701152, 1854930647, 2683774155, 1530637790, 2320258070, 1086704260, 691720499, 3286101825, 4280735926, 568632376, 1274202416, 4194385357, 2917109897, 3616771536, 1911799859, 3508013413, 4107695527, 2298948322, 2918077160, 2166469136, 1566810333, 666358996, 3019106791, 2355339603, 127848796, 309049116, 4172598364, 451598967, 3396456615, 3397855157, 4160530966, 3792287385, 2933569340, 3158078446, 1799420732, 4233446360, 1195562564, 3812502285, 3340167346, 832972998, 4236732687, 2813517707, 660207563, 1870462423, 3783309115, 2416745730, 1342644731, 3950482077, 754564373, 2068468799, 1031651649, 2486955671, 1715492367, 2327446476, 572839923, 1207676383, 2061193945, 196943407, 1925383922, 3870569689, 3894282386, 1212693631, 1404071290, 3046879375, 2924493982, 3410043685, 3809721374, 2324452872, 834464932, 2330176976, 2860749140, 2576919325, 1731581763, 2734233631, 1307721060, 1364026888, 3650978588, 1517407670, 2245310205, 1530006937, 2955120239, 3188064498, 3417491514, 2397531047, 459089077, 4053955510, 443399345, 1796085719, 1924236593, 2043248158, 186441553, 2326275902, 2472457891, 3132406322, 939610904, 2208858507, 945878056, 109589586, 548812211, 1240173202, 3204593993, 721035164, 733233892, 4129866603, 2629432835, 103841909, 4233482652, 850252801, 662253897, 1653616235, 3491341022, 475692128, 2503334188, 355990982, 2296073816, 838210242, 3287833079, 827619964, 3900262990, 1395434945, 3626514356, 3890029739, 3246993302, 2756484700, 1020069034, 3097281098, 448129499, 3328003156, 350913378, 2016280706, 2504330849, 1069039051, 2606681503, 4105161434, 3689449592, 2290999300, 890022371, 1248474502, 2805549586, 3381461720, 3786043854, 2720553646, 994750777, 915794947, 2191214736, 207734182, 2034668448, 3462612507, 2931162527, 3404334608, 3955422702, 4101042191, 4245026154, 3080687510, 241496869, 3336337400, 3961557738, 1762565244, 982272855, 201112283, 839026428, 1053533360, 3454224402, 1310868263, 2693848699, 2099992077, 3451484296, 769531652, 2865700439, 1878592864, 2721136361, 2214123932, 3828307988, 209596659, 741099773, 3199029910, 1334965681, 1544576669, 3504078266, 25459822, 3656324319, 2163345613, 3538725464, 4097275680, 3727367002, 2639924031, 3133419113, 1947965051, 1635935439, 1049126814, 3324554030, 3132507893, 3038879776, 329990888, 3582827510, 2917953579, 3027357988, 1091197630, 1490696964, 3381934548, 1603833838, 3482203517, 2183868252, 3349292431, 2412825052, 1716350840, 1475395653, 431844396, 2931362300, 1504434771, 2130908322, 4187762748, 2537958471, 4258559043, 679552214, 4150134889, 2946590212, 50221051, 3358795689, 2017939006, 850243093, 3672955852, 249808095, 2780041635, 942783646, 574550556, 3534933632, 547576575, 2354608175, 1511213271, 4913332, 1354252006, 3751258543, 137446248, 3343068228, 3233484676, 1974227414, 3362209801, 2838722690, 2222596348, 3987178007, 3099562962, 1447343198, 2266421070, 2171410780, 724137152, 3590971377, 642460272, 358356132, 2516335475, 2281385363, 1673631698, 567570379, 2820831217, 3459305266, 593075908, 685000684, 1637393981, 3084948420, 3808715397, 786812600, 1404480468, 3736739260, 4046356764, 2277487134, 273026609, 121302633, 4029458939, 3880997922, 824988722, 1623515710, 1297465894, 3789533293, 207730144, 2664490926, 331830354, 1133560012, 2069816875, 927389900, 1259060843, 941566906, 835436573, 3953926187, 3930136621, 389315519, 4183568153, 2881497915, 1324589511, 2048359325, 1043570944, 2083395465, 1478226476, 3087455126, 23006780, 606435476, 494531655, 2130588953, 3466795601, 3747817133, 3211608416, 1749127250, 1473512503, 4141802367, 4009020451, 2169062501, 486602714, 3005612659, 260005846, 1336645235, 3457721846, 4035714886, 3551248844, 543614424, 2620675740, 972732959, 2146370990, 2444228783, 3757226413, 1787341042, 1622705784, 1002005711, 2926567845, 2450964264, 275604472, 462449927, 372609898, 2731933797, 3288882335, 28113805, 2715428189, 3304559815, 2784867699, 3629574622, 3881588857, 772925804, 3315006110, 2788002026, 1552017355, 1158208914, 3011680491, 465445743, 1709915594, 120028768, 2125877175, 3730264378, 3321247512, 3213599728, 2381392366, 136022703, 55061352, 1075511919, 1650581160, 1366656740, 949909697, 1448637601, 361121780, 3717238255, 3525154122, 734851236, 76931684, 3646466304, 1361374960, 71539262, 2847860259, 103976093, 2690312143, 2203268177, 2351464896, 3376181365, 1606789530, 2476053465, 912604440, 3183693432, 3672535607, 2110979845, 636817180, 549564144, 412837280, 40658714, 1197547156, 4071713833, 3278498849, 2457986482, 2570925070, 2773406614, 2173220569, 1144522688, 3620770026, 1622735997, 2368411802, 2011454842, 1827133664, 2585489840, 3315467888, 1711780406, 288791384, 3181545850, 703884497, 3482409752, 3690605116, 1256264409, 3474559514, 2575819952, 347868803, 1414887195, 3049697508, 2286946758, 1924076795, 3990073716, 343931456, 1347596990, 1443539675, 2711173653, 1516854237, 2203898925, 3570346377, 785743864, 78265891, 1764085862, 1871947520, 3185452642, 2924684299, 2788837988, 2502034127, 2492118039, 12747637, 2645726181, 1779979579, 2417304151, 2013047543, 2938718119, 2151179730, 1964454002, 1323558464, 182394583, 25293277, 846729988, 3138476563, 1376710613, 3520885930, 1300980659, 2039674586, 3257949269, 2308477698, 2463152670, 1053115398, 3638265799, 96951187, 1351905935, 2209891741, 748742656, 1115277914, 3976004334, 3353523651, 2009742989, 115220113, 387413820, 12], null]\"}','2018-02-12 08:41:20','2018-02-12 08:41:20'); - --- INSERT INTO job_saved_state (job_definition_id, saved_state) VALUES (100003,'{}'); - -INSERT INTO job_suggested_param_value(id, job_execution_id, tuning_parameter_id, param_value, created_ts, updated_ts) VALUES (3209,1541,1,149.5239493606563,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3210,1541,2,1536,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3211,1541,3,10,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3212,1541,4,0.761466551875019,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3213,1541,5,2844.365182469904,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3214,1541,6,536870912,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3215,1541,7,2133.273886852428,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3216,1541,8,1152,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3217,1541,9,536870912,'2018-02-14 05:30:42','2018-02-14 05:30:42'), (3218,1542,1,162.7907384046847,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3219,1542,2,1536,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3220,1542,3,10,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3221,1542,4,0.7632081784681852,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3222,1542,5,2809.5806453243313,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3223,1542,6,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3224,1542,7,2107.1854839932485,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3225,1542,8,1152,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3226,1542,9,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3227,1543,1,124.05878355054111,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3228,1543,2,1536,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3229,1543,3,12.521341191290857,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3230,1543,4,0.7622909149004323,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3231,1543,5,2041.562366831904,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3232,1543,6,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3233,1543,7,1531.171775123928,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3234,1543,8,1152,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3235,1543,9,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3236,1544,1,149.51252503919468,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3237,1544,2,1536,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3238,1544,3,10,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3239,1544,4,0.7619961998308155,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3240,1544,5,2844.326081249364,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3241,1544,6,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3242,1544,7,2133.244560937023,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3243,1544,8,1152,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3244,1544,9,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3245,1545,1,159.88928672056016,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3246,1545,2,1536,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3247,1545,3,10,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3248,1545,4,0.770164839202443,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3249,1545,5,2863.372720073011,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3250,1545,6,536870912,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3251,1545,7,2147.5295400547584,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3252,1545,8,1152,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3253,1545,9,536870912,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3254,1546,1,201.64225529876035,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3255,1546,2,1536,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3256,1546,3,10.29839988592941,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3257,1546,4,0.7635183100860585,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3258,1546,5,2789.189282499988,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3259,1546,6,536870912,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3260,1546,7,2091.891961874991,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3261,1546,8,1152,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3262,1546,9,536870912,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3263,1547,1,149.52419594024295,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3264,1547,2,1536,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3265,1547,3,10,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3266,1547,4,0.7630834894363029,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3267,1547,5,2844.1716734703073,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3268,1547,6,536870912,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3269,1547,7,2133.1287551027303,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3270,1547,8,1152,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3271,1547,9,536870912,'2018-02-14 07:29:48','2018-02-14 07:29:48'); +INSERT INTO job_execution(id, job_exec_id, job_exec_url, job_definition_id, flow_execution_id, execution_state, resource_usage, execution_time, input_size_in_bytes, created_ts, updated_ts) VALUES +(1541,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0',100003,1496,'SUCCEEDED',21.132545572916666,3.2694833333333335,324713861757,'2018-02-14 05:30:42','2018-02-14 05:30:42'), +(1542,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416389&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416389&job=countByCountryFlow_countByCountry&attempt=0',100003,1497,'SUCCEEDED',23.334004991319443,3.6118166666666665,324713861757,'2018-02-14 06:29:45','2018-02-14 06:29:45'), +(1543,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416495&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416495&job=countByCountryFlow_countByCountry&attempt=0',100003,1498,'SUCCEEDED',21.28552951388889,3.2940833333333335,324713861757,'2018-02-14 06:29:45','2018-02-14 06:29:45'), +(1544,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416589&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416589&job=countByCountryFlow_countByCountry&attempt=0',100003,1499,'SUCCEEDED',21.630970052083335,3.9560833333333334,324713861757,'2018-02-14 06:29:45','2018-02-14 06:29:45'), +(1545,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416680&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416680&job=countByCountryFlow_countByCountry&attempt=0',100003,1500,'SUCCEEDED',22.328486328125,3.7285166666666667,324713861757,'2018-02-14 07:29:47','2018-02-14 07:29:48'), +(1546,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416818&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416818&job=countByCountryFlow_countByCountry&attempt=0',100003,1501,'SUCCEEDED',32.16945149739583,5.203783333333333,324713861757,'2018-02-14 07:29:48','2018-02-14 07:29:48'), +(1547,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5417057&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5417057&job=countByCountryFlow_countByCountry&attempt=0',100003,1502,'SUCCEEDED', 27.2955078125, 4.047583333333334, 324713861757,'2018-02-14 07:29:48','2018-02-14 07:29:48'); + +INSERT INTO job_suggested_param_set(fitness_job_execution_id, tuning_algorithm_id, param_set_state, is_param_set_default, fitness, is_param_set_best, are_constraints_violated, job_definition_id,id) VALUES +(1541,1,'FITNESS_COMPUTED',0,0.06987967161749142,0,0,100003,1541), +(1542,1,'FITNESS_COMPUTED',0,0.07715930864495756,0,0,100003,1542), +(1543,1,'FITNESS_COMPUTED',0,0.07038554856075895,0,0,100003,1543), +(1544,1,'FITNESS_COMPUTED',0,0.07152782795578526,0,0,100003,1544), +(1545,1,'FITNESS_COMPUTED',0,0.07383432757503201,0,0,100003,1545), +(1546,1,'FITNESS_COMPUTED',0,0.10637576523832741,0,0,100003,1546), +(1547,1,'FITNESS_COMPUTED',0,0.09025893809095505,0,0,100003,1547); + +INSERT INTO job_suggested_param_value(id, job_suggested_param_set_id, tuning_parameter_id, param_value, created_ts, updated_ts) VALUES +(3209,1541,1,149.5239493606563,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3210,1541,2,1536,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3211,1541,3,10,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3212,1541,4,0.761466551875019,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3213,1541,5,2844.365182469904,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3214,1541,6,536870912,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3215,1541,7,2133.273886852428,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3216,1541,8,1152,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3217,1541,9,536870912,'2018-02-14 05:30:42','2018-02-14 05:30:42'), (3218,1542,1,162.7907384046847,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3219,1542,2,1536,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3220,1542,3,10,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3221,1542,4,0.7632081784681852,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3222,1542,5,2809.5806453243313,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3223,1542,6,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3224,1542,7,2107.1854839932485,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3225,1542,8,1152,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3226,1542,9,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3227,1543,1,124.05878355054111,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3228,1543,2,1536,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3229,1543,3,12.521341191290857,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3230,1543,4,0.7622909149004323,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3231,1543,5,2041.562366831904,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3232,1543,6,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3233,1543,7,1531.171775123928,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3234,1543,8,1152,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3235,1543,9,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3236,1544,1,149.51252503919468,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3237,1544,2,1536,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3238,1544,3,10,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3239,1544,4,0.7619961998308155,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3240,1544,5,2844.326081249364,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3241,1544,6,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3242,1544,7,2133.244560937023,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3243,1544,8,1152,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3244,1544,9,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3245,1545,1,159.88928672056016,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3246,1545,2,1536,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3247,1545,3,10,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3248,1545,4,0.770164839202443,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3249,1545,5,2863.372720073011,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3250,1545,6,536870912,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3251,1545,7,2147.5295400547584,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3252,1545,8,1152,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3253,1545,9,536870912,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3254,1546,1,201.64225529876035,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3255,1546,2,1536,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3256,1546,3,10.29839988592941,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3257,1546,4,0.7635183100860585,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3258,1546,5,2789.189282499988,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3259,1546,6,536870912,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3260,1546,7,2091.891961874991,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3261,1546,8,1152,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3262,1546,9,536870912,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3263,1547,1,149.52419594024295,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3264,1547,2,1536,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3265,1547,3,10,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3266,1547,4,0.7630834894363029,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3267,1547,5,2844.1716734703073,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3268,1547,6,536870912,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3269,1547,7,2133.1287551027303,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3270,1547,8,1152,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3271,1547,9,536870912,'2018-02-14 07:29:48','2018-02-14 07:29:48'); diff --git a/test/resources/tunein-test1.sql b/test/resources/tunein-test1.sql index ebb33dd33..aee0f0d22 100644 --- a/test/resources/tunein-test1.sql +++ b/test/resources/tunein-test1.sql @@ -1,24 +1,44 @@ -INSERT INTO `flow_definition`(id, flow_def_id, flow_def_url) VALUES (57,'https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images'),(60,'https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall'); -INSERT INTO `job_definition`(id, job_def_id, flow_definition_id, job_name, job_def_url, scheduler, username, created_ts, updated_ts) VALUES (51,'https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action',57,'score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','azkaban','dev_svc',parsedatetime('2018-01-13 17:27:09','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-13 17:27:09','dd-MM-yyyy hh:mm:ss')),(54,'https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall&job=countByCountryFlowSmall_countByCountry',60,'countByCountryFlowSmall_countByCountry','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall&job=countByCountryFlowSmall_countByCountry','azkaban','mkumar1',parsedatetime('2018-01-22 11:41:12','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-22 11:41:12','dd-MM-yyyy hh:mm:ss')); -INSERT INTO `tuning_job_definition`(job_definition_id, client, tuning_algorithm_id, tuning_enabled, average_resource_usage, average_execution_time, average_input_size_in_bytes, allowed_max_resource_usage_percent, allowed_max_execution_time_percent, created_ts, updated_ts, tuning_disabled_reason) VALUES +INSERT INTO `flow_definition`(id, flow_def_id, flow_def_url) VALUES +(57,'https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images'), +(60,'https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall'); + + +INSERT INTO `job_definition`(id, job_def_id, flow_definition_id, job_name, job_def_url, scheduler, username, created_ts, updated_ts) VALUES +(51,'https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action',57,'score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','azkaban','dev_svc',parsedatetime('2018-01-13 17:27:09','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-13 17:27:09','dd-MM-yyyy hh:mm:ss')), +(54,'https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall&job=countByCountryFlowSmall_countByCountry',60,'countByCountryFlowSmall_countByCountry','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall&job=countByCountryFlowSmall_countByCountry','azkaban','mkumar1',parsedatetime('2018-01-22 11:41:12','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-22 11:41:12','dd-MM-yyyy hh:mm:ss')); + +INSERT INTO `tuning_job_definition`(job_definition_id, client, tuning_algorithm_id, tuning_enabled, average_resource_usage, average_execution_time, +average_input_size_in_bytes, allowed_max_resource_usage_percent, allowed_max_execution_time_percent, created_ts, updated_ts, tuning_disabled_reason) VALUES (51,'azkaban',1,1,458.4372180906033,61.92735384666667,5845242598595,200,200,parsedatetime('2018-01-13 17:27:09','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-13 17:28:34','dd-MM-yyyy hh:mm:ss'), NULL), (54,'azkaban',1,1,0.09060929361979166,1.8950733333333332,532614133,200,200,parsedatetime('2018-01-22 11:41:12','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-22 15:49:33','dd-MM-yyyy hh:mm:ss'), NULL); + INSERT INTO `flow_execution`(id, flow_exec_id, flow_exec_url, flow_definition_id) VALUES (846,'https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/executor?execid=5157830',57),(847,'https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/executor?execid=5158194',57),(848,'https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/executor?execid=5158533',57),(2035,'https://elephant.linkedin.com:8443/executor?execid=5356366','https://elephant.linkedin.com:8443/executor?execid=5356366',60),(2036,'https://elephant.linkedin.com:8443/executor?execid=5356377','https://elephant.linkedin.com:8443/executor?execid=5356377',60),(2047,'https://elephant.linkedin.com:8443/executor?execid=5356853','https://elephant.linkedin.com:8443/executor?execid=5356853',60); -INSERT INTO `job_execution`(id, job_exec_id, job_exec_url, job_definition_id, flow_execution_id, execution_state, resource_usage, execution_time, input_size_in_bytes, created_ts, updated_ts) VALUES (1624,'https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0',51,847,'SUCCEEDED',196.53583333333333,43.996566666666666,3990998122525,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:08:44','dd-MM-yyyy hh:mm:ss')),(1625,'https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0',51,846,'SUCCEEDED',169.09791666666666,34.86125,3990827471029,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:58:41','dd-MM-yyyy hh:mm:ss')),(1626,'https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0',51,848,'SUCCEEDED',166.11400634765624,33.7234,3990138654674,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(1627,NULL,NULL,51,NULL,NULL,NULL,NULL,NULL,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(1628,NULL,NULL,51,NULL,NULL,NULL,NULL,NULL,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(1629,NULL,NULL,51,NULL,NULL,NULL,NULL,NULL,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(1719,'https://elephant.linkedin.com:8443/executor?execid=5356366&job=countByCountryFlowSmall_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5356366&job=countByCountryFlowSmall_countByCountry&attempt=0',54,2035,'FAILED',0,0,0,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-09 05:52:42','dd-MM-yyyy hh:mm:ss')),(1720,'https://elephant.linkedin.com:8443/executor?execid=5356377&job=countByCountryFlowSmall_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5356377&job=countByCountryFlowSmall_countByCountry&attempt=0',54,2036,'SUCCEEDED',0.07686197916666666,3.5777,540598828,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-09 05:52:42','dd-MM-yyyy hh:mm:ss')),(1721,'https://elephant.linkedin.com:8443/executor?execid=5356853&job=countByCountryFlowSmall_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5356853&job=countByCountryFlowSmall_countByCountry&attempt=0',54,2047,'SUCCEEDED',0.21555555555555556,3.5456666666666665,540598828,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(2833,NULL,NULL,54,NULL,NULL,NULL,NULL,NULL,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(2834,NULL,NULL,54,NULL,NULL,NULL,NULL,NULL,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(2835,NULL,NULL,54,NULL,NULL,NULL,NULL,NULL,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')); -INSERT INTO `tuning_job_execution`(job_execution_id, tuning_algorithm_id, param_set_state, is_default_execution, fitness, is_param_set_best) VALUES (1624,1,'FITNESS_COMPUTED',0,0.05287618226970775,0), -(1625,1,'FITNESS_COMPUTED',0,0.04549620518409709,0), -(1626,1,'FITNESS_COMPUTED',0,0.044701092268747876,0), -(1627,1,'CREATED',0,NULL,0), -(1628,1,'CREATED',0,NULL,0), -(1629,1,'CREATED',0,NULL,0), -(1719,1,'FITNESS_COMPUTED',0,1.0960015760903588,0), -(1720,1,'FITNESS_COMPUTED',0,0.15266389313494158,0), -(1721,1,'FITNESS_COMPUTED',0,1.079813530812908,0), -(2833,1,'FITNESS_COMPUTED',0,0.54365576171875,0), -(2834,1,'CREATED',0,NULL,0), -(2835,1,'CREATED',0,NULL,0); ---INSERT INTO `job_saved_state` VALUES (51,'{\"current_population\":[{\"birthdate\":1.515995961168332E9,\"maximize\":false,\"candidate\":[171.02605253760447,1536.0,54.84590775607139,0.7115250205281223,1536.0,5.36870912E8],\"fitness\":1.0E9,\"paramSetId\":1627,\"_candidate\":[171.02605253760447,1536.0,54.84590775607139,0.7115250205281223,1536.0,5.36870912E8]},{\"birthdate\":1.515995961168334E9,\"maximize\":false,\"candidate\":[181.9245696813968,1536.0,42.047012100221565,0.7037704502574763,1536.0,5.36870912E8],\"fitness\":1.0E9,\"paramSetId\":1628,\"_candidate\":[181.9245696813968,1536.0,42.047012100221565,0.7037704502574763,1536.0,5.36870912E8]},{\"birthdate\":1.515995961168335E9,\"maximize\":false,\"candidate\":[213.25485474773512,1536.0,31.525782197486357,0.7476923282427653,1581.3588284589678,5.36870912E8],\"fitness\":1.0E9,\"paramSetId\":1629,\"_candidate\":[213.25485474773512,1536.0,31.525782197486357,0.7476923282427653,1581.3588284589678,5.36870912E8]}],\"prev_population\":[{\"_candidate\":[218.6447449212757,1536.0,43.796210012462936,0.7053483939633272,1536.0,5.36870912E8],\"maximize\":false,\"birthdate\":1.515995961167665E9,\"fitness\":0.05287618226970775},{\"_candidate\":[181.9245696813968,1536.0,42.047012100221565,0.7037704502574763,1536.0,5.36870912E8],\"maximize\":false,\"birthdate\":1.515995961167667E9,\"fitness\":0.04549620518409709},{\"_candidate\":[140.63900164342985,1536.0,51.426234701754865,0.6951498772580411,1976.5719588408601,5.36870912E8],\"maximize\":false,\"birthdate\":1.515995961167669E9,\"fitness\":0.044701092268747876}],\"archive\":[{\"birthday\":1.515985385404277E9,\"_candidate\":[219.39716200011478,1536.0,66.26037015196239,0.6912182600949739,1536.0,5.36870912E8],\"maximize\":false,\"birthdate\":1.515995961166964E9,\"fitness\":0.0428721215116997},{\"birthday\":1.515985385403604E9,\"_candidate\":[181.9245696813968,1536.0,42.047012100221565,0.7037704502574763,1536.0,5.36870912E8],\"maximize\":false,\"birthdate\":1.515995961166971E9,\"fitness\":0.039280067116591405},{\"birthday\":1.515985385403608E9,\"_candidate\":[275.86885706468235,1536.0,31.422304005393446,0.7301792632677065,2015.8347602357646,5.36870912E8],\"maximize\":false,\"birthdate\":1.515995961166975E9,\"fitness\":0.04447846959473768}],\"rnd_state\":\"[3, [700660853, 980765611, 350269045, 550794061, 3508660058, 2280194506, 372390680, 2086446374, 239386402, 971362604, 4241231975, 858459806, 3033502845, 814217089, 177306281, 1390604632, 2849717269, 3701489124, 2643708056, 1542339197, 1555385004, 3634877832, 573164398, 566979896, 2206975893, 1503874227, 1236610411, 2573036830, 2711844444, 4209994160, 3990240984, 128106381, 663247204, 1653265324, 3765434946, 3330504426, 4276961481, 837144373, 3095304940, 984937884, 139122733, 4043403675, 1544045642, 2533499146, 3571481965, 962064161, 3801182180, 2403202969, 3721566863, 1565524117, 822198358, 3375360787, 3509804184, 3870723125, 1875023332, 3008398872, 3751590940, 3309048627, 1182977075, 3777424954, 2883041912, 2657794984, 3378766220, 3152549006, 2684336427, 2187987130, 3983079230, 547491665, 1289795226, 854873566, 1972709764, 1765551945, 3392981810, 3517935071, 2310497346, 3676457131, 1798697065, 3091394835, 1290859076, 144558004, 1796870482, 1477299519, 1427562024, 549877811, 1597433131, 807147957, 3905500832, 261506349, 2560082346, 3179854567, 3240841291, 413948645, 2250452566, 1819800580, 3883891184, 632396012, 1519436197, 1409698969, 3375891321, 1473582645, 829906832, 2390021020, 3047321716, 2571315456, 2450064940, 2015213036, 3701906702, 1611515348, 4040359636, 3318495044, 1604983146, 2492332685, 789975929, 3194197706, 4147041789, 1135227509, 3723966140, 1499070128, 1495663007, 4004414959, 4129672433, 1784336180, 1506720970, 3367678585, 1049352334, 952862338, 1367689768, 3421301844, 3992023287, 3787999149, 607781132, 899298268, 2429143458, 1782950959, 3532504857, 1132237626, 2088890029, 1760382924, 1130657110, 3363127838, 3043321367, 2945251619, 1568851313, 2735984651, 766093001, 3088031430, 3822048557, 772213702, 1219747991, 1167235346, 1148978281, 1418791530, 2671812486, 3026023846, 215814664, 2474836935, 3742432459, 3886554830, 1239586734, 2033249033, 2061707261, 3582021973, 1205282201, 2940872178, 1647198252, 864907932, 2580542635, 4231988675, 1726503113, 2353555357, 1827771300, 3600123622, 4005348928, 2539198000, 4037701258, 2589333679, 3456867157, 1352943897, 1822690465, 2408381181, 1868778566, 378948152, 3749762946, 502288181, 3936503490, 2971031075, 2374321618, 31726935, 860889312, 1515652253, 844878931, 1684891814, 1454985016, 2424639673, 2260089345, 2799910537, 3496783614, 3225515131, 3459630561, 2432878302, 1646494133, 2227658705, 1257136958, 3706525986, 1094286407, 850469210, 2995793692, 2211513328, 1755894528, 3571575386, 1708963361, 2362822514, 2224335253, 4293751999, 1805463544, 2453039741, 1886293255, 3289381616, 2288127674, 2172693547, 2982972306, 3215964883, 888146238, 2417861490, 7187980, 3713409018, 738436957, 579588742, 2709938763, 3959994685, 1948771268, 2379578759, 1434401739, 1013355849, 2528670801, 2524228596, 1098550737, 3832353534, 4218333751, 2179914854, 3741310823, 1659901498, 415821831, 520299428, 1153497622, 1635045824, 2373128365, 1511012134, 2109214492, 3941675131, 2968940495, 1077608592, 546487837, 560757984, 2830915546, 1277936098, 2984058125, 3107790490, 1928094975, 883897300, 2378898857, 2535418754, 1991596485, 307019245, 4253943697, 2102844823, 2387327534, 1295679535, 257012774, 2644984219, 496082329, 3302964332, 799662144, 4064163474, 1223259384, 3774897138, 1573702522, 2325115513, 729338819, 353910412, 716335921, 1209662376, 4070293169, 1270690977, 1695305659, 697493579, 2651477605, 730652454, 3792581705, 1327538356, 2327699294, 562929078, 3595526182, 4113762841, 1829261651, 1794412928, 418993809, 4047909193, 3312977071, 1977410559, 3787797897, 684781661, 1694762123, 659979161, 1871116812, 915617124, 1116356383, 1136153717, 4222588389, 3252249914, 4119031047, 3508584846, 2789171200, 61632834, 1952607520, 3315629861, 856596078, 2412263936, 2305926159, 776419518, 271200364, 2257276232, 603198351, 3194379378, 1151535969, 2880976824, 1701962330, 930448142, 1207562140, 1406230557, 1188380320, 3125895500, 175152937, 4249855047, 216407144, 1520784966, 4072420197, 886879248, 4007320206, 506365179, 1851689952, 3197299601, 1213562517, 954668668, 3567328526, 3626984841, 255329757, 302263481, 1043515423, 2971370561, 1282039976, 1697427886, 3639571623, 3235043793, 1889879468, 1716754125, 2664459343, 383321614, 3172541016, 938820727, 1437985042, 1864633700, 1346599334, 3493651062, 813423282, 3136060154, 1298327746, 960933564, 2589411966, 407108904, 1328419689, 630919073, 1136835355, 2886715215, 3562424719, 495242583, 558900013, 4158088180, 2904510921, 2567796600, 1598024941, 2892566110, 3703661632, 1541123992, 4115353702, 1209327198, 4005599061, 3947903442, 2786172353, 1230913432, 1724749753, 2587833659, 965981368, 344510153, 3951342280, 3750898439, 2523499260, 3731616870, 293764110, 321250565, 3970020252, 2406840029, 2748315886, 1254048724, 1753412283, 3984132930, 3264149240, 3078734508, 1107941005, 3567886053, 2404486768, 1977906686, 4221601803, 610582347, 3157608813, 3522391435, 3369971915, 1634098420, 1035231476, 2305434161, 1097389492, 4129366118, 62724611, 436096798, 2879207403, 2045009166, 4184819944, 3434611449, 1783700513, 3421889769, 721589734, 3824308930, 73869467, 3677536367, 330561138, 167572255, 4289033113, 4228147727, 4102390954, 2285859672, 3394854823, 319437854, 1302681615, 2461968483, 225425264, 2398301733, 327178348, 3374586849, 2263215758, 1148329531, 1658710282, 3122582876, 4218881881, 1375881267, 3661335421, 2866320394, 2610257014, 1001734510, 326380055, 2920059200, 2935750748, 1949485840, 455098597, 863729740, 1151764749, 2716044236, 3942654125, 3622670978, 2544737265, 2421681654, 3782970175, 2918035276, 1850235112, 662428057, 4015171728, 2273865476, 3153121123, 670489394, 3680101668, 3765364135, 3414953034, 1729149762, 1092291501, 3631975206, 1968379780, 4033704983, 1038531297, 1379024270, 350261516, 662580983, 2666693522, 109449532, 3945334796, 520141517, 68422090, 428256539, 3160977087, 4150490870, 1519097291, 399058475, 4044038884, 429407636, 2681023844, 3959975732, 17861206, 1324335466, 3020138261, 4094586935, 1756933566, 3958575872, 405493436, 3255720634, 4187397878, 3412452110, 3660056351, 3684801802, 913893747, 3654082722, 3323334105, 1819591112, 1880678757, 889448818, 3748811275, 309205782, 2312707061, 1171686701, 2674633737, 1550440591, 2436816799, 2702844283, 2695031040, 201888479, 3452874601, 970377544, 1460081872, 2744496799, 748920411, 829469534, 1426574992, 1628101393, 3245963172, 2652123456, 3154243638, 566872445, 2861698301, 3936811112, 1790296253, 3506496756, 396730205, 1076765041, 128885341, 2260558166, 30213662, 3891947035, 2110845573, 2636313809, 1421844627, 1055180454, 3897169890, 2369585737, 2948350514, 1622381331, 2499854322, 3008886194, 1370901601, 2285886995, 2659664530, 452806941, 734310705, 38974153, 1266839691, 4040765272, 451966518, 1034593261, 3996792012, 3262702261, 2095968672, 2064347438, 1597581201, 2195067931, 3277290478, 181898705, 2859461848, 194913269, 3518323108, 1911542790, 1323553652, 1552994486, 2365121217, 3305517045, 1197074508, 1904733312, 2752495057, 3837469761, 2421121539, 1729750982, 2322937327, 638292048, 130682510, 4110785961, 709631645, 435142599, 2912303377, 3032706254, 3625438081, 2521030328, 481154033, 2357799157, 1791268394, 849004735, 481956827, 448804740, 135372808, 2781507049, 2021713577, 818843241, 3803606416, 1787431803, 3045742311, 2023574429, 1185539163, 1216654837, 789835050, 1535157273, 276], null]\"}',parsedatetime('2018-01-13 18:14:06','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(54,'{\"current_population\":[{\"birthdate\":1.518544975273291E9,\"maximize\":false,\"candidate\":[1459.6899166265118,2421.8450860158327,45.39309760195506,0.7338175361303875,2690.0010009930634,5.36870912E8],\"fitness\":10000.0,\"paramSetId\":2833,\"_candidate\":[1459.6899166265118,2421.8450860158327,45.39309760195506,0.7338175361303875,2690.0010009930634,5.36870912E8]},{\"birthdate\":1.518544975273293E9,\"maximize\":false,\"candidate\":[678.4611819565604,2233.783342819729,67.46065454683335,0.7337482196069645,3068.4288351321584,5.36870912E8],\"fitness\":10000.0,\"paramSetId\":2834,\"_candidate\":[678.4611819565604,2233.783342819729,67.46065454683335,0.7337482196069645,3068.4288351321584,5.36870912E8]},{\"birthdate\":1.518544975273295E9,\"maximize\":false,\"candidate\":[888.1718411828987,2391.4206410271354,63.83042592878694,0.7326134736198681,2987.2156090895137,5.36870912E8],\"fitness\":10000.0,\"paramSetId\":2835,\"_candidate\":[888.1718411828987,2391.4206410271354,63.83042592878694,0.7326134736198681,2987.2156090895137,5.36870912E8]}],\"prev_population\":[{\"_candidate\":[792.634660843181,2492.7849019072983,60.45853018964071,0.7336072933432902,2962.9708043614564,5.36870912E8],\"maximize\":false,\"birthdate\":1.518544975272647E9,\"fitness\":1.0960015760903588},{\"_candidate\":[872.5176554279782,2535.6175654819463,44.33267220160804,0.7337286195207247,3083.5815515791987,5.36870912E8],\"maximize\":false,\"birthdate\":1.518544975272649E9,\"fitness\":0.15266389313494158},{\"_candidate\":[885.9516064178075,2085.536516326114,42.55444706800388,0.7348330520625498,3354.63139252968,5.36870912E8],\"maximize\":false,\"birthdate\":1.518544975272651E9,\"fitness\":1.079813530812908}],\"archive\":[{\"birthday\":1.516680401836294E9,\"_candidate\":[1295.8008807732717,2637.186857988159,42.28551979944946,0.7335562104905394,3356.143076920849,5.36870912E8],\"maximize\":false,\"birthdate\":1.518544975271854E9,\"fitness\":0.10811803652893556},{\"birthday\":1.516680401837006E9,\"_candidate\":[814.7560112996814,2337.07365521413,54.984365215088005,0.7337628011079379,2957.576817126595,5.36870912E8],\"maximize\":false,\"birthdate\":1.51854497527186E9,\"fitness\":0.0912464245175376},{\"birthday\":1.516680401837007E9,\"_candidate\":[712.2723914467045,2063.6121485420485,62.79467361078451,0.732881328518512,3039.7339669001685,5.36870912E8],\"maximize\":false,\"birthdate\":1.518544975271864E9,\"fitness\":0.09649709137100561}],\"rnd_state\":\"[3, [3065724126, 3544512728, 2711128429, 3097451895, 967083626, 2260479002, 2011495360, 3348237249, 2378344902, 2134917540, 2742669276, 3328880153, 2822272025, 2880910783, 1441442737, 3231087398, 1146288325, 1625737643, 1228322673, 3721661216, 575561661, 2074324378, 3288388299, 1544300809, 2639678219, 2232788457, 1814056570, 2999882935, 2251501756, 2780554798, 2514692131, 687916693, 243141057, 610462178, 692982146, 3824954777, 365727273, 3022710399, 515788952, 4291459547, 667212329, 3358240868, 1492129036, 956252795, 2157732118, 1110061891, 2718486345, 4130618438, 2288404453, 1191040542, 507334165, 2697022142, 121071918, 1290502834, 1145014349, 3916513172, 3188936928, 3598964253, 1811700608, 797604560, 1459985284, 396308629, 536293065, 1861432351, 530738018, 2930646177, 3035022980, 3998314976, 1898847351, 1507100324, 1607137972, 1193970507, 1325837842, 959726587, 4042090604, 3224750321, 1424027500, 1443061177, 1237044770, 928822690, 1302176037, 3568612499, 3723148185, 1430469816, 692515008, 1093644720, 1683640516, 4174602693, 2331072580, 2762007312, 3686093550, 25874128, 229571105, 1580926638, 3078362483, 3211385514, 2382600957, 451342856, 1566049870, 877376711, 1242306867, 3657755180, 1303113933, 1237917316, 833353971, 2461580676, 148331876, 1558943638, 3170368962, 214736109, 3872311144, 2456901485, 1795447425, 1184953807, 3605341307, 580593700, 595817309, 3108476809, 3002792197, 1433369078, 2866216661, 3655047686, 3554797575, 1374808059, 2612494986, 1054441843, 3372403829, 897630532, 1434894456, 340091712, 1412255824, 153502320, 1082509383, 2192258062, 1212656009, 4097614680, 4211472344, 792137819, 326652612, 287540853, 649689138, 3991116082, 3709897114, 807708655, 645794625, 2900452471, 2514181414, 4279235972, 1223752520, 2150305618, 1702086023, 872116565, 3864479340, 1675395747, 2268218023, 1917912323, 2037116366, 2058231706, 2644080529, 2722765597, 4021962428, 3021783242, 3681936083, 425397342, 1090869761, 3299571049, 2177660117, 3662979482, 108545502, 3886174031, 2590960463, 695447006, 1487047828, 2178393656, 4025597868, 255761026, 1767121742, 3200850413, 2301247846, 214790591, 2641517169, 3048332331, 2844465691, 2738599240, 186184734, 2048643725, 1496536706, 3201056846, 2594104286, 1684879809, 1470534656, 1563583168, 1492709862, 3921781135, 1231021635, 1143335356, 868590525, 3688084842, 3363015415, 1081697634, 2064351382, 406335233, 1478861566, 2162286375, 2881004503, 1944164859, 3242596185, 2726419641, 816907820, 2930849747, 3927693308, 253252771, 2885236710, 584068619, 2055555515, 1618917124, 3511474096, 2610543554, 1371679093, 3262865804, 3139346620, 3540194239, 4002012437, 3625334564, 1812113079, 166562410, 106177145, 1899457561, 2200707317, 1039681884, 4172463313, 2407766685, 4146126707, 3004768120, 730301797, 3443113670, 3670981937, 586475702, 2947490368, 2948713681, 3450965965, 1293897586, 952983259, 889011649, 3267239110, 4161489389, 1360814786, 501755720, 2816906094, 2842137958, 2660216416, 419200669, 1408294612, 2373223348, 2328800876, 1584237670, 436351506, 3395527473, 1548715965, 3690533652, 2945734992, 674160573, 1989518247, 3100889221, 2763719839, 2357271150, 212780741, 2985115917, 1457888565, 3607299527, 980443153, 1542209158, 770291775, 746678077, 1412193049, 413891745, 1682193555, 765198090, 4230400852, 2045987415, 2340942203, 615884366, 2478755388, 833488046, 50735976, 2993136664, 889095813, 821147425, 2622280532, 706652556, 1844492101, 3799084036, 3330515368, 542924309, 3530527023, 1845988562, 2375814810, 3642831172, 3051224805, 4172514400, 1886513068, 3672472993, 4151775118, 2762329627, 2221399727, 1473049558, 3816789642, 2398938446, 2845606889, 4284897326, 2904177938, 1598351959, 1855445204, 2594015278, 1528024137, 1980953461, 3474774638, 3255002272, 2105370151, 3250245822, 2344078572, 2735834183, 318492123, 2905820022, 4143524390, 1758152212, 26419169, 3504744669, 922897638, 2284226546, 123574126, 2237741988, 171212654, 1776315349, 2849978615, 3228771518, 1184260950, 1032684138, 3181470798, 2673690187, 4173768130, 2623194448, 3133756108, 851870142, 28138823, 1206311932, 741037646, 3670265831, 404270308, 619902761, 3287601440, 3208299557, 2761870961, 2124804014, 796011685, 2971745401, 784966770, 784332828, 578351065, 725624721, 3683445932, 3771024580, 2422147209, 3762031872, 2347660671, 1933243950, 3711755263, 176475674, 1591596727, 3651471810, 4270080122, 353326368, 3734423915, 3894466421, 2605405796, 4108660146, 2956628083, 3316574506, 2589541445, 3572104663, 3153232913, 1053856203, 2895389043, 1365063224, 1671772292, 1925028221, 902873584, 3595807393, 3097817112, 4108789053, 1759616424, 3746272037, 3717258263, 2672653056, 4147971464, 949492224, 942543068, 1119921660, 4008015534, 3435957522, 3184057267, 4167360692, 1249859248, 7558097, 1133845986, 2943139418, 2073521243, 3617451110, 2229897027, 2555647933, 3726176874, 889452764, 820644837, 1327154049, 2265672064, 2129248553, 412606852, 63204922, 3234344952, 320707343, 3092740087, 44853598, 1945226434, 3159550172, 3086394144, 2012592804, 3990228399, 2867389493, 2781617757, 2882552002, 2806351707, 909035806, 1294586605, 3840428648, 602515456, 2997467429, 2755363986, 147729940, 870496959, 3924354611, 1525531182, 1923355511, 1562510856, 3915804036, 2592881318, 648423540, 3725293333, 2269428648, 467073447, 2595389432, 495610460, 2386665387, 116759568, 750721290, 1275499854, 1875377515, 2736580174, 3938718880, 290001066, 3522290030, 29953608, 3064174778, 4090117350, 2624194794, 1376276064, 2665796666, 634445370, 70168509, 32198189, 3001234425, 1631602258, 277495047, 3121947616, 453054115, 2838793569, 4146704714, 2831492717, 596378763, 2178259917, 1832432377, 716277901, 3659429932, 1323761615, 66127783, 384298520, 774313993, 1449039381, 1428144251, 2553931515, 281546735, 2989769939, 3020480002, 3096161781, 2155328679, 3728427667, 4115395479, 4229426936, 1338120410, 3780413236, 3122514188, 3388096654, 2062867830, 3244651183, 832086074, 3991330458, 1136227927, 3604120290, 3712104605, 3520462657, 495374736, 2067270593, 2387664894, 1602148084, 1240393197, 2419712601, 3040375177, 1081946879, 3023252227, 3890705166, 482850694, 4013987459, 1571040181, 1521329843, 1705401874, 1682094409, 1128205548, 1899120173, 241188982, 36185350, 78024647, 509896880, 3877417777, 3269955810, 2336805412, 1743390219, 2792517786, 3058532670, 3731231273, 3104120029, 1006319366, 748566713, 1149848531, 1009523072, 3209403427, 3084624506, 554189074, 4207439260, 2186236223, 2662732177, 335450920, 662450073, 1116746604, 3056019694, 702760901, 3778868255, 2871867096, 1719987053, 3734422564, 39611381, 2168517922, 148059197, 794689514, 3195118058, 2819533870, 3577103586, 1534358750, 2054529561, 1340212594, 1707171826, 980558901, 3541812777, 2304960896, 2998240578, 32794602, 1503372561, 2976532095, 2080503067, 1190284614, 448190919, 1424477249, 3703469626, 278436083, 2339142365, 849039042, 1068240596, 2250237233, 1480326781, 1116357061, 3778674107, 3209970900, 3562388741, 1667085518, 1988254279, 3554198962, 1065233563, 1150015936, 3712709535, 3084752011, 1219620914, 4018979140, 3511230272, 1047524801, 314747652, 4224631036, 570175320, 304193502, 2970362892, 3636317124, 3535211386, 1561392126, 4107979711, 687771063, 1576513160, 3777146659, 3706627510, 2997104755, 1060609260, 3065422355, 1894842799, 3754972445, 3533484985, 2371212970, 2398165984, 1668517555, 588], null]\"}',parsedatetime('2018-01-22 15:58:58','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')); -INSERT INTO `job_suggested_param_value`(id, job_execution_id, tuning_parameter_id, param_value, created_ts, updated_ts) VALUES (12569,1624,1,218.6447449212757,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12570,1624,2,1536,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12571,1624,3,43.796210012462936,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12572,1624,4,0.7053483939633272,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12573,1624,5,1536,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12574,1624,6,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12575,1624,7,1152,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12576,1624,8,1152,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12577,1624,9,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12578,1625,1,181.9245696813968,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12579,1625,2,1536,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12580,1625,3,42.047012100221565,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12581,1625,4,0.7037704502574763,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12582,1625,5,1536,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12583,1625,6,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12584,1625,7,1152,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12585,1625,8,1152,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12586,1625,9,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12587,1626,1,140.63900164342985,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12588,1626,2,1536,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12589,1626,3,51.426234701754865,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12590,1626,4,0.6951498772580411,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12591,1626,5,1976.5719588408601,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12592,1626,6,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12593,1626,7,1482.428969130645,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12594,1626,8,1152,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12595,1626,9,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12596,1627,1,171.02605253760447,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12597,1627,2,1536,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12598,1627,3,54.84590775607139,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12599,1627,4,0.7115250205281223,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12600,1627,5,1536,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12601,1627,6,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12602,1627,7,1152,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12603,1627,8,1152,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12604,1627,9,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12605,1628,1,181.9245696813968,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12606,1628,2,1536,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12607,1628,3,42.047012100221565,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12608,1628,4,0.7037704502574763,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12609,1628,5,1536,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12610,1628,6,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12611,1628,7,1152,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12612,1628,8,1152,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12613,1628,9,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12614,1629,1,213.25485474773512,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12615,1629,2,1536,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12616,1629,3,31.525782197486357,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12617,1629,4,0.7476923282427653,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12618,1629,5,1581.3588284589678,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12619,1629,6,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12620,1629,7,1186.0191213442258,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12621,1629,8,1152,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12622,1629,9,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(13330,1719,1,792.634660843181,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13331,1719,2,2492.7849019072983,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13332,1719,3,60.45853018964071,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13333,1719,4,0.7336072933432902,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13334,1719,5,2962.9708043614564,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13335,1719,6,536870912,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13336,1719,7,2222.2281032710926,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13337,1719,8,1869.5886764304737,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13338,1719,9,536870912,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13339,1720,1,872.5176554279782,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13340,1720,2,2535.6175654819463,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13341,1720,3,44.33267220160804,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13342,1720,4,0.7337286195207247,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13343,1720,5,3083.5815515791987,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13344,1720,6,536870912,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13345,1720,7,2312.686163684399,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13346,1720,8,1901.7131741114597,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13347,1720,9,536870912,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13348,1721,1,885.9516064178075,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13349,1721,2,2085.536516326114,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13350,1721,3,42.55444706800388,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13351,1721,4,0.7348330520625498,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13352,1721,5,3354.63139252968,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13353,1721,6,536870912,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13354,1721,7,2515.97354439726,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13355,1721,8,1564.1523872445855,parsedatetime('2018-01-23 04:06:42','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:42','dd-MM-yyyy hh:mm:ss')),(13356,1721,9,536870912,parsedatetime('2018-01-23 04:06:42','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:42','dd-MM-yyyy hh:mm:ss')),(20462,2833,1,1459.6899166265118,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20463,2833,2,2421.8450860158327,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20464,2833,3,45.39309760195506,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20465,2833,4,0.7338175361303875,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20466,2833,5,2690.0010009930634,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20467,2833,6,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20468,2833,7,2017.5007507447976,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20469,2833,8,1816.3838145118746,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20470,2833,9,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20471,2834,1,678.4611819565604,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20472,2834,2,2233.783342819729,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20473,2834,3,67.46065454683335,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20474,2834,4,0.7337482196069645,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20475,2834,5,3068.4288351321584,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20476,2834,6,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20477,2834,7,2301.3216263491186,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20478,2834,8,1675.3375071147968,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20479,2834,9,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20480,2835,1,888.1718411828987,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20481,2835,2,2391.4206410271354,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20482,2835,3,63.83042592878694,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20483,2835,4,0.7326134736198681,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20484,2835,5,2987.2156090895137,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20485,2835,6,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20486,2835,7,2240.4117068171354,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20487,2835,8,1793.5654807703515,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20488,2835,9,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')); + +INSERT INTO `job_execution`(id, job_exec_id, job_exec_url, job_definition_id, flow_execution_id, execution_state, resource_usage, execution_time, input_size_in_bytes, created_ts, updated_ts) VALUES +(1624,'https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0',51,847,'SUCCEEDED',196.53583333333333,43.996566666666666,3990998122525,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:08:44','dd-MM-yyyy hh:mm:ss')), +(1625,'https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0',51,846,'SUCCEEDED',169.09791666666666,34.86125,3990827471029,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:58:41','dd-MM-yyyy hh:mm:ss')), +(1626,'https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0',51,848,'SUCCEEDED',166.11400634765624,33.7234,3990138654674,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')), +(1719,'https://elephant.linkedin.com:8443/executor?execid=5356366&job=countByCountryFlowSmall_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5356366&job=countByCountryFlowSmall_countByCountry&attempt=0',54,2035,'FAILED',0,0,0,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-09 05:52:42','dd-MM-yyyy hh:mm:ss')), +(1720,'https://elephant.linkedin.com:8443/executor?execid=5356377&job=countByCountryFlowSmall_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5356377&job=countByCountryFlowSmall_countByCountry&attempt=0',54,2036,'SUCCEEDED',0.07686197916666666,3.5777,540598828,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-09 05:52:42','dd-MM-yyyy hh:mm:ss')), +(1721,'https://elephant.linkedin.com:8443/executor?execid=5356853&job=countByCountryFlowSmall_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5356853&job=countByCountryFlowSmall_countByCountry&attempt=0',54,2047,'SUCCEEDED',0.21555555555555556,3.5456666666666665,540598828,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')); + +INSERT INTO `job_suggested_param_set`(fitness_job_execution_id, tuning_algorithm_id, param_set_state, is_param_set_default, fitness, is_param_set_best, are_constraints_violated, job_definition_id,id) VALUES +(1624,1,'FITNESS_COMPUTED',0,0.05287618226970775,0,0,51,1624), +(1625,1,'FITNESS_COMPUTED',0,0.04549620518409709,0,0,51,1625), +(1626,1,'FITNESS_COMPUTED',0,0.044701092268747876,0,0,51,1626), +(NULL,1,'CREATED',0,NULL,0,0,51,1627), +(NULL,1,'CREATED',0,NULL,0,0,51,1628), +(NULL,1,'CREATED',0,NULL,0,0,51,1629), +(1719,1,'FITNESS_COMPUTED',0,1.0960015760903588,0,0,54,1719), +(1720,1,'FITNESS_COMPUTED',0,0.15266389313494158,0,0,54,1720), +(1721,1,'FITNESS_COMPUTED',0,1.079813530812908,0,0,54,1721), +(2833,1,'FITNESS_COMPUTED',0,0.54365576171875,0,0,54,2833), +(NULL,1,'CREATED',0,NULL,0,0,54,2834), +(NULL,1,'CREATED',0,NULL,0,0,54,2835); + +INSERT INTO `job_suggested_param_value`(id, job_suggested_param_set_id, tuning_parameter_id, param_value, created_ts, updated_ts) VALUES +(12569,1624,1,218.6447449212757,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')), +(12570,1624,2,1536,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12571,1624,3,43.796210012462936,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12572,1624,4,0.7053483939633272,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12573,1624,5,1536,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12574,1624,6,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12575,1624,7,1152,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12576,1624,8,1152,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12577,1624,9,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12578,1625,1,181.9245696813968,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12579,1625,2,1536,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12580,1625,3,42.047012100221565,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12581,1625,4,0.7037704502574763,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12582,1625,5,1536,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12583,1625,6,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12584,1625,7,1152,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12585,1625,8,1152,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12586,1625,9,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12587,1626,1,140.63900164342985,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12588,1626,2,1536,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12589,1626,3,51.426234701754865,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12590,1626,4,0.6951498772580411,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12591,1626,5,1976.5719588408601,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12592,1626,6,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12593,1626,7,1482.428969130645,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12594,1626,8,1152,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12595,1626,9,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12596,1627,1,171.02605253760447,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12597,1627,2,1536,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12598,1627,3,54.84590775607139,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12599,1627,4,0.7115250205281223,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12600,1627,5,1536,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12601,1627,6,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12602,1627,7,1152,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12603,1627,8,1152,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12604,1627,9,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12605,1628,1,181.9245696813968,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12606,1628,2,1536,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12607,1628,3,42.047012100221565,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12608,1628,4,0.7037704502574763,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12609,1628,5,1536,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12610,1628,6,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12611,1628,7,1152,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12612,1628,8,1152,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12613,1628,9,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12614,1629,1,213.25485474773512,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12615,1629,2,1536,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12616,1629,3,31.525782197486357,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12617,1629,4,0.7476923282427653,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12618,1629,5,1581.3588284589678,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12619,1629,6,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12620,1629,7,1186.0191213442258,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12621,1629,8,1152,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12622,1629,9,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(13330,1719,1,792.634660843181,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13331,1719,2,2492.7849019072983,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13332,1719,3,60.45853018964071,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13333,1719,4,0.7336072933432902,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13334,1719,5,2962.9708043614564,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13335,1719,6,536870912,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13336,1719,7,2222.2281032710926,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13337,1719,8,1869.5886764304737,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13338,1719,9,536870912,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13339,1720,1,872.5176554279782,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13340,1720,2,2535.6175654819463,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13341,1720,3,44.33267220160804,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13342,1720,4,0.7337286195207247,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13343,1720,5,3083.5815515791987,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13344,1720,6,536870912,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13345,1720,7,2312.686163684399,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13346,1720,8,1901.7131741114597,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13347,1720,9,536870912,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13348,1721,1,885.9516064178075,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13349,1721,2,2085.536516326114,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13350,1721,3,42.55444706800388,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13351,1721,4,0.7348330520625498,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13352,1721,5,3354.63139252968,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13353,1721,6,536870912,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13354,1721,7,2515.97354439726,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13355,1721,8,1564.1523872445855,parsedatetime('2018-01-23 04:06:42','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:42','dd-MM-yyyy hh:mm:ss')),(13356,1721,9,536870912,parsedatetime('2018-01-23 04:06:42','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:42','dd-MM-yyyy hh:mm:ss')),(20462,2833,1,1459.6899166265118,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20463,2833,2,2421.8450860158327,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20464,2833,3,45.39309760195506,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20465,2833,4,0.7338175361303875,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20466,2833,5,2690.0010009930634,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20467,2833,6,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20468,2833,7,2017.5007507447976,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20469,2833,8,1816.3838145118746,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20470,2833,9,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20471,2834,1,678.4611819565604,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20472,2834,2,2233.783342819729,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20473,2834,3,67.46065454683335,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20474,2834,4,0.7337482196069645,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20475,2834,5,3068.4288351321584,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20476,2834,6,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20477,2834,7,2301.3216263491186,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20478,2834,8,1675.3375071147968,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20479,2834,9,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20480,2835,1,888.1718411828987,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20481,2835,2,2391.4206410271354,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20482,2835,3,63.83042592878694,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20483,2835,4,0.7326134736198681,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20484,2835,5,2987.2156090895137,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20485,2835,6,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20486,2835,7,2240.4117068171354,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20487,2835,8,1793.5654807703515,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20488,2835,9,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')); INSERT INTO `yarn_app_result`(id,name,username,queue_name,start_time,finish_time,tracking_url,job_type,severity,score,workflow_depth,scheduler,job_name,job_exec_id,flow_exec_id,job_def_id,flow_def_id,job_exec_url,flow_exec_url,job_def_url,flow_def_url,resource_used,resource_wasted,total_delay) VALUES ('application_1506645932520_14618965','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515864742550,1515864947012,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14618965','Pig',3,1809,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',104318976,16860147,12093),('application_1506645932520_14618978','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515864747221,1515865052849,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14618978','Pig',3,5835,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',349622272,53433482,24885),('application_1506645932520_14618986','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515864749064,1515864871027,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14618986','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',423936,75348,3168),('application_1506645932520_14618990','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515864750944,1515864963817,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14618990','Pig',2,408,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',33875968,5673563,42593),('application_1506645932520_14618994','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515864752042,1515864968073,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14618994','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',985088,149591,3109),('application_1506645932520_14819554','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987211575,1515987345216,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819554','Pig',2,994,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',41111040,0,5629),('application_1506645932520_14819557','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987213364,1515987353613,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819557','Pig',4,6520,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',148128768,0,4803),('application_1506645932520_14819559','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987214957,1515987289457,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819559','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',159744,0,2335),('application_1506645932520_14819561','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987216374,1515987328839,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819561','Pig',2,298,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',12905472,0,2710),('application_1506645932520_14819564','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987217458,1515987328877,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819564','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',242688,0,3036),('application_1506645932520_14819566','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987218578,1515987337373,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819566','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',276480,0,2428),('application_1506645932520_14819567','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987219755,1515987345926,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819567','Pig',2,996,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',39914496,0,4385),('application_1506645932520_14819799','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987384869,1515987654239,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819799','Pig',3,3171,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',84836352,0,8879),('application_1506645932520_14819804','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987386490,1515987624024,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819804','Pig',3,3168,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',86373888,0,7496),('application_1506645932520_14819806','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987387715,1515987628906,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819806','Pig',4,10113,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',204218880,0,12516),('application_1506645932520_14820502','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987672309,1515987731074,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14820502','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',95232,0,4795),('application_1506645932520_14820601','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987743030,1515987809262,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14820601','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',79872,0,4812),('application_1506645932520_14820710','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987821752,1515987866959,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14820710','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',46080,0,5151),('application_1506645932520_14820781','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987880705,1515988060797,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14820781','Pig',1,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',2429952,0,30350),('application_1506645932520_14820903','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515988082745,1515988166282,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14820903','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',1453056,0,4934),('application_1506645932520_14820904','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515988083623,1515988172147,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14820904','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',615936,0,5217),('application_1506645932520_14820960','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515988182390,1515988255602,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14820960','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',328704,0,5271),('application_1506645932520_14820995','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515988270096,1515988320392,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14820995','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',145920,0,4778),('application_1506645932520_14824675','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991076714,1515991231731,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824675','Pig',3,1518,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',51489792,0,3885),('application_1506645932520_14824676','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991079060,1515991237503,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824676','Pig',3,4896,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',158307840,0,5956),('application_1506645932520_14824677','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991080425,1515991187851,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824677','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',233472,0,2877),('application_1506645932520_14824680','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991082049,1515991240032,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824680','Pig',2,302,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',13275648,0,43306),('application_1506645932520_14824683','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991083758,1515991247488,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824683','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',281088,0,40714),('application_1506645932520_14824685','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991085047,1515991265419,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824685','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',287232,0,41607),('application_1506645932520_14824687','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991086759,1515991294536,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824687','Pig',3,1500,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',41547264,0,70495),('application_1506645932520_14824996','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991326143,1515991641190,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824996','Pig',3,3213,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',92054016,0,6387),('application_1506645932520_14824999','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991327452,1515991670369,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824999','Pig',3,3165,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',117229056,0,11406),('application_1506645932520_14825000','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991328750,1515991660916,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14825000','Pig',4,10129,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',243571200,0,19257),('application_1506645932520_14825298','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991684212,1515991767524,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14825298','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',127488,0,4634),('application_1506645932520_14825364','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991778730,1515991873235,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14825364','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',124416,0,4678),('application_1506645932520_14825463','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991888162,1515991971005,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14825463','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',56832,0,37362),('application_1506645932520_14825523','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991982696,1515992198290,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14825523','Pig',1,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',3177984,0,6575),('application_1506645932520_14825763','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515992221397,1515992318476,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14825763','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',1463808,0,18682),('application_1506645932520_14825769','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515992222425,1515992350395,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14825769','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',737280,0,4593),('application_1506645932520_14826095','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515992373009,1515992451751,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14826095','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',347136,0,5224),('application_1506645932520_14826163','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515992466247,1515992538144,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14826163','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',198144,0,5388),('application_1506645932520_14828901','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994413751,1515994540123,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14828901','Pig',3,1527,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',41195520,0,3799),('application_1506645932520_14828909','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994415531,1515994564943,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14828909','Pig',4,6496,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',135762432,0,21059),('application_1506645932520_14828915','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994417787,1515994516440,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14828915','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',192000,0,2372),('application_1506645932520_14828918','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994418955,1515994535247,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14828918','Pig',2,300,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',11008512,0,3043),('application_1506645932520_14828920','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994420115,1515994548888,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14828920','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',271872,0,2402),('application_1506645932520_14828927','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994421136,1515994524042,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14828927','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',215040,0,3006),('application_1506645932520_14828931','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994422273,1515994554177,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14828931','Pig',2,990,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',40410624,0,17972),('application_1506645932520_14829374','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994598599,1515994826719,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14829374','Pig',3,3234,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',76623663,0,13428),('application_1506645932520_14829375','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994599701,1515994811605,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14829375','Pig',3,2643,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',73923525,0,12955),('application_1506645932520_14829377','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994601160,1515994871831,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14829377','Pig',4,10093,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',227204472,0,12798),('application_1506645932520_14829662','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994884732,1515994942466,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14829662','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',100554,0,5346),('application_1506645932520_14829738','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994954814,1515995030508,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14829738','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',102957,0,4962),('application_1506645932520_14829819','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515995045481,1515995093836,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14829819','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',58839,0,5399),('application_1506645932520_14829871','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515995113082,1515995270705,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14829871','Pig',1,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',2724270,0,4927),('application_1506645932520_14830034','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515995291854,1515995356948,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14830034','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',1442952,0,4843),('application_1506645932520_14830035','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515995293330,1515995363322,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14830035','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',609561,0,4607),('application_1506645932520_14830183','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515995379978,1515995475455,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14830183','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',346434,0,35580),('application_1506645932520_14830345','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515995489495,1515995541281,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14830345','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',169446,0,4860),('application_1506645932520_18304591','PigLatin:count_by_country_small.pig','mkumar1','sna_default',1517997086752,1517997308615,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_18304591','Pig',2,2,0,'azkaban','countByCountryFlowSmallNew_countByCountry','https://elephant.linkedin.com:8443/executor?execid=5356377&job=countByCountryFlowSmallNew_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5356377','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmallNew&job=countByCountryFlowSmallNew_countByCountry','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmallNew','https://elephant.linkedin.com:8443/executor?execid=5356377&job=countByCountryFlowSmallNew_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5356377','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmallNew&job=countByCountryFlowSmallNew_countByCountry','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmallNew',28334400,1248,7201),('application_1506645932520_18311651','PigLatin:count_by_country_small.pig','mkumar1','sna_default',1518001319781,1518001775270,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_18311651','Pig',3,6,0,'azkaban','countByCountryFlowSmall_countByCountry','https://elephant.linkedin.com:8443/executor?execid=5221700&job=countByCountryFlowSmall_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5221700','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall&job=countByCountryFlowSmall_countByCountry','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall','https://elephant.linkedin.com:8443/executor?execid=5221700&job=countByCountryFlowSmall_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5221700','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall&job=countByCountryFlowSmall_countByCountry','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall',794624,469908,242749); INSERT INTO `yarn_app_heuristic_result` VALUES (1511830,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1511831,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1511832,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1511833,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1511834,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1511835,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1511836,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1511837,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1511838,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1511839,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1511840,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1511841,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1511842,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1511843,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1511844,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1511845,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1511846,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1511847,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1511848,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1511849,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1511850,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1511851,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1511852,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1511853,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1511866,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1511867,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1511868,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',2,408),(1511869,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1511870,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1511871,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1511872,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1511873,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1511874,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1511875,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1511876,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1511877,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1511878,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1511879,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1511880,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1809),(1511881,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1511882,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1511883,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1511884,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1511885,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1511886,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1511887,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1511888,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1511889,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1511902,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1511903,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1511904,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,5835),(1511905,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1511906,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1511907,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1511908,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1511909,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1511910,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1511911,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1511912,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1511913,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519428,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519429,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519430,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519431,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519432,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519433,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519434,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519435,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519436,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519437,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519438,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519439,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519440,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519441,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519442,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519443,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519444,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519445,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519446,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519447,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519448,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519449,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519450,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519451,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519452,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519453,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519454,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519455,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519456,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519457,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519458,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519459,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519460,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519461,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519462,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519463,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519464,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519465,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519466,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',2,298),(1519467,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519468,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519469,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519470,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519471,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519472,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519473,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519474,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519475,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519476,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519477,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519478,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',2,996),(1519479,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519480,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519481,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519482,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519483,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519484,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519485,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519486,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519487,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519488,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519489,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519490,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',2,994),(1519491,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519492,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519493,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519494,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519495,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519496,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519497,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519498,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519499,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519512,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519513,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519514,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',4,6520),(1519515,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519516,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519517,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519518,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519519,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519520,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519521,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519522,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519523,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519524,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519525,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519526,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1503),(1519527,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519528,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519529,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519530,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519531,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519532,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,1665),(1519533,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519534,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519535,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519536,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519537,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519538,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519539,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519540,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519541,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519542,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519543,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519544,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519545,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519546,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519547,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519548,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519549,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519550,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519551,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519552,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519553,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519554,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519555,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519556,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519557,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519558,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519559,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519560,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519561,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519562,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519563,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519564,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519565,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519566,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519567,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519568,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519569,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519570,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519571,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519572,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519573,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519574,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519575,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519576,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519577,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519578,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',1,0),(1519579,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519580,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519581,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519582,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519583,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519584,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519585,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519586,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519587,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519588,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519589,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519590,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519591,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519592,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519593,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519594,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519595,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519596,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519597,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519598,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519599,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519600,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519601,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519602,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519603,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519604,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519605,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519606,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519607,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519608,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519609,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519610,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519611,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519612,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519613,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519614,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519615,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519616,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519617,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519618,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519619,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519620,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519621,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519622,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519623,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519624,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519625,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519626,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519627,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519628,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519629,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519630,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519631,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519632,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519633,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519634,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1500),(1519635,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519636,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519637,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519638,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519639,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519640,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,1671),(1519641,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519642,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519643,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519644,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519645,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519646,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519647,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519648,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519649,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519650,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519651,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519652,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519653,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519654,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519655,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519656,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519657,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519658,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',2,302),(1519659,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519660,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519661,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519662,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519663,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519664,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519665,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519666,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519667,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519668,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519669,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519670,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1518),(1519671,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519672,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519673,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519674,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519675,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519676,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519677,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519678,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519679,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519680,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519681,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519682,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',4,7116),(1519683,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519684,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519685,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519686,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519687,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519688,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,2997),(1519689,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519690,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519691,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519692,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519693,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519694,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519695,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519696,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519697,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519698,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519699,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519700,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519701,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519702,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519703,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519704,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519705,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519706,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519707,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519708,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519709,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519710,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519711,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519712,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519713,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519714,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519715,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519716,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519717,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519718,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1500),(1519719,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519720,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519721,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519722,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519723,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519724,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519725,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519726,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519727,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519728,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519729,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519730,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,4896),(1519731,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519732,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519733,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519734,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519735,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519736,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519737,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519738,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519739,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519740,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519741,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519742,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1527),(1519743,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519744,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519745,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519746,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519747,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519748,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,1686),(1519749,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519750,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519751,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519752,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519753,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519754,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519755,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519756,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519757,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519758,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519759,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519760,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519761,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519762,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519763,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519764,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519765,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519766,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519767,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519768,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519769,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519770,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519771,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519772,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519773,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519774,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519775,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519776,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519777,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519778,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519779,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519780,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519781,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519782,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519783,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519784,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519785,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519786,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519787,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519788,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519789,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519790,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519791,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519792,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519793,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519794,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',1,0),(1519795,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519796,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519797,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519798,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519799,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519800,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519801,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519802,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519803,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519804,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519805,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519806,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519807,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519808,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519809,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519810,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519811,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519812,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519813,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519814,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519815,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519816,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519817,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519818,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519819,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519820,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519821,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519822,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519823,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519824,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519825,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519826,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519827,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519828,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519829,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519830,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519831,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519832,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519833,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519834,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519835,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519836,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519837,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519838,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519839,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519840,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519841,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519842,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519843,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519844,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519845,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519846,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519847,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519848,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519849,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519850,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1509),(1519851,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519852,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519853,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519854,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519855,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519856,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,1656),(1519857,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519858,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519859,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519860,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519861,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519862,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519863,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519864,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519865,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519866,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519867,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519868,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519869,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519870,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519871,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519872,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519873,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519874,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',2,300),(1519875,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519876,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519877,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519878,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519879,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519880,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519881,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519882,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519883,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519884,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519885,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519886,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519887,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519888,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519889,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519890,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519891,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519892,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519893,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519894,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519895,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519896,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519897,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519898,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1527),(1519899,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519900,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519901,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519902,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519903,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519904,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519905,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519906,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519907,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519908,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519909,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519910,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519911,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519912,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519913,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519914,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519915,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519916,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519917,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519918,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519919,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519920,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519921,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519922,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',2,990),(1519923,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519924,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519925,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519926,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519927,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519928,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519929,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519930,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519931,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519932,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519933,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519934,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',4,7132),(1519935,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519936,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519937,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519938,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519939,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519940,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,2997),(1519941,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519942,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519943,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519944,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519945,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519946,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',4,6496),(1519947,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519948,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519949,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519950,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519951,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519952,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519953,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519954,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519955,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519956,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519957,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519958,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',2,996),(1519959,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519960,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519961,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519962,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519963,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519964,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,1647),(1519965,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519966,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519967,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519968,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519969,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519970,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519971,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519972,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519973,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519974,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519975,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519976,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519977,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519978,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519979,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519980,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519981,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519982,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519983,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519984,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519985,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519986,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519987,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519988,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519989,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519990,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519991,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519992,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519993,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519994,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519995,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519996,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519997,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519998,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519999,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1520000,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1520001,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1520002,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1520003,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1520004,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1520005,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1520006,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1520007,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1520008,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1520009,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1520010,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',1,0),(1520011,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1520012,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1520013,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1520014,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1520015,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1520016,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1520017,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1520018,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1520019,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1520020,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1520021,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1520022,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1520023,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1520024,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1520025,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1520026,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1520027,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1520028,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1520029,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1520030,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1520031,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1520032,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1520033,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1520034,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1520035,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1520036,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1520037,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1520038,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1520039,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1520040,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1520041,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1520042,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1520043,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1520044,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1520045,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1520046,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1520047,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1520048,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1520049,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1520050,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1520051,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1520052,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1520053,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1520054,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1520055,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1520056,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1520057,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1520058,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1520059,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1520060,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1520061,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1520062,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1520063,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1520064,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1520065,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1520066,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1536),(1520067,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1520068,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1520069,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1520070,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1520071,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1520072,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,1698),(1520073,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1520074,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1520075,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1520076,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1520077,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1520078,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',4,7096),(1520079,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1520080,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1520081,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1520082,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1520083,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1520084,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,2997),(1520085,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1520086,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1520087,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1536115,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1536116,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1536117,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1536118,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1536119,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1536120,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1536121,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1536122,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1536123,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1536124,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',2,2),(1536125,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1536126,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1536163,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1536164,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1536165,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1536166,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1536167,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1536168,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',3,3),(1536169,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1536170,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1536171,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1536172,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',3,3),(1536173,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1536174,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0); diff --git a/test/rest/RestAPITest.java b/test/rest/RestAPITest.java index 0a4b44330..5ac9b93f6 100644 --- a/test/rest/RestAPITest.java +++ b/test/rest/RestAPITest.java @@ -19,19 +19,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; -import com.google.gson.JsonElement; -import com.linkedin.drelephant.AutoTuner; import com.linkedin.drelephant.DrElephant; import com.linkedin.drelephant.ElephantContext; import com.linkedin.drelephant.tuning.BaselineComputeUtil; import com.linkedin.drelephant.tuning.FitnessComputeUtil; -import com.linkedin.drelephant.tuning.JobCompleteDetector; import com.linkedin.drelephant.util.Utils; -import common.DBTestUtil; -import controllers.AutoTuningMetricsController; - -import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; @@ -42,10 +35,11 @@ import models.JobDefinition; import models.JobExecution; import models.JobExecution.ExecutionState; +import models.JobSuggestedParamSet; import models.TuningJobDefinition; -import models.TuningJobExecution; -import models.TuningJobExecution.ParamSetStatus; +import models.JobSuggestedParamSet.ParamSetStatus; +import models.TuningJobExecutionParamSet; import org.apache.hadoop.conf.Configuration; import org.junit.Assert; import org.junit.Before; @@ -55,7 +49,6 @@ import play.Application; import play.GlobalSettings; -import play.libs.Json; import play.libs.WS; import play.test.FakeApplication; @@ -154,31 +147,29 @@ public void run() { jsonResponse.path("mapreduce.reduce.memory.mb").asDouble() > 0); assertTrue("Get current run param output size did not match", jsonResponse.size() == 9); - TuningJobExecution tuningJobExecution = TuningJobExecution.find.select("*") - .fetch(TuningJobExecution.TABLE.jobExecution, "*") - .fetch(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.job, "*") + TuningJobExecutionParamSet tuningJobExecutionParamSet = TuningJobExecutionParamSet.find.select("*") + .fetch(TuningJobExecutionParamSet.TABLE.jobExecution, "*") + .fetch(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet, "*") .where() - .eq(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.jobExecId, + .eq(TuningJobExecutionParamSet.TABLE.jobExecution + "." + JobExecution.TABLE.jobExecId, "https://elephant.linkedin.com:8443/executor?execid=5221700&job=countByCountryFlowSmall_countByCountry&attempt=0") .findUnique(); - tuningJobExecution.paramSetState = ParamSetStatus.EXECUTED; - tuningJobExecution.jobExecution.executionState = ExecutionState.SUCCEEDED; - tuningJobExecution.update(); + JobSuggestedParamSet jobSuggestedParamSet = tuningJobExecutionParamSet.jobSuggestedParamSet; + JobExecution jobExecution = tuningJobExecutionParamSet.jobExecution; + + jobExecution.executionState = ExecutionState.SUCCEEDED; + jobExecution.update(); + jobSuggestedParamSet.paramSetState = ParamSetStatus.EXECUTED; + jobSuggestedParamSet.update(); FitnessComputeUtil fitnessComputeUtil = new FitnessComputeUtil(); fitnessComputeUtil.updateFitness(); - tuningJobExecution = TuningJobExecution.find.select("*") - .fetch(TuningJobExecution.TABLE.jobExecution, "*") - .fetch(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.job, "*") - .where() - .eq(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.jobExecId, - "https://elephant.linkedin.com:8443/executor?execid=5221700&job=countByCountryFlowSmall_countByCountry&attempt=0") - .findUnique(); + jobSuggestedParamSet = JobSuggestedParamSet.find.byId(jobSuggestedParamSet.id); - assertTrue("Fitness not computed", tuningJobExecution.paramSetState == ParamSetStatus.FITNESS_COMPUTED); - assertTrue("Fitness not computed and have zero value", tuningJobExecution.fitness > 0); + assertTrue("Fitness not computed", jobSuggestedParamSet.paramSetState.equals(ParamSetStatus.FITNESS_COMPUTED)); + assertTrue("Fitness is non-positive", jobSuggestedParamSet.fitness > 0); } }); }