Skip to content

Commit ffe4d17

Browse files
ShubhamGupta29varunsaxena
ShubhamGupta29
authored and
varunsaxena
committed
Fix calculation of Spark heuristic score #423 
1 parent 4c365fa commit ffe4d17

12 files changed

+83
-13
lines changed

‎app/com/linkedin/drelephant/spark/heuristics/ConfigurationHeuristic.scala‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ import scala.util.Try
2222
import com.linkedin.drelephant.analysis._
2323
import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData
2424
import com.linkedin.drelephant.spark.data.SparkApplicationData
25-
import com.linkedin.drelephant.util.MemoryFormatUtils
25+
import com.linkedin.drelephant.util.{MemoryFormatUtils, Utils}
2626
import com.linkedin.drelephant.math.Statistics
2727

2828
/**
@@ -94,7 +94,7 @@ class ConfigurationHeuristic(private val heuristicConfigurationData: HeuristicCo
9494
heuristicConfigurationData.getClassName,
9595
heuristicConfigurationData.getHeuristicName,
9696
evaluator.severity,
97-
0,
97+
evaluator.score,
9898
mutableResultDetailsArrayList
9999
)
100100
if (evaluator.serializerSeverity != Severity.NONE) {
@@ -241,6 +241,13 @@ object ConfigurationHeuristic {
241241

242242
lazy val severity: Severity = Severity.max(serializerSeverity, shuffleAndDynamicAllocationSeverity, severityConfThresholds)
243243

244+
if (data.executorSummaries == null) {
245+
throw new Exception("Executor Summary is Null.")
246+
}
247+
val executorCount = data.executorSummaries.size
248+
249+
lazy val score = Utils.getHeuristicScore(severity, executorCount)
250+
244251
private val serializerIfNonNullRecommendation: String = configurationHeuristic.serializerIfNonNullRecommendation
245252

246253
private def getProperty(key: String): Option[String] = appConfigurationProperties.get(key)

‎app/com/linkedin/drelephant/spark/heuristics/DriverHeuristic.scala‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ import scala.util.Try
2323
import com.linkedin.drelephant.analysis._
2424
import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData
2525
import com.linkedin.drelephant.spark.data.SparkApplicationData
26-
import com.linkedin.drelephant.util.MemoryFormatUtils
26+
import com.linkedin.drelephant.util.{MemoryFormatUtils, Utils}
2727
import com.linkedin.drelephant.spark.fetchers.statusapiv1.ExecutorSummary
2828

2929
/**
@@ -98,7 +98,7 @@ class DriverHeuristic(private val heuristicConfigurationData: HeuristicConfigura
9898
heuristicConfigurationData.getClassName,
9999
heuristicConfigurationData.getHeuristicName,
100100
evaluator.severity,
101-
0,
101+
evaluator.score,
102102
mutableResultDetailsArrayList
103103
)
104104
result
@@ -187,6 +187,10 @@ object DriverHeuristic {
187187
//Severity for the configuration thresholds
188188
val severityConfThresholds: Severity = Severity.max(severityDriverCores, severityDriverMemory, severityDriverMemoryOverhead)
189189
lazy val severity: Severity = Severity.max(severityConfThresholds, severityGc, severityJvmUsedMemory)
190+
191+
val executorCount = 1 //For driver number of executor is 1
192+
lazy val score = Utils.getHeuristicScore(severity, executorCount)
193+
190194
private def getProperty(key: String): Option[String] = appConfigurationProperties.get(key)
191195
}
192196

‎app/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristic.scala‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ import com.linkedin.drelephant.analysis._
2222
import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData
2323
import com.linkedin.drelephant.spark.data.SparkApplicationData
2424
import com.linkedin.drelephant.math.Statistics
25-
25+
import com.linkedin.drelephant.util.Utils
2626

2727
import scala.collection.JavaConverters
2828

@@ -66,7 +66,7 @@ class ExecutorGcHeuristic(private val heuristicConfigurationData: HeuristicConfi
6666
heuristicConfigurationData.getClassName,
6767
heuristicConfigurationData.getHeuristicName,
6868
evaluator.severityTimeA,
69-
0,
69+
evaluator.score,
7070
resultDetails.asJava
7171
)
7272
result
@@ -116,6 +116,9 @@ object ExecutorGcHeuristic {
116116

117117
lazy val severityTimeD: Severity = executorGcHeuristic.gcSeverityDThresholds.severityOf(ratio)
118118

119+
val executorCount = executorSummaries.size
120+
lazy val score = Utils.getHeuristicScore(severityTimeA, executorCount)
121+
119122
/**
120123
* returns the total JVM GC Time and total executor Run Time across all stages
121124
* @param executorSummaries

‎app/com/linkedin/drelephant/spark/heuristics/ExecutorStorageSpillHeuristic.scala‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import com.linkedin.drelephant.spark.fetchers.statusapiv1.{ExecutorStageSummary,
2121
import com.linkedin.drelephant.analysis._
2222
import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData
2323
import com.linkedin.drelephant.spark.data.SparkApplicationData
24-
import com.linkedin.drelephant.util.MemoryFormatUtils
24+
import com.linkedin.drelephant.util.{MemoryFormatUtils, Utils}
2525

2626
import scala.collection.JavaConverters
2727

@@ -76,7 +76,7 @@ class ExecutorStorageSpillHeuristic(private val heuristicConfigurationData: Heur
7676
heuristicConfigurationData.getClassName,
7777
heuristicConfigurationData.getHeuristicName,
7878
evaluator.severity,
79-
0,
79+
evaluator.score,
8080
resultDetails.asJava
8181
)
8282
result
@@ -132,6 +132,10 @@ object ExecutorStorageSpillHeuristic {
132132
else Severity.NONE
133133
}
134134

135+
val executorCount = executorSummaries.size
136+
lazy val score = Utils.getHeuristicScore(severity, executorCount)
137+
138+
135139
lazy val sparkExecutorMemory: Long = (appConfigurationProperties.get(SPARK_EXECUTOR_MEMORY).map(MemoryFormatUtils.stringToBytes)).getOrElse(0)
136140
lazy val sparkExecutorCores: Int = (appConfigurationProperties.get(SPARK_EXECUTOR_CORES).map(_.toInt)).getOrElse(0)
137141
}

‎app/com/linkedin/drelephant/spark/heuristics/JvmUsedMemoryHeuristic.scala‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import com.linkedin.drelephant.analysis._
2020
import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData
2121
import com.linkedin.drelephant.spark.data.SparkApplicationData
2222
import com.linkedin.drelephant.spark.fetchers.statusapiv1.ExecutorSummary
23-
import com.linkedin.drelephant.util.MemoryFormatUtils
23+
import com.linkedin.drelephant.util.{MemoryFormatUtils, Utils}
2424

2525
import scala.collection.JavaConverters
2626

@@ -57,7 +57,7 @@ class JvmUsedMemoryHeuristic(private val heuristicConfigurationData: HeuristicCo
5757
heuristicConfigurationData.getClassName,
5858
heuristicConfigurationData.getHeuristicName,
5959
evaluator.severity,
60-
0,
60+
evaluator.score,
6161
resultDetails.asJava
6262
)
6363
result
@@ -79,6 +79,10 @@ object JvmUsedMemoryHeuristic {
7979
lazy val appConfigurationProperties: Map[String, String] =
8080
data.appConfigurationProperties
8181

82+
if (data.executorSummaries == null) {
83+
throw new Exception("Executor Summary is Null.")
84+
}
85+
8286
lazy val executorSummaries: Seq[ExecutorSummary] = data.executorSummaries
8387
val executorList: Seq[ExecutorSummary] = executorSummaries.filterNot(_.id.equals("driver"))
8488
val sparkExecutorMemory: Long = (appConfigurationProperties.get(SPARK_EXECUTOR_MEMORY).map(MemoryFormatUtils.stringToBytes)).getOrElse(0L)
@@ -100,6 +104,10 @@ object JvmUsedMemoryHeuristic {
100104
} else {
101105
MAX_EXECUTOR_PEAK_JVM_USED_MEMORY_THRESHOLDS.severityOf(sparkExecutorMemory)
102106
}
107+
108+
val executorCount = executorList.size
109+
lazy val score = Utils.getHeuristicScore(severity, executorCount)
110+
103111
}
104112

105113
}

‎app/com/linkedin/drelephant/spark/heuristics/StagesWithFailedTasksHeuristic.scala‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationDa
2121
import com.linkedin.drelephant.spark.data.SparkApplicationData
2222
import com.linkedin.drelephant.spark.fetchers.statusapiv1.{StageData, TaskData}
2323
import com.linkedin.drelephant.spark.fetchers.statusapiv1.StageStatus
24+
import com.linkedin.drelephant.util.Utils
2425

2526
import scala.collection.JavaConverters
2627

@@ -51,7 +52,7 @@ class StagesWithFailedTasksHeuristic(private val heuristicConfigurationData: Heu
5152
heuristicConfigurationData.getClassName,
5253
heuristicConfigurationData.getHeuristicName,
5354
evaluator.severity,
54-
0,
55+
evaluator.score,
5556
resultDetails.asJava
5657
)
5758
result
@@ -142,6 +143,14 @@ object StagesWithFailedTasksHeuristic {
142143

143144
lazy val (severityOOMStages: Severity, severityOverheadStages: Severity, stagesWithOOMError: Int, stagesWithOverheadError: Int) = getErrorsSeverity
144145
lazy val severity: Severity = Severity.max(severityOverheadStages, severityOOMStages)
146+
147+
if (data.executorSummaries == null) {
148+
throw new Exception("Executor Summary is Null.")
149+
}
150+
151+
val executorCount = data.executorSummaries.filterNot(_.id.equals("driver")).size
152+
lazy val score = Utils.getHeuristicScore(severity, executorCount)
153+
145154
}
146155

147156
}

‎app/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristic.scala‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import com.linkedin.drelephant.analysis._
2020
import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData
2121
import com.linkedin.drelephant.spark.data.SparkApplicationData
2222
import com.linkedin.drelephant.spark.fetchers.statusapiv1.ExecutorSummary
23-
import com.linkedin.drelephant.util.MemoryFormatUtils
23+
import com.linkedin.drelephant.util.{MemoryFormatUtils, Utils}
2424

2525
import scala.collection.JavaConverters
2626

@@ -56,7 +56,7 @@ class UnifiedMemoryHeuristic(private val heuristicConfigurationData: HeuristicCo
5656
heuristicConfigurationData.getClassName,
5757
heuristicConfigurationData.getHeuristicName,
5858
evaluator.severity,
59-
0,
59+
evaluator.score,
6060
resultDetails.asJava
6161
)
6262
result
@@ -128,5 +128,9 @@ object UnifiedMemoryHeuristic {
128128
} else {
129129
Severity.NONE
130130
}
131+
132+
val executorCount = executorList.size
133+
lazy val score = Utils.getHeuristicScore(severity, executorCount)
134+
131135
}
132136
}

‎test/com/linkedin/drelephant/spark/heuristics/DriverHeuristicTest.scala‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,10 @@ class DriverHeuristicTest extends FunSpec with Matchers {
3939
heuristicResult.getSeverity should be(Severity.SEVERE)
4040
}
4141

42+
it("has score") {
43+
heuristicResult.getScore should be(Severity.SEVERE.getValue * 1)
44+
}
45+
4246
describe("Evaluator") {
4347
val evaluator = new Evaluator(driverHeuristic, data)
4448
it("has max driver peak JVM memory") {

‎test/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristicTest.scala‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,10 +106,19 @@ class ExecutorGcHeuristicTest extends FunSpec with Matchers {
106106
heuristicResult.getSeverity should be(Severity.CRITICAL)
107107
}
108108

109+
it("returns non-zero score") {
110+
heuristicResult.getScore should be(Severity.CRITICAL.getValue * data.executorSummaries
111+
.filterNot(_.id.equals("driver")).size)
112+
}
113+
109114
it("return the low severity") {
110115
heuristicResult2.getSeverity should be(Severity.LOW)
111116
}
112117

118+
it("return the 0 score") {
119+
heuristicResult2.getScore should be(0)
120+
}
121+
113122
it("return NONE severity for runtime less than 5 min") {
114123
heuristicResult2.getSeverity should be(Severity.LOW)
115124
}

‎test/com/linkedin/drelephant/spark/heuristics/ExecutorStorageSpillHeuristicTest.scala‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,10 @@ class ExecutorStorageSpillHeuristicTest extends FunSpec with Matchers {
6868
heuristicResult.getSeverity should be(Severity.SEVERE)
6969
}
7070

71+
it("returns the score") {
72+
heuristicResult.getScore should be(Severity.SEVERE.getValue * data1.executorSummaries.size)
73+
}
74+
7175
it("returns the total memory spilled") {
7276
val details = heuristicResultDetails.get(0)
7377
details.getName should include("Total memory spilled")

0 commit comments

Comments
 (0)