Skip to content

Commit e8d8e6a

Browse files
yaooqinnyaooqinn
committed
[SPARK-55971][UI] Add Jobs table to SQL execution detail page
### What changes were proposed in this pull request? 1. **Jobs table on SQL execution page** — Replaces the old comma-separated job ID links with a full "Associated Jobs" table on the SQL execution detail page. | Column | Description | |--------|-------------| | Job ID | Link to job detail page | | Description | Stage name and description (via `ApiHelper.lastStageNameAndDescription`) | | Submitted | Submission time (sortable by epoch) | | Duration | Human-readable duration (sortable by millis) | | Stages: Succeeded/Total | Completed/total with failed/skipped counts | | Tasks: Succeeded/Total | Progress bar with task counts | **Features:** - Collapsible section using BS5 Collapse (consistent with Plan Details, SQL Properties) - Expanded by default, shows job count in header - Sortable columns via `sorttable.js` (Stages/Tasks excluded with `sorttable_nosort`) - Responsive table wrapper for small screens - Gracefully handles missing jobs (`NoSuchElementException` catch) 2. **Concise progress bar labels** — The `makeProgressBar` visible label now shows `(N killed)` instead of the full kill reason with stack trace. The truncated reason (120 chars) is kept in the tooltip for hover inspection. This applies globally across Jobs, Stages, and SQL execution pages. ### Why are the changes needed? Previously jobs were shown as comma-separated ID links (e.g., "Running Jobs: 0 1 2") in the summary section, providing no context about job status, duration, or progress. The table shows all relevant information at a glance, matching the Jobs page style. The progress bar kill reason text could be extremely long (full stack traces), making the bar unreadable. ### Does this PR introduce _any_ user-facing change? Yes: - New "Associated Jobs (N)" collapsible table on the SQL execution detail page - Progress bar labels across all pages now show concise `(N killed)` instead of full error text ### How was this patch tested? Compilation verified. Manual testing with succeeded, failed, and killed-task jobs. ### Was this patch authored or co-authored using generative AI tooling? Yes, co-authored with GitHub Copilot. Closes #54768 from yaooqinn/SPARK-55971. Authored-by: Kent Yao <kentyao@microsoft.com> Signed-off-by: Kent Yao <kentyao@microsoft.com>
1 parent e7bbd32 commit e8d8e6a

File tree

2 files changed

+97
-25
lines changed

2 files changed

+97
-25
lines changed

‎core/src/main/scala/org/apache/spark/ui/UIUtils.scala‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -494,22 +494,25 @@ private[spark] object UIUtils extends Logging {
494494
val startRatio = if (total == 0) 0.0 else (boundedStarted.toDouble / total) * 100
495495
val startWidth = "width: %s%%".format(startRatio)
496496

497-
val killTaskReasonText = reasonToNumKilled.toSeq.sortBy(-_._2).map {
498-
case (reason, count) => s" ($count killed: $reason)"
497+
val totalKilled = reasonToNumKilled.values.sum
498+
val killReasonTitle = reasonToNumKilled.toSeq.sortBy(-_._2).map {
499+
case (reason, count) =>
500+
val truncated = if (reason.length > 120) reason.take(120) + "..." else reason
501+
s" ($count killed: $truncated)"
499502
}.mkString
500503
val progressTitle = s"$completed/$total" + {
501504
if (started > 0) s" ($started running)" else ""
502505
} + {
503506
if (failed > 0) s" ($failed failed)" else ""
504507
} + {
505508
if (skipped > 0) s" ($skipped skipped)" else ""
506-
} + killTaskReasonText
509+
} + killReasonTitle
507510

508511
val progressLabel = s"$completed/$total" +
509512
(if (failed == 0 && skipped == 0 && started > 0) s" ($started running)" else "") +
510513
(if (failed > 0) s" ($failed failed)" else "") +
511514
(if (skipped > 0) s" ($skipped skipped)" else "") +
512-
killTaskReasonText
515+
(if (totalKilled > 0) s" ($totalKilled killed)" else "")
513516

514517
// scalastyle:off line.size.limit
515518
<div class="progress-stacked" title={progressTitle}>

‎sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala‎

Lines changed: 90 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,10 @@ import org.json4s.JNull
2424
import org.json4s.JsonAST.{JBool, JString}
2525
import org.json4s.jackson.JsonMethods.parse
2626

27-
import org.apache.spark.JobExecutionStatus
2827
import org.apache.spark.internal.Logging
2928
import org.apache.spark.internal.config.UI.UI_SQL_GROUP_SUB_EXECUTION_ENABLED
3029
import org.apache.spark.ui.{UIUtils, WebUIPage}
30+
import org.apache.spark.ui.jobs.ApiHelper
3131

3232
class ExecutionPage(parent: SQLTab) extends WebUIPage("execution") with Logging {
3333

@@ -48,23 +48,6 @@ class ExecutionPage(parent: SQLTab) extends WebUIPage("execution") with Logging
4848
val duration = executionUIData.completionTime.map(_.getTime()).getOrElse(currentTime) -
4949
executionUIData.submissionTime
5050

51-
def jobLinks(status: JobExecutionStatus, label: String): Seq[Node] = {
52-
val jobs = executionUIData.jobs.flatMap { case (jobId, jobStatus) =>
53-
if (jobStatus == status) Some(jobId) else None
54-
}
55-
if (jobs.nonEmpty) {
56-
<li class="job-url">
57-
<strong>{label} </strong>
58-
{jobs.toSeq.sorted.map { jobId =>
59-
<a href={jobURL(request, jobId.intValue())}>{jobId.toString}</a><span>&nbsp;</span>
60-
}}
61-
</li>
62-
} else {
63-
Nil
64-
}
65-
}
66-
67-
6851
val summary =
6952
<div>
7053
<ul class="list-unstyled">
@@ -107,9 +90,6 @@ class ExecutionPage(parent: SQLTab) extends WebUIPage("execution") with Logging
10790
}
10891
}
10992
}
110-
{jobLinks(JobExecutionStatus.RUNNING, "Running Jobs:")}
111-
{jobLinks(JobExecutionStatus.SUCCEEDED, "Succeeded Jobs:")}
112-
{jobLinks(JobExecutionStatus.FAILED, "Failed Jobs:")}
11393
</ul>
11494
<div id="plan-viz-download-btn-container">
11595
<select id="plan-viz-format-select">
@@ -130,6 +110,7 @@ class ExecutionPage(parent: SQLTab) extends WebUIPage("execution") with Logging
130110
summary ++
131111
planVisualization(request, metrics, graph) ++
132112
physicalPlanDescription(executionUIData.physicalPlanDescription) ++
113+
jobsTable(request, executionUIData) ++
133114
modifiedConfigs(configs.filter { case (k, _) => !k.startsWith(pandasOnSparkConfPrefix) }) ++
134115
modifiedPandasOnSparkConfigs(
135116
configs.filter { case (k, _) => k.startsWith(pandasOnSparkConfPrefix) }) ++
@@ -225,6 +206,94 @@ class ExecutionPage(parent: SQLTab) extends WebUIPage("execution") with Logging
225206
</div>
226207
}
227208

209+
private def jobsTable(
210+
request: HttpServletRequest,
211+
executionUIData: SQLExecutionUIData): Seq[Node] = {
212+
val jobIds = executionUIData.jobs.keys.toSeq.sorted.reverse
213+
if (jobIds.isEmpty) return Nil
214+
215+
val store = parent.parent.store
216+
val basePath = UIUtils.prependBaseUri(request, parent.basePath)
217+
val rows = jobIds.flatMap { jobId =>
218+
try {
219+
val job = store.job(jobId)
220+
val submissionTimeMs = job.submissionTime.map(_.getTime).getOrElse(-1L)
221+
val formattedTime = job.submissionTime.map(UIUtils.formatDate).getOrElse("")
222+
val durationMs = (job.submissionTime, job.completionTime) match {
223+
case (Some(start), Some(end)) => end.getTime - start.getTime
224+
case (Some(start), None) => System.currentTimeMillis() - start.getTime
225+
case _ => -1L
226+
}
227+
val duration = if (durationMs >= 0) UIUtils.formatDuration(durationMs) else ""
228+
val (lastStageName, lastStageDesc) =
229+
ApiHelper.lastStageNameAndDescription(store, job)
230+
val jobDesc = UIUtils.makeDescription(
231+
job.description.getOrElse(lastStageDesc), basePath, plainText = false)
232+
val detailUrl = s"$basePath/jobs/job/?id=$jobId"
233+
val stagesInfo = {
234+
val completed = job.numCompletedStages
235+
val total = job.stageIds.size - job.numSkippedStages
236+
val extra = Seq(
237+
if (job.numFailedStages > 0) s"(${job.numFailedStages} failed)" else "",
238+
if (job.numSkippedStages > 0) s"(${job.numSkippedStages} skipped)" else ""
239+
).filter(_.nonEmpty).mkString(" ")
240+
s"$completed/$total $extra"
241+
}
242+
Some(
243+
<tr id={"job-" + jobId}>
244+
<td><a href={jobURL(request, jobId)}>{jobId}</a></td>
245+
<td>
246+
{jobDesc}
247+
<a href={detailUrl} class="name-link">{lastStageName}</a>
248+
</td>
249+
<td sorttable_customkey={submissionTimeMs.toString}>{formattedTime}</td>
250+
<td sorttable_customkey={durationMs.toString}>{duration}</td>
251+
<td class="stage-progress-cell">{stagesInfo}</td>
252+
<td class="progress-cell">
253+
{UIUtils.makeProgressBar(started = job.numActiveTasks,
254+
completed = job.numCompletedIndices,
255+
failed = job.numFailedTasks, skipped = job.numSkippedTasks,
256+
reasonToNumKilled = job.killedTasksSummary,
257+
total = job.numTasks - job.numSkippedTasks)}
258+
</td>
259+
</tr>)
260+
} catch {
261+
case _: NoSuchElementException => None
262+
}
263+
}
264+
265+
// scalastyle:off
266+
<div>
267+
<span class="collapse-table" data-bs-toggle="collapse"
268+
data-bs-target="#sql-jobs-table"
269+
aria-expanded="true" aria-controls="sql-jobs-table"
270+
data-collapse-name="collapse-sql-jobs">
271+
<h4>
272+
<span class="collapse-table-arrow arrow-open"></span>
273+
<a>Associated Jobs ({jobIds.size})</a>
274+
</h4>
275+
</span>
276+
<div class="collapsible-table collapse show" id="sql-jobs-table">
277+
<div class="table-responsive">
278+
<table class="table table-bordered table-hover table-sm sortable">
279+
<thead>
280+
<tr>
281+
<th>Job ID</th>
282+
<th>Description</th>
283+
<th>Submitted</th>
284+
<th>Duration</th>
285+
<th class="sorttable_nosort">Stages: Succeeded/Total</th>
286+
<th class="sorttable_nosort">Tasks (for all stages): Succeeded/Total</th>
287+
</tr>
288+
</thead>
289+
<tbody>{rows}</tbody>
290+
</table>
291+
</div>
292+
</div>
293+
</div>
294+
// scalastyle:on
295+
}
296+
228297
private def modifiedConfigs(modifiedConfigs: Map[String, String]): Seq[Node] = {
229298
if (Option(modifiedConfigs).exists(_.isEmpty)) return Nil
230299

0 commit comments

Comments
 (0)