Skip to content

Commit 8919f96

Browse files
jrcastro2jrcastro2
committed
logging: add OpenSearch job log indexing and UI log viewer
- Add custom logging handler using contextvars and OpenSearch - Define JobLogEntrySchema and LogContextSchema - Support search_after pagination in log search API - Fetch logs incrementally from UI using search_after cursor - Add React log viewer with fade-in and scroll support - closes inveniosoftware#67
1 parent 39b0bd6 commit 8919f96

34 files changed

+999
-33
lines changed

‎invenio_jobs/administration/runs.py

+89-16
Original file line numberDiff line numberDiff line change
@@ -8,23 +8,96 @@
88

99
"""Invenio administration Runs view module."""
1010

11+
from datetime import datetime
12+
13+
from dateutil import parser
14+
from flask import g
1115
from invenio_administration.views.base import AdminResourceListView
1216
from invenio_i18n import lazy_gettext as _
1317

18+
from invenio_jobs.administration.jobs import JobsAdminMixin
19+
from invenio_jobs.proxies import current_jobs_logs_service, current_runs_service
20+
21+
22+
class RunsDetailsView(JobsAdminMixin, AdminResourceListView):
23+
"""Configuration for System Runs details view."""
24+
25+
url = "/runs/<pid_value>"
26+
search_request_headers = {"Accept": "application/json"}
27+
request_headers = {"Accept": "application/json"}
28+
name = "run-details"
29+
resource_config = "runs_resource"
30+
title = "Run Details"
31+
disabled = lambda _: True
32+
33+
template = "invenio_jobs/system/runs/runs-details.html"
34+
35+
list_view_name = "jobs"
36+
pid_value = "<pid_value>"
37+
38+
def get_context(self, **kwargs):
39+
"""Compute admin view context."""
40+
pid_value = kwargs.get("pid_value", "")
41+
logs, sort = self._get_logs(pid_value)
42+
if not logs:
43+
logs = []
44+
job_id = ""
45+
run_dict = {}
46+
run_duration = 0
47+
else:
48+
job_id = logs[0]["context"]["job_id"]
49+
run_dict = self._get_run_dict(job_id, pid_value)
50+
run_duration = self.get_duration_in_minutes(
51+
run_dict["started_at"], run_dict["finished_at"]
52+
)
53+
54+
ctx = super().get_context(**kwargs)
55+
ctx["logs"] = logs
56+
ctx["run"] = run_dict
57+
ctx["run_duration"] = run_duration
58+
ctx["sort"] = sort
59+
return ctx
60+
61+
def _get_logs(self, pid_value):
62+
"""Retrieve and format logs."""
63+
params = dict(q=f'"{pid_value}"')
64+
logs_result = current_jobs_logs_service.search(g.identity, params)
65+
result_dict = logs_result.to_dict()
66+
logs = result_dict["hits"]["hits"]
67+
sort = result_dict["hits"]["sort"]
68+
69+
for log in logs:
70+
log["formatted_timestamp"] = self._format_datetime(log["timestamp"])
71+
72+
return logs, sort
73+
74+
def _get_run_dict(self, job_id, pid_value):
75+
"""Retrieve and format run dictionary."""
76+
run_dict = current_runs_service.read(g.identity, job_id, pid_value).to_dict()
77+
if "started_at" in run_dict and run_dict["started_at"]:
78+
run_dict["formatted_started_at"] = self._format_datetime(
79+
run_dict["started_at"]
80+
)
81+
else:
82+
run_dict["formatted_started_at"] = None
83+
return run_dict
84+
85+
def _format_datetime(self, timestamp):
86+
"""Format ISO datetime to a user-friendly string."""
87+
dt = parser.isoparse(timestamp)
88+
return dt.strftime("%Y-%m-%d %H:%M")
89+
90+
def get_duration_in_minutes(self, started_at, finished_at):
91+
"""Calculate duration in minutes."""
92+
if not started_at:
93+
return 0
94+
95+
start_time = parser.isoparse(started_at)
96+
end_time = (
97+
parser.isoparse(finished_at)
98+
if finished_at
99+
else parser.isoparse(datetime.now().astimezone().isoformat())
100+
)
14101

15-
class RunsListView(AdminResourceListView):
16-
"""Configuration for System Runs sets list view."""
17-
18-
api_endpoint = "/runs"
19-
name = "Runs"
20-
search_request_headers = {"Accept": "application/vnd.inveniordm.v1+json"}
21-
title = "Runs"
22-
category = "System"
23-
resource_config = "jobs_resource"
24-
icon = "signal"
25-
extension_name = "invenio-rdm-records"
26-
display_search = False
27-
display_delete = False
28-
display_edit = False
29-
display_create = False
30-
actions = None
102+
duration = (end_time - start_time).total_seconds() / 60
103+
return int(duration)

‎invenio_jobs/assets/semantic-ui/js/invenio_jobs/administration/RunsLogs.js

+212
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
import React, { Component } from "react";
2+
import PropTypes from "prop-types";
3+
import { Label, Container, Divider, Grid, Header, Icon, List, Message, Segment } from "semantic-ui-react";
4+
import { http } from "react-invenio-forms";
5+
import { withCancel, ErrorMessage } from "react-invenio-forms";
6+
import { DateTime } from "luxon";
7+
8+
export class RunsLogs extends Component {
9+
constructor(props) {
10+
super(props);
11+
this.state = {
12+
error: null,
13+
logs: this.props.logs,
14+
run: this.props.run,
15+
sort: this.props.sort,
16+
runDuration: this.props.runDuration,
17+
};
18+
}
19+
20+
fetchLogs = async (runId, sort) => {
21+
try {
22+
const searchAfterParams = (sort || []).map((value) => `search_after=${value}`).join("&");
23+
this.cancellableFetch = withCancel(
24+
http.get(`/api/logs/jobs?q=${runId}&${searchAfterParams}`)
25+
);
26+
const response = await this.cancellableFetch.promise;
27+
if (response.status !== 200) {
28+
throw new Error(`Failed to fetch logs: ${response.statusText}`);
29+
}
30+
31+
const formattedLogs = response.data.hits.hits.map((log) => ({
32+
...log,
33+
formatted_timestamp: DateTime.fromISO(log.timestamp).toFormat("yyyy-MM-dd HH:mm"),
34+
}));
35+
const newSort = response.data.hits.sort;
36+
37+
this.setState((prevState) => ({
38+
logs: [...prevState.logs, ...formattedLogs],
39+
error: null,
40+
sort: newSort || prevState.sort, // Update sort only if newSort exists
41+
})); // Append logs and clear error
42+
} catch (err) {
43+
console.error("Error fetching logs:", err);
44+
this.setState({ error: err.message });
45+
}
46+
};
47+
48+
getDurationInMinutes(startedAt, finishedAt) {
49+
if (!startedAt) return 0;
50+
51+
const start = DateTime.fromISO(startedAt);
52+
const end = finishedAt
53+
? DateTime.fromISO(finishedAt)
54+
: DateTime.now();
55+
56+
const duration = end.diff(start, "minutes").minutes;
57+
58+
return Math.floor(duration);
59+
}
60+
61+
formatDatetime(timestamp) {
62+
if (!timestamp) return null;
63+
64+
return DateTime.fromISO(timestamp).toFormat("yyyy-MM-dd HH:mm");
65+
}
66+
67+
checkRunStatus = async (runId, jobId) => {
68+
try {
69+
this.cancellableFetch = withCancel(
70+
http.get(`/api/jobs/${jobId}/runs/${runId}`)
71+
);
72+
const response = await this.cancellableFetch.promise;
73+
if (response.status !== 200) {
74+
throw new Error(`Failed to fetch run status: ${response.statusText}`);
75+
}
76+
77+
const run = response.data;
78+
run.formatted_started_at = this.formatDatetime(run.started_at);
79+
const runDuration = this.getDurationInMinutes(run.started_at, run.finished_at);
80+
this.setState({ run: run, runDuration: runDuration });
81+
if (run.status === "SUCCESS" || run.status === "FAILED" || run.status === "PARTIAL_SUCCESS") {
82+
clearInterval(this.logsInterval); // Stop fetching logs if run finihsed
83+
}
84+
} catch (err) {
85+
console.error("Error checking run status:", err);
86+
this.setState({ error: err.message });
87+
}
88+
};
89+
90+
componentDidMount() {
91+
this.logsInterval = setInterval(async () => {
92+
const { run, sort } = this.state;
93+
if (run.status === "RUNNING") {
94+
await this.fetchLogs(run.id, sort); // Fetch logs only if the run is running
95+
await this.checkRunStatus(run.id, run.job_id); // Check the run status
96+
}
97+
}, 2000);
98+
}
99+
100+
componentWillUnmount() {
101+
clearInterval(this.logsInterval);
102+
}
103+
104+
render() {
105+
const { error, logs, run, runDuration, sort } = this.state;
106+
const levelClassMapping = {
107+
DEBUG: "",
108+
INFO: "primary",
109+
WARNING: "warning",
110+
ERROR: "negative",
111+
CRITICAL: "negative",
112+
};
113+
114+
const getClassForLogLevel = (level) => levelClassMapping[level] || "";
115+
return (
116+
<Container>
117+
<Header as="h2" className="mt-20">
118+
{run.title}
119+
</Header>
120+
<Divider />
121+
{error && (
122+
<Message negative>
123+
<Message.Header>Error Fetching Logs</Message.Header>
124+
<p>{error}</p>
125+
</Message>
126+
)}
127+
<Grid celled>
128+
<Grid.Row>
129+
<Grid.Column width={3}>
130+
<Header as="h4" color="grey">
131+
Job run
132+
</Header>
133+
<List>
134+
<List.Item>
135+
<Icon
136+
name={
137+
run.status === "SUCCESS"
138+
? "check circle"
139+
: run.status === "FAILED"
140+
? "times circle"
141+
: run.status === "RUNNING"
142+
? "spinner"
143+
: run.status === "PARTIAL_SUCCESS"
144+
? "exclamation circle"
145+
: "clock outline"
146+
}
147+
color={
148+
run.status === "SUCCESS"
149+
? "green"
150+
: run.status === "FAILED"
151+
? "red"
152+
: run.status === "RUNNING"
153+
? "blue"
154+
: run.status === "PARTIAL_SUCCESS"
155+
? "orange"
156+
: "grey"
157+
}
158+
/>
159+
<List.Content>
160+
{run.formatted_started_at ? (
161+
<>
162+
<p>
163+
<strong>{run.formatted_started_at}</strong>
164+
</p>
165+
<p className="description">{runDuration} mins</p>
166+
</>
167+
) : (
168+
<p className="description">Not yet started</p>
169+
)}
170+
{run.message && (
171+
<Label basic
172+
color={
173+
run.status === "SUCCESS"
174+
? "green"
175+
: run.status === "FAILED"
176+
? "red"
177+
: run.status === "PARTIAL_SUCCESS"
178+
? "orange"
179+
: "blue"
180+
}
181+
>
182+
<p>{run.message}</p>
183+
</Label>
184+
)}
185+
</List.Content>
186+
</List.Item>
187+
</List>
188+
</Grid.Column>
189+
<Grid.Column className="log-table" width={13}>
190+
<Segment>
191+
{logs.map((log, index) => (
192+
<div key={index} className={`log-line ${log.level.toLowerCase()}`}>
193+
<span className="log-timestamp">[{log.formatted_timestamp}]</span>{" "}
194+
<span className={getClassForLogLevel(log.level)}>{log.level}</span>{" "}
195+
<span className="log-message">{log.message}</span>
196+
</div>
197+
))}
198+
</Segment>
199+
</Grid.Column>
200+
</Grid.Row>
201+
</Grid>
202+
</Container>
203+
);
204+
}
205+
}
206+
207+
RunsLogs.propTypes = {
208+
run: PropTypes.object.isRequired,
209+
logs: PropTypes.array.isRequired,
210+
runDuration: PropTypes.number.isRequired,
211+
sort: PropTypes.array.isRequired,
212+
};

‎invenio_jobs/assets/semantic-ui/js/invenio_jobs/administration/RunsLogsView.js

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// This file is part of Invenio
2+
// Copyright (C) 2024 CERN.
3+
//
4+
// Invenio RDM is free software; you can redistribute it and/or modify it
5+
// under the terms of the MIT License; see LICENSE file for more details.
6+
7+
import _get from "lodash/get";
8+
import React from "react";
9+
import ReactDOM from "react-dom";
10+
11+
import { RunsLogs } from "./RunsLogs";
12+
13+
const detailsConfig = document.getElementById("runs-logs-config");
14+
15+
if (detailsConfig) {
16+
const logs = JSON.parse(detailsConfig.dataset.logs);
17+
const run = JSON.parse(detailsConfig.dataset.run);
18+
const runDuration = JSON.parse(detailsConfig.dataset.runDuration);
19+
const sort = JSON.parse(detailsConfig.dataset.sort);
20+
ReactDOM.render(
21+
<RunsLogs
22+
logs={logs}
23+
run={run}
24+
runDuration={runDuration}
25+
sort={sort}
26+
/>,
27+
detailsConfig
28+
);
29+
}

‎invenio_jobs/assets/semantic-ui/js/invenio_jobs/administration/RunsSearchResultItemLayout.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ class SearchResultItemComponent extends Component {
6969
className="word-break-all"
7070
>
7171
<StatusFormatter status={status} />
72-
<a href={result.links.self}>{createdFormatted}</a>
72+
<a href={`/administration/runs/${result.id}`}>{createdFormatted}</a>
7373
</Table.Cell>
7474
<Table.Cell
7575
key={`run-last-run-${status}`}

‎invenio_jobs/assets/semantic-ui/js/invenio_jobs/administration/StatusFormatter.js

+6
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,12 @@ export const StatusFormatter = ({ status }) => {
5454
color="red"
5555
value={status === "CANCELLED"}
5656
/>
57+
<BoolFormatter
58+
tooltip={i18next.t("Partial Success")}
59+
icon="exclamation circle"
60+
color="orange"
61+
value={status === "PARTIAL_SUCCESS"}
62+
/>
5763
</span>
5864
);
5965
};

0 commit comments

Comments
 (0)