Skip to content

Commit d533bf2

Browse files
jrcastro2jrcastro2
committed
logging: add OpenSearch job log indexing and UI log viewer
- Add custom logging handler using contextvars and OpenSearch - Define JobLogEntrySchema and LogContextSchema - Support search_after pagination in log search API - Fetch logs incrementally from UI using search_after cursor - Add React log viewer with fade-in and scroll support - closes inveniosoftware#67
1 parent 39b0bd6 commit d533bf2

File tree

27 files changed

+869
-32
lines changed

27 files changed

+869
-32
lines changed

‎invenio_jobs/administration/runs.py

+77-16
Original file line numberDiff line numberDiff line change
@@ -10,21 +10,82 @@
1010

1111
from invenio_administration.views.base import AdminResourceListView
1212
from invenio_i18n import lazy_gettext as _
13+
from invenio_jobs.administration.jobs import JobsAdminMixin
14+
from flask import g
15+
from invenio_jobs.proxies import current_jobs_logs_service, current_runs_service
16+
from dateutil import parser
17+
from datetime import datetime
18+
class RunsDetailsView(JobsAdminMixin, AdminResourceListView):
19+
"""Configuration for System Runs details view."""
1320

21+
url = "/runs/<pid_value>"
22+
search_request_headers = {"Accept": "application/json"}
23+
request_headers = {"Accept": "application/json"}
24+
name = "run-details"
25+
resource_config = "runs_resource"
26+
title = "Run Details"
27+
disabled = lambda _: True
1428

15-
class RunsListView(AdminResourceListView):
16-
"""Configuration for System Runs sets list view."""
17-
18-
api_endpoint = "/runs"
19-
name = "Runs"
20-
search_request_headers = {"Accept": "application/vnd.inveniordm.v1+json"}
21-
title = "Runs"
22-
category = "System"
23-
resource_config = "jobs_resource"
24-
icon = "signal"
25-
extension_name = "invenio-rdm-records"
26-
display_search = False
27-
display_delete = False
28-
display_edit = False
29-
display_create = False
30-
actions = None
29+
template = "invenio_jobs/system/runs/runs-details.html"
30+
31+
list_view_name = "jobs"
32+
pid_value = "<pid_value>"
33+
34+
def get_context(self, **kwargs):
35+
"""Compute admin view context."""
36+
pid_value = kwargs.get("pid_value", "")
37+
logs, sort = self._get_logs(pid_value)
38+
if not logs:
39+
logs = []
40+
job_id = ""
41+
run_dict = {}
42+
run_duration = 0
43+
else:
44+
job_id = logs[0]["context"]["job_id"]
45+
run_dict = self._get_run_dict(job_id, pid_value)
46+
run_duration = self.get_duration_in_minutes(run_dict["started_at"], run_dict["finished_at"])
47+
48+
ctx = super().get_context(**kwargs)
49+
ctx["logs"] = logs
50+
ctx["run"] = run_dict
51+
ctx["run_duration"] = run_duration
52+
ctx["sort"] = sort
53+
return ctx
54+
55+
def _get_logs(self, pid_value):
56+
"""Retrieve and format logs."""
57+
params = dict(q=pid_value)
58+
logs_result = current_jobs_logs_service.search(g.identity, params)
59+
result_dict = logs_result.to_dict()
60+
logs = result_dict["hits"]["hits"]
61+
sort = result_dict["hits"]["sort"]
62+
63+
for log in logs:
64+
log["formatted_timestamp"] = self._format_datetime(log["timestamp"])
65+
66+
return logs, sort
67+
68+
def _get_run_dict(self, job_id, pid_value):
69+
"""Retrieve and format run dictionary."""
70+
run_dict = current_runs_service.read(g.identity, job_id, pid_value).to_dict()
71+
if "started_at" in run_dict and run_dict["started_at"]:
72+
run_dict["formatted_started_at"] = self._format_datetime(run_dict["started_at"])
73+
else:
74+
run_dict["formatted_started_at"] = None
75+
return run_dict
76+
77+
def _format_datetime(self, timestamp):
78+
"""Format ISO datetime to a user-friendly string."""
79+
dt = parser.isoparse(timestamp)
80+
return dt.strftime("%Y-%m-%d %H:%M")
81+
82+
def get_duration_in_minutes(self, started_at, finished_at):
83+
"""Calculate duration in minutes."""
84+
if not started_at:
85+
return 0
86+
87+
start_time = parser.isoparse(started_at)
88+
end_time = parser.isoparse(finished_at) if finished_at else parser.isoparse(datetime.now().astimezone().isoformat())
89+
90+
duration = (end_time - start_time).total_seconds() / 60
91+
return int(duration)

‎invenio_jobs/assets/semantic-ui/js/invenio_jobs/administration/RunsLogs.js

+207
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,207 @@
1+
import React, { Component } from "react";
2+
import PropTypes from "prop-types";
3+
import { Label, Container, Divider, Grid, Header, Icon, List, Table, Message } from "semantic-ui-react";
4+
import { http } from "react-invenio-forms";
5+
import { withCancel, ErrorMessage } from "react-invenio-forms";
6+
import { DateTime } from "luxon";
7+
8+
export class RunsLogs extends Component {
9+
constructor(props) {
10+
super(props);
11+
this.state = {
12+
error: null,
13+
logs: this.props.logs,
14+
run: this.props.run,
15+
sort: this.props.sort,
16+
runDuration: this.props.runDuration,
17+
};
18+
}
19+
20+
fetchLogs = async (runId, sort) => {
21+
try {
22+
const searchAfterParams = (sort || []).map((value) => `search_after=${value}`).join("&");
23+
this.cancellableFetch = withCancel(
24+
http.get(`/api/logs/jobs?q=${runId}&${searchAfterParams}`)
25+
);
26+
const response = await this.cancellableFetch.promise;
27+
if (response.status !== 200) {
28+
throw new Error(`Failed to fetch logs: ${response.statusText}`);
29+
}
30+
31+
const formattedLogs = response.data.hits.hits.map((log) => ({
32+
...log,
33+
formatted_timestamp: DateTime.fromISO(log.timestamp).toFormat("yyyy-MM-dd HH:mm"),
34+
}));
35+
const newSort = response.data.hits.sort;
36+
37+
this.setState((prevState) => ({
38+
logs: [...prevState.logs, ...formattedLogs],
39+
error: null,
40+
sort: newSort || prevState.sort, // Update sort only if newSort exists
41+
})); // Append logs and clear error
42+
} catch (err) {
43+
console.error("Error fetching logs:", err);
44+
this.setState({ error: err.message });
45+
}
46+
};
47+
48+
getDurationInMinutes(startedAt, finishedAt) {
49+
if (!startedAt) return 0;
50+
51+
const start = DateTime.fromISO(startedAt);
52+
const end = finishedAt
53+
? DateTime.fromISO(finishedAt)
54+
: DateTime.now();
55+
56+
const duration = end.diff(start, "minutes").minutes;
57+
58+
return Math.floor(duration);
59+
}
60+
61+
formatDatetime(timestamp) {
62+
if (!timestamp) return null;
63+
64+
return DateTime.fromISO(timestamp).toFormat("yyyy-MM-dd HH:mm");
65+
}
66+
67+
checkRunStatus = async (runId, jobId) => {
68+
try {
69+
this.cancellableFetch = withCancel(
70+
http.get(`/api/jobs/${jobId}/runs/${runId}`)
71+
);
72+
const response = await this.cancellableFetch.promise;
73+
if (response.status !== 200) {
74+
throw new Error(`Failed to fetch run status: ${response.statusText}`);
75+
}
76+
77+
const run = response.data;
78+
run.formatted_started_at = this.formatDatetime(run.started_at);
79+
const runDuration = this.getDurationInMinutes(run.started_at, run.finished_at);
80+
this.setState({ run: run, runDuration: runDuration });
81+
if (run.status === "SUCCESS" || run.status === "FAILED") {
82+
clearInterval(this.logsInterval); // Stop fetching logs if run finihsed
83+
}
84+
} catch (err) {
85+
console.error("Error checking run status:", err);
86+
this.setState({ error: err.message });
87+
}
88+
};
89+
90+
componentDidMount() {
91+
this.logsInterval = setInterval(async () => {
92+
const { run, sort } = this.state;
93+
if (run.status === "RUNNING") {
94+
await this.fetchLogs(run.id, sort); // Fetch logs only if the run is running
95+
await this.checkRunStatus(run.id, run.job_id); // Check the run status
96+
}
97+
}, 2000);
98+
}
99+
100+
componentWillUnmount() {
101+
clearInterval(this.logsInterval);
102+
}
103+
104+
render() {
105+
const { error, logs, run, runDuration, sort } = this.state;
106+
const levelClassMapping = {
107+
DEBUG: "",
108+
INFO: "primary",
109+
WARNING: "warning",
110+
ERROR: "expired",
111+
CRITICAL: "negative",
112+
};
113+
114+
const getClassForLogLevel = (level) => levelClassMapping[level] || "";
115+
return (
116+
<Container>
117+
<Header as="h2" className="mt-20">
118+
{run.title}
119+
</Header>
120+
<Divider />
121+
{error && (
122+
<Message negative>
123+
<Message.Header>Error Fetching Logs</Message.Header>
124+
<p>{error}</p>
125+
</Message>
126+
)}
127+
<Grid celled>
128+
<Grid.Row>
129+
<Grid.Column width={3}>
130+
<Header as="h4" color="grey">
131+
Job run
132+
</Header>
133+
<List>
134+
<List.Item>
135+
<Icon
136+
name={
137+
run.status === "SUCCESS"
138+
? "check circle"
139+
: run.status === "FAILED"
140+
? "times circle"
141+
: run.status === "RUNNING"
142+
? "spinner"
143+
: "clock outline"
144+
}
145+
color={
146+
run.status === "SUCCESS"
147+
? "green"
148+
: run.status === "FAILED"
149+
? "red"
150+
: run.status === "RUNNING"
151+
? "blue"
152+
: "grey"
153+
}
154+
/>
155+
<List.Content>
156+
{run.formatted_started_at ? (
157+
<>
158+
<p>
159+
<strong>{run.formatted_started_at}</strong>
160+
</p>
161+
<p className="description">{runDuration} mins</p>
162+
</>
163+
) : (
164+
<p className="description">Not yet started</p>
165+
)}
166+
</List.Content>
167+
</List.Item>
168+
</List>
169+
</Grid.Column>
170+
<Grid.Column className="log-table" width={13}>
171+
<Table basic="very" compact>
172+
<Table.Header>
173+
<Table.Row>
174+
<Table.HeaderCell width={3}>Timestamp</Table.HeaderCell>
175+
<Table.HeaderCell width={2}>Level</Table.HeaderCell>
176+
<Table.HeaderCell width={11}>Message</Table.HeaderCell>
177+
</Table.Row>
178+
</Table.Header>
179+
<Table.Body>
180+
{logs.map((log, index) => (
181+
<Table.Row
182+
key={index}
183+
className={`log-line ${log.level.toLowerCase()}`}
184+
>
185+
<Table.Cell>{log.formatted_timestamp}</Table.Cell>
186+
<Table.Cell>
187+
<Label className={getClassForLogLevel(log.level)}>{log.level}</Label>
188+
</Table.Cell>
189+
<Table.Cell><pre className="m-0">{log.message}</pre></Table.Cell>
190+
</Table.Row>
191+
))}
192+
</Table.Body>
193+
</Table>
194+
</Grid.Column>
195+
</Grid.Row>
196+
</Grid>
197+
</Container>
198+
);
199+
}
200+
}
201+
202+
RunsLogs.propTypes = {
203+
run: PropTypes.object.isRequired,
204+
logs: PropTypes.array.isRequired,
205+
runDuration: PropTypes.number.isRequired,
206+
sort: PropTypes.array.isRequired,
207+
};

‎invenio_jobs/assets/semantic-ui/js/invenio_jobs/administration/RunsLogsView.js

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// This file is part of Invenio
2+
// Copyright (C) 2024 CERN.
3+
//
4+
// Invenio RDM is free software; you can redistribute it and/or modify it
5+
// under the terms of the MIT License; see LICENSE file for more details.
6+
7+
import _get from "lodash/get";
8+
import React from "react";
9+
import ReactDOM from "react-dom";
10+
11+
import { RunsLogs } from "./RunsLogs";
12+
13+
const detailsConfig = document.getElementById("runs-logs-config");
14+
15+
if (detailsConfig) {
16+
const logs = JSON.parse(detailsConfig.dataset.logs);
17+
const run = JSON.parse(detailsConfig.dataset.run);
18+
const runDuration = JSON.parse(detailsConfig.dataset.runDuration);
19+
const sort = JSON.parse(detailsConfig.dataset.sort);
20+
ReactDOM.render(
21+
<RunsLogs
22+
logs={logs}
23+
run={run}
24+
runDuration={runDuration}
25+
sort={sort}
26+
/>,
27+
detailsConfig
28+
);
29+
}

‎invenio_jobs/assets/semantic-ui/js/invenio_jobs/administration/RunsSearchResultItemLayout.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ class SearchResultItemComponent extends Component {
6969
className="word-break-all"
7070
>
7171
<StatusFormatter status={status} />
72-
<a href={result.links.self}>{createdFormatted}</a>
72+
<a href={`/administration/runs/${result.id}`}>{createdFormatted}</a>
7373
</Table.Cell>
7474
<Table.Cell
7575
key={`run-last-run-${status}`}

‎invenio_jobs/ext.py

+6
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323
RunsResourceConfig,
2424
TasksResource,
2525
TasksResourceConfig,
26+
JobLogResource,
27+
JobLogResourceConfig,
2628
)
2729
from .services import (
2830
JobsService,
@@ -31,6 +33,8 @@
3133
RunsServiceConfig,
3234
TasksService,
3335
TasksServiceConfig,
36+
JobLogService,
37+
JobLogServiceConfig,
3438
)
3539

3640

@@ -62,6 +66,7 @@ def init_services(self, app):
6266
self.service = JobsService(JobsServiceConfig.build(app))
6367
self.runs_service = RunsService(RunsServiceConfig.build(app))
6468
self.tasks_service = TasksService(TasksServiceConfig.build(app))
69+
self.jobs_log_service = JobLogService(JobLogServiceConfig.build(app))
6570

6671
def init_resource(self, app):
6772
"""Initialize resources."""
@@ -72,6 +77,7 @@ def init_resource(self, app):
7277
self.tasks_resource = TasksResource(
7378
TasksResourceConfig.build(app), self.tasks_service
7479
)
80+
self.job_log_resource = JobLogResource(JobLogResourceConfig.build(app), self.jobs_log_service)
7581

7682
def load_entry_point_group(self):
7783
"""Load actions from an entry point group."""

‎invenio_jobs/logging/__init__.py

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# -*- coding: utf-8 -*-
2+
#
3+
# This file is part of Invenio.
4+
# Copyright (C) 2025 CERN.
5+
#
6+
# Invenio-Jobs is free software; you can redistribute it and/or
7+
# modify it under the terms of the MIT License; see LICENSE file for more
8+
# details.
9+
10+
"""Job logging."""

0 commit comments

Comments
 (0)