Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add labels and childFilter support to statistics endpoints #6922

Open
wants to merge 2 commits into
base: develop
Choose a base branch
from
Open

feat: add labels and childFilter support to statistics endpoints #6922

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6cf00de
feat: add labels and childFilter support to statistics endpoints
Malaydewangan09 Jan 24, 2025
0b043bc
feat: add labels and childFilter support to statistics endpoints
Malaydewangan09 Jan 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions core/src/main/java/io/kestra/core/models/collectors/ExecutionUsage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ public static ExecutionUsage of(final String tenantId,
to,
DateUtils.GroupType.DAY,
null,
true);
true, null, null);
} catch (UnsupportedOperationException ignored) {

}
Expand All @@ -53,7 +53,7 @@ public static ExecutionUsage of(final String tenantId,
to,
DateUtils.GroupType.DAY,
null,
false))
false, null, null))
.dailyTaskRunsCount(dailyTaskRunsCount)
.build();
}
Expand All @@ -70,7 +70,9 @@ public static ExecutionUsage of(final ExecutionRepositoryInterface repository,
from,
to,
DateUtils.GroupType.DAY,
true
true,
null,
null
);
} catch (UnsupportedOperationException ignored) {}

Expand All @@ -82,7 +84,9 @@ public static ExecutionUsage of(final ExecutionRepositoryInterface repository,
from,
to,
DateUtils.GroupType.DAY,
false
false,
null,
null
))
.dailyTaskRunsCount(dailyTaskRunsCount)
.build();
Expand Down
13 changes: 10 additions & 3 deletions core/src/main/java/io/kestra/core/repositories/ExecutionRepositoryInterface.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,9 @@ List<DailyExecutionStatistics> dailyStatisticsForAllTenants(
@Nullable ZonedDateTime startDate,
@Nullable ZonedDateTime endDate,
@Nullable DateUtils.GroupType groupBy,
boolean isTaskRun
boolean isTaskRun,
@Nullable Map<String, String> labels,
@Nullable ChildFilter childFilter
);

List<DailyExecutionStatistics> dailyStatistics(
Expand All @@ -143,7 +145,9 @@ List<DailyExecutionStatistics> dailyStatistics(
@Nullable ZonedDateTime endDate,
@Nullable DateUtils.GroupType groupBy,
List<State.Type> state,
boolean isTaskRun
boolean isTaskRun,
@Nullable Map<String, String> labels,
@Nullable ChildFilter childFilter
);

List<Execution> lastExecutions(
Expand All @@ -159,7 +163,10 @@ Map<String, Map<String, List<DailyExecutionStatistics>>> dailyGroupByFlowStatist
@Nullable List<FlowFilter> flows,
@Nullable ZonedDateTime startDate,
@Nullable ZonedDateTime endDate,
boolean groupByNamespaceOnly
boolean groupByNamespaceOnly,
@Nullable Map<String, String> labels,
@Nullable ChildFilter childFilter

);

Map<String, ExecutionCountStatistics> executionCountsGroupedByNamespace(
Expand Down
28 changes: 14 additions & 14 deletions core/src/test/java/io/kestra/core/repositories/AbstractExecutionRepositoryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -286,8 +286,8 @@ protected void dailyGroupByFlowStatistics() throws InterruptedException {
null,
ZonedDateTime.now().minusDays(10),
ZonedDateTime.now(),
false
);
false,
null, null);

assertThat(result.size(), is(1));
assertThat(result.get("io.kestra.unittest").size(), is(2));
Expand Down Expand Up @@ -315,8 +315,8 @@ protected void dailyGroupByFlowStatistics() throws InterruptedException {
null,
ZonedDateTime.now().minusDays(10),
ZonedDateTime.now(),
true
);
true,
null, null);

assertThat(result.size(), is(1));
assertThat(result.get("io.kestra.unittest").size(), is(1));
Expand All @@ -336,8 +336,8 @@ protected void dailyGroupByFlowStatistics() throws InterruptedException {
List.of(ExecutionRepositoryInterface.FlowFilter.builder().namespace("io.kestra.unittest").id(FLOW).build()),
ZonedDateTime.now().minusDays(10),
ZonedDateTime.now(),
false
);
false,
null, null);

assertThat(result.size(), is(1));
assertThat(result.get("io.kestra.unittest").size(), is(1));
Expand Down Expand Up @@ -444,7 +444,7 @@ protected void dailyStatistics() throws InterruptedException {
ZonedDateTime.now(),
null,
null,
false);
false, null, null);

assertThat(result.size(), is(11));
assertThat(result.get(10).getExecutionCounts().size(), is(11));
Expand All @@ -464,7 +464,7 @@ protected void dailyStatistics() throws InterruptedException {
ZonedDateTime.now(),
null,
null,
false);
false, null, null);

assertThat(result.size(), is(11));
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS), is(21L));
Expand All @@ -479,7 +479,7 @@ protected void dailyStatistics() throws InterruptedException {
ZonedDateTime.now(),
null,
null,
false);
false, null, null);
assertThat(result.size(), is(11));
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS), is(20L));

Expand All @@ -493,7 +493,7 @@ protected void dailyStatistics() throws InterruptedException {
ZonedDateTime.now(),
null,
null,
false);
false, null, null);
assertThat(result.size(), is(11));
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS), is(1L));
}
Expand Down Expand Up @@ -522,7 +522,7 @@ protected void taskRunsDailyStatistics() {
ZonedDateTime.now(),
null,
null,
true);
true, null, null);

assertThat(result.size(), is(11));
assertThat(result.get(10).getExecutionCounts().size(), is(11));
Expand All @@ -542,7 +542,7 @@ protected void taskRunsDailyStatistics() {
ZonedDateTime.now(),
null,
null,
true);
true, null, null);

assertThat(result.size(), is(11));
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS), is(57L));
Expand All @@ -557,7 +557,7 @@ protected void taskRunsDailyStatistics() {
ZonedDateTime.now(),
null,
null,
true);
true, null, null);
assertThat(result.size(), is(11));
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS), is(55L));

Expand All @@ -571,7 +571,7 @@ protected void taskRunsDailyStatistics() {
ZonedDateTime.now(),
null,
null,
true);
true, null, null);
assertThat(result.size(), is(11));
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS), is(2L));
}
Expand Down
49 changes: 36 additions & 13 deletions jdbc/src/main/java/io/kestra/jdbc/repository/AbstractJdbcExecutionRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -368,7 +368,9 @@ public List<DailyExecutionStatistics> dailyStatisticsForAllTenants(
@Nullable ZonedDateTime startDate,
@Nullable ZonedDateTime endDate,
@Nullable DateUtils.GroupType groupBy,
boolean isTaskRun
boolean isTaskRun,
@Nullable Map<String, String> labels,
@Nullable ChildFilter childFilter
) {
if (isTaskRun) {
throw new UnsupportedOperationException();
Expand All @@ -388,7 +390,10 @@ public List<DailyExecutionStatistics> dailyStatisticsForAllTenants(
finalStartDate,
finalEndDate,
groupBy,
null
null,
labels,
childFilter

);

return dailyStatisticsQueryMapRecord(
Expand All @@ -412,8 +417,9 @@ public List<DailyExecutionStatistics> dailyStatistics(
@Nullable ZonedDateTime endDate,
@Nullable DateUtils.GroupType groupBy,
@Nullable List<State.Type> states,
boolean isTaskRun
) {
boolean isTaskRun,
@Nullable Map<String, String> labels,
@Nullable ChildFilter childFilter) {
if (isTaskRun) {
throw new UnsupportedOperationException();
}
Expand All @@ -434,7 +440,9 @@ public List<DailyExecutionStatistics> dailyStatistics(
finalStartDate,
finalEndDate,
groupBy,
states
states,
labels,
childFilter
);

return dailyStatisticsQueryMapRecord(
Expand Down Expand Up @@ -484,7 +492,9 @@ private Results dailyStatisticsQueryForAllTenants(
ZonedDateTime startDate,
ZonedDateTime endDate,
@Nullable DateUtils.GroupType groupBy,
@Nullable List<State.Type> state
@Nullable List<State.Type> state,
@Nullable Map<String, String> labels,
@Nullable ChildFilter childFilter
) {
return dailyStatisticsQuery(
this.defaultFilter(),
Expand All @@ -497,7 +507,9 @@ private Results dailyStatisticsQueryForAllTenants(
startDate,
endDate,
groupBy,
state
state,
labels,
childFilter
);
}

Expand All @@ -512,7 +524,9 @@ private Results dailyStatisticsQuery(
ZonedDateTime startDate,
ZonedDateTime endDate,
@Nullable DateUtils.GroupType groupBy,
@Nullable List<State.Type> state
@Nullable List<State.Type> state,
@Nullable Map<String, String> labels,
@Nullable ChildFilter childFilter
) {
return dailyStatisticsQuery(
this.defaultFilter(tenantId),
Expand All @@ -525,7 +539,9 @@ private Results dailyStatisticsQuery(
startDate,
endDate,
groupBy,
state
state,
labels,
childFilter
);
}

Expand All @@ -540,7 +556,10 @@ private Results dailyStatisticsQuery(
ZonedDateTime startDate,
ZonedDateTime endDate,
@Nullable DateUtils.GroupType groupBy,
@Nullable List<State.Type> state
@Nullable List<State.Type> state,
@Nullable Map<String, String> labels,
@Nullable ChildFilter childFilter

) {
List<Field<?>> dateFields = new ArrayList<>(groupByFields(Duration.between(startDate, endDate), "start_date", groupBy));
List<Field<?>> selectFields = new ArrayList<>(fields);
Expand All @@ -564,7 +583,7 @@ private Results dailyStatisticsQuery(
.and(START_DATE_FIELD.greaterOrEqual(startDate.toOffsetDateTime()))
.and(START_DATE_FIELD.lessOrEqual(endDate.toOffsetDateTime()));

select = filteringQuery(select, scope, namespace, flowId, flows, query, null, null, null);
select = filteringQuery(select, scope, namespace, flowId, flows, query, labels, null, childFilter);

if (state != null) {
select = select.and(this.statesFilter(state));
Expand Down Expand Up @@ -697,7 +716,9 @@ public Map<String, Map<String, List<DailyExecutionStatistics>>> dailyGroupByFlow
@Nullable List<FlowFilter> flows,
@Nullable ZonedDateTime startDate,
@Nullable ZonedDateTime endDate,
boolean groupByNamespaceOnly
boolean groupByNamespaceOnly,
@Nullable Map<String, String> labels,
@Nullable ChildFilter childFilter
) {
List<Field<?>> fields = new ArrayList<>();

Expand All @@ -722,7 +743,9 @@ public Map<String, Map<String, List<DailyExecutionStatistics>>> dailyGroupByFlow
finalStartDate,
finalEndDate,
null,
null
null,
labels,
childFilter
);

return results
Expand Down
26 changes: 18 additions & 8 deletions webserver/src/main/java/io/kestra/webserver/controllers/api/StatsController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,14 @@
import io.kestra.core.repositories.LogRepositoryInterface;
import io.kestra.core.repositories.TriggerRepositoryInterface;
import io.kestra.core.tenant.TenantService;
import io.kestra.webserver.utils.RequestUtils;
import io.micronaut.core.annotation.Introspected;
import io.micronaut.core.annotation.Nullable;
import io.micronaut.core.convert.format.Format;
import io.micronaut.http.annotation.Body;
import io.micronaut.http.annotation.Controller;
import io.micronaut.http.annotation.Post;
import io.micronaut.http.annotation.QueryValue;
import io.micronaut.scheduling.TaskExecutors;
import io.micronaut.scheduling.annotation.ExecuteOn;
import io.micronaut.validation.Validated;
Expand All @@ -31,7 +33,6 @@
import java.time.ZonedDateTime;
import java.util.List;
import java.util.Map;
import java.util.Optional;

@Validated
@Controller("/api/v1/stats")
Expand Down Expand Up @@ -76,7 +77,9 @@ public List<DailyExecutionStatistics> dailyStatistics(@Body @Valid StatisticRequ
statisticRequest.endDate() != null ? statisticRequest.endDate().withZoneSameInstant(ZoneId.systemDefault()) : null,
null,
statisticRequest.state(),
false);
false,
RequestUtils.toMap(statisticRequest.labels()),
statisticRequest.childFilter());
}

@ExecuteOn(TaskExecutors.IO)
Expand All @@ -93,7 +96,7 @@ public List<DailyExecutionStatistics> taskRunsDailyStatistics(@Body @Valid Stati
statisticRequest.endDate() != null ? statisticRequest.endDate().withZoneSameInstant(ZoneId.systemDefault()) : null,
null,
statisticRequest.state(),
true);
true, null, null);
}

@ExecuteOn(TaskExecutors.IO)
Expand All @@ -108,8 +111,9 @@ public Map<String, Map<String, List<DailyExecutionStatistics>>> dailyGroupByFlow
statisticRequest.flows() != null && statisticRequest.flows().getFirst().getNamespace() != null && statisticRequest.flows().getFirst().getId() != null ? statisticRequest.flows() : null,
statisticRequest.startDate() != null ? statisticRequest.startDate().withZoneSameInstant(ZoneId.systemDefault()) : null,
statisticRequest.endDate() != null ? statisticRequest.endDate().withZoneSameInstant(ZoneId.systemDefault()) : null,
Boolean.TRUE.equals(statisticRequest.namespaceOnly())
);
Boolean.TRUE.equals(statisticRequest.namespaceOnly()),
RequestUtils.toMap(statisticRequest.labels()),
statisticRequest.childFilter());
}

@ExecuteOn(TaskExecutors.IO)
Expand Down Expand Up @@ -168,7 +172,9 @@ public record StatisticRequest(
@Parameter(description = "A flow id filter") @Nullable String flowId,
@Parameter(description = "The start datetime, default to now") @Nullable @Format("yyyy-MM-dd'T'HH:mm[:ss][.SSS][XXX]") ZonedDateTime startDate,
@Parameter(description = "The end datetime, default to now") @Nullable @Format("yyyy-MM-dd'T'HH:mm[:ss][.SSS][XXX]")ZonedDateTime endDate,
@Parameter(description = "the state of the execution") @Nullable List<State.Type> state
@Parameter(description = "the state of the execution") @Nullable List<State.Type> state,
@Parameter(description = "A labels filter as a list of 'key:value'") @Nullable @QueryValue @Format("MULTI") List<String> labels,
@Parameter(description = "A execution child filter") @Nullable @QueryValue ExecutionRepositoryInterface.ChildFilter childFilter
) {}

@Introspected
Expand All @@ -185,7 +191,9 @@ public record LogStatisticRequest(
public record ByNamespaceStatisticRequest(
@Parameter(description = "A namespace filter prefix") @Nullable String namespace,
@Parameter(description = "The start datetime, default to now") @Nullable @Format("yyyy-MM-dd'T'HH:mm[:ss][.SSS][XXX]") ZonedDateTime startDate,
@Parameter(description = "The end datetime, default to now") @Nullable @Format("yyyy-MM-dd'T'HH:mm[:ss][.SSS][XXX]")ZonedDateTime endDate
@Parameter(description = "The end datetime, default to now") @Nullable @Format("yyyy-MM-dd'T'HH:mm[:ss][.SSS][XXX]")ZonedDateTime endDate,
@Parameter(description = "A labels filter as a list of 'key:value'") @Nullable @QueryValue @Format("MULTI") List<String> labels,
@Parameter(description = "A execution child filter") @Nullable @QueryValue ExecutionRepositoryInterface.ChildFilter childFilter
) {}

@Introspected
Expand All @@ -196,7 +204,9 @@ public record ByFlowStatisticRequest(
@Parameter(description = "A list of flows filter") @Nullable List<ExecutionRepositoryInterface.FlowFilter> flows,
@Parameter(description = "The start datetime, default to now") @Nullable @Format("yyyy-MM-dd'T'HH:mm[:ss][.SSS][XXX]") ZonedDateTime startDate,
@Parameter(description = "The end datetime, default to now") @Nullable @Format("yyyy-MM-dd'T'HH:mm[:ss][.SSS][XXX]")ZonedDateTime endDate,
@Nullable Boolean namespaceOnly
@Nullable Boolean namespaceOnly,
@Parameter(description = "A labels filter as a list of 'key:value'") @Nullable @QueryValue @Format("MULTI") List<String> labels,
@Parameter(description = "A execution child filter") @Nullable @QueryValue ExecutionRepositoryInterface.ChildFilter childFilter
) {}

@Introspected
Expand Down
Loading
Loading