-
Notifications
You must be signed in to change notification settings - Fork 8.5k
Gap auto fill scheduler UI and API #244719
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
31c0e58
b154793
26daa71
4c14e11
8e2c648
65ed6f5
213c9fb
977f06e
c3e43e7
4e19aa2
981580e
fb677d6
a2853d8
a74fce3
b65b5aa
25c7ba1
bd19d50
14839b1
de66f16
163655b
d84bd51
e6e0d53
5f2be71
62332f2
0f9f792
4fc5880
6e1efdf
63a4f54
7f641e4
bd4c92d
dc56f7b
ea5a982
9a49039
27d8fa2
fc45533
f64d923
072ce05
050b4ca
688257c
cc0892c
b22663d
646c24c
478e0c7
4020cf8
292c0fa
6da188b
529a9b3
186c110
78fba04
8332730
cdc2561
d82a383
e18a29a
c94c1b8
5ac0c42
2ae242b
b614f28
cb1bb77
119a32d
51afe46
12de6ea
51ed5b3
2b7afdd
418ac39
25134a1
110d04d
5998915
5e2c09c
7ff212e
8157587
2bd5d0d
da6ab37
fd116c2
efaab1e
ebf94e1
3983a1c
7c4d001
91f0f50
c592629
db0cd17
63c7a09
cb33530
a2732bc
e817da2
0b1d7ac
c64c8b3
005d146
cc26f35
de754bf
99ef613
61763d3
4d76346
25e216f
4da6ad9
4b8bc74
1371576
0d54d25
578e046
88beab9
95169c0
6296289
a1462b0
1b4c83b
0f5db8e
ff562e3
751e1c7
99748ea
ff30a37
58650e0
10592f2
4731e48
28e1e0b
3ae97c1
c3ee98a
d03a277
90873c5
a52f0b2
6c6cc56
7dc07c9
103c9c8
aa6df6a
be8e319
cd350a1
179591b
fe1bdd4
fec3dfc
cbbc161
a9d9786
11ae994
4f96e72
96a8896
f54ab39
0d57b63
7166908
d712d57
1a79b57
983be8f
99c7108
41d131f
275a718
d9913d9
93358c8
7949fc5
d8d992c
9cbaf7f
5bc0348
0ca2a76
9ad3195
3e77636
6f4ef97
6b77f76
d0f3fa1
4fbf1b9
1e1b973
598fe47
da5a467
a525775
707cfaa
bc8906e
4cf46c9
b907060
eab9aaa
1743cdc
4f7d4a6
8027070
6828719
71a7b7f
3fba6fd
2784fc9
fe01d5f
0d7f1a8
afb28f1
456d93c
1e61142
a9272db
bb04e68
4168a0b
0ed43aa
ea7c2e6
95ee9e1
c38a154
8b7970f
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -14,6 +14,51 @@ import { parseDuration } from '../../../../../parse_duration'; | |||||
|
|
||||||
| const { maxBackfills, numRetries, minScheduleIntervalInMs } = gapAutoFillSchedulerLimits; | ||||||
|
|
||||||
| const validateGapAutoFillSchedulerPayload = ( | ||||||
| gapFillRange: string, | ||||||
| schedule: { interval: string }, | ||||||
| ruleTypes: { type: string; consumer: string }[] | ||||||
| ) => { | ||||||
| const now = new Date(); | ||||||
| const parsed = dateMath.parse(gapFillRange, { forceNow: now }); | ||||||
| if (!parsed || !parsed.isValid()) { | ||||||
| return 'gap_fill_range is invalid'; | ||||||
| } | ||||||
|
|
||||||
| const maxLookbackExpression = `now-${MAX_SCHEDULE_BACKFILL_LOOKBACK_WINDOW_DAYS}d`; | ||||||
| const lookbackLimit = dateMath.parse(maxLookbackExpression, { forceNow: now }); | ||||||
| if (!lookbackLimit || !lookbackLimit.isValid()) { | ||||||
| return 'gap_fill_range is invalid'; | ||||||
| } | ||||||
|
|
||||||
| if (parsed.isBefore(lookbackLimit)) { | ||||||
| return `gap_fill_range cannot look back more than ${MAX_SCHEDULE_BACKFILL_LOOKBACK_WINDOW_DAYS} days`; | ||||||
| } | ||||||
|
|
||||||
| try { | ||||||
| const intervalMs = parseDuration(schedule.interval); | ||||||
| if (intervalMs < minScheduleIntervalInMs) { | ||||||
| return 'schedule.interval must be at least 1 minute'; | ||||||
| } | ||||||
| } catch (error) { | ||||||
| return `schedule.interval is invalid: ${(error as Error).message}`; | ||||||
| } | ||||||
|
|
||||||
| // Duplicate check for rule_types | ||||||
| const seen = new Set<string>(); | ||||||
| for (const ruleType of ruleTypes) { | ||||||
| const key = `${ruleType.type}:${ruleType.consumer}`; | ||||||
| if (seen.has(key)) { | ||||||
| return `rule_types contains duplicate entry: type="${ruleType.type}" consumer="${ruleType.consumer}"`; | ||||||
| } | ||||||
| seen.add(key); | ||||||
| } | ||||||
| }; | ||||||
|
|
||||||
| export const getGapAutoFillSchedulerParamsSchema = schema.object({ | ||||||
| id: schema.string(), | ||||||
| }); | ||||||
|
|
||||||
| export const gapAutoFillSchedulerBodySchema = schema.object( | ||||||
| { | ||||||
| id: schema.maybe(schema.string()), | ||||||
|
|
@@ -34,41 +79,41 @@ export const gapAutoFillSchedulerBodySchema = schema.object( | |||||
| ), | ||||||
| }, | ||||||
| { | ||||||
| validate({ gap_fill_range: gapFillRange, schedule, rule_types: ruleTypes }) { | ||||||
| const now = new Date(); | ||||||
| const parsed = dateMath.parse(gapFillRange, { forceNow: now }); | ||||||
| if (!parsed || !parsed.isValid()) { | ||||||
| return 'gap_fill_range is invalid'; | ||||||
| } | ||||||
|
|
||||||
| const maxLookbackExpression = `now-${MAX_SCHEDULE_BACKFILL_LOOKBACK_WINDOW_DAYS}d`; | ||||||
| const lookbackLimit = dateMath.parse(maxLookbackExpression, { forceNow: now }); | ||||||
| if (!lookbackLimit || !lookbackLimit.isValid()) { | ||||||
| return 'gap_fill_range is invalid'; | ||||||
| } | ||||||
|
|
||||||
| if (parsed.isBefore(lookbackLimit)) { | ||||||
| return `gap_fill_range cannot look back more than ${MAX_SCHEDULE_BACKFILL_LOOKBACK_WINDOW_DAYS} days`; | ||||||
| } | ||||||
|
|
||||||
| try { | ||||||
| const intervalMs = parseDuration(schedule.interval); | ||||||
| if (intervalMs < minScheduleIntervalInMs) { | ||||||
| return 'schedule.interval must be at least 1 minute'; | ||||||
| } | ||||||
| } catch (error) { | ||||||
| return `schedule.interval is invalid: ${(error as Error).message}`; | ||||||
| } | ||||||
| validate(payload) { | ||||||
| return validateGapAutoFillSchedulerPayload( | ||||||
| payload.gap_fill_range, | ||||||
| payload.schedule, | ||||||
| payload.rule_types | ||||||
| ); | ||||||
| }, | ||||||
| } | ||||||
| ); | ||||||
|
|
||||||
| // Duplicate check for rule_types | ||||||
| const seen = new Set<string>(); | ||||||
| for (const ruleType of ruleTypes) { | ||||||
| const key = `${ruleType.type}:${ruleType.consumer}`; | ||||||
| if (seen.has(key)) { | ||||||
| return `rule_types contains duplicate entry: type="${ruleType.type}" consumer="${ruleType.consumer}"`; | ||||||
| } | ||||||
| seen.add(key); | ||||||
| } | ||||||
| export const gapAutoFillSchedulerUpdateBodySchema = schema.object( | ||||||
| { | ||||||
| name: schema.string(), | ||||||
| enabled: schema.boolean(), | ||||||
| gap_fill_range: schema.string(), | ||||||
| max_backfills: schema.number({ min: 1, max: 5000 }), | ||||||
| num_retries: schema.number({ min: 1 }), | ||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. should this be
Suggested change
|
||||||
| schedule: schema.object({ | ||||||
| interval: schema.string(), | ||||||
| }), | ||||||
| scope: schema.arrayOf(schema.string()), | ||||||
| rule_types: schema.arrayOf( | ||||||
| schema.object({ | ||||||
| type: schema.string(), | ||||||
| consumer: schema.string(), | ||||||
| }) | ||||||
| ), | ||||||
| }, | ||||||
| { | ||||||
| validate(payload) { | ||||||
| return validateGapAutoFillSchedulerPayload( | ||||||
| payload.gap_fill_range, | ||||||
| payload.schedule, | ||||||
| payload.rule_types | ||||||
| ); | ||||||
| }, | ||||||
| } | ||||||
| ); | ||||||
|
|
@@ -95,3 +140,48 @@ export const gapAutoFillSchedulerResponseSchema = schema.object({ | |||||
| created_at: schema.string(), | ||||||
| updated_at: schema.string(), | ||||||
| }); | ||||||
|
|
||||||
| export const gapAutoFillSchedulerLogsRequestQuerySchema = schema.object({ | ||||||
| start: schema.string(), | ||||||
| end: schema.string(), | ||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. should we validate that these are valid dates? |
||||||
| page: schema.number({ defaultValue: 1, min: 1 }), | ||||||
| per_page: schema.number({ defaultValue: 50, min: 1, max: 1000 }), | ||||||
| sort_field: schema.oneOf([schema.literal('@timestamp')], { defaultValue: '@timestamp' }), | ||||||
| sort_direction: schema.oneOf([schema.literal('asc'), schema.literal('desc')], { | ||||||
| defaultValue: 'desc', | ||||||
| }), | ||||||
| statuses: schema.maybe( | ||||||
| schema.arrayOf( | ||||||
| schema.oneOf([ | ||||||
| schema.literal('success'), | ||||||
| schema.literal('error'), | ||||||
| schema.literal('skipped'), | ||||||
| schema.literal('no_gaps'), | ||||||
| ]) | ||||||
| ) | ||||||
| ), | ||||||
| }); | ||||||
|
|
||||||
| export const gapAutoFillSchedulerLogEntrySchema = schema.object({ | ||||||
| id: schema.string(), | ||||||
| timestamp: schema.maybe(schema.string()), | ||||||
| status: schema.maybe(schema.string()), | ||||||
| message: schema.maybe(schema.string()), | ||||||
| results: schema.maybe( | ||||||
| schema.arrayOf( | ||||||
| schema.object({ | ||||||
| rule_id: schema.maybe(schema.string()), | ||||||
| processed_gaps: schema.maybe(schema.number()), | ||||||
| status: schema.maybe(schema.string()), | ||||||
| error: schema.maybe(schema.string()), | ||||||
| }) | ||||||
| ) | ||||||
| ), | ||||||
| }); | ||||||
|
|
||||||
| export const gapAutoFillSchedulerLogsResponseSchema = schema.object({ | ||||||
| data: schema.arrayOf(gapAutoFillSchedulerLogEntrySchema), | ||||||
| total: schema.number(), | ||||||
| page: schema.number(), | ||||||
| per_page: schema.number(), | ||||||
| }); | ||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -112,35 +112,37 @@ Payload summary: ${JSON.stringify(otherParams, (key, value) => | |
| savedObjectOptions | ||
| ); | ||
|
|
||
| try { | ||
| await taskManager.ensureScheduled( | ||
| { | ||
| id: so.id, | ||
| taskType: GAP_AUTO_FILL_SCHEDULER_TASK_TYPE, | ||
| schedule: params.schedule, | ||
| scope: params.scope ?? [], | ||
| params: { | ||
| configId: so.id, | ||
| spaceId: context.spaceId, | ||
| }, | ||
| state: {}, | ||
| }, | ||
| { | ||
| request: params.request, | ||
| } | ||
| ); | ||
| } catch (e) { | ||
| context.logger.error( | ||
| `Failed to schedule task for gap auto fill scheduler ${so.id}. Will attempt to delete the saved object.` | ||
| ); | ||
| if (params.enabled) { | ||
| try { | ||
| await soClient.delete(GAP_AUTO_FILL_SCHEDULER_SAVED_OBJECT_TYPE, so.id); | ||
| } catch (deleteError) { | ||
| await taskManager.ensureScheduled( | ||
| { | ||
| id: so.id, | ||
| taskType: GAP_AUTO_FILL_SCHEDULER_TASK_TYPE, | ||
| schedule: params.schedule, | ||
| scope: params.scope ?? [], | ||
| params: { | ||
| configId: so.id, | ||
| spaceId: context.spaceId, | ||
| }, | ||
| state: {}, | ||
| }, | ||
| { | ||
| request: params.request, | ||
| } | ||
| ); | ||
| } catch (e) { | ||
| context.logger.error( | ||
| `Failed to delete gap auto fill saved object for gap auto fill scheduler ${so.id}.` | ||
| `Failed to schedule task for gap auto fill scheduler ${so.id}. Will attempt to delete the saved object.` | ||
| ); | ||
| try { | ||
| await soClient.delete(GAP_AUTO_FILL_SCHEDULER_SAVED_OBJECT_TYPE, so.id); | ||
| } catch (deleteError) { | ||
| context.logger.error( | ||
| `Failed to delete gap auto fill saved object for gap auto fill scheduler ${so.id}.` | ||
| ); | ||
| } | ||
| throw e; | ||
| } | ||
| throw e; | ||
| } | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. unit test for creating a scheduler that's disabled? |
||
|
|
||
| // Log successful creation | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
should this be