diff --git a/internal/scheduler/common.go b/internal/scheduler/common.go index 4b0bc6e2940..55d22c3bac1 100644 --- a/internal/scheduler/common.go +++ b/internal/scheduler/common.go @@ -24,6 +24,9 @@ type SchedulerResult struct { PreemptedJobs []interfaces.LegacySchedulerJob // Queued jobs that should be scheduled. ScheduledJobs []interfaces.LegacySchedulerJob + // Queued jobs that could not be scheduled. + // This is used to fail jobs that could not schedule above `minimumGangCardinality`. + FailedJobs []interfaces.LegacySchedulerJob // For each preempted job, maps the job id to the id of the node on which the job was running. // For each scheduled job, maps the job id to the id of the node on which the job should be scheduled. NodeIdByJobId map[string]string @@ -32,9 +35,10 @@ type SchedulerResult struct { SchedulingContexts []*schedulercontext.SchedulingContext } -func NewSchedulerResult[S ~[]T, T interfaces.LegacySchedulerJob]( +func NewSchedulerResultForTest[S ~[]T, T interfaces.LegacySchedulerJob]( preemptedJobs S, scheduledJobs S, + failedJobs S, nodeIdByJobId map[string]string, ) *SchedulerResult { castPreemptedJobs := make([]interfaces.LegacySchedulerJob, len(preemptedJobs)) @@ -45,10 +49,15 @@ func NewSchedulerResult[S ~[]T, T interfaces.LegacySchedulerJob]( for i, job := range scheduledJobs { castScheduledJobs[i] = job } + castFailedJobs := make([]interfaces.LegacySchedulerJob, len(failedJobs)) + for i, job := range failedJobs { + castFailedJobs[i] = job + } return &SchedulerResult{ PreemptedJobs: castPreemptedJobs, ScheduledJobs: castScheduledJobs, NodeIdByJobId: nodeIdByJobId, + FailedJobs: castFailedJobs, } } @@ -72,6 +81,16 @@ func ScheduledJobsFromSchedulerResult[T interfaces.LegacySchedulerJob](sr *Sched return rv } +// FailedJobsFromScheduleResult returns the slice of scheduled jobs in the result, +// cast to type T. +func FailedJobsFromSchedulerResult[T interfaces.LegacySchedulerJob](sr *SchedulerResult) []T { + rv := make([]T, len(sr.FailedJobs)) + for i, job := range sr.FailedJobs { + rv[i] = job.(T) + } + return rv +} + // JobsSummary returns a string giving an overview of the provided jobs meant for logging. // For example: "affected queues [A, B]; resources {A: {cpu: 1}, B: {cpu: 2}}; jobs [jobAId, jobBId]". func JobsSummary(jobs []interfaces.LegacySchedulerJob) string { @@ -132,22 +151,22 @@ func GangIdAndCardinalityFromLegacySchedulerJob(job interfaces.LegacySchedulerJo // GangIdAndCardinalityFromAnnotations returns a tuple (gangId, gangCardinality, gangMinimumCardinality, isGangJob, error). func GangIdAndCardinalityFromAnnotations(annotations map[string]string) (string, int, int, bool, error) { if annotations == nil { - return "", 0, 0, false, nil + return "", 1, 1, false, nil } gangId, ok := annotations[configuration.GangIdAnnotation] if !ok { - return "", 0, 0, false, nil + return "", 1, 1, false, nil } gangCardinalityString, ok := annotations[configuration.GangCardinalityAnnotation] if !ok { - return "", 0, 0, false, errors.Errorf("missing annotation %s", configuration.GangCardinalityAnnotation) + return "", 1, 1, false, errors.Errorf("missing annotation %s", configuration.GangCardinalityAnnotation) } gangCardinality, err := strconv.Atoi(gangCardinalityString) if err != nil { - return "", 0, 0, false, errors.WithStack(err) + return "", 1, 1, false, errors.WithStack(err) } if gangCardinality <= 0 { - return "", 0, 0, false, errors.Errorf("gang cardinality is non-positive %d", gangCardinality) + return "", 1, 1, false, errors.Errorf("gang cardinality is non-positive %d", gangCardinality) } gangMinimumCardinalityString, ok := annotations[configuration.GangMinimumCardinalityAnnotation] if !ok { @@ -156,10 +175,13 @@ func GangIdAndCardinalityFromAnnotations(annotations map[string]string) (string, } else { gangMinimumCardinality, err := strconv.Atoi(gangMinimumCardinalityString) if err != nil { - return "", 0, 0, false, errors.WithStack(err) + return "", 1, 1, false, errors.WithStack(err) } if gangMinimumCardinality <= 0 { - return "", 0, 0, false, errors.Errorf("gang minimum cardinality is non-positive %d", gangMinimumCardinality) + return "", 1, 1, false, errors.Errorf("gang minimum cardinality is non-positive %d", gangMinimumCardinality) + } + if gangMinimumCardinality > gangCardinality { + return "", 1, 1, false, errors.Errorf("gang minimum cardinality %d cannot be greater than gang cardinality %d", gangMinimumCardinality, gangCardinality) } return gangId, gangCardinality, gangMinimumCardinality, true, nil } diff --git a/internal/scheduler/context/context.go b/internal/scheduler/context/context.go index 29c12bbc374..8a52b497735 100644 --- a/internal/scheduler/context/context.go +++ b/internal/scheduler/context/context.go @@ -226,15 +226,20 @@ func (sctx *SchedulingContext) ReportString(verbosity int32) string { func (sctx *SchedulingContext) AddGangSchedulingContext(gctx *GangSchedulingContext) (bool, error) { allJobsEvictedInThisRound := true allJobsSuccessful := true + numberOfSuccessfulJobs := 0 for _, jctx := range gctx.JobSchedulingContexts { evictedInThisRound, err := sctx.AddJobSchedulingContext(jctx) if err != nil { return false, err } allJobsEvictedInThisRound = allJobsEvictedInThisRound && evictedInThisRound - allJobsSuccessful = allJobsSuccessful && jctx.IsSuccessful() + isSuccess := jctx.IsSuccessful() + allJobsSuccessful = allJobsSuccessful && isSuccess + if isSuccess { + numberOfSuccessfulJobs++ + } } - if allJobsSuccessful && !allJobsEvictedInThisRound { + if numberOfSuccessfulJobs >= gctx.GangMinCardinality && !allJobsEvictedInThisRound { sctx.NumScheduledGangs++ } return allJobsEvictedInThisRound, nil @@ -458,15 +463,6 @@ func (qctx *QueueSchedulingContext) ReportString(verbosity int32) string { return sb.String() } -func (qctx *QueueSchedulingContext) AddGangSchedulingContext(gctx *GangSchedulingContext) error { - for _, jctx := range gctx.JobSchedulingContexts { - if _, err := qctx.AddJobSchedulingContext(jctx); err != nil { - return err - } - } - return nil -} - // AddJobSchedulingContext adds a job scheduling context. // Automatically updates scheduled resources. func (qctx *QueueSchedulingContext) AddJobSchedulingContext(jctx *JobSchedulingContext) (bool, error) { @@ -542,6 +538,7 @@ type GangSchedulingContext struct { TotalResourceRequests schedulerobjects.ResourceList AllJobsEvicted bool NodeUniformityLabel string + GangMinCardinality int } func NewGangSchedulingContext(jctxs []*JobSchedulingContext) *GangSchedulingContext { @@ -550,12 +547,14 @@ func NewGangSchedulingContext(jctxs []*JobSchedulingContext) *GangSchedulingCont queue := "" priorityClassName := "" nodeUniformityLabel := "" + gangMinCardinality := 1 if len(jctxs) > 0 { queue = jctxs[0].Job.GetQueue() priorityClassName = jctxs[0].Job.GetPriorityClassName() if jctxs[0].PodRequirements != nil { nodeUniformityLabel = jctxs[0].PodRequirements.Annotations[configuration.GangNodeUniformityLabelAnnotation] } + gangMinCardinality = jctxs[0].GangMinCardinality } allJobsEvicted := true totalResourceRequests := schedulerobjects.NewResourceList(4) @@ -571,6 +570,7 @@ func NewGangSchedulingContext(jctxs []*JobSchedulingContext) *GangSchedulingCont TotalResourceRequests: totalResourceRequests, AllJobsEvicted: allJobsEvicted, NodeUniformityLabel: nodeUniformityLabel, + GangMinCardinality: gangMinCardinality, } } @@ -600,6 +600,10 @@ type JobSchedulingContext struct { UnschedulableReason string // Pod scheduling contexts for the individual pods that make up the job. PodSchedulingContext *PodSchedulingContext + // The minimum size of the gang associated with this job. + GangMinCardinality int + // If set, indicates this job should be failed back to the client when the gang is scheduled. + ShouldFail bool } func (jctx *JobSchedulingContext) String() string { @@ -615,6 +619,7 @@ func (jctx *JobSchedulingContext) String() string { if jctx.PodSchedulingContext != nil { fmt.Fprint(w, jctx.PodSchedulingContext.String()) } + fmt.Fprintf(w, "GangMinCardinality:\t%d\n", jctx.GangMinCardinality) w.Flush() return sb.String() } @@ -623,15 +628,25 @@ func (jctx *JobSchedulingContext) IsSuccessful() bool { return jctx.UnschedulableReason == "" } -func JobSchedulingContextsFromJobs[J interfaces.LegacySchedulerJob](priorityClasses map[string]types.PriorityClass, jobs []J) []*JobSchedulingContext { +func JobSchedulingContextsFromJobs[J interfaces.LegacySchedulerJob](priorityClasses map[string]types.PriorityClass, jobs []J, extractGangInfo func(map[string]string) (string, int, int, bool, error)) []*JobSchedulingContext { jctxs := make([]*JobSchedulingContext, len(jobs)) timestamp := time.Now() + for i, job := range jobs { + // TODO: Move min cardinality to gang context only and remove from here. + // Requires re-phrasing nodedb in terms of gang context, as well as feeding the value extracted from the annotations downstream. + _, _, gangMinCardinality, _, err := extractGangInfo(job.GetAnnotations()) + if err != nil { + gangMinCardinality = 1 + } + jctxs[i] = &JobSchedulingContext{ - Created: timestamp, - JobId: job.GetId(), - Job: job, - PodRequirements: job.GetPodRequirements(priorityClasses), + Created: timestamp, + JobId: job.GetId(), + Job: job, + PodRequirements: job.GetPodRequirements(priorityClasses), + GangMinCardinality: gangMinCardinality, + ShouldFail: false, } } return jctxs diff --git a/internal/scheduler/context/context_test.go b/internal/scheduler/context/context_test.go index 67fdabd483f..6fe905bfeaf 100644 --- a/internal/scheduler/context/context_test.go +++ b/internal/scheduler/context/context_test.go @@ -89,8 +89,9 @@ func testNSmallCpuJobSchedulingContext(queue, priorityClassName string, n int) [ func testSmallCpuJobSchedulingContext(queue, priorityClassName string) *JobSchedulingContext { job := testfixtures.Test1Cpu4GiJob(queue, priorityClassName) return &JobSchedulingContext{ - JobId: job.GetId(), - Job: job, - PodRequirements: job.GetPodRequirements(testfixtures.TestPriorityClasses), + JobId: job.GetId(), + Job: job, + PodRequirements: job.GetPodRequirements(testfixtures.TestPriorityClasses), + GangMinCardinality: 1, } } diff --git a/internal/scheduler/gang_scheduler.go b/internal/scheduler/gang_scheduler.go index fb9a3add118..f81e6bcaff4 100644 --- a/internal/scheduler/gang_scheduler.go +++ b/internal/scheduler/gang_scheduler.go @@ -38,11 +38,55 @@ func (sch *GangScheduler) SkipUnsuccessfulSchedulingKeyCheck() { sch.skipUnsuccessfulSchedulingKeyCheck = true } -func (sch *GangScheduler) Schedule(ctx *armadacontext.Context, gctx *schedulercontext.GangSchedulingContext) (ok bool, unschedulableReason string, err error) { - // Exit immediately if this is a new gang and we've exceeded any round limits. +func (sch *GangScheduler) updateGangSchedulingContextOnFailure(gctx *schedulercontext.GangSchedulingContext, gangAddedToSchedulingContext bool, unschedulableReason string) (err error) { + if gangAddedToSchedulingContext { + failedJobs := util.Map(gctx.JobSchedulingContexts, func(jctx *schedulercontext.JobSchedulingContext) interfaces.LegacySchedulerJob { return jctx.Job }) + if _, err = sch.schedulingContext.EvictGang(failedJobs); err != nil { + return + } + } + + for _, jctx := range gctx.JobSchedulingContexts { + jctx.UnschedulableReason = unschedulableReason + } + + if _, err = sch.schedulingContext.AddGangSchedulingContext(gctx); err != nil { + return + } + + // Register unfeasible scheduling keys. // - // Because this check occurs before adding the gctx to the sctx, - // the round limits can be exceeded by one gang. + // Only record unfeasible scheduling keys for single-job gangs. + // Since a gang may be unschedulable even if all its members are individually schedulable. + if !sch.skipUnsuccessfulSchedulingKeyCheck && gctx.Cardinality() == 1 { + jctx := gctx.JobSchedulingContexts[0] + schedulingKey := sch.schedulingContext.SchedulingKeyFromLegacySchedulerJob(jctx.Job) + if _, ok := sch.schedulingContext.UnfeasibleSchedulingKeys[schedulingKey]; !ok { + // Keep the first jctx for each unfeasible schedulingKey. + sch.schedulingContext.UnfeasibleSchedulingKeys[schedulingKey] = jctx + } + } + + return +} + +func (sch *GangScheduler) updateGangSchedulingContextOnSuccess(gctx *schedulercontext.GangSchedulingContext, gangAddedToSchedulingContext bool) (err error) { + if gangAddedToSchedulingContext { + jobs := util.Map(gctx.JobSchedulingContexts, func(jctx *schedulercontext.JobSchedulingContext) interfaces.LegacySchedulerJob { return jctx.Job }) + if _, err = sch.schedulingContext.EvictGang(jobs); err != nil { + return + } + } + + if _, err = sch.schedulingContext.AddGangSchedulingContext(gctx); err != nil { + return + } + + return +} + +func (sch *GangScheduler) Schedule(ctx *armadacontext.Context, gctx *schedulercontext.GangSchedulingContext) (ok bool, unschedulableReason string, err error) { + // Exit immediately if this is a new gang and we've hit any round limits. if !gctx.AllJobsEvicted { if ok, unschedulableReason, err = sch.constraints.CheckRoundConstraints(sch.schedulingContext, gctx.Queue); err != nil || !ok { return @@ -66,36 +110,15 @@ func (sch *GangScheduler) Schedule(ctx *armadacontext.Context, gctx *schedulerco } } - // Process unschedulable jobs. - if !ok { - // Register the job as unschedulable. If the job was added to the context, remove it first. - if gangAddedToSchedulingContext { - jobs := util.Map(gctx.JobSchedulingContexts, func(jctx *schedulercontext.JobSchedulingContext) interfaces.LegacySchedulerJob { return jctx.Job }) - if _, err = sch.schedulingContext.EvictGang(jobs); err != nil { - return - } - } - for _, jctx := range gctx.JobSchedulingContexts { - jctx.UnschedulableReason = unschedulableReason - } - if _, err = sch.schedulingContext.AddGangSchedulingContext(gctx); err != nil { - return - } - - // Register unfeasible scheduling keys. - // - // Only record unfeasible scheduling keys for single-job gangs. - // Since a gang may be unschedulable even if all its members are individually schedulable. - if !sch.skipUnsuccessfulSchedulingKeyCheck && gctx.Cardinality() == 1 { - jctx := gctx.JobSchedulingContexts[0] - schedulingKey := sch.schedulingContext.SchedulingKeyFromLegacySchedulerJob(jctx.Job) - if _, ok := sch.schedulingContext.UnfeasibleSchedulingKeys[schedulingKey]; !ok { - // Keep the first jctx for each unfeasible schedulingKey. - sch.schedulingContext.UnfeasibleSchedulingKeys[schedulingKey] = jctx - } - } + if ok { + err = sch.updateGangSchedulingContextOnSuccess(gctx, gangAddedToSchedulingContext) + } else { + err = sch.updateGangSchedulingContextOnFailure(gctx, gangAddedToSchedulingContext, unschedulableReason) } + + return }() + if _, err = sch.schedulingContext.AddGangSchedulingContext(gctx); err != nil { return } @@ -186,23 +209,38 @@ func (sch *GangScheduler) tryScheduleGang(ctx *armadacontext.Context, gctx *sche return } -func (sch *GangScheduler) tryScheduleGangWithTxn(ctx *armadacontext.Context, txn *memdb.Txn, gctx *schedulercontext.GangSchedulingContext) (ok bool, unschedulableReason string, err error) { - if ok, err = sch.nodeDb.ScheduleManyWithTxn(txn, gctx.JobSchedulingContexts); err != nil { - return - } else if !ok { - for _, jctx := range gctx.JobSchedulingContexts { - if jctx.PodSchedulingContext != nil { - // Clear any node bindings on failure to schedule. - jctx.PodSchedulingContext.NodeId = "" +func clearNodeBindings(jctx *schedulercontext.JobSchedulingContext) { + if jctx.PodSchedulingContext != nil { + // Clear any node bindings on failure to schedule. + jctx.PodSchedulingContext.NodeId = "" + } +} + +func (sch *GangScheduler) tryScheduleGangWithTxn(_ *armadacontext.Context, txn *memdb.Txn, gctx *schedulercontext.GangSchedulingContext) (ok bool, unschedulableReason string, err error) { + if ok, err = sch.nodeDb.ScheduleManyWithTxn(txn, gctx.JobSchedulingContexts); err == nil { + if !ok { + for _, jctx := range gctx.JobSchedulingContexts { + clearNodeBindings(jctx) + } + + if gctx.Cardinality() > 1 { + unschedulableReason = "unable to schedule gang since minimum cardinality not met" + } else { + unschedulableReason = "job does not fit on any node" } - } - if gctx.Cardinality() > 1 { - unschedulableReason = "at least one job in the gang does not fit on any node" } else { - unschedulableReason = "job does not fit on any node" + // When a gang schedules successfully, update state for failed jobs if they exist. + for _, jctx := range gctx.JobSchedulingContexts { + if jctx.ShouldFail { + clearNodeBindings(jctx) + jctx.UnschedulableReason = "job does not fit on any node" + } + } } + return } + return } diff --git a/internal/scheduler/gang_scheduler_test.go b/internal/scheduler/gang_scheduler_test.go index cc79703d2b2..cce5b2962c3 100644 --- a/internal/scheduler/gang_scheduler_test.go +++ b/internal/scheduler/gang_scheduler_test.go @@ -34,39 +34,74 @@ func TestGangScheduler(t *testing.T) { Gangs [][]*jobdb.Job // Indices of gangs expected to be scheduled. ExpectedScheduledIndices []int + // Cumulative number of jobs we expect to schedule successfully. + // Each index `i` is the expected value when processing gang `i`. + ExpectedScheduledJobs []int }{ "simple success": { SchedulingConfig: testfixtures.TestSchedulingConfig(), Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32)), }, ExpectedScheduledIndices: testfixtures.IntRange(0, 0), + ExpectedScheduledJobs: []int{32}, }, "simple failure": { SchedulingConfig: testfixtures.TestSchedulingConfig(), Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 33), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 33)), }, ExpectedScheduledIndices: nil, + ExpectedScheduledJobs: []int{0}, + }, + "simple success where min cardinality is met": { + SchedulingConfig: testfixtures.TestSchedulingConfig(), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Gangs: [][]*jobdb.Job{ + testfixtures.WithGangAnnotationsAndMinCardinalityJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 40), 32), + }, + ExpectedScheduledIndices: testfixtures.IntRange(0, 0), + ExpectedScheduledJobs: []int{32}, + }, + "simple failure where min cardinality is not met": { + SchedulingConfig: testfixtures.TestSchedulingConfig(), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Gangs: [][]*jobdb.Job{ + testfixtures.WithGangAnnotationsAndMinCardinalityJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 40), 33), + }, + ExpectedScheduledIndices: nil, + ExpectedScheduledJobs: []int{0}, }, "one success and one failure": { SchedulingConfig: testfixtures.TestSchedulingConfig(), Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), }, ExpectedScheduledIndices: testfixtures.IntRange(0, 0), + ExpectedScheduledJobs: []int{32, 32}, + }, + "one success and one failure using min cardinality": { + SchedulingConfig: testfixtures.TestSchedulingConfig(), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Gangs: [][]*jobdb.Job{ + testfixtures.WithGangAnnotationsAndMinCardinalityJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 33), 32), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), + }, + ExpectedScheduledIndices: testfixtures.IntRange(0, 0), + ExpectedScheduledJobs: []int{32, 32}, }, "multiple nodes": { SchedulingConfig: testfixtures.TestSchedulingConfig(), Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 64), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 64)), }, ExpectedScheduledIndices: testfixtures.IntRange(0, 0), + ExpectedScheduledJobs: []int{64}, }, "MaximumResourceFractionToSchedule": { SchedulingConfig: testfixtures.WithRoundLimitsConfig( @@ -75,11 +110,12 @@ func TestGangScheduler(t *testing.T) { ), Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 8), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 16), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 8), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 8)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 16)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 8)), }, ExpectedScheduledIndices: []int{0, 1}, + ExpectedScheduledJobs: []int{8, 24, 24}, }, "MaximumResourceFractionToScheduleByPool": { SchedulingConfig: testfixtures.WithRoundLimitsConfig( @@ -91,13 +127,14 @@ func TestGangScheduler(t *testing.T) { ), Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), }, ExpectedScheduledIndices: []int{0, 1, 2}, + ExpectedScheduledJobs: []int{1, 2, 3, 3, 3}, }, "MaximumResourceFractionToScheduleByPool non-existing pool": { SchedulingConfig: testfixtures.WithRoundLimitsConfig( @@ -109,13 +146,14 @@ func TestGangScheduler(t *testing.T) { ), Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), }, ExpectedScheduledIndices: []int{0, 1, 2, 3}, + ExpectedScheduledJobs: []int{1, 2, 3, 4, 4}, }, "MaximumResourceFractionPerQueue": { SchedulingConfig: testfixtures.WithPerPriorityLimitsConfig( @@ -129,16 +167,17 @@ func TestGangScheduler(t *testing.T) { ), Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 2), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass1, 2), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass1, 3), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass2, 3), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass2, 4), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass3, 4), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass3, 5), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 2)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass1, 2)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass1, 3)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass2, 3)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass2, 4)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass3, 4)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass3, 5)), }, ExpectedScheduledIndices: []int{0, 2, 4, 6}, + ExpectedScheduledJobs: []int{1, 1, 3, 3, 6, 6, 10, 10}, }, "resolution has no impact on jobs of size a multiple of the resolution": { SchedulingConfig: testfixtures.WithIndexedResourcesConfig( @@ -150,14 +189,15 @@ func TestGangScheduler(t *testing.T) { ), Nodes: testfixtures.N32CpuNodes(3, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), + testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), }, ExpectedScheduledIndices: testfixtures.IntRange(0, 5), + ExpectedScheduledJobs: testfixtures.IntRange(1, 6), }, "jobs of size not a multiple of the resolution blocks scheduling new jobs": { SchedulingConfig: testfixtures.WithIndexedResourcesConfig( @@ -169,12 +209,13 @@ func TestGangScheduler(t *testing.T) { ), Nodes: testfixtures.N32CpuNodes(3, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), + testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), }, ExpectedScheduledIndices: testfixtures.IntRange(0, 2), + ExpectedScheduledJobs: []int{1, 2, 3, 3}, }, "consider all nodes in the bucket": { SchedulingConfig: testfixtures.WithIndexedResourcesConfig( @@ -208,9 +249,10 @@ func TestGangScheduler(t *testing.T) { ), ), Gangs: [][]*jobdb.Job{ - testfixtures.N1GpuJobs("A", testfixtures.PriorityClass0, 1), + testfixtures.WithGangAnnotationsJobs(testfixtures.N1GpuJobs("A", testfixtures.PriorityClass0, 1)), }, ExpectedScheduledIndices: testfixtures.IntRange(0, 0), + ExpectedScheduledJobs: []int{1}, }, "NodeUniformityLabel set but not indexed": { SchedulingConfig: testfixtures.TestSchedulingConfig(), @@ -219,12 +261,14 @@ func TestGangScheduler(t *testing.T) { testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), Gangs: [][]*jobdb.Job{ - testfixtures.WithNodeUniformityLabelAnnotationJobs( - "foo", - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), - ), + testfixtures.WithGangAnnotationsJobs( + testfixtures.WithNodeUniformityLabelAnnotationJobs( + "foo", + testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), + )), }, ExpectedScheduledIndices: nil, + ExpectedScheduledJobs: []int{0}, }, "NodeUniformityLabel not set": { SchedulingConfig: testfixtures.WithIndexedNodeLabelsConfig( @@ -233,12 +277,14 @@ func TestGangScheduler(t *testing.T) { ), Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ - testfixtures.WithNodeUniformityLabelAnnotationJobs( - "foo", - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), - ), + testfixtures.WithGangAnnotationsJobs( + testfixtures.WithNodeUniformityLabelAnnotationJobs( + "foo", + testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), + )), }, ExpectedScheduledIndices: nil, + ExpectedScheduledJobs: []int{0}, }, "NodeUniformityLabel insufficient capacity": { SchedulingConfig: testfixtures.WithIndexedNodeLabelsConfig( @@ -256,12 +302,12 @@ func TestGangScheduler(t *testing.T) { ), ), Gangs: [][]*jobdb.Job{ - testfixtures.WithNodeUniformityLabelAnnotationJobs( - "foo", - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 3), + testfixtures.WithGangAnnotationsJobs( + testfixtures.WithNodeUniformityLabelAnnotationJobs("foo", testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 3)), ), }, ExpectedScheduledIndices: nil, + ExpectedScheduledJobs: []int{0}, }, "NodeUniformityLabel": { SchedulingConfig: testfixtures.WithIndexedNodeLabelsConfig( @@ -291,12 +337,14 @@ func TestGangScheduler(t *testing.T) { ), ), Gangs: [][]*jobdb.Job{ - testfixtures.WithNodeUniformityLabelAnnotationJobs( - "foo", - testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 4), - ), + testfixtures.WithGangAnnotationsJobs( + testfixtures.WithNodeUniformityLabelAnnotationJobs( + "foo", + testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 4), + )), }, ExpectedScheduledIndices: []int{0}, + ExpectedScheduledJobs: []int{4}, }, } for name, tc := range tests { @@ -369,8 +417,9 @@ func TestGangScheduler(t *testing.T) { require.NoError(t, err) var actualScheduledIndices []int + scheduledGangs := 0 for i, gang := range tc.Gangs { - jctxs := schedulercontext.JobSchedulingContextsFromJobs(testfixtures.TestPriorityClasses, gang) + jctxs := schedulercontext.JobSchedulingContextsFromJobs(testfixtures.TestPriorityClasses, gang, GangIdAndCardinalityFromAnnotations) gctx := schedulercontext.NewGangSchedulingContext(jctxs) ok, reason, err := sch.Schedule(armadacontext.Background(), gctx) require.NoError(t, err) @@ -394,8 +443,36 @@ func TestGangScheduler(t *testing.T) { "node uniformity constraint not met: %s", nodeUniformityLabelValues, ) } + + // Verify any excess jobs that failed have the correct state set + for _, jctx := range jctxs { + if jctx.ShouldFail { + if jctx.PodSchedulingContext != nil { + require.Equal(t, "", jctx.PodSchedulingContext.NodeId) + } + require.Equal(t, "job does not fit on any node", jctx.UnschedulableReason) + } + } + + // Verify accounting + scheduledGangs++ + require.Equal(t, scheduledGangs, sch.schedulingContext.NumScheduledGangs) + require.Equal(t, tc.ExpectedScheduledJobs[i], sch.schedulingContext.NumScheduledJobs) + require.Equal(t, 0, sch.schedulingContext.NumEvictedJobs) } else { require.NotEmpty(t, reason) + + // Verify all jobs have been correctly unbound from nodes + for _, jctx := range jctxs { + if jctx.PodSchedulingContext != nil { + require.Equal(t, "", jctx.PodSchedulingContext.NodeId) + } + } + + // Verify accounting + require.Equal(t, scheduledGangs, sch.schedulingContext.NumScheduledGangs) + require.Equal(t, tc.ExpectedScheduledJobs[i], sch.schedulingContext.NumScheduledJobs) + require.Equal(t, 0, sch.schedulingContext.NumEvictedJobs) } } assert.Equal(t, tc.ExpectedScheduledIndices, actualScheduledIndices) diff --git a/internal/scheduler/nodedb/nodedb.go b/internal/scheduler/nodedb/nodedb.go index a2351d1e75f..e529d870ff6 100644 --- a/internal/scheduler/nodedb/nodedb.go +++ b/internal/scheduler/nodedb/nodedb.go @@ -488,13 +488,9 @@ func NodeJobDiff(txnA, txnB *memdb.Txn) (map[string]*Node, map[string]*Node, err return preempted, scheduled, nil } -// ScheduleMany assigns a set of jobs to nodes. The assignment is atomic, i.e., either all jobs are -// successfully assigned to nodes or none are. The returned bool indicates whether assignment -// succeeded (true) or not (false). -// -// This method sets the PodSchedulingContext field on each JobSchedulingContext that it attempts to -// schedule; if it returns early (e.g., because it finds an unschedulable JobSchedulingContext), -// then this field will not be set on the remaining items. +// ScheduleMany assigns a set of jobs to nodes. +// If N jobs can be scheduled, where N >= `GangMinCardinality`, it will return true, nil and set ShouldFail on any excess jobs. +// Otherwise, it will return false, nil. // TODO: Pass through contexts to support timeouts. func (nodeDb *NodeDb) ScheduleMany(jctxs []*schedulercontext.JobSchedulingContext) (bool, error) { txn := nodeDb.db.Txn(true) @@ -507,25 +503,42 @@ func (nodeDb *NodeDb) ScheduleMany(jctxs []*schedulercontext.JobSchedulingContex return ok, err } +// TODO: Remove me once we re-phrase nodedb in terms of gang context (and therefore can just take this value from the gang scheduling context provided) +func gangMinCardinality(jctxs []*schedulercontext.JobSchedulingContext) int { + if len(jctxs) > 0 { + return jctxs[0].GangMinCardinality + } else { + return 1 + } +} + func (nodeDb *NodeDb) ScheduleManyWithTxn(txn *memdb.Txn, jctxs []*schedulercontext.JobSchedulingContext) (bool, error) { // Attempt to schedule pods one by one in a transaction. + cumulativeScheduled := 0 + gangMinCardinality := gangMinCardinality(jctxs) + for _, jctx := range jctxs { + // Defensively reset `ShouldFail` (this should always be false as the state is re-constructed per cycle but just in case) + jctx.ShouldFail = false + node, err := nodeDb.SelectNodeForJobWithTxn(txn, jctx) if err != nil { return false, err } + if node == nil { + // Indicates that when the min cardinality is met, we should fail this job back to the client. + jctx.ShouldFail = true + continue + } + // If we found a node for this pod, bind it and continue to the next pod. - if node != nil { - if node, err := bindJobToNode(nodeDb.priorityClasses, jctx.Job, node); err != nil { + if node, err := bindJobToNode(nodeDb.priorityClasses, jctx.Job, node); err != nil { + return false, err + } else { + if err := nodeDb.UpsertWithTxn(txn, node); err != nil { return false, err - } else { - if err := nodeDb.UpsertWithTxn(txn, node); err != nil { - return false, err - } } - } else { - return false, nil } // Once a job is scheduled, it should no longer be considered for preemption. @@ -534,7 +547,14 @@ func (nodeDb *NodeDb) ScheduleManyWithTxn(txn *memdb.Txn, jctxs []*schedulercont return false, err } } + + cumulativeScheduled++ } + + if cumulativeScheduled < gangMinCardinality { + return false, nil + } + return true, nil } diff --git a/internal/scheduler/nodedb/nodedb_test.go b/internal/scheduler/nodedb/nodedb_test.go index 50f7f9c5a9c..afeeadecb0e 100644 --- a/internal/scheduler/nodedb/nodedb_test.go +++ b/internal/scheduler/nodedb/nodedb_test.go @@ -2,14 +2,17 @@ package nodedb import ( "fmt" + "strconv" "testing" + "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/exp/maps" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + "github.com/armadaproject/armada/internal/armada/configuration" armadamaps "github.com/armadaproject/armada/internal/common/maps" schedulerconfig "github.com/armadaproject/armada/internal/scheduler/configuration" schedulercontext "github.com/armadaproject/armada/internal/scheduler/context" @@ -71,7 +74,7 @@ func TestSelectNodeForPod_NodeIdLabel_Success(t *testing.T) { map[string]string{schedulerconfig.NodeIdLabel: nodeId}, testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), ) - jctxs := schedulercontext.JobSchedulingContextsFromJobs(testfixtures.TestPriorityClasses, jobs) + jctxs := schedulercontext.JobSchedulingContextsFromJobs(testfixtures.TestPriorityClasses, jobs, func(_ map[string]string) (string, int, int, bool, error) { return "", 1, 1, true, nil }) for _, jctx := range jctxs { txn := db.Txn(false) node, err := db.SelectNodeForJobWithTxn(txn, jctx) @@ -100,7 +103,7 @@ func TestSelectNodeForPod_NodeIdLabel_Failure(t *testing.T) { map[string]string{schedulerconfig.NodeIdLabel: "this node does not exist"}, testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), ) - jctxs := schedulercontext.JobSchedulingContextsFromJobs(testfixtures.TestPriorityClasses, jobs) + jctxs := schedulercontext.JobSchedulingContextsFromJobs(testfixtures.TestPriorityClasses, jobs, func(_ map[string]string) (string, int, int, bool, error) { return "", 1, 1, true, nil }) for _, jctx := range jctxs { txn := db.Txn(false) node, err := db.SelectNodeForJobWithTxn(txn, jctx) @@ -435,7 +438,7 @@ func TestScheduleIndividually(t *testing.T) { nodeDb, err := newNodeDbWithNodes(tc.Nodes) require.NoError(t, err) - jctxs := schedulercontext.JobSchedulingContextsFromJobs(testfixtures.TestPriorityClasses, tc.Jobs) + jctxs := schedulercontext.JobSchedulingContextsFromJobs(testfixtures.TestPriorityClasses, tc.Jobs, func(_ map[string]string) (string, int, int, bool, error) { return "", 1, 1, true, nil }) for i, jctx := range jctxs { ok, err := nodeDb.ScheduleMany([]*schedulercontext.JobSchedulingContext{jctx}) @@ -474,6 +477,9 @@ func TestScheduleIndividually(t *testing.T) { } func TestScheduleMany(t *testing.T) { + gangSuccess := testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32)) + gangFailure := testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 33)) + tests := map[string]struct { // Nodes to schedule across. Nodes []*schedulerobjects.Node @@ -483,22 +489,30 @@ func TestScheduleMany(t *testing.T) { // For each group, whether we expect scheduling to succeed. ExpectSuccess []bool }{ + // Attempts to schedule 32 jobs with a minimum gang cardinality of 1 job. All jobs get scheduled. "simple success": { Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), - Jobs: [][]*jobdb.Job{testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32)}, + Jobs: [][]*jobdb.Job{gangSuccess}, ExpectSuccess: []bool{true}, }, - "simple failure": { + // Attempts to schedule 33 jobs with a minimum gang cardinality of 32 jobs. One fails, but the overall result is a success. + "simple success with min cardinality": { Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), - Jobs: [][]*jobdb.Job{testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 33)}, + Jobs: [][]*jobdb.Job{testfixtures.WithGangAnnotationsAndMinCardinalityJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 33), 32)}, + ExpectSuccess: []bool{true}, + }, + // Attempts to schedule 33 jobs with a minimum gang cardinality of 33. The overall result fails. + "simple failure with min cardinality": { + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Jobs: [][]*jobdb.Job{gangFailure}, ExpectSuccess: []bool{false}, }, "correct rollback": { Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), Jobs: [][]*jobdb.Job{ - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 33), - testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32), + gangSuccess, + gangFailure, + gangSuccess, }, ExpectSuccess: []bool{true, false, true}, }, @@ -519,14 +533,27 @@ func TestScheduleMany(t *testing.T) { nodeDb, err := newNodeDbWithNodes(tc.Nodes) require.NoError(t, err) for i, jobs := range tc.Jobs { - jctxs := schedulercontext.JobSchedulingContextsFromJobs(testfixtures.TestPriorityClasses, jobs) - ok, err := nodeDb.ScheduleMany(jctxs) + minCardinalityStr, ok := jobs[0].GetAnnotations()[configuration.GangMinimumCardinalityAnnotation] + if !ok { + minCardinalityStr = "1" + } + minCardinality, err := strconv.Atoi(minCardinalityStr) + if err != nil { + minCardinality = 1 + } + extractGangInfo := func(_ map[string]string) (string, int, int, bool, error) { + id, _ := uuid.NewUUID() + return id.String(), 1, minCardinality, true, nil + } + + jctxs := schedulercontext.JobSchedulingContextsFromJobs(testfixtures.TestPriorityClasses, jobs, extractGangInfo) + ok, err = nodeDb.ScheduleMany(jctxs) require.NoError(t, err) assert.Equal(t, tc.ExpectSuccess[i], ok) for _, jctx := range jctxs { pctx := jctx.PodSchedulingContext require.NotNil(t, pctx) - if tc.ExpectSuccess[i] { + if tc.ExpectSuccess[i] && !jctx.ShouldFail { assert.NotEqual(t, "", pctx.NodeId) } } @@ -591,7 +618,7 @@ func benchmarkScheduleMany(b *testing.B, nodes []*schedulerobjects.Node, jobs [] b.ResetTimer() for n := 0; n < b.N; n++ { - jctxs := schedulercontext.JobSchedulingContextsFromJobs(testfixtures.TestPriorityClasses, jobs) + jctxs := schedulercontext.JobSchedulingContextsFromJobs(testfixtures.TestPriorityClasses, jobs, func(_ map[string]string) (string, int, int, bool, error) { return "", 1, 1, true, nil }) txn := nodeDb.Txn(true) _, err := nodeDb.ScheduleManyWithTxn(txn, jctxs) txn.Abort() diff --git a/internal/scheduler/pool_assigner.go b/internal/scheduler/pool_assigner.go index 9ff1f9b140c..9d636570a61 100644 --- a/internal/scheduler/pool_assigner.go +++ b/internal/scheduler/pool_assigner.go @@ -135,10 +135,11 @@ func (p *DefaultPoolAssigner) AssignPool(j *jobdb.Job) (string, error) { nodeDb := e.nodeDb txn := nodeDb.Txn(true) jctx := &schedulercontext.JobSchedulingContext{ - Created: time.Now(), - JobId: j.GetId(), - Job: j, - PodRequirements: j.GetPodRequirements(p.priorityClasses), + Created: time.Now(), + JobId: j.GetId(), + Job: j, + PodRequirements: j.GetPodRequirements(p.priorityClasses), + GangMinCardinality: 1, } node, err := nodeDb.SelectNodeForJobWithTxn(txn, jctx) txn.Abort() diff --git a/internal/scheduler/preempting_queue_scheduler.go b/internal/scheduler/preempting_queue_scheduler.go index adfbe4d86b5..7be10464132 100644 --- a/internal/scheduler/preempting_queue_scheduler.go +++ b/internal/scheduler/preempting_queue_scheduler.go @@ -260,6 +260,7 @@ func (sch *PreemptingQueueScheduler) Schedule(ctx *armadacontext.Context) (*Sche return &SchedulerResult{ PreemptedJobs: preemptedJobs, ScheduledJobs: scheduledJobs, + FailedJobs: schedulerResult.FailedJobs, NodeIdByJobId: sch.nodeIdByJobId, SchedulingContexts: []*schedulercontext.SchedulingContext{sch.schedulingContext}, }, nil diff --git a/internal/scheduler/preempting_queue_scheduler_test.go b/internal/scheduler/preempting_queue_scheduler_test.go index 84538cdccc2..86077955363 100644 --- a/internal/scheduler/preempting_queue_scheduler_test.go +++ b/internal/scheduler/preempting_queue_scheduler_test.go @@ -516,7 +516,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { { // Schedule a gang across two nodes. JobsByQueue: map[string][]*jobdb.Job{ - "A": testfixtures.WithGangAnnotationsJobs(testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 2)), + "A": testfixtures.WithGangAnnotationsAndMinCardinalityJobs(testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 2), 1), }, ExpectedScheduledIndices: map[string][]int{ "A": testfixtures.IntRange(0, 1), diff --git a/internal/scheduler/queue_scheduler.go b/internal/scheduler/queue_scheduler.go index cf03c7af3fc..95683904eec 100644 --- a/internal/scheduler/queue_scheduler.go +++ b/internal/scheduler/queue_scheduler.go @@ -63,6 +63,7 @@ func (sch *QueueScheduler) SkipUnsuccessfulSchedulingKeyCheck() { func (sch *QueueScheduler) Schedule(ctx *armadacontext.Context) (*SchedulerResult, error) { nodeIdByJobId := make(map[string]string) scheduledJobs := make([]interfaces.LegacySchedulerJob, 0) + failedJobs := make([]interfaces.LegacySchedulerJob, 0) for { // Peek() returns the next gang to try to schedule. Call Clear() before calling Peek() again. // Calling Clear() after (failing to) schedule ensures we get the next gang in order of smallest fair share. @@ -91,13 +92,21 @@ func (sch *QueueScheduler) Schedule(ctx *armadacontext.Context) (*SchedulerResul if ok, unschedulableReason, err := sch.gangScheduler.Schedule(ctx, gctx); err != nil { return nil, err } else if ok { + // We scheduled the minimum number of gang jobs required. for _, jctx := range gctx.JobSchedulingContexts { - scheduledJobs = append(scheduledJobs, jctx.Job) pctx := jctx.PodSchedulingContext if pctx != nil && pctx.NodeId != "" { + scheduledJobs = append(scheduledJobs, jctx.Job) nodeIdByJobId[jctx.JobId] = pctx.NodeId } } + + // Report any excess gang jobs that failed + for _, jctx := range gctx.JobSchedulingContexts { + if jctx.ShouldFail { + failedJobs = append(failedJobs, jctx.Job) + } + } } else if schedulerconstraints.IsTerminalUnschedulableReason(unschedulableReason) { // If unschedulableReason indicates no more new jobs can be scheduled, // instruct the underlying iterator to only yield evicted jobs from now on. @@ -107,6 +116,7 @@ func (sch *QueueScheduler) Schedule(ctx *armadacontext.Context) (*SchedulerResul // instruct the underlying iterator to only yield evicted jobs for this queue from now on. sch.candidateGangIterator.OnlyYieldEvictedForQueue(gctx.Queue) } + // Clear() to get the next gang in order of smallest fair share. // Calling clear here ensures the gang scheduled in this iteration is accounted for. if err := sch.candidateGangIterator.Clear(); err != nil { @@ -122,6 +132,7 @@ func (sch *QueueScheduler) Schedule(ctx *armadacontext.Context) (*SchedulerResul return &SchedulerResult{ PreemptedJobs: nil, ScheduledJobs: scheduledJobs, + FailedJobs: failedJobs, NodeIdByJobId: nodeIdByJobId, SchedulingContexts: []*schedulercontext.SchedulingContext{sch.schedulingContext}, }, nil @@ -208,6 +219,8 @@ func (it *QueuedGangIterator) Peek() (*schedulercontext.GangSchedulingContext, e Job: job, UnschedulableReason: unsuccessfulJctx.UnschedulableReason, PodSchedulingContext: unsuccessfulJctx.PodSchedulingContext, + // TODO: Move this into gang scheduling context + GangMinCardinality: 1, } if _, err := it.schedulingContext.AddJobSchedulingContext(jctx); err != nil { return nil, err @@ -232,6 +245,7 @@ func (it *QueuedGangIterator) Peek() (*schedulercontext.GangSchedulingContext, e schedulercontext.JobSchedulingContextsFromJobs( it.schedulingContext.PriorityClasses, gang, + GangIdAndCardinalityFromAnnotations, ), ) return it.next, nil @@ -241,6 +255,7 @@ func (it *QueuedGangIterator) Peek() (*schedulercontext.GangSchedulingContext, e schedulercontext.JobSchedulingContextsFromJobs( it.schedulingContext.PriorityClasses, []interfaces.LegacySchedulerJob{job}, + GangIdAndCardinalityFromAnnotations, ), ) return it.next, nil diff --git a/internal/scheduler/queue_scheduler_test.go b/internal/scheduler/queue_scheduler_test.go index 3832db7ceba..d5b720aa35a 100644 --- a/internal/scheduler/queue_scheduler_test.go +++ b/internal/scheduler/queue_scheduler_test.go @@ -410,9 +410,19 @@ func TestQueueScheduler(t *testing.T) { SchedulingConfig: testfixtures.TestSchedulingConfig(), Nodes: testfixtures.N32CpuNodes(3, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( - testfixtures.WithAnnotationsJobs(map[string]string{configuration.GangIdAnnotation: "my-gang", configuration.GangCardinalityAnnotation: "2"}, testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithAnnotationsJobs(map[string]string{ + configuration.GangIdAnnotation: "my-gang", + configuration.GangCardinalityAnnotation: "2", + configuration.GangMinimumCardinalityAnnotation: "1", + }, + testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1)), testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.WithAnnotationsJobs(map[string]string{configuration.GangIdAnnotation: "my-gang", configuration.GangCardinalityAnnotation: "2"}, testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithAnnotationsJobs(map[string]string{ + configuration.GangIdAnnotation: "my-gang", + configuration.GangCardinalityAnnotation: "2", + configuration.GangMinimumCardinalityAnnotation: "1", + }, + testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1)), ), PriorityFactorByQueue: map[string]float64{"A": 1}, ExpectedScheduledIndices: []int{0, 1, 2}, @@ -428,9 +438,19 @@ func TestQueueScheduler(t *testing.T) { SchedulingConfig: testfixtures.TestSchedulingConfig(), Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( - testfixtures.WithAnnotationsJobs(map[string]string{configuration.GangIdAnnotation: "my-gang", configuration.GangCardinalityAnnotation: "2"}, testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithAnnotationsJobs(map[string]string{ + configuration.GangIdAnnotation: "my-gang", + configuration.GangCardinalityAnnotation: "2", + configuration.GangMinimumCardinalityAnnotation: "2", + }, + testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1)), testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), - testfixtures.WithAnnotationsJobs(map[string]string{configuration.GangIdAnnotation: "my-gang", configuration.GangCardinalityAnnotation: "2"}, testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1)), + testfixtures.WithAnnotationsJobs(map[string]string{ + configuration.GangIdAnnotation: "my-gang", + configuration.GangCardinalityAnnotation: "2", + configuration.GangMinimumCardinalityAnnotation: "2", + }, + testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1)), ), PriorityFactorByQueue: map[string]float64{"A": 1}, ExpectedScheduledIndices: []int{1}, diff --git a/internal/scheduler/reports_test.go b/internal/scheduler/reports_test.go index fcc0837188a..d989e498fad 100644 --- a/internal/scheduler/reports_test.go +++ b/internal/scheduler/reports_test.go @@ -253,7 +253,7 @@ func withSuccessfulJobSchedulingContext(sctx *schedulercontext.SchedulingContext qctx.SchedulingContext = nil qctx.Created = time.Time{} } - qctx.SuccessfulJobSchedulingContexts[jobId] = &schedulercontext.JobSchedulingContext{JobId: jobId} + qctx.SuccessfulJobSchedulingContexts[jobId] = &schedulercontext.JobSchedulingContext{JobId: jobId, GangMinCardinality: 1} rl := schedulerobjects.ResourceList{Resources: map[string]resource.Quantity{"cpu": resource.MustParse("1")}} qctx.ScheduledResourcesByPriorityClass.AddResourceList("foo", rl) sctx.ScheduledResourcesByPriorityClass.AddResourceList("foo", rl) @@ -293,7 +293,7 @@ func withUnsuccessfulJobSchedulingContext(sctx *schedulercontext.SchedulingConte qctx.SchedulingContext = nil qctx.Created = time.Time{} } - qctx.UnsuccessfulJobSchedulingContexts[jobId] = &schedulercontext.JobSchedulingContext{JobId: jobId, UnschedulableReason: "unknown"} + qctx.UnsuccessfulJobSchedulingContexts[jobId] = &schedulercontext.JobSchedulingContext{JobId: jobId, UnschedulableReason: "unknown", GangMinCardinality: 1} return sctx } diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index 2d4f4a25c2f..58ba50b6f1b 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -17,6 +17,7 @@ import ( "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/common/stringinterner" "github.com/armadaproject/armada/internal/scheduler/database" + "github.com/armadaproject/armada/internal/scheduler/interfaces" "github.com/armadaproject/armada/internal/scheduler/jobdb" "github.com/armadaproject/armada/internal/scheduler/kubernetesobjects/affinity" "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" @@ -397,7 +398,7 @@ func (s *Scheduler) eventsFromSchedulerResult(result *SchedulerResult) ([]*armad // EventsFromSchedulerResult generates necessary EventSequences from the provided SchedulerResult. func EventsFromSchedulerResult(result *SchedulerResult, time time.Time) ([]*armadaevents.EventSequence, error) { - eventSequences := make([]*armadaevents.EventSequence, 0, len(result.PreemptedJobs)+len(result.ScheduledJobs)) + eventSequences := make([]*armadaevents.EventSequence, 0, len(result.PreemptedJobs)+len(result.ScheduledJobs)+len(result.FailedJobs)) eventSequences, err := AppendEventSequencesFromPreemptedJobs(eventSequences, PreemptedJobsFromSchedulerResult[*jobdb.Job](result), time) if err != nil { return nil, err @@ -406,6 +407,10 @@ func EventsFromSchedulerResult(result *SchedulerResult, time time.Time) ([]*arma if err != nil { return nil, err } + eventSequences, err = AppendEventSequencesFromUnschedulableJobs(eventSequences, result.FailedJobs, time) + if err != nil { + return nil, err + } return eventSequences, nil } @@ -505,6 +510,32 @@ func AppendEventSequencesFromScheduledJobs(eventSequences []*armadaevents.EventS return eventSequences, nil } +func AppendEventSequencesFromUnschedulableJobs(eventSequences []*armadaevents.EventSequence, jobs []interfaces.LegacySchedulerJob, time time.Time) ([]*armadaevents.EventSequence, error) { + for _, job := range jobs { + jobId, err := armadaevents.ProtoUuidFromUlidString(job.GetId()) + if err != nil { + return nil, err + } + gangJobUnschedulableError := &armadaevents.Error{ + Terminal: true, + Reason: &armadaevents.Error_GangJobUnschedulable{GangJobUnschedulable: &armadaevents.GangJobUnschedulable{Message: "Job did not meet the minimum gang cardinality"}}, + } + eventSequences = append(eventSequences, &armadaevents.EventSequence{ + Queue: job.GetQueue(), + JobSetName: job.GetJobSet(), + Events: []*armadaevents.EventSequence_Event{ + { + Created: &time, + Event: &armadaevents.EventSequence_Event_JobErrors{ + JobErrors: &armadaevents.JobErrors{JobId: jobId, Errors: []*armadaevents.Error{gangJobUnschedulableError}}, + }, + }, + }, + }) + } + return eventSequences, nil +} + // generateUpdateMessages generates EventSequences representing the state changes on updated jobs // If there are no state changes then an empty slice will be returned func (s *Scheduler) generateUpdateMessages(ctx *armadacontext.Context, updatedJobs []*jobdb.Job, txn *jobdb.Txn) ([]*armadaevents.EventSequence, error) { diff --git a/internal/scheduler/scheduler_test.go b/internal/scheduler/scheduler_test.go index 8c599474bd2..f513d79a8f5 100644 --- a/internal/scheduler/scheduler_test.go +++ b/internal/scheduler/scheduler_test.go @@ -212,6 +212,7 @@ func TestScheduler_TestCycle(t *testing.T) { expectedJobRunLeased []string // ids of jobs we expect to have produced leased messages expectedJobRunErrors []string // ids of jobs we expect to have produced jobRunErrors messages expectedJobErrors []string // ids of jobs we expect to have produced jobErrors messages + expectedJobsToFail []string // ids of jobs we expect to fail without having failed the overall scheduling cycle expectedJobRunPreempted []string // ids of jobs we expect to have produced jobRunPreempted messages expectedJobCancelled []string // ids of jobs we expect to have produced cancelled messages expectedJobRequestCancel []string // ids of jobs we expect to have produced request cancel @@ -254,6 +255,12 @@ func TestScheduler_TestCycle(t *testing.T) { expectedQueued: []string{queuedJob.Id()}, expectedQueuedVersion: queuedJob.QueuedVersion(), }, + "FailedJobs in scheduler result will publish appropriate messages": { + initialJobs: []*jobdb.Job{queuedJob}, + expectedJobErrors: []string{queuedJob.Id()}, + expectedJobsToFail: []string{queuedJob.Id()}, + expectedTerminal: []string{queuedJob.Id()}, + }, "No updates to an already leased job": { initialJobs: []*jobdb.Job{leasedJob}, expectedLeased: []string{leasedJob.Id()}, @@ -592,6 +599,7 @@ func TestScheduler_TestCycle(t *testing.T) { schedulingAlgo := &testSchedulingAlgo{ jobsToSchedule: tc.expectedJobRunLeased, jobsToPreempt: tc.expectedJobRunPreempted, + jobsToFail: tc.expectedJobsToFail, shouldError: tc.scheduleError, } publisher := &testPublisher{shouldError: tc.publishError} @@ -1104,6 +1112,7 @@ type testSchedulingAlgo struct { numberOfScheduleCalls int jobsToPreempt []string jobsToSchedule []string + jobsToFail []string shouldError bool } @@ -1114,6 +1123,7 @@ func (t *testSchedulingAlgo) Schedule(ctx *armadacontext.Context, txn *jobdb.Txn } preemptedJobs := make([]*jobdb.Job, 0, len(t.jobsToPreempt)) scheduledJobs := make([]*jobdb.Job, 0, len(t.jobsToSchedule)) + failedJobs := make([]*jobdb.Job, 0, len(t.jobsToFail)) for _, id := range t.jobsToPreempt { job := jobDb.GetById(txn, id) if job == nil { @@ -1141,13 +1151,27 @@ func (t *testSchedulingAlgo) Schedule(ctx *armadacontext.Context, txn *jobdb.Txn job = job.WithQueuedVersion(job.QueuedVersion()+1).WithQueued(false).WithNewRun("test-executor", "test-node", "node") scheduledJobs = append(scheduledJobs, job) } + for _, id := range t.jobsToFail { + job := jobDb.GetById(txn, id) + if job == nil { + return nil, errors.Errorf("was asked to lease %s but job does not exist", id) + } + if !job.Queued() { + return nil, errors.Errorf("was asked to lease %s but job was already leased", job.Id()) + } + job = job.WithQueued(false).WithFailed(true) + failedJobs = append(failedJobs, job) + } if err := jobDb.Upsert(txn, preemptedJobs); err != nil { return nil, err } if err := jobDb.Upsert(txn, scheduledJobs); err != nil { return nil, err } - return NewSchedulerResult(preemptedJobs, scheduledJobs, nil), nil + if err := jobDb.Upsert(txn, failedJobs); err != nil { + return nil, err + } + return NewSchedulerResultForTest(preemptedJobs, scheduledJobs, failedJobs, nil), nil } type testPublisher struct { diff --git a/internal/scheduler/scheduling_algo.go b/internal/scheduler/scheduling_algo.go index 563ab93f000..58be9e48936 100644 --- a/internal/scheduler/scheduling_algo.go +++ b/internal/scheduler/scheduling_algo.go @@ -95,6 +95,7 @@ func (l *FairSchedulingAlgo) Schedule( overallSchedulerResult := &SchedulerResult{ NodeIdByJobId: make(map[string]string), SchedulingContexts: make([]*schedulercontext.SchedulingContext, 0, 0), + FailedJobs: make([]interfaces.LegacySchedulerJob, 0), } // Exit immediately if scheduling is disabled. @@ -167,19 +168,23 @@ func (l *FairSchedulingAlgo) Schedule( } } - // Update jobDb. preemptedJobs := PreemptedJobsFromSchedulerResult[*jobdb.Job](schedulerResult) scheduledJobs := ScheduledJobsFromSchedulerResult[*jobdb.Job](schedulerResult) + failedJobs := FailedJobsFromSchedulerResult[*jobdb.Job](schedulerResult) if err := jobDb.Upsert(txn, preemptedJobs); err != nil { return nil, err } if err := jobDb.Upsert(txn, scheduledJobs); err != nil { return nil, err } + if err := jobDb.Upsert(txn, failedJobs); err != nil { + return nil, err + } // Aggregate changes across executors. overallSchedulerResult.PreemptedJobs = append(overallSchedulerResult.PreemptedJobs, schedulerResult.PreemptedJobs...) overallSchedulerResult.ScheduledJobs = append(overallSchedulerResult.ScheduledJobs, schedulerResult.ScheduledJobs...) + overallSchedulerResult.FailedJobs = append(overallSchedulerResult.FailedJobs, schedulerResult.FailedJobs...) overallSchedulerResult.SchedulingContexts = append(overallSchedulerResult.SchedulingContexts, schedulerResult.SchedulingContexts...) maps.Copy(overallSchedulerResult.NodeIdByJobId, schedulerResult.NodeIdByJobId) @@ -459,6 +464,10 @@ func (l *FairSchedulingAlgo) scheduleOnExecutors( result.ScheduledJobs[i] = jobDbJob.WithQueuedVersion(jobDbJob.QueuedVersion()+1).WithQueued(false).WithNewRun(node.Executor, node.Id, node.Name) } } + for i, job := range result.FailedJobs { + jobDbJob := job.(*jobdb.Job) + result.FailedJobs[i] = jobDbJob.WithQueued(false).WithFailed(true) + } return result, sctx, nil } diff --git a/internal/scheduler/scheduling_algo_test.go b/internal/scheduler/scheduling_algo_test.go index 2bf766ecd40..9ebb813fae8 100644 --- a/internal/scheduler/scheduling_algo_test.go +++ b/internal/scheduler/scheduling_algo_test.go @@ -46,6 +46,9 @@ func TestSchedule(t *testing.T) { // Indices of queued jobs expected to be scheduled. expectedScheduledIndices []int + + // Count of jobs expected to fail + expectedFailedJobCount int }{ "scheduling": { schedulingConfig: testfixtures.TestSchedulingConfig(), @@ -249,13 +252,21 @@ func TestSchedule(t *testing.T) { }, expectedScheduledIndices: []int{0}, }, - "gang scheduling": { + "gang scheduling successful": { schedulingConfig: testfixtures.TestSchedulingConfig(), executors: []*schedulerobjects.Executor{testfixtures.Test1Node32CoreExecutor("executor1")}, queues: []*database.Queue{{Name: "A", Weight: 100}}, queuedJobs: testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 2)), expectedScheduledIndices: []int{0, 1}, }, + "gang scheduling successful with some jobs failing to schedule above min cardinality": { + schedulingConfig: testfixtures.TestSchedulingConfig(), + executors: []*schedulerobjects.Executor{testfixtures.Test1Node32CoreExecutor("executor1")}, + queues: []*database.Queue{{Name: "A", Weight: 100}}, + queuedJobs: testfixtures.WithGangAnnotationsAndMinCardinalityJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 10), 2), + expectedScheduledIndices: []int{0, 1}, + expectedFailedJobCount: 8, + }, "not scheduling a gang that does not fit on any executor": { schedulingConfig: testfixtures.TestSchedulingConfig(), executors: []*schedulerobjects.Executor{ @@ -433,6 +444,10 @@ func TestSchedule(t *testing.T) { assert.Equal(t, tc.expectedScheduledIndices, actualScheduledIndices) } + // Check that we failed the correct number of excess jobs when a gang schedules >= minimum cardinality + failedJobs := FailedJobsFromSchedulerResult[*jobdb.Job](schedulerResult) + assert.Equal(t, tc.expectedFailedJobCount, len(failedJobs)) + // Check that preempted jobs are marked as such consistently. for _, job := range preemptedJobs { dbJob := jobDb.GetById(txn, job.Id()) @@ -451,6 +466,13 @@ func TestSchedule(t *testing.T) { assert.NotEmpty(t, dbRun.NodeName()) } + // Check that failed jobs are marked as such consistently. + for _, job := range failedJobs { + dbJob := jobDb.GetById(txn, job.Id()) + assert.True(t, dbJob.Failed()) + assert.False(t, dbJob.Queued()) + } + // Check that jobDb was updated correctly. // TODO: Check that there are no unexpected jobs in the jobDb. for _, job := range preemptedJobs { @@ -461,6 +483,10 @@ func TestSchedule(t *testing.T) { dbJob := jobDb.GetById(txn, job.Id()) assert.Equal(t, job, dbJob) } + for _, job := range failedJobs { + dbJob := jobDb.GetById(txn, job.Id()) + assert.Equal(t, job, dbJob) + } }) } } diff --git a/internal/scheduler/simulator/simulator.go b/internal/scheduler/simulator/simulator.go index 94fa8989b84..1c282e8c303 100644 --- a/internal/scheduler/simulator/simulator.go +++ b/internal/scheduler/simulator/simulator.go @@ -534,6 +534,10 @@ func (s *Simulator) handleScheduleEvent() error { if err != nil { return err } + eventSequences, err = scheduler.AppendEventSequencesFromUnschedulableJobs(eventSequences, result.FailedJobs, s.time) + if err != nil { + return err + } } } txn.Commit() diff --git a/internal/scheduler/submitcheck.go b/internal/scheduler/submitcheck.go index d390f3c5037..4fe71c9c59f 100644 --- a/internal/scheduler/submitcheck.go +++ b/internal/scheduler/submitcheck.go @@ -134,17 +134,21 @@ func (srv *SubmitChecker) updateExecutors(ctx *armadacontext.Context) { } func (srv *SubmitChecker) CheckApiJobs(jobs []*api.Job) (bool, string) { - return srv.check(schedulercontext.JobSchedulingContextsFromJobs(srv.priorityClasses, jobs)) + return srv.check(schedulercontext.JobSchedulingContextsFromJobs(srv.priorityClasses, jobs, GangIdAndCardinalityFromAnnotations)) } func (srv *SubmitChecker) CheckJobDbJobs(jobs []*jobdb.Job) (bool, string) { - return srv.check(schedulercontext.JobSchedulingContextsFromJobs(srv.priorityClasses, jobs)) + return srv.check(schedulercontext.JobSchedulingContextsFromJobs(srv.priorityClasses, jobs, GangIdAndCardinalityFromAnnotations)) } func (srv *SubmitChecker) check(jctxs []*schedulercontext.JobSchedulingContext) (bool, string) { // First, check if all jobs can be scheduled individually. for i, jctx := range jctxs { + // Override min cardinality to enable individual job scheduling checks, but reset after + originalGangMinCardinality := jctx.GangMinCardinality + jctx.GangMinCardinality = 1 schedulingResult := srv.getIndividualSchedulingResult(jctx) + jctx.GangMinCardinality = originalGangMinCardinality if !schedulingResult.isSchedulable { return schedulingResult.isSchedulable, fmt.Sprintf("%d-th job unschedulable:\n%s", i, schedulingResult.reason) } @@ -247,7 +251,7 @@ func (srv *SubmitChecker) getSchedulingResult(jctxs []*schedulercontext.JobSched sb.WriteString("\n") } else { sb.WriteString(":") - sb.WriteString(fmt.Sprintf(" %d out of %d pods schedulable\n", numSuccessfullyScheduled, len(jctxs))) + sb.WriteString(fmt.Sprintf(" %d out of %d pods schedulable (minCardinality %d)\n", numSuccessfullyScheduled, len(jctxs), jctxs[0].GangMinCardinality)) } } return schedulingResult{isSchedulable: isSchedulable, reason: sb.String()} diff --git a/internal/scheduler/submitcheck_test.go b/internal/scheduler/submitcheck_test.go index 87be5674bf8..8e418db09a2 100644 --- a/internal/scheduler/submitcheck_test.go +++ b/internal/scheduler/submitcheck_test.go @@ -1,6 +1,7 @@ package scheduler import ( + "fmt" "testing" "time" @@ -218,7 +219,11 @@ func testNJobGang(n int) []*api.Job { gang := make([]*api.Job, n) for i := 0; i < n; i++ { job := test1CoreCpuJob() - job.Annotations = map[string]string{configuration.GangIdAnnotation: gangId} + job.Annotations = map[string]string{ + configuration.GangIdAnnotation: gangId, + configuration.GangCardinalityAnnotation: fmt.Sprintf("%d", n), + configuration.GangMinimumCardinalityAnnotation: fmt.Sprintf("%d", n), + } gang[i] = job } return gang diff --git a/internal/scheduler/testfixtures/testfixtures.go b/internal/scheduler/testfixtures/testfixtures.go index e73d246c74a..a4e45d8a14a 100644 --- a/internal/scheduler/testfixtures/testfixtures.go +++ b/internal/scheduler/testfixtures/testfixtures.go @@ -321,7 +321,17 @@ func WithGangAnnotationsJobs(jobs []*jobdb.Job) []*jobdb.Job { gangId := uuid.NewString() gangCardinality := fmt.Sprintf("%d", len(jobs)) return WithAnnotationsJobs( - map[string]string{configuration.GangIdAnnotation: gangId, configuration.GangCardinalityAnnotation: gangCardinality}, + map[string]string{configuration.GangIdAnnotation: gangId, configuration.GangCardinalityAnnotation: gangCardinality, configuration.GangMinimumCardinalityAnnotation: gangCardinality}, + jobs, + ) +} + +func WithGangAnnotationsAndMinCardinalityJobs(jobs []*jobdb.Job, minimumCardinality int) []*jobdb.Job { + gangId := uuid.NewString() + gangCardinality := fmt.Sprintf("%d", len(jobs)) + gangMinCardinality := fmt.Sprintf("%d", minimumCardinality) + return WithAnnotationsJobs( + map[string]string{configuration.GangIdAnnotation: gangId, configuration.GangCardinalityAnnotation: gangCardinality, configuration.GangMinimumCardinalityAnnotation: gangMinCardinality}, jobs, ) } diff --git a/pkg/armadaevents/events.pb.go b/pkg/armadaevents/events.pb.go index 690ea10745d..3feeac79696 100644 --- a/pkg/armadaevents/events.pb.go +++ b/pkg/armadaevents/events.pb.go @@ -2373,6 +2373,7 @@ type Error struct { // *Error_PodLeaseReturned // *Error_PodTerminated // *Error_JobRunPreemptedError + // *Error_GangJobUnschedulable Reason isError_Reason `protobuf_oneof:"reason"` } @@ -2445,6 +2446,9 @@ type Error_PodTerminated struct { type Error_JobRunPreemptedError struct { JobRunPreemptedError *JobRunPreemptedError `protobuf:"bytes,11,opt,name=jobRunPreemptedError,proto3,oneof" json:"jobRunPreemptedError,omitempty"` } +type Error_GangJobUnschedulable struct { + GangJobUnschedulable *GangJobUnschedulable `protobuf:"bytes,12,opt,name=gangJobUnschedulable,proto3,oneof" json:"gangJobUnschedulable,omitempty"` +} func (*Error_KubernetesError) isError_Reason() {} func (*Error_ContainerError) isError_Reason() {} @@ -2456,6 +2460,7 @@ func (*Error_PodError) isError_Reason() {} func (*Error_PodLeaseReturned) isError_Reason() {} func (*Error_PodTerminated) isError_Reason() {} func (*Error_JobRunPreemptedError) isError_Reason() {} +func (*Error_GangJobUnschedulable) isError_Reason() {} func (m *Error) GetReason() isError_Reason { if m != nil { @@ -2541,6 +2546,13 @@ func (m *Error) GetJobRunPreemptedError() *JobRunPreemptedError { return nil } +func (m *Error) GetGangJobUnschedulable() *GangJobUnschedulable { + if x, ok := m.GetReason().(*Error_GangJobUnschedulable); ok { + return x.GangJobUnschedulable + } + return nil +} + // XXX_OneofWrappers is for the internal use of the proto package. func (*Error) XXX_OneofWrappers() []interface{} { return []interface{}{ @@ -2554,6 +2566,7 @@ func (*Error) XXX_OneofWrappers() []interface{} { (*Error_PodLeaseReturned)(nil), (*Error_PodTerminated)(nil), (*Error_JobRunPreemptedError)(nil), + (*Error_GangJobUnschedulable)(nil), } } @@ -3140,6 +3153,50 @@ func (m *JobRunPreemptedError) XXX_DiscardUnknown() { var xxx_messageInfo_JobRunPreemptedError proto.InternalMessageInfo +type GangJobUnschedulable struct { + Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` +} + +func (m *GangJobUnschedulable) Reset() { *m = GangJobUnschedulable{} } +func (m *GangJobUnschedulable) String() string { return proto.CompactTextString(m) } +func (*GangJobUnschedulable) ProtoMessage() {} +func (*GangJobUnschedulable) Descriptor() ([]byte, []int) { + return fileDescriptor_6aab92ca59e015f8, []int{38} +} +func (m *GangJobUnschedulable) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *GangJobUnschedulable) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_GangJobUnschedulable.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *GangJobUnschedulable) XXX_Merge(src proto.Message) { + xxx_messageInfo_GangJobUnschedulable.Merge(m, src) +} +func (m *GangJobUnschedulable) XXX_Size() int { + return m.Size() +} +func (m *GangJobUnschedulable) XXX_DiscardUnknown() { + xxx_messageInfo_GangJobUnschedulable.DiscardUnknown(m) +} + +var xxx_messageInfo_GangJobUnschedulable proto.InternalMessageInfo + +func (m *GangJobUnschedulable) GetMessage() string { + if m != nil { + return m.Message + } + return "" +} + // Generated by the scheduler whenever it detects a SubmitJob message that includes a previously used deduplication id // (i.e., when it detects a duplicate job submission). type JobDuplicateDetected struct { @@ -3151,7 +3208,7 @@ func (m *JobDuplicateDetected) Reset() { *m = JobDuplicateDetected{} } func (m *JobDuplicateDetected) String() string { return proto.CompactTextString(m) } func (*JobDuplicateDetected) ProtoMessage() {} func (*JobDuplicateDetected) Descriptor() ([]byte, []int) { - return fileDescriptor_6aab92ca59e015f8, []int{38} + return fileDescriptor_6aab92ca59e015f8, []int{39} } func (m *JobDuplicateDetected) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3210,7 +3267,7 @@ func (m *JobRunPreempted) Reset() { *m = JobRunPreempted{} } func (m *JobRunPreempted) String() string { return proto.CompactTextString(m) } func (*JobRunPreempted) ProtoMessage() {} func (*JobRunPreempted) Descriptor() ([]byte, []int) { - return fileDescriptor_6aab92ca59e015f8, []int{39} + return fileDescriptor_6aab92ca59e015f8, []int{40} } func (m *JobRunPreempted) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3279,7 +3336,7 @@ func (m *PartitionMarker) Reset() { *m = PartitionMarker{} } func (m *PartitionMarker) String() string { return proto.CompactTextString(m) } func (*PartitionMarker) ProtoMessage() {} func (*PartitionMarker) Descriptor() ([]byte, []int) { - return fileDescriptor_6aab92ca59e015f8, []int{40} + return fileDescriptor_6aab92ca59e015f8, []int{41} } func (m *PartitionMarker) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3332,7 +3389,7 @@ func (m *JobRunPreemptionRequested) Reset() { *m = JobRunPreemptionReque func (m *JobRunPreemptionRequested) String() string { return proto.CompactTextString(m) } func (*JobRunPreemptionRequested) ProtoMessage() {} func (*JobRunPreemptionRequested) Descriptor() ([]byte, []int) { - return fileDescriptor_6aab92ca59e015f8, []int{41} + return fileDescriptor_6aab92ca59e015f8, []int{42} } func (m *JobRunPreemptionRequested) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3423,6 +3480,7 @@ func init() { proto.RegisterType((*LeaseExpired)(nil), "armadaevents.LeaseExpired") proto.RegisterType((*MaxRunsExceeded)(nil), "armadaevents.MaxRunsExceeded") proto.RegisterType((*JobRunPreemptedError)(nil), "armadaevents.JobRunPreemptedError") + proto.RegisterType((*GangJobUnschedulable)(nil), "armadaevents.GangJobUnschedulable") proto.RegisterType((*JobDuplicateDetected)(nil), "armadaevents.JobDuplicateDetected") proto.RegisterType((*JobRunPreempted)(nil), "armadaevents.JobRunPreempted") proto.RegisterType((*PartitionMarker)(nil), "armadaevents.PartitionMarker") @@ -3432,224 +3490,226 @@ func init() { func init() { proto.RegisterFile("pkg/armadaevents/events.proto", fileDescriptor_6aab92ca59e015f8) } var fileDescriptor_6aab92ca59e015f8 = []byte{ - // 3461 bytes of a gzipped FileDescriptorProto + // 3498 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x5b, 0x4b, 0x6c, 0x1b, 0xd7, - 0xd5, 0xf6, 0x90, 0x12, 0x1f, 0x87, 0x92, 0x48, 0x5f, 0xcb, 0x0a, 0xad, 0xd8, 0xa2, 0x33, 0xce, - 0xff, 0xc7, 0x09, 0x12, 0x32, 0x71, 0x1e, 0xc8, 0xa3, 0x48, 0x20, 0xda, 0x4a, 0x6c, 0xc7, 0xb2, - 0x1d, 0xca, 0x4e, 0xdd, 0x20, 0x05, 0x33, 0xe4, 0x5c, 0x51, 0x63, 0x0d, 0x67, 0x26, 0xf3, 0x90, - 0x25, 0x20, 0x8b, 0xb6, 0x68, 0xd3, 0x5d, 0x6b, 0xa0, 0x5d, 0x14, 0xe8, 0x22, 0xdd, 0x36, 0x40, - 0xd7, 0x5d, 0x77, 0x97, 0x02, 0x45, 0x91, 0x76, 0xd5, 0x15, 0x5b, 0x24, 0xe8, 0xa2, 0x5c, 0x74, - 0xdd, 0x76, 0xd3, 0xe2, 0xbe, 0x66, 0xee, 0x1d, 0x0e, 0x6d, 0xc5, 0x8f, 0x3a, 0x85, 0x57, 0xd2, - 0x7c, 0xe7, 0x39, 0xf7, 0x71, 0xe6, 0x9c, 0x73, 0x2f, 0xe1, 0x98, 0xb7, 0x3d, 0x68, 0x19, 0xfe, - 0xd0, 0x30, 0x0d, 0xbc, 0x83, 0x9d, 0x30, 0x68, 0xb1, 0x3f, 0x4d, 0xcf, 0x77, 0x43, 0x17, 0xcd, - 0xc9, 0xa4, 0x65, 0x7d, 0xfb, 0xe5, 0xa0, 0x69, 0xb9, 0x2d, 0xc3, 0xb3, 0x5a, 0x7d, 0xd7, 0xc7, - 0xad, 0x9d, 0xe7, 0x5a, 0x03, 0xec, 0x60, 0xdf, 0x08, 0xb1, 0xc9, 0x24, 0x96, 0x4f, 0x4a, 0x3c, - 0x0e, 0x0e, 0x6f, 0xb8, 0xfe, 0xb6, 0xe5, 0x0c, 0xb2, 0x38, 0x1b, 0x03, 0xd7, 0x1d, 0xd8, 0xb8, - 0x45, 0x9f, 0x7a, 0xd1, 0x66, 0x2b, 0xb4, 0x86, 0x38, 0x08, 0x8d, 0xa1, 0xc7, 0x19, 0x5e, 0x48, - 0x54, 0x0d, 0x8d, 0xfe, 0x96, 0xe5, 0x60, 0x7f, 0xaf, 0x45, 0xfd, 0xf5, 0xac, 0x96, 0x8f, 0x03, - 0x37, 0xf2, 0xfb, 0x78, 0x42, 0xed, 0x33, 0x03, 0x2b, 0xdc, 0x8a, 0x7a, 0xcd, 0xbe, 0x3b, 0x6c, - 0x0d, 0xdc, 0x81, 0x9b, 0xe8, 0x27, 0x4f, 0xf4, 0x81, 0xfe, 0xc7, 0xd9, 0x5f, 0xb5, 0x9c, 0x10, - 0xfb, 0x8e, 0x61, 0xb7, 0x82, 0xfe, 0x16, 0x36, 0x23, 0x1b, 0xfb, 0xc9, 0x7f, 0x6e, 0xef, 0x3a, - 0xee, 0x87, 0xc1, 0x04, 0xc0, 0x64, 0xf5, 0x9b, 0x8b, 0x30, 0xbf, 0x46, 0x86, 0x66, 0x03, 0x7f, - 0x18, 0x61, 0xa7, 0x8f, 0xd1, 0x93, 0x30, 0xfb, 0x61, 0x84, 0x23, 0x5c, 0xd7, 0x8e, 0x6b, 0x27, - 0xcb, 0xed, 0x43, 0xe3, 0x51, 0xa3, 0x4a, 0x81, 0xa7, 0xdd, 0xa1, 0x15, 0xe2, 0xa1, 0x17, 0xee, - 0x75, 0x18, 0x07, 0x7a, 0x15, 0xe6, 0xae, 0xbb, 0xbd, 0x6e, 0x80, 0xc3, 0xae, 0x63, 0x0c, 0x71, - 0x3d, 0x47, 0x25, 0xea, 0xe3, 0x51, 0x63, 0xf1, 0xba, 0xdb, 0xdb, 0xc0, 0xe1, 0x45, 0x63, 0x28, - 0x8b, 0x41, 0x82, 0xa2, 0x67, 0xa0, 0x18, 0x05, 0xd8, 0xef, 0x5a, 0x66, 0x3d, 0x4f, 0xc5, 0x16, - 0xc7, 0xa3, 0x46, 0x8d, 0x40, 0xe7, 0x4c, 0x49, 0xa4, 0xc0, 0x10, 0xf4, 0x34, 0x14, 0x06, 0xbe, - 0x1b, 0x79, 0x41, 0x7d, 0xe6, 0x78, 0x5e, 0x70, 0x33, 0x44, 0xe6, 0x66, 0x08, 0xba, 0x04, 0x05, - 0x36, 0xdf, 0xf5, 0xd9, 0xe3, 0xf9, 0x93, 0x95, 0x53, 0x8f, 0x35, 0xe5, 0x45, 0xd0, 0x54, 0x5e, - 0x98, 0x3d, 0x31, 0x85, 0x8c, 0x2e, 0x2b, 0xe4, 0xcb, 0xe6, 0x6f, 0x07, 0x61, 0x96, 0xf2, 0xa1, - 0x4b, 0x50, 0xec, 0xfb, 0x98, 0x4c, 0x56, 0x1d, 0x1d, 0xd7, 0x4e, 0x56, 0x4e, 0x2d, 0x37, 0xd9, - 0x22, 0x68, 0x8a, 0x49, 0x6a, 0x5e, 0x11, 0x8b, 0xa0, 0x7d, 0x64, 0x3c, 0x6a, 0x1c, 0xe4, 0xec, - 0x89, 0xd6, 0x9b, 0x7f, 0x6e, 0x68, 0x1d, 0xa1, 0x05, 0x5d, 0x86, 0x72, 0x10, 0xf5, 0x86, 0x56, - 0x78, 0xde, 0xed, 0xd1, 0x31, 0xaf, 0x9c, 0x7a, 0x44, 0x75, 0x77, 0x43, 0x90, 0xdb, 0x8f, 0x8c, - 0x47, 0x8d, 0x43, 0x31, 0x77, 0xa2, 0xf1, 0xec, 0x81, 0x4e, 0xa2, 0x04, 0x6d, 0x41, 0xd5, 0xc7, - 0x9e, 0x6f, 0xb9, 0xbe, 0x15, 0x5a, 0x01, 0x26, 0x7a, 0x73, 0x54, 0xef, 0x31, 0x55, 0x6f, 0x47, - 0x65, 0x6a, 0x1f, 0x1b, 0x8f, 0x1a, 0x47, 0x52, 0x92, 0x8a, 0x8d, 0xb4, 0x5a, 0x14, 0x02, 0x4a, - 0x41, 0x1b, 0x38, 0xa4, 0xf3, 0x59, 0x39, 0x75, 0xfc, 0x96, 0xc6, 0x36, 0x70, 0xd8, 0x3e, 0x3e, - 0x1e, 0x35, 0x8e, 0x4e, 0xca, 0x2b, 0x26, 0x33, 0xf4, 0x23, 0x1b, 0x6a, 0x32, 0x6a, 0x92, 0x17, - 0x9c, 0xa1, 0x36, 0x57, 0xa6, 0xdb, 0x24, 0x5c, 0xed, 0x95, 0xf1, 0xa8, 0xb1, 0x9c, 0x96, 0x55, - 0xec, 0x4d, 0x68, 0x26, 0xf3, 0xd3, 0x37, 0x9c, 0x3e, 0xb6, 0x89, 0x99, 0xd9, 0xac, 0xf9, 0x39, - 0x2d, 0xc8, 0x6c, 0x7e, 0x62, 0x6e, 0x75, 0x7e, 0x62, 0x18, 0xbd, 0x0f, 0x73, 0xf1, 0x03, 0x19, - 0xaf, 0x02, 0x5f, 0x47, 0xd9, 0x4a, 0xc9, 0x48, 0x2d, 0x8f, 0x47, 0x8d, 0x25, 0x59, 0x46, 0x51, - 0xad, 0x68, 0x4b, 0xb4, 0xdb, 0x6c, 0x64, 0x8a, 0xd3, 0xb5, 0x33, 0x0e, 0x59, 0xbb, 0x3d, 0x39, - 0x22, 0x8a, 0x36, 0xa2, 0x9d, 0x6c, 0xe2, 0xa8, 0xdf, 0xc7, 0xd8, 0xc4, 0x66, 0xbd, 0x94, 0xa5, - 0xfd, 0xbc, 0xc4, 0xc1, 0xb4, 0xcb, 0x32, 0xaa, 0x76, 0x99, 0x42, 0xc6, 0xfa, 0xba, 0xdb, 0x5b, - 0xf3, 0x7d, 0xd7, 0x0f, 0xea, 0xe5, 0xac, 0xb1, 0x3e, 0x2f, 0xc8, 0x6c, 0xac, 0x63, 0x6e, 0x75, - 0xac, 0x63, 0x98, 0xfb, 0xdb, 0x89, 0x9c, 0x0b, 0xd8, 0x08, 0xb0, 0x59, 0x87, 0x29, 0xfe, 0xc6, - 0x1c, 0xb1, 0xbf, 0x31, 0x32, 0xe1, 0x6f, 0x4c, 0x41, 0x26, 0x2c, 0xb0, 0xe7, 0xd5, 0x20, 0xb0, - 0x06, 0x0e, 0x36, 0xeb, 0x15, 0xaa, 0xff, 0x68, 0x96, 0x7e, 0xc1, 0xd3, 0x3e, 0x3a, 0x1e, 0x35, - 0xea, 0xaa, 0x9c, 0x62, 0x23, 0xa5, 0x13, 0x7d, 0x00, 0xf3, 0x0c, 0xe9, 0x44, 0x8e, 0x63, 0x39, - 0x83, 0xfa, 0x1c, 0x35, 0xf2, 0x68, 0x96, 0x11, 0xce, 0xd2, 0x7e, 0x74, 0x3c, 0x6a, 0x3c, 0xa2, - 0x48, 0x29, 0x26, 0x54, 0x85, 0x24, 0x62, 0x30, 0x20, 0x99, 0xd8, 0xf9, 0xac, 0x88, 0x71, 0x5e, - 0x65, 0x62, 0x11, 0x23, 0x25, 0xa9, 0x46, 0x8c, 0x14, 0x31, 0x99, 0x0f, 0x3e, 0xc9, 0x0b, 0xd3, - 0xe7, 0x83, 0xcf, 0xb3, 0x34, 0x1f, 0x19, 0x53, 0xad, 0x68, 0x43, 0x1f, 0x01, 0xf9, 0xf0, 0x9c, - 0x89, 0x3c, 0xdb, 0xea, 0x1b, 0x21, 0x3e, 0x83, 0x43, 0xdc, 0x27, 0x91, 0xba, 0x4a, 0xad, 0xe8, - 0x13, 0x56, 0x26, 0x38, 0xdb, 0xfa, 0x78, 0xd4, 0x58, 0xc9, 0xd2, 0xa1, 0x58, 0xcd, 0xb4, 0x82, - 0xbe, 0xa3, 0xc1, 0xe1, 0x20, 0x34, 0x1c, 0xd3, 0xb0, 0x5d, 0x07, 0x9f, 0x73, 0x06, 0x3e, 0x0e, - 0x82, 0x73, 0xce, 0xa6, 0x5b, 0xaf, 0x51, 0xfb, 0x27, 0x52, 0x61, 0x3d, 0x8b, 0xb5, 0x7d, 0x62, - 0x3c, 0x6a, 0x34, 0x32, 0xb5, 0x28, 0x1e, 0x64, 0x1b, 0x42, 0xbb, 0x70, 0x48, 0x64, 0x15, 0x57, - 0x43, 0xcb, 0xb6, 0x02, 0x23, 0xb4, 0x5c, 0xa7, 0x7e, 0x90, 0xda, 0x7f, 0x2c, 0x1d, 0x1d, 0x27, - 0x18, 0xdb, 0x8f, 0x8d, 0x47, 0x8d, 0x63, 0x19, 0x1a, 0x14, 0xdb, 0x59, 0x26, 0x92, 0x25, 0x74, - 0xd9, 0xc7, 0x84, 0x11, 0x9b, 0xf5, 0x43, 0xd3, 0x97, 0x50, 0xcc, 0x24, 0x2f, 0xa1, 0x18, 0xcc, - 0x5a, 0x42, 0x31, 0x91, 0x58, 0xf2, 0x0c, 0x3f, 0xb4, 0x88, 0xd9, 0x75, 0xc3, 0xdf, 0xc6, 0x7e, - 0x7d, 0x31, 0xcb, 0xd2, 0x65, 0x95, 0x89, 0x59, 0x4a, 0x49, 0xaa, 0x96, 0x52, 0x44, 0x74, 0x53, - 0x03, 0xd5, 0x35, 0xcb, 0x75, 0x3a, 0x24, 0x6d, 0x08, 0xc8, 0xeb, 0x1d, 0xa6, 0x46, 0x9f, 0xb8, - 0xc5, 0xeb, 0xc9, 0xec, 0xed, 0x27, 0xc6, 0xa3, 0xc6, 0x89, 0xa9, 0xda, 0x14, 0x47, 0xa6, 0x1b, - 0x45, 0xd7, 0xa0, 0x42, 0x88, 0x98, 0x26, 0x60, 0x66, 0x7d, 0x89, 0xfa, 0x70, 0x64, 0xd2, 0x07, - 0xce, 0x40, 0x33, 0x90, 0xc3, 0x92, 0x84, 0x62, 0x47, 0x56, 0xd5, 0x2e, 0xc2, 0x2c, 0x95, 0xd7, - 0xc7, 0x05, 0x38, 0x94, 0xb1, 0x36, 0xd0, 0xeb, 0x50, 0xf0, 0x23, 0x87, 0x24, 0x6c, 0x2c, 0x4b, - 0x41, 0xaa, 0xd5, 0xab, 0x91, 0x65, 0xb2, 0x6c, 0xd1, 0x8f, 0x1c, 0x25, 0x87, 0x9b, 0xa5, 0x00, - 0x91, 0x27, 0xd9, 0xa2, 0x65, 0xf2, 0x6c, 0x64, 0xaa, 0xfc, 0x75, 0xb7, 0xa7, 0xca, 0x53, 0x00, - 0x61, 0x98, 0x17, 0x0b, 0xaf, 0x6b, 0x91, 0x5d, 0xc5, 0xf2, 0x8c, 0xc7, 0x55, 0x35, 0x6f, 0x47, - 0x3d, 0xec, 0x3b, 0x38, 0xc4, 0x81, 0x78, 0x07, 0xba, 0xad, 0x68, 0x14, 0xf1, 0x25, 0x44, 0xd2, - 0x3f, 0x27, 0xe3, 0xe8, 0xa7, 0x1a, 0xd4, 0x87, 0xc6, 0x6e, 0x57, 0x80, 0x41, 0x77, 0xd3, 0xf5, - 0xbb, 0x1e, 0xf6, 0x2d, 0xd7, 0xa4, 0xc9, 0x67, 0xe5, 0xd4, 0x37, 0x6e, 0xbb, 0x91, 0x9a, 0xeb, - 0xc6, 0xae, 0x80, 0x83, 0x37, 0x5d, 0xff, 0x32, 0x15, 0x5f, 0x73, 0x42, 0x7f, 0xaf, 0x7d, 0xec, - 0xb3, 0x51, 0xe3, 0x00, 0x99, 0x96, 0x61, 0x16, 0x4f, 0x27, 0x1b, 0x46, 0x3f, 0xd6, 0x60, 0x29, - 0x74, 0x43, 0xc3, 0xee, 0xf6, 0xa3, 0x61, 0x64, 0x1b, 0xa1, 0xb5, 0x83, 0xbb, 0x51, 0x60, 0x0c, - 0x30, 0xcf, 0x71, 0x5f, 0xbb, 0xbd, 0x53, 0x57, 0x88, 0xfc, 0xe9, 0x58, 0xfc, 0x2a, 0x91, 0x66, - 0x3e, 0x1d, 0xe5, 0x3e, 0x2d, 0x86, 0x19, 0x2c, 0x9d, 0x4c, 0x74, 0xf9, 0x17, 0x1a, 0x2c, 0x4f, - 0x7f, 0x4d, 0x74, 0x02, 0xf2, 0xdb, 0x78, 0x8f, 0x57, 0x11, 0x07, 0xc7, 0xa3, 0xc6, 0xfc, 0x36, - 0xde, 0x93, 0x46, 0x9d, 0x50, 0xd1, 0xb7, 0x60, 0x76, 0xc7, 0xb0, 0x23, 0xcc, 0x97, 0x44, 0xb3, - 0xc9, 0xea, 0xa5, 0xa6, 0x5c, 0x2f, 0x35, 0xbd, 0xed, 0x01, 0x01, 0x9a, 0x62, 0x46, 0x9a, 0xef, - 0x44, 0x86, 0x13, 0x5a, 0xe1, 0x1e, 0x5b, 0x2e, 0x54, 0x81, 0xbc, 0x5c, 0x28, 0xf0, 0x6a, 0xee, - 0x65, 0x6d, 0xf9, 0x13, 0x0d, 0x8e, 0x4c, 0x7d, 0xe9, 0xaf, 0x83, 0x87, 0x7a, 0x17, 0x66, 0xc8, - 0xc2, 0x27, 0xf5, 0xcd, 0x96, 0x35, 0xd8, 0x7a, 0xe9, 0x05, 0xea, 0x4e, 0x81, 0x95, 0x23, 0x0c, - 0x91, 0xcb, 0x11, 0x86, 0x90, 0x1a, 0xcd, 0x76, 0x6f, 0xbc, 0xf4, 0x02, 0x75, 0xaa, 0xc0, 0x8c, - 0x50, 0x40, 0x36, 0x42, 0x01, 0xfd, 0xdf, 0x05, 0x28, 0xc7, 0x05, 0x84, 0xb4, 0x07, 0xb5, 0x3b, - 0xda, 0x83, 0x67, 0xa1, 0x66, 0x62, 0x93, 0x7f, 0xf9, 0x2c, 0xd7, 0x11, 0xbb, 0xb9, 0xcc, 0xa2, - 0xab, 0x42, 0x53, 0xe4, 0xab, 0x29, 0x12, 0x3a, 0x05, 0x25, 0x9e, 0x68, 0xef, 0xd1, 0x8d, 0x3c, - 0xdf, 0x5e, 0x1a, 0x8f, 0x1a, 0x48, 0x60, 0x92, 0x68, 0xcc, 0x87, 0x3a, 0x00, 0xac, 0x7a, 0x5d, - 0xc7, 0xa1, 0xc1, 0x53, 0xfe, 0xba, 0xfa, 0x06, 0x97, 0x62, 0x3a, 0xab, 0x43, 0x13, 0x7e, 0xb9, - 0x0e, 0x4d, 0x50, 0xf4, 0x3e, 0xc0, 0xd0, 0xb0, 0x1c, 0x26, 0xc7, 0xf3, 0x7b, 0x7d, 0x5a, 0x48, - 0x59, 0x8f, 0x39, 0x99, 0xf6, 0x44, 0x52, 0xd6, 0x9e, 0xa0, 0xa4, 0x5a, 0xe4, 0xf5, 0x76, 0xbd, - 0x40, 0x77, 0xe9, 0xca, 0x34, 0xd5, 0x5c, 0xed, 0x61, 0x52, 0x31, 0x72, 0x11, 0x49, 0xa7, 0xd0, - 0x42, 0x86, 0xcd, 0xb6, 0x36, 0x71, 0x68, 0x0d, 0x31, 0xcd, 0xec, 0xf9, 0xb0, 0x09, 0x4c, 0x1e, - 0x36, 0x81, 0xa1, 0x97, 0x01, 0x8c, 0x70, 0xdd, 0x0d, 0xc2, 0x4b, 0x4e, 0x1f, 0xd3, 0x8c, 0xbd, - 0xc4, 0xdc, 0x4f, 0x50, 0xd9, 0xfd, 0x04, 0x45, 0xaf, 0x41, 0xc5, 0xe3, 0x1f, 0xa1, 0x9e, 0x8d, - 0x69, 0x46, 0x5e, 0x62, 0x9f, 0x14, 0x09, 0x96, 0x64, 0x65, 0x6e, 0xf4, 0x16, 0x54, 0xfb, 0xae, - 0xd3, 0x8f, 0x7c, 0x1f, 0x3b, 0xfd, 0xbd, 0x0d, 0x63, 0x13, 0xd3, 0xec, 0xbb, 0xc4, 0x96, 0x4a, - 0x8a, 0x24, 0x2f, 0x95, 0x14, 0x09, 0xbd, 0x08, 0xe5, 0xb8, 0x7b, 0x41, 0x13, 0xec, 0x32, 0x2f, - 0x84, 0x05, 0x28, 0x09, 0x27, 0x9c, 0xc4, 0x79, 0x2b, 0x88, 0xb3, 0x34, 0x9a, 0x34, 0x73, 0xe7, - 0x25, 0x58, 0x76, 0x5e, 0x82, 0xd1, 0x39, 0x38, 0x48, 0xbf, 0x8b, 0xdd, 0x30, 0xb4, 0xbb, 0x01, - 0xee, 0xbb, 0x8e, 0x19, 0xd0, 0x9c, 0x38, 0xcf, 0xdc, 0xa7, 0xc4, 0x2b, 0xa1, 0xbd, 0xc1, 0x48, - 0xb2, 0xfb, 0x29, 0x92, 0xfe, 0x3b, 0x0d, 0x16, 0xb3, 0x96, 0x50, 0x6a, 0x39, 0x6b, 0xf7, 0x64, - 0x39, 0xbf, 0x0b, 0x25, 0xcf, 0x35, 0xbb, 0x81, 0x87, 0xfb, 0x3c, 0x62, 0xa5, 0x16, 0xf3, 0x65, - 0xd7, 0xdc, 0xf0, 0x70, 0xff, 0x9b, 0x56, 0xb8, 0xb5, 0xba, 0xe3, 0x5a, 0xe6, 0x05, 0x2b, 0xe0, - 0xab, 0xce, 0x63, 0x14, 0x25, 0x43, 0x28, 0x72, 0xb0, 0x5d, 0x82, 0x02, 0xb3, 0xa2, 0xff, 0x3e, - 0x0f, 0xb5, 0xf4, 0xb2, 0xfd, 0x5f, 0x7a, 0x15, 0x74, 0x0d, 0x8a, 0x16, 0x4b, 0x99, 0x79, 0x06, - 0xf1, 0x7f, 0x52, 0x4c, 0x6f, 0x26, 0x0d, 0xbf, 0xe6, 0xce, 0x73, 0x4d, 0x9e, 0x5b, 0xd3, 0x21, - 0xa0, 0x9a, 0xb9, 0xa4, 0xaa, 0x99, 0x83, 0xa8, 0x03, 0xc5, 0x00, 0xfb, 0x3b, 0x56, 0x1f, 0xf3, - 0xe0, 0xd4, 0x90, 0x35, 0xf7, 0x5d, 0x1f, 0x13, 0x9d, 0x1b, 0x8c, 0x25, 0xd1, 0xc9, 0x65, 0x54, - 0x9d, 0x1c, 0x44, 0xef, 0x42, 0xb9, 0xef, 0x3a, 0x9b, 0xd6, 0x60, 0xdd, 0xf0, 0x78, 0x78, 0x3a, - 0x96, 0xa5, 0xf5, 0xb4, 0x60, 0xe2, 0x4d, 0x08, 0xf1, 0x98, 0x6a, 0x42, 0xc4, 0x5c, 0xc9, 0x84, - 0xfe, 0x7d, 0x06, 0x20, 0x99, 0x1c, 0xf4, 0x0a, 0x54, 0xf0, 0x2e, 0xee, 0x47, 0xa1, 0xeb, 0x8b, - 0xef, 0x04, 0xef, 0xe9, 0x09, 0x58, 0x09, 0xec, 0x90, 0xa0, 0x64, 0xa3, 0x3a, 0xc6, 0x10, 0x07, - 0x9e, 0xd1, 0x17, 0xcd, 0x40, 0xea, 0x4c, 0x0c, 0xca, 0x1b, 0x35, 0x06, 0xd1, 0xff, 0xc3, 0x0c, - 0x6d, 0x1f, 0xb2, 0x3e, 0x20, 0x1a, 0x8f, 0x1a, 0x0b, 0x8e, 0xda, 0x38, 0xa4, 0x74, 0xf4, 0x06, - 0xcc, 0x6f, 0xc7, 0x0b, 0x8f, 0xf8, 0x36, 0x43, 0x05, 0x68, 0x6a, 0x97, 0x10, 0x14, 0xef, 0xe6, - 0x64, 0x1c, 0x6d, 0x42, 0xc5, 0x70, 0x1c, 0x37, 0xa4, 0xdf, 0x20, 0xd1, 0x1b, 0x7c, 0x72, 0xda, - 0x32, 0x6d, 0xae, 0x26, 0xbc, 0x2c, 0x4b, 0xa2, 0xc1, 0x43, 0xd2, 0x20, 0x07, 0x0f, 0x09, 0x46, - 0x1d, 0x28, 0xd8, 0x46, 0x0f, 0xdb, 0x22, 0xe8, 0x3f, 0x3e, 0xd5, 0xc4, 0x05, 0xca, 0xc6, 0xb4, - 0xd3, 0x4f, 0x3e, 0x93, 0x93, 0x3f, 0xf9, 0x0c, 0x59, 0xde, 0x84, 0x5a, 0xda, 0x9f, 0xfd, 0x25, - 0x30, 0x4f, 0xca, 0x09, 0x4c, 0xf9, 0xb6, 0x29, 0x93, 0x01, 0x15, 0xc9, 0xa9, 0xfb, 0x61, 0x42, - 0xff, 0xa5, 0x06, 0x8b, 0x59, 0x7b, 0x17, 0xad, 0x4b, 0x3b, 0x5e, 0xe3, 0x3d, 0x8e, 0x8c, 0xa5, - 0xce, 0x65, 0xa7, 0x6c, 0xf5, 0x64, 0xa3, 0xb7, 0x61, 0xc1, 0x71, 0x4d, 0xdc, 0x35, 0x88, 0x01, - 0xdb, 0x0a, 0xc2, 0x7a, 0x8e, 0xf6, 0x8e, 0x69, 0x6f, 0x84, 0x50, 0x56, 0x05, 0x41, 0x92, 0x9e, - 0x57, 0x08, 0xfa, 0x0f, 0x34, 0xa8, 0xa6, 0x5a, 0x97, 0x77, 0x9d, 0x44, 0xc9, 0xa9, 0x4f, 0x6e, - 0x7f, 0xa9, 0x8f, 0xfe, 0x93, 0x1c, 0x54, 0xa4, 0xba, 0xee, 0xae, 0x7d, 0xb8, 0x0e, 0x55, 0xfe, - 0xa5, 0xb4, 0x9c, 0x01, 0x2b, 0xa7, 0x72, 0xbc, 0x49, 0x31, 0x71, 0x52, 0x70, 0xde, 0xed, 0x6d, - 0xc4, 0xbc, 0xb4, 0x9a, 0xa2, 0x1d, 0xac, 0x40, 0xc1, 0x24, 0x13, 0x0b, 0x2a, 0x05, 0x5d, 0x83, - 0xa5, 0xc8, 0x33, 0x8d, 0x10, 0x77, 0x03, 0xde, 0x73, 0xef, 0x3a, 0xd1, 0xb0, 0x87, 0x7d, 0xba, - 0xe3, 0x67, 0x59, 0xcf, 0x85, 0x71, 0x88, 0xa6, 0xfc, 0x45, 0x4a, 0x97, 0x74, 0x2e, 0x66, 0xd1, - 0xf5, 0xb3, 0x80, 0x26, 0xfb, 0xca, 0xca, 0xf8, 0x6a, 0xfb, 0x1c, 0xdf, 0x8f, 0x35, 0xa8, 0xa5, - 0xdb, 0xc5, 0x0f, 0x64, 0xa2, 0xf7, 0xa0, 0x1c, 0xb7, 0x7e, 0xef, 0xda, 0x81, 0xa7, 0xa1, 0xe0, - 0x63, 0x23, 0x70, 0x1d, 0xbe, 0x33, 0x69, 0x88, 0x61, 0x88, 0x1c, 0x62, 0x18, 0xa2, 0x5f, 0x81, - 0x39, 0x36, 0x82, 0x6f, 0x5a, 0x76, 0x88, 0x7d, 0x74, 0x06, 0x0a, 0x41, 0x68, 0x84, 0x38, 0xa8, - 0x6b, 0xc7, 0xf3, 0x27, 0x17, 0x4e, 0x2d, 0x4d, 0x76, 0x79, 0x09, 0x99, 0x69, 0x65, 0x9c, 0xb2, - 0x56, 0x86, 0xe8, 0xdf, 0xd3, 0x60, 0x4e, 0x6e, 0x66, 0xdf, 0x1b, 0xb5, 0x5f, 0xf1, 0xd5, 0x3e, - 0x12, 0x3e, 0xd8, 0xf7, 0x66, 0x66, 0xbf, 0x9a, 0xf5, 0x5f, 0x6b, 0x6c, 0x64, 0xe3, 0x2e, 0xe8, - 0xdd, 0x9a, 0x1f, 0x24, 0xad, 0x10, 0xb2, 0xc3, 0x02, 0x1a, 0xd8, 0xf6, 0xdb, 0x0a, 0xa1, 0xe1, - 0x4f, 0x11, 0x97, 0xc3, 0x9f, 0x42, 0xd0, 0xff, 0x98, 0xa3, 0x9e, 0x27, 0x1d, 0xef, 0x07, 0xdd, - 0x04, 0x4a, 0x65, 0x27, 0xf9, 0xaf, 0x90, 0x9d, 0x3c, 0x03, 0x45, 0xfa, 0x39, 0x88, 0x13, 0x07, - 0x3a, 0x69, 0x04, 0x52, 0x4f, 0x1c, 0x19, 0x72, 0x8b, 0xa8, 0x35, 0x7b, 0x97, 0x51, 0xeb, 0x9f, - 0x1a, 0x2c, 0xa8, 0x47, 0x02, 0x0f, 0x7c, 0x58, 0x27, 0x16, 0x54, 0xfe, 0x3e, 0x2d, 0xa8, 0x7f, - 0x68, 0x30, 0xaf, 0x9c, 0x54, 0x3c, 0x3c, 0xaf, 0xfe, 0xb3, 0x1c, 0x2c, 0x65, 0xab, 0xb9, 0x2f, - 0xe5, 0xd3, 0x59, 0x20, 0x89, 0xd0, 0xb9, 0xe4, 0xcb, 0x7e, 0x78, 0xa2, 0x7a, 0xa2, 0xaf, 0x20, - 0xb2, 0xa8, 0x89, 0x23, 0x06, 0x21, 0x8e, 0xae, 0x41, 0xc5, 0x92, 0x0e, 0x33, 0xf2, 0x59, 0x3d, - 0x67, 0xf9, 0x08, 0x83, 0xd5, 0xd8, 0x53, 0x0e, 0x2e, 0x64, 0x55, 0xed, 0x02, 0xcc, 0x90, 0xd4, - 0x43, 0xdf, 0x81, 0x22, 0x77, 0x07, 0x3d, 0x0f, 0x65, 0xba, 0x4b, 0x69, 0x45, 0xc0, 0xd2, 0x4e, - 0xfa, 0xd1, 0x24, 0x60, 0xea, 0x3a, 0x41, 0x49, 0x60, 0xe8, 0x25, 0x00, 0x92, 0x38, 0xf2, 0xfd, - 0x99, 0xa3, 0xfb, 0x93, 0x56, 0x1e, 0x9e, 0x6b, 0x4e, 0x6c, 0xca, 0x72, 0x0c, 0xea, 0xbf, 0xca, - 0x41, 0x45, 0x3e, 0x3e, 0xb9, 0x23, 0xe3, 0x1f, 0x81, 0xa8, 0x0a, 0xbb, 0x86, 0x69, 0x92, 0xbf, - 0x58, 0x04, 0xe4, 0xd6, 0xd4, 0x41, 0x12, 0xff, 0xaf, 0x0a, 0x09, 0x56, 0x03, 0xd0, 0x03, 0x6a, - 0x2b, 0x45, 0x92, 0xac, 0xd6, 0xd2, 0xb4, 0xe5, 0x6d, 0x38, 0x9c, 0xa9, 0x4a, 0xce, 0xdc, 0x67, - 0xef, 0x55, 0xe6, 0xfe, 0x9b, 0x59, 0x38, 0x9c, 0x79, 0x6c, 0xf5, 0xc0, 0x77, 0xb1, 0xba, 0x83, - 0xf2, 0xf7, 0x64, 0x07, 0x7d, 0xac, 0x65, 0xcd, 0x2c, 0x3b, 0x02, 0x78, 0x65, 0x1f, 0x67, 0x79, - 0xf7, 0x6a, 0x8e, 0xd5, 0x65, 0x39, 0x7b, 0x47, 0x7b, 0xa2, 0xb0, 0xdf, 0x3d, 0x81, 0x9e, 0x65, - 0x45, 0x18, 0xb5, 0x55, 0xa4, 0xb6, 0x44, 0x84, 0x48, 0x99, 0x2a, 0x72, 0x88, 0xd4, 0xe5, 0x42, - 0x82, 0x95, 0xfe, 0xa5, 0xa4, 0x2e, 0xe7, 0x3c, 0xe9, 0xea, 0x7f, 0x4e, 0xc6, 0xff, 0xbb, 0x6b, - 0xf8, 0x5f, 0x1a, 0x54, 0x53, 0xe7, 0xd8, 0x0f, 0xcf, 0x37, 0xe8, 0x47, 0x1a, 0x94, 0xe3, 0x2b, - 0x14, 0x77, 0x9d, 0x86, 0xae, 0x42, 0x01, 0xb3, 0x63, 0x7c, 0x16, 0xee, 0x0e, 0xa5, 0xae, 0x59, - 0x11, 0x1a, 0xbf, 0x58, 0x95, 0x3a, 0xb9, 0xef, 0x70, 0x41, 0xfd, 0x0f, 0x9a, 0x48, 0x30, 0x13, - 0x9f, 0x1e, 0xe8, 0x54, 0x24, 0xef, 0x94, 0xbf, 0xd3, 0x77, 0xfa, 0x6d, 0x09, 0x66, 0x29, 0x1f, - 0x29, 0x00, 0x43, 0xec, 0x0f, 0x2d, 0xc7, 0xb0, 0xe9, 0xeb, 0x94, 0xd8, 0xbe, 0x15, 0x98, 0xbc, - 0x6f, 0x05, 0x86, 0xb6, 0xa0, 0x9a, 0x34, 0xad, 0xa8, 0x9a, 0xec, 0xdb, 0x5b, 0x6f, 0xab, 0x4c, - 0xac, 0x2d, 0x9d, 0x92, 0x54, 0x8f, 0xb7, 0x53, 0x44, 0x64, 0xc2, 0x42, 0xdf, 0x75, 0x42, 0xc3, - 0x72, 0xb0, 0xcf, 0x0c, 0xe5, 0xb3, 0x6e, 0xaf, 0x9c, 0x56, 0x78, 0x58, 0xed, 0xaf, 0xca, 0xa9, - 0xb7, 0x57, 0x54, 0x1a, 0xfa, 0x00, 0xe6, 0x45, 0x12, 0xce, 0x8c, 0xcc, 0x64, 0xdd, 0x5e, 0x59, - 0x93, 0x59, 0xd8, 0x92, 0x56, 0xa4, 0xd4, 0xdb, 0x2b, 0x0a, 0x09, 0xd9, 0x50, 0xf3, 0x5c, 0xf3, - 0xaa, 0xc3, 0xdb, 0x0e, 0x46, 0xcf, 0xc6, 0xbc, 0x53, 0xba, 0x32, 0x91, 0xf2, 0x28, 0x5c, 0x2c, - 0x14, 0xa7, 0x65, 0xd5, 0xfb, 0x60, 0x69, 0x2a, 0x7a, 0x1f, 0xe6, 0x6c, 0x52, 0x0b, 0xad, 0xed, - 0x7a, 0x96, 0x8f, 0xcd, 0xec, 0xdb, 0x5b, 0x17, 0x24, 0x0e, 0x16, 0x08, 0x65, 0x19, 0xf5, 0x06, - 0x8b, 0x4c, 0x21, 0xb3, 0x3f, 0x34, 0x76, 0x3b, 0x91, 0x13, 0xac, 0xed, 0xf2, 0x9b, 0x38, 0xc5, - 0xac, 0xd9, 0x5f, 0x57, 0x99, 0xd8, 0xec, 0xa7, 0x24, 0xd5, 0xd9, 0x4f, 0x11, 0xd1, 0x05, 0x1a, - 0xe7, 0xd9, 0x94, 0xb0, 0x5b, 0x5c, 0x4b, 0x13, 0xa3, 0xc5, 0x66, 0x83, 0x35, 0x2d, 0xf8, 0x93, - 0xa2, 0x34, 0xd6, 0xc0, 0xe7, 0x80, 0xbe, 0x76, 0x07, 0x87, 0x91, 0xef, 0x60, 0x93, 0x5f, 0xe0, - 0x9a, 0x9c, 0x03, 0x85, 0x2b, 0x9e, 0x03, 0x05, 0x9d, 0x98, 0x03, 0x85, 0x4a, 0xd6, 0x94, 0xe7, - 0x9a, 0x57, 0xd8, 0x96, 0x09, 0xe3, 0x6b, 0x5d, 0x8f, 0x4e, 0x98, 0x4a, 0x58, 0xd8, 0x9a, 0x52, - 0xa4, 0xd4, 0x35, 0xa5, 0x90, 0xf8, 0x4d, 0x22, 0xf9, 0xde, 0x09, 0x1b, 0xa9, 0xca, 0x94, 0x9b, - 0x44, 0x13, 0x9c, 0xf1, 0x4d, 0xa2, 0x09, 0xca, 0xc4, 0x4d, 0xa2, 0x49, 0xd9, 0x92, 0x68, 0x2f, - 0xe8, 0x9f, 0x68, 0x50, 0x4d, 0xed, 0x74, 0xf4, 0x3a, 0xc4, 0x37, 0x16, 0xae, 0xec, 0x79, 0x22, - 0x51, 0x55, 0x6e, 0x38, 0x10, 0x3c, 0xeb, 0x86, 0x03, 0xc1, 0xd1, 0x05, 0x80, 0xf8, 0xab, 0x70, - 0xab, 0x30, 0x49, 0xb3, 0xa4, 0x84, 0x53, 0xce, 0x92, 0x12, 0x54, 0xff, 0x3c, 0x0f, 0x25, 0xb1, - 0x54, 0xee, 0x4b, 0x21, 0xd3, 0x82, 0xe2, 0x10, 0x07, 0xf4, 0xa6, 0x43, 0x2e, 0xc9, 0x47, 0x38, - 0x24, 0xe7, 0x23, 0x1c, 0x52, 0xd3, 0xa5, 0xfc, 0x1d, 0xa5, 0x4b, 0x33, 0xfb, 0x4e, 0x97, 0x30, - 0x3d, 0xe5, 0x94, 0x02, 0x9e, 0x38, 0x57, 0xb8, 0x75, 0x14, 0x15, 0x67, 0xa0, 0xb2, 0x60, 0xea, - 0x0c, 0x54, 0x26, 0xa1, 0x6d, 0x38, 0x28, 0x9d, 0x7d, 0xf0, 0xde, 0x13, 0x09, 0x3d, 0x0b, 0xd3, - 0x8f, 0x94, 0x3b, 0x94, 0x8b, 0x6d, 0xb0, 0xed, 0x14, 0x2a, 0xe7, 0x9b, 0x69, 0x9a, 0xfe, 0xd7, - 0x1c, 0x2c, 0xa8, 0xfe, 0xde, 0x97, 0x89, 0x7d, 0x1e, 0xca, 0x78, 0xd7, 0x0a, 0xbb, 0x7d, 0xd7, - 0xc4, 0xbc, 0x68, 0xa3, 0xf3, 0x44, 0xc0, 0xd3, 0xae, 0xa9, 0xcc, 0x93, 0xc0, 0xe4, 0xd5, 0x90, - 0xdf, 0xd7, 0x6a, 0x48, 0x5a, 0x75, 0x33, 0xb7, 0x6f, 0xd5, 0x65, 0x8f, 0x73, 0xf9, 0x3e, 0x8d, - 0xf3, 0xcd, 0x1c, 0xd4, 0xd2, 0xf1, 0xf0, 0xeb, 0xb1, 0x85, 0xd4, 0xdd, 0x90, 0xdf, 0xf7, 0x6e, - 0x78, 0x03, 0xe6, 0x49, 0xf6, 0x66, 0x84, 0x21, 0xbf, 0x03, 0x38, 0x43, 0xb3, 0x1e, 0x16, 0x9b, - 0x22, 0x67, 0x55, 0xe0, 0x4a, 0x6c, 0x92, 0x70, 0xfd, 0xbb, 0x39, 0x98, 0x57, 0xe2, 0xf6, 0xc3, - 0x17, 0x52, 0xf4, 0x2a, 0xcc, 0x2b, 0xe9, 0x90, 0xfe, 0x7d, 0xb6, 0x4e, 0xd4, 0x3c, 0xe4, 0xe1, - 0x1b, 0x97, 0x05, 0x98, 0x93, 0xf3, 0x2a, 0xbd, 0x0d, 0xd5, 0x54, 0x1a, 0x24, 0xbf, 0x80, 0xb6, - 0x9f, 0x17, 0xd0, 0x97, 0x60, 0x31, 0xeb, 0xeb, 0xad, 0x7f, 0xaa, 0x51, 0xc2, 0xe4, 0x25, 0xdf, - 0xb3, 0x00, 0x0e, 0xbe, 0xd1, 0xbd, 0x6d, 0xdd, 0xc4, 0x86, 0x01, 0xdf, 0x38, 0x9f, 0x2a, 0x33, - 0x4a, 0x02, 0x23, 0x9a, 0x5c, 0xdb, 0xec, 0xde, 0xb6, 0x5a, 0xa1, 0x9a, 0x5c, 0xdb, 0x9c, 0xd0, - 0x24, 0x30, 0xfd, 0x87, 0x79, 0x51, 0xd2, 0x26, 0xb7, 0x64, 0xdf, 0x83, 0x9a, 0x27, 0x1e, 0x6e, - 0xef, 0x2d, 0x4d, 0xea, 0x63, 0xfe, 0xb4, 0xa5, 0x05, 0x95, 0xa2, 0xea, 0xe6, 0xd5, 0x5a, 0x6e, - 0x9f, 0xba, 0x3b, 0xa9, 0xb2, 0x6d, 0x41, 0xa5, 0xa0, 0x6f, 0xc3, 0x41, 0x71, 0x89, 0x68, 0x07, - 0x0b, 0xc7, 0xf3, 0x53, 0x95, 0xb3, 0x4b, 0xbd, 0xb1, 0x40, 0xda, 0xf3, 0x6a, 0x8a, 0x94, 0x52, - 0xcf, 0x7d, 0x9f, 0xd9, 0xaf, 0xfa, 0xb4, 0xf3, 0xd5, 0x14, 0x89, 0xd4, 0xd7, 0xd5, 0xd4, 0xbd, - 0x63, 0x74, 0x06, 0x4a, 0xf4, 0x67, 0x49, 0xb7, 0x9e, 0x01, 0xba, 0x50, 0x29, 0x9f, 0x62, 0xa1, - 0xc8, 0x21, 0xf4, 0x22, 0x94, 0xe3, 0xeb, 0xc9, 0xfc, 0x30, 0x91, 0xed, 0x19, 0x01, 0x2a, 0x7b, - 0x46, 0x80, 0xfa, 0xcf, 0x35, 0x38, 0x32, 0xf5, 0x4e, 0xf2, 0x83, 0x2e, 0xb6, 0x9f, 0x7a, 0x16, - 0x4a, 0xe2, 0xb8, 0x0f, 0x01, 0x14, 0xde, 0xb9, 0xba, 0x76, 0x75, 0xed, 0x4c, 0xed, 0x00, 0xaa, - 0x40, 0xf1, 0xf2, 0xda, 0xc5, 0x33, 0xe7, 0x2e, 0xbe, 0x55, 0xd3, 0xc8, 0x43, 0xe7, 0xea, 0xc5, - 0x8b, 0xe4, 0x21, 0xf7, 0xd4, 0x05, 0xf9, 0xf2, 0x11, 0xfb, 0x8c, 0xa2, 0x39, 0x28, 0xad, 0x7a, - 0x1e, 0xdd, 0xb7, 0x4c, 0x76, 0x6d, 0xc7, 0x22, 0x7b, 0xb5, 0xa6, 0xa1, 0x22, 0xe4, 0x2f, 0x5d, - 0x5a, 0xaf, 0xe5, 0xd0, 0x22, 0xd4, 0xce, 0x60, 0xc3, 0xb4, 0x2d, 0x07, 0x8b, 0x60, 0x51, 0xcb, - 0xb7, 0xaf, 0x7f, 0xf6, 0xc5, 0x8a, 0xf6, 0xf9, 0x17, 0x2b, 0xda, 0x5f, 0xbe, 0x58, 0xd1, 0x6e, - 0x7e, 0xb9, 0x72, 0xe0, 0xf3, 0x2f, 0x57, 0x0e, 0xfc, 0xe9, 0xcb, 0x95, 0x03, 0xef, 0x3d, 0x2b, - 0xfd, 0x04, 0x8f, 0xbd, 0x93, 0xe7, 0xbb, 0x24, 0x4e, 0xf2, 0xa7, 0x56, 0xfa, 0x47, 0x87, 0x9f, - 0xe6, 0x8e, 0xad, 0xd2, 0xc7, 0xcb, 0x8c, 0xaf, 0x79, 0xce, 0x6d, 0x32, 0x80, 0xfe, 0x6e, 0x2c, - 0xe8, 0x15, 0xe8, 0xef, 0xc3, 0x9e, 0xff, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa1, 0x54, 0x1c, - 0xef, 0xaf, 0x38, 0x00, 0x00, + 0xb9, 0xf6, 0x90, 0x12, 0x1f, 0x3f, 0x25, 0x91, 0x3e, 0x96, 0x15, 0x5a, 0xb1, 0x45, 0x67, 0x9c, + 0x7b, 0xe3, 0x04, 0x09, 0x99, 0x38, 0x0f, 0xe4, 0x71, 0x91, 0x40, 0xb4, 0x15, 0x3f, 0x62, 0xd9, + 0x0e, 0x65, 0xe7, 0xfa, 0x06, 0xb9, 0x60, 0x86, 0x9c, 0x23, 0x6a, 0xac, 0xe1, 0xcc, 0x64, 0x1e, + 0xb2, 0x04, 0x64, 0x71, 0xef, 0xc5, 0xbd, 0xb9, 0xbb, 0xd4, 0x40, 0xbb, 0x28, 0xd0, 0x45, 0xba, + 0x6d, 0x80, 0xae, 0xbb, 0xee, 0xaa, 0x59, 0x14, 0x45, 0xda, 0x55, 0x57, 0x6c, 0x91, 0xa0, 0x8b, + 0x72, 0xd1, 0x75, 0xdb, 0x4d, 0x8b, 0xf3, 0x9a, 0x39, 0x67, 0x38, 0xb4, 0xe5, 0x57, 0x9d, 0xc2, + 0x2b, 0x69, 0xbe, 0xff, 0x39, 0xe7, 0xf1, 0xcf, 0xff, 0xff, 0xe7, 0x10, 0x8e, 0x79, 0xdb, 0x83, + 0x96, 0xe1, 0x0f, 0x0d, 0xd3, 0xc0, 0x3b, 0xd8, 0x09, 0x83, 0x16, 0xfb, 0xd3, 0xf4, 0x7c, 0x37, + 0x74, 0xd1, 0x9c, 0x4c, 0x5a, 0xd6, 0xb7, 0x5f, 0x0f, 0x9a, 0x96, 0xdb, 0x32, 0x3c, 0xab, 0xd5, + 0x77, 0x7d, 0xdc, 0xda, 0x79, 0xa9, 0x35, 0xc0, 0x0e, 0xf6, 0x8d, 0x10, 0x9b, 0x4c, 0x62, 0xf9, + 0xa4, 0xc4, 0xe3, 0xe0, 0xf0, 0xa6, 0xeb, 0x6f, 0x5b, 0xce, 0x20, 0x8b, 0xb3, 0x31, 0x70, 0xdd, + 0x81, 0x8d, 0x5b, 0xf4, 0xa9, 0x17, 0x6d, 0xb6, 0x42, 0x6b, 0x88, 0x83, 0xd0, 0x18, 0x7a, 0x9c, + 0xe1, 0x95, 0x44, 0xd5, 0xd0, 0xe8, 0x6f, 0x59, 0x0e, 0xf6, 0xf7, 0x5a, 0xd4, 0x5f, 0xcf, 0x6a, + 0xf9, 0x38, 0x70, 0x23, 0xbf, 0x8f, 0x27, 0xd4, 0xbe, 0x30, 0xb0, 0xc2, 0xad, 0xa8, 0xd7, 0xec, + 0xbb, 0xc3, 0xd6, 0xc0, 0x1d, 0xb8, 0x89, 0x7e, 0xf2, 0x44, 0x1f, 0xe8, 0x7f, 0x9c, 0xfd, 0x4d, + 0xcb, 0x09, 0xb1, 0xef, 0x18, 0x76, 0x2b, 0xe8, 0x6f, 0x61, 0x33, 0xb2, 0xb1, 0x9f, 0xfc, 0xe7, + 0xf6, 0x6e, 0xe0, 0x7e, 0x18, 0x4c, 0x00, 0x4c, 0x56, 0xbf, 0xb5, 0x08, 0xf3, 0x6b, 0x64, 0x68, + 0x36, 0xf0, 0x27, 0x11, 0x76, 0xfa, 0x18, 0x3d, 0x0b, 0xb3, 0x9f, 0x44, 0x38, 0xc2, 0x75, 0xed, + 0xb8, 0x76, 0xb2, 0xdc, 0x3e, 0x34, 0x1e, 0x35, 0xaa, 0x14, 0x78, 0xde, 0x1d, 0x5a, 0x21, 0x1e, + 0x7a, 0xe1, 0x5e, 0x87, 0x71, 0xa0, 0x37, 0x61, 0xee, 0x86, 0xdb, 0xeb, 0x06, 0x38, 0xec, 0x3a, + 0xc6, 0x10, 0xd7, 0x73, 0x54, 0xa2, 0x3e, 0x1e, 0x35, 0x16, 0x6f, 0xb8, 0xbd, 0x0d, 0x1c, 0x5e, + 0x32, 0x86, 0xb2, 0x18, 0x24, 0x28, 0x7a, 0x01, 0x8a, 0x51, 0x80, 0xfd, 0xae, 0x65, 0xd6, 0xf3, + 0x54, 0x6c, 0x71, 0x3c, 0x6a, 0xd4, 0x08, 0x74, 0xde, 0x94, 0x44, 0x0a, 0x0c, 0x41, 0xcf, 0x43, + 0x61, 0xe0, 0xbb, 0x91, 0x17, 0xd4, 0x67, 0x8e, 0xe7, 0x05, 0x37, 0x43, 0x64, 0x6e, 0x86, 0xa0, + 0xcb, 0x50, 0x60, 0xf3, 0x5d, 0x9f, 0x3d, 0x9e, 0x3f, 0x59, 0x39, 0xf5, 0x54, 0x53, 0x5e, 0x04, + 0x4d, 0xe5, 0x85, 0xd9, 0x13, 0x53, 0xc8, 0xe8, 0xb2, 0x42, 0xbe, 0x6c, 0xfe, 0x78, 0x10, 0x66, + 0x29, 0x1f, 0xba, 0x0c, 0xc5, 0xbe, 0x8f, 0xc9, 0x64, 0xd5, 0xd1, 0x71, 0xed, 0x64, 0xe5, 0xd4, + 0x72, 0x93, 0x2d, 0x82, 0xa6, 0x98, 0xa4, 0xe6, 0x55, 0xb1, 0x08, 0xda, 0x47, 0xc6, 0xa3, 0xc6, + 0x41, 0xce, 0x9e, 0x68, 0xbd, 0xf5, 0xbb, 0x86, 0xd6, 0x11, 0x5a, 0xd0, 0x15, 0x28, 0x07, 0x51, + 0x6f, 0x68, 0x85, 0x17, 0xdc, 0x1e, 0x1d, 0xf3, 0xca, 0xa9, 0x27, 0x54, 0x77, 0x37, 0x04, 0xb9, + 0xfd, 0xc4, 0x78, 0xd4, 0x38, 0x14, 0x73, 0x27, 0x1a, 0xcf, 0x1d, 0xe8, 0x24, 0x4a, 0xd0, 0x16, + 0x54, 0x7d, 0xec, 0xf9, 0x96, 0xeb, 0x5b, 0xa1, 0x15, 0x60, 0xa2, 0x37, 0x47, 0xf5, 0x1e, 0x53, + 0xf5, 0x76, 0x54, 0xa6, 0xf6, 0xb1, 0xf1, 0xa8, 0x71, 0x24, 0x25, 0xa9, 0xd8, 0x48, 0xab, 0x45, + 0x21, 0xa0, 0x14, 0xb4, 0x81, 0x43, 0x3a, 0x9f, 0x95, 0x53, 0xc7, 0x6f, 0x6b, 0x6c, 0x03, 0x87, + 0xed, 0xe3, 0xe3, 0x51, 0xe3, 0xe8, 0xa4, 0xbc, 0x62, 0x32, 0x43, 0x3f, 0xb2, 0xa1, 0x26, 0xa3, + 0x26, 0x79, 0xc1, 0x19, 0x6a, 0x73, 0x65, 0xba, 0x4d, 0xc2, 0xd5, 0x5e, 0x19, 0x8f, 0x1a, 0xcb, + 0x69, 0x59, 0xc5, 0xde, 0x84, 0x66, 0x32, 0x3f, 0x7d, 0xc3, 0xe9, 0x63, 0x9b, 0x98, 0x99, 0xcd, + 0x9a, 0x9f, 0xd3, 0x82, 0xcc, 0xe6, 0x27, 0xe6, 0x56, 0xe7, 0x27, 0x86, 0xd1, 0x47, 0x30, 0x17, + 0x3f, 0x90, 0xf1, 0x2a, 0xf0, 0x75, 0x94, 0xad, 0x94, 0x8c, 0xd4, 0xf2, 0x78, 0xd4, 0x58, 0x92, + 0x65, 0x14, 0xd5, 0x8a, 0xb6, 0x44, 0xbb, 0xcd, 0x46, 0xa6, 0x38, 0x5d, 0x3b, 0xe3, 0x90, 0xb5, + 0xdb, 0x93, 0x23, 0xa2, 0x68, 0x23, 0xda, 0xc9, 0x26, 0x8e, 0xfa, 0x7d, 0x8c, 0x4d, 0x6c, 0xd6, + 0x4b, 0x59, 0xda, 0x2f, 0x48, 0x1c, 0x4c, 0xbb, 0x2c, 0xa3, 0x6a, 0x97, 0x29, 0x64, 0xac, 0x6f, + 0xb8, 0xbd, 0x35, 0xdf, 0x77, 0xfd, 0xa0, 0x5e, 0xce, 0x1a, 0xeb, 0x0b, 0x82, 0xcc, 0xc6, 0x3a, + 0xe6, 0x56, 0xc7, 0x3a, 0x86, 0xb9, 0xbf, 0x9d, 0xc8, 0xb9, 0x88, 0x8d, 0x00, 0x9b, 0x75, 0x98, + 0xe2, 0x6f, 0xcc, 0x11, 0xfb, 0x1b, 0x23, 0x13, 0xfe, 0xc6, 0x14, 0x64, 0xc2, 0x02, 0x7b, 0x5e, + 0x0d, 0x02, 0x6b, 0xe0, 0x60, 0xb3, 0x5e, 0xa1, 0xfa, 0x8f, 0x66, 0xe9, 0x17, 0x3c, 0xed, 0xa3, + 0xe3, 0x51, 0xa3, 0xae, 0xca, 0x29, 0x36, 0x52, 0x3a, 0xd1, 0xc7, 0x30, 0xcf, 0x90, 0x4e, 0xe4, + 0x38, 0x96, 0x33, 0xa8, 0xcf, 0x51, 0x23, 0x4f, 0x66, 0x19, 0xe1, 0x2c, 0xed, 0x27, 0xc7, 0xa3, + 0xc6, 0x13, 0x8a, 0x94, 0x62, 0x42, 0x55, 0x48, 0x22, 0x06, 0x03, 0x92, 0x89, 0x9d, 0xcf, 0x8a, + 0x18, 0x17, 0x54, 0x26, 0x16, 0x31, 0x52, 0x92, 0x6a, 0xc4, 0x48, 0x11, 0x93, 0xf9, 0xe0, 0x93, + 0xbc, 0x30, 0x7d, 0x3e, 0xf8, 0x3c, 0x4b, 0xf3, 0x91, 0x31, 0xd5, 0x8a, 0x36, 0xf4, 0x29, 0x90, + 0x0f, 0xcf, 0x99, 0xc8, 0xb3, 0xad, 0xbe, 0x11, 0xe2, 0x33, 0x38, 0xc4, 0x7d, 0x12, 0xa9, 0xab, + 0xd4, 0x8a, 0x3e, 0x61, 0x65, 0x82, 0xb3, 0xad, 0x8f, 0x47, 0x8d, 0x95, 0x2c, 0x1d, 0x8a, 0xd5, + 0x4c, 0x2b, 0xe8, 0xbf, 0x34, 0x38, 0x1c, 0x84, 0x86, 0x63, 0x1a, 0xb6, 0xeb, 0xe0, 0xf3, 0xce, + 0xc0, 0xc7, 0x41, 0x70, 0xde, 0xd9, 0x74, 0xeb, 0x35, 0x6a, 0xff, 0x44, 0x2a, 0xac, 0x67, 0xb1, + 0xb6, 0x4f, 0x8c, 0x47, 0x8d, 0x46, 0xa6, 0x16, 0xc5, 0x83, 0x6c, 0x43, 0x68, 0x17, 0x0e, 0x89, + 0xac, 0xe2, 0x5a, 0x68, 0xd9, 0x56, 0x60, 0x84, 0x96, 0xeb, 0xd4, 0x0f, 0x52, 0xfb, 0x4f, 0xa5, + 0xa3, 0xe3, 0x04, 0x63, 0xfb, 0xa9, 0xf1, 0xa8, 0x71, 0x2c, 0x43, 0x83, 0x62, 0x3b, 0xcb, 0x44, + 0xb2, 0x84, 0xae, 0xf8, 0x98, 0x30, 0x62, 0xb3, 0x7e, 0x68, 0xfa, 0x12, 0x8a, 0x99, 0xe4, 0x25, + 0x14, 0x83, 0x59, 0x4b, 0x28, 0x26, 0x12, 0x4b, 0x9e, 0xe1, 0x87, 0x16, 0x31, 0xbb, 0x6e, 0xf8, + 0xdb, 0xd8, 0xaf, 0x2f, 0x66, 0x59, 0xba, 0xa2, 0x32, 0x31, 0x4b, 0x29, 0x49, 0xd5, 0x52, 0x8a, + 0x88, 0x6e, 0x69, 0xa0, 0xba, 0x66, 0xb9, 0x4e, 0x87, 0xa4, 0x0d, 0x01, 0x79, 0xbd, 0xc3, 0xd4, + 0xe8, 0x33, 0xb7, 0x79, 0x3d, 0x99, 0xbd, 0xfd, 0xcc, 0x78, 0xd4, 0x38, 0x31, 0x55, 0x9b, 0xe2, + 0xc8, 0x74, 0xa3, 0xe8, 0x3a, 0x54, 0x08, 0x11, 0xd3, 0x04, 0xcc, 0xac, 0x2f, 0x51, 0x1f, 0x8e, + 0x4c, 0xfa, 0xc0, 0x19, 0x68, 0x06, 0x72, 0x58, 0x92, 0x50, 0xec, 0xc8, 0xaa, 0xda, 0x45, 0x98, + 0xa5, 0xf2, 0xfa, 0xb8, 0x00, 0x87, 0x32, 0xd6, 0x06, 0x7a, 0x1b, 0x0a, 0x7e, 0xe4, 0x90, 0x84, + 0x8d, 0x65, 0x29, 0x48, 0xb5, 0x7a, 0x2d, 0xb2, 0x4c, 0x96, 0x2d, 0xfa, 0x91, 0xa3, 0xe4, 0x70, + 0xb3, 0x14, 0x20, 0xf2, 0x24, 0x5b, 0xb4, 0x4c, 0x9e, 0x8d, 0x4c, 0x95, 0xbf, 0xe1, 0xf6, 0x54, + 0x79, 0x0a, 0x20, 0x0c, 0xf3, 0x62, 0xe1, 0x75, 0x2d, 0xb2, 0xab, 0x58, 0x9e, 0xf1, 0xb4, 0xaa, + 0xe6, 0xbd, 0xa8, 0x87, 0x7d, 0x07, 0x87, 0x38, 0x10, 0xef, 0x40, 0xb7, 0x15, 0x8d, 0x22, 0xbe, + 0x84, 0x48, 0xfa, 0xe7, 0x64, 0x1c, 0xfd, 0x40, 0x83, 0xfa, 0xd0, 0xd8, 0xed, 0x0a, 0x30, 0xe8, + 0x6e, 0xba, 0x7e, 0xd7, 0xc3, 0xbe, 0xe5, 0x9a, 0x34, 0xf9, 0xac, 0x9c, 0xfa, 0xb7, 0x3b, 0x6e, + 0xa4, 0xe6, 0xba, 0xb1, 0x2b, 0xe0, 0xe0, 0x5d, 0xd7, 0xbf, 0x42, 0xc5, 0xd7, 0x9c, 0xd0, 0xdf, + 0x6b, 0x1f, 0xfb, 0x6a, 0xd4, 0x38, 0x40, 0xa6, 0x65, 0x98, 0xc5, 0xd3, 0xc9, 0x86, 0xd1, 0xf7, + 0x34, 0x58, 0x0a, 0xdd, 0xd0, 0xb0, 0xbb, 0xfd, 0x68, 0x18, 0xd9, 0x46, 0x68, 0xed, 0xe0, 0x6e, + 0x14, 0x18, 0x03, 0xcc, 0x73, 0xdc, 0xb7, 0xee, 0xec, 0xd4, 0x55, 0x22, 0x7f, 0x3a, 0x16, 0xbf, + 0x46, 0xa4, 0x99, 0x4f, 0x47, 0xb9, 0x4f, 0x8b, 0x61, 0x06, 0x4b, 0x27, 0x13, 0x5d, 0xfe, 0xb1, + 0x06, 0xcb, 0xd3, 0x5f, 0x13, 0x9d, 0x80, 0xfc, 0x36, 0xde, 0xe3, 0x55, 0xc4, 0xc1, 0xf1, 0xa8, + 0x31, 0xbf, 0x8d, 0xf7, 0xa4, 0x51, 0x27, 0x54, 0xf4, 0x1f, 0x30, 0xbb, 0x63, 0xd8, 0x11, 0xe6, + 0x4b, 0xa2, 0xd9, 0x64, 0xf5, 0x52, 0x53, 0xae, 0x97, 0x9a, 0xde, 0xf6, 0x80, 0x00, 0x4d, 0x31, + 0x23, 0xcd, 0xf7, 0x23, 0xc3, 0x09, 0xad, 0x70, 0x8f, 0x2d, 0x17, 0xaa, 0x40, 0x5e, 0x2e, 0x14, + 0x78, 0x33, 0xf7, 0xba, 0xb6, 0xfc, 0x85, 0x06, 0x47, 0xa6, 0xbe, 0xf4, 0x77, 0xc1, 0x43, 0xbd, + 0x0b, 0x33, 0x64, 0xe1, 0x93, 0xfa, 0x66, 0xcb, 0x1a, 0x6c, 0xbd, 0xf6, 0x0a, 0x75, 0xa7, 0xc0, + 0xca, 0x11, 0x86, 0xc8, 0xe5, 0x08, 0x43, 0x48, 0x8d, 0x66, 0xbb, 0x37, 0x5f, 0x7b, 0x85, 0x3a, + 0x55, 0x60, 0x46, 0x28, 0x20, 0x1b, 0xa1, 0x80, 0xfe, 0xb7, 0x02, 0x94, 0xe3, 0x02, 0x42, 0xda, + 0x83, 0xda, 0x3d, 0xed, 0xc1, 0x73, 0x50, 0x33, 0xb1, 0xc9, 0xbf, 0x7c, 0x96, 0xeb, 0x88, 0xdd, + 0x5c, 0x66, 0xd1, 0x55, 0xa1, 0x29, 0xf2, 0xd5, 0x14, 0x09, 0x9d, 0x82, 0x12, 0x4f, 0xb4, 0xf7, + 0xe8, 0x46, 0x9e, 0x6f, 0x2f, 0x8d, 0x47, 0x0d, 0x24, 0x30, 0x49, 0x34, 0xe6, 0x43, 0x1d, 0x00, + 0x56, 0xbd, 0xae, 0xe3, 0xd0, 0xe0, 0x29, 0x7f, 0x5d, 0x7d, 0x83, 0xcb, 0x31, 0x9d, 0xd5, 0xa1, + 0x09, 0xbf, 0x5c, 0x87, 0x26, 0x28, 0xfa, 0x08, 0x60, 0x68, 0x58, 0x0e, 0x93, 0xe3, 0xf9, 0xbd, + 0x3e, 0x2d, 0xa4, 0xac, 0xc7, 0x9c, 0x4c, 0x7b, 0x22, 0x29, 0x6b, 0x4f, 0x50, 0x52, 0x2d, 0xf2, + 0x7a, 0xbb, 0x5e, 0xa0, 0xbb, 0x74, 0x65, 0x9a, 0x6a, 0xae, 0xf6, 0x30, 0xa9, 0x18, 0xb9, 0x88, + 0xa4, 0x53, 0x68, 0x21, 0xc3, 0x66, 0x5b, 0x9b, 0x38, 0xb4, 0x86, 0x98, 0x66, 0xf6, 0x7c, 0xd8, + 0x04, 0x26, 0x0f, 0x9b, 0xc0, 0xd0, 0xeb, 0x00, 0x46, 0xb8, 0xee, 0x06, 0xe1, 0x65, 0xa7, 0x8f, + 0x69, 0xc6, 0x5e, 0x62, 0xee, 0x27, 0xa8, 0xec, 0x7e, 0x82, 0xa2, 0xb7, 0xa0, 0xe2, 0xf1, 0x8f, + 0x50, 0xcf, 0xc6, 0x34, 0x23, 0x2f, 0xb1, 0x4f, 0x8a, 0x04, 0x4b, 0xb2, 0x32, 0x37, 0x3a, 0x0b, + 0xd5, 0xbe, 0xeb, 0xf4, 0x23, 0xdf, 0xc7, 0x4e, 0x7f, 0x6f, 0xc3, 0xd8, 0xc4, 0x34, 0xfb, 0x2e, + 0xb1, 0xa5, 0x92, 0x22, 0xc9, 0x4b, 0x25, 0x45, 0x42, 0xaf, 0x42, 0x39, 0xee, 0x5e, 0xd0, 0x04, + 0xbb, 0xcc, 0x0b, 0x61, 0x01, 0x4a, 0xc2, 0x09, 0x27, 0x71, 0xde, 0x0a, 0xe2, 0x2c, 0x8d, 0x26, + 0xcd, 0xdc, 0x79, 0x09, 0x96, 0x9d, 0x97, 0x60, 0x74, 0x1e, 0x0e, 0xd2, 0xef, 0x62, 0x37, 0x0c, + 0xed, 0x6e, 0x80, 0xfb, 0xae, 0x63, 0x06, 0x34, 0x27, 0xce, 0x33, 0xf7, 0x29, 0xf1, 0x6a, 0x68, + 0x6f, 0x30, 0x92, 0xec, 0x7e, 0x8a, 0xa4, 0xff, 0x52, 0x83, 0xc5, 0xac, 0x25, 0x94, 0x5a, 0xce, + 0xda, 0x03, 0x59, 0xce, 0x1f, 0x40, 0xc9, 0x73, 0xcd, 0x6e, 0xe0, 0xe1, 0x3e, 0x8f, 0x58, 0xa9, + 0xc5, 0x7c, 0xc5, 0x35, 0x37, 0x3c, 0xdc, 0xff, 0x77, 0x2b, 0xdc, 0x5a, 0xdd, 0x71, 0x2d, 0xf3, + 0xa2, 0x15, 0xf0, 0x55, 0xe7, 0x31, 0x8a, 0x92, 0x21, 0x14, 0x39, 0xd8, 0x2e, 0x41, 0x81, 0x59, + 0xd1, 0x7f, 0x95, 0x87, 0x5a, 0x7a, 0xd9, 0xfe, 0x33, 0xbd, 0x0a, 0xba, 0x0e, 0x45, 0x8b, 0xa5, + 0xcc, 0x3c, 0x83, 0xf8, 0x17, 0x29, 0xa6, 0x37, 0x93, 0x86, 0x5f, 0x73, 0xe7, 0xa5, 0x26, 0xcf, + 0xad, 0xe9, 0x10, 0x50, 0xcd, 0x5c, 0x52, 0xd5, 0xcc, 0x41, 0xd4, 0x81, 0x62, 0x80, 0xfd, 0x1d, + 0xab, 0x8f, 0x79, 0x70, 0x6a, 0xc8, 0x9a, 0xfb, 0xae, 0x8f, 0x89, 0xce, 0x0d, 0xc6, 0x92, 0xe8, + 0xe4, 0x32, 0xaa, 0x4e, 0x0e, 0xa2, 0x0f, 0xa0, 0xdc, 0x77, 0x9d, 0x4d, 0x6b, 0xb0, 0x6e, 0x78, + 0x3c, 0x3c, 0x1d, 0xcb, 0xd2, 0x7a, 0x5a, 0x30, 0xf1, 0x26, 0x84, 0x78, 0x4c, 0x35, 0x21, 0x62, + 0xae, 0x64, 0x42, 0xff, 0x34, 0x03, 0x90, 0x4c, 0x0e, 0x7a, 0x03, 0x2a, 0x78, 0x17, 0xf7, 0xa3, + 0xd0, 0xf5, 0xc5, 0x77, 0x82, 0xf7, 0xf4, 0x04, 0xac, 0x04, 0x76, 0x48, 0x50, 0xb2, 0x51, 0x1d, + 0x63, 0x88, 0x03, 0xcf, 0xe8, 0x8b, 0x66, 0x20, 0x75, 0x26, 0x06, 0xe5, 0x8d, 0x1a, 0x83, 0xe8, + 0x5f, 0x61, 0x86, 0xb6, 0x0f, 0x59, 0x1f, 0x10, 0x8d, 0x47, 0x8d, 0x05, 0x47, 0x6d, 0x1c, 0x52, + 0x3a, 0x7a, 0x07, 0xe6, 0xb7, 0xe3, 0x85, 0x47, 0x7c, 0x9b, 0xa1, 0x02, 0x34, 0xb5, 0x4b, 0x08, + 0x8a, 0x77, 0x73, 0x32, 0x8e, 0x36, 0xa1, 0x62, 0x38, 0x8e, 0x1b, 0xd2, 0x6f, 0x90, 0xe8, 0x0d, + 0x3e, 0x3b, 0x6d, 0x99, 0x36, 0x57, 0x13, 0x5e, 0x96, 0x25, 0xd1, 0xe0, 0x21, 0x69, 0x90, 0x83, + 0x87, 0x04, 0xa3, 0x0e, 0x14, 0x6c, 0xa3, 0x87, 0x6d, 0x11, 0xf4, 0x9f, 0x9e, 0x6a, 0xe2, 0x22, + 0x65, 0x63, 0xda, 0xe9, 0x27, 0x9f, 0xc9, 0xc9, 0x9f, 0x7c, 0x86, 0x2c, 0x6f, 0x42, 0x2d, 0xed, + 0xcf, 0xfe, 0x12, 0x98, 0x67, 0xe5, 0x04, 0xa6, 0x7c, 0xc7, 0x94, 0xc9, 0x80, 0x8a, 0xe4, 0xd4, + 0xc3, 0x30, 0xa1, 0xff, 0x44, 0x83, 0xc5, 0xac, 0xbd, 0x8b, 0xd6, 0xa5, 0x1d, 0xaf, 0xf1, 0x1e, + 0x47, 0xc6, 0x52, 0xe7, 0xb2, 0x53, 0xb6, 0x7a, 0xb2, 0xd1, 0xdb, 0xb0, 0xe0, 0xb8, 0x26, 0xee, + 0x1a, 0xc4, 0x80, 0x6d, 0x05, 0x61, 0x3d, 0x47, 0x7b, 0xc7, 0xb4, 0x37, 0x42, 0x28, 0xab, 0x82, + 0x20, 0x49, 0xcf, 0x2b, 0x04, 0xfd, 0xff, 0x34, 0xa8, 0xa6, 0x5a, 0x97, 0xf7, 0x9d, 0x44, 0xc9, + 0xa9, 0x4f, 0x6e, 0x7f, 0xa9, 0x8f, 0xfe, 0xfd, 0x1c, 0x54, 0xa4, 0xba, 0xee, 0xbe, 0x7d, 0xb8, + 0x01, 0x55, 0xfe, 0xa5, 0xb4, 0x9c, 0x01, 0x2b, 0xa7, 0x72, 0xbc, 0x49, 0x31, 0x71, 0x52, 0x70, + 0xc1, 0xed, 0x6d, 0xc4, 0xbc, 0xb4, 0x9a, 0xa2, 0x1d, 0xac, 0x40, 0xc1, 0x24, 0x13, 0x0b, 0x2a, + 0x05, 0x5d, 0x87, 0xa5, 0xc8, 0x33, 0x8d, 0x10, 0x77, 0x03, 0xde, 0x73, 0xef, 0x3a, 0xd1, 0xb0, + 0x87, 0x7d, 0xba, 0xe3, 0x67, 0x59, 0xcf, 0x85, 0x71, 0x88, 0xa6, 0xfc, 0x25, 0x4a, 0x97, 0x74, + 0x2e, 0x66, 0xd1, 0xf5, 0x73, 0x80, 0x26, 0xfb, 0xca, 0xca, 0xf8, 0x6a, 0xfb, 0x1c, 0xdf, 0xcf, + 0x34, 0xa8, 0xa5, 0xdb, 0xc5, 0x8f, 0x64, 0xa2, 0xf7, 0xa0, 0x1c, 0xb7, 0x7e, 0xef, 0xdb, 0x81, + 0xe7, 0xa1, 0xe0, 0x63, 0x23, 0x70, 0x1d, 0xbe, 0x33, 0x69, 0x88, 0x61, 0x88, 0x1c, 0x62, 0x18, + 0xa2, 0x5f, 0x85, 0x39, 0x36, 0x82, 0xef, 0x5a, 0x76, 0x88, 0x7d, 0x74, 0x06, 0x0a, 0x41, 0x68, + 0x84, 0x38, 0xa8, 0x6b, 0xc7, 0xf3, 0x27, 0x17, 0x4e, 0x2d, 0x4d, 0x76, 0x79, 0x09, 0x99, 0x69, + 0x65, 0x9c, 0xb2, 0x56, 0x86, 0xe8, 0xff, 0xa3, 0xc1, 0x9c, 0xdc, 0xcc, 0x7e, 0x30, 0x6a, 0xef, + 0xf2, 0xd5, 0x3e, 0x15, 0x3e, 0xd8, 0x0f, 0x66, 0x66, 0xef, 0xce, 0xfa, 0xcf, 0x34, 0x36, 0xb2, + 0x71, 0x17, 0xf4, 0x7e, 0xcd, 0x0f, 0x92, 0x56, 0x08, 0xd9, 0x61, 0x01, 0x0d, 0x6c, 0xfb, 0x6d, + 0x85, 0xd0, 0xf0, 0xa7, 0x88, 0xcb, 0xe1, 0x4f, 0x21, 0xe8, 0xbf, 0xc9, 0x51, 0xcf, 0x93, 0x8e, + 0xf7, 0xa3, 0x6e, 0x02, 0xa5, 0xb2, 0x93, 0xfc, 0x5d, 0x64, 0x27, 0x2f, 0x40, 0x91, 0x7e, 0x0e, + 0xe2, 0xc4, 0x81, 0x4e, 0x1a, 0x81, 0xd4, 0x13, 0x47, 0x86, 0xdc, 0x26, 0x6a, 0xcd, 0xde, 0x67, + 0xd4, 0xfa, 0x8b, 0x06, 0x0b, 0xea, 0x91, 0xc0, 0x23, 0x1f, 0xd6, 0x89, 0x05, 0x95, 0x7f, 0x48, + 0x0b, 0xea, 0xcf, 0x1a, 0xcc, 0x2b, 0x27, 0x15, 0x8f, 0xcf, 0xab, 0xff, 0x30, 0x07, 0x4b, 0xd9, + 0x6a, 0x1e, 0x4a, 0xf9, 0x74, 0x0e, 0x48, 0x22, 0x74, 0x3e, 0xf9, 0xb2, 0x1f, 0x9e, 0xa8, 0x9e, + 0xe8, 0x2b, 0x88, 0x2c, 0x6a, 0xe2, 0x88, 0x41, 0x88, 0xa3, 0xeb, 0x50, 0xb1, 0xa4, 0xc3, 0x8c, + 0x7c, 0x56, 0xcf, 0x59, 0x3e, 0xc2, 0x60, 0x35, 0xf6, 0x94, 0x83, 0x0b, 0x59, 0x55, 0xbb, 0x00, + 0x33, 0x24, 0xf5, 0xd0, 0x77, 0xa0, 0xc8, 0xdd, 0x41, 0x2f, 0x43, 0x99, 0xee, 0x52, 0x5a, 0x11, + 0xb0, 0xb4, 0x93, 0x7e, 0x34, 0x09, 0x98, 0xba, 0x4e, 0x50, 0x12, 0x18, 0x7a, 0x0d, 0x80, 0x24, + 0x8e, 0x7c, 0x7f, 0xe6, 0xe8, 0xfe, 0xa4, 0x95, 0x87, 0xe7, 0x9a, 0x13, 0x9b, 0xb2, 0x1c, 0x83, + 0xfa, 0x4f, 0x73, 0x50, 0x91, 0x8f, 0x4f, 0xee, 0xc9, 0xf8, 0xa7, 0x20, 0xaa, 0xc2, 0xae, 0x61, + 0x9a, 0xe4, 0x2f, 0x16, 0x01, 0xb9, 0x35, 0x75, 0x90, 0xc4, 0xff, 0xab, 0x42, 0x82, 0xd5, 0x00, + 0xf4, 0x80, 0xda, 0x4a, 0x91, 0x24, 0xab, 0xb5, 0x34, 0x6d, 0x79, 0x1b, 0x0e, 0x67, 0xaa, 0x92, + 0x33, 0xf7, 0xd9, 0x07, 0x95, 0xb9, 0xff, 0x7c, 0x16, 0x0e, 0x67, 0x1e, 0x5b, 0x3d, 0xf2, 0x5d, + 0xac, 0xee, 0xa0, 0xfc, 0x03, 0xd9, 0x41, 0x9f, 0x69, 0x59, 0x33, 0xcb, 0x8e, 0x00, 0xde, 0xd8, + 0xc7, 0x59, 0xde, 0x83, 0x9a, 0x63, 0x75, 0x59, 0xce, 0xde, 0xd3, 0x9e, 0x28, 0xec, 0x77, 0x4f, + 0xa0, 0x17, 0x59, 0x11, 0x46, 0x6d, 0x15, 0xa9, 0x2d, 0x11, 0x21, 0x52, 0xa6, 0x8a, 0x1c, 0x22, + 0x75, 0xb9, 0x90, 0x60, 0xa5, 0x7f, 0x29, 0xa9, 0xcb, 0x39, 0x4f, 0xba, 0xfa, 0x9f, 0x93, 0xf1, + 0x7f, 0xec, 0x1a, 0xfe, 0xab, 0x06, 0xd5, 0xd4, 0x39, 0xf6, 0xe3, 0xf3, 0x0d, 0xfa, 0x5c, 0x83, + 0x72, 0x7c, 0x85, 0xe2, 0xbe, 0xd3, 0xd0, 0x55, 0x28, 0x60, 0x76, 0x8c, 0xcf, 0xc2, 0xdd, 0xa1, + 0xd4, 0x35, 0x2b, 0x42, 0xe3, 0x17, 0xab, 0x52, 0x27, 0xf7, 0x1d, 0x2e, 0xa8, 0xff, 0x5a, 0x13, + 0x09, 0x66, 0xe2, 0xd3, 0x23, 0x9d, 0x8a, 0xe4, 0x9d, 0xf2, 0xf7, 0xfa, 0x4e, 0xbf, 0x28, 0xc3, + 0x2c, 0xe5, 0x23, 0x05, 0x60, 0x88, 0xfd, 0xa1, 0xe5, 0x18, 0x36, 0x7d, 0x9d, 0x12, 0xdb, 0xb7, + 0x02, 0x93, 0xf7, 0xad, 0xc0, 0xd0, 0x16, 0x54, 0x93, 0xa6, 0x15, 0x55, 0x93, 0x7d, 0x7b, 0xeb, + 0x3d, 0x95, 0x89, 0xb5, 0xa5, 0x53, 0x92, 0xea, 0xf1, 0x76, 0x8a, 0x88, 0x4c, 0x58, 0xe8, 0xbb, + 0x4e, 0x68, 0x58, 0x0e, 0xf6, 0x99, 0xa1, 0x7c, 0xd6, 0xed, 0x95, 0xd3, 0x0a, 0x0f, 0xab, 0xfd, + 0x55, 0x39, 0xf5, 0xf6, 0x8a, 0x4a, 0x43, 0x1f, 0xc3, 0xbc, 0x48, 0xc2, 0x99, 0x91, 0x99, 0xac, + 0xdb, 0x2b, 0x6b, 0x32, 0x0b, 0x5b, 0xd2, 0x8a, 0x94, 0x7a, 0x7b, 0x45, 0x21, 0x21, 0x1b, 0x6a, + 0x9e, 0x6b, 0x5e, 0x73, 0x78, 0xdb, 0xc1, 0xe8, 0xd9, 0x98, 0x77, 0x4a, 0x57, 0x26, 0x52, 0x1e, + 0x85, 0x8b, 0x85, 0xe2, 0xb4, 0xac, 0x7a, 0x1f, 0x2c, 0x4d, 0x45, 0x1f, 0xc1, 0x9c, 0x4d, 0x6a, + 0xa1, 0xb5, 0x5d, 0xcf, 0xf2, 0xb1, 0x99, 0x7d, 0x7b, 0xeb, 0xa2, 0xc4, 0xc1, 0x02, 0xa1, 0x2c, + 0xa3, 0xde, 0x60, 0x91, 0x29, 0x64, 0xf6, 0x87, 0xc6, 0x6e, 0x27, 0x72, 0x82, 0xb5, 0x5d, 0x7e, + 0x13, 0xa7, 0x98, 0x35, 0xfb, 0xeb, 0x2a, 0x13, 0x9b, 0xfd, 0x94, 0xa4, 0x3a, 0xfb, 0x29, 0x22, + 0xba, 0x48, 0xe3, 0x3c, 0x9b, 0x12, 0x76, 0x8b, 0x6b, 0x69, 0x62, 0xb4, 0xd8, 0x6c, 0xb0, 0xa6, + 0x05, 0x7f, 0x52, 0x94, 0xc6, 0x1a, 0xf8, 0x1c, 0xd0, 0xd7, 0xee, 0xe0, 0x30, 0xf2, 0x1d, 0x6c, + 0xf2, 0x0b, 0x5c, 0x93, 0x73, 0xa0, 0x70, 0xc5, 0x73, 0xa0, 0xa0, 0x13, 0x73, 0xa0, 0x50, 0xc9, + 0x9a, 0xf2, 0x5c, 0xf3, 0x2a, 0xdb, 0x32, 0x61, 0x7c, 0xad, 0xeb, 0xc9, 0x09, 0x53, 0x09, 0x0b, + 0x5b, 0x53, 0x8a, 0x94, 0xba, 0xa6, 0x14, 0x12, 0xbf, 0x49, 0x24, 0xdf, 0x3b, 0x61, 0x23, 0x55, + 0x99, 0x72, 0x93, 0x68, 0x82, 0x33, 0xbe, 0x49, 0x34, 0x41, 0x99, 0xb8, 0x49, 0x34, 0xc1, 0x41, + 0xac, 0x0f, 0x0c, 0x67, 0x70, 0xc1, 0xed, 0xa9, 0xab, 0x7a, 0x2e, 0xcb, 0xfa, 0xd9, 0x0c, 0x4e, + 0x66, 0x3d, 0x4b, 0x87, 0x6a, 0x3d, 0x8b, 0xa3, 0x5d, 0x12, 0xcd, 0x0d, 0xfd, 0x0b, 0x0d, 0xaa, + 0xa9, 0x38, 0x83, 0xde, 0x86, 0xf8, 0xbe, 0xc4, 0xd5, 0x3d, 0x4f, 0xa4, 0xc9, 0xca, 0xfd, 0x0a, + 0x82, 0x67, 0xdd, 0xaf, 0x20, 0x38, 0xba, 0x08, 0x10, 0x7f, 0x93, 0x6e, 0x17, 0xa4, 0x69, 0x8e, + 0x96, 0x70, 0xca, 0x39, 0x5a, 0x82, 0xea, 0x5f, 0xe7, 0xa1, 0x24, 0x16, 0xea, 0x43, 0x29, 0xa3, + 0x5a, 0x50, 0x1c, 0xe2, 0x80, 0xde, 0xb3, 0xc8, 0x25, 0xd9, 0x10, 0x87, 0xe4, 0x6c, 0x88, 0x43, + 0x6a, 0xb2, 0x96, 0xbf, 0xa7, 0x64, 0x6d, 0x66, 0xdf, 0xc9, 0x1a, 0xa6, 0x67, 0xac, 0x52, 0xb8, + 0x15, 0xa7, 0x1a, 0xb7, 0x8f, 0xe1, 0xe2, 0x04, 0x56, 0x16, 0x4c, 0x9d, 0xc0, 0xca, 0x24, 0xb4, + 0x0d, 0x07, 0xa5, 0x93, 0x17, 0xde, 0xf9, 0x22, 0x81, 0x6f, 0x61, 0xfa, 0x81, 0x76, 0x87, 0x72, + 0xb1, 0xed, 0xbd, 0x9d, 0x42, 0xe5, 0x6c, 0x37, 0x4d, 0xd3, 0xff, 0x90, 0x83, 0x05, 0xd5, 0xdf, + 0x87, 0x32, 0xb1, 0x2f, 0x43, 0x19, 0xef, 0x5a, 0x61, 0xb7, 0xef, 0x9a, 0x98, 0x97, 0x8c, 0x74, + 0x9e, 0x08, 0x78, 0xda, 0x35, 0x95, 0x79, 0x12, 0x98, 0xbc, 0x1a, 0xf2, 0xfb, 0x5a, 0x0d, 0x49, + 0xa3, 0x70, 0xe6, 0xce, 0x8d, 0xc2, 0xec, 0x71, 0x2e, 0x3f, 0xa4, 0x71, 0xbe, 0x95, 0x83, 0x5a, + 0x3a, 0x1a, 0x7f, 0x37, 0xb6, 0x90, 0xba, 0x1b, 0xf2, 0xfb, 0xde, 0x0d, 0xef, 0xc0, 0x3c, 0xc9, + 0x1d, 0x8d, 0x30, 0xe4, 0x37, 0x10, 0x67, 0x68, 0xce, 0xc5, 0x62, 0x53, 0xe4, 0xac, 0x0a, 0x5c, + 0x89, 0x4d, 0x12, 0xae, 0xff, 0x77, 0x0e, 0xe6, 0x95, 0xaf, 0xc6, 0xe3, 0x17, 0x52, 0xf4, 0x2a, + 0xcc, 0x2b, 0xc9, 0x98, 0xfe, 0xbf, 0x6c, 0x9d, 0xa8, 0x59, 0xd0, 0xe3, 0x37, 0x2e, 0x0b, 0x30, + 0x27, 0x67, 0x75, 0x7a, 0x1b, 0xaa, 0xa9, 0x24, 0x4c, 0x7e, 0x01, 0x6d, 0x3f, 0x2f, 0xa0, 0x2f, + 0xc1, 0x62, 0x56, 0xee, 0xa0, 0x9f, 0x85, 0xc5, 0xac, 0xaf, 0xfa, 0xdd, 0x1b, 0xf8, 0x52, 0xa3, + 0x16, 0x26, 0xef, 0x2a, 0x9f, 0x03, 0x70, 0xf0, 0xcd, 0xee, 0x1d, 0xcb, 0x3f, 0x36, 0x9e, 0xf8, + 0xe6, 0x85, 0x54, 0xb5, 0x54, 0x12, 0x18, 0xd1, 0xe4, 0xda, 0x66, 0xf7, 0x8e, 0x45, 0x17, 0xd5, + 0xe4, 0xda, 0xe6, 0x84, 0x26, 0x81, 0xe9, 0xff, 0x9f, 0x17, 0x95, 0x79, 0x72, 0xd9, 0xf7, 0x43, + 0xa8, 0x79, 0xe2, 0xe1, 0xce, 0xde, 0xd2, 0xda, 0x24, 0xe6, 0x4f, 0x5b, 0x5a, 0x50, 0x29, 0xaa, + 0x6e, 0x5e, 0x74, 0xe6, 0xf6, 0xa9, 0xbb, 0x93, 0xaa, 0x3e, 0x17, 0x54, 0x0a, 0xfa, 0x4f, 0x38, + 0x28, 0xee, 0x42, 0xed, 0x60, 0xe1, 0x78, 0x7e, 0xaa, 0x72, 0x76, 0x37, 0x39, 0x16, 0x48, 0x7b, + 0x5e, 0x4d, 0x91, 0x52, 0xea, 0xb9, 0xef, 0x33, 0xfb, 0x55, 0x9f, 0x76, 0xbe, 0x9a, 0x22, 0xe9, + 0x9f, 0x6b, 0x50, 0x4d, 0x5d, 0x9f, 0x46, 0x67, 0xa0, 0x44, 0x7f, 0x5d, 0x75, 0xfb, 0x19, 0xa0, + 0x0b, 0x92, 0xf2, 0x29, 0x16, 0x8a, 0x1c, 0x42, 0xaf, 0x42, 0x39, 0xbe, 0x65, 0xcd, 0xcf, 0x44, + 0xd9, 0xe6, 0x13, 0xa0, 0xb2, 0xf9, 0x04, 0xa8, 0xff, 0x48, 0x83, 0x23, 0x53, 0xaf, 0x56, 0x3f, + 0xea, 0x9e, 0xc1, 0x73, 0x2f, 0x42, 0x49, 0x9c, 0x5a, 0x22, 0x80, 0xc2, 0xfb, 0xd7, 0xd6, 0xae, + 0xad, 0x9d, 0xa9, 0x1d, 0x40, 0x15, 0x28, 0x5e, 0x59, 0xbb, 0x74, 0xe6, 0xfc, 0xa5, 0xb3, 0x35, + 0x8d, 0x3c, 0x74, 0xae, 0x5d, 0xba, 0x44, 0x1e, 0x72, 0xcf, 0x5d, 0x94, 0xef, 0x50, 0xb1, 0xef, + 0x31, 0x9a, 0x83, 0xd2, 0xaa, 0xe7, 0xd1, 0x00, 0xc0, 0x64, 0xd7, 0x76, 0x2c, 0xb2, 0x57, 0x6b, + 0x1a, 0x2a, 0x42, 0xfe, 0xf2, 0xe5, 0xf5, 0x5a, 0x0e, 0x2d, 0x42, 0xed, 0x0c, 0x36, 0x4c, 0xdb, + 0x72, 0xb0, 0x88, 0x3a, 0xb5, 0x7c, 0xfb, 0xc6, 0x57, 0xdf, 0xac, 0x68, 0x5f, 0x7f, 0xb3, 0xa2, + 0xfd, 0xfe, 0x9b, 0x15, 0xed, 0xd6, 0xb7, 0x2b, 0x07, 0xbe, 0xfe, 0x76, 0xe5, 0xc0, 0x6f, 0xbf, + 0x5d, 0x39, 0xf0, 0xe1, 0x8b, 0xd2, 0x2f, 0x09, 0xd9, 0x3b, 0x79, 0xbe, 0x4b, 0x02, 0x2e, 0x7f, + 0x6a, 0xa5, 0x7f, 0x3b, 0xf9, 0x65, 0xee, 0xd8, 0x2a, 0x7d, 0xbc, 0xc2, 0xf8, 0x9a, 0xe7, 0xdd, + 0x26, 0x03, 0xe8, 0xcf, 0xdf, 0x82, 0x5e, 0x81, 0xfe, 0xcc, 0xed, 0xe5, 0xbf, 0x07, 0x00, 0x00, + 0xff, 0xff, 0x2c, 0x75, 0x86, 0x00, 0x76, 0x39, 0x00, 0x00, } func (m *EventSequence) Marshal() (dAtA []byte, err error) { @@ -6089,6 +6149,27 @@ func (m *Error_JobRunPreemptedError) MarshalToSizedBuffer(dAtA []byte) (int, err } return len(dAtA) - i, nil } +func (m *Error_GangJobUnschedulable) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Error_GangJobUnschedulable) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.GangJobUnschedulable != nil { + { + size, err := m.GangJobUnschedulable.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintEvents(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x62 + } + return len(dAtA) - i, nil +} func (m *KubernetesError) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -6527,6 +6608,36 @@ func (m *JobRunPreemptedError) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *GangJobUnschedulable) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *GangJobUnschedulable) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *GangJobUnschedulable) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Message) > 0 { + i -= len(m.Message) + copy(dAtA[i:], m.Message) + i = encodeVarintEvents(dAtA, i, uint64(len(m.Message))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func (m *JobDuplicateDetected) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -7854,6 +7965,18 @@ func (m *Error_JobRunPreemptedError) Size() (n int) { } return n } +func (m *Error_GangJobUnschedulable) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.GangJobUnschedulable != nil { + l = m.GangJobUnschedulable.Size() + n += 1 + l + sovEvents(uint64(l)) + } + return n +} func (m *KubernetesError) Size() (n int) { if m == nil { return 0 @@ -8042,6 +8165,19 @@ func (m *JobRunPreemptedError) Size() (n int) { return n } +func (m *GangJobUnschedulable) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Message) + if l > 0 { + n += 1 + l + sovEvents(uint64(l)) + } + return n +} + func (m *JobDuplicateDetected) Size() (n int) { if m == nil { return 0 @@ -14164,6 +14300,41 @@ func (m *Error) Unmarshal(dAtA []byte) error { } m.Reason = &Error_JobRunPreemptedError{v} iNdEx = postIndex + case 12: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field GangJobUnschedulable", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthEvents + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthEvents + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &GangJobUnschedulable{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Reason = &Error_GangJobUnschedulable{v} + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipEvents(dAtA[iNdEx:]) @@ -15440,6 +15611,88 @@ func (m *JobRunPreemptedError) Unmarshal(dAtA []byte) error { } return nil } +func (m *GangJobUnschedulable) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: GangJobUnschedulable: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: GangJobUnschedulable: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Message", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthEvents + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthEvents + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Message = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipEvents(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthEvents + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *JobDuplicateDetected) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/pkg/armadaevents/events.proto b/pkg/armadaevents/events.proto index 8450951895f..dff09a6783f 100644 --- a/pkg/armadaevents/events.proto +++ b/pkg/armadaevents/events.proto @@ -406,6 +406,7 @@ message Error { PodLeaseReturned podLeaseReturned = 9; PodTerminated podTerminated = 10; JobRunPreemptedError jobRunPreemptedError = 11; + GangJobUnschedulable gangJobUnschedulable = 12; } } @@ -488,6 +489,10 @@ message MaxRunsExceeded { message JobRunPreemptedError{ } +message GangJobUnschedulable{ + string message = 1; +} + // Generated by the scheduler whenever it detects a SubmitJob message that includes a previously used deduplication id // (i.e., when it detects a duplicate job submission). message JobDuplicateDetected {