Skip to content

feat(resourcemanager): load resource groups in the background #9336

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+16 −6
Open

feat(resourcemanager): load resource groups in the background #9336

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 16 additions & 6 deletions pkg/mcs/resourcemanager/server/manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,10 @@ import (
)

const (
persistLoopInterval = 1 * time.Minute
persistLoopInterval = time.Minute
metricsCleanupInterval = time.Minute
metricsCleanupTimeout = 20 * time.Minute
metricsAvailableRUInterval = 1 * time.Second
metricsAvailableRUInterval = time.Second
defaultCollectIntervalSec = 20
tickPerSecond = time.Second
)
Expand Down Expand Up @@ -141,9 +141,17 @@ func (m *Manager) Init(ctx context.Context) error {
}

// Load keyspace resource groups from the storage.
if err := m.loadKeyspaceResourceGroups(); err != nil {
return err
}
// This operation is potentially time-consuming when there are many keyspaces and resource groups.
// So we run it in a new goroutine to prevent it from blocking the leader startup.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we move it to the background, could you please add reporting of resource group loading to the /ready API as well.

go func() {
start := time.Now()
log.Info("start to load keyspace resource groups in the background")
if err := m.loadKeyspaceResourceGroups(start); err != nil {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This behavior is different from the old one if the background task failed(maybe due to IO errors). Silently omit the error may cause undefined behavior in some cases.

log.Error("failed to load keyspace resource groups", zap.Duration("time-cost", time.Since(start)), zap.Error(err))
} else {
log.Info("load keyspace resource groups finished", zap.Duration("time-cost", time.Since(start)))
}
}()

// Start the background metrics flusher.
go m.backgroundMetricsFlush(ctx)
Expand All @@ -155,7 +163,7 @@ func (m *Manager) Init(ctx context.Context) error {
return nil
}

func (m *Manager) loadKeyspaceResourceGroups() error {
func (m *Manager) loadKeyspaceResourceGroups(start time.Time) error {
// Empty the keyspace resource group manager map before the loading.
m.Lock()
m.krgms = make(map[uint32]*keyspaceResourceGroupManager)
Expand All @@ -170,6 +178,7 @@ func (m *Manager) loadKeyspaceResourceGroups() error {
}); err != nil {
return err
}
log.Info("load keyspace resource groups meta info finished", zap.Duration("time-cost", time.Since(start)))
// Load keyspace resource group states from the storage.
if err := m.storage.LoadResourceGroupStates(func(keyspaceID uint32, name string, rawValue string) {
krgm := m.getKeyspaceResourceGroupManager(keyspaceID)
Expand All @@ -186,6 +195,7 @@ func (m *Manager) loadKeyspaceResourceGroups() error {
}); err != nil {
return err
}
log.Info("load keyspace resource groups states finished", zap.Duration("time-cost", time.Since(start)))
// Initialize the reserved keyspace resource group manager and default resource groups.
m.initReserved()
return nil
Expand Down
Loading