conflict

Signed-off-by: 童剑 <[email protected]>

Author: bufferflies

client/go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c
 	github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86
-	github.com/pingcap/kvproto v0.0.0-20250224053625-b6a98c6bf02d
+	github.com/pingcap/kvproto v0.0.0-20250512085210-1174a93e5067
 	github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3
 	github.com/prometheus/client_golang v1.20.5
 	github.com/stretchr/testify v1.9.0

client/go.sum

@@ -49,8 +49,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm
 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg=
 github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE=
 github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4=
-github.com/pingcap/kvproto v0.0.0-20250224053625-b6a98c6bf02d h1:52qhTQG8G8V/pHo/w7F4d2Tw98KMk2C+gAe3U8SWRAg=
-github.com/pingcap/kvproto v0.0.0-20250224053625-b6a98c6bf02d/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8=
+github.com/pingcap/kvproto v0.0.0-20250512085210-1174a93e5067 h1:UJpCOXMB68EGVUOp7xklJ46+sgNDzYqFK/k2aDG8JQQ=
+github.com/pingcap/kvproto v0.0.0-20250512085210-1174a93e5067/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8=
 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw=
 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

client/servicediscovery/service_discovery_test.go

@@ -159,7 +159,7 @@ func (suite *serviceClientTestSuite) SetupSuite() {
 		}
 		time.Sleep(50 * time.Millisecond)
 	}
-	suite.NotNil(suite.leaderClient)
+	suite.Require().NotNil(suite.leaderClient)
 }
 
 func (suite *serviceClientTestSuite) TearDownTest() {

go.mod

@@ -35,7 +35,7 @@ require (
 	github.com/pingcap/errcode v0.3.0
 	github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c
 	github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86
-	github.com/pingcap/kvproto v0.0.0-20250224053625-b6a98c6bf02d
+	github.com/pingcap/kvproto v0.0.0-20250512085210-1174a93e5067
 	github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3
 	github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21
 	github.com/pingcap/tidb-dashboard v0.0.0-20250219061340-d62018124ae2

go.sum

@@ -392,8 +392,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue
 github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE=
 github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4=
 github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w=
-github.com/pingcap/kvproto v0.0.0-20250224053625-b6a98c6bf02d h1:52qhTQG8G8V/pHo/w7F4d2Tw98KMk2C+gAe3U8SWRAg=
-github.com/pingcap/kvproto v0.0.0-20250224053625-b6a98c6bf02d/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8=
+github.com/pingcap/kvproto v0.0.0-20250512085210-1174a93e5067 h1:UJpCOXMB68EGVUOp7xklJ46+sgNDzYqFK/k2aDG8JQQ=
+github.com/pingcap/kvproto v0.0.0-20250512085210-1174a93e5067/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8=
 github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM=
 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw=
 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=

pkg/gc/gc_state_manager.go

@@ -23,6 +23,7 @@ import (
 
 	"go.uber.org/zap"
 
+	"github.com/pingcap/kvproto/pkg/keyspacepb"
 	"github.com/pingcap/log"
 
 	"github.com/tikv/pd/pkg/errs"
@@ -432,11 +433,14 @@ func (*GCStateManager) logAdvancingTxnSafePoint(keyspaceID uint32, result Advanc
 // GC barrier can extend its lifetime arbitrarily.
 //
 // Passing non-positive value to ttl is not allowed. Passing `time.Duration(math.MaxInt64)` to ttl indicates that the
-// GC barrier should never expire.
+// GC barrier should never expire. The ttl might be rounded up, and the actual ttl is guaranteed no less than the
+// specified duration.
 //
 // The barrierID must be non-empty. For NullKeyspace, "gc_worker" is a reserved name and cannot be used as a barrierID.
 //
 // The given barrierTS must be greater than or equal to the current txn safe point, or an error will be returned.
+//
+// When this function executes successfully, its result is never nil.
 func (m *GCStateManager) SetGCBarrier(keyspaceID uint32, barrierID string, barrierTS uint64, ttl time.Duration, now time.Time) (*endpoint.GCBarrier, error) {
 	if ttl <= 0 {
 		return nil, errs.ErrInvalidArgument.GenWithStackByArgs("ttl", ttl)
@@ -617,6 +621,7 @@ func (m *GCStateManager) GetGCState(keyspaceID uint32) (GCState, error) {
 
 // GetAllKeyspacesGCStates returns the GC state of all keyspaces.
 // Returns a map from keyspaceID to GCState. Keyspaces without keyspace-level GC enabled will not be included.
+// Note, it returns only the GC states of active keyspace. If a keyspace is in DISABLE/ARCHIVED/TOMBSTONE state, it's ignored here.
 func (m *GCStateManager) GetAllKeyspacesGCStates() (map[uint32]GCState, error) {
 	// TODO: Handle the case that there are too many keyspaces and loading them at once is not suitable.
 	allKeyspaces, err := m.keyspaceManager.LoadRangeKeyspace(0, 0)
@@ -639,6 +644,11 @@ func (m *GCStateManager) GetAllKeyspacesGCStates() (map[uint32]GCState, error) {
 	}
 
 	for _, keyspaceMeta := range allKeyspaces {
+		// Just handle the active keyspace, leave the others up to keyspace management.
+		if keyspaceMeta.State != keyspacepb.KeyspaceState_ENABLED {
+			continue
+		}
+
 		if keyspaceMeta.Config[keyspace.GCManagementType] != keyspace.KeyspaceLevelGC {
 			results[keyspaceMeta.Id] = GCState{
 				KeyspaceID:      keyspaceMeta.Id,

pkg/gc/gc_state_manager_test.go

@@ -27,6 +27,7 @@ import (
 	"github.com/stretchr/testify/suite"
 
 	"github.com/pingcap/errors"
+	"github.com/pingcap/kvproto/pkg/keyspacepb"
 
 	"github.com/tikv/pd/pkg/errs"
 	"github.com/tikv/pd/pkg/id"
@@ -125,6 +126,16 @@ func newGCStateManagerForTest(t *testing.T) (storage *endpoint.StorageEndpoint,
 	re.NoError(err)
 	re.Equal(uint32(3), ks3.Id)
 
+	ks4, err := keyspaceManager.CreateKeyspace(&keyspace.CreateKeyspaceRequest{
+		Name:       "ks4",
+		Config:     map[string]string{},
+		CreateTime: time.Now().Unix(),
+	})
+	re.NoError(err)
+	_, err = keyspaceManager.UpdateKeyspaceState("ks4", keyspacepb.KeyspaceState_DISABLED, time.Now().Unix())
+	re.NoError(err)
+	re.Equal(uint32(4), ks4.Id)
+
 	return s, s.GetGCStateProvider(), gcStateManager, clean
 }
 
@@ -135,7 +146,7 @@ func (s *gcStateManagerTestSuite) SetupTest() {
 	s.keyspacePresets.manageable = []uint32{constant.NullKeyspaceID, 2}
 	s.keyspacePresets.unmanageable = []uint32{0, 1, 3}
 	s.keyspacePresets.unifiedGC = []uint32{constant.NullKeyspaceID, 0, 1, 3}
-	s.keyspacePresets.notExisting = []uint32{4, 0xffffff}
+	s.keyspacePresets.notExisting = []uint32{5, 0xffffff}
 	s.keyspacePresets.nullSynonyms = []uint32{constant.NullKeyspaceID, 0x1000000, 0xfffffffe}
 }
 

pkg/keyspace/keyspace.go

@@ -107,6 +107,16 @@ type CreateKeyspaceRequest struct {
 	CreateTime int64
 }
 
+// CreateKeyspaceByIDRequest represents necessary arguments to create a keyspace.
+type CreateKeyspaceByIDRequest struct {
+	// ID of the keyspace to be created.
+	// Using an existing ID will result in error.
+	ID     *uint32
+	Config map[string]string
+	// CreateTime is the timestamp used to record creation time.
+	CreateTime int64
+}
+
 // NewKeyspaceManager creates a Manager of keyspace related data.
 func NewKeyspaceManager(
 	ctx context.Context,
@@ -280,6 +290,81 @@ func (manager *Manager) CreateKeyspace(request *CreateKeyspaceRequest) (*keyspac
 	return keyspace, nil
 }
 
+// CreateKeyspaceByID create a keyspace meta with given config and save it to storage.
+func (manager *Manager) CreateKeyspaceByID(request *CreateKeyspaceByIDRequest) (*keyspacepb.KeyspaceMeta, error) {
+	if request.ID == nil {
+		return nil, errors.New("keyspace id is empty")
+	}
+	id := *request.ID
+	name := strconv.FormatUint(uint64(id), 10)
+	userKind := endpoint.StringUserKind(request.Config[UserKindKey])
+	config, err := manager.kgm.GetKeyspaceConfigByKind(userKind)
+	if err != nil {
+		return nil, err
+	}
+	if len(config) != 0 {
+		if request.Config == nil {
+			request.Config = config
+		} else {
+			request.Config[TSOKeyspaceGroupIDKey] = config[TSOKeyspaceGroupIDKey]
+			request.Config[UserKindKey] = config[UserKindKey]
+		}
+	}
+	// Create a disabled keyspace meta for tikv-server to get the config on keyspace split.
+	keyspace := &keyspacepb.KeyspaceMeta{
+		Id:             id,
+		Name:           name,
+		State:          keyspacepb.KeyspaceState_DISABLED,
+		CreatedAt:      request.CreateTime,
+		StateChangedAt: request.CreateTime,
+		Config:         request.Config,
+	}
+	err = manager.saveNewKeyspace(keyspace)
+	if err != nil {
+		log.Warn("[keyspace] failed to save keyspace before split",
+			zap.Uint32("keyspace-id", keyspace.GetId()),
+			zap.String("keyspace-name", keyspace.GetName()),
+			zap.Error(err),
+		)
+		return nil, err
+	}
+	// Split keyspace region.
+	err = manager.splitKeyspaceRegion(id, manager.config.ToWaitRegionSplit())
+	if err != nil {
+		err2 := manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error {
+			metaPath := keypath.KeyspaceMetaPath(id)
+			return txn.Remove(metaPath)
+		})
+		if err2 != nil {
+			log.Warn("[keyspace] failed to remove pre-created keyspace after split failed",
+				zap.Uint32("keyspace-id", keyspace.GetId()),
+				zap.String("keyspace-name", keyspace.GetName()),
+				zap.Error(err2),
+			)
+		}
+		return nil, err
+	}
+	// enable the keyspace metadata after split.
+	keyspace.State = keyspacepb.KeyspaceState_ENABLED
+	_, err = manager.UpdateKeyspaceStateByID(id, keyspacepb.KeyspaceState_ENABLED, request.CreateTime)
+	if err != nil {
+		log.Warn("[keyspace] failed to create keyspace",
+			zap.Uint32("keyspace-id", keyspace.GetId()),
+			zap.String("keyspace-name", keyspace.GetName()),
+			zap.Error(err),
+		)
+		return nil, err
+	}
+	if err := manager.kgm.UpdateKeyspaceForGroup(userKind, config[TSOKeyspaceGroupIDKey], keyspace.GetId(), opAdd); err != nil {
+		return nil, err
+	}
+	log.Info("[keyspace] keyspace created",
+		zap.Uint32("keyspace-id", keyspace.GetId()),
+		zap.String("keyspace-name", keyspace.GetName()),
+	)
+	return keyspace, nil
+}
+
 func (manager *Manager) saveNewKeyspace(keyspace *keyspacepb.KeyspaceMeta) error {
 	manager.metaLock.Lock(keyspace.Id)
 	defer manager.metaLock.Unlock(keyspace.Id)

pkg/keyspace/keyspace_test.go

@@ -149,6 +149,52 @@ func (suite *keyspaceTestSuite) TestCreateKeyspace() {
 	re.Error(err)
 }
 
+func makeCreateKeyspaceByIDRequests(count int) []*CreateKeyspaceByIDRequest {
+	now := time.Now().Unix()
+	requests := make([]*CreateKeyspaceByIDRequest, count)
+	for i := range count {
+		id := uint32(i + 1)
+		requests[i] = &CreateKeyspaceByIDRequest{
+			ID: &id,
+			Config: map[string]string{
+				testConfig1: "100",
+				testConfig2: "200",
+			},
+			CreateTime: now,
+		}
+	}
+	return requests
+}
+
+func (suite *keyspaceTestSuite) TestCreateKeyspaceByID() {
+	re := suite.Require()
+	manager := suite.manager
+	requests := makeCreateKeyspaceByIDRequests(10)
+
+	for i, request := range requests {
+		created, err := manager.CreateKeyspaceByID(request)
+		re.NoError(err)
+		re.Equal(uint32(i+1), created.Id)
+		checkCreateByIDRequest(re, request, created)
+
+		loaded, err := manager.LoadKeyspaceByID(*request.ID)
+		re.NoError(err)
+		checkCreateByIDRequest(re, request, loaded)
+
+		loaded, err = manager.LoadKeyspaceByID(created.Id)
+		re.NoError(err)
+		checkCreateByIDRequest(re, request, loaded)
+	}
+
+	// Create a keyspace with existing ID must return error.
+	_, err := manager.CreateKeyspaceByID(requests[0])
+	re.Error(err)
+
+	// Create a keyspace with empty id must return error.
+	_, err = manager.CreateKeyspaceByID(&CreateKeyspaceByIDRequest{})
+	re.Error(err)
+}
+
 func makeMutations() []*Mutation {
 	return []*Mutation{
 		{
@@ -343,6 +389,15 @@ func checkCreateRequest(re *require.Assertions, request *CreateKeyspaceRequest,
 	re.Equal(request.Config, meta.GetConfig())
 }
 
+// checkCreateByIDRequest verifies a keyspace meta matches a create request.
+func checkCreateByIDRequest(re *require.Assertions, request *CreateKeyspaceByIDRequest, meta *keyspacepb.KeyspaceMeta) {
+	re.Equal(*request.ID, meta.GetId())
+	re.Equal(request.CreateTime, meta.GetCreatedAt())
+	re.Equal(request.CreateTime, meta.GetStateChangedAt())
+	re.Equal(keyspacepb.KeyspaceState_ENABLED, meta.GetState())
+	re.Equal(request.Config, meta.GetConfig())
+}
+
 // checkMutations verifies that performing mutations on old config would result in new config.
 func checkMutations(re *require.Assertions, oldConfig, newConfig map[string]string, mutations []*Mutation) {
 	// Copy oldConfig to expected to avoid modifying its content.

pkg/schedule/checker/rule_checker_test.go

@@ -2079,6 +2079,7 @@ func (suite *ruleCheckerTestAdvancedSuite) TestReplaceAnExistingPeerCases() {
 }
 
 func (suite *ruleCheckerTestSuite) TestRemoveOrphanPeer() {
+	re := suite.Require()
 	suite.cluster.AddLabelsStore(1, 1, map[string]string{"zone": "z1", "host": "h1"})
 	suite.cluster.AddLabelsStore(2, 1, map[string]string{"zone": "z1", "host": "h1"})
 	suite.cluster.AddLabelsStore(3, 1, map[string]string{"zone": "z1", "host": "h1"})
@@ -2104,14 +2105,14 @@ func (suite *ruleCheckerTestSuite) TestRemoveOrphanPeer() {
 	// case1: regionA has 3 peers but not extra peer can be removed, so it needs to add peer first
 	suite.cluster.AddLeaderRegionWithRange(1, "200", "300", 1, 2, 3)
 	op := suite.rc.Check(suite.cluster.GetRegion(1))
-	suite.NotNil(op)
-	suite.Equal("add-rule-peer", op.Desc())
+	re.NotNil(op)
+	re.Equal("add-rule-peer", op.Desc())
 
 	// case2: regionB has 4 peers and one extra peer can be removed, so it needs to remove extra peer first
 	suite.cluster.AddLeaderRegionWithRange(2, "300", "400", 1, 2, 3, 4)
 	op = suite.rc.Check(suite.cluster.GetRegion(2))
-	suite.NotNil(op)
-	suite.Equal("remove-orphan-peer", op.Desc())
+	re.NotNil(op)
+	re.Equal("remove-orphan-peer", op.Desc())
 }
 
 func (suite *ruleCheckerTestSuite) TestIssue7808() {

pkg/schedule/operator/operator_test.go

@@ -542,6 +542,7 @@ func (suite *operatorTestSuite) TestRecord() {
 }
 
 func (suite *operatorTestSuite) TestToJSONObject() {
+	re := suite.Require()
 	steps := []OpStep{
 		AddPeer{ToStore: 1, PeerID: 1},
 		TransferLeader{FromStore: 3, ToStore: 1},
@@ -550,28 +551,28 @@ func (suite *operatorTestSuite) TestToJSONObject() {
 	op := NewTestOperator(101, &metapb.RegionEpoch{}, OpLeader|OpRegion, steps...)
 	op.Start()
 	obj := op.ToJSONObject()
-	suite.Equal("test", obj.Desc)
-	suite.Equal("test", obj.Brief)
-	suite.Equal(uint64(101), obj.RegionID)
-	suite.Equal(OpLeader|OpRegion, obj.Kind)
-	suite.Equal("12m0s", obj.Timeout)
-	suite.Equal(STARTED, obj.Status)
+	re.Equal("test", obj.Desc)
+	re.Equal("test", obj.Brief)
+	re.Equal(uint64(101), obj.RegionID)
+	re.Equal(OpLeader|OpRegion, obj.Kind)
+	re.Equal("12m0s", obj.Timeout)
+	re.Equal(STARTED, obj.Status)
 
 	// Test SUCCESS status.
 	region := newTestRegion(1, 1, [2]uint64{1, 1}, [2]uint64{2, 2})
-	suite.Nil(op.Check(region))
-	suite.Equal(SUCCESS, op.Status())
+	re.Nil(op.Check(region))
+	re.Equal(SUCCESS, op.Status())
 	obj = op.ToJSONObject()
-	suite.Equal(SUCCESS, obj.Status)
+	re.Equal(SUCCESS, obj.Status)
 
 	// Test TIMEOUT status.
 	steps = []OpStep{TransferLeader{FromStore: 2, ToStore: 1}}
 	op = NewTestOperator(1, &metapb.RegionEpoch{}, OpLeader, steps...)
 	op.Start()
 	op.SetStatusReachTime(STARTED, op.GetStartTime().Add(-FastStepWaitTime-time.Second))
-	suite.True(op.CheckTimeout())
+	re.True(op.CheckTimeout())
 	obj = op.ToJSONObject()
-	suite.Equal(TIMEOUT, obj.Status)
+	re.Equal(TIMEOUT, obj.Status)
 }
 
 func TestOperatorCheckConcurrently(t *testing.T) {

pkg/schedule/schedulers/evict_slow_trend.go

@@ -39,9 +39,11 @@ import (
 )
 
 const (
-	alterEpsilon               = 1e-9
-	minReCheckDurationGap      = 120 // default gap for re-check the slow node, unit: s
-	defaultRecoveryDurationGap = 600 // default gap for recovery, unit: s.
+	alterEpsilon          = 1e-9
+	minReCheckDurationGap = 120 // default gap for re-check the slow node, unit: s
+	// We use 1800 seconds as the default gap for recovery, which is 30 minutes.
+	// This is based on the SLA level reflected by AWS EBS. And we can adjust it later if needed.
+	defaultRecoveryDurationGap = 1800 // default gap for recovery, unit: s.
 )
 
 type slowCandidate struct {