tests: add more cases for TestReadyAPI (tikv#9170)

ref tikv#4399

Signed-off-by: lhy1024 <[email protected]>

Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com>

Author: lhy1024

pkg/storage/storage.go

@@ -20,7 +20,6 @@ import (
 
 	clientv3 "go.etcd.io/etcd/client/v3"
 
-	"github.com/pingcap/failpoint"
 	"github.com/pingcap/kvproto/pkg/metapb"
 
 	"github.com/tikv/pd/pkg/core"
@@ -221,9 +220,6 @@ func AreRegionsLoaded(s Storage) bool {
 	ps := s.(*coreStorage)
 	ps.mu.RLock()
 	defer ps.mu.RUnlock()
-	failpoint.Inject("loadRegionSlow", func() {
-		failpoint.Return(false)
-	})
 	if ps.useRegionStorage.Load() {
 		return ps.regionLoaded == fromLeveldb
 	}

server/apiv2/handlers/ready.go

@@ -19,6 +19,8 @@ import (
 
 	"github.com/gin-gonic/gin"
 
+	"github.com/pingcap/failpoint"
+
 	"github.com/tikv/pd/pkg/storage"
 	"github.com/tikv/pd/server"
 	"github.com/tikv/pd/server/apiv2/middlewares"
@@ -39,6 +41,13 @@ func Ready(c *gin.Context) {
 	svr := c.MustGet(middlewares.ServerContextKey).(*server.Server)
 	s := svr.GetStorage()
 	regionLoaded := storage.AreRegionsLoaded(s)
+	failpoint.Inject("loadRegionSlow", func(val failpoint.Value) {
+		if s, ok := val.(string); ok {
+			if svr.GetAddr() == s {
+				regionLoaded = false
+			}
+		}
+	})
 	if regionLoaded {
 		c.Status(http.StatusOK)
 	} else {

tests/server/apiv2/handlers/ready_test.go

@@ -25,48 +25,72 @@ import (
 
 	"github.com/pingcap/failpoint"
 
-	tu "github.com/tikv/pd/pkg/utils/testutil"
+	"github.com/tikv/pd/pkg/utils/apiutil"
 	"github.com/tikv/pd/server/apiv2/handlers"
 	"github.com/tikv/pd/tests"
 )
 
-func TestReady(t *testing.T) {
+func TestReadyAPI(t *testing.T) {
 	re := require.New(t)
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	cluster, err := tests.NewTestCluster(ctx, 1)
+	cluster, err := tests.NewTestCluster(ctx, 3)
 	re.NoError(err)
 	defer cluster.Destroy()
 	re.NoError(cluster.RunInitialServers())
 	re.NotEmpty(cluster.WaitLeader())
-	server := cluster.GetLeaderServer()
-	re.NoError(server.BootstrapCluster())
-	url := server.GetConfig().ClientUrls + v2Prefix + "/ready"
-	failpoint.Enable("github.com/tikv/pd/pkg/storage/loadRegionSlow", `return()`)
-	checkReady(re, url, false)
-	failpoint.Disable("github.com/tikv/pd/pkg/storage/loadRegionSlow")
-	checkReady(re, url, true)
+	leader := cluster.GetLeaderServer()
+	re.NoError(leader.BootstrapCluster())
+	url := leader.GetConfig().ClientUrls + v2Prefix + "/ready"
+	// check ready status when region is not loaded for leader
+	failpoint.Enable("github.com/tikv/pd/server/apiv2/handlers/loadRegionSlow", `return("`+leader.GetAddr()+`")`)
+	checkReadyAPI(re, url, false)
+	// check ready status when region is loaded for leader
+	failpoint.Disable("github.com/tikv/pd/server/apiv2/handlers/loadRegionSlow")
+	checkReadyAPI(re, url, true)
+	// check ready status when region is not loaded for follower
+	followerServer := cluster.GetServer(cluster.GetFollower())
+	url = followerServer.GetConfig().ClientUrls + v2Prefix + "/ready"
+	failpoint.Enable("github.com/tikv/pd/server/apiv2/handlers/loadRegionSlow", `return("`+followerServer.GetAddr()+`")`)
+	checkReadyAPI(re, url, true)
+	checkReadyAPI(re, url, false, apiutil.PDAllowFollowerHandleHeader)
+	// check ready status when region is loaded for follower
+	failpoint.Disable("github.com/tikv/pd/server/apiv2/handlers/loadRegionSlow")
+	checkReadyAPI(re, url, true)
+	checkReadyAPI(re, url, true, apiutil.PDAllowFollowerHandleHeader)
 }
 
-func checkReady(re *require.Assertions, url string, isReady bool) {
+func checkReadyAPI(re *require.Assertions, url string, isReady bool, headers ...string) {
 	expectCode := http.StatusOK
 	if !isReady {
 		expectCode = http.StatusInternalServerError
 	}
-	resp, err := tests.TestDialClient.Get(url)
+	// check ready status
+	req, err := http.NewRequest(http.MethodGet, url, http.NoBody)
+	re.NoError(err)
+	if len(headers) > 0 {
+		req.Header.Add(headers[0], "true")
+	}
+	resp, err := tests.TestDialClient.Do(req)
 	re.NoError(err)
 	defer resp.Body.Close()
 	buf, err := io.ReadAll(resp.Body)
 	re.NoError(err)
 	re.Empty(buf)
 	re.Equal(expectCode, resp.StatusCode)
-	r := &handlers.ReadyStatus{}
-	if isReady {
-		r.RegionLoaded = true
+	// check ready status with verbose
+	req, err = http.NewRequest(http.MethodGet, url+"?verbose", http.NoBody)
+	re.NoError(err)
+	if len(headers) > 0 {
+		req.Header.Add(headers[0], "true")
 	}
-	data, err := json.Marshal(r)
+	resp, err = tests.TestDialClient.Do(req)
 	re.NoError(err)
-	err = tu.CheckGetJSON(tests.TestDialClient, url+"?verbose", data,
-		tu.Status(re, expectCode))
+	defer resp.Body.Close()
+	buf, err = io.ReadAll(resp.Body)
 	re.NoError(err)
+	r := &handlers.ReadyStatus{}
+	re.NoError(json.Unmarshal(buf, &r))
+	re.Equal(expectCode, resp.StatusCode)
+	re.Equal(isReady, r.RegionLoaded)
 }