Skip to content

Commit 854cbca

Browse files
peknurpeknur
authored
fix(service): run only single attach/detach operation simultaneously (#94)
1 parent 1d65a40 commit 854cbca

File tree

5 files changed

+190
-3
lines changed

5 files changed

+190
-3
lines changed

‎Makefile

+1
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ clean-tests:
3232

3333
.PHONY: test
3434
test:
35+
go vet ./...
3536
go test -race ./...
3637

3738
test-integration:

‎internal/controller/controller.go

-1
Original file line numberDiff line numberDiff line change
@@ -272,7 +272,6 @@ func (c *Controller) ControllerPublishVolume(ctx context.Context, req *csi.Contr
272272
if errors.As(err, &svcError) && svcError.Status != http.StatusConflict && svcError.ErrorCode() == upcloud.ErrCodeStorageDeviceLimitReached {
273273
return nil, status.Error(codes.ResourceExhausted, "The limit of the number of attached devices has been reached")
274274
}
275-
// already attached to the node
276275
return nil, err
277276
}
278277

‎internal/service/mock/upcloud_client.go

+108
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
package mock
2+
3+
import (
4+
"context"
5+
"errors"
6+
"fmt"
7+
"math/rand"
8+
"sync"
9+
"time"
10+
11+
"github.com/UpCloudLtd/upcloud-go-api/v6/upcloud"
12+
"github.com/UpCloudLtd/upcloud-go-api/v6/upcloud/request"
13+
upsvc "github.com/UpCloudLtd/upcloud-go-api/v6/upcloud/service"
14+
)
15+
16+
type UpCloudClient struct {
17+
upsvc.Storage
18+
19+
servers sync.Map
20+
}
21+
22+
func (u *UpCloudClient) StoreServer(s *upcloud.ServerDetails) {
23+
u.servers.LoadOrStore(s.UUID, s)
24+
}
25+
26+
func (u *UpCloudClient) getServer(id string) *upcloud.ServerDetails {
27+
if s, ok := u.servers.Load(id); ok {
28+
return s.(*upcloud.ServerDetails)
29+
}
30+
return nil
31+
}
32+
33+
func (u *UpCloudClient) WaitForServerState(ctx context.Context, r *request.WaitForServerStateRequest) (*upcloud.ServerDetails, error) {
34+
s, _ := u.GetServerDetails(ctx, &request.GetServerDetailsRequest{
35+
UUID: r.UUID,
36+
})
37+
return s, nil
38+
}
39+
40+
func (u *UpCloudClient) GetServers(ctx context.Context) (*upcloud.Servers, error) {
41+
s := []upcloud.Server{}
42+
u.servers.Range(func(key, value any) bool {
43+
if d, ok := value.(*upcloud.ServerDetails); ok {
44+
s = append(s, d.Server)
45+
}
46+
return true
47+
})
48+
return &upcloud.Servers{Servers: s}, nil
49+
}
50+
51+
func (u *UpCloudClient) GetServerDetails(ctx context.Context, r *request.GetServerDetailsRequest) (*upcloud.ServerDetails, error) {
52+
if s := u.getServer(r.UUID); s != nil {
53+
return s, nil
54+
}
55+
return nil, fmt.Errorf("server '%s' not found", r.UUID)
56+
}
57+
58+
func (u *UpCloudClient) AttachStorage(ctx context.Context, r *request.AttachStorageRequest) (*upcloud.ServerDetails, error) {
59+
server := u.getServer(r.ServerUUID)
60+
if server == nil {
61+
return server, errors.New("server not found")
62+
}
63+
if server.State != upcloud.ServerStateStarted {
64+
return nil, fmt.Errorf("server %s state is %s", r.ServerUUID, server.State)
65+
}
66+
server.State = upcloud.ServerStateMaintenance
67+
u.StoreServer(server)
68+
time.Sleep(time.Duration(rand.Intn(200)+100) * time.Millisecond) //nolint:gosec // using weak random number doesn't affect the result.
69+
server.State = upcloud.ServerStateStarted
70+
if server.StorageDevices == nil {
71+
server.StorageDevices = make(upcloud.ServerStorageDeviceSlice, 0)
72+
}
73+
server.StorageDevices = append(server.StorageDevices, upcloud.ServerStorageDevice{
74+
Address: fmt.Sprintf("%s:%d", r.Address, len(server.StorageDevices)+1),
75+
UUID: r.StorageUUID,
76+
Size: 10,
77+
})
78+
u.StoreServer(server)
79+
80+
return u.getServer(r.ServerUUID), nil
81+
}
82+
83+
func (u *UpCloudClient) DetachStorage(ctx context.Context, r *request.DetachStorageRequest) (*upcloud.ServerDetails, error) {
84+
server := u.getServer(r.ServerUUID)
85+
if server == nil {
86+
return server, fmt.Errorf("server %s not found", r.ServerUUID)
87+
}
88+
if server.State != upcloud.ServerStateStarted {
89+
return nil, fmt.Errorf("server %s state is %s", r.ServerUUID, server.State)
90+
}
91+
server.State = upcloud.ServerStateMaintenance
92+
u.StoreServer(server)
93+
time.Sleep(time.Duration(rand.Intn(200)+100) * time.Millisecond) //nolint:gosec // using weak random number doesn't affect the result.
94+
server = u.getServer(r.ServerUUID)
95+
server.State = upcloud.ServerStateStarted
96+
if len(server.StorageDevices) > 0 {
97+
storage := make([]upcloud.ServerStorageDevice, 0)
98+
for i := range server.StorageDevices {
99+
if server.StorageDevices[i].Address != r.Address {
100+
storage = append(storage, server.StorageDevices[i])
101+
}
102+
}
103+
server.StorageDevices = storage
104+
}
105+
u.StoreServer(server)
106+
107+
return server, nil
108+
}

‎internal/service/service_test.go

+59
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,18 @@ import (
55
"fmt"
66
"net/http"
77
"net/http/httptest"
8+
"sync"
89
"testing"
10+
"time"
911

1012
"github.com/UpCloudLtd/upcloud-csi/internal/service"
13+
"github.com/UpCloudLtd/upcloud-csi/internal/service/mock"
1114
"github.com/UpCloudLtd/upcloud-go-api/v6/upcloud"
1215
"github.com/UpCloudLtd/upcloud-go-api/v6/upcloud/client"
16+
"github.com/UpCloudLtd/upcloud-go-api/v6/upcloud/request"
1317
upsvc "github.com/UpCloudLtd/upcloud-go-api/v6/upcloud/service"
1418
"github.com/stretchr/testify/assert"
19+
"github.com/stretchr/testify/require"
1520
)
1621

1722
func TestUpCloudService_ListStorage(t *testing.T) {
@@ -168,3 +173,57 @@ func TestUpCloudService_ListStorageBackups(t *testing.T) {
168173
assert.NoError(t, err)
169174
assert.Len(t, storages, 3)
170175
}
176+
177+
func TestUpCloudService_AttachDetachStorage_Concurrency(t *testing.T) {
178+
t.Parallel()
179+
180+
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
181+
defer cancel()
182+
183+
client := &mock.UpCloudClient{}
184+
s := service.NewUpCloudService(client)
185+
c := 10
186+
187+
var wg sync.WaitGroup
188+
for i := 0; i < c; i++ {
189+
wg.Add(1)
190+
// populate backend with two nodes and add 5 storages per node
191+
serverUUID := fmt.Sprintf("test-node-%d", i%2)
192+
volUUID := fmt.Sprintf("test-vol-%d", i)
193+
client.StoreServer(&upcloud.ServerDetails{
194+
Server: upcloud.Server{
195+
UUID: serverUUID,
196+
State: upcloud.ServerStateStarted,
197+
},
198+
StorageDevices: make([]upcloud.ServerStorageDevice, 0),
199+
})
200+
go func(volUUID, serverUUID string) {
201+
defer wg.Done()
202+
t1 := time.Now()
203+
err := s.AttachStorage(ctx, volUUID, serverUUID)
204+
t.Logf("attached %s to node %s in %s", volUUID, serverUUID, time.Since(t1))
205+
assert.NoError(t, err)
206+
}(volUUID, serverUUID)
207+
}
208+
wg.Wait()
209+
servers, err := client.GetServers(ctx)
210+
require.NoError(t, err)
211+
require.Len(t, servers.Servers, 2)
212+
for _, srv := range servers.Servers {
213+
d, err := client.GetServerDetails(ctx, &request.GetServerDetailsRequest{UUID: srv.UUID})
214+
if !assert.NoError(t, err) {
215+
continue
216+
}
217+
for _, storage := range d.StorageDevices {
218+
wg.Add(1)
219+
go func(volUUID, serverUUID string) {
220+
defer wg.Done()
221+
t1 := time.Now()
222+
err := s.DetachStorage(ctx, volUUID, serverUUID)
223+
t.Logf("detached %s from node %s in %s", volUUID, serverUUID, time.Since(t1))
224+
assert.NoError(t, err)
225+
}(storage.UUID, d.UUID)
226+
}
227+
}
228+
wg.Wait()
229+
}

‎internal/service/upcloud_service.go

+22-2
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"errors"
66
"fmt"
7+
"sync"
78
"time"
89

910
"github.com/UpCloudLtd/upcloud-go-api/v6/upcloud"
@@ -30,6 +31,9 @@ type upCloudClient interface {
3031

3132
type UpCloudService struct {
3233
client upCloudClient
34+
35+
// nodeSync holds per node mutex lock so that only one detach/attach operation can run simultaneously towards the node.
36+
nodeSync sync.Map
3337
}
3438

3539
func NewUpCloudService(svc upCloudClient) *UpCloudService {
@@ -123,6 +127,13 @@ func (u *UpCloudService) DeleteStorage(ctx context.Context, storageUUID string)
123127
}
124128

125129
func (u *UpCloudService) AttachStorage(ctx context.Context, storageUUID, serverUUID string) error {
130+
// Lock attach operation per node because node can only attach single storage at the time.
131+
mu, _ := u.nodeSync.LoadOrStore(serverUUID, &sync.Mutex{})
132+
if mu != nil {
133+
mu.(*sync.Mutex).Lock()
134+
defer mu.(*sync.Mutex).Unlock()
135+
}
136+
126137
if err := u.waitForServerOnline(ctx, serverUUID); err != nil {
127138
return fmt.Errorf("failed to attach storage, pre-condition failed: %w", err)
128139
}
@@ -142,12 +153,21 @@ func (u *UpCloudService) AttachStorage(ctx context.Context, storageUUID, serverU
142153
}
143154

144155
func (u *UpCloudService) DetachStorage(ctx context.Context, storageUUID, serverUUID string) error {
156+
// Lock detach operation per node because node can only detach single storage at the time.
157+
mu, _ := u.nodeSync.LoadOrStore(serverUUID, &sync.Mutex{})
158+
if mu != nil {
159+
mu.(*sync.Mutex).Lock()
160+
defer mu.(*sync.Mutex).Unlock()
161+
}
162+
145163
sd, err := u.client.GetServerDetails(ctx, &request.GetServerDetailsRequest{UUID: serverUUID})
146164
if err != nil {
147165
return err
148166
}
149-
if err := u.waitForServerOnline(ctx, serverUUID); err != nil {
150-
return fmt.Errorf("failed to detach storage, pre-condition failed: %w", err)
167+
if sd.State != upcloud.ServerStateStarted {
168+
if err := u.waitForServerOnline(ctx, serverUUID); err != nil {
169+
return fmt.Errorf("failed to detach storage, pre-condition failed: %w", err)
170+
}
151171
}
152172
for _, device := range sd.StorageDevices {
153173
if device.UUID == storageUUID {

0 commit comments

Comments
 (0)