Skip to content

Commit 3099b31

Browse files
authored
cache auth results and use as client conf (#845)
* cache auth results and use as client conf
1 parent 87fb2de commit 3099b31

File tree

10 files changed

+173
-8
lines changed

10 files changed

+173
-8
lines changed

.gitignore

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ bin/
33
vendor/
44
profile.cov
55
Secret-*
6-
secret*
76
/deploy-dev/
87
.idea
98
.DS_Store
@@ -20,4 +19,4 @@ cov2.out
2019
.yalc/
2120
yalc.lock
2221
.ropeproject
23-
dist/
22+
dist/

cmd/app/mount_manager.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,10 @@ func (m *MountManager) Start(ctx context.Context) {
105105
klog.Errorf("Register job controller error: %v", err)
106106
return
107107
}
108+
if err := (mountctrl.NewSecretController(m.client)).SetupWithManager(m.mgr); err != nil {
109+
klog.Errorf("Register secret controller error: %v", err)
110+
return
111+
}
108112
klog.Info("Mount manager started.")
109113
if err := m.mgr.Start(ctx); err != nil {
110114
klog.Errorf("Mount manager start error: %v", err)

deploy/kubernetes/base/resources.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ rules:
3232
verbs: ["get", "list", "watch"]
3333
- apiGroups: [""]
3434
resources: ["secrets"]
35-
verbs: ["get", "create", "update", "patch", "delete"]
35+
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
3636
- apiGroups: [""]
3737
resources: ["pods", "pods/log"]
3838
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]

pkg/config/config.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,8 @@ var (
6060
CeCliPath = "/usr/local/bin/juicefs"
6161
CeMountPath = "/bin/mount.juicefs"
6262
JfsMountPath = "/sbin/mount.juicefs"
63+
ClientConfPath = "/root/.juicefs"
64+
ROConfPath = "/etc/juicefs"
6365
)
6466

6567
const (

pkg/config/setting.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ type JfsSetting struct {
8181
Options []string // mount options
8282
FormatCmd string // format or auth
8383
SubPath string // subPath which is to be created or deleted
84-
SecretName string // secret name which is set env in pod
84+
SecretName string // secret with JuiceFS volume credentials
8585

8686
Attr PodAttr
8787
}
Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
/*
2+
Copyright 2023 Juicedata Inc
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package controller
18+
19+
import (
20+
"context"
21+
"os"
22+
"path/filepath"
23+
24+
corev1 "k8s.io/api/core/v1"
25+
k8serrors "k8s.io/apimachinery/pkg/api/errors"
26+
"k8s.io/klog"
27+
ctrl "sigs.k8s.io/controller-runtime"
28+
"sigs.k8s.io/controller-runtime/pkg/controller"
29+
"sigs.k8s.io/controller-runtime/pkg/event"
30+
"sigs.k8s.io/controller-runtime/pkg/handler"
31+
"sigs.k8s.io/controller-runtime/pkg/predicate"
32+
"sigs.k8s.io/controller-runtime/pkg/reconcile"
33+
"sigs.k8s.io/controller-runtime/pkg/source"
34+
35+
"github.com/juicedata/juicefs-csi-driver/pkg/config"
36+
"github.com/juicedata/juicefs-csi-driver/pkg/juicefs"
37+
"github.com/juicedata/juicefs-csi-driver/pkg/k8sclient"
38+
)
39+
40+
type SecretController struct {
41+
*k8sclient.K8sClient
42+
}
43+
44+
func NewSecretController(client *k8sclient.K8sClient) *SecretController {
45+
return &SecretController{client}
46+
}
47+
48+
func (m *SecretController) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) {
49+
klog.V(6).Infof("Receive secret %s %s", request.Name, request.Namespace)
50+
secrets, err := m.GetSecret(ctx, request.Name, request.Namespace)
51+
if err != nil && !k8serrors.IsNotFound(err) {
52+
klog.Errorf("get secret %s error: %v", request.Name, err)
53+
return reconcile.Result{}, err
54+
}
55+
if secrets == nil {
56+
klog.V(6).Infof("secret %s has been deleted.", request.Name)
57+
return reconcile.Result{}, nil
58+
}
59+
if _, found := secrets.Data["token"]; !found {
60+
klog.V(6).Infof("token not found in secret %s", request.Name)
61+
return reconcile.Result{}, nil
62+
}
63+
if _, found := secrets.Data["name"]; !found {
64+
klog.V(6).Infof("name not found in secret %s", request.Name)
65+
return reconcile.Result{}, nil
66+
}
67+
jfs := juicefs.NewJfsProvider(nil, nil)
68+
secretsMap := make(map[string]string)
69+
for k, v := range secrets.Data {
70+
secretsMap[k] = string(v[:])
71+
}
72+
jfsSetting, err := jfs.Settings(ctx, "", secretsMap, nil, nil)
73+
if err != nil {
74+
return reconcile.Result{}, err
75+
}
76+
output, err := jfs.AuthFs(ctx, secretsMap, jfsSetting, true)
77+
klog.V(6).Infof("auth output: %s.", output)
78+
if err != nil {
79+
return reconcile.Result{}, err
80+
}
81+
conf := jfsSetting.Name + ".conf"
82+
confPath := filepath.Join(config.ClientConfPath, conf)
83+
b, err := os.ReadFile(confPath)
84+
if err != nil {
85+
return reconcile.Result{}, err
86+
}
87+
confs := string(b)
88+
secretsMap["initConfig"] = confs
89+
secrets.StringData = secretsMap
90+
err = m.UpdateSecret(ctx, secrets)
91+
if err != nil {
92+
return reconcile.Result{}, err
93+
}
94+
return reconcile.Result{}, nil
95+
}
96+
97+
func (m *SecretController) SetupWithManager(mgr ctrl.Manager) error {
98+
c, err := controller.New("secret", mgr, controller.Options{Reconciler: m})
99+
if err != nil {
100+
return err
101+
}
102+
103+
return c.Watch(&source.Kind{Type: &corev1.Secret{}}, &handler.EnqueueRequestForObject{}, predicate.Funcs{
104+
CreateFunc: func(event event.CreateEvent) bool {
105+
secret := event.Object.(*corev1.Secret)
106+
klog.V(6).Infof("watch secret %s created", secret.GetName())
107+
return true
108+
},
109+
UpdateFunc: func(updateEvent event.UpdateEvent) bool {
110+
secretNew, ok := updateEvent.ObjectNew.(*corev1.Secret)
111+
klog.V(6).Infof("watch secret %s updated", secretNew.GetName())
112+
if !ok {
113+
klog.V(6).Infof("secret.onUpdateFunc Skip object: %v", updateEvent.ObjectNew)
114+
return false
115+
}
116+
117+
secretOld, ok := updateEvent.ObjectOld.(*corev1.Secret)
118+
if !ok {
119+
klog.V(6).Infof("secret.onUpdateFunc Skip object: %v", updateEvent.ObjectOld)
120+
return false
121+
}
122+
123+
if secretNew.GetResourceVersion() == secretOld.GetResourceVersion() {
124+
klog.V(6).Info("secret.onUpdateFunc Skip due to resourceVersion not changed")
125+
return false
126+
}
127+
return true
128+
},
129+
DeleteFunc: func(deleteEvent event.DeleteEvent) bool {
130+
secret := deleteEvent.Object.(*corev1.Secret)
131+
klog.V(6).Infof("watch secret %s deleted", secret.GetName())
132+
return true
133+
},
134+
})
135+
}

pkg/juicefs/juicefs.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ type Interface interface {
6969
Settings(ctx context.Context, volumeID string, secrets, volCtx map[string]string, options []string) (*config.JfsSetting, error)
7070
GetSubPath(ctx context.Context, volumeID string) (string, error)
7171
CreateTarget(ctx context.Context, target string) error
72+
AuthFs(ctx context.Context, secrets map[string]string, jfsSetting *config.JfsSetting, force bool) (string, error)
7273
}
7374

7475
type juicefs struct {
@@ -345,7 +346,7 @@ func (j *juicefs) Settings(ctx context.Context, volumeID string, secrets, volCtx
345346
}
346347
jfsSetting, err := config.ParseSetting(secrets, volCtx, mountOptions, !config.ByProcess)
347348
if err != nil {
348-
klog.V(5).Infof("Parse config error: %v", err)
349+
klog.V(5).Infof("Parse config for %s error: %v", secrets["name"], err)
349350
return nil, err
350351
}
351352
jfsSetting.VolumeId = volumeID
@@ -731,7 +732,7 @@ func (j *juicefs) AuthFs(ctx context.Context, secrets map[string]string, setting
731732
}
732733
if config.ByProcess && secrets["initconfig"] != "" {
733734
conf := secrets["name"] + ".conf"
734-
confPath := filepath.Join("/root/.juicefs", conf)
735+
confPath := filepath.Join(config.ClientConfPath, conf)
735736
if _, err := os.Stat(confPath); os.IsNotExist(err) {
736737
err = ioutil.WriteFile(confPath, []byte(secrets["initconfig"]), 0644)
737738
if err != nil {

pkg/juicefs/mocks/mock_juicefs.go

Lines changed: 15 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

pkg/juicefs/mount/builder/common.go

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package builder
1919
import (
2020
"fmt"
2121
"path"
22+
"path/filepath"
2223
"regexp"
2324
"strconv"
2425
"strings"
@@ -155,7 +156,12 @@ func (r *BaseBuilder) genInitCommand() string {
155156
formatCmd = formatCmd + " --encrypt-rsa-key=/root/.rsa/rsa-key.pem"
156157
}
157158
}
158-
159+
if r.jfsSetting.InitConfig != "" {
160+
confPath := filepath.Join(config.ROConfPath, r.jfsSetting.Name+".conf")
161+
args := []string{"cp", confPath, config.ClientConfPath}
162+
confCmd := strings.Join(args, " ")
163+
formatCmd = strings.Join([]string{confCmd, formatCmd}, "\n")
164+
}
159165
return formatCmd
160166
}
161167

@@ -304,7 +310,7 @@ func (r *BaseBuilder) _genJuiceVolumes() ([]corev1.Volume, []corev1.VolumeMount)
304310
volumeMounts = append(volumeMounts,
305311
corev1.VolumeMount{
306312
Name: "init-config",
307-
MountPath: "/root/.juicefs",
313+
MountPath: config.ROConfPath,
308314
},
309315
)
310316
}

tests/sanity/fake_juicefs_provider.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,9 @@ func (j *fakeJfsProvider) JfsCleanupMountPoint(ctx context.Context, mountPath st
8080
return nil
8181
}
8282

83+
func (j *fakeJfsProvider) AuthFs(ctx context.Context, secrets map[string]string, setting *config.JfsSetting, force bool) (string, error) {
84+
return "", nil
85+
}
8386
func (j *fakeJfsProvider) JfsUnmount(ctx context.Context, volumeId, mountPath string) error {
8487
return nil
8588
}

0 commit comments

Comments
 (0)