Skip to content

Commit 7fac7e0

Browse files
anisurrahman75anisurrahman75
authored
Add restic check and rebuild-index cmd for better repository debugging (#203)
Signed-off-by: Anisur Rahman <[email protected]>
1 parent 6b48bb8 commit 7fac7e0

File tree

3 files changed

+374
-0
lines changed

3 files changed

+374
-0
lines changed

‎pkg/check.go

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
/*
2+
Copyright AppsCode Inc. and Contributors
3+
4+
Licensed under the AppsCode Community License 1.0.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package pkg
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"os"
23+
"path/filepath"
24+
25+
"stash.appscode.dev/apimachinery/apis/stash/v1alpha1"
26+
cs "stash.appscode.dev/apimachinery/client/clientset/versioned"
27+
"stash.appscode.dev/apimachinery/pkg/restic"
28+
"stash.appscode.dev/stash/pkg/util"
29+
30+
"github.com/pkg/errors"
31+
"github.com/spf13/cobra"
32+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
33+
"k8s.io/cli-runtime/pkg/genericclioptions"
34+
"k8s.io/client-go/kubernetes"
35+
"k8s.io/client-go/rest"
36+
"k8s.io/klog/v2"
37+
)
38+
39+
type checkOptions struct {
40+
kubeClient *kubernetes.Clientset
41+
config *rest.Config
42+
repo *v1alpha1.Repository
43+
44+
// All restic options for the 'check' command.
45+
readData bool
46+
readDataSubset string
47+
withCache bool
48+
}
49+
50+
func NewCmdCheckRepository(clientGetter genericclioptions.RESTClientGetter) *cobra.Command {
51+
opt := checkOptions{}
52+
cmd := &cobra.Command{
53+
Use: "check",
54+
Short: `Check the repository for errors`,
55+
DisableAutoGenTag: true,
56+
RunE: func(cmd *cobra.Command, args []string) error {
57+
if len(args) == 0 || args[0] == "" {
58+
return fmt.Errorf("repository name not found")
59+
}
60+
repositoryName := args[0]
61+
62+
var err error
63+
opt.config, err = clientGetter.ToRESTConfig()
64+
if err != nil {
65+
return errors.Wrap(err, "failed to read kubeconfig")
66+
}
67+
namespace, _, err = clientGetter.ToRawKubeConfigLoader().Namespace()
68+
if err != nil {
69+
return err
70+
}
71+
72+
opt.kubeClient, err = kubernetes.NewForConfig(opt.config)
73+
if err != nil {
74+
return err
75+
}
76+
77+
stashClient, err = cs.NewForConfig(opt.config)
78+
if err != nil {
79+
return err
80+
}
81+
82+
// get source repository
83+
opt.repo, err = stashClient.StashV1alpha1().Repositories(namespace).Get(context.TODO(), repositoryName, metav1.GetOptions{})
84+
if err != nil {
85+
return err
86+
}
87+
88+
extraArgs := opt.getUserExtraArguments()
89+
if opt.repo.Spec.Backend.Local != nil {
90+
return opt.checkLocalRepository(extraArgs)
91+
}
92+
93+
return opt.checkRepository(extraArgs)
94+
},
95+
}
96+
97+
cmd.Flags().BoolVar(&opt.readData, "read-data", false, "read all data blobs")
98+
cmd.Flags().BoolVar(&opt.withCache, "with-cache", false, "use existing cache, only read uncached data from repository")
99+
cmd.Flags().StringVar(&opt.readDataSubset, "read-data-subset", "", "read a `subset` of data packs, specified as 'n/t' for specific part, or either 'x%' or 'x.y%' or a size in bytes with suffixes k/K, m/M, g/G, t/T for a random subset")
100+
return cmd
101+
}
102+
103+
func (opt *checkOptions) checkLocalRepository(extraArgs []string) error {
104+
// get the pod that mount this repository as volume
105+
pod, err := getBackendMountingPod(opt.kubeClient, opt.repo)
106+
if err != nil {
107+
return err
108+
}
109+
110+
command := []string{"/stash-enterprise", "check"}
111+
command = append(command, extraArgs...)
112+
command = append(command, "--repo-name="+opt.repo.Name, "--repo-namespace="+opt.repo.Namespace)
113+
114+
out, err := execCommandOnPod(opt.kubeClient, opt.config, pod, command)
115+
if string(out) != "" {
116+
klog.Infoln("Output:", string(out))
117+
}
118+
if err != nil {
119+
return err
120+
}
121+
klog.Infof("Repository %s/%s has been checked successfully", opt.repo.Namespace, opt.repo.Name)
122+
return nil
123+
}
124+
125+
func (opt *checkOptions) checkRepository(extraArgs []string) error {
126+
// get source repository secret
127+
secret, err := opt.kubeClient.CoreV1().Secrets(namespace).Get(context.TODO(), opt.repo.Spec.Backend.StorageSecretName, metav1.GetOptions{})
128+
if err != nil {
129+
return err
130+
}
131+
132+
if err = os.MkdirAll(ScratchDir, 0o755); err != nil {
133+
return err
134+
}
135+
defer os.RemoveAll(ScratchDir)
136+
137+
// configure restic wrapper
138+
extraOpt := util.ExtraOptions{
139+
StorageSecret: secret,
140+
ScratchDir: ScratchDir,
141+
}
142+
// configure setupOption
143+
setupOpt, err := util.SetupOptionsForRepository(*opt.repo, extraOpt)
144+
if err != nil {
145+
return fmt.Errorf("setup option for repository failed")
146+
}
147+
// init restic wrapper
148+
resticWrapper, err := restic.NewResticWrapper(setupOpt)
149+
if err != nil {
150+
return err
151+
}
152+
153+
localDirs := &cliLocalDirectories{
154+
configDir: filepath.Join(ScratchDir, configDirName),
155+
}
156+
157+
// dump restic's environments into `restic-env` file.
158+
// we will pass this env file to restic docker container.
159+
160+
err = resticWrapper.DumpEnv(localDirs.configDir, ResticEnvs)
161+
if err != nil {
162+
return err
163+
}
164+
165+
// For TLS secured Minio/REST server, specify cert path
166+
if resticWrapper.GetCaPath() != "" {
167+
extraArgs = append(extraArgs, "--cacert", resticWrapper.GetCaPath())
168+
}
169+
170+
// run unlock inside docker
171+
if err = runCmdViaDocker(*localDirs, "check", extraArgs); err != nil {
172+
return err
173+
}
174+
klog.Infof("Repository %s/%s has been checked successfully", opt.repo.Namespace, opt.repo.Name)
175+
return nil
176+
}
177+
178+
func (opt *checkOptions) getUserExtraArguments() []string {
179+
var extraArgs []string
180+
if opt.readData {
181+
extraArgs = append(extraArgs, "--read-data")
182+
}
183+
if opt.readDataSubset != "" {
184+
extraArgs = append(extraArgs,
185+
fmt.Sprintf("--read-data-subset=%s", opt.readDataSubset))
186+
}
187+
if opt.withCache {
188+
extraArgs = append(extraArgs, "--with-cache")
189+
}
190+
return extraArgs
191+
}

‎pkg/rebuild_index.go

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
/*
2+
Copyright AppsCode Inc. and Contributors
3+
4+
Licensed under the AppsCode Community License 1.0.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package pkg
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"os"
23+
"path/filepath"
24+
25+
"stash.appscode.dev/apimachinery/apis/stash/v1alpha1"
26+
cs "stash.appscode.dev/apimachinery/client/clientset/versioned"
27+
"stash.appscode.dev/apimachinery/pkg/restic"
28+
"stash.appscode.dev/stash/pkg/util"
29+
30+
"github.com/pkg/errors"
31+
"github.com/spf13/cobra"
32+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
33+
"k8s.io/cli-runtime/pkg/genericclioptions"
34+
"k8s.io/client-go/kubernetes"
35+
"k8s.io/client-go/rest"
36+
"k8s.io/klog/v2"
37+
)
38+
39+
type rebuildIndexOptions struct {
40+
kubeClient *kubernetes.Clientset
41+
config *rest.Config
42+
repo *v1alpha1.Repository
43+
44+
// All restic options for the 'rebuild-index' command.
45+
readAllPacks bool
46+
}
47+
48+
func NewCmdRebuildIndex(clientGetter genericclioptions.RESTClientGetter) *cobra.Command {
49+
opt := rebuildIndexOptions{}
50+
cmd := &cobra.Command{
51+
Use: "rebuild-index",
52+
Short: `Build a new index`,
53+
DisableAutoGenTag: true,
54+
RunE: func(cmd *cobra.Command, args []string) error {
55+
if len(args) == 0 || args[0] == "" {
56+
return fmt.Errorf("repository name not found")
57+
}
58+
repositoryName := args[0]
59+
60+
var err error
61+
opt.config, err = clientGetter.ToRESTConfig()
62+
if err != nil {
63+
return errors.Wrap(err, "failed to read kubeconfig")
64+
}
65+
namespace, _, err = clientGetter.ToRawKubeConfigLoader().Namespace()
66+
if err != nil {
67+
return err
68+
}
69+
70+
opt.kubeClient, err = kubernetes.NewForConfig(opt.config)
71+
if err != nil {
72+
return err
73+
}
74+
75+
stashClient, err = cs.NewForConfig(opt.config)
76+
if err != nil {
77+
return err
78+
}
79+
80+
// get source repository
81+
opt.repo, err = stashClient.StashV1alpha1().Repositories(namespace).Get(context.TODO(), repositoryName, metav1.GetOptions{})
82+
if err != nil {
83+
return err
84+
}
85+
86+
extraArgs := opt.getUserExtraArguments()
87+
if opt.repo.Spec.Backend.Local != nil {
88+
return opt.rebuildIndexToLocalRepository(extraArgs)
89+
}
90+
91+
return opt.rebuildIndex(extraArgs)
92+
},
93+
}
94+
95+
cmd.Flags().BoolVar(&opt.readAllPacks, "read-all-packs", false, "read all pack files to generate new index from scratch")
96+
return cmd
97+
}
98+
99+
func (opt *rebuildIndexOptions) rebuildIndex(extraArgs []string) error {
100+
// get source repository secret
101+
secret, err := opt.kubeClient.CoreV1().Secrets(namespace).Get(context.TODO(), opt.repo.Spec.Backend.StorageSecretName, metav1.GetOptions{})
102+
if err != nil {
103+
return err
104+
}
105+
106+
if err = os.MkdirAll(ScratchDir, 0o755); err != nil {
107+
return err
108+
}
109+
defer os.RemoveAll(ScratchDir)
110+
111+
// configure restic wrapper
112+
extraOpt := util.ExtraOptions{
113+
StorageSecret: secret,
114+
ScratchDir: ScratchDir,
115+
}
116+
// configure setupOption
117+
setupOpt, err := util.SetupOptionsForRepository(*opt.repo, extraOpt)
118+
if err != nil {
119+
return fmt.Errorf("setup option for repository failed")
120+
}
121+
// init restic wrapper
122+
resticWrapper, err := restic.NewResticWrapper(setupOpt)
123+
if err != nil {
124+
return err
125+
}
126+
127+
localDirs := &cliLocalDirectories{
128+
configDir: filepath.Join(ScratchDir, configDirName),
129+
}
130+
131+
// dump restic's environments into `restic-env` file.
132+
// we will pass this env file to restic docker container.
133+
134+
err = resticWrapper.DumpEnv(localDirs.configDir, ResticEnvs)
135+
if err != nil {
136+
return err
137+
}
138+
139+
// For TLS secured Minio/REST server, specify cert path
140+
if resticWrapper.GetCaPath() != "" {
141+
extraArgs = append(extraArgs, "--cacert", resticWrapper.GetCaPath())
142+
}
143+
144+
// run unlock inside docker
145+
if err = runCmdViaDocker(*localDirs, "rebuild-index", extraArgs); err != nil {
146+
return err
147+
}
148+
klog.Infof("Repository %s/%s has been rebuild-indexed successfully", opt.repo.Namespace, opt.repo.Name)
149+
return nil
150+
}
151+
152+
func (opt *rebuildIndexOptions) rebuildIndexToLocalRepository(extraArgs []string) error {
153+
// get the pod that mount this repository as volume
154+
pod, err := getBackendMountingPod(opt.kubeClient, opt.repo)
155+
if err != nil {
156+
return err
157+
}
158+
159+
command := []string{"/stash-enterprise", "rebuild-index"}
160+
command = append(command, extraArgs...)
161+
command = append(command, "--repo-name="+opt.repo.Name, "--repo-namespace="+opt.repo.Namespace)
162+
163+
out, err := execCommandOnPod(opt.kubeClient, opt.config, pod, command)
164+
if string(out) != "" {
165+
klog.Infoln("Output:", string(out))
166+
}
167+
if err != nil {
168+
return err
169+
}
170+
171+
klog.Infof("Repository %s/%s has been rebuild-indexed successfully", opt.repo.Namespace, opt.repo.Name)
172+
return nil
173+
}
174+
175+
func (opt *rebuildIndexOptions) getUserExtraArguments() []string {
176+
var extraArgs []string
177+
if opt.readAllPacks {
178+
extraArgs = append(extraArgs, "--read-all-packs")
179+
}
180+
return extraArgs
181+
}

‎pkg/root.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,5 +64,7 @@ func NewRootCmd() *cobra.Command {
6464
rootCmd.AddCommand(NewCmdDebug(f))
6565
rootCmd.AddCommand(NewCmdGen(f))
6666
rootCmd.AddCommand(NewCmdKey(f))
67+
rootCmd.AddCommand(NewCmdCheckRepository(f))
68+
rootCmd.AddCommand(NewCmdRebuildIndex(f))
6769
return rootCmd
6870
}

0 commit comments

Comments
 (0)