Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support user-supplied bootstrap configuration #66

Merged
merged 6 commits into from
Oct 15, 2024
Merged

Support user-supplied bootstrap configuration #66

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
469cfb2
initial implementation
eaudetcobello Oct 8, 2024
413e3c5
change switch to if
eaudetcobello Oct 15, 2024
28eeba6
revert templates/aws
eaudetcobello Oct 15, 2024
bffa5e4
revert
eaudetcobello Oct 15, 2024
a909e9c
update documentation
eaudetcobello Oct 15, 2024
4ed4fea
add secret name to err
eaudetcobello Oct 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions bootstrap/api/v1beta2/ck8sconfig_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@ type CK8sConfigSpec struct {
// +optional
Files []File `json:"files,omitempty"`

// BootstrapConfig is the data to be passed to the bootstrap script.
BootstrapConfig *BootstrapConfig `json:"bootstrapConfig,omitempty"`

// BootCommands specifies extra commands to run in cloud-init early in the boot process.
// +optional
BootCommands []string `json:"bootCommands,omitempty"`
Expand Down Expand Up @@ -281,6 +284,17 @@ const (
GzipBase64 Encoding = "gzip+base64"
)

type BootstrapConfig struct {
// Content is the actual content of the file.
// If this is set, ContentFrom is ignored.
// +optional
Content string `json:"content,omitempty"`

// ContentFrom is a referenced source of content to populate the file.
// +optional
ContentFrom *FileSource `json:"contentFrom,omitempty"`
}

// File defines the input for generating write_files in cloud-init.
type File struct {
// Path specifies the full path on disk where to store the file.
Expand Down
25 changes: 25 additions & 0 deletions bootstrap/api/v1beta2/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

33 changes: 33 additions & 0 deletions bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_ck8sconfigs.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,39 @@ spec:
channel:
description: Channel is the channel to use for the snap install.
type: string
bootstrapConfig:
description: BootstrapConfig is the data to be passed to the bootstrap
script.
properties:
content:
description: |-
Content is the actual content of the file.
If this is set, ContentFrom is ignored.
type: string
contentFrom:
description: ContentFrom is a referenced source of content to
populate the file.
properties:
secret:
description: Secret represents a secret that should populate
this file.
properties:
key:
description: Key is the key in the secret's data map for
this value.
type: string
name:
description: Name of the secret in the CK8sBootstrapConfig's
namespace to use.
type: string
required:
- key
- name
type: object
required:
- secret
type: object
type: object
controlPlane:
description: CK8sControlPlaneConfig is configuration for the control
plane node.
Expand Down
33 changes: 33 additions & 0 deletions bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_ck8sconfigtemplates.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,39 @@ spec:
channel:
description: Channel is the channel to use for the snap install.
type: string
bootstrapConfig:
description: BootstrapConfig is the data to be passed to the
bootstrap script.
properties:
content:
description: |-
Content is the actual content of the file.
If this is set, ContentFrom is ignored.
type: string
contentFrom:
description: ContentFrom is a referenced source of content
to populate the file.
properties:
secret:
description: Secret represents a secret that should
populate this file.
properties:
key:
description: Key is the key in the secret's data
map for this value.
type: string
name:
description: Name of the secret in the CK8sBootstrapConfig's
namespace to use.
type: string
required:
- key
- name
type: object
required:
- secret
type: object
type: object
controlPlane:
description: CK8sControlPlaneConfig is configuration for the
control plane node.
Expand Down
43 changes: 38 additions & 5 deletions bootstrap/controllers/ck8sconfig_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -404,6 +404,32 @@ func (r *CK8sConfigReconciler) joinWorker(ctx context.Context, scope *Scope) err
return nil
}

// resolveUserBootstrapConfig returns the bootstrap configuration provided by the user.
// It can resolve string content, a reference to a secret, or an empty string if no configuration was provided.
func (r *CK8sConfigReconciler) resolveUserBootstrapConfig(ctx context.Context, cfg *bootstrapv1.CK8sConfig) (string, error) {
// User did not provide a bootstrap configuration
if cfg.Spec.BootstrapConfig == nil {
return "", nil
}

// User provided a bootstrap configuration through content
if cfg.Spec.BootstrapConfig.Content != "" {
return cfg.Spec.BootstrapConfig.Content, nil
}

// User referenced a secret for the bootstrap configuration
if cfg.Spec.BootstrapConfig.ContentFrom == nil {
return "", nil
}

data, err := r.resolveSecretFileContent(ctx, cfg.Namespace, *cfg.Spec.BootstrapConfig.ContentFrom)
if err != nil {
return "", fmt.Errorf("failed to read bootstrap configuration from secret %q: %w", cfg.Spec.BootstrapConfig.ContentFrom.Secret.Name, err)
}

return string(data), nil
}

// resolveFiles maps .Spec.Files into cloudinit.Files, resolving any object references
// along the way.
func (r *CK8sConfigReconciler) resolveFiles(ctx context.Context, cfg *bootstrapv1.CK8sConfig) ([]bootstrapv1.File, error) {
Expand All @@ -412,7 +438,7 @@ func (r *CK8sConfigReconciler) resolveFiles(ctx context.Context, cfg *bootstrapv
for i := range cfg.Spec.Files {
in := cfg.Spec.Files[i]
if in.ContentFrom != nil {
data, err := r.resolveSecretFileContent(ctx, cfg.Namespace, in)
data, err := r.resolveSecretFileContent(ctx, cfg.Namespace, *in.ContentFrom)
if err != nil {
return nil, fmt.Errorf("failed to resolve file source: %w", err)
}
Expand Down Expand Up @@ -505,18 +531,18 @@ func (r *CK8sConfigReconciler) getSnapInstallDataFromSpec(spec bootstrapv1.CK8sC
}

// resolveSecretFileContent returns file content fetched from a referenced secret object.
func (r *CK8sConfigReconciler) resolveSecretFileContent(ctx context.Context, ns string, source bootstrapv1.File) ([]byte, error) {
func (r *CK8sConfigReconciler) resolveSecretFileContent(ctx context.Context, ns string, source bootstrapv1.FileSource) ([]byte, error) {
Comment on lines -508 to +534
Copy link
Contributor Author

@eaudetcobello eaudetcobello Oct 8, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I changed the parameter type from File to FileSource because 1. We do not use any of the fields from the File type, except ContentFrom. 2. This makes the function reusable for any other types that have a FileSource (and therefore reference a secret), but are not an explicit abstraction of a File.

secret := &corev1.Secret{}
key := types.NamespacedName{Namespace: ns, Name: source.ContentFrom.Secret.Name}
key := types.NamespacedName{Namespace: ns, Name: source.Secret.Name}
if err := r.Client.Get(ctx, key, secret); err != nil {
if apierrors.IsNotFound(err) {
return nil, fmt.Errorf("secret not found %s: %w", key, err)
}
return nil, fmt.Errorf("failed to retrieve Secret %q: %w", key, err)
}
data, ok := secret.Data[source.ContentFrom.Secret.Key]
data, ok := secret.Data[source.Secret.Key]
if !ok {
return nil, fmt.Errorf("secret references non-existent secret key %q: %w", source.ContentFrom.Secret.Key, ErrInvalidRef)
return nil, fmt.Errorf("secret references non-existent secret key %q: %w", source.Secret.Key, ErrInvalidRef)
}
return data, nil
}
Expand Down Expand Up @@ -636,6 +662,12 @@ func (r *CK8sConfigReconciler) handleClusterNotInitialized(ctx context.Context,
return ctrl.Result{}, err
}

userSuppliedBootstrapConfig, err := r.resolveUserBootstrapConfig(ctx, scope.Config)
if err != nil {
conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, bootstrapv1.DataSecretGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error())
return ctrl.Result{}, err
}

Comment on lines +665 to +670
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

resolveUserBootstrapConfig returns an empty string if there is no user-supplied configuration, we check this later in common.go.

microclusterPort := scope.Config.Spec.ControlPlaneConfig.GetMicroclusterPort()
ds, err := ck8s.RenderK8sdProxyDaemonSetManifest(ck8s.K8sdProxyDaemonSetInput{K8sdPort: microclusterPort})
if err != nil {
Expand All @@ -654,6 +686,7 @@ func (r *CK8sConfigReconciler) handleClusterNotInitialized(ctx context.Context,
PreRunCommands: scope.Config.Spec.PreRunCommands,
PostRunCommands: scope.Config.Spec.PostRunCommands,
KubernetesVersion: scope.Config.Spec.Version,
BootstrapConfig: userSuppliedBootstrapConfig,
SnapInstallData: snapInstallData,
ExtraFiles: cloudinit.FilesFromAPI(files),
ConfigFileContents: string(initConfig),
Expand Down
33 changes: 33 additions & 0 deletions controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_ck8scontrolplanes.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,39 @@ spec:
channel:
description: Channel is the channel to use for the snap install.
type: string
bootstrapConfig:
description: BootstrapConfig is the data to be passed to the bootstrap
script.
properties:
content:
description: |-
Content is the actual content of the file.
If this is set, ContentFrom is ignored.
type: string
contentFrom:
description: ContentFrom is a referenced source of content
to populate the file.
properties:
secret:
description: Secret represents a secret that should populate
this file.
properties:
key:
description: Key is the key in the secret's data map
for this value.
type: string
name:
description: Name of the secret in the CK8sBootstrapConfig's
namespace to use.
type: string
required:
- key
- name
type: object
required:
- secret
type: object
type: object
controlPlane:
description: CK8sControlPlaneConfig is configuration for the control
plane node.
Expand Down
33 changes: 33 additions & 0 deletions controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_ck8scontrolplanetemplates.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,39 @@ spec:
description: Channel is the channel to use for the snap
install.
type: string
bootstrapConfig:
description: BootstrapConfig is the data to be passed
to the bootstrap script.
properties:
content:
description: |-
Content is the actual content of the file.
If this is set, ContentFrom is ignored.
type: string
contentFrom:
description: ContentFrom is a referenced source of
content to populate the file.
properties:
secret:
description: Secret represents a secret that should
populate this file.
properties:
key:
description: Key is the key in the secret's
data map for this value.
type: string
name:
description: Name of the secret in the CK8sBootstrapConfig's
namespace to use.
type: string
required:
- key
- name
type: object
required:
- secret
type: object
type: object
controlPlane:
description: CK8sControlPlaneConfig is configuration for
the control plane node.
Expand Down
11 changes: 10 additions & 1 deletion pkg/cloudinit/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ type BaseUserData struct {
PreRunCommands []string
// PostRunCommands is a list of commands to run after k8s installation.
PostRunCommands []string
// BootstrapConfig is the contents of the bootstrap configuration file.
BootstrapConfig string
// ExtraFiles is a list of extra files to load on the host.
ExtraFiles []File
// ConfigFileContents is the contents of the k8s configuration file.
Expand Down Expand Up @@ -93,14 +95,21 @@ func NewBaseCloudConfig(data BaseUserData) (CloudConfig, error) {
config.RunCommands = append(config.RunCommands, "/capi/scripts/configure-snapstore-proxy.sh")
}

var configFileContents string
if data.BootstrapConfig != "" {
configFileContents = data.BootstrapConfig
} else {
configFileContents = data.ConfigFileContents
}

// write files
config.WriteFiles = append(
config.WriteFiles,
append(
data.ExtraFiles,
File{
Path: "/capi/etc/config.yaml",
Content: data.ConfigFileContents,
Content: configFileContents,
Permissions: "0400",
Owner: "root:root",
},
Expand Down
28 changes: 28 additions & 0 deletions pkg/cloudinit/controlplane_init_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"testing"

. "github.com/onsi/gomega"
format "github.com/onsi/gomega/format"
"github.com/onsi/gomega/gstruct"

"github.com/canonical/cluster-api-k8s/pkg/cloudinit"
Expand All @@ -29,6 +30,8 @@ import (
func TestNewInitControlPlane(t *testing.T) {
g := NewWithT(t)

format.MaxLength = 20000

config, err := cloudinit.NewInitControlPlane(cloudinit.InitControlPlaneInput{
BaseUserData: cloudinit.BaseUserData{
KubernetesVersion: "v1.30.0",
Expand Down Expand Up @@ -102,6 +105,31 @@ func TestNewInitControlPlane(t *testing.T) {
), "Some /capi/scripts files are missing")
}

func TestUserSuppliedBootstrapConfig(t *testing.T) {
g := NewWithT(t)

config, err := cloudinit.NewInitControlPlane(cloudinit.InitControlPlaneInput{
BaseUserData: cloudinit.BaseUserData{
KubernetesVersion: "v1.30.0",
eaudetcobello marked this conversation as resolved.
Show resolved Hide resolved
BootstrapConfig: "### bootstrap config ###",
ConfigFileContents: "### config file ###",
},
})

g.Expect(err).ToNot(HaveOccurred())

// Test that user-supplied bootstrap configuration takes precedence over ConfigFileContents.
g.Expect(config.WriteFiles).To(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
"Path": Equal("/capi/etc/config.yaml"),
"Content": Equal("### bootstrap config ###"),
})))

g.Expect(config.WriteFiles).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
"Path": Equal("/capi/etc/config.yaml"),
"Content": Equal("### config file ###"),
})))
}

func TestNewInitControlPlaneInvalidVersionError(t *testing.T) {
g := NewWithT(t)

Expand Down