Merge pull request #1245 from bkneis/POD-823/cache

Pod 823/cache

Author: bkneis

cmd/agent/container/setup.go

@@ -302,6 +302,10 @@ func dockerlessBuild(
 	args = append(args, parseIgnorePaths(dockerlessOptions.IgnorePaths)...)
 	args = append(args, "--build-arg", "TARGETOS="+runtime.GOOS)
 	args = append(args, "--build-arg", "TARGETARCH="+runtime.GOARCH)
+	if dockerlessOptions.RegistryCache != "" {
+		log.Debug("Appending registry cache to dockerless build arguments ", dockerlessOptions.RegistryCache)
+		args = append(args, "--registry-cache", dockerlessOptions.RegistryCache)
+	}
 
 	// ignore mounts
 	args = append(args, "--ignore-path", setupInfo.SubstitutionContext.ContainerWorkspaceFolder)

cmd/agent/workspace/build.go

@@ -81,9 +81,10 @@ func (cmd *BuildCmd) Run(ctx context.Context) error {
 	for _, platform := range platforms {
 		// build the image
 		imageName, err := runner.Build(ctx, provider2.BuildOptions{
-			CLIOptions: workspaceInfo.CLIOptions,
-
-			Platform: platform,
+			CLIOptions:    workspaceInfo.CLIOptions,
+			RegistryCache: workspaceInfo.RegistryCache,
+			Platform:      platform,
+			ExportCache:   true,
 		})
 		if err != nil {
 			logger.Errorf("Error building image: %v", err)

cmd/agent/workspace/up.go

@@ -126,7 +126,8 @@ func (cmd *UpCmd) devPodUp(ctx context.Context, workspaceInfo *provider2.AgentWo
 
 	// start the devcontainer
 	result, err := runner.Up(ctx, devcontainer.UpOptions{
-		CLIOptions: workspaceInfo.CLIOptions,
+		CLIOptions:    workspaceInfo.CLIOptions,
+		RegistryCache: workspaceInfo.RegistryCache,
 	}, workspaceInfo.InjectTimeout)
 	if err != nil {
 		return nil, err
@@ -206,7 +207,7 @@ func initWorkspace(ctx context.Context, cancel context.CancelFunc, workspaceInfo
 	}
 
 	// install docker in background
-	errChan := make(chan error, 1)
+	errChan := make(chan error, 2)
 	go func() {
 		if !workspaceInfo.Agent.IsDockerDriver() || workspaceInfo.Agent.Docker.Install == "false" {
 			errChan <- nil
@@ -229,12 +230,28 @@ func initWorkspace(ctx context.Context, cancel context.CancelFunc, workspaceInfo
 		}
 	}
 
-	// wait until docker is installed
+	// wait until docker is installed before configuring docker daemon
 	err = <-errChan
 	if err != nil {
 		return nil, nil, "", errors.Wrap(err, "install docker")
 	}
 
+	// If we are provisioning the machine, ensure the daemon has required options
+	local, err := workspaceInfo.Agent.Local.Bool()
+	if workspaceInfo.Agent.IsDockerDriver() && err != nil && !local {
+		errChan <- configureDockerDaemon(ctx, logger)
+	} else {
+		logger.Debug("Skipping configuring daemon")
+		errChan <- nil
+	}
+
+	// wait until docker daemon is configured
+	err = <-errChan
+	if err != nil {
+		logger.Warn("Could not find docker daemon config file, if using the registry cache, please ensure the daemon is configured with containerd-snapshotter=true")
+		logger.Warn("More info at https://docs.docker.com/engine/storage/containerd/")
+	}
+
 	return tunnelClient, logger, dockerCredentialsDir, nil
 }
 
@@ -410,7 +427,7 @@ func prepareImage(workspaceDir, image string) error {
 	return nil
 }
 
-func installDocker(log log.Logger) error {
+func installDocker(log log.Logger) (err error) {
 	if !command.Exists("docker") {
 		writer := log.Writer(logrus.InfoLevel, false)
 		defer writer.Close()
@@ -420,11 +437,33 @@ func installDocker(log log.Logger) error {
 		shellCommand := exec.Command("sh", "-c", scripts.InstallDocker)
 		shellCommand.Stdout = writer
 		shellCommand.Stderr = writer
-		err := shellCommand.Run()
-		if err != nil {
+		err = shellCommand.Run()
+	}
+	return err
+}
+
+func configureDockerDaemon(ctx context.Context, log log.Logger) (err error) {
+	log.Info("Configuring docker daemon ...")
+	// Enable image snapshotter in the dameon
+	var daemonConfig = []byte(`{
+		"features": {
+			"containerd-snapshotter": true
+		}
+	}`)
+	// Check rootless docker
+	homeDir, err := os.UserHomeDir()
+	if err != nil {
+		return err
+	}
+	if _, err = os.Stat(fmt.Sprintf("%s/.config/docker", homeDir)); !errors.Is(err, os.ErrNotExist) {
+		err = os.WriteFile(fmt.Sprintf("%s/.config/docker/daemon.json", homeDir), daemonConfig, 0644)
+	}
+	// otherwise assume default
+	if err != nil {
+		if err = os.WriteFile("/etc/docker/daemon.json", daemonConfig, 0644); err != nil {
 			return err
 		}
 	}
-
-	return nil
+	// reload docker daemon
+	return exec.CommandContext(ctx, "pkill", "-HUP", "dockerd").Run()
 }

e2e/tests/build/build.go

@@ -57,13 +57,18 @@ var _ = DevPodDescribe("devpod build test suite", func() {
 			err = f.DevPodBuild(ctx, tempDir, "--force-build", "--platform", "linux/amd64,linux/arm64", "--repository", prebuildRepo, "--skip-push")
 			framework.ExpectNoError(err)
 
+			// parse the dockerfile
+			file, err := dockerfile.Parse(modifiedDockerfileContents)
+			framework.ExpectNoError(err)
+			info := &config.ImageBuildInfo{Dockerfile: file}
+
 			// make sure images are there
-			prebuildHash, err := config.CalculatePrebuildHash(cfg, "linux/amd64", "amd64", filepath.Dir(cfg.Origin), dockerfilePath, modifiedDockerfileContents, log.Default)
+			prebuildHash, err := config.CalculatePrebuildHash(cfg, "linux/amd64", "amd64", filepath.Dir(cfg.Origin), dockerfilePath, modifiedDockerfileContents, info, log.Default)
 			framework.ExpectNoError(err)
 			_, err = dockerHelper.InspectImage(ctx, prebuildRepo+":"+prebuildHash, false)
 			framework.ExpectNoError(err)
 
-			prebuildHash, err = config.CalculatePrebuildHash(cfg, "linux/arm64", "arm64", filepath.Dir(cfg.Origin), dockerfilePath, modifiedDockerfileContents, log.Default)
+			prebuildHash, err = config.CalculatePrebuildHash(cfg, "linux/arm64", "arm64", filepath.Dir(cfg.Origin), dockerfilePath, modifiedDockerfileContents, info, log.Default)
 			framework.ExpectNoError(err)
 			_, err = dockerHelper.InspectImage(ctx, prebuildRepo+":"+prebuildHash, false)
 			framework.ExpectNoError(err)
@@ -98,8 +103,13 @@ var _ = DevPodDescribe("devpod build test suite", func() {
 			err = f.DevPodBuild(ctx, tempDir, "--skip-push")
 			framework.ExpectNoError(err)
 
+			// parse the dockerfile
+			file, err := dockerfile.Parse(modifiedDockerfileContents)
+			framework.ExpectNoError(err)
+			info := &config.ImageBuildInfo{Dockerfile: file}
+
 			// make sure images are there
-			prebuildHash, err := config.CalculatePrebuildHash(cfg, "linux/amd64", "amd64", filepath.Dir(cfg.Origin), dockerfilePath, modifiedDockerfileContents, log.Default)
+			prebuildHash, err := config.CalculatePrebuildHash(cfg, "linux/amd64", "amd64", filepath.Dir(cfg.Origin), dockerfilePath, modifiedDockerfileContents, info, log.Default)
 			framework.ExpectNoError(err)
 			_, err = dockerHelper.InspectImage(ctx, dockerdriver.GetImageName(tempDir, prebuildHash), false)
 			framework.ExpectNoError(err)
@@ -154,8 +164,13 @@ var _ = DevPodDescribe("devpod build test suite", func() {
 			err = f.DevPodBuild(ctx, tempDir, "--force-build", "--force-internal-buildkit", "--repository", prebuildRepo, "--skip-push")
 			framework.ExpectNoError(err)
 
+			// parse the dockerfile
+			file, err := dockerfile.Parse(modifiedDockerfileContents)
+			framework.ExpectNoError(err)
+			info := &config.ImageBuildInfo{Dockerfile: file}
+
 			// make sure images are there
-			prebuildHash, err := config.CalculatePrebuildHash(cfg, "linux/amd64", "amd64", filepath.Dir(cfg.Origin), dockerfilePath, modifiedDockerfileContents, log.Default)
+			prebuildHash, err := config.CalculatePrebuildHash(cfg, "linux/amd64", "amd64", filepath.Dir(cfg.Origin), dockerfilePath, modifiedDockerfileContents, info, log.Default)
 			framework.ExpectNoError(err)
 
 			_, err = dockerHelper.InspectImage(ctx, prebuildRepo+":"+prebuildHash, false)

examples/build/.devcontainer.json

@@ -0,0 +1,10 @@
+{
+  "name": "My dev env",
+  "build": {
+    "context": ".",
+    "dockerfile": "./Dockerfile"
+  },
+  "features":{
+    "ghcr.io/devcontainers/features/github-cli:1": {}
+  }
+}

examples/build/Dockerfile

@@ -0,0 +1,30 @@
+FROM mcr.microsoft.com/devcontainers/go:1.22-bullseye
+
+ARG TARGETOS
+ARG TARGETARCH
+
+# Install Node.js
+RUN \
+    curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends nodejs \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set environment variables for Rust
+ENV RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    PATH=/usr/local/cargo/bin:$PATH \
+    RUST_VERSION=1.69.0
+
+# Install Protobuf compiler
+RUN \
+    apt-get update \
+    && apt-get install -y --no-install-recommends protobuf-compiler \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY app /app
+COPY files /files
+
+RUN echo hello

examples/build/README.md

@@ -0,0 +1,6 @@
+## Build Example
+
+This folder holds a super simple devcontainer configuration that builds a local Dockerfile with a devcontainer feature. You can start this project via:
+```
+devpod up ./examples/build
+```

pkg/client/clientimplementation/workspace_client.go

@@ -203,6 +203,9 @@ func (s *workspaceClient) agentInfo(cliOptions provider.CLIOptions) (string, *pr
 	// Get the timeout from the context options
 	agentInfo.InjectTimeout = config.ParseTimeOption(s.devPodConfig, config.ContextOptionAgentInjectTimeout)
 
+	// Set registry cache from context option
+	agentInfo.RegistryCache = s.devPodConfig.ContextOption(config.ContextOptionRegistryCache)
+
 	// marshal config
 	out, err := json.Marshal(agentInfo)
 	if err != nil {

pkg/config/context.go

@@ -14,6 +14,7 @@ const (
 	ContextOptionSSHAgentForwarding         = "SSH_AGENT_FORWARDING"
 	ContextOptionSSHConfigPath              = "SSH_CONFIG_PATH"
 	ContextOptionAgentInjectTimeout         = "AGENT_INJECT_TIMEOUT"
+	ContextOptionRegistryCache              = "REGISTRY_CACHE"
 )
 
 var ContextOptions = []ContextOption{
@@ -86,4 +87,9 @@ var ContextOptions = []ContextOption{
 		Description: "Specifies the timeout to inject the agent",
 		Default:     "20",
 	},
+	{
+		Name:        ContextOptionRegistryCache,
+		Description: "Specifies the registry to use as a build cache, e.g. gcr.io/my-project/my-dev-env",
+		Default:     "",
+	},
 }

pkg/devcontainer/build.go

@@ -103,6 +103,7 @@ func (r *runner) build(
 			ImageMetadata: imageMetadata,
 			ImageName:     overrideBuildImageName,
 			PrebuildHash:  imageTag,
+			RegistryCache: options.RegistryCache,
 		}, nil
 	}
 
@@ -133,6 +134,7 @@ func (r *runner) extendImage(
 			ImageDetails:  imageBuildInfo.ImageDetails,
 			ImageMetadata: extendedBuildInfo.MetadataConfig,
 			ImageName:     imageBase,
+			RegistryCache: options.RegistryCache,
 		}, nil
 	}
 
@@ -287,7 +289,7 @@ func (r *runner) buildImage(
 		return nil, err
 	}
 
-	prebuildHash, err := config.CalculatePrebuildHash(parsedConfig.Config, options.Platform, targetArch, config.GetContextPath(parsedConfig.Config), dockerfilePath, dockerfileContent, r.Log)
+	prebuildHash, err := config.CalculatePrebuildHash(parsedConfig.Config, options.Platform, targetArch, config.GetContextPath(parsedConfig.Config), dockerfilePath, dockerfileContent, buildInfo, r.Log)
 	if err != nil {
 		return nil, err
 	}
@@ -319,6 +321,7 @@ func (r *runner) buildImage(
 					ImageMetadata: extendedBuildInfo.MetadataConfig,
 					ImageName:     prebuildImage,
 					PrebuildHash:  prebuildHash,
+					RegistryCache: options.RegistryCache,
 				}, nil
 			} else if err != nil {
 				r.Log.Debugf("Error trying to find prebuild image %s: %v", prebuildImage, err)
@@ -333,7 +336,7 @@ func (r *runner) buildImage(
 			return nil, fmt.Errorf("cannot build devcontainer because driver is non-docker and dockerless fallback is disabled")
 		}
 
-		return dockerlessFallback(r.LocalWorkspaceFolder, substitutionContext.ContainerWorkspaceFolder, parsedConfig, buildInfo, extendedBuildInfo, dockerfileContent)
+		return dockerlessFallback(r.LocalWorkspaceFolder, substitutionContext.ContainerWorkspaceFolder, parsedConfig, buildInfo, extendedBuildInfo, dockerfileContent, options)
 	}
 
 	return dockerDriver.BuildDevContainer(ctx, prebuildHash, parsedConfig, extendedBuildInfo, dockerfilePath, dockerfileContent, r.LocalWorkspaceFolder, options)
@@ -346,6 +349,7 @@ func dockerlessFallback(
 	buildInfo *config.ImageBuildInfo,
 	extendedBuildInfo *feature.ExtendedBuildInfo,
 	dockerfileContent string,
+	options provider.BuildOptions,
 ) (*config.BuildInfo, error) {
 	contextPath := config.GetContextPath(parsedConfig.Config)
 	devPodInternalFolder := filepath.Join(contextPath, config.DevPodContextFeatureFolder)
@@ -380,6 +384,7 @@ func dockerlessFallback(
 
 			User: buildInfo.User,
 		},
+		RegistryCache: options.RegistryCache,
 	}, nil
 }
 

pkg/devcontainer/build/options.go

@@ -8,6 +8,7 @@ type BuildOptions struct {
 
 	Images    []string
 	CacheFrom []string
+	CacheTo   []string
 
 	Dockerfile string
 	Context    string

pkg/devcontainer/buildkit/buildkit.go

@@ -28,6 +28,10 @@ func Build(ctx context.Context, client *buildkit.Client, writer io.Writer, platf
 	if err != nil {
 		return err
 	}
+	cacheTo, err := ParseCacheEntry(options.CacheTo)
+	if err != nil {
+		return err
+	}
 
 	// is context stream?
 	attachable := []session.Attachable{}
@@ -42,6 +46,7 @@ func Build(ctx context.Context, client *buildkit.Client, writer io.Writer, platf
 		},
 		Session:      attachable,
 		CacheImports: cacheFrom,
+		CacheExports: cacheTo,
 	}
 
 	// set options target

pkg/devcontainer/config/build.go

@@ -21,6 +21,7 @@ type BuildInfo struct {
 	ImageMetadata *ImageMetadataConfig
 	ImageName     string
 	PrebuildHash  string
+	RegistryCache string
 
 	Dockerless *BuildInfoDockerless
 }

pkg/devcontainer/config/prebuild.go

@@ -15,7 +15,11 @@ import (
 	"github.com/loft-sh/log/hash"
 )
 
-func CalculatePrebuildHash(originalConfig *DevContainerConfig, platform, architecture, contextPath, dockerfilePath, dockerfileContent string, log log.Logger) (string, error) {
+func CalculatePrebuildHash(
+	originalConfig *DevContainerConfig,
+	platform, architecture, contextPath, dockerfilePath, dockerfileContent string,
+	buildInfo *ImageBuildInfo,
+	log log.Logger) (string, error) {
 	parsedConfig := CloneDevContainerConfig(originalConfig)
 
 	if platform != "" {
@@ -57,8 +61,17 @@ func CalculatePrebuildHash(originalConfig *DevContainerConfig, platform, archite
 	}
 	excludes = append(excludes, DevPodContextFeatureFolder+"/")
 
+	// find exact files to hash
+	// todo pass down target or search all
+	// todo update DirectoryHash function
+	var includes []string
+	if buildInfo.Dockerfile != nil {
+		includes = buildInfo.Dockerfile.BuildContextFiles()
+	}
+	log.Debug("Build context files to use for hash are ", includes)
+
 	// get hash of the context directory
-	contextHash, err := util.DirectoryHash(contextPath, excludes)
+	contextHash, err := util.DirectoryHash(contextPath, excludes, includes)
 	if err != nil {
 		return "", err
 	}

pkg/devcontainer/run.go

@@ -85,8 +85,9 @@ type runner struct {
 type UpOptions struct {
 	provider2.CLIOptions
 
-	NoBuild    bool
-	ForceBuild bool
+	NoBuild       bool
+	ForceBuild    bool
+	RegistryCache string
 }
 
 func (r *runner) Up(ctx context.Context, options UpOptions, timeout time.Duration) (*config.Result, error) {