refactor: mirror-resources (#2975)

Signed-off-by: Philip Laine <[email protected]>

Author: phillebaba

.gitattributes

@@ -1,5 +1 @@
-# Set this repository to use unix style line endings
 * text eol=lf
-*.png -text
-*.gif -text
-*.jpg -text

site/src/content/docs/commands/zarf_package_mirror-resources.md

@@ -25,7 +25,7 @@ zarf package mirror-resources [ PACKAGE_SOURCE ] [flags]
 
 # Mirror resources to internal Zarf resources
 $ zarf package mirror-resources <your-package.tar.zst> \
-	--registry-url 127.0.0.1:31999 \
+	--registry-url http://zarf-docker-registry.zarf.svc.cluster.local:5000 \
 	--registry-push-username zarf-push \
 	--registry-push-password <generated-registry-push-password> \
 	--git-url http://zarf-gitea-http.zarf.svc.cluster.local:3000 \
@@ -57,6 +57,7 @@ $ zarf package mirror-resources <your-package.tar.zst> \
       --registry-push-username string   Username to access to the registry Zarf is configured to use (default "zarf-push")
       --registry-url string             External registry url address to use for this Zarf cluster
       --retries int                     Number of retries to perform for Zarf deploy operations like git/image pushes or Helm installs (default 3)
+      --shasum string                   Shasum of the package to pull. Required if pulling a https package. A shasum can be retrieved using 'zarf dev sha256sum <url>'
       --skip-signature-validation       Skip validating the signature of the Zarf package
 ```
 

src/cmd/package.go

@@ -11,26 +11,27 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"runtime"
 	"strings"
 
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/defenseunicorns/pkg/helpers/v2"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"oras.land/oras-go/v2/registry"
+
 	"github.com/zarf-dev/zarf/src/cmd/common"
+	"github.com/zarf-dev/zarf/src/config"
 	"github.com/zarf-dev/zarf/src/config/lang"
+	"github.com/zarf-dev/zarf/src/internal/dns"
 	"github.com/zarf-dev/zarf/src/internal/packager2"
+	"github.com/zarf-dev/zarf/src/pkg/cluster"
 	"github.com/zarf-dev/zarf/src/pkg/lint"
 	"github.com/zarf-dev/zarf/src/pkg/message"
+	"github.com/zarf-dev/zarf/src/pkg/packager"
 	"github.com/zarf-dev/zarf/src/pkg/packager/filters"
 	"github.com/zarf-dev/zarf/src/pkg/packager/sources"
 	"github.com/zarf-dev/zarf/src/types"
-
-	"oras.land/oras-go/v2/registry"
-
-	"github.com/AlecAivazis/survey/v2"
-	"github.com/defenseunicorns/pkg/helpers/v2"
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-	"github.com/zarf-dev/zarf/src/config"
-	"github.com/zarf-dev/zarf/src/pkg/cluster"
-	"github.com/zarf-dev/zarf/src/pkg/packager"
 )
 
 var packageCmd = &cobra.Command{
@@ -128,18 +129,47 @@ var packageMirrorCmd = &cobra.Command{
 		}
 	},
 	RunE: func(cmd *cobra.Command, args []string) error {
-		packageSource, err := choosePackage(args)
+		var c *cluster.Cluster
+		if dns.IsServiceURL(pkgConfig.InitOpts.RegistryInfo.Address) || dns.IsServiceURL(pkgConfig.InitOpts.GitServer.Address) {
+			var err error
+			c, err = cluster.NewCluster()
+			if err != nil {
+				return err
+			}
+		}
+		src, err := choosePackage(args)
 		if err != nil {
 			return err
 		}
-		pkgConfig.PkgOpts.PackageSource = packageSource
-		pkgClient, err := packager.New(&pkgConfig)
+		filter := filters.Combine(
+			filters.ByLocalOS(runtime.GOOS),
+			filters.BySelectState(pkgConfig.PkgOpts.OptionalComponents),
+		)
+
+		loadOpt := packager2.LoadOptions{
+			Source:                  src,
+			Shasum:                  pkgConfig.PkgOpts.Shasum,
+			PublicKeyPath:           pkgConfig.PkgOpts.PublicKeyPath,
+			SkipSignatureValidation: pkgConfig.PkgOpts.SkipSignatureValidation,
+			Filter:                  filter,
+		}
+		pkgPaths, err := packager2.LoadPackage(cmd.Context(), loadOpt)
 		if err != nil {
 			return err
 		}
-		defer pkgClient.ClearTempPaths()
-		if err := pkgClient.Mirror(cmd.Context()); err != nil {
-			return fmt.Errorf("failed to mirror package: %w", err)
+		defer os.RemoveAll(pkgPaths.Base)
+		mirrorOpt := packager2.MirrorOptions{
+			Cluster:         c,
+			PackagePaths:    *pkgPaths,
+			Filter:          filter,
+			RegistryInfo:    pkgConfig.InitOpts.RegistryInfo,
+			GitInfo:         pkgConfig.InitOpts.GitServer,
+			NoImageChecksum: pkgConfig.MirrorOpts.NoImgChecksum,
+			Retries:         pkgConfig.PkgOpts.Retries,
+		}
+		err = packager2.Mirror(cmd.Context(), mirrorOpt)
+		if err != nil {
+			return err
 		}
 		return nil
 	},
@@ -482,6 +512,7 @@ func bindMirrorFlags(v *viper.Viper) {
 	// Always require confirm flag (no viper)
 	mirrorFlags.BoolVar(&config.CommonOptions.Confirm, "confirm", false, lang.CmdPackageDeployFlagConfirm)
 
+	mirrorFlags.StringVar(&pkgConfig.PkgOpts.Shasum, "shasum", "", lang.CmdPackagePullFlagShasum)
 	mirrorFlags.BoolVar(&pkgConfig.MirrorOpts.NoImgChecksum, "no-img-checksum", false, lang.CmdPackageMirrorFlagNoChecksum)
 	mirrorFlags.BoolVar(&pkgConfig.PkgOpts.SkipSignatureValidation, "skip-signature-validation", false, lang.CmdPackageFlagSkipSignatureValidation)
 

src/config/lang/english.go

@@ -233,7 +233,7 @@ $ zarf init --artifact-push-password={PASSWORD} --artifact-push-username={USERNA
 	CmdPackageMirrorExample = `
 # Mirror resources to internal Zarf resources
 $ zarf package mirror-resources <your-package.tar.zst> \
-	--registry-url 127.0.0.1:31999 \
+	--registry-url http://zarf-docker-registry.zarf.svc.cluster.local:5000 \
 	--registry-push-username zarf-push \
 	--registry-push-password <generated-registry-push-password> \
 	--git-url http://zarf-gitea-http.zarf.svc.cluster.local:3000 \

src/internal/dns/dns.go

@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: 2021-Present The Zarf Authors
+
+// Package dns contains DNS related functionality.
+package dns
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"regexp"
+	"strconv"
+)
+
+var (
+	// localClusterServiceRegex is used to match the local cluster service format:
+	localClusterServiceRegex = regexp.MustCompile(`^(?P<name>[^\.]+)\.(?P<namespace>[^\.]+)\.svc\.cluster\.local$`)
+)
+
+// IsServiceURL returns true if the give url complies with the service url format.
+func IsServiceURL(serviceURL string) bool {
+	_, _, _, err := ParseServiceURL(serviceURL)
+	return err == nil
+}
+
+// ParseServiceURL takes a serviceURL and parses it to find the service info for connecting to the cluster. The string is expected to follow the following format:
+// Example serviceURL: http://{SERVICE_NAME}.{NAMESPACE}.svc.cluster.local:{PORT}.
+func ParseServiceURL(serviceURL string) (string, string, int, error) {
+	if serviceURL == "" {
+		return "", "", 0, errors.New("service url cannot be empty")
+	}
+	parsedURL, err := url.Parse(serviceURL)
+	if err != nil {
+		return "", "", 0, err
+	}
+	if parsedURL.Port() == "" {
+		return "", "", 0, errors.New("service url does not have a port")
+	}
+	remotePort, err := strconv.Atoi(parsedURL.Port())
+	if err != nil {
+		return "", "", 0, err
+	}
+	matches := localClusterServiceRegex.FindStringSubmatch(parsedURL.Hostname())
+	if len(matches) != 3 {
+		return "", "", 0, fmt.Errorf("invalid service url %s", serviceURL)
+	}
+	return matches[2], matches[1], remotePort, nil
+}

src/internal/dns/dns_test.go

@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: 2021-Present The Zarf Authors
+
+package dns
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestServiceURL(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name              string
+		serviceURL        string
+		expectedErr       string
+		expectedNamespace string
+		expectedName      string
+		expectedPort      int
+	}{
+		{
+			name:              "correct service url",
+			serviceURL:        "http://foo.bar.svc.cluster.local:5000",
+			expectedNamespace: "bar",
+			expectedName:      "foo",
+			expectedPort:      5000,
+		},
+		{
+			name:        "invalid service url without port",
+			serviceURL:  "http://google.com",
+			expectedErr: "service url does not have a port",
+		},
+		{
+			name:        "invalid service url with port",
+			serviceURL:  "http://google.com:3000",
+			expectedErr: "invalid service url http://google.com:3000",
+		},
+		{
+			name:        "empty service url",
+			serviceURL:  "",
+			expectedErr: "service url cannot be empty",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			isServiceURL := IsServiceURL(tt.serviceURL)
+			namespace, name, port, err := ParseServiceURL(tt.serviceURL)
+			if tt.expectedErr != "" {
+				require.False(t, isServiceURL)
+				require.EqualError(t, err, tt.expectedErr)
+				return
+			}
+			require.True(t, isServiceURL)
+			require.Equal(t, tt.expectedNamespace, namespace)
+			require.Equal(t, tt.expectedName, name)
+			require.Equal(t, tt.expectedPort, port)
+		})
+	}
+}