Fixing Google Cloud Storage (GCS) Support

Author: aaronmaxlevy

server/datastore/s3/s3.go

@@ -5,18 +5,22 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
+	"strings"
 
 	"github.com/fleetdm/fleet/v4/pkg/fleethttp"
 	"github.com/fleetdm/fleet/v4/server/aws_common"
 	"github.com/fleetdm/fleet/v4/server/config"
 	"github.com/fleetdm/fleet/v4/server/fleet"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
 	aws_config "github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	types "github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/aws/smithy-go/middleware"
+	smithyhttp "github.com/aws/smithy-go/transport/http"
 )
 
 const awsRegionHint = "us-east-1"
@@ -105,6 +109,14 @@ func newS3Store(cfg config.S3ConfigInternal) (*s3store, error) {
 
 	s3Client := s3.NewFromConfig(conf, func(o *s3.Options) {
 		o.UsePathStyle = cfg.ForceS3PathStyle
+
+		// Apply workaround if using Google Cloud Storage (GCS) endpoint
+		// This fixes signature issues with AWS SDK v2 when using GCS
+		// See: https://github.com/aws/aws-sdk-go-v2/issues/1816#issuecomment-1927281540
+		if cfg.EndpointURL != "" && strings.Contains(cfg.EndpointURL, "storage.googleapis.com") {
+			// GCS alters the Accept-Encoding header which breaks the request signature
+			ignoreSigningHeaders(o, []string{"Accept-Encoding"})
+		}
 	})
 
 	return &s3store{
@@ -133,3 +145,65 @@ func (s *s3store) CreateTestBucket(ctx context.Context, name string) error {
 	}
 	return err
 }
+
+// GCS workaround middleware functions to fix signature issues
+// See: https://github.com/aws/aws-sdk-go-v2/issues/1816#issuecomment-1927281540
+
+type ignoredHeadersKey struct{}
+
+// ignoreSigningHeaders excludes the listed headers from the request signature
+// because some providers (like GCS) may alter them, causing signature mismatches.
+func ignoreSigningHeaders(o *s3.Options, headers []string) {
+	o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error {
+		if err := stack.Finalize.Insert(ignoreHeaders(headers), "Signing", middleware.Before); err != nil {
+			return err
+		}
+
+		if err := stack.Finalize.Insert(restoreIgnored(), "Signing", middleware.After); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func ignoreHeaders(headers []string) middleware.FinalizeMiddleware {
+	return middleware.FinalizeMiddlewareFunc(
+		"IgnoreHeaders",
+		func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {
+			req, ok := in.Request.(*smithyhttp.Request)
+			if !ok {
+				return out, metadata, &v4.SigningError{Err: fmt.Errorf("(ignoreHeaders) unexpected request middleware type %T", in.Request)}
+			}
+
+			ignored := make(map[string]string, len(headers))
+			for _, h := range headers {
+				ignored[h] = req.Header.Get(h)
+				req.Header.Del(h)
+			}
+
+			ctx = middleware.WithStackValue(ctx, ignoredHeadersKey{}, ignored)
+
+			return next.HandleFinalize(ctx, in)
+		},
+	)
+}
+
+func restoreIgnored() middleware.FinalizeMiddleware {
+	return middleware.FinalizeMiddlewareFunc(
+		"RestoreIgnored",
+		func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {
+			req, ok := in.Request.(*smithyhttp.Request)
+			if !ok {
+				return out, metadata, &v4.SigningError{Err: fmt.Errorf("(restoreIgnored) unexpected request middleware type %T", in.Request)}
+			}
+
+			ignored, _ := middleware.GetStackValue(ctx, ignoredHeadersKey{}).(map[string]string)
+			for k, v := range ignored {
+				req.Header.Set(k, v)
+			}
+
+			return next.HandleFinalize(ctx, in)
+		},
+	)
+}