storage: CRC32 ignored for >16MiB uploads #12474
Description
Client
GCS
Environment
go 1.24.3
cloud.google.com/go/storage v1.55.0
Code and Dependencies
import (
"context"
"hash/crc32"
"io"
"log"
"os"
"cloud.google.com/go/storage"
)
const objectName = "test-crc32-file-upload"
func main() {
if len(os.Args) != 2 {
log.Fatalf("Usage: %s <bucket>", os.Args[0])
}
size := 16 * (1 << 20)
run(os.Args[1], size)
run(os.Args[1], size+1)
}
func run(bucketName string, size int) {
ctx := context.Background()
client, err := storage.NewGRPCClient(ctx, storage.WithDisabledClientMetrics())
if err != nil {
log.Fatalf("NewGRPCClient: %v", err)
}
defer client.Close()
bucket := client.Bucket(bucketName)
w := bucket.Object(objectName).NewWriter(ctx)
w.SendCRC32C = true
crc32c := crc32.New(crc32.MakeTable(crc32.Castagnoli))
mw := io.MultiWriter(w, crc32c)
if _, err := mw.Write(make([]byte, size)); err != nil {
log.Fatalf("Error Write: %v", err)
}
w.ObjectAttrs.CRC32C = crc32c.Sum32()
log.Printf("Sending size=%d checksum=%x", size, w.ObjectAttrs.CRC32C)
if err := w.Close(); err != nil {
log.Printf("Error closing GCS writer: %v", err)
return
}
log.Printf("Success")
}
go.mod
module example.org/t
go 1.24.3
require cloud.google.com/go/storage v1.55.0
require (
cel.dev/expr v0.20.0 // indirect
cloud.google.com/go v0.121.1 // indirect
cloud.google.com/go/auth v0.16.1 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.7.0 // indirect
cloud.google.com/go/iam v1.5.2 // indirect
cloud.google.com/go/monitoring v1.24.2 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 // indirect
github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/go-jose/go-jose/v4 v4.0.4 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/google/s2a-go v0.1.9 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
github.com/googleapis/gax-go/v2 v2.14.2 // indirect
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
github.com/zeebo/errs v1.4.0 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
go.opentelemetry.io/otel v1.36.0 // indirect
go.opentelemetry.io/otel/metric v1.36.0 // indirect
go.opentelemetry.io/otel/sdk v1.36.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect
go.opentelemetry.io/otel/trace v1.36.0 // indirect
golang.org/x/crypto v0.38.0 // indirect
golang.org/x/net v0.40.0 // indirect
golang.org/x/oauth2 v0.30.0 // indirect
golang.org/x/sync v0.14.0 // indirect
golang.org/x/sys v0.33.0 // indirect
golang.org/x/text v0.25.0 // indirect
golang.org/x/time v0.11.0 // indirect
google.golang.org/api v0.235.0 // indirect
google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250512202823-5a2f75b736a9 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250512202823-5a2f75b736a9 // indirect
google.golang.org/grpc v1.72.1 // indirect
google.golang.org/protobuf v1.36.6 // indirect
)
Expected behavior
Both files (16MiB and 16MiB+1) are uploaded.
Actual behavior
Only the smaller, 16MiB file, is uploaded correctly. For the larger one, the checksum is ignored:
$ go run main.go my-example-bucket
2025/06/20 08:12:54 Sending size=16777216 checksum=a3ab8542
2025/06/20 08:12:57 Success
2025/06/20 08:13:01 Sending size=16777217 checksum=f29e959f
2025/06/20 08:13:01 Error closing GCS writer: rpc error: code = InvalidArgument desc = Object crc32c mismatch: server computed: 'f29e959f', client provided: '00000000'.