How to Use AWS S3 with Credentials (curl, Go)

How to Use AWS S3 with Credentials (curl, Go)

source tree

.
|-- download-object
|   |-- download-5mb.sh
|   `-- main.go
`-- list-bucket
    |-- list-buckets.sh
    `-- main.go

---

1) List S3 Buckets

Note: Bucket operations request must be signed with the us-east-1 region scope.

Bash (curl) version

#!/bin/bash

# Load AWS credentials from environment variables
AK="${AWS_ACCESS_KEY_ID}"
SK="${AWS_SECRET_ACCESS_KEY}"

if [[ -z "$AK" || -z "$SK" ]]; then
  echo "Error: AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY not set"
  exit 1
fi

HOST="s3.amazonaws.com"
URL="https://${HOST}/"
REGION_SCOPE="us-east-1"  # Important: ListBuckets must be signed with us-east-1
NOW=$(date -u "+%Y%m%dT%H%M%SZ")
DS=$(date -u "+%Y%m%d")
PH="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"

SIGNED_HEADERS="host;x-amz-content-sha256;x-amz-date"
CR="GET
/

host:${HOST}
x-amz-content-sha256:${PH}
x-amz-date:${NOW}

${SIGNED_HEADERS}
${PH}"

# Calculate Canonical Request hash
CRH=$(printf "%s" "$CR" | openssl sha256 -binary | xxd -p -c 256)

# Build StringToSign
STS="AWS4-HMAC-SHA256
${NOW}
${DS}/${REGION_SCOPE}/s3/aws4_request
${CRH}"

# Derive signing key
kDate=$(printf "%s" "$DS" | openssl dgst -binary -sha256 -hmac "AWS4${SK}" | xxd -p -c 256)
kReg=$( printf "%s" "$REGION_SCOPE" | openssl dgst -binary -sha256 -mac HMAC -macopt hexkey:$kDate | xxd -p -c 256)
kSvc=$( printf "%s" "s3" | openssl dgst -binary -sha256 -mac HMAC -macopt hexkey:$kReg | xxd -p -c 256)
kSig=$( printf "%s" "aws4_request" | openssl dgst -binary -sha256 -mac HMAC -macopt hexkey:$kSvc | xxd -p -c 256)

# Compute final signature
SIG=$(printf "%s" "$STS" | openssl dgst -sha256 -mac HMAC -macopt hexkey:$kSig | awk '{print $2}')

# Build Authorization header
AUTH="AWS4-HMAC-SHA256 Credential=${AK}/${DS}/${REGION_SCOPE}/s3/aws4_request, SignedHeaders=${SIGNED_HEADERS}, Signature=${SIG}"

# Send request
curl -s \
  -H "x-amz-date: ${NOW}" \
  -H "x-amz-content-sha256: ${PH}" \
  -H "Authorization: ${AUTH}" \
  "${URL}"

Usage

cd list-bucket
export AWS_ACCESS_KEY_ID="access123"
export AWS_SECRET_ACCESS_KEY="secret123"
./list-buckets.sh

---

Go (minio-go SDK) version

package main

import (
	"context"
	"fmt"
	"log"
	"os"

	"github.com/minio/minio-go/v7"
	"github.com/minio/minio-go/v7/pkg/credentials"
)

func main() {
	// Load credentials from environment variables
	ak := os.Getenv("AWS_ACCESS_KEY_ID")
	sk := os.Getenv("AWS_SECRET_ACCESS_KEY")

	if ak == "" || sk == "" {
		log.Fatal("Error: AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY not set")
	}

	endpoint := "s3.amazonaws.com"
	useTLS := true

	// Initialize MinIO client (also works for AWS S3)
	client, err := minio.New(endpoint, &minio.Options{
		Creds:  credentials.NewStaticV4(ak, sk, ""),
		Secure: useTLS,
	})
	if err != nil {
		log.Fatal(err)
	}

	ctx := context.Background()

	// List all buckets
	buckets, err := client.ListBuckets(ctx)
	if err != nil {
		log.Fatal(err)
	}

	for _, b := range buckets {
		fmt.Println(b.Name)
	}
}

Usage

cd list-bucket
export AWS_ACCESS_KEY_ID="access123"
export AWS_SECRET_ACCESS_KEY="secret123"
go run main.go

---

2) Download 5MB.json from mig-test-aws-seoul-bucket

Note: Object operations must be signed with the bucket’s region scope (ap-northeast-2).

Bash (curl) version — download-5mb.sh

#!/bin/bash
set -euo pipefail

# Load AWS credentials from environment variables
AK="${AWS_ACCESS_KEY_ID:-}"
SK="${AWS_SECRET_ACCESS_KEY:-}"
ST="${AWS_SESSION_TOKEN:-}"   # optional (for STS / assumed roles)
BUCKET="mig-test-aws-seoul-bucket"
OBJECT_KEY="5MB.json"

if [[ -z "$AK" || -z "$SK" ]]; then
  echo "Error: AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY not set" >&2
  exit 1
fi

REGION_SCOPE="ap-northeast-2"                      # bucket region (Seoul)
HOST="s3.${REGION_SCOPE}.amazonaws.com"            # regional endpoint
URI_PATH="/${BUCKET}/${OBJECT_KEY}"                # canonical URI
URL="https://${HOST}${URI_PATH}"

NOW=$(date -u "+%Y%m%dT%H%M%SZ")
DS=$(date -u "+%Y%m%d")

# Payload hash: GET has empty payload; use SHA256 of empty string
PH="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"

# Build canonical headers and signed headers (conditionally add security token)
CANONICAL_HEADERS="host:${HOST}
x-amz-content-sha256:${PH}
x-amz-date:${NOW}
"
SIGNED_HEADERS="host;x-amz-content-sha256;x-amz-date"

if [[ -n "$ST" ]]; then
  CANONICAL_HEADERS="${CANONICAL_HEADERS}x-amz-security-token:${ST}
"
  SIGNED_HEADERS="${SIGNED_HEADERS};x-amz-security-token"
fi

# Canonical Request
CR="GET
${URI_PATH}

${CANONICAL_HEADERS}
${SIGNED_HEADERS}
${PH}"

# Hash Canonical Request
CRH=$(printf "%s" "$CR" | openssl sha256 -binary | xxd -p -c 256)

# String to Sign
STS="AWS4-HMAC-SHA256
${NOW}
${DS}/${REGION_SCOPE}/s3/aws4_request
${CRH}"

# Derive signing key (kSig)
kDate=$(printf "%s" "$DS" | openssl dgst -binary -sha256 -hmac "AWS4${SK}" | xxd -p -c 256)
kReg=$( printf "%s" "$REGION_SCOPE" | openssl dgst -binary -sha256 -mac HMAC -macopt hexkey:$kDate | xxd -p -c 256)
kSvc=$( printf "%s" "s3" | openssl dgst -binary -sha256 -mac HMAC -macopt hexkey:$kReg | xxd -p -c 256)
kSig=$( printf "%s" "aws4_request" | openssl dgst -binary -sha256 -mac HMAC -macopt hexkey:$kSvc | xxd -p -c 256)

# Signature
SIG=$(printf "%s" "$STS" | openssl dgst -sha256 -mac HMAC -macopt hexkey:$kSig | awk '{print $2}')

# Authorization header
AUTH="AWS4-HMAC-SHA256 Credential=${AK}/${DS}/${REGION_SCOPE}/s3/aws4_request, SignedHeaders=${SIGNED_HEADERS}, Signature=${SIG}"

# Perform request -> save as local file "5MB.json"
curl -sS \
  -H "x-amz-date: ${NOW}" \
  -H "x-amz-content-sha256: ${PH}" \
  -H "Authorization: ${AUTH}" \
  ${ST:+-H "x-amz-security-token: ${ST}"} \
  -o "5MB.json" \
  "${URL}"

echo "Downloaded: 5MB.json"

Usage

cd download-object
export AWS_ACCESS_KEY_ID="access123"
export AWS_SECRET_ACCESS_KEY="secret123"
# export AWS_SESSION_TOKEN="..."   # if using temporary credentials
./download-5mb.sh

---

Go (minio-go SDK) version — download-object/main.go

package main

import (
	"context"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"

	"github.com/minio/minio-go/v7"
	"github.com/minio/minio-go/v7/pkg/credentials"
)

func main() {
	// Load credentials from environment variables
	ak := os.Getenv("AWS_ACCESS_KEY_ID")
	sk := os.Getenv("AWS_SECRET_ACCESS_KEY")
	st := os.Getenv("AWS_SESSION_TOKEN") // optional

	if ak == "" || sk == "" {
		log.Fatal("Error: AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY not set")
	}

	// Use regional endpoint for the bucket's region (ap-northeast-2)
	endpoint := "s3.ap-northeast-2.amazonaws.com"
	useTLS := true

	cli, err := minio.New(endpoint, &minio.Options{
		Creds:  credentials.NewStaticV4(ak, sk, st),
		Secure: useTLS,
		// Region: can be omitted; client will discover bucket region if needed
	})
	if err != nil {
		log.Fatal(err)
	}

	ctx := context.Background()

	bucket := "mig-test-aws-seoul-bucket"
	object := "5MB.json"
	local  := filepath.Base(object)

	rc, err := cli.GetObject(ctx, bucket, object, minio.GetObjectOptions{})
	if err != nil {
		log.Fatal(err)
	}
	defer rc.Close()

	out, err := os.Create(local)
	if err != nil {
		log.Fatal(err)
	}
	defer out.Close()

	if _, err := io.Copy(out, rc); err != nil {
		log.Fatal(err)
	}

	fmt.Printf("Downloaded: %s\n", local)
}

Usage

cd download-object
export AWS_ACCESS_KEY_ID="access123"
export AWS_SECRET_ACCESS_KEY="secret123"
# export AWS_SESSION_TOKEN="..."   # if using temporary credentials
go run main.go

---