Merge #620

620: Support `credential_process` config setting r=omus a=omus

Adds support for the AWS config setting [`credential_process`](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-sourcing-external.html). I want to add support for this feature as I want I'm using [`aws-vault`'s integration](https://github.com/99designs/aws-vault/blob/master/USAGE.md#use-case-2-aws-vault-is-a-master-credentials-vault-for-aws-sdk) which means I can store my AWS credentials in one place.

As part of this I encountered that if you have an `~/.aws/credentials` file that the `credential_process` setting was ignored. This made me do some testing into if the credential preference order mirrors what the AWS CLI does. The results of these tests found the following:

- config file `sso_*` used over `~/.aws/credentials`
- `~/.aws/credentials` used over config file `credential_process`
- config file `credential_process` used over config file `aws_access_key_id`/`aws_secret_access_key`

This was using the AWS CLI version `aws-cli/2.11.13 Python/3.11.3 Darwin/22.4.0 source/arm64 prompt/off`. In this PR the `credential_process` is preferred over config file `aws_access_key_id`/`aws_secret_access_key` but the `sso_*` credentials have the wrong preference ordering. I'll try to address the credential preference ordering issues in another PR.


Co-authored-by: Curtis Vogt <[email protected]>

Author: bors[bot]

Project.toml

@@ -1,7 +1,7 @@
 name = "AWS"
 uuid = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc"
 license = "MIT"
-version = "1.84.1"
+version = "1.85.0"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
@@ -23,7 +23,7 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 XMLDict = "228000da-037f-5747-90a9-8195ccbf91a5"
 
 [compat]
-Compat = "3.29, 4"
+Compat = "3.32, 4"
 GitHub = "5"
 HTTP = "1"
 IniFile = "0.5"

src/AWS.jl

@@ -1,6 +1,6 @@
 module AWS
 
-using Compat: Compat, @something
+using Compat: Compat, @compat, @something
 using Base64
 using Dates
 using Downloads: Downloads, Downloader, Curl

src/AWSCredentials.jl

@@ -8,21 +8,22 @@ using ..AWSExceptions
 
 export AWSCredentials,
     aws_account_number,
-    aws_get_region,
     aws_get_profile_settings,
+    aws_get_region,
     aws_user_arn,
     check_credentials,
+    credentials_from_webtoken,
     dot_aws_config,
+    dot_aws_config_file,
     dot_aws_credentials,
     dot_aws_credentials_file,
-    dot_aws_config_file,
     ec2_instance_credentials,
     ecs_instance_credentials,
     env_var_credentials,
+    external_process_credentials,
     localhost_is_ec2,
-    localhost_maybe_ec2,
     localhost_is_lambda,
-    credentials_from_webtoken
+    localhost_maybe_ec2
 
 function localhost_maybe_ec2()
     return localhost_is_ec2() || isfile("/sys/devices/virtual/dmi/id/product_uuid")
@@ -424,10 +425,14 @@ function dot_aws_config(profile=nothing)
         settings = _aws_profile_config(ini, p)
         isempty(settings) && return nothing
 
+        credential_process = get(settings, "credential_process", nothing)
         access_key = get(settings, "aws_access_key_id", nothing)
         sso_start_url = get(settings, "sso_start_url", nothing)
 
-        if !isnothing(access_key)
+        if !isnothing(credential_process)
+            cmd = Cmd(Base.shell_split(credential_process))
+            return external_process_credentials(cmd)
+        elseif !isnothing(access_key)
             access_key, secret_key, token = _aws_get_credential_details(p, ini)
             return AWSCredentials(access_key, secret_key, token)
         elseif !isnothing(sso_start_url)
@@ -559,6 +564,26 @@ function credentials_from_webtoken()
     )
 end
 
+"""
+    external_process_credentials(cmd::Base.AbstractCmd) -> AWSCredentials
+
+Sources AWS credentials from an external process as defined in the AWS CLI config file.
+See https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-sourcing-external.html
+for details.
+"""
+function external_process_credentials(cmd::Base.AbstractCmd)
+    nt = open(cmd, "r") do io
+        _read_credential_process(io)
+    end
+    return AWSCredentials(
+        nt.access_key_id,
+        nt.secret_access_key,
+        @something(nt.session_token, "");
+        expiry=@something(nt.expiration, typemax(DateTime)),
+        renew=() -> external_process_credentials(cmd),
+    )
+end
+
 """
     aws_get_region(; profile=nothing, config=nothing, default="$DEFAULT_REGION")
 

src/utilities/credentials.jl

@@ -212,3 +212,38 @@ function _aws_get_sso_credential_details(profile::AbstractString, ini::Inifile)
 
     return (access_key, secret_key, token, expiry)
 end
+
+"""
+    _read_credential_process(io::IO) -> NamedTuple
+
+Parse the AWS CLI external process output out as defined in:
+https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-sourcing-external.html
+"""
+function _read_credential_process(io::IO)
+    # `JSON.parse` chokes on `Base.Process` I/O streams.
+    json = JSON.parse(read(io, String))
+
+    version = json["Version"]
+    if version != 1
+        error(
+            "Credential process returned unhandled version $version:\n",
+            sprint(JSON.print, json, 2),
+        )
+    end
+
+    access_key_id = json["AccessKeyId"]
+    secret_access_key = json["SecretAccessKey"]
+
+    # The presence of the "Expiration" key determines if the provided credentials are
+    # long-term credentials or temporary credentials. Temporary credentials must include a
+    # session token (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html)
+    if haskey(json, "Expiration") || haskey(json, "SessionToken")
+        expiration = parse(DateTime, json["Expiration"], dateformat"yyyy-mm-dd\THH:MM:SS\Z")
+        session_token = json["SessionToken"]
+    else
+        expiration = nothing
+        session_token = nothing
+    end
+
+    return @compat (; access_key_id, secret_access_key, session_token, expiration)
+end

test/AWSCredentials.jl

@@ -511,6 +511,69 @@ end
         end
     end
 
+    @testset "~/.aws/config - Credential Process" begin
+        mktempdir() do dir
+            config_file = joinpath(dir, "config")
+            credential_process_file = joinpath(dir, "cred_process")
+            open(credential_process_file, "w") do io
+                println(io, "#!/bin/sh")
+                println(io, "cat <<EOF")
+                json = Dict(
+                    "Version" => 1,
+                    "AccessKeyId" => test_values["Test-AccessKeyId"],
+                    "SecretAccessKey" => test_values["Test-SecretAccessKey"],
+                )
+                JSON.print(io, json)
+                println(io, "\nEOF")
+            end
+            chmod(credential_process_file, 0o700)
+
+            withenv("AWS_CONFIG_FILE" => config_file) do
+                @testset "support" begin
+                    open(config_file, "w") do io
+                        write(
+                            io,
+                            """
+                            [profile $(test_values["Test-Config-Profile"])]
+                            credential_process = $(abspath(credential_process_file))
+                            """,
+                        )
+                    end
+
+                    result = dot_aws_config(test_values["Test-Config-Profile"])
+
+                    @test result.access_key_id == test_values["Test-AccessKeyId"]
+                    @test result.secret_key == test_values["Test-SecretAccessKey"]
+                    @test isempty(result.token)
+                    @test result.expiry == typemax(DateTime)
+                end
+
+                # The AWS CLI uses the config file `credential_process` setting over
+                # specifying the config file `aws_access_key_id`/`aws_secret_access_key`.
+                @testset "precedence" begin
+                    open(config_file, "w") do io
+                        write(
+                            io,
+                            """
+                            [profile $(test_values["Test-Config-Profile"])]
+                            aws_access_key_id = invalid
+                            aws_secret_access_key = invalid
+                            credential_process = $(abspath(credential_process_file))
+                            """,
+                        )
+                    end
+
+                    result = dot_aws_config(test_values["Test-Config-Profile"])
+
+                    @test result.access_key_id == test_values["Test-AccessKeyId"]
+                    @test result.secret_key == test_values["Test-SecretAccessKey"]
+                    @test isempty(result.token)
+                    @test result.expiry == typemax(DateTime)
+                end
+            end
+        end
+    end
+
     @testset "~/.aws/creds - Default Profile" begin
         mktemp() do creds_file, creds_io
             write(
@@ -696,6 +759,59 @@ end
         end
     end
 
+    @testset "Credential Process" begin
+        gen_process(json) = Cmd(["echo", JSON.json(json)])
+
+        long_term_resp = Dict(
+            "Version" => 1,
+            "AccessKeyId" => "access-key",
+            "SecretAccessKey" => "secret-key",
+            # format trick: using this comment to force use of multiple lines
+        )
+        creds = external_process_credentials(gen_process(long_term_resp))
+        @test creds.access_key_id == long_term_resp["AccessKeyId"]
+        @test creds.secret_key == long_term_resp["SecretAccessKey"]
+        @test isempty(creds.token)
+        @test creds.expiry == typemax(DateTime)
+
+        expiration = floor(now(UTC), Second)
+        temporary_resp = Dict(
+            "Version" => 1,
+            "AccessKeyId" => "access-key",
+            "SecretAccessKey" => "secret-key",
+            "SessionToken" => "session-token",
+            "Expiration" => Dates.format(expiration, dateformat"yyyy-mm-dd\THH:MM:SS\Z"),
+        )
+        creds = external_process_credentials(gen_process(temporary_resp))
+        @test creds.access_key_id == temporary_resp["AccessKeyId"]
+        @test creds.secret_key == temporary_resp["SecretAccessKey"]
+        @test creds.token == temporary_resp["SessionToken"]
+        @test creds.expiry == expiration
+
+        unhandled_version_resp = Dict("Version" => 2)
+        json = sprint(JSON.print, unhandled_version_resp, 2)
+        ex = ErrorException("Credential process returned unhandled version 2:\n$json")
+        @test_throws ex external_process_credentials(gen_process(unhandled_version_resp))
+
+        missing_token_resp = Dict(
+            "Version" => 1,
+            "AccessKeyId" => "access-key",
+            "SecretAccessKey" => "secret-key",
+            "Expiration" => Dates.format(expiration, dateformat"yyyy-mm-dd\THH:MM:SS\Z"),
+        )
+        ex = KeyError("SessionToken")
+        @test_throws ex external_process_credentials(gen_process(missing_token_resp))
+
+        missing_expiration_resp = Dict(
+            "Version" => 1,
+            "AccessKeyId" => "access-key",
+            "SecretAccessKey" => "secret-key",
+            "SessionToken" => "session-token",
+        )
+        ex = KeyError("Expiration")
+        @test_throws ex external_process_credentials(gen_process(missing_expiration_resp))
+    end
+
     @testset "Credentials Not Found" begin
         patches = [
             @patch HTTP.request(method::String, url; kwargs...) = nothing