Skip to content

Commit 12cd283

Browse files
committed
Add support for scopes to the Token API
Signed-off-by: Matheus Pimenta <[email protected]>
1 parent dc1567d commit 12cd283

File tree

10 files changed

+168
-59
lines changed

10 files changed

+168
-59
lines changed

README.md

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -260,9 +260,10 @@ occupy port 80 in the network namespace of the Node.
260260
### The `metadata.google.internal` DNS record
261261

262262
Google has documented the `http://metadata.google.internal` endpoint as where to fetch
263-
metadata from. Some Google libraries may use this endpoint to fetch metadata from the
264-
emulator. If that's the case, you can configure your Pods with the following DNS
265-
configuration:
263+
metadata from, and that it should resolve to `http://169.254.169.254`. While some Google
264+
libraries hard-code the IP address and hit it directly without DNS resolution, others
265+
may use the DNS hostname to fetch metadata from the emulator. If that's your case, you
266+
can add to your Pods the following DNS configuration:
266267

267268
```yaml
268269
spec:
@@ -271,12 +272,6 @@ spec:
271272
ip: 169.254.169.254
272273
```
273274
274-
This IP address is also documented by Google as the address `metadata.google.internal`
275-
should resolve to. Not all Google libraries use the DNS hostname, some use this IP
276-
address directly in their code, so this configuration may not be necessary for all
277-
libraries (example: [Go](https://github.com/googleapis/google-cloud-go/blob/a961cb5e85ed07e2eaf088faa29ed9b60882212b/compute/metadata/metadata.go#L472-L485)).
278-
Pods running the `gcloud` CLI will require this configuration to work properly.
279-
280275
### Limitations and Security Risks
281276
282277
#### Pod identification by IP address

internal/googlecredentials/google_credentials.go

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,13 +66,19 @@ func (c *Config) WorkloadIdentityProviderAudience() string {
6666
return fmt.Sprintf("//iam.googleapis.com/%s", c.opts.WorkloadIdentityProvider)
6767
}
6868

69-
func (c *Config) NewToken(ctx context.Context, subjectToken string, googleServiceAccountEmail *string) (*oauth2.Token, error) {
69+
func (c *Config) NewToken(ctx context.Context, subjectToken string,
70+
googleServiceAccountEmail *string, scopes []string) (*oauth2.Token, error) {
71+
72+
if len(scopes) == 0 {
73+
scopes = AccessScopes()
74+
}
75+
7076
conf := externalaccount.Config{
7177
UniverseDomain: "googleapis.com",
7278
Audience: c.WorkloadIdentityProviderAudience(),
7379
SubjectTokenType: "urn:ietf:params:oauth:token-type:jwt",
7480
TokenURL: "https://sts.googleapis.com/v1/token",
75-
Scopes: AccessScopes(),
81+
Scopes: scopes,
7682
SubjectTokenSupplier: tokenSupplier(subjectToken),
7783
}
7884

internal/server/gke_apis.go

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ Refer to https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identit
105105
if err != nil {
106106
return nil, err
107107
}
108-
accessTokens, _, r, err := s.getPodGoogleAccessTokens(w, r)
108+
accessTokens, _, r, err := s.getPodGoogleAccessTokens(w, r, nil)
109109
if err != nil {
110110
return nil, err
111111
}
@@ -130,7 +130,13 @@ func (s *Server) gkeServiceAccountScopesAPI() pkghttp.MetadataHandlerFunc {
130130

131131
func (s *Server) gkeServiceAccountTokenAPI() pkghttp.MetadataHandler {
132132
mh := func(w http.ResponseWriter, r *http.Request) (any, error) {
133-
tokens, expiresAt, _, err := s.getPodGoogleAccessTokens(w, r)
133+
var scopes []string
134+
for scope := range strings.SplitSeq(r.URL.Query().Get("scopes"), ",") {
135+
if s := strings.TrimSpace(scope); s != "" {
136+
scopes = append(scopes, s)
137+
}
138+
}
139+
tokens, expiresAt, _, err := s.getPodGoogleAccessTokens(w, r, scopes)
134140
if err != nil {
135141
return nil, err
136142
}

internal/server/gke_apis_test.go

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ import (
4343
"github.com/stretchr/testify/require"
4444
"golang.org/x/oauth2/google"
4545
"google.golang.org/api/googleapi"
46+
oauth2 "google.golang.org/api/oauth2/v2"
4647
)
4748

4849
const (
@@ -55,6 +56,12 @@ var gkeHeaders = http.Header{
5556
"Metadata-Flavor": []string{gkeMetadataFlavor},
5657
}
5758

59+
func TestOnGCE(t *testing.T) {
60+
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
61+
defer cancel()
62+
assert.True(t, metadata.OnGCEWithContext(ctx))
63+
}
64+
5865
func TestGKEServiceAccountTokenAPI(t *testing.T) {
5966
const url = "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token"
6067

@@ -131,8 +138,6 @@ func TestGKEServiceAccountTokenAPI_Implicitly(t *testing.T) {
131138
// GKE Service Account Token API. The Go library will internally
132139
// call this API to get an Access Token for GCS operations.
133140

134-
require.True(t, metadata.OnGCE())
135-
136141
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
137142
defer cancel()
138143

@@ -170,16 +175,25 @@ func TestGKEServiceAccountTokenAPI_Implicitly(t *testing.T) {
170175
}
171176

172177
func TestGKEServiceAccountTokenAPI_DefaultTokenSource(t *testing.T) {
173-
require.True(t, metadata.OnGCE())
174-
175178
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
176179
defer cancel()
177180

178-
ts, err := google.DefaultTokenSource(ctx)
181+
const scope = "https://www.googleapis.com/auth/bigtable.admin.table"
182+
ts, err := google.DefaultTokenSource(ctx, scope)
179183
require.NoError(t, err)
184+
180185
token, err := ts.Token()
181186
require.NoError(t, err)
182-
assert.NotEmpty(t, token.AccessToken)
187+
require.NotEmpty(t, token.AccessToken)
188+
189+
if os.Getenv("HOSTNAME") != "test-direct-access" {
190+
svc, err := oauth2.NewService(ctx)
191+
require.NoError(t, err)
192+
193+
tokenInfo, err := svc.Tokeninfo().AccessToken(token.AccessToken).Context(ctx).Do()
194+
require.NoError(t, err)
195+
assert.Equal(t, scope, tokenInfo.Scope)
196+
}
183197
}
184198

185199
func TestGKEServiceAccountIdentityAPI(t *testing.T) {

internal/server/pods.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ func (s *Server) listPodGoogleServiceAccounts(w http.ResponseWriter, r *http.Req
106106
// impersonation.
107107
// If there's an error this function sends the response to the client.
108108
func (s *Server) getPodGoogleAccessTokens(w http.ResponseWriter, r *http.Request,
109-
) (*serviceaccounttokens.AccessTokens, time.Time, *http.Request, error) {
109+
scopes []string) (*serviceaccounttokens.AccessTokens, time.Time, *http.Request, error) {
110110
saRef, r, err := s.getPodServiceAccountReference(w, r)
111111
if err != nil {
112112
return nil, time.Time{}, nil, err
@@ -122,7 +122,7 @@ func (s *Server) getPodGoogleAccessTokens(w http.ResponseWriter, r *http.Request
122122
return nil, time.Time{}, nil, err
123123
}
124124
tokens, expiresAt, err := s.opts.ServiceAccountTokens.GetGoogleAccessTokens(
125-
r.Context(), saToken, googleEmail)
125+
r.Context(), saToken, googleEmail, scopes)
126126
if err != nil {
127127
respondGoogleAPIErrorf(w, r, "error getting google access token: %w", err)
128128
return nil, time.Time{}, nil, err

internal/serviceaccounttokens/cache/provider.go

Lines changed: 89 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
"context"
2727
"errors"
2828
"fmt"
29+
"strings"
2930
"sync"
3031
"time"
3132

@@ -38,18 +39,20 @@ import (
3839
)
3940

4041
type Provider struct {
41-
opts ProviderOptions
42-
numTokens prometheus.Gauge
43-
cacheMisses prometheus.Counter
44-
serviceAccounts map[serviceaccounts.Reference]*serviceAccount
45-
googleIDTokens map[googleIDTokenReference]*tokenAndExpiration[string]
46-
nodeServiceAccountRef *serviceaccounts.Reference
47-
ctx context.Context
48-
cancelCtx context.CancelFunc
49-
serviceAccountsMutex sync.Mutex
50-
googleIDTokensMutex sync.RWMutex
51-
wg sync.WaitGroup
52-
semaphore chan struct{}
42+
opts ProviderOptions
43+
numTokens prometheus.Gauge
44+
cacheMisses prometheus.Counter
45+
serviceAccounts map[serviceaccounts.Reference]*serviceAccount
46+
googleIDTokens map[googleIDTokenReference]*tokenAndExpiration[string]
47+
googleScopedAccessTokens map[googleScopedAccessTokenReference]*tokenAndExpiration[string]
48+
nodeServiceAccountRef *serviceaccounts.Reference
49+
ctx context.Context
50+
cancelCtx context.CancelFunc
51+
serviceAccountsMutex sync.Mutex
52+
googleIDTokensMutex sync.RWMutex
53+
googleScopedAccessTokensMutex sync.RWMutex
54+
wg sync.WaitGroup
55+
semaphore chan struct{}
5356
}
5457

5558
type ProviderOptions struct {
@@ -80,17 +83,18 @@ func NewProvider(ctx context.Context, opts ProviderOptions) *Provider {
8083
backgroundCtx, cancel := context.WithCancel(backgroundCtx)
8184

8285
p := &Provider{
83-
opts: opts,
84-
numTokens: numTokens,
85-
cacheMisses: cacheMisses,
86-
serviceAccounts: make(map[serviceaccounts.Reference]*serviceAccount),
87-
googleIDTokens: make(map[googleIDTokenReference]*tokenAndExpiration[string]),
88-
ctx: backgroundCtx,
89-
cancelCtx: cancel,
90-
semaphore: make(chan struct{}, opts.Concurrency),
86+
opts: opts,
87+
numTokens: numTokens,
88+
cacheMisses: cacheMisses,
89+
serviceAccounts: make(map[serviceaccounts.Reference]*serviceAccount),
90+
googleIDTokens: make(map[googleIDTokenReference]*tokenAndExpiration[string]),
91+
googleScopedAccessTokens: make(map[googleScopedAccessTokenReference]*tokenAndExpiration[string]),
92+
ctx: backgroundCtx,
93+
cancelCtx: cancel,
94+
semaphore: make(chan struct{}, opts.Concurrency),
9195
}
9296

93-
// start garbage collector for google ID tokens
97+
// start garbage collector for input-dependant tokens
9498
p.wg.Add(1)
9599
go func() {
96100
defer p.wg.Done()
@@ -108,6 +112,14 @@ func NewProvider(ctx context.Context, opts ProviderOptions) *Provider {
108112
}
109113
}
110114
p.googleIDTokensMutex.Unlock()
115+
116+
p.googleScopedAccessTokensMutex.Lock()
117+
for ref, token := range p.googleScopedAccessTokens {
118+
if token.isExpired() {
119+
delete(p.googleScopedAccessTokens, ref)
120+
}
121+
}
122+
p.googleScopedAccessTokensMutex.Unlock()
111123
}
112124
}
113125
}()
@@ -131,14 +143,65 @@ func (p *Provider) GetServiceAccountToken(ctx context.Context, ref *serviceaccou
131143
}
132144

133145
func (p *Provider) GetGoogleAccessTokens(ctx context.Context, saToken string,
134-
googleEmail *string) (*serviceaccounttokens.AccessTokens, time.Time, error) {
135-
ref := serviceaccounts.ReferenceFromToken(saToken)
136-
tokens, err := p.getTokens(ctx, ref)
146+
googleEmail *string, scopes []string) (*serviceaccounttokens.AccessTokens, time.Time, error) {
147+
148+
saRef := serviceaccounts.ReferenceFromToken(saToken)
149+
150+
// easy case: no scopes
151+
if len(scopes) == 0 {
152+
tokens, err := p.getTokens(ctx, saRef)
153+
if err != nil {
154+
return nil, time.Time{}, err
155+
}
156+
token := tokens.googleAccessTokens
157+
return token.token, token.expiration(), nil
158+
}
159+
160+
// handle case with custom scopes
161+
162+
var email string
163+
if googleEmail != nil {
164+
email = *googleEmail
165+
}
166+
ref := googleScopedAccessTokenReference{*saRef, email, strings.Join(scopes, ",")}
167+
168+
// check cache first
169+
p.googleScopedAccessTokensMutex.RLock()
170+
token, ok := p.googleScopedAccessTokens[ref]
171+
p.googleScopedAccessTokensMutex.RUnlock()
172+
if ok && !token.isExpired() {
173+
return &serviceaccounttokens.AccessTokens{DirectAccess: token.token}, token.expiration(), nil
174+
}
175+
176+
// cache miss or token expired. need to cache a new token, so acquire semaphore to limit concurrency
177+
select {
178+
case p.semaphore <- struct{}{}:
179+
case <-ctx.Done():
180+
return nil, time.Time{}, fmt.Errorf("request context done while acquiring semaphore: %w", ctx.Err())
181+
case <-p.ctx.Done():
182+
return nil, time.Time{}, fmt.Errorf("process terminated while acquiring semaphore: %w", p.ctx.Err())
183+
}
184+
185+
tokens, expiration, err := p.opts.Source.GetGoogleAccessTokens(ctx, saToken, googleEmail, scopes)
186+
187+
// release concurrency semaphore
188+
<-p.semaphore
189+
190+
// check error
137191
if err != nil {
138192
return nil, time.Time{}, err
139193
}
140-
token := tokens.googleAccessTokens
141-
return token.token, token.expiration(), nil
194+
195+
// token issued successfully. cache it and return
196+
tokenString := tokens.DirectAccess
197+
if tokenString == "" {
198+
tokenString = tokens.Impersonated
199+
}
200+
token = newToken(tokenString, expiration)
201+
p.googleScopedAccessTokensMutex.Lock()
202+
p.googleScopedAccessTokens[ref] = token
203+
p.googleScopedAccessTokensMutex.Unlock()
204+
return &serviceaccounttokens.AccessTokens{DirectAccess: token.token}, token.expiration(), nil
142205
}
143206

144207
func (p *Provider) GetGoogleIdentityToken(ctx context.Context, saRef *serviceaccounts.Reference,

internal/serviceaccounttokens/cache/tokens.go

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,12 @@ type googleIDTokenReference struct {
5353
audience string
5454
}
5555

56+
type googleScopedAccessTokenReference struct {
57+
serviceAccountRefernce serviceaccounts.Reference
58+
email string
59+
scopes string
60+
}
61+
5662
func (p *Provider) createTokens(ctx context.Context, saRef *serviceaccounts.Reference) (*tokens, *string, error) {
5763
sa, err := p.opts.ServiceAccounts.Get(ctx, saRef)
5864
if err != nil {
@@ -69,7 +75,7 @@ func (p *Provider) createTokens(ctx context.Context, saRef *serviceaccounts.Refe
6975
return nil, nil, fmt.Errorf("error creating token for kubernetes service account: %w", err)
7076
}
7177

72-
accessTokens, accessTokenExpiration, err := p.opts.Source.GetGoogleAccessTokens(ctx, saToken, email)
78+
accessTokens, accessTokenExpiration, err := p.opts.Source.GetGoogleAccessTokens(ctx, saToken, email, nil)
7379
if err != nil {
7480
return nil, nil, fmt.Errorf("error creating google access token: %w", err)
7581
}

internal/serviceaccounttokens/create/provider.go

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -71,30 +71,40 @@ func (p *Provider) GetServiceAccountToken(ctx context.Context, ref *serviceaccou
7171
}
7272

7373
func (p *Provider) GetGoogleAccessTokens(ctx context.Context, saToken string,
74-
googleEmail *string) (*serviceaccounttokens.AccessTokens, time.Time, error) {
74+
googleEmail *string, scopes []string) (*serviceaccounttokens.AccessTokens, time.Time, error) {
7575

76-
directAccess, err := p.opts.GoogleCredentialsConfig.NewToken(ctx, saToken, nil)
77-
if err != nil {
78-
return nil, time.Time{}, err
76+
expiration := time.Now().Add(365 * 24 * time.Hour)
77+
78+
// Optimization: No need for a direct access token if the token was requested with custom
79+
// scopes and a google service account email is configured for impersonation. Tokens with
80+
// custom scopes are not used for fetching google identity tokens, so we only need to
81+
// cache the token that was requested by a client pod.
82+
var directAccess string
83+
if !(googleEmail != nil && len(scopes) > 0) {
84+
token, err := p.opts.GoogleCredentialsConfig.NewToken(ctx, saToken, nil, scopes)
85+
if err != nil {
86+
return nil, time.Time{}, err
87+
}
88+
directAccess = token.AccessToken
89+
expiration = token.Expiry
7990
}
80-
expiry := directAccess.Expiry
8191

8292
var impersonated string
8393
if googleEmail != nil {
84-
token, err := p.opts.GoogleCredentialsConfig.NewToken(ctx, saToken, googleEmail)
94+
token, err := p.opts.GoogleCredentialsConfig.NewToken(ctx, saToken, googleEmail, scopes)
8595
if err != nil {
8696
return nil, time.Time{}, err
8797
}
8898
impersonated = token.AccessToken
89-
if token.Expiry.Before(expiry) {
90-
expiry = token.Expiry
99+
if token.Expiry.Before(expiration) {
100+
expiration = token.Expiry
91101
}
92102
}
93103

94104
return &serviceaccounttokens.AccessTokens{
95-
DirectAccess: directAccess.AccessToken,
105+
DirectAccess: directAccess,
96106
Impersonated: impersonated,
97-
}, expiry, nil
107+
}, expiration, nil
98108
}
99109

100110
func (p *Provider) GetGoogleIdentityToken(ctx context.Context, _ *serviceaccounts.Reference,

internal/serviceaccounttokens/provider.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,8 @@ type AccessTokens struct {
3636

3737
type Provider interface {
3838
GetServiceAccountToken(ctx context.Context, ref *serviceaccounts.Reference) (string, time.Time, error)
39-
GetGoogleAccessTokens(ctx context.Context, saToken string, googleEmail *string) (*AccessTokens, time.Time, error)
39+
GetGoogleAccessTokens(ctx context.Context, saToken string, googleEmail *string,
40+
scopes []string) (*AccessTokens, time.Time, error)
4041
GetGoogleIdentityToken(ctx context.Context, saRef *serviceaccounts.Reference,
4142
accessToken, googleEmail, audience string) (string, time.Time, error)
4243
}

0 commit comments

Comments
 (0)