Skip to content

Commit adc74db

Browse files
committed
Support custom prefixes for S3 and STS
Closes trinodb#57
1 parent 4da2f10 commit adc74db

21 files changed

+253
-77
lines changed

Diff for: trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/TrinoS3ProxyServerModule.java

+13-2
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import io.trino.s3.proxy.server.security.SecurityResponse;
3636
import io.trino.s3.proxy.server.signing.SigningController;
3737
import io.trino.s3.proxy.server.signing.SigningControllerConfig;
38+
import org.glassfish.jersey.server.model.Resource;
3839

3940
import java.util.Optional;
4041
import java.util.ServiceLoader;
@@ -52,11 +53,16 @@ public class TrinoS3ProxyServerModule
5253
@Override
5354
protected void setup(Binder binder)
5455
{
56+
configBinder(binder).bindConfig(SigningControllerConfig.class);
57+
configBinder(binder).bindConfig(TrinoS3ProxyConfig.class);
58+
59+
TrinoS3ProxyConfig builtConfig = buildConfigObject(TrinoS3ProxyConfig.class);
60+
5561
jaxrsBinder(binder).bind(TrinoS3ProxyResource.class);
62+
jaxrsBinder(binder).bindInstance(buildPrefixedResource(TrinoS3ProxyResource.class, builtConfig.getS3Prefix()));
5663
jaxrsBinder(binder).bind(TrinoStsResource.class);
64+
jaxrsBinder(binder).bindInstance(buildPrefixedResource(TrinoStsResource.class, builtConfig.getStsPrefix()));
5765

58-
configBinder(binder).bindConfig(SigningControllerConfig.class);
59-
configBinder(binder).bindConfig(TrinoS3ProxyConfig.class);
6066
binder.bind(SigningController.class).in(Scopes.SINGLETON);
6167
binder.bind(CredentialsController.class).in(Scopes.SINGLETON);
6268
binder.bind(SecurityController.class).in(Scopes.SINGLETON);
@@ -96,4 +102,9 @@ private void installPlugins()
96102
install(plugin.module());
97103
});
98104
}
105+
106+
private static Resource buildPrefixedResource(Class<?> resourceClass, String resourcePathPrefix)
107+
{
108+
return Resource.builder(resourceClass).path(resourcePathPrefix).build();
109+
}
99110
}

Diff for: trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoS3ProxyConfig.java

+32-2
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,11 @@
2222
public class TrinoS3ProxyConfig
2323
{
2424
private Optional<String> hostName = Optional.empty();
25+
private String s3Prefix = TrinoS3ProxyRestConstants.DEFAULT_S3_PATH;
26+
private String stsPrefix = TrinoS3ProxyRestConstants.DEFAULT_STS_PATH;
2527

26-
@Config("s3proxy.hostname")
27-
@ConfigDescription("Hostname to use for REST operations, virtual-host style addressing is only supported if this is set")
28+
@Config("s3proxy.s3.hostname")
29+
@ConfigDescription("Hostname to use for S3 REST operations, virtual-host style addressing is only supported if this is set")
2830
public TrinoS3ProxyConfig setHostName(String hostName)
2931
{
3032
this.hostName = Optional.ofNullable(hostName);
@@ -36,4 +38,32 @@ public Optional<String> getHostName()
3638
{
3739
return hostName;
3840
}
41+
42+
@Config("s3proxy.s3.prefix")
43+
@ConfigDescription("URL Prefix for S3 operations, optional")
44+
public TrinoS3ProxyConfig setS3Prefix(String s3Prefix)
45+
{
46+
this.s3Prefix = s3Prefix;
47+
return this;
48+
}
49+
50+
@NotNull
51+
public String getS3Prefix()
52+
{
53+
return s3Prefix;
54+
}
55+
56+
@Config("s3proxy.sts.prefix")
57+
@ConfigDescription("URL Prefix for STS operations, optional")
58+
public TrinoS3ProxyConfig setStsPrefix(String stsPrefix)
59+
{
60+
this.stsPrefix = stsPrefix;
61+
return this;
62+
}
63+
64+
@NotNull
65+
public String getStsPrefix()
66+
{
67+
return stsPrefix;
68+
}
3969
}

Diff for: trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoS3ProxyResource.java

-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333
import static io.trino.s3.proxy.server.rest.RequestBuilder.fromRequest;
3434
import static java.util.Objects.requireNonNull;
3535

36-
@Path(TrinoS3ProxyRestConstants.S3_PATH)
3736
public class TrinoS3ProxyResource
3837
{
3938
private final SigningController signingController;

Diff for: trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoS3ProxyRestConstants.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ public final class TrinoS3ProxyRestConstants
1717
{
1818
private TrinoS3ProxyRestConstants() {}
1919

20-
public static final String BASE_PATH = "/api/v1/s3Proxy/";
21-
public static final String S3_PATH = BASE_PATH + "s3";
22-
public static final String STS_PATH = BASE_PATH + "sts";
20+
private static final String BASE_PATH = "/api/v1/s3Proxy/";
21+
public static final String DEFAULT_S3_PATH = BASE_PATH + "s3";
22+
public static final String DEFAULT_STS_PATH = BASE_PATH + "sts";
2323
}

Diff for: trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoStsResource.java

-2
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
import io.trino.s3.proxy.server.signing.SigningMetadata;
2626
import io.trino.s3.proxy.server.signing.SigningServiceType;
2727
import jakarta.ws.rs.POST;
28-
import jakarta.ws.rs.Path;
2928
import jakarta.ws.rs.WebApplicationException;
3029
import jakarta.ws.rs.core.Context;
3130
import jakarta.ws.rs.core.MediaType;
@@ -43,7 +42,6 @@
4342
import static io.trino.s3.proxy.server.signing.SigningController.formatResponseInstant;
4443
import static java.util.Objects.requireNonNull;
4544

46-
@Path(TrinoS3ProxyRestConstants.STS_PATH)
4745
public class TrinoStsResource
4846
{
4947
private static final Logger log = Logger.get(TrinoStsResource.class);

Diff for: trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestStsRequests.java renamed to trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/AbstractTestStsRequests.java

+4-9
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,11 @@
1313
*/
1414
package io.trino.s3.proxy.server;
1515

16-
import com.google.inject.Inject;
1716
import io.airlift.http.server.testing.TestingHttpServer;
1817
import io.trino.s3.proxy.server.credentials.Credential;
1918
import io.trino.s3.proxy.server.credentials.Credentials;
2019
import io.trino.s3.proxy.server.credentials.CredentialsProvider;
21-
import io.trino.s3.proxy.server.rest.TrinoS3ProxyRestConstants;
22-
import io.trino.s3.proxy.server.testing.TestingUtil.ForTesting;
23-
import io.trino.s3.proxy.server.testing.harness.TrinoS3ProxyTest;
20+
import io.trino.s3.proxy.server.rest.TrinoS3ProxyConfig;
2421
import org.junit.jupiter.api.Test;
2522
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
2623
import software.amazon.awssdk.endpoints.Endpoint;
@@ -36,18 +33,16 @@
3633
import static java.util.concurrent.CompletableFuture.completedFuture;
3734
import static org.assertj.core.api.Assertions.assertThat;
3835

39-
@TrinoS3ProxyTest
40-
public class TestStsRequests
36+
public abstract class AbstractTestStsRequests
4137
{
4238
private final StsClient stsClient;
4339
private final CredentialsProvider credentialsProvider;
4440

45-
@Inject
46-
public TestStsRequests(@ForTesting Credentials testingCredentials, TestingHttpServer httpServer, CredentialsProvider credentialsProvider)
41+
public AbstractTestStsRequests(Credentials testingCredentials, TestingHttpServer httpServer, CredentialsProvider credentialsProvider, TrinoS3ProxyConfig s3ProxyConfig)
4742
{
4843
this.credentialsProvider = requireNonNull(credentialsProvider, "credentialsProvider is null");
4944

50-
URI localProxyServerUri = httpServer.getBaseUrl().resolve(TrinoS3ProxyRestConstants.STS_PATH);
45+
URI localProxyServerUri = httpServer.getBaseUrl().resolve(s3ProxyConfig.getStsPrefix());
5146
Endpoint endpoint = Endpoint.builder().url(localProxyServerUri).build();
5247

5348
stsClient = StsClient.builder()

Diff for: trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/LocalServer.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ public static void main(String[] args)
4040
Credentials testingCredentials = trinoS3ProxyServer.getInjector().getInstance(Key.get(Credentials.class, ForTesting.class));
4141

4242
log.info("");
43-
log.info("Endpoint: %s", httpServer.getBaseUrl().resolve(TrinoS3ProxyRestConstants.S3_PATH));
43+
log.info("Endpoint: %s", httpServer.getBaseUrl().resolve(TrinoS3ProxyRestConstants.DEFAULT_S3_PATH));
4444
log.info("Access Key: %s", testingCredentials.emulated().accessKey());
4545
log.info("Secret Key: %s", testingCredentials.emulated().secretKey());
4646
log.info("");

Diff for: trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestGenericRestRequests.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ public void testAwsChunkedUpload()
117117
private StatusResponse doAwsChunkedUpload(String content)
118118
{
119119
URI uri = UriBuilder.fromUri(baseUri)
120-
.replacePath(TrinoS3ProxyRestConstants.S3_PATH)
120+
.replacePath(TrinoS3ProxyRestConstants.DEFAULT_S3_PATH)
121121
.path("two")
122122
.path("test")
123123
.build();

Diff for: trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestProxiedAssumedRoleRequests.java

+7-6
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
import com.google.inject.Inject;
1717
import io.airlift.http.server.testing.TestingHttpServer;
1818
import io.trino.s3.proxy.server.credentials.Credentials;
19-
import io.trino.s3.proxy.server.rest.TrinoS3ProxyRestConstants;
19+
import io.trino.s3.proxy.server.rest.TrinoS3ProxyConfig;
2020
import io.trino.s3.proxy.server.testing.ManagedS3MockContainer.ForS3MockContainer;
2121
import io.trino.s3.proxy.server.testing.TestingCredentialsRolesProvider;
2222
import io.trino.s3.proxy.server.testing.TestingUtil.ForTesting;
@@ -51,9 +51,10 @@ public TestProxiedAssumedRoleRequests(
5151
@ForTesting Credentials testingCredentials,
5252
TestingCredentialsRolesProvider credentialsController,
5353
@ForS3MockContainer S3Client storageClient,
54-
@ForS3MockContainer List<String> configuredBuckets)
54+
@ForS3MockContainer List<String> configuredBuckets,
55+
TrinoS3ProxyConfig trinoS3ProxyConfig)
5556
{
56-
this(buildClient(httpServer, testingCredentials), testingCredentials, credentialsController, storageClient, configuredBuckets);
57+
this(buildClient(httpServer, testingCredentials, trinoS3ProxyConfig.getS3Prefix(), trinoS3ProxyConfig.getStsPrefix()), testingCredentials, credentialsController, storageClient, configuredBuckets);
5758
}
5859

5960
protected TestProxiedAssumedRoleRequests(
@@ -75,11 +76,11 @@ public void validateCount()
7576
credentialsController.resetAssumedRoles();
7677
}
7778

78-
protected static S3Client buildClient(TestingHttpServer httpServer, Credentials credentials)
79+
protected static S3Client buildClient(TestingHttpServer httpServer, Credentials credentials, String s3Prefix, String stsPrefix)
7980
{
8081
URI baseUrl = httpServer.getBaseUrl();
81-
URI localProxyServerUri = baseUrl.resolve(TrinoS3ProxyRestConstants.S3_PATH);
82-
URI localStsServerUri = baseUrl.resolve(TrinoS3ProxyRestConstants.STS_PATH);
82+
URI localProxyServerUri = baseUrl.resolve(s3Prefix);
83+
URI localStsServerUri = baseUrl.resolve(stsPrefix);
8384

8485
AwsBasicCredentials awsBasicCredentials = AwsBasicCredentials.create(credentials.emulated().accessKey(), credentials.emulated().secretKey());
8586

Diff for: trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestProxiedEmulatedAndRemoteAssumedRoleRequests.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import com.google.inject.Inject;
1717
import io.airlift.http.server.testing.TestingHttpServer;
1818
import io.trino.s3.proxy.server.credentials.Credentials;
19+
import io.trino.s3.proxy.server.rest.TrinoS3ProxyConfig;
1920
import io.trino.s3.proxy.server.testing.ManagedS3MockContainer.ForS3MockContainer;
2021
import io.trino.s3.proxy.server.testing.TestingCredentialsRolesProvider;
2122
import io.trino.s3.proxy.server.testing.TestingTrinoS3ProxyServerModule.ForTestingRemoteCredentials;
@@ -34,8 +35,9 @@ public TestProxiedEmulatedAndRemoteAssumedRoleRequests(
3435
TestingCredentialsRolesProvider credentialsController,
3536
@ForS3MockContainer S3Client storageClient,
3637
@ForS3MockContainer List<String> configuredBuckets,
37-
@ForTestingRemoteCredentials Credentials remoteCredentials)
38+
@ForTestingRemoteCredentials Credentials remoteCredentials,
39+
TrinoS3ProxyConfig trinoS3ProxyConfig)
3840
{
39-
super(buildClient(httpServer, remoteCredentials), testingCredentials, credentialsController, storageClient, configuredBuckets);
41+
super(buildClient(httpServer, remoteCredentials, trinoS3ProxyConfig.getS3Prefix(), trinoS3ProxyConfig.getStsPrefix()), testingCredentials, credentialsController, storageClient, configuredBuckets);
4042
}
4143
}

Diff for: trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestProxiedRequests.java

+13-1
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,28 @@
1515

1616
import com.google.inject.Inject;
1717
import io.trino.s3.proxy.server.testing.ManagedS3MockContainer.ForS3MockContainer;
18+
import io.trino.s3.proxy.server.testing.TestingTrinoS3ProxyServer;
19+
import io.trino.s3.proxy.server.testing.harness.BuilderFilter;
1820
import io.trino.s3.proxy.server.testing.harness.TrinoS3ProxyTest;
1921
import io.trino.s3.proxy.server.testing.harness.TrinoS3ProxyTestCommonModules.WithConfiguredBuckets;
2022
import software.amazon.awssdk.services.s3.S3Client;
2123

2224
import java.util.List;
2325

24-
@TrinoS3ProxyTest(filters = WithConfiguredBuckets.class)
26+
@TrinoS3ProxyTest(filters = {WithConfiguredBuckets.class, TestProxiedRequests.Filter.class})
2527
public class TestProxiedRequests
2628
extends AbstractTestProxiedRequests
2729
{
30+
public static class Filter
31+
implements BuilderFilter
32+
{
33+
@Override
34+
public TestingTrinoS3ProxyServer.Builder filter(TestingTrinoS3ProxyServer.Builder builder)
35+
{
36+
return builder.withProperty("s3proxy.s3.prefix", "/api/some/s3/prefix");
37+
}
38+
}
39+
2840
@Inject
2941
public TestProxiedRequests(S3Client s3Client, @ForS3MockContainer S3Client storageClient, @ForS3MockContainer List<String> configuredBuckets)
3042
{
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.trino.s3.proxy.server;
15+
16+
import com.google.inject.Inject;
17+
import io.trino.s3.proxy.server.testing.ManagedS3MockContainer.ForS3MockContainer;
18+
import io.trino.s3.proxy.server.testing.TestingTrinoS3ProxyServer;
19+
import io.trino.s3.proxy.server.testing.harness.BuilderFilter;
20+
import io.trino.s3.proxy.server.testing.harness.TrinoS3ProxyTest;
21+
import io.trino.s3.proxy.server.testing.harness.TrinoS3ProxyTestCommonModules.WithConfiguredBuckets;
22+
import software.amazon.awssdk.services.s3.S3Client;
23+
24+
import java.util.List;
25+
26+
@TrinoS3ProxyTest(filters = {WithConfiguredBuckets.class, TestProxiedRequestsWithoutPrefix.Filter.class})
27+
public class TestProxiedRequestsWithoutPrefix
28+
extends AbstractTestProxiedRequests
29+
{
30+
public static class Filter
31+
implements BuilderFilter
32+
{
33+
@Override
34+
public TestingTrinoS3ProxyServer.Builder filter(TestingTrinoS3ProxyServer.Builder builder)
35+
{
36+
return builder.withProperty("s3proxy.s3.prefix", "");
37+
}
38+
}
39+
40+
@Inject
41+
public TestProxiedRequestsWithoutPrefix(S3Client s3Client, @ForS3MockContainer S3Client storageClient, @ForS3MockContainer List<String> configuredBuckets)
42+
{
43+
super(s3Client, storageClient, configuredBuckets);
44+
}
45+
}

Diff for: trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestRemoteSessionProxiedRequests.java

+8-6
Original file line numberDiff line numberDiff line change
@@ -16,32 +16,34 @@
1616
import com.google.inject.Inject;
1717
import io.airlift.http.server.testing.TestingHttpServer;
1818
import io.trino.s3.proxy.server.credentials.Credentials;
19+
import io.trino.s3.proxy.server.rest.TrinoS3ProxyConfig;
1920
import io.trino.s3.proxy.server.testing.ManagedS3MockContainer.ForS3MockContainer;
2021
import io.trino.s3.proxy.server.testing.TestingTrinoS3ProxyServerModule.ForTestingRemoteCredentials;
2122
import io.trino.s3.proxy.server.testing.harness.TrinoS3ProxyTest;
22-
import io.trino.s3.proxy.server.testing.harness.TrinoS3ProxyTestCommonModules;
23+
import io.trino.s3.proxy.server.testing.harness.TrinoS3ProxyTestCommonModules.WithConfiguredBuckets;
2324
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
2425
import software.amazon.awssdk.services.s3.S3Client;
2526

2627
import java.util.List;
28+
import java.util.Optional;
2729

2830
import static io.trino.s3.proxy.server.testing.TestingUtil.clientBuilder;
2931

30-
@TrinoS3ProxyTest(filters = TrinoS3ProxyTestCommonModules.WithConfiguredBuckets.class)
32+
@TrinoS3ProxyTest(filters = WithConfiguredBuckets.class)
3133
public class TestRemoteSessionProxiedRequests
3234
extends AbstractTestProxiedRequests
3335
{
3436
@Inject
35-
public TestRemoteSessionProxiedRequests(@ForS3MockContainer S3Client storageClient, @ForTestingRemoteCredentials Credentials remoteCredentials, TestingHttpServer httpServer, @ForS3MockContainer List<String> configuredBuckets)
37+
public TestRemoteSessionProxiedRequests(@ForS3MockContainer S3Client storageClient, @ForTestingRemoteCredentials Credentials remoteCredentials, TestingHttpServer httpServer, @ForS3MockContainer List<String> configuredBuckets, TrinoS3ProxyConfig trinoS3ProxyConfig)
3638
{
37-
super(buildInternalClient(remoteCredentials, httpServer), storageClient, configuredBuckets);
39+
super(buildInternalClient(remoteCredentials, httpServer, trinoS3ProxyConfig.getS3Prefix()), storageClient, configuredBuckets);
3840
}
3941

40-
private static S3Client buildInternalClient(Credentials credentials, TestingHttpServer httpServer)
42+
private static S3Client buildInternalClient(Credentials credentials, TestingHttpServer httpServer, String s3Prefix)
4143
{
4244
AwsBasicCredentials awsBasicCredentials = AwsBasicCredentials.create(credentials.emulated().accessKey(), credentials.emulated().secretKey());
4345

44-
return clientBuilder(httpServer.getBaseUrl())
46+
return clientBuilder(httpServer.getBaseUrl(), Optional.of(s3Prefix))
4547
.credentialsProvider(() -> awsBasicCredentials)
4648
.build();
4749
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.trino.s3.proxy.server;
15+
16+
import com.google.inject.Inject;
17+
import io.airlift.http.server.testing.TestingHttpServer;
18+
import io.trino.s3.proxy.server.credentials.Credentials;
19+
import io.trino.s3.proxy.server.credentials.CredentialsProvider;
20+
import io.trino.s3.proxy.server.rest.TrinoS3ProxyConfig;
21+
import io.trino.s3.proxy.server.testing.TestingTrinoS3ProxyServer;
22+
import io.trino.s3.proxy.server.testing.TestingUtil.ForTesting;
23+
import io.trino.s3.proxy.server.testing.harness.BuilderFilter;
24+
import io.trino.s3.proxy.server.testing.harness.TrinoS3ProxyTest;
25+
26+
@TrinoS3ProxyTest(filters = TestStsRequestsWithPrefix.Filter.class)
27+
public class TestStsRequestsWithPrefix
28+
extends AbstractTestStsRequests
29+
{
30+
public static class Filter
31+
implements BuilderFilter
32+
{
33+
@Override
34+
public TestingTrinoS3ProxyServer.Builder filter(TestingTrinoS3ProxyServer.Builder builder)
35+
{
36+
return builder.withProperty("s3proxy.sts.prefix", "/api/some/sts/prefix");
37+
}
38+
}
39+
40+
@Inject
41+
public TestStsRequestsWithPrefix(@ForTesting Credentials testingCredentials, TestingHttpServer httpServer, CredentialsProvider credentialsProvider, TrinoS3ProxyConfig s3ProxyConfig)
42+
{
43+
super(testingCredentials, httpServer, credentialsProvider, s3ProxyConfig);
44+
}
45+
}

0 commit comments

Comments
 (0)