Skip to content

Commit bfe6c02

Browse files
posullivebyhr
posulliv
authored and
ebyhr
committed
Add support for Oracle as backend database
1 parent 3136c68 commit bfe6c02

25 files changed

+372
-79
lines changed

‎docs/installation.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,9 @@ distribution is installed.
3838

3939
### Backend database
4040

41-
Trino Gateway requires a MySQL or PostgreSQL database. Database initialization
42-
is performed automatically when the Trino Gateway process starts. Migrations
43-
are performed using `Flyway`.
41+
Trino Gateway requires a MySQL, PostgreSQL, or Oracle database. Database
42+
initialization is performed automatically when the Trino Gateway process
43+
starts. Migrations are performed using `Flyway`.
4444

4545
The migration files can viewed in the `gateway-ha/src/main/resources/` folder.
4646
Each database type supported has its own sub-folder.

‎docs/quickstart.md‎

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ under the project root folder, and run it at the temporary directory.
1616
It copies the following, necessary files to current directory:
1717

1818
- gateway-ha.jar
19-
- gateway-ha-persistence-postgres.sql
2019
- quickstart-config.yaml
2120

2221
```shell
@@ -41,26 +40,18 @@ else
4140
cp ../docs/quickstart-config.yaml ./quickstart-config.yaml
4241
fi
4342

44-
# Check and get the postgres.sql
45-
if [[ -f "gateway-ha-persistence-postgres.sql" ]]; then
46-
echo "Found gateway-ha-persistence-postgres.sql file in current directory."
47-
else
48-
cp ../gateway-ha/src/main/resources/gateway-ha-persistence-postgres.sql ./gateway-ha-persistence-postgres.sql
49-
fi
50-
5143
#Check if DB is running
5244
if docker ps --format '{{.Names}}' | grep -q '^local-postgres$'; then
5345
echo "PostgreSQL database container 'localhost-postgres' is already running. Only starting Trino Gateway."
5446
else
5547
echo "PostgreSQL database container 'localhost-postgres' is not running. Proceeding to initialize and run database server."
5648
export PGPASSWORD=mysecretpassword
57-
docker run -v "$(pwd)"/gateway-ha-persistence-postgres.sql:/tmp/gateway-ha-persistence-postgres.sql --name local-postgres -p 5432:5432 -e POSTGRES_PASSWORD=$PGPASSWORD -d postgres:latest
49+
docker run --name local-postgres -p 5432:5432 -e POSTGRES_PASSWORD=$PGPASSWORD -d postgres:latest
5850
#Make sure the DB has time to initialize
5951
sleep 5
6052

6153
#Initialize the DB
6254
docker exec local-postgres psql -U postgres -h localhost -c 'CREATE DATABASE gateway'
63-
docker exec local-postgres psql -U postgres -h localhost -d gateway -f /tmp/gateway-ha-persistence-postgres.sql
6455
fi
6556

6657

‎gateway-ha/pom.xml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,14 @@
277277
<scope>runtime</scope>
278278
</dependency>
279279

280+
<dependency>
281+
<groupId>com.oracle.database.jdbc</groupId>
282+
<artifactId>ojdbc11-production</artifactId>
283+
<version>23.6.0.24.10</version>
284+
<type>pom</type>
285+
<scope>runtime</scope>
286+
</dependency>
287+
280288
<!-- Required for ClusterStatsJdbcMonitor -->
281289
<dependency>
282290
<groupId>io.trino</groupId>
@@ -285,6 +293,13 @@
285293
<scope>runtime</scope>
286294
</dependency>
287295

296+
<dependency>
297+
<groupId>org.flywaydb</groupId>
298+
<artifactId>flyway-database-oracle</artifactId>
299+
<version>${dep.flyway.version}</version>
300+
<scope>runtime</scope>
301+
</dependency>
302+
288303
<dependency>
289304
<groupId>org.flywaydb</groupId>
290305
<artifactId>flyway-database-postgresql</artifactId>
@@ -385,6 +400,12 @@
385400
<scope>test</scope>
386401
</dependency>
387402

403+
<dependency>
404+
<groupId>org.testcontainers</groupId>
405+
<artifactId>oracle-xe</artifactId>
406+
<scope>test</scope>
407+
</dependency>
408+
388409
<dependency>
389410
<groupId>org.testcontainers</groupId>
390411
<artifactId>postgresql</artifactId>

‎gateway-ha/src/main/java/io/trino/gateway/ha/module/RouterBaseModule.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ public RouterBaseModule(HaGatewayConfiguration configuration)
3939
connectionManager = new JdbcConnectionManager(jdbi, configuration.getDataStore());
4040
resourceGroupsManager = new HaResourceGroupsManager(connectionManager);
4141
gatewayBackendManager = new HaGatewayManager(jdbi);
42-
queryHistoryManager = new HaQueryHistoryManager(jdbi);
42+
queryHistoryManager = new HaQueryHistoryManager(jdbi, configuration.getDataStore().getJdbcUrl().startsWith("jdbc:oracle"));
4343
}
4444

4545
@Provides

‎gateway-ha/src/main/java/io/trino/gateway/ha/persistence/FlywayMigration.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,10 @@ private static String getLocation(String configDbUrl)
3434
if (configDbUrl.startsWith("jdbc:mysql")) {
3535
return "mysql";
3636
}
37-
throw new IllegalArgumentException(format("Invalid JDBC URL: %s. Only PostgreSQL and MySQL are supported.", configDbUrl));
37+
if (configDbUrl.startsWith("jdbc:oracle")) {
38+
return "oracle";
39+
}
40+
throw new IllegalArgumentException(format("Invalid JDBC URL: %s. Only PostgreSQL, MySQL, and Oracle are supported.", configDbUrl));
3841
}
3942

4043
public static void migrate(DataStoreConfiguration config)

‎gateway-ha/src/main/java/io/trino/gateway/ha/persistence/dao/QueryHistoryDao.java‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,21 @@ public interface QueryHistoryDao
3232
""")
3333
List<QueryHistory> findRecentQueries();
3434

35+
@SqlQuery("""
36+
SELECT * FROM query_history
37+
ORDER BY created DESC
38+
FETCH FIRST 2000 ROWS ONLY
39+
""")
40+
List<QueryHistory> findRecentQueriesWithFetch();
41+
42+
default List<QueryHistory> findRecentQueries(boolean isLimitUnsupported)
43+
{
44+
if (isLimitUnsupported) {
45+
return findRecentQueriesWithFetch();
46+
}
47+
return findRecentQueries();
48+
}
49+
3550
@SqlQuery("""
3651
SELECT * FROM query_history
3752
WHERE user_name = :userName
@@ -40,6 +55,22 @@ public interface QueryHistoryDao
4055
""")
4156
List<QueryHistory> findRecentQueriesByUserName(String userName);
4257

58+
@SqlQuery("""
59+
SELECT * FROM query_history
60+
WHERE user_name = :userName
61+
ORDER BY created DESC
62+
FETCH FIRST 2000 ROWS ONLY
63+
""")
64+
List<QueryHistory> findRecentQueriesByUserNameWithFetch(String userName);
65+
66+
default List<QueryHistory> findRecentQueriesByUserName(String userName, boolean isLimitUnsupported)
67+
{
68+
if (isLimitUnsupported) {
69+
return findRecentQueriesByUserNameWithFetch(userName);
70+
}
71+
return findRecentQueriesByUserName(userName);
72+
}
73+
4374
@SqlQuery("""
4475
SELECT backend_url FROM query_history
4576
WHERE query_id = :queryId
@@ -67,7 +98,7 @@ SELECT FLOOR(created / 1000 / 60) AS minute,
6798
COUNT(1) AS query_count
6899
FROM query_history
69100
WHERE created > :created
70-
GROUP BY minute, backend_url
101+
GROUP BY FLOOR(created / 1000 / 60), backend_url
71102
""")
72103
@UseRowMapper(MapMapper.class)
73104
List<Map<String, Object>> findDistribution(long created);

‎gateway-ha/src/main/java/io/trino/gateway/ha/router/HaQueryHistoryManager.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,10 +38,12 @@ public class HaQueryHistoryManager
3838
private static final int FIRST_PAGE_NO = 1;
3939

4040
private final QueryHistoryDao dao;
41+
private final boolean isOracleBackend;
4142

42-
public HaQueryHistoryManager(Jdbi jdbi)
43+
public HaQueryHistoryManager(Jdbi jdbi, boolean isOracleBackend)
4344
{
4445
dao = requireNonNull(jdbi, "jdbi is null").onDemand(QueryHistoryDao.class);
46+
this.isOracleBackend = isOracleBackend;
4547
}
4648

4749
@Override
@@ -66,10 +68,10 @@ public List<QueryDetail> fetchQueryHistory(Optional<String> user)
6668
{
6769
List<QueryHistory> histories;
6870
if (user.isPresent()) {
69-
histories = dao.findRecentQueriesByUserName(user.orElseThrow());
71+
histories = dao.findRecentQueriesByUserName(user.orElseThrow(), isOracleBackend);
7072
}
7173
else {
72-
histories = dao.findRecentQueries();
74+
histories = dao.findRecentQueries(isOracleBackend);
7375
}
7476
return upcast(histories);
7577
}

‎gateway-ha/src/main/resources/oracle/V1__create_schema.sql‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
CREATE TABLE gateway_backend (
2+
name VARCHAR(256) PRIMARY KEY,
3+
routing_group VARCHAR (256),
4+
backend_url VARCHAR (256),
5+
external_url VARCHAR (256),
6+
active NUMBER(1)
7+
);
8+
9+
CREATE TABLE query_history (
10+
query_id VARCHAR(256) PRIMARY KEY,
11+
query_text VARCHAR (256),
12+
created NUMBER,
13+
backend_url VARCHAR (256),
14+
user_name VARCHAR(256),
15+
source VARCHAR(256)
16+
);
17+
CREATE INDEX query_history_created_idx ON query_history(created);
18+
19+
CREATE TABLE resource_groups (
20+
resource_group_id NUMBER GENERATED ALWAYS as IDENTITY(START with 1 INCREMENT by 1),
21+
name VARCHAR(250) NOT NULL,
22+
-- OPTIONAL POLICY CONTROLS
23+
parent NUMBER,
24+
jmx_export CHAR(1),
25+
scheduling_policy VARCHAR(128),
26+
scheduling_weight NUMBER,
27+
-- REQUIRED QUOTAS
28+
soft_memory_limit VARCHAR(128) NOT NULL,
29+
max_queued INT NOT NULL,
30+
hard_concurrency_limit NUMBER NOT NULL,
31+
-- OPTIONAL QUOTAS
32+
soft_concurrency_limit NUMBER,
33+
soft_cpu_limit VARCHAR(128),
34+
hard_cpu_limit VARCHAR(128),
35+
environment VARCHAR(128),
36+
PRIMARY KEY(resource_group_id),
37+
FOREIGN KEY (parent) REFERENCES resource_groups (resource_group_id) ON DELETE CASCADE
38+
);
39+
40+
CREATE TABLE selectors (
41+
resource_group_id NUMBER NOT NULL,
42+
priority NUMBER NOT NULL,
43+
-- Regex fields -- these will be used as a regular expression pattern to
44+
-- match against the field of the same name on queries
45+
user_regex VARCHAR(512),
46+
source_regex VARCHAR(512),
47+
-- Selector fields -- these must match exactly.
48+
query_type VARCHAR(512),
49+
client_tags VARCHAR(512),
50+
selector_resource_estimate VARCHAR(1024),
51+
FOREIGN KEY (resource_group_id) REFERENCES resource_groups (resource_group_id) ON DELETE CASCADE
52+
);
53+
54+
CREATE TABLE resource_groups_global_properties (
55+
name VARCHAR(128) NOT NULL PRIMARY KEY,
56+
value VARCHAR(512) NULL,
57+
CHECK (name in ('cpu_quota_period'))
58+
);
59+
60+
CREATE TABLE exact_match_source_selectors(
61+
environment VARCHAR(256),
62+
update_time TIMESTAMP NOT NULL,
63+
-- Selector fields which must exactly match a query
64+
source VARCHAR(512) NOT NULL,
65+
query_type VARCHAR(512),
66+
resource_group_id VARCHAR(256) NOT NULL,
67+
PRIMARY KEY (environment, source, resource_group_id),
68+
UNIQUE (source, environment, query_type, resource_group_id)
69+
);

‎gateway-ha/src/test/java/io/trino/gateway/ha/HaGatewayTestUtils.java‎

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -76,16 +76,16 @@ public static void seedRequiredData(String h2DbFilePath)
7676
}
7777
}
7878

79-
public static TestConfig buildGatewayConfig(int routerPort, String configFile, PostgreSQLContainer postgresqlContainer)
79+
public static File buildGatewayConfig(PostgreSQLContainer postgreSqlContainer, int routerPort, String configFile)
8080
throws Exception
8181
{
8282
URL resource = HaGatewayTestUtils.class.getClassLoader().getResource("auth/localhost.jks");
8383
String configStr =
8484
getResourceFileContent(configFile)
8585
.replace("REQUEST_ROUTER_PORT", String.valueOf(routerPort))
86-
.replace("POSTGRESQL_JDBC_URL", postgresqlContainer.getJdbcUrl())
87-
.replace("POSTGRESQL_USER", postgresqlContainer.getUsername())
88-
.replace("POSTGRESQL_PASSWORD", postgresqlContainer.getPassword())
86+
.replace("POSTGRESQL_JDBC_URL", postgreSqlContainer.getJdbcUrl())
87+
.replace("POSTGRESQL_USER", postgreSqlContainer.getUsername())
88+
.replace("POSTGRESQL_PASSWORD", postgreSqlContainer.getPassword())
8989
.replace(
9090
"APPLICATION_CONNECTOR_PORT", String.valueOf(30000 + (int) (Math.random() * 1000)))
9191
.replace("ADMIN_CONNECTOR_PORT", String.valueOf(31000 + (int) (Math.random() * 1000)))
@@ -99,7 +99,7 @@ public static TestConfig buildGatewayConfig(int routerPort, String configFile, P
9999
}
100100

101101
log.info("Test Gateway Config \n[%s]", configStr);
102-
return new TestConfig(target.getAbsolutePath());
102+
return target;
103103
}
104104

105105
public static String getResourceFileContent(String fileName)
@@ -169,12 +169,4 @@ private static void verifyTrinoStatus(int port, String name)
169169
}
170170
throw new IllegalStateException("Trino cluster is not healthy");
171171
}
172-
173-
public record TestConfig(String configFilePath)
174-
{
175-
public TestConfig
176-
{
177-
requireNonNull(configFilePath, "configFilePath is null");
178-
}
179-
}
180172
}

‎gateway-ha/src/test/java/io/trino/gateway/ha/TestGatewayHaMultipleBackend.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import org.testcontainers.containers.PostgreSQLContainer;
3939
import org.testcontainers.containers.TrinoContainer;
4040

41+
import java.io.File;
4142
import java.io.IOException;
4243
import java.util.Base64;
4344
import java.util.List;
@@ -63,7 +64,7 @@ final class TestGatewayHaMultipleBackend
6364

6465
private TrinoContainer adhocTrino;
6566
private TrinoContainer scheduledTrino;
66-
private PostgreSQLContainer postgresql;
67+
private final PostgreSQLContainer postgresql = new PostgreSQLContainer("postgres:16");
6768

6869
public static String oauthInitiatePath = OAuth2GatewayCookie.OAUTH2_PATH;
6970
public static String oauthCallbackPath = oauthInitiatePath + "/callback";
@@ -87,7 +88,6 @@ void setup()
8788
scheduledTrino = new TrinoContainer("trinodb/trino");
8889
scheduledTrino.withCopyFileToContainer(forClasspathResource("trino-config.properties"), "/etc/trino/config.properties");
8990
scheduledTrino.start();
90-
postgresql = new PostgreSQLContainer("postgres:16");
9191
postgresql.start();
9292

9393
int backend1Port = adhocTrino.getMappedPort(8080);
@@ -115,11 +115,11 @@ public MockResponse dispatch(RecordedRequest request)
115115
}
116116
});
117117

118-
HaGatewayTestUtils.TestConfig testConfig =
119-
HaGatewayTestUtils.buildGatewayConfig(routerPort, "test-config-template.yml", postgresql);
118+
File testConfigFile =
119+
HaGatewayTestUtils.buildGatewayConfig(postgresql, routerPort, "test-config-template.yml");
120120

121121
// Start Gateway
122-
String[] args = {testConfig.configFilePath()};
122+
String[] args = {testConfigFile.getAbsolutePath()};
123123
HaGatewayLauncher.main(args);
124124
// Now populate the backend
125125
HaGatewayTestUtils.setUpBackend(

0 commit comments

Comments
 (0)